From 9622bd329482b59c8aa108566e6ec9c0ebecd08c Mon Sep 17 00:00:00 2001 From: cchallu Date: Tue, 30 Jan 2024 18:52:06 -0600 Subject: [PATCH 01/14] first commit --- experiments/nf_evaluation/README.md | 1 + experiments/nf_evaluation/datasets.py | 68 ++++++++++++++++++++ experiments/nf_evaluation/evaluation.py | 0 experiments/nf_evaluation/models.py | 30 +++++++++ experiments/nf_evaluation/run_experiments.py | 47 ++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 experiments/nf_evaluation/README.md create mode 100644 experiments/nf_evaluation/datasets.py create mode 100644 experiments/nf_evaluation/evaluation.py create mode 100644 experiments/nf_evaluation/models.py create mode 100644 experiments/nf_evaluation/run_experiments.py diff --git a/experiments/nf_evaluation/README.md b/experiments/nf_evaluation/README.md new file mode 100644 index 000000000..d2130477f --- /dev/null +++ b/experiments/nf_evaluation/README.md @@ -0,0 +1 @@ +# Comprehensive Evaluation of Neuralforecast models diff --git a/experiments/nf_evaluation/datasets.py b/experiments/nf_evaluation/datasets.py new file mode 100644 index 000000000..9a8ef56a6 --- /dev/null +++ b/experiments/nf_evaluation/datasets.py @@ -0,0 +1,68 @@ +import pandas as pd + +from datasetsforecast.m4 import M4 +from datasetsforecast.m3 import M3 +from datasetsforecast.long_horizon import LongHorizon + + +def get_dataset(name): + + # Read data and parameters + if name == 'M4-yearly': + Y_df, *_ = M4.load(directory='./', group='Yearly') + freq = 'Y' + h = 6 + val_size = 6 + test_size = 6 + elif name == 'M4-quarterly': + Y_df, *_ = M4.load(directory='./', group='Quarterly') + freq = 'Q' + h = 8 + val_size = 8 + test_size = 8 + elif name == 'M4-monthly': + Y_df, *_ = M4.load(directory='./', group='Monthly') + freq = 'M' + h = 18 + val_size = 18 + test_size = 18 + elif name == 'M4-daily': + Y_df, *_ = M4.load(directory='./', group='Daily') + freq = 'D' + h = 14 + val_size = 14 + test_size = 14 + elif name == 'ETTm2': + Y_df, *_ = LongHorizon.load(directory='./', group='ETTm2') + freq = '15T' + h = 720 + val_size = 11520 + test_size = 11520 + elif name == 'Electricity': + Y_df, *_ = LongHorizon.load(directory='./', group='ECL') + freq = 'H' + h = 720 + val_size = 2632 + test_size = 5260 + elif name == 'Weather': + Y_df, *_ = LongHorizon.load(directory='./', group='Weather') + freq = '10T' + h = 720 + val_size = 5270 + test_size = 10539 + elif name == 'Traffic': + Y_df, *_ = LongHorizon.load(directory='./', group='Traffic') + freq = 'H' + h = 720 + val_size = 1756 + test_size = 3508 + elif name == 'ILI': + Y_df, *_ = LongHorizon.load(directory='./', group='ILI') + freq = 'W' + h = 60 + val_size = 97 + test_size = 193 + else: + raise Exception("Frequency not defined") + + return Y_df, X_df, h, freq, val_size, test_size \ No newline at end of file diff --git a/experiments/nf_evaluation/evaluation.py b/experiments/nf_evaluation/evaluation.py new file mode 100644 index 000000000..e69de29bb diff --git a/experiments/nf_evaluation/models.py b/experiments/nf_evaluation/models.py new file mode 100644 index 000000000..85ac97055 --- /dev/null +++ b/experiments/nf_evaluation/models.py @@ -0,0 +1,30 @@ +from neuralforecast.auto import * +from neuralforecast.losses.pytorch import HuberLoss, DistributionLoss + + +def get_model(model_name, horizon, num_samples): + """Returns the model class given the model name. + """ + model_dict = { + 'lstm': AutoLSTM(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'rnn': AutoRNN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'gru': AutoGRU(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'dilatedrnn': AutoDilatedRNN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples), + 'deepar': AutoDeepAR(config=None, horizon=horizon, + loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=False), + num_samples=num_samples, alias='model'), + 'tcn': AutoTCN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'mlp': AutoMLP(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'nbeats': AutoNBEATS(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'nhits': AutoNHITS(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'dlinear': AutoDLinear(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'tft': AutoTFT(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'vanillatransformer': AutoVanillaTransformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'informer': AutoInformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'autoformer': AutoAutoformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'fedformer': AutoFEDformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'timesnet': AutoTimesNet(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'patchtst': AutoPatchTST(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model') + } + + return model_dict[model_name] \ No newline at end of file diff --git a/experiments/nf_evaluation/run_experiments.py b/experiments/nf_evaluation/run_experiments.py new file mode 100644 index 000000000..f868b3c87 --- /dev/null +++ b/experiments/nf_evaluation/run_experiments.py @@ -0,0 +1,47 @@ +import os +import argparse +import time + +import pandas as pd +from neuralforecast.core import NeuralForecast +from neuralforecast.losses.numpy import mae, mse, smape, mape + +from .models import get_model +from .datasets import get_dataset + + +def main(args): + + # Load dataset and model + Y_df, h, freq, val_size, test_size = get_dataset(args.dataset) + model = get_model(model=args.model, horizon=h, num_samples=20) + + # create experiment directory + start_time = time.time() + + # Train model + nf = NeuralForecast(model=[model], freq=freq) + forecasts_df = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, verbose=True) + elapsed_time = time.time() - start_time + + # Evaluation + metrics_df = pd.DataFrame() + metrics_df['MAE'] = mae(y=forecasts_df['y'], y_hat=forecasts_df['model']) + metrics_df['MSE'] = mse(y=forecasts_df['y'], y_hat=forecasts_df['model']) + metrics_df['SMAPE'] = smape(y=forecasts_df['y'], y_hat=forecasts_df['model']) + metrics_df['MAPE'] = mape(y=forecasts_df['y'], y_hat=forecasts_df['model']) + metrics_df['elapsed_time'] = elapsed_time + + # Save results + os.makedirs(f'./results/{args.model}_{args.dataset}_{args.experiment_id}', exist_ok=True) + +def parse_args(): + parser = argparse.ArgumentParser(description="script arguments") + parser.add_argument('--dataset', type=str, help='dataset to train models on') + parser.add_argument('--model', type=int, help='number of steps to fin tune model') + parser.add_argument('--experiment_id', type=str, help='identify experiment') + return parser.parse_args() + +if __name__ == '__main__': + # parse arguments + args = parse_args() \ No newline at end of file From d7430d79d302f84488b0b7b1d55ab13b90855fc3 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Wed, 14 Feb 2024 11:06:52 -0500 Subject: [PATCH 02/14] WIP - Script works, but some auto models have bugs --- experiments/nf_evaluation/datasets.py | 19 +++- experiments/nf_evaluation/models.py | 34 +++---- experiments/nf_evaluation/run_experiments.py | 102 ++++++++++++++----- 3 files changed, 109 insertions(+), 46 deletions(-) diff --git a/experiments/nf_evaluation/datasets.py b/experiments/nf_evaluation/datasets.py index 9a8ef56a6..ecabeace0 100644 --- a/experiments/nf_evaluation/datasets.py +++ b/experiments/nf_evaluation/datasets.py @@ -10,54 +10,63 @@ def get_dataset(name): # Read data and parameters if name == 'M4-yearly': Y_df, *_ = M4.load(directory='./', group='Yearly') - freq = 'Y' + Y_df['ds'] = Y_df['ds'].astype(int) + freq = 1 h = 6 val_size = 6 test_size = 6 elif name == 'M4-quarterly': Y_df, *_ = M4.load(directory='./', group='Quarterly') - freq = 'Q' + Y_df['ds'] = Y_df['ds'].astype(int) + freq = 4 h = 8 val_size = 8 test_size = 8 elif name == 'M4-monthly': Y_df, *_ = M4.load(directory='./', group='Monthly') - freq = 'M' + Y_df['ds'] = Y_df['ds'].astype(int) + freq = 12 h = 18 val_size = 18 test_size = 18 elif name == 'M4-daily': Y_df, *_ = M4.load(directory='./', group='Daily') - freq = 'D' + Y_df['ds'] = Y_df['ds'].astype(int) + freq = 365 h = 14 val_size = 14 test_size = 14 elif name == 'ETTm2': Y_df, *_ = LongHorizon.load(directory='./', group='ETTm2') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = '15T' h = 720 val_size = 11520 test_size = 11520 elif name == 'Electricity': Y_df, *_ = LongHorizon.load(directory='./', group='ECL') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 val_size = 2632 test_size = 5260 elif name == 'Weather': Y_df, *_ = LongHorizon.load(directory='./', group='Weather') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = '10T' h = 720 val_size = 5270 test_size = 10539 elif name == 'Traffic': Y_df, *_ = LongHorizon.load(directory='./', group='Traffic') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 val_size = 1756 test_size = 3508 elif name == 'ILI': Y_df, *_ = LongHorizon.load(directory='./', group='ILI') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'W' h = 60 val_size = 97 @@ -65,4 +74,4 @@ def get_dataset(name): else: raise Exception("Frequency not defined") - return Y_df, X_df, h, freq, val_size, test_size \ No newline at end of file + return Y_df, h, freq, val_size, test_size \ No newline at end of file diff --git a/experiments/nf_evaluation/models.py b/experiments/nf_evaluation/models.py index 85ac97055..69c94c34c 100644 --- a/experiments/nf_evaluation/models.py +++ b/experiments/nf_evaluation/models.py @@ -6,25 +6,25 @@ def get_model(model_name, horizon, num_samples): """Returns the model class given the model name. """ model_dict = { - 'lstm': AutoLSTM(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'rnn': AutoRNN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'gru': AutoGRU(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'dilatedrnn': AutoDilatedRNN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples), - 'deepar': AutoDeepAR(config=None, horizon=horizon, + 'AutoLSTM': AutoLSTM(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoRNN': AutoRNN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoGRU': AutoGRU(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoDilatedRNN': AutoDilatedRNN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoDeepAR': AutoDeepAR(config=None, h=horizon, loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=False), num_samples=num_samples, alias='model'), - 'tcn': AutoTCN(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'mlp': AutoMLP(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'nbeats': AutoNBEATS(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'nhits': AutoNHITS(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'dlinear': AutoDLinear(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'tft': AutoTFT(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'vanillatransformer': AutoVanillaTransformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'informer': AutoInformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'autoformer': AutoAutoformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'fedformer': AutoFEDformer(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'timesnet': AutoTimesNet(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'patchtst': AutoPatchTST(config=None, horizon=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model') + 'AutoTCN': AutoTCN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoMLP': AutoMLP(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoNBEATS': AutoNBEATS(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoNHITS': AutoNHITS(config=None, h=horizon, loss=HuberLoss(), backend='optuna', num_samples=num_samples, alias='model'), + 'AutoDLinear': AutoDLinear(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoTFT': AutoTFT(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoVanillaTransformer': AutoVanillaTransformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoInformer': AutoInformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoAutoformer': AutoAutoformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoFEDformer': AutoFEDformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoTimesNet': AutoTimesNet(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoPatchTST': AutoPatchTST(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model') } return model_dict[model_name] \ No newline at end of file diff --git a/experiments/nf_evaluation/run_experiments.py b/experiments/nf_evaluation/run_experiments.py index f868b3c87..4788f33b3 100644 --- a/experiments/nf_evaluation/run_experiments.py +++ b/experiments/nf_evaluation/run_experiments.py @@ -6,42 +6,96 @@ from neuralforecast.core import NeuralForecast from neuralforecast.losses.numpy import mae, mse, smape, mape -from .models import get_model -from .datasets import get_dataset +from models import get_model +from datasets import get_dataset +DATASETS = [ + 'M4-yearly', + # 'M4-quarterly', + # 'M4-monthly', + # 'M4-daily', + # 'ETTm2', + # 'Electricity', + # 'Weather', + # 'Traffic', + # 'ILI' + ] + +MODELS = [ + # 'AutoLSTM', + # 'AutoRNN', + # 'AutoGRU', + # 'AutoDilatedRNN', + # 'AutoDeepAR', + # 'AutoTCN', + # 'AutoMLP', + # 'AutoNBEATS', + 'AutoNHITS', + # 'AutoDLinear', + # 'AutoTFT', + # 'AutoVanillaTransformer', + # 'AutoInformer', + # 'AutoAutoformer', + # 'AutoFEDformer', + # 'AutoTimesNet', + # 'AutoPatchTST', +] -def main(args): - - # Load dataset and model - Y_df, h, freq, val_size, test_size = get_dataset(args.dataset) - model = get_model(model=args.model, horizon=h, num_samples=20) - # create experiment directory - start_time = time.time() +def main(): - # Train model - nf = NeuralForecast(model=[model], freq=freq) - forecasts_df = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, verbose=True) - elapsed_time = time.time() - start_time + for dataset in DATASETS: + + # Initialize a empty list for performance metrics + metrics_list = [] + + for model_name in MODELS: - # Evaluation - metrics_df = pd.DataFrame() - metrics_df['MAE'] = mae(y=forecasts_df['y'], y_hat=forecasts_df['model']) - metrics_df['MSE'] = mse(y=forecasts_df['y'], y_hat=forecasts_df['model']) - metrics_df['SMAPE'] = smape(y=forecasts_df['y'], y_hat=forecasts_df['model']) - metrics_df['MAPE'] = mape(y=forecasts_df['y'], y_hat=forecasts_df['model']) - metrics_df['elapsed_time'] = elapsed_time + # Load dataset and model + Y_df, h, freq, val_size, test_size = get_dataset(dataset) + model = get_model(model_name=model_name, horizon=h, num_samples=20) + + # Start time + start_time = time.time() - # Save results - os.makedirs(f'./results/{args.model}_{args.dataset}_{args.experiment_id}', exist_ok=True) + # Train model + nf = NeuralForecast(models=[model], freq=freq) + forecasts_df = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, n_windows=None, verbose=True) + + # Calculate elapsed time + elapsed_time = time.time() - start_time + + # Evaluation + model_mae = mae(y=forecasts_df['y'], y_hat=forecasts_df['model1']) + model_mse = mse(y=forecasts_df['y'], y_hat=forecasts_df['model1']) + model_smape = smape(y=forecasts_df['y'], y_hat=forecasts_df['model1']) + model_mape = mape(y=forecasts_df['y'], y_hat=forecasts_df['model1']) + + metrics_list.append({ + 'Model': model_name, + 'MAE': model_mae, + 'MSE': model_mse, + 'sMAPE': model_smape, + 'MAPE': model_mape, + 'time': elapsed_time + }) + + + # Save results + results_path = f'./results/{dataset}' + os.makedirs(results_path, exist_ok=True) + + metrics_df = pd.DataFrame(metrics_list) + metrics_df.to_csv(f'{results_path}/metrics.csv', header=True, index=True) def parse_args(): parser = argparse.ArgumentParser(description="script arguments") parser.add_argument('--dataset', type=str, help='dataset to train models on') - parser.add_argument('--model', type=int, help='number of steps to fin tune model') + parser.add_argument('--model', type=str, help='name of the model') parser.add_argument('--experiment_id', type=str, help='identify experiment') return parser.parse_args() if __name__ == '__main__': # parse arguments - args = parse_args() \ No newline at end of file + # args = parse_args() + main() \ No newline at end of file From 8cef6cae90bdb6718dac680f83ae14d42c38cb5f Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 15 Feb 2024 14:08:27 -0500 Subject: [PATCH 03/14] Experiment script for running a model on a dataset --- experiments/nf_evaluation/README.md | 48 ++++++++ experiments/nf_evaluation/environment.yml | 9 ++ experiments/nf_evaluation/models.py | 28 ++--- experiments/nf_evaluation/run_experiments.py | 110 +++++++------------ 4 files changed, 109 insertions(+), 86 deletions(-) create mode 100644 experiments/nf_evaluation/environment.yml diff --git a/experiments/nf_evaluation/README.md b/experiments/nf_evaluation/README.md index d2130477f..b8cc98c9e 100644 --- a/experiments/nf_evaluation/README.md +++ b/experiments/nf_evaluation/README.md @@ -1 +1,49 @@ # Comprehensive Evaluation of Neuralforecast models + +In this experiment, we tested all available models in Neuralforecast on benchmark datasets to evaluate their speed and forecasting performance. + +The datasets used for this benchmark are: +- M4 (yearly) +- M4 (quarterly) +- M4 (monthly) +- M4 (daily) +- Ettm2 (15 min) +- Electricity (hourly) +- Weather (10 min) +- Traffic (hourly) +- ILI (weekly) + +Each model went through hyperparameter optimization. The test was completed locally on CPU. + +The table below summarizes the results + +
+ +## Reproducibility + +1. Create a conda environment `nf_evaluation` using the `environment.yml` file. + ```shell + conda env create -f environment.yml + ``` + +3. Activate the conda environment using + ```shell + conda activate nf_evaluation + ``` + +Alternatively simply installing neuralforecast and datasetsforecast with pip may suffice: +``` +pip install git+https://github.com/Nixtla/datasetsforecast.git +pip install git+https://github.com/Nixtla/neuralforecast.git +``` + +4. Run the experiments for each dataset and each model using with +- `--dataset` parameter in `[M4-yearly, M4-quarterly, M4-monthly, M4-daily, Ettm2, Electricity, Weather, Traffic, ILI]` +- `--model` parameter in `['AutoLSTM', 'AutoRNN', 'AutoGRU', 'AutoDilatedRNN', 'AutoDeepAR', 'AutoTCN', 'AutoMLP', 'AutoNBEATS', 'AutoNHITS', 'AutoDLinear', 'AutoTFT', 'AutoVanillaTransformer', 'AutoInformer', 'AutoAutoformer', 'AutoFEDformer', 'AutoTimesNet', 'AutoPatchTST']` +
+ +```shell +python run_experiments.py --dataset M4-yearly --model AutoMLP +``` + +5. The script creates a folder `results/` which contains a CSV file with the metrics for the specified model. \ No newline at end of file diff --git a/experiments/nf_evaluation/environment.yml b/experiments/nf_evaluation/environment.yml new file mode 100644 index 000000000..744060d0f --- /dev/null +++ b/experiments/nf_evaluation/environment.yml @@ -0,0 +1,9 @@ +name: nf_evaluation +channels: + - conda-forge +dependencies: + - numpy<1.24 + - pip + - pip: + - "git+https://github.com/Nixtla/datasetsforecast.git" + - "git+https://github.com/Nixtla/neuralforecast.git" \ No newline at end of file diff --git a/experiments/nf_evaluation/models.py b/experiments/nf_evaluation/models.py index 69c94c34c..017c0ff03 100644 --- a/experiments/nf_evaluation/models.py +++ b/experiments/nf_evaluation/models.py @@ -8,23 +8,23 @@ def get_model(model_name, horizon, num_samples): model_dict = { 'AutoLSTM': AutoLSTM(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), 'AutoRNN': AutoRNN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), - 'AutoGRU': AutoGRU(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), + 'AutoGRU': AutoGRU(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), 'AutoDilatedRNN': AutoDilatedRNN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), 'AutoDeepAR': AutoDeepAR(config=None, h=horizon, loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=False), - num_samples=num_samples, alias='model'), - 'AutoTCN': AutoTCN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoMLP': AutoMLP(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoNBEATS': AutoNBEATS(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoNHITS': AutoNHITS(config=None, h=horizon, loss=HuberLoss(), backend='optuna', num_samples=num_samples, alias='model'), - 'AutoDLinear': AutoDLinear(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoTFT': AutoTFT(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoVanillaTransformer': AutoVanillaTransformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoInformer': AutoInformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoAutoformer': AutoAutoformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoFEDformer': AutoFEDformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoTimesNet': AutoTimesNet(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model'), - 'AutoPatchTST': AutoPatchTST(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, alias='model') + num_samples=num_samples), + 'AutoTCN': AutoTCN(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoMLP': AutoMLP(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoNBEATS': AutoNBEATS(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoNHITS': AutoNHITS(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoDLinear': AutoDLinear(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoTFT': AutoTFT(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoVanillaTransformer': AutoVanillaTransformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoInformer': AutoInformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoAutoformer': AutoAutoformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoFEDformer': AutoFEDformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoTimesNet': AutoTimesNet(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoPatchTST': AutoPatchTST(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples) } return model_dict[model_name] \ No newline at end of file diff --git a/experiments/nf_evaluation/run_experiments.py b/experiments/nf_evaluation/run_experiments.py index 4788f33b3..17d5e2e1f 100644 --- a/experiments/nf_evaluation/run_experiments.py +++ b/experiments/nf_evaluation/run_experiments.py @@ -9,93 +9,59 @@ from models import get_model from datasets import get_dataset -DATASETS = [ - 'M4-yearly', - # 'M4-quarterly', - # 'M4-monthly', - # 'M4-daily', - # 'ETTm2', - # 'Electricity', - # 'Weather', - # 'Traffic', - # 'ILI' - ] +# For compatibitlity with Mac with M chip +os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" -MODELS = [ - # 'AutoLSTM', - # 'AutoRNN', - # 'AutoGRU', - # 'AutoDilatedRNN', - # 'AutoDeepAR', - # 'AutoTCN', - # 'AutoMLP', - # 'AutoNBEATS', - 'AutoNHITS', - # 'AutoDLinear', - # 'AutoTFT', - # 'AutoVanillaTransformer', - # 'AutoInformer', - # 'AutoAutoformer', - # 'AutoFEDformer', - # 'AutoTimesNet', - # 'AutoPatchTST', -] +def main(args): -def main(): - - for dataset in DATASETS: + # Load dataset and model + Y_df, h, freq, val_size, test_size = get_dataset(args.dataset) + model = get_model(model_name=args.model, horizon=h, num_samples=20) - # Initialize a empty list for performance metrics - metrics_list = [] + # Start time + start_time = time.time() - for model_name in MODELS: - - # Load dataset and model - Y_df, h, freq, val_size, test_size = get_dataset(dataset) - model = get_model(model_name=model_name, horizon=h, num_samples=20) + # Train model + nf = NeuralForecast(models=[model], freq=freq) + forecasts_df = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, n_windows=None, verbose=True) - # Start time - start_time = time.time() - - # Train model - nf = NeuralForecast(models=[model], freq=freq) - forecasts_df = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, n_windows=None, verbose=True) + # Calculate elapsed time + elapsed_time = time.time() - start_time - # Calculate elapsed time - elapsed_time = time.time() - start_time - - # Evaluation - model_mae = mae(y=forecasts_df['y'], y_hat=forecasts_df['model1']) - model_mse = mse(y=forecasts_df['y'], y_hat=forecasts_df['model1']) - model_smape = smape(y=forecasts_df['y'], y_hat=forecasts_df['model1']) - model_mape = mape(y=forecasts_df['y'], y_hat=forecasts_df['model1']) + # Evaluation + model_mae = mae(y=forecasts_df['y'], y_hat=forecasts_df[f'{args.model}']) + model_mse = mse(y=forecasts_df['y'], y_hat=forecasts_df[f'{args.model}']) + model_smape = smape(y=forecasts_df['y'], y_hat=forecasts_df[f'{args.model}']) + model_mape = mape(y=forecasts_df['y'], y_hat=forecasts_df[f'{args.model}']) - metrics_list.append({ - 'Model': model_name, - 'MAE': model_mae, - 'MSE': model_mse, - 'sMAPE': model_smape, - 'MAPE': model_mape, - 'time': elapsed_time - }) - - - # Save results - results_path = f'./results/{dataset}' - os.makedirs(results_path, exist_ok=True) + metrics = { + 'Model': [args.model], + 'MAE': [model_mae], + 'MSE': [model_mse], + 'sMAPE': [model_smape], + 'MAPE': [model_mape], + 'time': [elapsed_time] + } + + + # Save results + results_path = f'./results/{args.dataset}' + os.makedirs(results_path, exist_ok=True) - metrics_df = pd.DataFrame(metrics_list) - metrics_df.to_csv(f'{results_path}/metrics.csv', header=True, index=True) + metrics_df = pd.DataFrame(metrics) + metrics_df.to_csv(f'{results_path}/{args.model}_metrics.csv', header=True, index=False) def parse_args(): parser = argparse.ArgumentParser(description="script arguments") parser.add_argument('--dataset', type=str, help='dataset to train models on') parser.add_argument('--model', type=str, help='name of the model') - parser.add_argument('--experiment_id', type=str, help='identify experiment') + # parser.add_argument('--experiment_id', type=str, help='identify experiment') return parser.parse_args() if __name__ == '__main__': # parse arguments - # args = parse_args() - main() \ No newline at end of file + args = parse_args() + + # Run experiment + main(args) \ No newline at end of file From 695cb09c9a64fff7db7957eb212b60b0e977e68a Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 15 Feb 2024 14:20:17 -0500 Subject: [PATCH 04/14] Small fix to readme --- experiments/nf_evaluation/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/experiments/nf_evaluation/README.md b/experiments/nf_evaluation/README.md index b8cc98c9e..734a2d887 100644 --- a/experiments/nf_evaluation/README.md +++ b/experiments/nf_evaluation/README.md @@ -16,6 +16,7 @@ The datasets used for this benchmark are: Each model went through hyperparameter optimization. The test was completed locally on CPU. The table below summarizes the results +*Table will be updated as results are obtained*
@@ -46,4 +47,4 @@ pip install git+https://github.com/Nixtla/neuralforecast.git python run_experiments.py --dataset M4-yearly --model AutoMLP ``` -5. The script creates a folder `results/` which contains a CSV file with the metrics for the specified model. \ No newline at end of file +5. The script creates a folder `results/` which contains a CSV file with the metrics for the specified model \ No newline at end of file From a5b977e9400a88b1ba63fee1a08a3971f33fb322 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 15 Feb 2024 15:30:54 -0500 Subject: [PATCH 05/14] Remove empty file --- experiments/nf_evaluation/evaluation.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 experiments/nf_evaluation/evaluation.py diff --git a/experiments/nf_evaluation/evaluation.py b/experiments/nf_evaluation/evaluation.py deleted file mode 100644 index e69de29bb..000000000 From 7de642901a6bff3cf72445ce41337b5645f6ad33 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Fri, 16 Feb 2024 14:08:45 -0500 Subject: [PATCH 06/14] Add datasets and remove comment in experiment --- experiments/nf_evaluation/datasets.py | 60 +++++++++++++++++++- experiments/nf_evaluation/run_experiments.py | 1 - 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/experiments/nf_evaluation/datasets.py b/experiments/nf_evaluation/datasets.py index ecabeace0..1f9133a9e 100644 --- a/experiments/nf_evaluation/datasets.py +++ b/experiments/nf_evaluation/datasets.py @@ -8,7 +8,28 @@ def get_dataset(name): # Read data and parameters - if name == 'M4-yearly': + if name == 'M3-yearly': + Y_df, *_ = M3.load(directory='./', group='Yearly') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'Y' + h = 6 + val_size = 6 + test_size = 6 + elif name == 'M3-quarterly': + Y_df, *_ = M3.load(directory='./', group='Quarterly') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'Q' + h = 8 + val_size = 8 + test_size = 8 + elif name == 'M3-monthly': + Y_df, *_ = M3.load(directory='./', group='Monthly') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'M' + h = 12 + val_size = 12 + test_size = 12 + elif name == 'M4-yearly': Y_df, *_ = M4.load(directory='./', group='Yearly') Y_df['ds'] = Y_df['ds'].astype(int) freq = 1 @@ -36,6 +57,13 @@ def get_dataset(name): h = 14 val_size = 14 test_size = 14 + elif name == 'M4-hourly': + Y_df, *_ = M4.load(directory='./', group='Hourly') + Y_df['ds'] = Y_df['ds'].astype(int) + freq = 24 + h = 48 + val_size = 48 + test_size = 48 elif name == 'ETTm2': Y_df, *_ = LongHorizon.load(directory='./', group='ETTm2') Y_df['ds'] = pd.to_datetime(Y_df['ds']) @@ -43,13 +71,41 @@ def get_dataset(name): h = 720 val_size = 11520 test_size = 11520 + elif name == 'Ettm1': + Y_df, *_ = LongHorizon.load(directory='./', group='ETTm1') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = '15T' + h = 720 + val_size = 11520 + test_size = 11520 + elif name == 'Etth1': + Y_df, *_ = LongHorizon.load(directory='./', group='ETTh1') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'H' + h = 720 + val_size = 11520 + test_size = 11520 + elif name == 'Etth2': + Y_df, *_ = LongHorizon.load(directory='./', group='ETTh2') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'H' + h = 720 + val_size = 11520 + test_size = 11520 elif name == 'Electricity': Y_df, *_ = LongHorizon.load(directory='./', group='ECL') Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 val_size = 2632 - test_size = 5260 + test_size = 5260 + elif name == 'Exchange': + Y_df, *_ = LongHorizon.load(directory='./', group='Exchange') + Y_df['ds'] = pd.to_datetime(Y_df['ds']) + freq = 'D' + h = 720 + val_size = 760 + test_size = 1517 elif name == 'Weather': Y_df, *_ = LongHorizon.load(directory='./', group='Weather') Y_df['ds'] = pd.to_datetime(Y_df['ds']) diff --git a/experiments/nf_evaluation/run_experiments.py b/experiments/nf_evaluation/run_experiments.py index 17d5e2e1f..1edf7f251 100644 --- a/experiments/nf_evaluation/run_experiments.py +++ b/experiments/nf_evaluation/run_experiments.py @@ -56,7 +56,6 @@ def parse_args(): parser = argparse.ArgumentParser(description="script arguments") parser.add_argument('--dataset', type=str, help='dataset to train models on') parser.add_argument('--model', type=str, help='name of the model') - # parser.add_argument('--experiment_id', type=str, help='identify experiment') return parser.parse_args() if __name__ == '__main__': From dd4f27fc81751b9151f720afa84f621abe182be2 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Mon, 19 Feb 2024 10:00:50 -0500 Subject: [PATCH 07/14] Bash file to run all experiments --- experiments/nf_evaluation/run.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 experiments/nf_evaluation/run.sh diff --git a/experiments/nf_evaluation/run.sh b/experiments/nf_evaluation/run.sh new file mode 100644 index 000000000..1e75bd3ae --- /dev/null +++ b/experiments/nf_evaluation/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +datasets=("M3-yearly" "M3-quarterly" "M3-monthly" "M4-yearly" "M4-quarterly" "M4-monthly" "M4-daily" "M4-hourly" "Ettm2" "Ettm1" "Etth1" "Etth2" "Electricity" "Exchange" "Weather" "Traffic" "ILI") + +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") + +for dataset in "${datasets[@]}"; do + for model in "${models[@]}"; do + python run_experiments.py --dataset "$dataset" --model "$model" + done +done \ No newline at end of file From 223f3f63af49c3020bb105cb51c054ddf93eb5d9 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Mon, 19 Feb 2024 10:04:06 -0500 Subject: [PATCH 08/14] Update readme --- experiments/nf_evaluation/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/experiments/nf_evaluation/README.md b/experiments/nf_evaluation/README.md index 734a2d887..52273a52a 100644 --- a/experiments/nf_evaluation/README.md +++ b/experiments/nf_evaluation/README.md @@ -38,7 +38,7 @@ pip install git+https://github.com/Nixtla/datasetsforecast.git pip install git+https://github.com/Nixtla/neuralforecast.git ``` -4. Run the experiments for each dataset and each model using with +4. Run the experiments for each dataset and each model using the - `--dataset` parameter in `[M4-yearly, M4-quarterly, M4-monthly, M4-daily, Ettm2, Electricity, Weather, Traffic, ILI]` - `--model` parameter in `['AutoLSTM', 'AutoRNN', 'AutoGRU', 'AutoDilatedRNN', 'AutoDeepAR', 'AutoTCN', 'AutoMLP', 'AutoNBEATS', 'AutoNHITS', 'AutoDLinear', 'AutoTFT', 'AutoVanillaTransformer', 'AutoInformer', 'AutoAutoformer', 'AutoFEDformer', 'AutoTimesNet', 'AutoPatchTST']`
@@ -47,4 +47,10 @@ pip install git+https://github.com/Nixtla/neuralforecast.git python run_experiments.py --dataset M4-yearly --model AutoMLP ``` +You can also run all experiments in a single command using +
+```shell +run.sh +``` + 5. The script creates a folder `results/` which contains a CSV file with the metrics for the specified model \ No newline at end of file From 67eb304c8d905f75c93350d667a9b920b0b871e8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 22 Feb 2024 18:52:12 +0000 Subject: [PATCH 09/14] Fix test and val size for Etth datasets --- experiments/nf_evaluation/datasets.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/experiments/nf_evaluation/datasets.py b/experiments/nf_evaluation/datasets.py index 1f9133a9e..2e1d425dc 100644 --- a/experiments/nf_evaluation/datasets.py +++ b/experiments/nf_evaluation/datasets.py @@ -64,7 +64,7 @@ def get_dataset(name): h = 48 val_size = 48 test_size = 48 - elif name == 'ETTm2': + elif name == 'Ettm2': Y_df, *_ = LongHorizon.load(directory='./', group='ETTm2') Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = '15T' @@ -83,15 +83,15 @@ def get_dataset(name): Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 - val_size = 11520 - test_size = 11520 + val_size = 2880 + test_size = 2880 elif name == 'Etth2': Y_df, *_ = LongHorizon.load(directory='./', group='ETTh2') Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 - val_size = 11520 - test_size = 11520 + val_size = 2880 + test_size = 2880 elif name == 'Electricity': Y_df, *_ = LongHorizon.load(directory='./', group='ECL') Y_df['ds'] = pd.to_datetime(Y_df['ds']) From 4ac1d8953d0d94fed0878c5297bb14db14b24fe7 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 22 Feb 2024 18:56:09 +0000 Subject: [PATCH 10/14] Add environment variable for memory allocation --- experiments/nf_evaluation/run_experiments.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/experiments/nf_evaluation/run_experiments.py b/experiments/nf_evaluation/run_experiments.py index 1edf7f251..9bdfc7af9 100644 --- a/experiments/nf_evaluation/run_experiments.py +++ b/experiments/nf_evaluation/run_experiments.py @@ -12,6 +12,8 @@ # For compatibitlity with Mac with M chip os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" +# For memory allocation +os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" def main(args): From e661312e56d20b5993bc38b705ab1934687a9215 Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 22 Feb 2024 18:58:07 +0000 Subject: [PATCH 11/14] Bash script to run experiments on all GPUs --- experiments/nf_evaluation/gpu_1.sh | 11 +++++++++++ experiments/nf_evaluation/gpu_2.sh | 11 +++++++++++ experiments/nf_evaluation/gpu_3.sh | 11 +++++++++++ experiments/nf_evaluation/gpu_4.sh | 11 +++++++++++ 4 files changed, 44 insertions(+) create mode 100755 experiments/nf_evaluation/gpu_1.sh create mode 100755 experiments/nf_evaluation/gpu_2.sh create mode 100755 experiments/nf_evaluation/gpu_3.sh create mode 100755 experiments/nf_evaluation/gpu_4.sh diff --git a/experiments/nf_evaluation/gpu_1.sh b/experiments/nf_evaluation/gpu_1.sh new file mode 100755 index 000000000..6a0c90e79 --- /dev/null +++ b/experiments/nf_evaluation/gpu_1.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +datasets=("M3-yearly" "M3-quarterly" "M3-monthly" "M4-yearly") + +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") + +for dataset in "${datasets[@]}"; do + for model in "${models[@]}"; do + CUDA_VISIBLE_DEVICES=0 python run_experiments.py --dataset "$dataset" --model "$model" + done +done \ No newline at end of file diff --git a/experiments/nf_evaluation/gpu_2.sh b/experiments/nf_evaluation/gpu_2.sh new file mode 100755 index 000000000..aaeeba38e --- /dev/null +++ b/experiments/nf_evaluation/gpu_2.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +datasets=("M4-quarterly" "M4-monthly" "M4-daily" "M4-hourly") + +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") + +for dataset in "${datasets[@]}"; do + for model in "${models[@]}"; do + CUDA_VISIBLE_DEVICES=1 python run_experiments.py --dataset "$dataset" --model "$model" + done +done \ No newline at end of file diff --git a/experiments/nf_evaluation/gpu_3.sh b/experiments/nf_evaluation/gpu_3.sh new file mode 100755 index 000000000..d1a89903f --- /dev/null +++ b/experiments/nf_evaluation/gpu_3.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +datasets=("Ettm2" "Ettm1" "Etth1" "Etth2") + +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") + +for dataset in "${datasets[@]}"; do + for model in "${models[@]}"; do + CUDA_VISIBLE_DEVICES=2 python run_experiments.py --dataset "$dataset" --model "$model" + done +done \ No newline at end of file diff --git a/experiments/nf_evaluation/gpu_4.sh b/experiments/nf_evaluation/gpu_4.sh new file mode 100755 index 000000000..5553f506b --- /dev/null +++ b/experiments/nf_evaluation/gpu_4.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +datasets=("Electricity" "Exchange" "Weather" "Traffic" "ILI") + +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") + +for dataset in "${datasets[@]}"; do + for model in "${models[@]}"; do + CUDA_VISIBLE_DEVICES=3 python run_experiments.py --dataset "$dataset" --model "$model" + done +done \ No newline at end of file From 066a704fcbfe395a2e5ea2f05e2321be25409a2c Mon Sep 17 00:00:00 2001 From: Marco Peixeiro Date: Thu, 22 Feb 2024 19:04:52 +0000 Subject: [PATCH 12/14] Assign same max_steps to all auto models --- nbs/models.ipynb | 38 ++++++++++----------- neuralforecast/auto.py | 76 +++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 38 deletions(-) diff --git a/nbs/models.ipynb b/nbs/models.ipynb index eb5aa557b..0b3411fb6 100644 --- a/nbs/models.ipynb +++ b/nbs/models.ipynb @@ -157,7 +157,7 @@ " \"context_size\": tune.choice([5, 10, 50]),\n", " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([16, 32]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20)\n", @@ -287,7 +287,7 @@ " \"context_size\": tune.choice([5, 10, 50]),\n", " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([16, 32]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20)\n", @@ -385,7 +385,7 @@ " \"context_size\": tune.choice([5, 10, 50]),\n", " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([16, 32]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20)\n", @@ -484,7 +484,7 @@ " \"context_size\": tune.choice([5, 10, 50]),\n", " \"decoder_hidden_size\": tune.choice([64, 128]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([16, 32]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20)\n", @@ -583,7 +583,7 @@ " \"lstm_dropout\": tune.uniform(0.0, 0.5),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice(['robust', 'minmax1']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -687,7 +687,7 @@ " \"context_size\": tune.choice([5, 10, 50]),\n", " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([16, 32]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20)\n", @@ -794,7 +794,7 @@ " \"num_layers\": tune.randint(2, 6),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -893,7 +893,7 @@ " \"h\": None,\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -993,7 +993,7 @@ " \"h\": None,\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1098,7 +1098,7 @@ " [40, 20, 1], [1, 1, 1]]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.quniform(lower=500, upper=1500, q=100),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1200,7 +1200,7 @@ " \"moving_avg_window\": tune.choice([10, 25, 50]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.quniform(lower=500, upper=1500, q=100),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1311,7 +1311,7 @@ " \"n_head\": tune.choice([4, 8]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1412,7 +1412,7 @@ " \"n_head\": tune.choice([4, 8]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1513,7 +1513,7 @@ " \"n_head\": tune.choice([4, 8]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1614,7 +1614,7 @@ " \"n_head\": tune.choice([4, 8]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1714,7 +1714,7 @@ " \"hidden_size\": tune.choice([64, 128, 256]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1817,7 +1817,7 @@ " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", " \"revin\": tune.choice([False, True]),\n", - " \"max_steps\": tune.choice([500, 1000, 5000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"windows_batch_size\": tune.choice([128, 256, 512, 1024]),\n", " \"loss\": None,\n", @@ -1927,7 +1927,7 @@ " \"conv_hidden_size\": tune.choice([32, 64, 128]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice(['robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128]),\n", " \"windows_batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"loss\": None,\n", @@ -2034,7 +2034,7 @@ " \"multi_layer\": tune.choice([3, 5, 7]),\n", " \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n", " \"scaler_type\": tune.choice([None, 'robust', 'standard']),\n", - " \"max_steps\": tune.choice([500, 1000, 2000]),\n", + " \"max_steps\": tune.quniform(lower=500, upper=5000, q=500),\n", " \"batch_size\": tune.choice([32, 64, 128, 256]),\n", " \"loss\": None,\n", " \"random_seed\": tune.randint(1, 20),\n", diff --git a/neuralforecast/auto.py b/neuralforecast/auto.py index b2e982e79..3e7a0b375 100644 --- a/neuralforecast/auto.py +++ b/neuralforecast/auto.py @@ -42,6 +42,7 @@ # %% ../nbs/models.ipynb 9 class AutoRNN(BaseAuto): + default_config = { "input_size_multiplier": [-1, 4, 16, 64], "inference_input_size_multiplier": [-1], @@ -51,7 +52,7 @@ class AutoRNN(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -111,6 +112,7 @@ def __init__( # %% ../nbs/models.ipynb 14 class AutoLSTM(BaseAuto): + default_config = { "input_size_multiplier": [-1, 4, 16, 64], "inference_input_size_multiplier": [-1], @@ -120,7 +122,7 @@ class AutoLSTM(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -141,6 +143,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -175,6 +178,7 @@ def __init__( # %% ../nbs/models.ipynb 17 class AutoGRU(BaseAuto): + default_config = { "input_size_multiplier": [-1, 4, 16, 64], "inference_input_size_multiplier": [-1], @@ -184,7 +188,7 @@ class AutoGRU(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -206,6 +210,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -241,6 +246,7 @@ def __init__( # %% ../nbs/models.ipynb 20 class AutoTCN(BaseAuto): + default_config = { "input_size_multiplier": [-1, 4, 16, 64], "inference_input_size_multiplier": [-1], @@ -249,7 +255,7 @@ class AutoTCN(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -271,6 +277,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -306,6 +313,7 @@ def __init__( # %% ../nbs/models.ipynb 23 class AutoDeepAR(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -314,7 +322,7 @@ class AutoDeepAR(BaseAuto): "lstm_dropout": tune.uniform(0.0, 0.5), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice(["robust", "minmax1"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -339,6 +347,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -372,6 +381,7 @@ def __init__( # %% ../nbs/models.ipynb 26 class AutoDilatedRNN(BaseAuto): + default_config = { "input_size_multiplier": [-1, 4, 16, 64], "inference_input_size_multiplier": [-1], @@ -382,7 +392,7 @@ class AutoDilatedRNN(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -404,6 +414,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -439,6 +450,7 @@ def __init__( # %% ../nbs/models.ipynb 30 class AutoMLP(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -446,7 +458,7 @@ class AutoMLP(BaseAuto): "num_layers": tune.randint(2, 6), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -469,6 +481,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -502,12 +515,13 @@ def __init__( # %% ../nbs/models.ipynb 33 class AutoNBEATS(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -530,6 +544,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -563,12 +578,13 @@ def __init__( # %% ../nbs/models.ipynb 36 class AutoNBEATSx(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -591,6 +607,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -624,6 +641,7 @@ def __init__( # %% ../nbs/models.ipynb 39 class AutoNHITS(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -642,7 +660,7 @@ class AutoNHITS(BaseAuto): ), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.quniform(lower=500, upper=1500, q=100), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -665,6 +683,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -698,13 +717,14 @@ def __init__( # %% ../nbs/models.ipynb 42 class AutoDLinear(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, "moving_avg_window": tune.choice([10, 25, 50]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.quniform(lower=500, upper=1500, q=100), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -727,6 +747,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -760,6 +781,7 @@ def __init__( # %% ../nbs/models.ipynb 46 class AutoTFT(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -767,7 +789,7 @@ class AutoTFT(BaseAuto): "n_head": tune.choice([4, 8]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -790,6 +812,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -823,6 +846,7 @@ def __init__( # %% ../nbs/models.ipynb 49 class AutoVanillaTransformer(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -830,7 +854,7 @@ class AutoVanillaTransformer(BaseAuto): "n_head": tune.choice([4, 8]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -853,6 +877,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -886,6 +911,7 @@ def __init__( # %% ../nbs/models.ipynb 52 class AutoInformer(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -893,7 +919,7 @@ class AutoInformer(BaseAuto): "n_head": tune.choice([4, 8]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -916,6 +942,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -949,6 +976,7 @@ def __init__( # %% ../nbs/models.ipynb 55 class AutoAutoformer(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -956,7 +984,7 @@ class AutoAutoformer(BaseAuto): "n_head": tune.choice([4, 8]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -979,6 +1007,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -1012,13 +1041,14 @@ def __init__( # %% ../nbs/models.ipynb 58 class AutoFEDformer(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, "hidden_size": tune.choice([64, 128, 256]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -1041,6 +1071,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -1074,6 +1105,7 @@ def __init__( # %% ../nbs/models.ipynb 61 class AutoPatchTST(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3], "h": None, @@ -1083,7 +1115,7 @@ class AutoPatchTST(BaseAuto): "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), "revin": tune.choice([False, True]), - "max_steps": tune.choice([500, 1000, 5000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "windows_batch_size": tune.choice([128, 256, 512, 1024]), "loss": None, @@ -1106,6 +1138,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -1139,6 +1172,7 @@ def __init__( # %% ../nbs/models.ipynb 65 class AutoTimesNet(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4, 5], "h": None, @@ -1146,7 +1180,7 @@ class AutoTimesNet(BaseAuto): "conv_hidden_size": tune.choice([32, 64, 128]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice(["robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128]), "windows_batch_size": tune.choice([32, 64, 128, 256]), "loss": None, @@ -1169,6 +1203,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -1202,6 +1237,7 @@ def __init__( # %% ../nbs/models.ipynb 69 class AutoStemGNN(BaseAuto): + default_config = { "input_size_multiplier": [1, 2, 3, 4], "h": None, @@ -1210,7 +1246,7 @@ class AutoStemGNN(BaseAuto): "multi_layer": tune.choice([3, 5, 7]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice([None, "robust", "standard"]), - "max_steps": tune.choice([500, 1000, 2000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([32, 64, 128, 256]), "loss": None, "random_seed": tune.randint(1, 20), @@ -1233,6 +1269,7 @@ def __init__( backend="ray", callbacks=None, ): + # Define search space, input/output sizes if config is None: config = self.default_config.copy() @@ -1269,6 +1306,7 @@ def __init__( # %% ../nbs/models.ipynb 73 class AutoHINT(BaseAuto): + def __init__( self, cls_model, From 84b32709e17107b1a95eeaaf20af760272f90839 Mon Sep 17 00:00:00 2001 From: marcopeix Date: Mon, 8 Apr 2024 13:22:17 -0400 Subject: [PATCH 13/14] Export models.ipynb and clean notebook --- neuralforecast/auto.py | 150 +---------------------------------------- 1 file changed, 3 insertions(+), 147 deletions(-) diff --git a/neuralforecast/auto.py b/neuralforecast/auto.py index 2858df17f..a64676dac 100644 --- a/neuralforecast/auto.py +++ b/neuralforecast/auto.py @@ -49,7 +49,6 @@ # %% ../nbs/models.ipynb 13 class AutoRNN(BaseAuto): -<<<<<<< HEAD default_config = { "input_size_multiplier": [-1, 4, 16, 64], @@ -66,145 +65,6 @@ class AutoRNN(BaseAuto): "random_seed": tune.randint(1, 20), } - def __init__( - self, - h, - loss=MAE(), - valid_loss=None, - config=None, - search_alg=BasicVariantGenerator(random_state=1), - num_samples=10, - refit_with_val=False, - cpus=cpu_count(), - gpus=torch.cuda.device_count(), - verbose=False, - backend="ray", - callbacks=None, - ): - """Auto RNN - - **Parameters:**
- - """ - # Define search space, input/output sizes - if config is None: - config = self.default_config.copy() - config["input_size"] = tune.choice( - [h * x for x in self.default_config["input_size_multiplier"]] - ) - config["inference_input_size"] = tune.choice( - [h * x for x in self.default_config["inference_input_size_multiplier"]] - ) - del ( - config["input_size_multiplier"], - config["inference_input_size_multiplier"], - ) - if backend == "optuna": - config = self._ray_config_to_optuna(config) - - super(AutoRNN, self).__init__( - cls_model=RNN, - h=h, - loss=loss, - valid_loss=valid_loss, - config=config, - search_alg=search_alg, - num_samples=num_samples, - refit_with_val=refit_with_val, - cpus=cpus, - gpus=gpus, - verbose=verbose, - backend=backend, - callbacks=callbacks, - ) - -# %% ../nbs/models.ipynb 14 -class AutoLSTM(BaseAuto): - - default_config = { - "input_size_multiplier": [-1, 4, 16, 64], - "inference_input_size_multiplier": [-1], - "h": None, - "encoder_hidden_size": tune.choice([50, 100, 200, 300]), - "encoder_n_layers": tune.randint(1, 4), - "context_size": tune.choice([5, 10, 50]), - "decoder_hidden_size": tune.choice([64, 128, 256, 512]), - "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.quniform(lower=500, upper=5000, q=500), - "batch_size": tune.choice([16, 32]), - "loss": None, - "random_seed": tune.randint(1, 20), - } - - def __init__( - self, - h, - loss=MAE(), - valid_loss=None, - config=None, - search_alg=BasicVariantGenerator(random_state=1), - num_samples=10, - refit_with_val=False, - cpus=cpu_count(), - gpus=torch.cuda.device_count(), - verbose=False, - backend="ray", - callbacks=None, - ): - - # Define search space, input/output sizes - if config is None: - config = self.default_config.copy() - config["input_size"] = tune.choice( - [h * x for x in self.default_config["input_size_multiplier"]] - ) - config["inference_input_size"] = tune.choice( - [h * x for x in self.default_config["inference_input_size_multiplier"]] - ) - del ( - config["input_size_multiplier"], - config["inference_input_size_multiplier"], - ) - if backend == "optuna": - config = self._ray_config_to_optuna(config) - - super(AutoLSTM, self).__init__( - cls_model=LSTM, - h=h, - loss=loss, - valid_loss=valid_loss, - config=config, - search_alg=search_alg, - num_samples=num_samples, - refit_with_val=refit_with_val, - cpus=cpus, - gpus=gpus, - verbose=verbose, - backend=backend, - callbacks=callbacks, - ) - -# %% ../nbs/models.ipynb 17 -class AutoGRU(BaseAuto): - -======= - ->>>>>>> main - default_config = { - "input_size_multiplier": [-1, 4, 16, 64], - "inference_input_size_multiplier": [-1], - "h": None, - "encoder_hidden_size": tune.choice([50, 100, 200, 300]), - "encoder_n_layers": tune.randint(1, 4), - "context_size": tune.choice([5, 10, 50]), - "decoder_hidden_size": tune.choice([64, 128, 256, 512]), - "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.quniform(lower=500, upper=5000, q=500), - "batch_size": tune.choice([16, 32]), - "loss": None, - "random_seed": tune.randint(1, 20), - } - def __init__( self, h, @@ -221,15 +81,11 @@ def __init__( backend="ray", callbacks=None, ): -<<<<<<< HEAD - -======= """Auto RNN **Parameters:**
""" ->>>>>>> main # Define search space, input/output sizes if config is None: config = self.get_default_config(h=h, backend=backend) @@ -278,7 +134,7 @@ class AutoLSTM(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -349,7 +205,7 @@ class AutoGRU(BaseAuto): "context_size": tune.choice([5, 10, 50]), "decoder_hidden_size": tune.choice([64, 128, 256, 512]), "learning_rate": tune.loguniform(1e-4, 1e-1), - "max_steps": tune.choice([500, 1000]), + "max_steps": tune.quniform(lower=500, upper=5000, q=500), "batch_size": tune.choice([16, 32]), "loss": None, "random_seed": tune.randint(1, 20), @@ -1537,7 +1393,7 @@ class AutoTimesNet(BaseAuto): "conv_hidden_size": tune.choice([32, 64, 128]), "learning_rate": tune.loguniform(1e-4, 1e-1), "scaler_type": tune.choice(["robust", "standard"]), - "max_steps": tune.quniform(lower=500, upper=5000, q=500), + "max_steps": tune.choice([500, 1000, 2000]), "batch_size": tune.choice([32, 64, 128]), "windows_batch_size": tune.choice([32, 64, 128, 256]), "loss": None, From 5f907dea94a799f047cb44759e678a9088fd3be2 Mon Sep 17 00:00:00 2001 From: marcopeix Date: Mon, 8 Apr 2024 13:41:31 -0400 Subject: [PATCH 14/14] Add models, fix Traffic dataset name --- experiments/nf_evaluation/datasets.py | 2 +- experiments/nf_evaluation/gpu_1.sh | 2 +- experiments/nf_evaluation/gpu_2.sh | 2 +- experiments/nf_evaluation/gpu_3.sh | 2 +- experiments/nf_evaluation/gpu_4.sh | 2 +- experiments/nf_evaluation/models.py | 4 +++- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/experiments/nf_evaluation/datasets.py b/experiments/nf_evaluation/datasets.py index 2e1d425dc..d95b09e41 100644 --- a/experiments/nf_evaluation/datasets.py +++ b/experiments/nf_evaluation/datasets.py @@ -114,7 +114,7 @@ def get_dataset(name): val_size = 5270 test_size = 10539 elif name == 'Traffic': - Y_df, *_ = LongHorizon.load(directory='./', group='Traffic') + Y_df, *_ = LongHorizon.load(directory='./', group='TrafficL') Y_df['ds'] = pd.to_datetime(Y_df['ds']) freq = 'H' h = 720 diff --git a/experiments/nf_evaluation/gpu_1.sh b/experiments/nf_evaluation/gpu_1.sh index 6a0c90e79..59cbf1ecf 100755 --- a/experiments/nf_evaluation/gpu_1.sh +++ b/experiments/nf_evaluation/gpu_1.sh @@ -2,7 +2,7 @@ datasets=("M3-yearly" "M3-quarterly" "M3-monthly" "M4-yearly") -models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST", "AutoTSMixer", "AutoiTransformer") for dataset in "${datasets[@]}"; do for model in "${models[@]}"; do diff --git a/experiments/nf_evaluation/gpu_2.sh b/experiments/nf_evaluation/gpu_2.sh index aaeeba38e..301914c31 100755 --- a/experiments/nf_evaluation/gpu_2.sh +++ b/experiments/nf_evaluation/gpu_2.sh @@ -2,7 +2,7 @@ datasets=("M4-quarterly" "M4-monthly" "M4-daily" "M4-hourly") -models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST" "AutoTSMixer", "AutoiTransformer") for dataset in "${datasets[@]}"; do for model in "${models[@]}"; do diff --git a/experiments/nf_evaluation/gpu_3.sh b/experiments/nf_evaluation/gpu_3.sh index d1a89903f..358dc38f1 100755 --- a/experiments/nf_evaluation/gpu_3.sh +++ b/experiments/nf_evaluation/gpu_3.sh @@ -2,7 +2,7 @@ datasets=("Ettm2" "Ettm1" "Etth1" "Etth2") -models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST" "AutoTSMixer", "AutoiTransformer") for dataset in "${datasets[@]}"; do for model in "${models[@]}"; do diff --git a/experiments/nf_evaluation/gpu_4.sh b/experiments/nf_evaluation/gpu_4.sh index 5553f506b..562cc8a3c 100755 --- a/experiments/nf_evaluation/gpu_4.sh +++ b/experiments/nf_evaluation/gpu_4.sh @@ -2,7 +2,7 @@ datasets=("Electricity" "Exchange" "Weather" "Traffic" "ILI") -models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST") +models=("AutoLSTM" "AutoRNN" "AutoGRU" "AutoDilatedRNN" "AutoDeepAR" "AutoTCN" "AutoMLP" "AutoNBEATS" "AutoNHITS" "AutoDLinear" "AutoTFT" "AutoVanillaTransformer" "AutoInformer" "AutoAutoformer" "AutoFEDformer" "AutoTimesNet" "AutoPatchTST" "AutoTSMixer", "AutoiTransformer") for dataset in "${datasets[@]}"; do for model in "${models[@]}"; do diff --git a/experiments/nf_evaluation/models.py b/experiments/nf_evaluation/models.py index 017c0ff03..bd6e228c7 100644 --- a/experiments/nf_evaluation/models.py +++ b/experiments/nf_evaluation/models.py @@ -24,7 +24,9 @@ def get_model(model_name, horizon, num_samples): 'AutoAutoformer': AutoAutoformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), 'AutoFEDformer': AutoFEDformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), 'AutoTimesNet': AutoTimesNet(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), - 'AutoPatchTST': AutoPatchTST(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples) + 'AutoPatchTST': AutoPatchTST(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples), + 'AutoTSMixer': AutoTSMixer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, n_series=1), + 'AutoiTransformer': AutoiTransformer(config=None, h=horizon, loss=HuberLoss(), num_samples=num_samples, n_series=1) } return model_dict[model_name] \ No newline at end of file