Merge pull request #174 from ibm-granite/ttm_benchmark

TTM full benchmarking

notebooks/hfdemo/tinytimemixer/full_benchmarking/README.md

@@ -0,0 +1,23 @@
+# Steps to run the full benchmarking
+
+1. In terminal, the any one of the three bash scripts `granite-r2.sh`, `granite-r1.sh`, or `research-use-r2.sh`.
+2. Run `summarize_results.py`. For example, 
+```
+sh granite-r2.sh
+python summarize_results.py -rd=results-granite-r2/
+```
+
+It will run all benchmarking and dump the results. The dumped results are available in the CSV files. 
+1. TTM-Research-Use model results:
+    - `combined_results-research-use-r2.csv`: Across all datasets, all TTM models, and all forecast horizons.
+    - `combined_avg_results-research-use-r2.csv`: Across all datasets and all TTM models average over forecast horizons.
+2. TTM-Granite-R2 model results:
+    - `combined_results-granite-r2.csv`: Across all datasets, all TTM models, and all forecast horizons.
+    - `combined_avg_results-granite-r2.csv`: Across all datasets and all TTM models average over forecast horizons.
+2. TTM-Granite-R1 model results:
+    - `combined_results-granite-r1.csv`: Across all datasets, all TTM models, and all forecast horizons.
+    - `combined_avg_results-granite-r1.csv`: Across all datasets and all TTM models average over forecast horizons.  
+    Note that TTM-Granite-R1 models supports 512/1024 as context length, and 96 as the forecast horizon.
+
+# Sample benchmarking notebooks
+We also provide a bunch of sample benchmarking notebooks in the `sample_notebooks` folder. These notebooks can be directly run or modifed according to the need. 

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_avg_results-granite-r1.csv

@@ -0,0 +1,15 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,512,0.363,0.363
+etth2,512,0.286,0.285
+ettm1,512,0.415,0.36
+ettm2,512,0.186,0.175
+weather,512,0.152,0.15
+electricity,512,0.17,0.144
+traffic,512,0.509,0.401
+etth1,1024,0.362,0.36
+etth2,1024,0.281,0.28
+ettm1,1024,0.387,0.373
+ettm2,1024,0.175,0.172
+weather,1024,0.152,0.151
+electricity,1024,0.156,0.143
+traffic,1024,0.458,0.409

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_avg_results-granite-r2.csv

@@ -0,0 +1,22 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,512,0.407,0.383
+etth2,512,0.356,0.334
+ettm1,512,0.392,0.386
+ettm2,512,0.288,0.283
+weather,512,0.23,0.228
+electricity,512,0.212,0.18
+traffic,512,0.561,0.424
+etth1,1024,0.405,0.385
+etth2,1024,0.34,0.32
+ettm1,1024,0.39,0.389
+ettm2,1024,0.274,0.271
+weather,1024,0.227,0.227
+electricity,1024,0.198,0.201
+traffic,1024,0.518,0.438
+etth1,1536,0.396,0.384
+etth2,1536,0.338,0.322
+ettm1,1536,0.366,0.367
+ettm2,1536,0.256,0.256
+weather,1536,0.226,0.224
+electricity,1536,0.196,0.179
+traffic,1536,0.505,0.488

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_avg_results-research-use-r2.csv

@@ -0,0 +1,22 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,512,0.394,0.383
+etth2,512,0.345,0.324
+ettm1,512,0.386,0.376
+ettm2,512,0.281,0.272
+weather,512,0.237,0.234
+electricity,512,0.205,0.183
+traffic,512,0.564,0.434
+etth1,1024,0.404,0.385
+etth2,1024,0.335,0.318
+ettm1,1024,0.38,0.379
+ettm2,1024,0.271,0.269
+weather,1024,0.238,0.24
+electricity,1024,0.194,0.201
+traffic,1024,0.514,0.436
+etth1,1536,0.4,0.386
+etth2,1536,0.333,0.315
+ettm1,1536,0.362,0.361
+ettm2,1536,0.252,0.252
+weather,1536,0.231,0.228
+electricity,1536,0.192,0.18
+traffic,1536,0.502,0.49

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_results-granite-r1.csv

@@ -0,0 +1,15 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,512,96,0.363,0.363
+etth2,512,96,0.286,0.285
+ettm1,512,96,0.415,0.36
+ettm2,512,96,0.186,0.175
+weather,512,96,0.152,0.15
+electricity,512,96,0.17,0.144
+traffic,512,96,0.509,0.401
+etth1,1024,96,0.362,0.36
+etth2,1024,96,0.281,0.28
+ettm1,1024,96,0.387,0.373
+ettm2,1024,96,0.175,0.172
+weather,1024,96,0.152,0.151
+electricity,1024,96,0.156,0.143
+traffic,1024,96,0.458,0.409

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_results-granite-r2.csv

@@ -0,0 +1,85 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,512,96,0.363,0.362
+etth1,512,192,0.387,0.386
+etth1,512,336,0.402,0.4
+etth1,512,720,0.475,
+etth2,512,96,0.276,0.273
+etth2,512,192,0.346,0.345
+etth2,512,336,0.385,0.384
+etth2,512,720,0.419,
+ettm1,512,96,0.338,0.341
+ettm1,512,192,0.38,0.373
+ettm1,512,336,0.402,0.389
+ettm1,512,720,0.446,0.442
+ettm2,512,96,0.176,0.176
+ettm2,512,192,0.246,0.242
+ettm2,512,336,0.324,0.315
+ettm2,512,720,0.406,0.398
+weather,512,96,0.15,0.15
+weather,512,192,0.195,0.195
+weather,512,336,0.256,0.247
+weather,512,720,0.319,0.319
+electricity,512,96,0.18,0.145
+electricity,512,192,0.194,0.162
+electricity,512,336,0.213,0.183
+electricity,512,720,0.26,0.231
+traffic,512,96,0.518,0.41
+traffic,512,192,0.538,0.421
+traffic,512,336,0.571,0.44
+traffic,512,720,0.617,
+etth1,1024,96,0.359,0.359
+etth1,1024,192,0.389,0.389
+etth1,1024,336,0.409,0.406
+etth1,1024,720,0.462,
+etth2,1024,96,0.269,0.269
+etth2,1024,192,0.331,0.331
+etth2,1024,336,0.359,0.359
+etth2,1024,720,0.402,
+ettm1,1024,96,0.337,0.336
+ettm1,1024,192,0.387,0.387
+ettm1,1024,336,0.411,0.407
+ettm1,1024,720,0.427,0.427
+ettm2,1024,96,0.176,0.176
+ettm2,1024,192,0.239,0.238
+ettm2,1024,336,0.29,0.289
+ettm2,1024,720,0.391,0.383
+weather,1024,96,0.15,0.15
+weather,1024,192,0.195,0.197
+weather,1024,336,0.245,0.244
+weather,1024,720,0.318,0.316
+electricity,1024,96,0.158,0.147
+electricity,1024,192,0.181,0.173
+electricity,1024,336,0.196,0.197
+electricity,1024,720,0.255,0.286
+traffic,1024,96,0.474,0.418
+traffic,1024,192,0.496,0.435
+traffic,1024,336,0.521,0.461
+traffic,1024,720,0.581,
+etth1,1536,96,0.357,0.357
+etth1,1536,192,0.386,0.387
+etth1,1536,336,0.405,0.407
+etth1,1536,720,0.435,
+etth2,1536,96,0.274,0.277
+etth2,1536,192,0.331,0.331
+etth2,1536,336,0.356,0.357
+etth2,1536,720,0.392,
+ettm1,1536,96,0.327,0.331
+ettm1,1536,192,0.353,0.352
+ettm1,1536,336,0.387,0.386
+ettm1,1536,720,0.399,0.398
+ettm2,1536,96,0.168,0.168
+ettm2,1536,192,0.231,0.23
+ettm2,1536,336,0.276,0.276
+ettm2,1536,720,0.35,0.348
+weather,1536,96,0.15,0.149
+weather,1536,192,0.192,0.192
+weather,1536,336,0.248,0.245
+weather,1536,720,0.316,0.308
+electricity,1536,96,0.155,0.138
+electricity,1536,192,0.178,0.158
+electricity,1536,336,0.197,0.176
+electricity,1536,720,0.253,0.244
+traffic,1536,96,0.463,0.466
+traffic,1536,192,0.488,0.488
+traffic,1536,336,0.504,0.509
+traffic,1536,720,0.566,

notebooks/hfdemo/tinytimemixer/full_benchmarking/combined_results-research-use-r2.csv

@@ -0,0 +1,85 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,512,96,0.364,0.364
+etth1,512,192,0.386,0.387
+etth1,512,336,0.404,0.398
+etth1,512,720,0.424,
+etth2,512,96,0.277,0.277
+etth2,512,192,0.334,0.334
+etth2,512,336,0.362,0.361
+etth2,512,720,0.408,
+ettm1,512,96,0.322,0.311
+ettm1,512,192,0.376,0.359
+ettm1,512,336,0.407,0.396
+ettm1,512,720,0.439,0.437
+ettm2,512,96,0.171,0.171
+ettm2,512,192,0.238,0.231
+ettm2,512,336,0.304,0.293
+ettm2,512,720,0.41,0.394
+weather,512,96,0.158,0.153
+weather,512,192,0.206,0.204
+weather,512,336,0.256,0.252
+weather,512,720,0.328,0.327
+electricity,512,96,0.166,0.146
+electricity,512,192,0.191,0.164
+electricity,512,336,0.207,0.185
+electricity,512,720,0.255,0.238
+traffic,512,96,0.514,0.418
+traffic,512,192,0.544,0.425
+traffic,512,336,0.575,0.458
+traffic,512,720,0.622,
+etth1,1024,96,0.363,0.363
+etth1,1024,192,0.393,0.393
+etth1,1024,336,0.406,0.4
+etth1,1024,720,0.452,
+etth2,1024,96,0.271,0.271
+etth2,1024,192,0.324,0.325
+etth2,1024,336,0.357,0.357
+etth2,1024,720,0.388,
+ettm1,1024,96,0.327,0.328
+ettm1,1024,192,0.377,0.372
+ettm1,1024,336,0.395,0.4
+ettm1,1024,720,0.419,0.417
+ettm2,1024,96,0.178,0.178
+ettm2,1024,192,0.238,0.237
+ettm2,1024,336,0.29,0.286
+ettm2,1024,720,0.379,0.375
+weather,1024,96,0.166,0.165
+weather,1024,192,0.214,0.213
+weather,1024,336,0.254,0.262
+weather,1024,720,0.319,0.32
+electricity,1024,96,0.157,0.148
+electricity,1024,192,0.174,0.168
+electricity,1024,336,0.195,0.2
+electricity,1024,720,0.25,0.289
+traffic,1024,96,0.476,0.414
+traffic,1024,192,0.5,0.438
+traffic,1024,336,0.51,0.456
+traffic,1024,720,0.571,
+etth1,1536,96,0.359,0.359
+etth1,1536,192,0.389,0.393
+etth1,1536,336,0.405,0.406
+etth1,1536,720,0.448,
+etth2,1536,96,0.264,0.269
+etth2,1536,192,0.321,0.321
+etth2,1536,336,0.351,0.355
+etth2,1536,720,0.395,
+ettm1,1536,96,0.318,0.317
+ettm1,1536,192,0.354,0.355
+ettm1,1536,336,0.376,0.375
+ettm1,1536,720,0.398,0.398
+ettm2,1536,96,0.169,0.169
+ettm2,1536,192,0.223,0.222
+ettm2,1536,336,0.276,0.274
+ettm2,1536,720,0.342,0.341
+weather,1536,96,0.159,0.155
+weather,1536,192,0.203,0.201
+weather,1536,336,0.247,0.243
+weather,1536,720,0.314,0.315
+electricity,1536,96,0.152,0.14
+electricity,1536,192,0.179,0.159
+electricity,1536,336,0.193,0.179
+electricity,1536,720,0.243,0.241
+traffic,1536,96,0.462,0.469
+traffic,1536,192,0.491,0.491
+traffic,1536,336,0.509,0.509
+traffic,1536,720,0.547,

notebooks/hfdemo/tinytimemixer/full_benchmarking/granite-r1.sh

@@ -0,0 +1,9 @@
+data_root_path=$1
+for cl in 512 1024; do 
+    for fl in 96; do
+        python ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm-granite/granite-timeseries-ttm-r1 \
+        --data_root_path $data_root_path \
+        --save_dir results-granite-r1/
+    done;
+done;

notebooks/hfdemo/tinytimemixer/full_benchmarking/granite-r2.sh

@@ -0,0 +1,9 @@
+data_root_path=$1
+for cl in 512 1024 1536; do 
+    for fl in 96 192 336 720; do
+        python ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm-granite/granite-timeseries-ttm-r2 \
+        --data_root_path $data_root_path \
+        --save_dir results-granite-r2/
+    done;
+done;

notebooks/hfdemo/tinytimemixer/full_benchmarking/research-use-r2.sh

@@ -0,0 +1,10 @@
+data_root_path=$1
+for cl in 512 1024 1536; do 
+    for fl in 96 192 336 720; do
+        python ttm_full_benchmarking.py --context_length $cl --forecast_length $fl \
+        --num_epochs 50 --num_workers 16 --enable_prefix_tuning 1 \
+        --hf_model_path ibm/ttm-research-r2 \
+        --data_root_path $data_root_path \
+        --save_dir results-research-use-r2/
+    done;
+done;