Merge pull request #198 from ibm-granite/vj_1

foundts comparison

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_avg_results-ttm-a.csv

@@ -0,0 +1,10 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,1536,0.396,
+etth2,1536,0.338,
+ettm1,1536,0.366,
+ettm2,1536,0.256,
+weather,1536,0.226,
+electricity,1536,0.196,
+traffic,1536,0.505,
+exchange,1536,0.356,
+zafnoo,1536,0.505,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_avg_results-ttm-b1.csv

@@ -0,0 +1,10 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,512,0.394,
+etth2,512,0.345,
+ettm1,512,0.386,
+ettm2,512,0.281,
+weather,512,0.237,
+electricity,512,0.205,
+traffic,512,0.564,
+exchange,512,0.438,
+zafnoo,512,0.527,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_avg_results-ttm-b2.csv

@@ -0,0 +1,10 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,512,0.407,
+etth2,512,0.356,
+ettm1,512,0.392,
+ettm2,512,0.288,
+weather,512,0.23,
+electricity,512,0.212,
+traffic,512,0.561,
+exchange,512,0.387,
+zafnoo,512,0.517,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_avg_results-ttm-e.csv

@@ -0,0 +1,10 @@
+dataset,CL,zs_mse,fs5_mse
+etth1,1024,0.405,
+etth2,1024,0.34,
+ettm1,1024,0.39,
+ettm2,1024,0.274,
+weather,1024,0.227,
+electricity,1024,0.198,
+traffic,1024,0.518,
+exchange,1024,0.404,
+zafnoo,1024,0.496,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_results-ttm-a.csv

@@ -0,0 +1,37 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,1536,96,0.357,
+etth1,1536,192,0.386,
+etth1,1536,336,0.405,
+etth1,1536,720,0.435,
+etth2,1536,96,0.274,
+etth2,1536,192,0.331,
+etth2,1536,336,0.356,
+etth2,1536,720,0.392,
+ettm1,1536,96,0.327,
+ettm1,1536,192,0.353,
+ettm1,1536,336,0.387,
+ettm1,1536,720,0.399,
+ettm2,1536,96,0.168,
+ettm2,1536,192,0.231,
+ettm2,1536,336,0.276,
+ettm2,1536,720,0.35,
+weather,1536,96,0.15,
+weather,1536,192,0.192,
+weather,1536,336,0.248,
+weather,1536,720,0.316,
+electricity,1536,96,0.155,
+electricity,1536,192,0.178,
+electricity,1536,336,0.197,
+electricity,1536,720,0.253,
+traffic,1536,96,0.463,
+traffic,1536,192,0.488,
+traffic,1536,336,0.504,
+traffic,1536,720,0.566,
+exchange,1536,96,0.092,
+zafnoo,1536,96,0.425,
+exchange,1536,192,0.185,
+zafnoo,1536,192,0.473,
+exchange,1536,336,0.385,
+zafnoo,1536,336,0.518,
+exchange,1536,720,0.761,
+zafnoo,1536,720,0.604,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_results-ttm-b1.csv

@@ -0,0 +1,37 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,512,96,0.364,
+etth1,512,192,0.386,
+etth1,512,336,0.404,
+etth1,512,720,0.424,
+etth2,512,96,0.277,
+etth2,512,192,0.334,
+etth2,512,336,0.362,
+etth2,512,720,0.408,
+ettm1,512,96,0.322,
+ettm1,512,192,0.376,
+ettm1,512,336,0.407,
+ettm1,512,720,0.439,
+ettm2,512,96,0.171,
+ettm2,512,192,0.238,
+ettm2,512,336,0.304,
+ettm2,512,720,0.41,
+weather,512,96,0.158,
+weather,512,192,0.206,
+weather,512,336,0.256,
+weather,512,720,0.328,
+electricity,512,96,0.166,
+electricity,512,192,0.191,
+electricity,512,336,0.207,
+electricity,512,720,0.255,
+traffic,512,96,0.514,
+traffic,512,192,0.544,
+traffic,512,336,0.575,
+traffic,512,720,0.622,
+exchange,512,96,0.116,
+zafnoo,512,96,0.436,
+exchange,512,192,0.253,
+zafnoo,512,192,0.503,
+exchange,512,336,0.428,
+zafnoo,512,336,0.558,
+exchange,512,720,0.954,
+zafnoo,512,720,0.611,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_results-ttm-b2.csv

@@ -0,0 +1,37 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,512,96,0.363,
+etth1,512,192,0.387,
+etth1,512,336,0.402,
+etth1,512,720,0.475,
+etth2,512,96,0.276,
+etth2,512,192,0.346,
+etth2,512,336,0.385,
+etth2,512,720,0.419,
+ettm1,512,96,0.338,
+ettm1,512,192,0.38,
+ettm1,512,336,0.402,
+ettm1,512,720,0.446,
+ettm2,512,96,0.176,
+ettm2,512,192,0.246,
+ettm2,512,336,0.324,
+ettm2,512,720,0.406,
+weather,512,96,0.15,
+weather,512,192,0.195,
+weather,512,336,0.256,
+weather,512,720,0.319,
+electricity,512,96,0.18,
+electricity,512,192,0.194,
+electricity,512,336,0.213,
+electricity,512,720,0.26,
+traffic,512,96,0.518,
+traffic,512,192,0.538,
+traffic,512,336,0.571,
+traffic,512,720,0.617,
+exchange,512,96,0.094,
+zafnoo,512,96,0.431,
+exchange,512,192,0.189,
+zafnoo,512,192,0.488,
+exchange,512,336,0.315,
+zafnoo,512,336,0.548,
+exchange,512,720,0.949,
+zafnoo,512,720,0.602,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/combined_results-ttm-e.csv

@@ -0,0 +1,37 @@
+dataset,CL,FL,zs_mse,fs5_mse
+etth1,1024,96,0.359,
+etth1,1024,192,0.389,
+etth1,1024,336,0.409,
+etth1,1024,720,0.462,
+etth2,1024,96,0.269,
+etth2,1024,192,0.331,
+etth2,1024,336,0.359,
+etth2,1024,720,0.402,
+ettm1,1024,96,0.337,
+ettm1,1024,192,0.387,
+ettm1,1024,336,0.411,
+ettm1,1024,720,0.427,
+ettm2,1024,96,0.176,
+ettm2,1024,192,0.239,
+ettm2,1024,336,0.29,
+ettm2,1024,720,0.391,
+weather,1024,96,0.15,
+weather,1024,192,0.195,
+weather,1024,336,0.245,
+weather,1024,720,0.318,
+electricity,1024,96,0.158,
+electricity,1024,192,0.181,
+electricity,1024,336,0.196,
+electricity,1024,720,0.255,
+traffic,1024,96,0.474,
+traffic,1024,192,0.496,
+traffic,1024,336,0.521,
+traffic,1024,720,0.581,
+exchange,1024,96,0.092,
+zafnoo,1024,96,0.417,
+exchange,1024,192,0.211,
+zafnoo,1024,192,0.469,
+exchange,1024,336,0.413,
+zafnoo,1024,336,0.509,
+exchange,1024,720,0.901,
+zafnoo,1024,720,0.591,

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/readme.md

@@ -0,0 +1,49 @@
+# NOTE TO FOUNDTS LEADERBOARD TEAM
+
+
+
+- [FoundTS-Leaderboard](https://arxiv.org/abs/2410.11802) provides a general benchmarking framework for zero-shot and few-shot forecasting using time-series foundation models.
+- Thanks to the authors of the paper for providing this benchmarking system which is very essential in the current literature space.
+- However, we observe that the TTM numbers reported as part of the benchmarking is not computed correctly. Hence we are sharing the scripts here to rightly run the TTM models on the benchmarked datasets (ETTH1, ETTH2, ETTM1, ETTM2, Weather, Traffic, Electricity, Zafnoo, Exchange). Please note that, Solar dataset is used as part of the TTM pretraining. Hence TTM's results on Solar is extremely good and should not be considered for Zero-shot evaluation. Likewise, Weather and Electricity is part of TimesFM pretraining data and Traffic is part of both TimesFM and Moirai. Hence, we should exclude these results in zero/few-shot comparisons.
+- Since TTM requires atleast 512 time-points in the context length, we ignore short-context length datasets such as ILI, NASDAQ, NN5, Wike2000 where TTM cannot be applied currently.
+- We request the authors to update their benchmarking/leaderboard table accordingly.
+- As per the below zeroshot results, TTM outperform most of the datasets by a good margin. 
+
+
+
+# Updated Zeroshot FoundTS Table with correct TTM numbers
+
+![fig1](foundts_zeroshot_corrected_view.jpg)
+![fig2](foundts_zeroshot_corrected_view_avg.jpg)
+
+
+# Steps to reproduce
+
+## Fetching the data
+The evaluation data can be downloaded from any of the previous time-series github repos like autoformer or timesnet or informer or foundTS repo. [Sample download link](https://drive.google.com/drive/folders/1vE0ONyqPlym2JaaAoEe0XNDR8FS_d322). The ETT datasets can also be downloaded from [ETT-Github-Repository](https://github.com/zhouhaoyi/ETDataset). 
+
+Download and save the datasets in a directory. For example, in `data_root_path`. 
+
+CSVs of each data should reside in location `data_root_path/$dataset_name/$dataset_name.csv` for our data utils to process them automatically.
+
+
+
+## Running the scripts
+
+1. In terminal, run `sh ttm-a-1536.sh`
+2. Run `summarize_results.py`. For example, 
+
+```
+sh ttm-a-1536.sh data_root_path/
+python summarize_results.py -rd=results-ttm-a/
+```
+
+It will run all benchmarking and dump the results. The dumped results are available in the CSV files. 
+
+## Different TTM Models:
+- TTM-B1: Refers to TTM results with 512 context length and frequency prefix tuning enabled: `ttm-b1-512.sh`
+- TTM-B2: Refers to TTM results with 512 context length: `ttm-b1-512.sh`
+- TTM-E: Refers to TTM results with 1024 context length: `ttm-b1-512.sh`
+- TTM-A: Refers to TTM results with 1536 context length: `ttm-b1-512.sh`
+
+

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/ttm-a-1536.sh

@@ -0,0 +1,12 @@
+data_root_path=$1
+for cl in 1536; do 
+    for fl in 96 192 336 720; do
+        python ../ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm-granite/granite-timeseries-ttm-r2 \
+        --data_root_path $data_root_path \
+        --fewshot 0 \
+        --plot 0 \
+        --datasets etth1,etth2,ettm1,ettm2,weather,electricity,traffic,exchange,zafnoo \
+        --save_dir results-ttm-a/
+    done;
+done;

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/ttm-b1-512.sh

@@ -0,0 +1,13 @@
+data_root_path=$1
+for cl in 512; do 
+    for fl in 96 192 336 720; do
+        python ../ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm/ttm-research-r2  \
+        --data_root_path $data_root_path \
+        --fewshot 0 \
+        --plot 0 \
+        --enable_prefix_tuning 1 \
+        --datasets etth1,etth2,ettm1,ettm2,weather,electricity,traffic,exchange,zafnoo \
+        --save_dir results-ttm-b1/
+    done;
+done;

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/ttm-b2-512.sh

@@ -0,0 +1,12 @@
+data_root_path=$1
+for cl in 512; do 
+    for fl in 96 192 336 720; do
+        python ../ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm-granite/granite-timeseries-ttm-r2 \
+        --data_root_path $data_root_path \
+        --fewshot 0 \
+        --plot 0 \
+        --datasets etth1,etth2,ettm1,ettm2,weather,electricity,traffic,exchange,zafnoo \
+        --save_dir results-ttm-b2/
+    done;
+done;

notebooks/hfdemo/tinytimemixer/full_benchmarking/foundts_leaderboard/ttm-e-1024.sh

@@ -0,0 +1,12 @@
+data_root_path=$1
+for cl in 1024; do 
+    for fl in 96 192 336 720; do
+        python ../ttm_full_benchmarking.py --context_length $cl --forecast_length $fl --num_epochs 50 --num_workers 16 \
+        --hf_model_path ibm-granite/granite-timeseries-ttm-r2 \
+        --data_root_path $data_root_path \
+        --fewshot 0 \
+        --plot 0 \
+        --datasets etth1,etth2,ettm1,ettm2,weather,electricity,traffic,exchange,zafnoo \
+        --save_dir results-ttm-e/
+    done;
+done;