use google stoarge for slow tests

.github/workflows/cairo-ci.yaml

@@ -37,15 +37,14 @@ jobs:
         working-directory: ./stwo_cairo_prover
     steps:
       - uses: actions/checkout@v3
-        with:
-          lfs: false
+      - run: ./scripts/fetch_large_files.sh 
       - uses: dtolnay/rust-toolchain@master
         with:
           components: rustfmt
           profile: minimal
           toolchain: nightly-2024-12-16
       - uses: Swatinem/rust-cache@v2
-      - run: cargo +nightly-2024-12-16 test --release
+      - run: cargo +nightly-2024-12-16 test --release --features slow-tests
 
   format:
     runs-on: ubuntu-latest

README.md

@@ -1,11 +1 @@
 # stwo-cairo-air
-
-This repo is using Git LFS
-```
-https://packagecloud.io/github/git-lfs/install
-sudo apt-get update
-sudo apt-get install git-lfs
-git lfs install
-git lfs fetch --all
-git lfs pull
-```

stwo_cairo_prover/.gitignore

@@ -1 +1,2 @@
 /target
+/crates/prover/test_data

stwo_cairo_prover/crates/prover/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [features]
 parallel = ["rayon"]
+slow-tests = []
 
 [dependencies]
 air_structs_derive = { path = "../air_structs_derive" }

stwo_cairo_prover/crates/prover/src/input/README.md

@@ -0,0 +1,31 @@
+# Slow Tests Using Google Cloud Storage
+
+[Google Cloud Storage Browser](https://console.cloud.google.com/storage/browser/stwo-cairo-testing-artifacts?project=starkware-thirdparties)
+
+## How to Test
+
+1. **Download Tests Files**  
+   ```bash
+   ./scripts/fetch_large_files.sh
+   ```
+
+2. **Execute Tests**  
+   ```bash
+   cargo test test_read_from_large_files --features "slow-tests"
+   cargo test test_read_from_small_file --features "slow-tests"
+   ```
+
+## Adding a New "Slow-Test"
+
+1. **Create a Directory**  
+   Create a new directory at the Google Cloud Storage URL above and add the required files.
+
+2. **Mark the Test**  
+   Mark the new test with `#[cfg(feature = "slow-tests")]`.
+
+3. **Run Tests Locally**  
+    Run the test locally using:
+    ```bash
+    cargo test "<test_name>" --features "slow-tests"
+
+

stwo_cairo_prover/crates/prover/src/input/vm_import/mod.rs

@@ -127,14 +127,13 @@ impl<R: Read> Iterator for MemEntryIter<'_, R> {
 
 #[cfg(test)]
 pub mod tests {
-
     use std::path::PathBuf;
 
     use super::*;
 
     pub fn large_cairo_input() -> CairoInput {
         let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        d.push("test_data/large_cairo_input");
+        d.push("test_data/test_read_from_large_files");
 
         import_from_vm_output(
             d.join("pub.json").as_path(),
@@ -143,27 +142,27 @@ pub mod tests {
         )
         .expect(
             "
-            Failed to read test files. Maybe git-lfs is not installed? Checkout README.md.",
+            Failed to read test files. Checkout input/README.md.",
         )
     }
 
     pub fn small_cairo_input() -> CairoInput {
         let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        d.push("test_data/small_cairo_input");
+        d.push("test_data/test_read_from_small_files");
         import_from_vm_output(
             d.join("pub.json").as_path(),
             d.join("priv.json").as_path(),
             false,
         )
         .expect(
             "
-            Failed to read test files. Maybe git-lfs is not installed? Checkout README.md.",
+            Failed to read test files. Checkout input/README.md.",
         )
     }
 
     // TODO (Stav): Once all the components are in, verify the proof to ensure the sort was correct.
-    #[ignore]
     #[test]
+    #[cfg(feature = "slow-tests")]
     fn test_read_from_large_files() {
         let input = large_cairo_input();
         let components = input.state_transitions.casm_states_by_opcode;
@@ -228,7 +227,8 @@ pub mod tests {
         assert_eq!(components.ret_opcode.len(), 49472);
     }
 
-    #[ignore]
+    // TODO (Stav): Once all the components are in, verify the proof to ensure the sort was correct.
+    #[cfg(feature = "slow-tests")]
     #[test]
     fn test_read_from_small_files() {
         let input = small_cairo_input();

stwo_cairo_prover/scripts/fetch_large_files.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+BUCKET_URL="https://storage.googleapis.com/stwo-cairo-testing-artifacts"
+DEST_DIR="crates/prover/test_data"
+
+# Create the destination directory
+mkdir -p "${DEST_DIR}"
+
+curl -s "${BUCKET_URL}" | grep -oP '(?<=<Key>).*?(?=</Key>)' | while read -r FILE; do
+    echo "Downloading ${FILE}..."
+
+    # Create the necessary directories
+    mkdir -p "${DEST_DIR}/$(dirname "${FILE}")"
+
+    # Download the file
+    curl -Lo "${DEST_DIR}/${FILE}" "${BUCKET_URL}/${FILE}" || {
+        echo "Failed to download ${FILE}" >&2
+    }
+done