combining sdk and cli draft 1

.github/workflows/ci-tests.yml

@@ -1,36 +1,45 @@
-name: "ci tests"
-
+name: Rust checks and tests
 on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - "**.md"
   pull_request:
     paths-ignore:
       - "**.md"
+      - ".gitignore"
+
   workflow_dispatch:
     inputs:
       test-macos-and-windows:
         description: "run macOS and Windows tests"
         required: true
         default: false
         type: boolean
-  merge_group:
-    types: [checks_requested]
+
+concurrency:
+  group: push-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   # Not needed in CI, should make things a bit faster
   CARGO_INCREMENTAL: 0
   CARGO_TERM_COLOR: always
-  # Build smaller artifacts to avoid running out of space in CI
-  # TODO: Try to remove once https://github.com/paritytech/substrate/issues/11538 is resolved
-  RUSTFLAGS: -C strip=symbols -C opt-level=s
-  # Remove unnecessary WASM build artefacts
-  WASM_BUILD_CLEAN_TARGET: 1
+  MAX_TARGET_SIZE: 1024 # MB
+  # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to at least
+  #  1.61: https://github.com/RustCrypto/block-ciphers/issues/373
+  RUSTFLAGS: -C strip=symbols -C opt-level=s --cfg aes_armv8
 
 jobs:
-  clippy-fmt-test:
+  fmt:
+    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }}
+    steps:
+      - name: git checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+      - name: cargo fmt
+        uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
     strategy:
       fail-fast: false
       matrix:
@@ -45,15 +54,14 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0
+      - name: git checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
       # On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@8852e4d5c58653ed05135c0a5d949d9c2febcb00 # v1.6.1
         with:
           version: "15.0"
-        if: runner.os == 'macOS'
 
       - name: Install Protoc
         uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9 # v2.0.0
@@ -69,35 +77,129 @@ jobs:
       - name: Remove msys64
         run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
         if: runner.os == 'Windows'
-        # May not exist on self-hosted runners
         continue-on-error: true
 
-      - name: cargo fmt
-        uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
+      - name: Configure cache
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # @v3.3.2
         with:
-          command: fmt
-          args: --all -- --check
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/bin
+            ~/.cargo/git
+          key: ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ runner.arch }}-cargo-
 
-      - name: cargo test
-        uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
-        with:
-          command: test
+      - name: Clean unused crate source checkouts and git repo checkouts
+        run: cargo cache
 
-      - name: cargo clippy --locked (Linux & Windows)
-        uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # @v1.0.7
+      - name: cargo clippy (Linux & Windows)
+        uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # v1.0.7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --locked --all-targets -- -D warnings
         if: runner.os != 'macOS'
 
-      - name: cargo clippy --locked (macOS)
+      - name: cargo clippy (MacOS)
         uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # @v1.0.7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --locked --all-targets --no-default-features -- -D warnings
+          args: --locked  --no-default-features -- -D warnings
         if: runner.os == 'macOS'
 
+      - name: Clean unused artifacts
+        run: cargo sweep --maxsize ${{ env.MAX_TARGET_SIZE }}
+
+  test:
+    strategy:
+      matrix:
+        os: ${{ fromJson(github.repository_owner == 'subspace' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-20.04", "macos-12", "windows-2022"]') }}
+        run-all:
+          - ${{ inputs.test-macos-and-windows || github.ref == 'refs/heads/master' }}
+        exclude: # exclude macos-12 and window-2022 when the condition is false
+          - run-all: false
+            os: macos-12
+          - run-all: false
+            os: windows-2022
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: git checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+      # On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support
+      - name: Install LLVM and Clang
+        uses: KyleMayes/install-llvm-action@8852e4d5c58653ed05135c0a5d949d9c2febcb00 # v1.6.1
+        with:
+          version: "15.0"
+        if: runner.os == 'macOS'
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9 # v2.0.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Needed for hwloc
+      - name: Install automake (macOS)
+        run: brew install automake
+        if: runner.os == 'macOS'
+
+      # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
+      - name: Remove msys64
+        run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
+        if: runner.os == 'Windows'
+        continue-on-error: true
+
+      - name: Add cache
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # @v3.3.2
+        if: runner.os != 'Windows'
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/bin
+            ~/.cargo/git
+            target
+          key: test-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: test-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}
+
+      - name: Install utils for caching
+        if: runner.os != 'Windows'
+        run: |
+          test -x ~/.cargo/bin/cargo-sweep || cargo install cargo-sweep
+          test -x ~/.cargo/bin/cargo-cache || cargo install cargo-cache --no-default-features --features ci-autoclean cargo-cache
+
+      - name: Clean unused crate source checkouts and git repo checkouts
+        if: runner.os != 'Windows'
+        run: cargo cache
+
+      - name: Build and run tests (linux & windows)
+        run: cd sdk && cargo test --locked --release -- --test-threads=1 --nocapture && cd .. && cargo test --locked --release -- --test-threads=1 --nocapture
+        if: runner.os != 'macOS'
+
+      - name: Build and run tests (macOS)
+        run: cd sdk && cargo test --locked --release --no-default-features -- --test-threads=1 --nocapture && cd .. && cargo test --locked --release --no-default-features -- --test-threads=1 --nocapture
+        if: runner.os == 'macOS'
+
+      - name: Clean unused artifacts
+        if: runner.os != 'Windows'
+        run: cargo sweep --maxsize ${{ env.MAX_TARGET_SIZE }}
+
+  docs:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: git checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9 # v2.0.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure cache
+        uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
+
       - name: Check Documentation
-        run: cargo doc --locked --all --no-deps
+        run: cargo doc --locked --no-deps --lib
         env:
           RUSTDOCFLAGS: "-D rustdoc::broken-intra-doc-links -D rustdoc::private_intra_doc_links"

Cargo.toml

@@ -29,7 +29,7 @@ single-instance = "0.3.3"
 sp-core = { version = "21.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", features = ["full_crypto"] }
 strum = "0.24.1"
 strum_macros = "0.24.3"
-subspace-sdk = { git = "https://github.com/subspace/subspace-sdk", rev = "000c6c774f3dd995e783d6d78d1d59669540b454", default-features = false }
+subspace-sdk = { path = "./sdk", default-features = false }
 thiserror = "1"
 tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread", "tracing", "signal"] }
 toml = "0.7"
@@ -111,7 +111,6 @@ httparse = { opt-level = 3 }
 integer-sqrt = { opt-level = 3 }
 k256 = { opt-level = 3 }
 keccak = { opt-level = 3 }
-libm = { opt-level = 3 }
 libsecp256k1 = { opt-level = 3 }
 libz-sys = { opt-level = 3 }
 mio = { opt-level = 3 }
@@ -132,7 +131,6 @@ subspace-core-primitives = { opt-level = 3 }
 subspace-archiving = { opt-level = 3 }
 twox-hash = { opt-level = 3 }
 uint = { opt-level = 3 }
-wasmi = { opt-level = 3 }
 x25519-dalek = { opt-level = 3 }
 yamux = { opt-level = 3 }
 zeroize = { opt-level = 3 }

sdk/.gitignore

@@ -0,0 +1 @@
+/target

sdk/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added 
+
+- Added initial support for farmer
+  - Starting
+  - Farming
+  - Getting info
+  - Getting events
+    - Receiving new solutions
+    - Receiving updates on plotting progress
+  - Closing
+  - Wiping state
+
+- Added initial node support
+  - Starting
+  - Syncing
+  - Getting info
+  - Getting events
+    - Receiving new blocks
+    - Receiving updates on plotting progress
+  - Extensive customization
+  - Support different chain specs
+  - Closing
+  - Wiping state