From b017bf2710f990f24ad0de95e6c88a28bdca1a4f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 13:58:46 +0100 Subject: [PATCH 01/29] [CI] Try out building package on OSX --- .github/workflows/osx.yml | 176 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 .github/workflows/osx.yml diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml new file mode 100644 index 000000000..e2ce0e319 --- /dev/null +++ b/.github/workflows/osx.yml @@ -0,0 +1,176 @@ +# GitHub CI build pipeline +name: OSX Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + build: + runs-on: macos-latest + strategy: + matrix: + python-version: ["3.8"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + id: cache-venv # name for referring later + with: + path: | + .venv/ + dist/ + # The cache key depends on requirements[_dev].txt + key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv- + - name: Cleanup old artifacts + run: | + rm -rf .venv # Clear venv becaus of a cache bug... + make clean + - name: Initialize Virtualenv + run: | + python -m pip install --upgrade pip virtualenv + python -m venv .venv + - name: Install dependencies + run: | + source .venv/bin/activate + pip install -r requirements.txt + pip install wheel + - name: Run package creation + run: | + source .venv/bin/activate + make dist + release: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + id: cache-venv # name for referring later + with: + path: | + .venv/ + dist/ + # The cache key depends on requirements[_dev].txt + key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} + restore-keys: | + build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} + - name: Archive package + uses: actions/upload-artifact@v2 + with: + name: mlonmcu + path: dist/ + coverage: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + id: cache-venv # name for referring later + with: + path: | + .venv/ + dist/ + # The cache key depends on requirements[_dev].txt + key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} + restore-keys: | + build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} + - name: Initialize Virtualenv + run: | + python -m pip install --upgrade pip virtualenv + python -m venv .venv + - name: Install dependencies + run: | + source .venv/bin/activate + pip install -r requirements_dev.txt + - name: Install package into virtualenv + run: | + source .venv/bin/activate + make install + - name: Run Unit Tests and generate coverage report + run: | + source .venv/bin/activate + make coverage + - name: Archive code coverage html report + uses: actions/upload-artifact@v2 + with: + name: code-coverage-report + path: htmlcov + - name: Get coverage percentage (WIP) + id: report + run: "##[set-output name=percent;]$(coverage report | awk '$1 == \"TOTAL\" {print $NF+0 \"%\"}')" + # - name: Create coverage badge (WIP) + # uses: RubbaBoy/BYOB@v1.2.1 + # with: + # NAME: coverage + # LABEL: 'Coverage' + # STATUS: ${{ steps.report.outputs.percent }} + # COLOR: orange + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + docs: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + id: cache-venv # name for referring later + with: + path: | + .venv/ + dist/ + # The cache key depends on requirements[_dev].txt + key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} + restore-keys: | + build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} + - name: Initialize Virtualenv + run: | + python -m pip install --upgrade pip virtualenv + python -m venv .venv + - name: Install dependencies + run: | # TODO: get rid of if-else + source .venv/bin/activate + pip install -r requirements.txt + pip install -r docs/requirements.txt + - name: Install package into virtualenv + run: | + source .venv/bin/activate + make install + - name: Build docs + run: | + source .venv/bin/activate + make docs + - name: Deploy docs + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/main' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/_build/html + From e0eefeb92f7d6a58a775cbdcee1579cf40d0ea4e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:05:05 +0100 Subject: [PATCH 02/29] [CI] Run macos tests only on specific branch --- .github/workflows/osx.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index e2ce0e319..9c2f3e0b1 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -4,10 +4,7 @@ name: OSX Tests on: push: branches: - - main - pull_request: - branches: - - main + - macos-support jobs: build: runs-on: macos-latest From c1b1aed044e3a78189d82a0c3975dd84f7b7f3fb Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:14:10 +0100 Subject: [PATCH 03/29] [CI] Add osx.yml.j2 template and demo --- .github/workflows/osx.yml | 53 +++++++++++- resources/templates/osx.yml.j2 | 151 +++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 resources/templates/osx.yml.j2 diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 9c2f3e0b1..75af8d349 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -170,4 +170,55 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/_build/html - + demo: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.8] + template: ["osx"] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + # TODO: caching + - name: Install dependencies + run: | + true + - name: Initialize Virtualenv + run: | + python -m pip install --upgrade pip + python -m venv .venv + - name: Install dependencies + run: | + source .venv/bin/activate + pip install -r requirements.txt + - name: Run package creation + run: | + source .venv/bin/activate + make install + - name: Initialize MLonMCU environment + run: | + source .venv/bin/activate + mlonmcu init home/ --non-interactive --template ${{ matrix.template }} --clone-models + - name: Setup MLonMCU dependencies + run: | + source .venv/bin/activate + mlonmcu setup -H home/ -v + - name: Run MLonMCU flow + run: | + source .venv/bin/activate + mlonmcu flow run sine_model -H home/ -v + mlonmcu export -H home/ session.zip + - name: Remove dependencies (too large for artifact) + run: | + source .venv/bin/activate + mlonmcu cleanup -H home/ -f --deps + - name: Archive environment (without deps) + uses: actions/upload-artifact@v2 + with: + name: mlonmcu_home + path: home/ diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 new file mode 100644 index 000000000..684c0dbb3 --- /dev/null +++ b/resources/templates/osx.yml.j2 @@ -0,0 +1,151 @@ +--- +# The MLONMCU_HOME is filled in automatically when creating the environment +home: "{{ home_dir }}" +logging: + level: DEBUG + to_file: false + rotate: false +# Default locations for certain directoriescan be changed here +# Non-absolute paths will always be threated relative to the MLONMCU_HOME +paths: + # Where the dependencies are downloaded and installed + deps: deps + # If logging to file is used keep logs in this directory + logs: logs + # Location where reports and artifacts are written to + results: results + # Directory for intermediate build products, should be located on a large enough drive + temp: temp + # A collection of models which will be used to look for models + # The paths will be checked in the order defined here stopping at the first match + # Non-existant paths will be skipped without throwing an error + models: + - "{{ home_dir }}/models" + - "{{ config_dir }}/models" +# Here default clone_urls +repos: + tensorflow: # TODO: rename to tflite-micro? + url: "git@github.com:tensorflow/tflite-micro.git" + ref: main + tflite_micro_compiler: + url: "git@github.com:PhilippvK/tflite_micro_compiler.git" + ref: patches + tvm: + url: "git@github.com:tum-ei-eda/tvm.git" + ref: latest_patched + utvm_staticrt_codegen: + url: "git@github.com:tum-ei-eda/utvm_staticrt_codegen.git" + ref: master + etiss: + url: "git@github.com:tum-ei-eda/etiss.git" + ref: master + muriscvnn: + url: "git@github.com:fabianpedd/muriscv_nn.git" + ref: main + packer: + url: "git@gitlab.lrz.de:de-tum-ei-eda-esl/tflite-pack.git" + ref: main + spike: + url: "git@github.com:riscv-software-src/riscv-isa-sim.git" + ref: master + spikepk: + url: "git@github.com:riscv-software-src/riscv-pk.git" + ref: master + cmsis: + url: "https://github.com/ARM-software/CMSIS_5.git" + ref: 8c60448c0e1e50e426180b26db9bc31ddf774361 + +# Here all supported frameworks with their specific features are defined +# Optionally disable unwanted or incomatible backends or features here +# The configured defaults are used if no backend was specified in the command line options +frameworks: + default: tvm + tflite: + enabled: true + backends: + default: tflmi + tflmc: + enabled: false + tflmi: + enabled: true + features: + debug_arena: true + features: + muriscvnn: true + cmsisnn: true + tvm: + enabled: true + backends: + default: tvmaot + tvmrt: + enabled: true + features: + debug_arena: true + tvmaot: + enabled: true + features: + debug_arena: true + unpacked_api: true + usmp: true + tvmcg: + enabled: true + features: + debug_arena: true + features: + cmsisnnbyoc: false +# Some frontends are eperimental and therefore disabled here +# Features like packing are only available in certain environments +# +frontends: + tflite: + enabled: true + packed: + enabled: false + features: + packing: true + packed: true + onnx: + enabled: false + # TODO: saved_model (TF->TFLITE), ipynb (IPYNB->?) +# List of supported targets in the environment +targets: + default: etiss_pulpino + etiss_pulpino: + enabled: true + features: + gdbserver: true + etissdbg: true + trace: true + host_x86: + enabled: true + features: + gdbserver: true + spike: + enabled: false + features: + vext: true + ovpsim: + enabled: false + features: + vext: true + corstone300: + enabled: false + features: + ethosu: false + esp32: + enabled: true + esp32c3: + enabled: true +platforms: + mlif: + enabled: true + espidf: + enabled: true +postprocesses: +- detailed_cycles +# This is where further options such as specific versions of dependencies can be set in the furture +vars: + # tvm.make_tool: "ninja" + llvm.version: "11.1.0" + runs_per_stage: true + # riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi89JQF89pEiWwP6aQh7cM4H/rv32gcv.tar.xz" From 4441b597bc0bb63a60f0ac1ba80af11f59dba188 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:17:50 +0100 Subject: [PATCH 04/29] [OSX] Print warning --- mlonmcu/cli/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mlonmcu/cli/main.py b/mlonmcu/cli/main.py index 7125e6ba6..a2251a20a 100644 --- a/mlonmcu/cli/main.py +++ b/mlonmcu/cli/main.py @@ -71,11 +71,13 @@ def handle_docker(args): logger.warning(f"Docker compose process completed with exit code: {exit_code}") sys.exit(exit_code) - if platform.system() in ["Darwin", "Windows"]: + if platform.system() in ["Windows"]: raise RuntimeError( "Only Linux is supported at the Moment. If you have Docker installed, you may want to" + " try running this script using the `--docker` flag." ) + elif platform.system() in ["Darwin"]: + logger.warning("Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal compatibility.") # def main(args): From 225b642d0975652acc01906d838fd00c34ce2abb Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:18:33 +0100 Subject: [PATCH 05/29] [OSX] Only test demo in CI --- .github/workflows/osx.yml | 328 +++++++++++++++++++------------------- 1 file changed, 164 insertions(+), 164 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 75af8d349..5a03d361b 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -6,170 +6,170 @@ on: branches: - macos-support jobs: - build: - runs-on: macos-latest - strategy: - matrix: - python-version: ["3.8"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - id: cache-venv # name for referring later - with: - path: | - .venv/ - dist/ - # The cache key depends on requirements[_dev].txt - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - restore-keys: | - ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv- - - name: Cleanup old artifacts - run: | - rm -rf .venv # Clear venv becaus of a cache bug... - make clean - - name: Initialize Virtualenv - run: | - python -m pip install --upgrade pip virtualenv - python -m venv .venv - - name: Install dependencies - run: | - source .venv/bin/activate - pip install -r requirements.txt - pip install wheel - - name: Run package creation - run: | - source .venv/bin/activate - make dist - release: - needs: build - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - id: cache-venv # name for referring later - with: - path: | - .venv/ - dist/ - # The cache key depends on requirements[_dev].txt - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} - restore-keys: | - build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - - name: Archive package - uses: actions/upload-artifact@v2 - with: - name: mlonmcu - path: dist/ - coverage: - needs: build - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - id: cache-venv # name for referring later - with: - path: | - .venv/ - dist/ - # The cache key depends on requirements[_dev].txt - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} - restore-keys: | - build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - - name: Initialize Virtualenv - run: | - python -m pip install --upgrade pip virtualenv - python -m venv .venv - - name: Install dependencies - run: | - source .venv/bin/activate - pip install -r requirements_dev.txt - - name: Install package into virtualenv - run: | - source .venv/bin/activate - make install - - name: Run Unit Tests and generate coverage report - run: | - source .venv/bin/activate - make coverage - - name: Archive code coverage html report - uses: actions/upload-artifact@v2 - with: - name: code-coverage-report - path: htmlcov - - name: Get coverage percentage (WIP) - id: report - run: "##[set-output name=percent;]$(coverage report | awk '$1 == \"TOTAL\" {print $NF+0 \"%\"}')" - # - name: Create coverage badge (WIP) - # uses: RubbaBoy/BYOB@v1.2.1 - # with: - # NAME: coverage - # LABEL: 'Coverage' - # STATUS: ${{ steps.report.outputs.percent }} - # COLOR: orange - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - docs: - needs: build - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - id: cache-venv # name for referring later - with: - path: | - .venv/ - dist/ - # The cache key depends on requirements[_dev].txt - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} - restore-keys: | - build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} - - name: Initialize Virtualenv - run: | - python -m pip install --upgrade pip virtualenv - python -m venv .venv - - name: Install dependencies - run: | # TODO: get rid of if-else - source .venv/bin/activate - pip install -r requirements.txt - pip install -r docs/requirements.txt - - name: Install package into virtualenv - run: | - source .venv/bin/activate - make install - - name: Build docs - run: | - source .venv/bin/activate - make docs - - name: Deploy docs - uses: peaceiris/actions-gh-pages@v3 - if: ${{ github.ref == 'refs/heads/main' }} - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/_build/html +# build: +# runs-on: macos-latest +# strategy: +# matrix: +# python-version: ["3.8"] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - uses: actions/cache@v2 +# id: cache-venv # name for referring later +# with: +# path: | +# .venv/ +# dist/ +# # The cache key depends on requirements[_dev].txt +# key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} +# restore-keys: | +# ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv- +# - name: Cleanup old artifacts +# run: | +# rm -rf .venv # Clear venv becaus of a cache bug... +# make clean +# - name: Initialize Virtualenv +# run: | +# python -m pip install --upgrade pip virtualenv +# python -m venv .venv +# - name: Install dependencies +# run: | +# source .venv/bin/activate +# pip install -r requirements.txt +# pip install wheel +# - name: Run package creation +# run: | +# source .venv/bin/activate +# make dist +# release: +# needs: build +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: [3.8] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - uses: actions/cache@v2 +# id: cache-venv # name for referring later +# with: +# path: | +# .venv/ +# dist/ +# # The cache key depends on requirements[_dev].txt +# key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} +# restore-keys: | +# build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} +# - name: Archive package +# uses: actions/upload-artifact@v2 +# with: +# name: mlonmcu +# path: dist/ +# coverage: +# needs: build +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: [3.8] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - uses: actions/cache@v2 +# id: cache-venv # name for referring later +# with: +# path: | +# .venv/ +# dist/ +# # The cache key depends on requirements[_dev].txt +# key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} +# restore-keys: | +# build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} +# - name: Initialize Virtualenv +# run: | +# python -m pip install --upgrade pip virtualenv +# python -m venv .venv +# - name: Install dependencies +# run: | +# source .venv/bin/activate +# pip install -r requirements_dev.txt +# - name: Install package into virtualenv +# run: | +# source .venv/bin/activate +# make install +# - name: Run Unit Tests and generate coverage report +# run: | +# source .venv/bin/activate +# make coverage +# - name: Archive code coverage html report +# uses: actions/upload-artifact@v2 +# with: +# name: code-coverage-report +# path: htmlcov +# - name: Get coverage percentage (WIP) +# id: report +# run: "##[set-output name=percent;]$(coverage report | awk '$1 == \"TOTAL\" {print $NF+0 \"%\"}')" +# # - name: Create coverage badge (WIP) +# # uses: RubbaBoy/BYOB@v1.2.1 +# # with: +# # NAME: coverage +# # LABEL: 'Coverage' +# # STATUS: ${{ steps.report.outputs.percent }} +# # COLOR: orange +# # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# docs: +# needs: build +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: [3.8] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - uses: actions/cache@v2 +# id: cache-venv # name for referring later +# with: +# path: | +# .venv/ +# dist/ +# # The cache key depends on requirements[_dev].txt +# key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }} +# restore-keys: | +# build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }} +# - name: Initialize Virtualenv +# run: | +# python -m pip install --upgrade pip virtualenv +# python -m venv .venv +# - name: Install dependencies +# run: | # TODO: get rid of if-else +# source .venv/bin/activate +# pip install -r requirements.txt +# pip install -r docs/requirements.txt +# - name: Install package into virtualenv +# run: | +# source .venv/bin/activate +# make install +# - name: Build docs +# run: | +# source .venv/bin/activate +# make docs +# - name: Deploy docs +# uses: peaceiris/actions-gh-pages@v3 +# if: ${{ github.ref == 'refs/heads/main' }} +# with: +# github_token: ${{ secrets.GITHUB_TOKEN }} +# publish_dir: ./docs/_build/html demo: runs-on: macos-latest strategy: From 1e4b394aad1772c2bd2d4c4c9597adbe24c6915f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:27:52 +0100 Subject: [PATCH 06/29] [OSX] fix environment clone urls --- resources/templates/osx.yml.j2 | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index 684c0dbb3..b8888782f 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -25,31 +25,31 @@ paths: # Here default clone_urls repos: tensorflow: # TODO: rename to tflite-micro? - url: "git@github.com:tensorflow/tflite-micro.git" + url: "https://github.com/tensorflow/tflite-micro.git" ref: main tflite_micro_compiler: - url: "git@github.com:PhilippvK/tflite_micro_compiler.git" + url: "https://github.com/PhilippvK/tflite_micro_compiler.git" ref: patches tvm: - url: "git@github.com:tum-ei-eda/tvm.git" + url: "https://github.com/tum-ei-eda/tvm.git" ref: latest_patched utvm_staticrt_codegen: - url: "git@github.com:tum-ei-eda/utvm_staticrt_codegen.git" + url: "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git" ref: master etiss: - url: "git@github.com:tum-ei-eda/etiss.git" + url: "https://github.com/tum-ei-eda/etiss.git" ref: master muriscvnn: - url: "git@github.com:fabianpedd/muriscv_nn.git" + url: "https://github.com/fabianpedd/muriscv_nn.git" ref: main packer: - url: "git@gitlab.lrz.de:de-tum-ei-eda-esl/tflite-pack.git" + url: "https://gitlab.lrz.de/de-tum-ei-eda-esl/tflite-pack.git" ref: main spike: - url: "git@github.com:riscv-software-src/riscv-isa-sim.git" + url: "https://github.com/riscv-software-src/riscv-isa-sim.git" ref: master spikepk: - url: "git@github.com:riscv-software-src/riscv-pk.git" + url: "https://github.com/riscv-software-src/riscv-pk.git" ref: master cmsis: url: "https://github.com/ARM-software/CMSIS_5.git" @@ -71,7 +71,7 @@ frameworks: features: debug_arena: true features: - muriscvnn: true + muriscvnn: false cmsisnn: true tvm: enabled: true From f20b61495ea9ca7900a32d7afa92655d2a7a1543 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:37:53 +0100 Subject: [PATCH 07/29] [OSX] [CI] Install latest make and ninja using homebrew --- .github/workflows/osx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 5a03d361b..c3afe9530 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -187,7 +187,8 @@ jobs: # TODO: caching - name: Install dependencies run: | - true + brew install make + brew install ninja - name: Initialize Virtualenv run: | python -m pip install --upgrade pip From 1cc0c46b0ddf914a491e1757e84c708b7a74d979 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 14:55:47 +0100 Subject: [PATCH 08/29] [OSX] [CI] fix --- .github/workflows/osx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index c3afe9530..83f87879e 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -188,6 +188,7 @@ jobs: - name: Install dependencies run: | brew install make + echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH brew install ninja - name: Initialize Virtualenv run: | From 966bfacd8426c890cec1f820849de8ccb512e211 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 15:05:45 +0100 Subject: [PATCH 09/29] [OSX] [CI] install cmake and libbost using homebrew --- .github/workflows/osx.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 83f87879e..bf3153e1c 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -187,9 +187,8 @@ jobs: # TODO: caching - name: Install dependencies run: | - brew install make + brew install make ninja cmake boost echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH - brew install ninja - name: Initialize Virtualenv run: | python -m pip install --upgrade pip From 47cc6037c4ccad0d819a30234e514275155c9228 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 15:44:32 +0100 Subject: [PATCH 10/29] [OSX] [CI] debug llvm etiss issue --- .github/workflows/osx.yml | 2 +- mlonmcu/setup/tasks/etiss.py | 7 +++---- resources/templates/osx.yml.j2 | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index bf3153e1c..7b6cf61b6 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -187,7 +187,7 @@ jobs: # TODO: caching - name: Install dependencies run: | - brew install make ninja cmake boost + brew install make ninja cmake boost coreutils echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH - name: Initialize Virtualenv run: | diff --git a/mlonmcu/setup/tasks/etiss.py b/mlonmcu/setup/tasks/etiss.py index f3e3fc827..f1449193d 100644 --- a/mlonmcu/setup/tasks/etiss.py +++ b/mlonmcu/setup/tasks/etiss.py @@ -63,8 +63,7 @@ def clone_etiss( context.cache["etiss.src_dir"] = etissSrcDir -# @Tasks.needs(["etiss.src_dir", "llvm.install_dir"]) -@Tasks.needs(["etiss.src_dir"]) +@Tasks.needs(["etiss.src_dir", "llvm.install_dir"]) @Tasks.provides(["etiss.build_dir", "etiss.install_dir"]) @Tasks.param("dbg", [False, True]) @Tasks.validate(_validate_etiss) @@ -79,14 +78,14 @@ def build_etiss( etissName = utils.makeDirName("etiss", flags=flags) etissBuildDir = context.environment.paths["deps"].path / "build" / etissName etissInstallDir = context.environment.paths["deps"].path / "install" / etissName - # llvmInstallDir = context.cache["llvm.install_dir"] + llvmInstallDir = context.cache["llvm.install_dir"] user_vars = context.environment.vars if "etiss.build_dir" in user_vars or "etiss.install_dir" in user_vars: return False if rebuild or not utils.is_populated(etissBuildDir): utils.mkdirs(etissBuildDir) env = os.environ.copy() - # env["LLVM_DIR"] = str(llvmInstallDir) + env["LLVM_DIR"] = str(llvmInstallDir) utils.cmake( context.cache["etiss.src_dir"], "-DCMAKE_INSTALL_PREFIX=" + str(etissInstallDir), diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index b8888782f..11eec96af 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -146,6 +146,7 @@ postprocesses: # This is where further options such as specific versions of dependencies can be set in the furture vars: # tvm.make_tool: "ninja" - llvm.version: "11.1.0" runs_per_stage: true # riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi89JQF89pEiWwP6aQh7cM4H/rv32gcv.tar.xz" + llvm.version: "11.0.0" + llvm.distribution: "x86_64-apple-darwin" # TODO: automatically figure this out during setup, let user specify url instead? From b0f5a8c0f48d71f9f3306f9f11f975eeb5108aee Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 16:02:47 +0100 Subject: [PATCH 11/29] [OSX] do not build etiss -> broken --- resources/templates/osx.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index 11eec96af..06c6cb63b 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -109,9 +109,9 @@ frontends: # TODO: saved_model (TF->TFLITE), ipynb (IPYNB->?) # List of supported targets in the environment targets: - default: etiss_pulpino + default: host_x86 etiss_pulpino: - enabled: true + enabled: false features: gdbserver: true etissdbg: true From db537d656021152e5b5c1dc18c06a08a14a61dcf Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 28 Feb 2022 17:06:20 +0100 Subject: [PATCH 12/29] [OSX] fix validate bug with cmsisnn --- mlonmcu/setup/tasks/cmsisnn.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mlonmcu/setup/tasks/cmsisnn.py b/mlonmcu/setup/tasks/cmsisnn.py index 13b0b2fd3..b1f9abf77 100644 --- a/mlonmcu/setup/tasks/cmsisnn.py +++ b/mlonmcu/setup/tasks/cmsisnn.py @@ -28,6 +28,8 @@ from mlonmcu.logging import get_logger from .common import get_task_factory +from .arm_gcc import _validate_arm_gcc +from .riscv_gcc import _validate_riscv_gcc logger = get_logger() @@ -41,6 +43,8 @@ def _validate_cmsisnn(context: MlonMcuContext, params=None): dsp = params.get("dsp", False) target_arch = params.get("target_arch", None) if target_arch == "arm": + if not _validate_arm_gcc(context, params=params): + return False if dsp and not context.environment.has_feature("arm_dsp"): return False if mvei and not context.environment.has_feature("arm_mvei"): @@ -48,6 +52,9 @@ def _validate_cmsisnn(context: MlonMcuContext, params=None): else: if mvei or dsp: return False + if target_arch == "riscv": + if not _validate_riscv_gcc(context, params=params): + return False return True From 3900149060a98b66682daac7734dec823be0be7d Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 1 Mar 2022 12:55:40 +0100 Subject: [PATCH 13/29] [OSX] update osx.yml --- .github/workflows/osx.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 7b6cf61b6..950f445cd 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -185,10 +185,12 @@ jobs: with: python-version: ${{ matrix.python-version }} # TODO: caching - - name: Install dependencies + - name: Install SW dependencies run: | - brew install make ninja cmake boost coreutils + brew tap riscv-software-src/riscv + brew install make ninja cmake boost coreutils gnu-sed install riscv-tools echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH + echo "/usr/local/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH - name: Initialize Virtualenv run: | python -m pip install --upgrade pip From 31446abd65315db12178b9f7e14a5e656a3cc4ad Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 1 Mar 2022 13:00:31 +0100 Subject: [PATCH 14/29] [OSX] update template (try out spike and another llvm version) --- resources/templates/osx.yml.j2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index 06c6cb63b..d1fa4e415 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -121,9 +121,9 @@ targets: features: gdbserver: true spike: - enabled: false + enabled: true features: - vext: true + vext: false ovpsim: enabled: false features: @@ -148,5 +148,7 @@ vars: # tvm.make_tool: "ninja" runs_per_stage: true # riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi89JQF89pEiWwP6aQh7cM4H/rv32gcv.tar.xz" - llvm.version: "11.0.0" + llvm.version: "13.0.1" llvm.distribution: "x86_64-apple-darwin" # TODO: automatically figure this out during setup, let user specify url instead? + spike.exe: "/usr/local/Cellar/riscv-isa-sim/main/bin/spike" + spike.pk: "/usr/local/Cellar/riscv-pk/main/riscv64-unknown-elf/bin/pk" From a67873b143471fc95c421ad53461caf0918957ab Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 1 Mar 2022 13:06:11 +0100 Subject: [PATCH 15/29] [OSX] fix typo --- .github/workflows/osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 950f445cd..4b1b138ea 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -188,7 +188,7 @@ jobs: - name: Install SW dependencies run: | brew tap riscv-software-src/riscv - brew install make ninja cmake boost coreutils gnu-sed install riscv-tools + brew install make ninja cmake boost coreutils gnu-sed riscv-tools echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH echo "/usr/local/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH - name: Initialize Virtualenv From b3c54f16f8c8d6995a3d19f0f8362c8a42c8fc30 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Tue, 1 Mar 2022 15:32:59 +0100 Subject: [PATCH 16/29] [OSX] Update environment file prebuild spike still does not work. Reason: Brew builds a 64bit toolchain and spike does not want to use multilib... --- resources/templates/osx.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index d1fa4e415..ee4a4e19c 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -152,3 +152,5 @@ vars: llvm.distribution: "x86_64-apple-darwin" # TODO: automatically figure this out during setup, let user specify url instead? spike.exe: "/usr/local/Cellar/riscv-isa-sim/main/bin/spike" spike.pk: "/usr/local/Cellar/riscv-pk/main/riscv64-unknown-elf/bin/pk" + riscv_gcc.install_dir: "/usr/local/Cellar/riscv-gnu-toolchain/main/" + riscv_gcc.name: "riscv64-unknown-elf" From d93d5d3523b17b20c2cb28599275707d25a33fad Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 2 Mar 2022 08:06:15 +0100 Subject: [PATCH 17/29] Use prebuild RISCV toolchain on macos --- .github/workflows/osx.yml | 5 +++-- resources/templates/osx.yml.j2 | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 4b1b138ea..668d48c67 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -187,8 +187,9 @@ jobs: # TODO: caching - name: Install SW dependencies run: | - brew tap riscv-software-src/riscv - brew install make ninja cmake boost coreutils gnu-sed riscv-tools + # brew tap riscv-software-src/riscv + # brew install make ninja cmake boost coreutils gnu-sed riscv-tools + brew install make ninja cmake boost coreutils gnu-sed dtc echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH echo "/usr/local/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH - name: Initialize Virtualenv diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index ee4a4e19c..ef47160ea 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -150,7 +150,8 @@ vars: # riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi89JQF89pEiWwP6aQh7cM4H/rv32gcv.tar.xz" llvm.version: "13.0.1" llvm.distribution: "x86_64-apple-darwin" # TODO: automatically figure this out during setup, let user specify url instead? - spike.exe: "/usr/local/Cellar/riscv-isa-sim/main/bin/spike" - spike.pk: "/usr/local/Cellar/riscv-pk/main/riscv64-unknown-elf/bin/pk" - riscv_gcc.install_dir: "/usr/local/Cellar/riscv-gnu-toolchain/main/" - riscv_gcc.name: "riscv64-unknown-elf" + # spike.exe: "/usr/local/Cellar/riscv-isa-sim/main/bin/spike" + # spike.pk: "/usr/local/Cellar/riscv-pk/main/riscv32-unknown-elf/bin/pk" + riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi6cG9jUGFkRY78ztqQzZRXh/rv32gc_osx.tar.xz" + # riscv_gcc.install_dir: "/usr/local/Cellar/riscv-gnu-toolchain/main/" + # riscv_gcc.name: "riscv32-unknown-elf" From 5a33c4ab6123a1a84972141a3ea82be3918de815 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Thu, 14 Apr 2022 13:19:27 +0200 Subject: [PATCH 18/29] Fix tf version in osx template due to breaking change --- resources/templates/osx.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index ef47160ea..f3e7361bb 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -26,7 +26,7 @@ paths: repos: tensorflow: # TODO: rename to tflite-micro? url: "https://github.com/tensorflow/tflite-micro.git" - ref: main + ref: 746f880a405aeca2db6d005fd9fb8e451f3cea34 tflite_micro_compiler: url: "https://github.com/PhilippvK/tflite_micro_compiler.git" ref: patches From 719db02ae147422fc71d0b1acd240005a4345a5c Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:18:32 +0100 Subject: [PATCH 19/29] [CI][OSX] use macos bigsur --- .github/workflows/osx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 668d48c67..15e0e4672 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -29,7 +29,7 @@ jobs: # ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv- # - name: Cleanup old artifacts # run: | -# rm -rf .venv # Clear venv becaus of a cache bug... +# rm -rf .venv # Clear venv because of a cache bug... # make clean # - name: Initialize Virtualenv # run: | @@ -171,7 +171,7 @@ jobs: # github_token: ${{ secrets.GITHUB_TOKEN }} # publish_dir: ./docs/_build/html demo: - runs-on: macos-latest + runs-on: macos-11 strategy: matrix: python-version: [3.8] From 5a74d773a1776df875640c57c98d06124636e4c3 Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:19:03 +0100 Subject: [PATCH 20/29] [OSX] align osx.yml.j2 with default.yml.j2 --- resources/templates/osx.yml.j2 | 168 +++++++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 30 deletions(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index f3e7361bb..37517d7bf 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -5,6 +5,9 @@ logging: level: DEBUG to_file: false rotate: false +cleanup: + auto: true + keep: 50 # Default locations for certain directoriescan be changed here # Non-absolute paths will always be threated relative to the MLONMCU_HOME paths: @@ -14,6 +17,8 @@ paths: logs: logs # Location where reports and artifacts are written to results: results + # Directory where custom extensions can be integrated + plugins: plugins # Directory for intermediate build products, should be located on a large enough drive temp: temp # A collection of models which will be used to look for models @@ -24,55 +29,68 @@ paths: - "{{ config_dir }}/models" # Here default clone_urls repos: - tensorflow: # TODO: rename to tflite-micro? + tensorflow: url: "https://github.com/tensorflow/tflite-micro.git" - ref: 746f880a405aeca2db6d005fd9fb8e451f3cea34 + ref: f050eec7e32a0895f7658db21a4bdbd0975087a5 tflite_micro_compiler: url: "https://github.com/PhilippvK/tflite_micro_compiler.git" ref: patches tvm: - url: "https://github.com/tum-ei-eda/tvm.git" - ref: latest_patched + url: "https://github.com/apache/tvm.git" + ref: de6d8067754d746d88262c530b5241b5577b9aae utvm_staticrt_codegen: url: "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git" - ref: master + ref: 1b296f1efcc28671ee21ccbe86a81b968db8b1e5 + tvm_extensions: + url: "https://github.com/tum-ei-eda/tvm_extensions.git" + ref: 3ccd1ad8e4eed0cd86bfbd866d066833fd174dcb + muriscvnn: + url: "https://github.com/tum-ei-eda/muriscv-nn.git" + ref: c023b80a51c1b48ec62b9b092d047e9ac0bab3e8 etiss: url: "https://github.com/tum-ei-eda/etiss.git" - ref: master - muriscvnn: - url: "https://github.com/fabianpedd/muriscv_nn.git" - ref: main - packer: - url: "https://gitlab.lrz.de/de-tum-ei-eda-esl/tflite-pack.git" - ref: main + ref: 4d2d26fb1fdb17e1da3a397c35d6f8877bf3ceab spike: url: "https://github.com/riscv-software-src/riscv-isa-sim.git" - ref: master + ref: 0bc176b3fca43560b9e8586cdbc41cfde073e17a spikepk: url: "https://github.com/riscv-software-src/riscv-pk.git" - ref: master + ref: 7e9b671c0415dfd7b562ac934feb9380075d4aa2 cmsis: - url: "https://github.com/ARM-software/CMSIS_5.git" - ref: 8c60448c0e1e50e426180b26db9bc31ddf774361 + url: "https://github.com/PhilippvK/CMSIS_5.git" + ref: ad1c3cad8f1240ef14a2c55381a78d792d76ec4d + mlif: + url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" + ref: 4b9a32659f7c5340e8de26a0b8c4135ca67d64ac + espidf: + url: "https://github.com/espressif/esp-idf.git" + ref: release/v4.4 + microtvm_etissvp: + url: "https://github.com/PhilippvK/microtvm-etissvp-template.git" + ref: 95ed42e4cc1dfbcccb39a0a5b8f95ca22bbef9cc + zephyr: + url: "https://github.com/zephyrproject-rtos/zephyr.git" + ref: v3.2.0 + tflite_pack: + url: "https://github.com/tum-ei-eda/tflite-pack.git" + ref: 439b78d36456f716629ad9dbaff9734baaa75db9 # Here all supported frameworks with their specific features are defined # Optionally disable unwanted or incomatible backends or features here # The configured defaults are used if no backend was specified in the command line options frameworks: default: tvm - tflite: + tflm: enabled: true backends: default: tflmi - tflmc: - enabled: false tflmi: enabled: true features: debug_arena: true features: muriscvnn: false - cmsisnn: true + cmsisnn: false tvm: enabled: true backends: @@ -81,32 +99,96 @@ frameworks: enabled: true features: debug_arena: true + disable_legalize: false + autotuned: true + moiopt: true tvmaot: enabled: true features: debug_arena: true unpacked_api: true usmp: true + disable_legalize: false + autotuned: true + moiopt: true + tvmaotplus: + enabled: true + features: + debug_arena: true + disable_legalize: false + autotuned: true + moiopt: true + tvmllvm: + enabled: true + features: + # unpacked_api: true + disable_legalize: false + autotuned: true + moiopt: true tvmcg: enabled: true features: debug_arena: true + disable_legalize: false + autotuned: true features: cmsisnnbyoc: false -# Some frontends are eperimental and therefore disabled here + muriscvnnbyoc: false +# Some frontends are experimental and therefore disabled here # Features like packing are only available in certain environments -# frontends: tflite: enabled: true + features: + validate: true + visualize: true + split_layers: true + relay: + enabled: true + features: + relayviz: true packed: enabled: false features: packing: true packed: true onnx: + enabled: true + pb: + enabled: true + paddle: + enabled: true +# Some targets/platforms support multiple toolchains +toolchains: + gcc: true + llvm: true +# Platform extend the number of supported targets +platforms: + mlif: + enabled: true + features: + debug: true + validate: true + benchmark: true + espidf: enabled: false - # TODO: saved_model (TF->TFLITE), ipynb (IPYNB->?) + features: + debug: true + zephyr: + enabled: false + features: + debug: true + tvm: + enabled: true + features: + benchmark: true + tvm_rpc: true + autotune: true + tvm_profile: true + microtvm: + enabled: true + features: [] + # validate: true # List of supported targets in the environment targets: default: host_x86 @@ -116,6 +198,9 @@ targets: gdbserver: true etissdbg: true trace: true + log_instrs: true + # vext: true + # pext: true host_x86: enabled: true features: @@ -124,25 +209,48 @@ targets: enabled: true features: vext: false + pext: false + cachesim: true + log_instrs: true ovpsim: enabled: false features: - vext: true + vext: false + pext: false + log_instrs: true corstone300: enabled: false features: ethosu: false - esp32: + arm_mvei: true + arm_dsp: true + riscv_qemu: enabled: true - esp32c3: + features: + vext: false + microtvm_host: enabled: true -platforms: - mlif: + microtvm_spike: enabled: true - espidf: + microtvm_zephyr: + enabled: false + microtvm_arduino: + enabled: false + microtvm_etissvp: + enabled: false + microtvm_espidf: + enabled: false + tvm_cpu: enabled: true postprocesses: -- detailed_cycles + use: + # - detailed_cycles + # - average_cycles + # - filter_cols + # - features2cols + # - config2cols + # - bytes2kb + # - visualize # This is where further options such as specific versions of dependencies can be set in the furture vars: # tvm.make_tool: "ninja" From 159ee24581c28deda4050bdfda69dd642846161a Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:19:24 +0100 Subject: [PATCH 21/29] [OSX] use homebrew bottle for riscv_gcc.dl_url --- resources/templates/osx.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index 37517d7bf..a073c4714 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -260,6 +260,6 @@ vars: llvm.distribution: "x86_64-apple-darwin" # TODO: automatically figure this out during setup, let user specify url instead? # spike.exe: "/usr/local/Cellar/riscv-isa-sim/main/bin/spike" # spike.pk: "/usr/local/Cellar/riscv-pk/main/riscv32-unknown-elf/bin/pk" - riscv_gcc.dl_url: "https://syncandshare.lrz.de/dl/fi6cG9jUGFkRY78ztqQzZRXh/rv32gc_osx.tar.xz" + riscv_gcc.dl_url: "http://riscv.org.s3.amazonaws.com/bottles/riscv-gnu-toolchain-master.big_sur.bottle.4.tar.gz" # riscv_gcc.install_dir: "/usr/local/Cellar/riscv-gnu-toolchain/main/" # riscv_gcc.name: "riscv32-unknown-elf" From 62c6053e03df83d8c95fbf99198c04e22eb50a43 Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:41:51 +0100 Subject: [PATCH 22/29] OSX: lint --- mlonmcu/cli/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mlonmcu/cli/main.py b/mlonmcu/cli/main.py index a2251a20a..ebdf72084 100644 --- a/mlonmcu/cli/main.py +++ b/mlonmcu/cli/main.py @@ -77,7 +77,9 @@ def handle_docker(args): + " try running this script using the `--docker` flag." ) elif platform.system() in ["Darwin"]: - logger.warning("Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal compatibility.") + logger.warning( + "Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal compatibility." + ) # def main(args): From 962afad604973696c83b92a2fea8ca7441da013b Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:42:04 +0100 Subject: [PATCH 23/29] OSX: setup workaround --- mlonmcu/setup/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index a35c70afc..19db2ce10 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -367,6 +367,12 @@ def download_and_extract(url, archive, dest, progress=False): tmp_dir_new = Path(tmp_dir) / contents[0] if tmp_dir_new.is_dir(): # Archive contains a single subdirectory with a different name tmp_dir = tmp_dir_new + # workaround + contents = list(Path(tmp_dir).glob("*")) + if len(contents) == 1: + tmp_dir_new = Path(tmp_dir) / contents[0] + if tmp_dir_new.is_dir(): # Archive contains a single subdirectory with a different name + tmp_dir = tmp_dir_new move(tmp_dir, dest) From e4405b2cf49b607935120ed1b0f640aecdf8f659 Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:47:37 +0100 Subject: [PATCH 24/29] OSX: disable spike --- resources/templates/osx.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index a073c4714..b7dd5e420 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -206,7 +206,7 @@ targets: features: gdbserver: true spike: - enabled: true + enabled: false features: vext: false pext: false @@ -231,7 +231,7 @@ targets: microtvm_host: enabled: true microtvm_spike: - enabled: true + enabled: false microtvm_zephyr: enabled: false microtvm_arduino: From 37dd03d21a56892a994b1b2cbfd043189619f2e9 Mon Sep 17 00:00:00 2001 From: "Philipp v. K" Date: Fri, 3 Mar 2023 10:47:41 +0100 Subject: [PATCH 25/29] OSX: lint --- mlonmcu/cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlonmcu/cli/main.py b/mlonmcu/cli/main.py index ebdf72084..0a5e0c134 100644 --- a/mlonmcu/cli/main.py +++ b/mlonmcu/cli/main.py @@ -78,7 +78,8 @@ def handle_docker(args): ) elif platform.system() in ["Darwin"]: logger.warning( - "Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal compatibility." + "Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal" + " compatibility." ) From 4a1fbc2a93414a9d27dd50dfcd99fdf7b80a5a82 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 4 Mar 2023 08:03:22 +0100 Subject: [PATCH 26/29] osx: fix --- .github/workflows/osx.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 15e0e4672..dd91d6b0d 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -208,6 +208,11 @@ jobs: run: | source .venv/bin/activate mlonmcu init home/ --non-interactive --template ${{ matrix.template }} --clone-models + - name: Install environment-specific Python packages + run: | + source .venv/bin/activate + MLONMCU_HOME=$(pwd)/home/ mlonmcu setup -g + pip install -r home/requirements_addition.txt - name: Setup MLonMCU dependencies run: | source .venv/bin/activate From 03d8428ed9a42cc4d2c7e7b2f24cf589ac246db7 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 4 Mar 2023 08:38:39 +0100 Subject: [PATCH 27/29] fixup --- .github/workflows/osx.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index dd91d6b0d..715bb2ba3 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -209,10 +209,10 @@ jobs: source .venv/bin/activate mlonmcu init home/ --non-interactive --template ${{ matrix.template }} --clone-models - name: Install environment-specific Python packages - run: | - source .venv/bin/activate - MLONMCU_HOME=$(pwd)/home/ mlonmcu setup -g - pip install -r home/requirements_addition.txt + run: | + source .venv/bin/activate + MLONMCU_HOME=$(pwd)/home/ mlonmcu setup -g + pip install -r home/requirements_addition.txt - name: Setup MLonMCU dependencies run: | source .venv/bin/activate From 34dc634afc88bf37cf18c99cdfa80dd0d067c596 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 4 Mar 2023 13:19:39 +0100 Subject: [PATCH 28/29] osx: update mlif ref --- resources/templates/osx.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/osx.yml.j2 b/resources/templates/osx.yml.j2 index b7dd5e420..5a1a5f405 100644 --- a/resources/templates/osx.yml.j2 +++ b/resources/templates/osx.yml.j2 @@ -61,7 +61,7 @@ repos: ref: ad1c3cad8f1240ef14a2c55381a78d792d76ec4d mlif: url: "https://github.com/tum-ei-eda/mlonmcu-sw.git" - ref: 4b9a32659f7c5340e8de26a0b8c4135ca67d64ac + ref: f663e263a0ab8a93613c09c0bbbc8b82213eccaf espidf: url: "https://github.com/espressif/esp-idf.git" ref: release/v4.4 From acd4fae4befc4a129c1c2e15d0f6d5213aaa6ee9 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 4 Mar 2023 13:26:41 +0100 Subject: [PATCH 29/29] fix: cmsisnnbyoc build validate --- mlonmcu/setup/tasks/tvm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mlonmcu/setup/tasks/tvm.py b/mlonmcu/setup/tasks/tvm.py index 4465a1016..64a126bf9 100644 --- a/mlonmcu/setup/tasks/tvm.py +++ b/mlonmcu/setup/tasks/tvm.py @@ -46,6 +46,10 @@ def _validate_tvm_build(context: MlonMcuContext, params=None): user_vars = context.environment.vars use_tlcpack = user_vars.get("tvm.use_tlcpack", False) patch = bool(params.get("patch", False)) + cmsisnn = bool(params.get("cmsisnn", False)) + if cmsisnn: + if not (context.environment.has_feature("cmsisnnbyoc") or context.environment.has_feature("muriscvnnbyoc")): + return False if patch: if not context.environment.has_feature("disable_legalize"): return False