Draft: Initial MacOS support (very limited)

.github/workflows/osx.yml

@@ -0,0 +1,233 @@
+# GitHub CI build pipeline
+name: OSX Tests
+
+on:
+  push:
+    branches:
+      - macos-support
+jobs:
+#   build:
+#     runs-on: macos-latest
+#     strategy:
+#       matrix:
+#         python-version: ["3.8"]
+#     steps:
+#     - uses: actions/checkout@v2
+#     - name: Set up Python ${{ matrix.python-version }}
+#       uses: actions/setup-python@v2
+#       with:
+#         python-version: ${{ matrix.python-version }}
+#     - uses: actions/cache@v2
+#       id: cache-venv  # name for referring later
+#       with:
+#         path: |
+#           .venv/
+#           dist/
+#         # The cache key depends on requirements[_dev].txt
+#         key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+#         restore-keys: |
+#           ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-
+#     - name: Cleanup old artifacts
+#       run: |
+#         rm -rf .venv  # Clear venv because of a cache bug...
+#         make clean
+#     - name: Initialize Virtualenv
+#       run: |
+#         python -m pip install --upgrade pip virtualenv
+#         python -m venv .venv
+#     - name: Install dependencies
+#       run: |
+#         source .venv/bin/activate
+#         pip install -r requirements.txt
+#         pip install wheel
+#     - name: Run package creation
+#       run: |
+#         source .venv/bin/activate
+#         make dist
+#   release:
+#     needs: build
+#     runs-on: ubuntu-latest
+#     strategy:
+#       matrix:
+#         python-version: [3.8]
+#     steps:
+#     - uses: actions/checkout@v2
+#     - name: Set up Python ${{ matrix.python-version }}
+#       uses: actions/setup-python@v2
+#       with:
+#         python-version: ${{ matrix.python-version }}
+#     - uses: actions/cache@v2
+#       id: cache-venv  # name for referring later
+#       with:
+#         path: |
+#           .venv/
+#           dist/
+#         # The cache key depends on requirements[_dev].txt
+#         key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+#         restore-keys: |
+#           build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+#     - name: Archive package
+#       uses: actions/upload-artifact@v2
+#       with:
+#         name: mlonmcu
+#         path: dist/
+#   coverage:
+#     needs: build
+#     runs-on: ubuntu-latest
+#     strategy:
+#       matrix:
+#         python-version: [3.8]
+#     steps:
+#     - uses: actions/checkout@v2
+#     - name: Set up Python ${{ matrix.python-version }}
+#       uses: actions/setup-python@v2
+#       with:
+#         python-version: ${{ matrix.python-version }}
+#     - uses: actions/cache@v2
+#       id: cache-venv  # name for referring later
+#       with:
+#         path: |
+#           .venv/
+#           dist/
+#         # The cache key depends on requirements[_dev].txt
+#         key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+#         restore-keys: |
+#           build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+#     - name: Initialize Virtualenv
+#       run: |
+#         python -m pip install --upgrade pip virtualenv
+#         python -m venv .venv
+#     - name: Install dependencies
+#       run: |
+#         source .venv/bin/activate
+#         pip install -r requirements_dev.txt
+#     - name: Install package into virtualenv
+#       run: |
+#         source .venv/bin/activate
+#         make install
+#     - name: Run Unit Tests and generate coverage report
+#       run: |
+#         source .venv/bin/activate
+#         make coverage
+#     - name: Archive code coverage html report
+#       uses: actions/upload-artifact@v2
+#       with:
+#         name: code-coverage-report
+#         path: htmlcov
+#     - name: Get coverage percentage (WIP)
+#       id: report
+#       run: "##[set-output name=percent;]$(coverage report | awk '$1 == \"TOTAL\" {print $NF+0 \"%\"}')"
+#     # - name: Create coverage badge (WIP)
+#     #   uses: RubbaBoy/[email protected]
+#     #   with:
+#     #     NAME: coverage
+#     #     LABEL: 'Coverage'
+#     #     STATUS: ${{ steps.report.outputs.percent }}
+#     #     COLOR: orange
+#     #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#   docs:
+#     needs: build
+#     runs-on: ubuntu-latest
+#     strategy:
+#       matrix:
+#         python-version: [3.8]
+#     steps:
+#     - uses: actions/checkout@v2
+#     - name: Set up Python ${{ matrix.python-version }}
+#       uses: actions/setup-python@v2
+#       with:
+#         python-version: ${{ matrix.python-version }}
+#     - uses: actions/cache@v2
+#       id: cache-venv  # name for referring later
+#       with:
+#         path: |
+#           .venv/
+#           dist/
+#         # The cache key depends on requirements[_dev].txt
+#         key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+#         restore-keys: |
+#           build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+#     - name: Initialize Virtualenv
+#       run: |
+#         python -m pip install --upgrade pip virtualenv
+#         python -m venv .venv
+#     - name: Install dependencies
+#       run: | # TODO: get rid of if-else
+#         source .venv/bin/activate
+#         pip install -r requirements.txt
+#         pip install -r docs/requirements.txt
+#     - name: Install package into virtualenv
+#       run: |
+#         source .venv/bin/activate
+#         make install
+#     - name: Build docs
+#       run: |
+#         source .venv/bin/activate
+#         make docs
+#     - name: Deploy docs
+#       uses: peaceiris/actions-gh-pages@v3
+#       if: ${{ github.ref == 'refs/heads/main' }}
+#       with:
+#         github_token: ${{ secrets.GITHUB_TOKEN }}
+#         publish_dir: ./docs/_build/html
+  demo:
+    runs-on: macos-11
+    strategy:
+      matrix:
+        python-version: [3.8]
+        template: ["osx"]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    # TODO: caching
+    - name: Install SW dependencies
+      run: |
+          # brew tap riscv-software-src/riscv
+          # brew install make ninja cmake boost coreutils gnu-sed riscv-tools
+          brew install make ninja cmake boost coreutils gnu-sed dtc
+          echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH
+          echo "/usr/local/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
+    - name: Initialize Virtualenv
+      run: |
+        python -m pip install --upgrade pip
+        python -m venv .venv
+    - name: Install dependencies
+      run: |
+        source .venv/bin/activate
+        pip install -r requirements.txt
+    - name: Run package creation
+      run: |
+        source .venv/bin/activate
+        make install
+    - name: Initialize MLonMCU environment
+      run: |
+        source .venv/bin/activate
+        mlonmcu init home/ --non-interactive --template ${{ matrix.template }} --clone-models
+    - name: Install environment-specific Python packages
+      run: |
+        source .venv/bin/activate
+        MLONMCU_HOME=$(pwd)/home/ mlonmcu setup -g
+        pip install -r home/requirements_addition.txt
+    - name: Setup MLonMCU dependencies
+      run: |
+        source .venv/bin/activate
+        mlonmcu setup -H home/ -v
+    - name: Run MLonMCU flow
+      run: |
+        source .venv/bin/activate
+        mlonmcu flow run sine_model -H home/ -v
+        mlonmcu export -H home/ session.zip
+    - name: Remove dependencies (too large for artifact)
+      run: |
+        source .venv/bin/activate
+        mlonmcu cleanup -H home/ -f --deps
+    - name: Archive environment (without deps)
+      uses: actions/upload-artifact@v2
+      with:
+        name: mlonmcu_home
+        path: home/

mlonmcu/cli/main.py

@@ -71,11 +71,16 @@ def handle_docker(args):
             logger.warning(f"Docker compose process completed with exit code: {exit_code}")
         sys.exit(exit_code)
 
-    if platform.system() in ["Darwin", "Windows"]:
+    if platform.system() in ["Windows"]:
         raise RuntimeError(
             "Only Linux is supported at the Moment. If you have Docker installed, you may want to"
             + " try running this script using the `--docker` flag."
         )
+    elif platform.system() in ["Darwin"]:
+        logger.warning(
+            "Support for MacOS is currently very limited. It is recommended to use the --docker flag for optimal"
+            " compatibility."
+        )
 
 
 # def main(args):

mlonmcu/setup/tasks/cmsisnn.py

@@ -28,6 +28,8 @@
 from mlonmcu.logging import get_logger
 
 from .common import get_task_factory
+from .arm_gcc import _validate_arm_gcc
+from .riscv_gcc import _validate_riscv_gcc
 
 logger = get_logger()
 
@@ -41,13 +43,18 @@ def _validate_cmsisnn(context: MlonMcuContext, params=None):
     dsp = params.get("dsp", False)
     target_arch = params.get("target_arch", None)
     if target_arch == "arm":
+        if not _validate_arm_gcc(context, params=params):
+            return False
         if dsp and not context.environment.has_feature("arm_dsp"):
             return False
         if mvei and not context.environment.has_feature("arm_mvei"):
             return False
     else:
         if mvei or dsp:
             return False
+        if target_arch == "riscv":
+            if not _validate_riscv_gcc(context, params=params):
+                return False
     return True
 
 

mlonmcu/setup/tasks/etiss.py

@@ -63,8 +63,7 @@ def clone_etiss(
     context.cache["etiss.src_dir"] = etissSrcDir
 
 
-# @Tasks.needs(["etiss.src_dir", "llvm.install_dir"])
-@Tasks.needs(["etiss.src_dir"])
+@Tasks.needs(["etiss.src_dir", "llvm.install_dir"])
 @Tasks.provides(["etiss.build_dir", "etiss.install_dir"])
 @Tasks.param("dbg", [False, True])
 @Tasks.validate(_validate_etiss)
@@ -79,14 +78,14 @@ def build_etiss(
     etissName = utils.makeDirName("etiss", flags=flags)
     etissBuildDir = context.environment.paths["deps"].path / "build" / etissName
     etissInstallDir = context.environment.paths["deps"].path / "install" / etissName
-    # llvmInstallDir = context.cache["llvm.install_dir"]
+    llvmInstallDir = context.cache["llvm.install_dir"]
     user_vars = context.environment.vars
     if "etiss.build_dir" in user_vars or "etiss.install_dir" in user_vars:
         return False
     if rebuild or not utils.is_populated(etissBuildDir):
         utils.mkdirs(etissBuildDir)
         env = os.environ.copy()
-        # env["LLVM_DIR"] = str(llvmInstallDir)
+        env["LLVM_DIR"] = str(llvmInstallDir)
         utils.cmake(
             context.cache["etiss.src_dir"],
             "-DCMAKE_INSTALL_PREFIX=" + str(etissInstallDir),

mlonmcu/setup/tasks/tvm.py

@@ -46,6 +46,10 @@ def _validate_tvm_build(context: MlonMcuContext, params=None):
     user_vars = context.environment.vars
     use_tlcpack = user_vars.get("tvm.use_tlcpack", False)
     patch = bool(params.get("patch", False))
+    cmsisnn = bool(params.get("cmsisnn", False))
+    if cmsisnn:
+        if not (context.environment.has_feature("cmsisnnbyoc") or context.environment.has_feature("muriscvnnbyoc")):
+            return False
     if patch:
         if not context.environment.has_feature("disable_legalize"):
             return False

mlonmcu/setup/utils.py

@@ -367,6 +367,12 @@ def download_and_extract(url, archive, dest, progress=False):
                 tmp_dir_new = Path(tmp_dir) / contents[0]
                 if tmp_dir_new.is_dir():  # Archive contains a single subdirectory with a different name
                     tmp_dir = tmp_dir_new
+                    # workaround
+                    contents = list(Path(tmp_dir).glob("*"))
+                    if len(contents) == 1:
+                        tmp_dir_new = Path(tmp_dir) / contents[0]
+                        if tmp_dir_new.is_dir():  # Archive contains a single subdirectory with a different name
+                            tmp_dir = tmp_dir_new
             move(tmp_dir, dest)