diff --git a/.github/workflows/beta-build.yml b/.github/workflows/beta-build.yml index c5c09dcb5..bdc277231 100644 --- a/.github/workflows/beta-build.yml +++ b/.github/workflows/beta-build.yml @@ -67,7 +67,7 @@ jobs: cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} build-linux-x64: - uses: ./.github/workflows/template-build-linux-x64.yml + uses: ./.github/workflows/template-build-linux.yml secrets: inherit needs: [get-update-version, create-draft-release, get-cortex-llamacpp-latest-version] with: @@ -79,6 +79,22 @@ jobs: channel: beta upload_url: ${{ needs.create-draft-release.outputs.upload_url }} cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: amd64 + + build-linux-arm64: + uses: ./.github/workflows/template-build-linux.yml + secrets: inherit + needs: [get-update-version, create-draft-release, get-cortex-llamacpp-latest-version] + with: + ref: ${{ github.ref }} + public_provider: github + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: ubuntu-2004-arm64 + cmake-flags: "-DCORTEX_VARIANT=beta -DCORTEX_CPP_VERSION='v${{ needs.get-update-version.outputs.new_version }}' -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: beta + upload_url: ${{ needs.create-draft-release.outputs.upload_url }} + cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: arm64 build-docker-x64: uses: ./.github/workflows/template-build-docker-x64.yml @@ -111,7 +127,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} noti-discord: - needs: [get-update-version, create-draft-release, build-macos, build-windows-x64, build-linux-x64, update_release] + needs: [get-update-version, create-draft-release, build-macos, build-windows-x64, build-linux-x64, build-linux-arm64, update_release] runs-on: ubuntu-latest permissions: contents: write @@ -136,9 +152,13 @@ jobs: - Network Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-mac-universal-network-installer.pkg - Local Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-mac-universal-local-installer.pkg - Binary: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-mac-universal.tar.gz - - Linux Deb: + - Linux amd64 Deb: - Network Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-amd64-network-installer.deb - Local Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-amd64-local-installer.deb - Binary: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-amd64.tar.gz + - Linux amd64 Deb: + - Network Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-arm64-network-installer.deb + - Local Installer: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-arm64-local-installer.deb + - Binary: https://github.com/janhq/cortex.cpp/releases/download/v${{ env.VERSION }}/cortex-${{ env.VERSION }}-linux-arm64.tar.gz - Docker: menloltd/cortex:beta-${{ env.VERSION }} - Github Release: https://github.com/janhq/cortex.cpp/releases/tag/v${{ env.VERSION }} \ No newline at end of file diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 8a76e4669..fd98930a1 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -20,6 +20,12 @@ jobs: fail-fast: false matrix: include: + - os: "linux" + name: "arm64" + runs-on: "ubuntu-2004-arm64" + cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake" + build-deps-cmake-flags: "" + ccache-dir: "" - os: "linux" name: "amd64" runs-on: "ubuntu-20-04-cuda-12-0" @@ -52,6 +58,7 @@ jobs: submodules: recursive - name: use python + continue-on-error: true uses: actions/setup-python@v5 with: python-version: "3.10" @@ -90,15 +97,44 @@ jobs: AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" - name: Configure vcpkg + if: runner.os != 'Linux' + run: | + cd engine + make configure-vcpkg + + - name: Configure vcpkg linux amd64 + if: runner.os != 'Linux' + run: | + cd engine + make configure-vcpkg + + - name: Configure vcpkg linux arm64 + if: runner.os == 'Linux' run: | cd engine + # Set env if arch is arm64 + if [ "${{ matrix.name }}" == "arm64" ]; then + sudo apt install ninja-build pkg-config -y + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi make configure-vcpkg - name: Build + if: runner.os != 'Linux' run: | cd engine make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}" + - name: Build + if: runner.os == 'Linux' + run: | + cd engine + if [ "${{ matrix.name }}" == "arm64" ]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi + make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}" + + - name: Run setup config run: | cd engine diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 9a31ef5ff..1f076dc97 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -74,7 +74,7 @@ jobs: cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} build-linux-x64: - uses: ./.github/workflows/template-build-linux-x64.yml + uses: ./.github/workflows/template-build-linux.yml secrets: inherit needs: [get-update-version, set-public-provider, get-cortex-llamacpp-latest-version] with: @@ -85,11 +85,26 @@ jobs: cmake-flags: "-DCORTEX_VARIANT=nightly -DCORTEX_CPP_VERSION='v${{ needs.get-update-version.outputs.new_version }}' -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" channel: nightly cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: amd64 + + build-linux-arm64: + uses: ./.github/workflows/template-build-linux.yml + secrets: inherit + needs: [get-update-version, set-public-provider, get-cortex-llamacpp-latest-version] + with: + ref: ${{ needs.set-public-provider.outputs.ref }} + public_provider: ${{ needs.set-public-provider.outputs.public_provider }} + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: ubuntu-2004-arm64 + cmake-flags: "-DCORTEX_VARIANT=nightly -DCORTEX_CPP_VERSION='v${{ needs.get-update-version.outputs.new_version }}' -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: nightly + cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: arm64 update-latest-version: runs-on: ubuntu-latest if: needs.set-public-provider.outputs.public_provider == 'aws-s3' - needs: [get-update-version, set-public-provider, build-linux-x64, build-macos, build-windows-x64, get-cortex-llamacpp-latest-version] + needs: [get-update-version, set-public-provider, build-linux-x64, build-linux-arm64, build-macos, build-windows-x64, get-cortex-llamacpp-latest-version] steps: - name: Update latest version id: update-latest-version @@ -100,9 +115,11 @@ jobs: aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/mac-universal-cortex-nightly.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/mac-arm64/cortex-nightly.tar.gz aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/mac-universal-cortex-nightly.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/mac-universal/cortex-nightly.tar.gz aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/linux-amd64-cortex-nightly.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-amd64/cortex-nightly.tar.gz + aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/linux-arm64-cortex-nightly.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-arm64/cortex-nightly.tar.gz aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/windows-amd64-cortex-nightly.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/windows-amd64/cortex-nightly.tar.gz aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-mac-universal-network-installer.pkg s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/mac-universal/cortex-mac-universal-network-installer.pkg aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-linux-amd64-network-installer.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-amd64/cortex-linux-amd64-network-installer.deb + aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-linux-arm64-network-installer.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-arm64/cortex-linux-arm64-network-installer.deb aws s3 cp s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-windows-amd64-network-installer.exe s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/windows-amd64/cortex-windows-amd64-network-installer.exe env: diff --git a/.github/workflows/stable-build.yml b/.github/workflows/stable-build.yml index 2b0523771..b05df983d 100644 --- a/.github/workflows/stable-build.yml +++ b/.github/workflows/stable-build.yml @@ -67,7 +67,7 @@ jobs: cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} build-linux-x64: - uses: ./.github/workflows/template-build-linux-x64.yml + uses: ./.github/workflows/template-build-linux.yml secrets: inherit needs: [get-update-version, create-draft-release, get-cortex-llamacpp-latest-version] with: @@ -79,6 +79,22 @@ jobs: channel: stable upload_url: ${{ needs.create-draft-release.outputs.upload_url }} cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: amd64 + + build-linux-arm64: + uses: ./.github/workflows/template-build-linux.yml + secrets: inherit + needs: [get-update-version, create-draft-release, get-cortex-llamacpp-latest-version] + with: + ref: ${{ github.ref }} + public_provider: github + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: ubuntu-2004-arm64 + cmake-flags: "-DCORTEX_VARIANT=prod -DCORTEX_CPP_VERSION='v${{ needs.get-update-version.outputs.new_version }}' -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: stable + upload_url: ${{ needs.create-draft-release.outputs.upload_url }} + cortex-llamacpp-version: ${{ needs.get-cortex-llamacpp-latest-version.outputs.cortex_llamacpp_latest_version }} + arch: arm64 build-docker-x64: uses: ./.github/workflows/template-build-docker-x64.yml diff --git a/.github/workflows/template-build-linux-x64.yml b/.github/workflows/template-build-linux.yml similarity index 67% rename from .github/workflows/template-build-linux-x64.yml rename to .github/workflows/template-build-linux.yml index d1ca73844..0abefed8b 100644 --- a/.github/workflows/template-build-linux-x64.yml +++ b/.github/workflows/template-build-linux.yml @@ -1,4 +1,4 @@ -name: build-linux-x64 +name: build-linux on: workflow_call: inputs: @@ -49,6 +49,11 @@ on: type: string default: '0.0.0' description: 'The version of cortex-llamacpp to use for this job' + arch: + required: false + type: string + default: 'amd64' + description: 'The architecture to use for this job' secrets: DELTA_AWS_S3_BUCKET_NAME: required: false @@ -60,7 +65,7 @@ on: required: false jobs: - build-linux-x64: + build-linux: runs-on: ${{ inputs.runs-on }} permissions: contents: write @@ -124,14 +129,24 @@ jobs: - name: Configure vcpkg run: | cd engine + # Set env if arch is arm64 + if [ "${{ inputs.arch }}" == "arm64" ]; then + sudo apt install ninja-build pkg-config -y + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi make configure-vcpkg - name: Build run: | cd engine + # Set env if arch is arm64 + if [ "${{ inputs.arch }}" == "arm64" ]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}" - name: Install Python + continue-on-error: true uses: actions/setup-python@v4 with: python-version: '3.10' @@ -152,21 +167,25 @@ jobs: run: | mkdir -p engine/templates/linux/dependencies cd engine/templates/linux/dependencies - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx-cuda-11-7.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx-cuda-12-0.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2-cuda-11-7.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2-cuda-12-0.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512-cuda-11-7.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512-cuda-12-0.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx-cuda-11-7.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx-cuda-12-0.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-vulkan.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cuda-11-7-linux-amd64.tar.gz - wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cuda-12-0-linux-amd64.tar.gz + if [ "${{ inputs.arch }}" == "amd64" ]; then + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx-cuda-11-7.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx-cuda-12-0.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2-cuda-11-7.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2-cuda-12-0.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx2.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512-cuda-11-7.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512-cuda-12-0.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-avx512.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx-cuda-11-7.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx-cuda-12-0.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-noavx.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-amd64-vulkan.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cuda-11-7-linux-amd64.tar.gz + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cuda-12-0-linux-amd64.tar.gz + else + wget https://github.com/janhq/cortex.llamacpp/releases/download/v${{ inputs.cortex-llamacpp-version }}/cortex.llamacpp-${{ inputs.cortex-llamacpp-version }}-linux-arm64.tar.gz + fi cd .. # Remove network package @@ -185,30 +204,30 @@ jobs: - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: cortex-${{ inputs.new_version }}-linux-amd64 + name: cortex-${{ inputs.new_version }}-linux-${{ inputs.arch }} path: ./engine/cortex - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: cortex-${{ inputs.new_version }}-linux-amd64-network-installer + name: cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-network-installer path: ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: cortex-${{ inputs.new_version }}-linux-amd64-local-installer + name: cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-local-installer path: ./engine/${{ steps.set-output-params.outputs.package_name }}-local.deb - name: upload to aws s3 if public provider is aws if: inputs.public_provider == 'aws-s3' run: | - aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/linux-amd64-cortex-nightly.tar.gz - aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-linux-amd64-network-installer.deb + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/linux--${{ inputs.arch }}-cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/temp-latest/cortex-linux--${{ inputs.arch }}-network-installer.deb - aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux-amd64/cortex-nightly.tar.gz - aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux-amd64/cortex-${{ inputs.new_version }}-linux-amd64-network-installer.deb - aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-local.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux-amd64/cortex-${{ inputs.new_version }}-linux-amd64-local-installer.deb + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux--${{ inputs.arch }}/cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux--${{ inputs.arch }}/cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-network-installer.deb + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}-local.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/v${{ inputs.new_version }}/linux--${{ inputs.arch }}/cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-local-installer.deb env: AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} @@ -223,7 +242,7 @@ jobs: with: upload_url: ${{ inputs.upload_url }} asset_path: ./engine/cortex.tar.gz - asset_name: cortex-${{ inputs.new_version }}-linux-amd64.tar.gz + asset_name: cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}.tar.gz asset_content_type: application/zip - name: Upload release assert if public provider is github @@ -234,7 +253,7 @@ jobs: with: upload_url: ${{ inputs.upload_url }} asset_path: ./engine/${{ steps.set-output-params.outputs.package_name }}-network.deb - asset_name: cortex-${{ inputs.new_version }}-linux-amd64-network-installer.deb + asset_name: cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-network-installer.deb asset_content_type: application/octet-stream - name: Upload release assert if public provider is github @@ -245,5 +264,5 @@ jobs: with: upload_url: ${{ inputs.upload_url }} asset_path: ./engine/${{ steps.set-output-params.outputs.package_name }}-local.deb - asset_name: cortex-${{ inputs.new_version }}-linux-amd64-local-installer.deb + asset_name: cortex-${{ inputs.new_version }}-linux--${{ inputs.arch }}-local-installer.deb asset_content_type: application/octet-stream \ No newline at end of file diff --git a/engine/e2e-test/test_api_engine.py b/engine/e2e-test/test_api_engine.py index 57b47b879..e652e4495 100644 --- a/engine/e2e-test/test_api_engine.py +++ b/engine/e2e-test/test_api_engine.py @@ -28,14 +28,14 @@ def test_engines_get_llamacpp_should_be_successful(self): # engines install def test_engines_install_llamacpp_specific_version_and_variant(self): - data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} + data = {"version": "v0.1.40-b4354", "variant": "linux-amd64-avx-cuda-11-7"} response = requests.post( "http://localhost:3928/v1/engines/llama-cpp/install", json=data ) assert response.status_code == 200 def test_engines_install_llamacpp_specific_version_and_null_variant(self): - data = {"version": "v0.1.35-27.10.24"} + data = {"version": "v0.1.40-b4354"} response = requests.post( "http://localhost:3928/v1/engines/llama-cpp/install", json=data ) diff --git a/engine/e2e-test/test_api_engine_update.py b/engine/e2e-test/test_api_engine_update.py index 23939f038..be8685dba 100644 --- a/engine/e2e-test/test_api_engine_update.py +++ b/engine/e2e-test/test_api_engine_update.py @@ -25,7 +25,7 @@ def setup_and_teardown(self): @pytest.mark.asyncio async def test_engines_update_should_be_successfully(self): - requests.post("http://localhost:3928/v1/engines/llama-cpp?version=0.1.34") + requests.post("http://localhost:3928/v1/engines/llama-cpp?version=0.1.43") response = requests.post("http://localhost:3928/v1/engines/llama-cpp/update") assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_model.py b/engine/e2e-test/test_api_model.py index 8f2e4b07a..d75aa6831 100644 --- a/engine/e2e-test/test_api_model.py +++ b/engine/e2e-test/test_api_model.py @@ -85,6 +85,7 @@ async def test_model_pull_with_direct_url_should_have_desired_name(self): ], ) + @pytest.mark.asyncio async def test_models_start_stop_should_be_successful(self): print("Install engine") response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") diff --git a/engine/e2e-test/test_cli_engine_install.py b/engine/e2e-test/test_cli_engine_install.py index a998f3183..dbbc16e8a 100644 --- a/engine/e2e-test/test_cli_engine_install.py +++ b/engine/e2e-test/test_cli_engine_install.py @@ -49,7 +49,7 @@ def test_engines_install_onnx_on_tensorrt_should_be_failed(self): @pytest.mark.skipif(platform.system() == "Windows", reason="Progress bar log issue on Windows") def test_engines_install_pre_release_llamacpp(self): - engine_version = "v0.1.29" + engine_version = "v0.1.43" exit_code, output, error = run( "Install Engine", ["engines", "install", "llama-cpp", "-v", engine_version], @@ -69,7 +69,7 @@ def test_engines_install_pre_release_llamacpp(self): assert is_engine_version_exist, f"Engine version {engine_version} is not found" assert exit_code == 0, f"Install engine failed with error: {error}" - @pytest.mark.skipif(platform.system() == "Windows", reason="Progress bar log issue on Windows") + @pytest.mark.skipif(platform.system() == "Windows" or platform.system() == "Linux", reason="Progress bar log issue on Windows") def test_engines_should_fallback_to_download_llamacpp_engine_if_not_exists(self): exit_code, output, error = run( "Install Engine", diff --git a/engine/e2e-test/test_cli_engine_install_nightly.py b/engine/e2e-test/test_cli_engine_install_nightly.py index 8c66c284c..bbb56ac9b 100644 --- a/engine/e2e-test/test_cli_engine_install_nightly.py +++ b/engine/e2e-test/test_cli_engine_install_nightly.py @@ -47,6 +47,7 @@ def test_engines_install_onnx_on_tensorrt_should_be_failed(self): assert "is not supported on" in output, "Should display error message" assert exit_code == 0, f"Install engine failed with error: {error}" + @pytest.mark.skipif(platform.system() == "Linux", reason="Wait for linux arm ready") def test_engines_should_fallback_to_download_llamacpp_engine_if_not_exists(self): exit_code, output, error = run( "Install Engine", diff --git a/engine/e2e-test/test_cli_engine_uninstall.py b/engine/e2e-test/test_cli_engine_uninstall.py index fcc5f5c73..6b640d45d 100644 --- a/engine/e2e-test/test_cli_engine_uninstall.py +++ b/engine/e2e-test/test_cli_engine_uninstall.py @@ -24,7 +24,7 @@ def setup_and_teardown(self): @pytest.mark.asyncio async def test_engines_uninstall_llamacpp_should_be_successfully(self): - requests.post("http://127.0.0.1:3928/v1/engines/llama-cpp/install") + response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") await wait_for_websocket_download_success_event(timeout=None) exit_code, output, error = run( "Uninstall engine", ["engines", "uninstall", "llama-cpp"] diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index d855c8f61..a38dbe70e 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -140,10 +140,10 @@ cpp::result DownloadService::GetFileSize( curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); auto headers = curl_utils::GetHeaders(url); - if (headers.has_value()) { + if (headers) { curl_slist* curl_headers = nullptr; - for (const auto& [key, value] : headers.value()) { + for (const auto& [key, value] : headers->m) { auto header = key + ": " + value; curl_headers = curl_slist_append(curl_headers, header.c_str()); } @@ -227,10 +227,10 @@ cpp::result DownloadService::Download( curl_easy_setopt(curl, CURLOPT_URL, download_item.downloadUrl.c_str()); auto headers = curl_utils::GetHeaders(download_item.downloadUrl); - if (headers.has_value()) { + if (headers) { curl_slist* curl_headers = nullptr; - for (const auto& [key, value] : headers.value()) { + for (const auto& [key, value] : headers->m) { auto header = key + ": " + value; curl_headers = curl_slist_append(curl_headers, header.c_str()); } @@ -469,7 +469,7 @@ void DownloadService::SetUpCurlHandle(CURL* handle, const DownloadItem& item, auto headers = curl_utils::GetHeaders(item.downloadUrl); if (headers) { curl_slist* curl_headers = nullptr; - for (const auto& [key, value] : headers.value()) { + for (const auto& [key, value] : headers->m) { curl_headers = curl_slist_append(curl_headers, (key + ": " + value).c_str()); } diff --git a/engine/templates/linux/install.sh b/engine/templates/linux/install.sh index e11b879c6..ade0b134a 100644 --- a/engine/templates/linux/install.sh +++ b/engine/templates/linux/install.sh @@ -6,6 +6,17 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi +# Determine architecture +ARCH=$(uname -m) +if [ "$ARCH" = "x86_64" ]; then + ARCH="amd64" +elif [ "$ARCH" = "aarch64" ]; then + ARCH="arm64" +else + echo "Unsupported architecture: $ARCH" + exit 1 +fi + # Determine the home directory based on the user USER_TO_RUN_AS=${SUDO_USER:-$(whoami)} if [ "$USER_TO_RUN_AS" = "root" ]; then @@ -142,19 +153,19 @@ install_cortex() { case $channel in stable) - url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz" - url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb" - url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb" + url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}.tar.gz" + url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}-local-installer.deb" + url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}-network-installer.deb" ;; beta) - url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz" - url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb" - url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb" + url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}.tar.gz" + url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}-local-installer.deb" + url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-${ARCH}-network-installer.deb" ;; nightly) - url_binary="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-nightly.tar.gz" - url_deb_local="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-local-installer.deb" - url_deb_network="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-network-installer.deb" + url_binary="https://delta.jan.ai/cortex/v${version}/linux-${ARCH}/cortex-nightly.tar.gz" + url_deb_local="https://delta.jan.ai/cortex/v${version}/linux-${ARCH}/cortex-${version}-linux-${ARCH}-local-installer.deb" + url_deb_network="https://delta.jan.ai/cortex/v${version}/linux-${ARCH}/cortex-${version}-linux-${ARCH}-network-installer.deb" ;; esac diff --git a/engine/test/components/test_engine_matcher_utils.cc b/engine/test/components/test_engine_matcher_utils.cc index 7da4e3cd1..1d1ed47a8 100644 --- a/engine/test/components/test_engine_matcher_utils.cc +++ b/engine/test/components/test_engine_matcher_utils.cc @@ -19,6 +19,7 @@ class EngineMatcherUtilsTestSuite : public ::testing::Test { "cortex.llamacpp-0.1.25-25.08.24-linux-amd64-noavx-cuda-12-0.tar.gz", "cortex.llamacpp-0.1.25-25.08.24-linux-amd64-noavx.tar.gz", "cortex.llamacpp-0.1.25-25.08.24-linux-amd64-vulkan.tar.gz", + "cortex.llamacpp-0.1.43-linux-arm64.tar.gz", "cortex.llamacpp-0.1.25-25.08.24-mac-amd64.tar.gz", "cortex.llamacpp-0.1.25-25.08.24-mac-arm64.tar.gz", "cortex.llamacpp-0.1.25-25.08.24-windows-amd64-avx-cuda-11-7.tar.gz", @@ -134,6 +135,18 @@ TEST_F(EngineMatcherUtilsTestSuite, TestValidate) { EXPECT_EQ(variant, "cortex.llamacpp-0.1.25-25.08.24-windows-amd64-avx2.tar.gz"); } + + { + auto os{"linux"}; + auto cpu_arch{"arm64"}; + auto suitable_avx{""}; + auto cuda_version{""}; + + auto variant = engine_matcher_utils::Validate( + cortex_llamacpp_variants, os, cpu_arch, suitable_avx, cuda_version); + + EXPECT_EQ(variant, "cortex.llamacpp-0.1.43-linux-arm64.tar.gz"); + } } TEST_F(EngineMatcherUtilsTestSuite, TestGetVersionAndArch) { diff --git a/engine/utils/curl_utils.cc b/engine/utils/curl_utils.cc index 71f263a6a..1817948e2 100644 --- a/engine/utils/curl_utils.cc +++ b/engine/utils/curl_utils.cc @@ -9,12 +9,24 @@ namespace curl_utils { namespace { -size_t WriteCallback(void* contents, size_t size, size_t nmemb, - std::string* output) { - size_t totalSize = size * nmemb; - output->append((char*)contents, totalSize); - return totalSize; -} +class CurlResponse { + public: + static size_t WriteCallback(char* buffer, size_t size, size_t nitems, + void* userdata) { + auto* response = static_cast(userdata); + return response->Append(buffer, size * nitems); + } + + size_t Append(const char* buffer, size_t size) { + data_.append(buffer, size); + return size; + } + + const std::string& GetData() const { return data_; } + + private: + std::string data_; +}; void SetUpProxy(CURL* handle, const std::string& url) { auto config = file_manager_utils::GetCortexConfig(); @@ -59,19 +71,18 @@ void SetUpProxy(CURL* handle, const std::string& url) { } } // namespace -std::optional> GetHeaders( - const std::string& url) { +std::shared_ptr
GetHeaders(const std::string& url) { auto url_obj = url_parser::FromUrlString(url); if (url_obj.has_error()) { - return std::nullopt; + return nullptr; } if (url_obj->host == kHuggingFaceHost) { - std::unordered_map headers{}; - headers["Content-Type"] = "application/json"; + auto headers = std::make_shared
(); + headers->m["Content-Type"] = "application/json"; auto const& token = file_manager_utils::GetCortexConfig().huggingFaceToken; if (!token.empty()) { - headers["Authorization"] = "Bearer " + token; + headers->m["Authorization"] = "Bearer " + token; // for debug purpose auto min_token_size = 6; @@ -87,15 +98,15 @@ std::optional> GetHeaders( } if (url_obj->host == kGitHubHost) { - std::unordered_map headers{}; - headers["Accept"] = "application/vnd.github.v3+json"; + auto headers = std::make_shared
(); + headers->m["Accept"] = "application/vnd.github.v3+json"; // github API requires user-agent https://docs.github.com/en/rest/using-the-rest-api/getting-started-with-the-rest-api?apiVersion=2022-11-28#user-agent auto user_agent = file_manager_utils::GetCortexConfig().gitHubUserAgent; auto gh_token = file_manager_utils::GetCortexConfig().gitHubToken; - headers["User-Agent"] = + headers->m["User-Agent"] = user_agent.empty() ? kDefaultGHUserAgent : user_agent; if (!gh_token.empty()) { - headers["Authorization"] = "Bearer " + gh_token; + headers->m["Authorization"] = "Bearer " + gh_token; // for debug purpose auto min_token_size = 6; @@ -109,7 +120,7 @@ std::optional> GetHeaders( return headers; } - return std::nullopt; + return nullptr; } cpp::result SimpleGet(const std::string& url, @@ -122,8 +133,8 @@ cpp::result SimpleGet(const std::string& url, auto headers = GetHeaders(url); curl_slist* curl_headers = nullptr; - if (headers.has_value()) { - for (const auto& [key, value] : headers.value()) { + if (headers) { + for (const auto& [key, value] : headers->m) { auto header = key + ": " + value; curl_headers = curl_slist_append(curl_headers, header.c_str()); } @@ -131,12 +142,14 @@ cpp::result SimpleGet(const std::string& url, curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); } - std::string readBuffer; + auto* response = new CurlResponse(); + std::shared_ptr s(response, + std::default_delete()); SetUpProxy(curl, url); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlResponse::WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); if (timeout > 0) { curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); } @@ -155,10 +168,10 @@ cpp::result SimpleGet(const std::string& url, if (http_code >= 400) { CTL_ERR("HTTP request failed with status code: " + std::to_string(http_code)); - return cpp::fail(readBuffer); + return cpp::fail(response->GetData()); } - return readBuffer; + return response->GetData(); } cpp::result SimpleRequest( @@ -176,13 +189,15 @@ cpp::result SimpleRequest( curl_slist_append(curl_headers, "Content-Type: application/json"); curl_headers = curl_slist_append(curl_headers, "Expect:"); - if (headers.has_value()) { - for (const auto& [key, value] : headers.value()) { + if (headers) { + for (const auto& [key, value] : headers->m) { auto header = key + ": " + value; curl_headers = curl_slist_append(curl_headers, header.c_str()); } } - std::string readBuffer; + auto* response = new CurlResponse(); + std::shared_ptr s(response, + std::default_delete()); SetUpProxy(curl, url); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); @@ -196,8 +211,8 @@ cpp::result SimpleRequest( curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); } curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlResponse::WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.length()); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); @@ -221,10 +236,10 @@ cpp::result SimpleRequest( if (http_code >= 400) { CTL_ERR("HTTP request failed with status code: " + std::to_string(http_code)); - return cpp::fail(readBuffer); + return cpp::fail(response->GetData()); } - return readBuffer; + return response->GetData(); } cpp::result ReadRemoteYaml(const std::string& url) { diff --git a/engine/utils/curl_utils.h b/engine/utils/curl_utils.h index 64b5fc339..69369def0 100644 --- a/engine/utils/curl_utils.h +++ b/engine/utils/curl_utils.h @@ -15,8 +15,11 @@ enum class RequestType { GET, PATCH, POST, DEL }; namespace curl_utils { -std::optional> GetHeaders( - const std::string& url); +struct Header { + std::unordered_map m; +}; + +std::shared_ptr
GetHeaders(const std::string& url); cpp::result SimpleGet(const std::string& url, const int timeout = -1); diff --git a/engine/utils/engine_matcher_utils.h b/engine/utils/engine_matcher_utils.h index a6135e532..28c0f0c2a 100644 --- a/engine/utils/engine_matcher_utils.h +++ b/engine/utils/engine_matcher_utils.h @@ -156,6 +156,11 @@ inline std::string Validate(const std::vector& variants, if (os == "mac" && !os_and_arch_compatible_list.empty()) return os_and_arch_compatible_list[0]; + if (os == "linux" && cpu_arch == "arm64" && + !os_and_arch_compatible_list.empty()) { + return os_and_arch_compatible_list[0]; + } + std::vector avx_compatible_list; std::copy_if(os_and_arch_compatible_list.begin(),