Merge branch 'sgl-project:main' into main

stbaione · Nov 13, 2024 · 3058e99 · 3058e99
2 parents b5edcec + 54479d6
commit 3058e99
Show file tree

Hide file tree

Showing 89 changed files with 4,188 additions and 1,811 deletions.
diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml
@@ -18,22 +18,31 @@ concurrency:
 jobs:
   unit-test-rust:
     if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
-    runs-on: 1-gpu-runner
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
       - name: Install dependencies
         run: |
           bash scripts/ci_install_rust.sh
+
       - name: Run fmt
         run: |
           source "$HOME/.cargo/env"
           cd rust/
           cargo fmt -- --check
+
       - name: Run test
         timeout-minutes: 20
         run: |
           source "$HOME/.cargo/env"
           cd rust/
           cargo test
+
+  finish:
+    needs: [unit-test-rust]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finish
+        run: echo "This is an empty step to ensure that all jobs are completed."
diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml
@@ -47,10 +47,10 @@ jobs:
           bash scripts/ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           cd test/srt
-          python3 run_suite.py --suite minimal --range-begin 0 --range-end 6
+          python3 run_suite.py --suite minimal --range-begin 0 --range-end 5
 
   unit-test-backend-part-2:
     if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
@@ -64,10 +64,10 @@ jobs:
           bash scripts/ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           cd test/srt
-          python3 run_suite.py --suite minimal --range-begin 6 --range-end 14
+          python3 run_suite.py --suite minimal --range-begin 5 --range-end 14
 
   unit-test-backend-part-3:
     if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
@@ -81,10 +81,10 @@ jobs:
           bash scripts/ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           cd test/srt
-          python3 run_suite.py --suite minimal --range-begin 14 --range-end 20
+          python3 run_suite.py --suite minimal --range-begin 14 --range-end 21
 
   unit-test-backend-part-4:
     if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
@@ -98,10 +98,10 @@ jobs:
           bash scripts/ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           cd test/srt
-          python3 run_suite.py --suite minimal --range-begin 20
+          python3 run_suite.py --suite minimal --range-begin 21
 
   performance-test-1-gpu-part-1:
     if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
@@ -114,25 +114,25 @@ jobs:
         run: |
           bash scripts/ci_install_dependency.sh
 
-      - name: Benchmark Single Latency
+      - name: Benchmark single latency
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_latency.TestBenchLatency.test_default
 
-      - name: Benchmark Online Latency
+      - name: Benchmark online latency
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default
 
-      - name: Benchmark Offline Throughput
+      - name: Benchmark offline throughput
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default
 
-      - name: Benchmark Offline Throughput (Non-streaming, small batch size)
+      - name: Benchmark offline throughput (Non-streaming, small batch size)
         timeout-minutes: 10
         run: |
           cd test/srt
@@ -149,19 +149,19 @@ jobs:
         run: |
           bash scripts/ci_install_dependency.sh
 
-      - name: Benchmark Offline Throughput (w/o RadixAttention)
+      - name: Benchmark offline throughput (w/o RadixAttention)
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache
 
-      - name: Benchmark Offline Throughput (w/ Triton)
+      - name: Benchmark offline throughput (w/ Triton)
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend
 
-      - name: Benchmark Offline Throughput (w/ FP8)
+      - name: Benchmark offline throughput (w/ FP8)
         timeout-minutes: 10
         run: |
           cd test/srt
@@ -178,19 +178,19 @@ jobs:
         run: |
           bash scripts/ci_install_dependency.sh
 
-      - name: Benchmark Offline Throughput (TP=2)
+      - name: Benchmark offline throughput (TP=2)
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default
 
-      - name: Benchmark Offline Throughput (w/o RadixAttention) (TP=2)
+      - name: Benchmark offline throughput (w/o RadixAttention) (TP=2)
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache
 
-      - name: Benchmark Single Latency (TP=2)
+      - name: Benchmark single latency (TP=2)
         timeout-minutes: 10
         run: |
           cd test/srt
@@ -211,7 +211,7 @@ jobs:
           cd human-eval
           pip install -e .
 
-      - name: Evaluate Accuracy
+      - name: Evaluate accuracy
         timeout-minutes: 20
         run: |
           cd test/srt
@@ -232,20 +232,20 @@ jobs:
           cd human-eval
           pip install -e .
 
-      - name: Evaluate Accuracy (TP=2)
+      - name: Evaluate accuracy (TP=2)
         timeout-minutes: 20
         run: |
           cd test/srt
           python3 test_moe_eval_accuracy_large.py
 
-      - name: Evaluate MLA Accuracy (TP=2)
+      - name: Evaluate MLA accuracy (TP=2)
         timeout-minutes: 10
         run: |
           cd test/srt
           python3 test_mla.py
           python3 test_mla_fp8.py
 
-      - name: Evaluate Data Parallelism Accuracy (DP=2)
+      - name: Evaluate data parallelism accuracy (DP=2)
         timeout-minutes: 10
         run: |
           cd test/srt

diff --git a/.github/workflows/release-pypi-router.yml b/.github/workflows/release-pypi-router.yml
@@ -0,0 +1,104 @@
+# Reference: https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1
+
+name: Release SGLang Router to PyPI
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build on ${{ matrix.os }} (${{ matrix.target }})
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu
+            target: x86_64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move rust folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/rust/* .
+          rm -rf sglang-repo
+          ls -alt
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install build twine auditwheel
+
+      - name: Build package
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64"
+          CIBW_BEFORE_ALL: |
+            yum update && yum install -y openssl-devel && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH"
+
+      - name: List built packages
+        run: ls -lh wheelhouse/
+
+      - name: Check packages
+        run: twine check --strict wheelhouse/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: packages-${{ matrix.os }}-${{ matrix.target }}
+          path: wheelhouse/
+
+  build-sdist:
+    name: Build SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move rust folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/rust/* .
+          rm -rf sglang-repo
+          ls -alt
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build SDist
+        run: |
+          pip install build
+          python -m build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  upload:
+    name: Upload to PyPI
+    needs: [build, build-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_ROUTER }}
+        run: |
+          pip install twine
+          twine upload dist/* --verbose
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,7 +17,6 @@ repos:
       - id: check-ast
       - id: check-added-large-files
       - id: check-merge-conflict
-      - id: check-executables-have-shebangs
       - id: check-shebang-scripts-are-executable
       - id: detect-private-key
       - id: debug-statements