diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 0000000000000..6e49532b7b484 --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,14 @@ +name: Setup Python and UV +runs: + using: "composite" + steps: + # Use the setup-python action to take advantage of the cache. uv will symlink + # to this version + - name: Setup system python + uses: actions/setup-python@v5 + with: + python-version-file: '.python-version' + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.5.6" diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 6077daeedcf3a..5ce7fad25ec6d 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -59,8 +59,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Set up JDK 17 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f546469f962c7..74530c40e1f77 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,9 +13,10 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap --skip-lints - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1d900e1c7bb7f..4527200526a47 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,9 +31,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 - with: - python-version: '3.10' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Install taplo uses: baptiste0928/cargo-install@v3 with: @@ -46,8 +45,6 @@ jobs: locked: true # 0.16.2 requires Rust 1.81 or newer. version: '0.16.1' - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip - name: Bootstrap dependencies run: | sudo apt update @@ -57,4 +54,4 @@ jobs: run: | python3 ./mach clippy --use-crown --locked -- -- --deny warnings - name: Tidy - run: python3 ./mach test-tidy --no-progress --all \ No newline at end of file + run: python3 ./mach test-tidy --no-progress --all diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index 34857bdc9d0c1..f710518b462a2 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -52,9 +52,10 @@ jobs: path: ${{ inputs.profile }}-binary-linux - name: unPackage binary run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: | - python3 -m pip install --upgrade pip sudo apt update sudo apt install -qy --no-install-recommends mesa-vulkan-drivers python3 ./mach bootstrap --skip-lints diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 02ed12e95988b..ec6ff65eb8e58 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -131,13 +131,9 @@ jobs: - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 + - name: Setup Python if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - with: - python-version: '3.10' - - name: Bootstrap Python - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - run: python3 -m pip install --upgrade pip + uses: ./.github/actions/setup-python - name: Bootstrap dependencies if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index 74a1e1439fe09..610da047a5f22 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -41,15 +41,11 @@ jobs: - uses: actions/download-artifact@v4 with: name: ${{ inputs.profile }}-binary-macos - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep test environment run: | gtar -xzf target.tar.gz - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints - name: Smoketest run: python3 ./mach smoketest --${{ inputs.profile }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 4467b5644264e..f0edacb9fb8c9 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -77,18 +77,14 @@ jobs: if: github.event_name == 'pull_request_target' with: ref: ${{ github.event.pull_request.head.sha }} - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - name: Bootstrap run: | - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints brew install gnu-tar - name: Build (${{ inputs.profile }}) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6208d6613540e..2cb7245d615c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,15 +20,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Configuration id: configuration run: | diff --git a/.github/workflows/ohos.yml b/.github/workflows/ohos.yml index b60f7f3f12198..2690ea100b843 100644 --- a/.github/workflows/ohos.yml +++ b/.github/workflows/ohos.yml @@ -53,8 +53,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Setup OpenHarmony SDK diff --git a/.github/workflows/pull-request-wpt-export.yml b/.github/workflows/pull-request-wpt-export.yml index 2942fe9f2cd19..f398fe8d3a323 100644 --- a/.github/workflows/pull-request-wpt-export.yml +++ b/.github/workflows/pull-request-wpt-export.yml @@ -28,10 +28,14 @@ jobs: # using the token specified here. # See https://github.com/actions/checkout/issues/162. token: ${{ secrets.WPT_SYNC_TOKEN }} + - name: Setup Python + uses: ./.github/actions/setup-python - name: Install requirements - run: pip install -r servo/python/requirements.txt + run: uv pip install -r servo/python/requirements.txt - name: Process pull request - run: servo/python/wpt/export.py + run: | + source .venv/bin/activate + servo/python/wpt/export.py env: GITHUB_CONTEXT: ${{ toJson(github) }} WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }} diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index 1ed65b12c3296..145133b817cfb 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -35,9 +35,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: wpt-full-logs-linux-layout-2020 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep environment run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Add upstream remote diff --git a/.github/workflows/try-label.yml b/.github/workflows/try-label.yml index 49000da846612..c0abfc5d11555 100644 --- a/.github/workflows/try-label.yml +++ b/.github/workflows/try-label.yml @@ -77,14 +77,15 @@ jobs: } return try_string; - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Parse Labels if: ${{ steps.try_string.outputs.result }} id: configuration diff --git a/.github/workflows/try.yml b/.github/workflows/try.yml index 8e1879dfc7b1f..f92cb40217c78 100644 --- a/.github/workflows/try.yml +++ b/.github/workflows/try.yml @@ -29,15 +29,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Full Configuration id: full_config run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a59c23dc91954..22f6ea3e6a7aa 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -114,13 +114,12 @@ jobs: choco install wixtoolset echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - uses: actions/setup-python@v5 + - name: Setup Python if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - with: - python-version: "3.10" + uses: ./.github/actions/setup-python - name: Bootstrap if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | - python -m pip install --upgrade pip python mach fetch python mach bootstrap-gstreamer # For some reason WiX isn't currently on the GitHub runner path. This is a diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000000..04e207918ddc8 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.8 diff --git a/README.md b/README.md index 0c83a837d29f8..e1828176b18e1 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,14 @@ For more detailed build instructions, see the Servo book under [Setting up your [Building for Android]: https://book.servo.org/hacking/building-for-android.html [Building for OpenHarmony]: https://book.servo.org/hacking/building-for-openharmony.html +**NOTE**: In the below steps, please ensure you install at least version 3.9 of Python, preferably the latest version supported on your platform. +Servo will use its own pinned version of Python (in the `.python-version` file), which it installs using [`uv`](https://docs.astral.sh/uv). +The system version is required for now just to bootstrap the process of creating and activating the virtual environment. + ### macOS -- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). +- Download and install [`python`](https://www.python.org/downloads/macos/), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -25,11 +30,12 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Linux -- Install `curl` and `python` (version 3.10 to 3.12): - - Arch: `sudo pacman -S --needed curl python python-pip` - - Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools` - - Fedora: `sudo dnf install curl python3 python3-pip python3-devel` - - Gentoo: `sudo emerge net-misc/curl dev-python/pip` +- Install `curl` and `python`: + - Arch: `sudo pacman -S --needed curl python` + - Debian, Ubuntu: `sudo apt install curl` + - Fedora: `sudo dnf install curl python3` + - Gentoo: `sudo emerge net-misc/curl` +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -37,7 +43,7 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Windows -- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) +- Download and install [`python`](https://www.python.org/downloads/windows/), [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) - Be sure to select *Quick install via the Visual Studio Community installer* - In the Visual Studio Installer, ensure the following components are installed: - **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`) diff --git a/mach b/mach index 2ff80f6b51dc4..bd7cbed025fd8 100755 --- a/mach +++ b/mach @@ -14,11 +14,10 @@ import sys # Destructure because version_info > max_ver is true when running the same version. ver = (sys.version_info[0], sys.version_info[1]) -min_ver = (3, 10) -max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095. -if ver < min_ver or ver > max_ver: - print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \ - .format(ver[0], ver[1], min_ver[1], max_ver[1])) +min_ver = (3, 9) +if ver < min_ver: + print("mach requires at least version 3.{0} of Python. The version of Python installed in this system is {1}.{2}" \ + .format(min_ver[1], ver[0], ver[1])) sys.exit(1) def main(args): diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 43690c3d951de..43b626db123b4 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -131,11 +131,8 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_ requirements_hash = requirements_hasher.hexdigest() if marker_hash != requirements_hash: - print(" * Upgrading pip...") - _process_exec([python, "-m", "pip", "install", "--upgrade", "pip"]) - print(" * Installing Python requirements...") - _process_exec([python, "-m", "pip", "install", "-I", + _process_exec(["uv", "pip", "install", "-r", requirements_paths[0], "-r", requirements_paths[1], "-r", requirements_paths[2]]) @@ -144,14 +141,14 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_ def _activate_virtualenv(topdir): - virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1])) + virtualenv_path = os.path.join(topdir, ".venv") python = sys.executable if os.environ.get("VIRTUAL_ENV") != virtualenv_path: venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir()) if not os.path.exists(virtualenv_path): print(" * Setting up virtual environment...") - _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path]) + _process_exec(["uv", "venv"]) # This general approach is taken from virtualenv's `activate_this.py`. os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)]) @@ -171,6 +168,11 @@ def _activate_virtualenv(topdir): install_virtual_env_requirements(topdir, python, virtualenv_path) + # Turn off warnings about deprecated syntax in our indirect dependencies. + # TODO: find a better approach to do this. + import warnings + warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv') + def _ensure_case_insensitive_if_windows(): # The folder is called 'python'. By deliberately checking for it with the wrong case, we determine if the file diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index c48999f99f955..64a518f4d622b 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -175,7 +175,7 @@ def build(self, build_type: BuildType, jobs=None, params=None, no_package=False, return status @Command('clean', - description='Clean the target/ and python/_venv[version]/ directories', + description='Clean the target/ and python virtual environment directories', category='build') @CommandArgument('--manifest-path', default=None, @@ -188,8 +188,8 @@ def build(self, build_type: BuildType, jobs=None, params=None, no_package=False, def clean(self, manifest_path=None, params=[], verbose=False): self.ensure_bootstrapped() - virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1]) - virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname) + virtualenv_fname = '.venv' + virtualenv_path = path.join(self.get_top_dir(), virtualenv_fname) if path.exists(virtualenv_path): print('Removing virtualenv directory: %s' % virtualenv_path) shutil.rmtree(virtualenv_path) diff --git a/servo-tidy.toml b/servo-tidy.toml index 64d33117a6ae1..7460f4092ced1 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -117,7 +117,6 @@ directories = [ "./tests/wpt/mozilla/tests/mozilla/referrer-policy", "./tests/wpt/mozilla/tests/webgl", "./python/tidy/tests", - "./python/_v*", "./python/mach", # Generated and upstream code combined with our own. Could use cleanup "./target", diff --git a/shell.nix b/shell.nix index 2576e0e454fae..fd110442cd695 100644 --- a/shell.nix +++ b/shell.nix @@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // { # Build utilities cmake dbus gcc git pkg-config which llvm perl yasm m4 - (python3.withPackages (ps: with ps; [virtualenv pip dbus])) + + # Ensure the Python version is same as the one in `.python-version` file so + # that `uv` will just symlink to the one in nix store. Otherwise `uv` will + # download a pre-built binary that won't work on nix. + # FIXME: dbus python module needs to be installed into the virtual environment. + python312 + uv # This pins gnumake to 4.3 since 4.4 breaks jobserver # functionality in mozjs and causes builds to be extremely