From 868f8c7bc15c6bfb03dba2b22a190d54ff6c8468 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 7 Jun 2024 09:51:29 +0200 Subject: [PATCH 1/5] relax pyproject dependency versions and introduce requirements.txt closes #2053 closes #2079 --- .github/workflows/build.yml | 4 +- .github/workflows/publish.yml | 1 + .github/workflows/tests.yml | 20 ++++++-- pyproject.toml | 87 +++++++++++++++++++++++++++-------- requirements.txt | 46 ++++++++++++++++++ 5 files changed, 133 insertions(+), 25 deletions(-) create mode 100644 requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a9bf555f..85b898b4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,9 @@ jobs: - name: Upgrade pip, setuptools run: python -m pip install --upgrade pip setuptools - name: Install capa with build requirements - run: pip install -e .[build] + run: | + pip install -r requirements.txt + pip install -e .[build] - name: Build standalone executable run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec - name: Does it run (PE)? diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb2a00f97..4a591d778 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,6 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt pip install -e .[build] - name: build package run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 812528564..5553ceae5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,9 @@ jobs: with: python-version: "3.11" - name: Install dependencies - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Lint with ruff run: pre-commit run ruff - name: Lint with isort @@ -61,7 +63,9 @@ jobs: with: python-version: "3.11" - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run rule linter run: python scripts/lint.py rules/ @@ -96,7 +100,9 @@ jobs: if: matrix.os == 'ubuntu-20.04' run: sudo apt-get install -y libyaml-dev - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run tests (fast) # this set of tests runs about 80% of the cases in 20% of the time, # and should catch most errors quickly. @@ -131,7 +137,9 @@ jobs: run: sudo apt-get install -y libyaml-dev - name: Install capa if: ${{ env.BN_SERIAL != 0 }} - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: install Binary Ninja if: ${{ env.BN_SERIAL != 0 }} run: | @@ -188,7 +196,9 @@ jobs: - name: Install pyyaml run: sudo apt-get install -y libyaml-dev - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run tests run: | mkdir ./.github/ghidra/project diff --git a/pyproject.toml b/pyproject.toml index 714a567b3..0052d2660 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,25 +32,74 @@ classifiers = [ "Topic :: Security", ] dependencies = [ - "tqdm==4.66.4", - "pyyaml==6.0.1", - "tabulate==0.9.0", - "colorama==0.4.6", - "termcolor==2.4.0", - "wcwidth==0.2.13", - "ida-settings==2.1.0", - "viv-utils[flirt]==0.7.9", - "networkx==3.1", - "ruamel.yaml==0.18.6", - "vivisect==1.1.1", - "pefile==2023.2.7", - "pyelftools==0.31", - "dnfile==0.14.1", - "dncil==1.0.2", - "pydantic==2.7.1", - "rich==13.7.1", - "humanize==4.9.0", - "protobuf==5.27.0", + # --------------------------------------- + # As a library, capa uses lower version bounds + # when specifying its dependencies. This lets + # other programs that use capa (and other libraries) + # to find a compatible set of dependency versions. + # + # We can optionally pin to specific versions or + # limit the upper bound when there's a good reason; + # but the default is to assume all greater versions + # probably work with capa until proven otherwise. + # + # The following link provides good background: + # https://iscinumpy.dev/post/bound-version-constraints/ + # + # When we develop capa, and when we distribute it as + # a standalone binary, we'll use specific versions + # that are pinned in requirements.txt. + # But the requirements for a library are specified here + # and are looser. + # + # Related discussions: + # + # - https://github.com/mandiant/capa/issues/2053 + # - https://github.com/mandiant/capa/pull/2059 + # - https://github.com/mandiant/capa/pull/2079 + # + # --------------------------------------- + # The following dependency versions were imported + # during June 2024 by truncating specific versions to + # their major-most version (major version when possible, + # or minor otherwise). + # As specific constraints are identified, please provide + # comments and context. + "tqdm>=4", + "pyyaml>=6", + "tabulate>=0.9", + "colorama>=0.4", + "termcolor>=2", + "wcwidth>=0.2", + "ida-settings>=2", + "ruamel.yaml>=0.18", + "pefile>=2023.2.7", + "pyelftools>=0.31", + "pydantic>=2", + "rich>=13", + "humanize>=4", + "protobuf>=5", + + # --------------------------------------- + # Dependencies that we develop + # + # These dependencies are often actively influenced by capa, + # so we provide a minimum patch version that includes the + # latest bug fixes we need here. + "viv-utils[flirt]>=0.7.9", + "vivisect>=1.1.1", + "dnfile>=0.14.1", + "dncil>=1.0.2", + + # --------------------------------------- + # Dependencies with version caps + # + # These dependencies must not exceed the version cap, + # typically due to dropping support for python releases + # we still support. + + # networkx 3.2 doesn't support python 3.8 + "networkx>=3,<3.2", ] dynamic = ["version"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..004f98725 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,46 @@ +# Dependencies with specific version constraints +# used during development and building the standalone executables. +# For these environments, use `pip install -r requirements.txt` +# before installing capa from source/pypi. This will ensure +# the following specific versions are used. +# +# Initially generated via: pip freeze | grep -v -- "-e" +# Kept up to date by dependabot. +annotated-types==0.7.0 +colorama==0.4.6 +cxxfilt==0.2.2 +dncil==1.0.2 +dnfile==0.15.0 +funcy==2.0 +humanize==4.9.0 +ida-netnode==3.0 +ida-settings==2.1.0 +intervaltree==3.1.0 +markdown-it-py==3.0.0 +mdurl==0.1.2 +msgpack==1.0.8 +networkx==3.1 +pefile==2023.2.7 +pip==24.0 +protobuf==5.27.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.22 +pydantic==2.7.3 +pydantic-core==2.18.4 +pyelftools==0.31 +pygments==2.18.0 +python-flirt==0.8.6 +pyyaml==6.0.1 +rich==13.7.1 +ruamel-yaml==0.18.6 +ruamel-yaml-clib==0.2.8 +setuptools==65.5.0 +six==1.16.0 +sortedcontainers==2.4.0 +tabulate==0.9.0 +termcolor==2.4.0 +tqdm==4.66.4 +viv-utils==0.7.9 +vivisect==1.1.1 +wcwidth==0.2.13 From a7fe047d423d77dcc0c11c682025e396e5549b74 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 7 Jun 2024 09:55:31 +0200 Subject: [PATCH 2/5] pyproject: document dev/build profile dependency policies --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0052d2660..527a135a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,10 @@ namespaces = false [project.optional-dependencies] dev = [ + # Dev and build dependencies are not relaxed because + # we want all developer environments to be consistent. + # These dependencies are not used in production environments + # and should not conflict with other libraries/tooling. "pre-commit==3.5.0", "pytest==8.0.0", "pytest-sugar==1.0.0", @@ -148,6 +152,10 @@ dev = [ "deptry==0.16.1" ] build = [ + # Dev and build dependencies are not relaxed because + # we want all developer environments to be consistent. + # These dependencies are not used in production environments + # and should not conflict with other libraries/tooling. "pyinstaller==6.7.0", "setuptools==69.5.1", "build==1.2.1" From ecac530e2f47b8215b14a593da0dcdaa8a0166cd Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 7 Jun 2024 09:56:41 +0200 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da9482f0d..8a682a51e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - render maec/* fields #843 @s-ff - replace Halo spinner with Rich #2086 @s-ff - optimize rule matching #2080 @williballenthin +- relax dependency version requirements for the capa library #2053 @williballenthin ### Breaking Changes From 7fd9e504c5c6301de1c1b0a3c2ff4fab8e961120 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 7 Jun 2024 09:59:05 +0200 Subject: [PATCH 4/5] doc: installation: describe requirements.txt usage --- doc/installation.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/installation.md b/doc/installation.md index 57c939c2b..93df732c2 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -91,6 +91,12 @@ For more details about creating and using virtual environments, check out the [v ##### Install development dependencies +When developing capa, please use the pinned dependencies found in `requirements.txt`. +This ensures that everyone has the exact same, reproducible environment. +Please install these dependencies before install capa (from source or from PyPI): + +`$ pip install -r requirements.txt` + We use the following tools to ensure consistent code style and formatting: - [black](https://github.com/psf/black) code formatter - [isort](https://pypi.org/project/isort/) code formatter From 26c1859e9ab0d24a6ca98d0cf492af8185dea5bf Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 7 Jun 2024 10:04:33 +0200 Subject: [PATCH 5/5] pyproject: don't use dnfile 0.15 yet --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 527a135a3..268950764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,6 @@ dependencies = [ # latest bug fixes we need here. "viv-utils[flirt]>=0.7.9", "vivisect>=1.1.1", - "dnfile>=0.14.1", "dncil>=1.0.2", # --------------------------------------- @@ -98,8 +97,13 @@ dependencies = [ # typically due to dropping support for python releases # we still support. - # networkx 3.2 doesn't support python 3.8 + # TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does. + # https://github.com/mandiant/capa/issues/1966 "networkx>=3,<3.2", + + # TODO(williballenthin): dnfile 0.15 changes UserString API and we havent updated yet. + # https://github.com/mandiant/capa/pull/2037 + "dnfile>=0.14.1,<0.15", ] dynamic = ["version"]