relax pyproject dependency versions and introduce requirements.txt (#…

…1012)

closes #1011

.github/workflows/build.yml

@@ -35,7 +35,9 @@ jobs:
         with:
           python-version: '3.8'
       - name: Install floss [build]
-        run: pip install -e .[build]
+        run: |
+          pip install -r requirements.txt
+          pip install -e .[build]
       - name: Build standalone executable
         run: pyinstaller .github/pyinstaller/floss.spec
       - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1

.github/workflows/publish.yml

@@ -25,6 +25,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           pip install -e .[build]
       - name: build package
         run: |

.github/workflows/tests.yml

@@ -22,7 +22,9 @@ jobs:
       with:
         python-version: '3.8'
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Lint with isort
       run: pre-commit run isort
     - name: Lint with black
@@ -59,6 +61,8 @@ jobs:
       if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
     - name: Install FLOSS
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Run tests
       run: pytest tests/

.gitignore

@@ -29,3 +29,6 @@ flare_floss.egg-info
 
 # vscode
 .vscode
+.direnv/
+.env/
+.envrc

doc/installation.md

@@ -90,6 +90,12 @@ You'll find that the `floss.exe` (Windows) or `floss` (Linux, macOS) executables
 
 ### Step 3: Install development and testing dependencies
 
+When developing FLOSS, please use the pinned dependencies found in `requirements.txt`.
+This ensures that everyone has the exact same, reproducible environment.
+Please install these dependencies before install FLOSS (from source or from PyPI):
+
+`$ pip install -r requirements.txt`
+
 To install all testing and development dependencies, run:
 
 `$ pip install -e /local/path/to/src[dev]`

pyproject.toml

@@ -31,16 +31,67 @@ classifiers = [
 ]
 
 dependencies = [
-    "tabulate==0.9.0",
-    "vivisect==1.1.1",
-    "viv-utils[flirt]==0.7.9",
-    "pydantic==2.7.3",
-    "tqdm==4.66.4",
-    "networkx==3.1",
-    "halo==0.0.31",
-    "rich==13.7.1",
+    # ---------------------------------------
+    # As a library, FLOSS uses lower version bounds
+    # when specifying its dependencies. This lets
+    # other programs that use FLOSS (and other libraries)
+    # to find a compatible set of dependency versions.
+    #
+    # We can optionally pin to specific versions or
+    # limit the upper bound when there's a good reason;
+    # but the default is to assume all greater versions
+    # probably work with FLOSS until proven otherwise.
+    #
+    # The following link provides good background:
+    # https://iscinumpy.dev/post/bound-version-constraints/
+    #
+    # When we develop FLOSS, and when we distribute it as
+    # a standalone binary, we'll use specific versions
+    # that are pinned in requirements.txt.
+    # But the requirements for a library are specified here
+    # and are looser.
+    #
+    # Related discussions:
+    # 
+    #   - https://github.com/mandiant/capa/issues/2053
+    #   - https://github.com/mandiant/capa/pull/2059
+    #   - https://github.com/mandiant/capa/pull/2079
+    #
+    # ---------------------------------------
+    # The following dependency versions were imported
+    # during June 2024 by truncating specific versions to
+    # their major-most version (major version when possible, 
+    # or minor otherwise).
+    # As specific constraints are identified, please provide
+    # comments and context.
+    "tabulate>=0.9",
+    "pydantic>=2",
+    "tqdm>=4",
+    "halo>=0.0.31",
+    "rich>=13",
     "pefile>=2022.5.30",
-    "binary2strings==0.1.13",
+    "binary2strings>=0.1",
+
+    # ---------------------------------------
+    # Dependencies that we develop
+    #
+    # These dependencies are often actively influenced by capa,
+    # so we provide a minimum patch version that includes the
+    # latest bug fixes we need here.
+    "viv-utils[flirt]>=0.7.9",
+    "vivisect>=1.1.1",
+    "dncil>=1.0.2",
+
+    # ---------------------------------------
+    # Dependencies with version caps
+    #
+    # These dependencies must not exceed the version cap,
+    # typically due to dropping support for python releases
+    # we still support.
+
+    # TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does.
+    # https://github.com/mandiant/capa/issues/1966
+    "networkx>=3,<3.2",
 ]
 dynamic = ["version", "readme"]
 
@@ -52,6 +103,10 @@ readme = {file = "README.md", content-type = "text/markdown"}
 packages = ["floss", "floss.sigs"]
 
 [project.optional-dependencies]
+# Dev and build dependencies are not relaxed because
+# we want all developer environments to be consistent.
+# These dependencies are not used in production environments
+# and should not conflict with other libraries/tooling.
 dev = [
     "pre-commit==3.5.0",
     "pyyaml==6.0.1",

requirements.txt

@@ -0,0 +1,57 @@
+annotated-types==0.7.0
+binary2strings==0.1.13
+black==24.4.2
+cfgv==3.4.0
+click==8.1.7
+colorama==0.4.6
+coverage==7.5.3
+cxxfilt==0.2.2
+distlib==0.3.8
+dncil==1.0.2
+filelock==3.14.0
+funcy==2.0
+halo==0.0.31
+identify==2.5.36
+iniconfig==2.0.0
+intervaltree==3.1.0
+isort==5.13.2
+log-symbols==0.0.14
+markdown-it-py==3.0.0
+mdurl==0.1.2
+msgpack==1.0.8
+mypy==1.10.0
+networkx==3.1
+nodeenv==1.9.1
+packaging==24.1
+pathspec==0.12.1
+pefile==2023.2.7
+pip==24.0
+platformdirs==4.2.2
+pluggy==1.5.0
+pre-commit==3.5.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.11.1
+pycparser==2.22
+pydantic==2.7.2
+pydantic-core==2.18.3
+pygments==2.18.0
+pytest==8.2.1
+pytest-cov==5.0.0
+pytest-instafail==0.5.0
+pytest-sugar==1.0.0
+python-flirt==0.8.6
+pyyaml==6.0.1
+rich==13.7.1
+setuptools==65.5.0
+six==1.16.0
+sortedcontainers==2.4.0
+spinners==0.0.24
+tabulate==0.9.0
+termcolor==2.4.0
+tqdm==4.66.4
+types-pyyaml==6.0.10
+types-tabulate==0.9.0.20240106
+virtualenv==20.26.2
+viv-utils==0.7.9
+vivisect==1.1.1