diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fba93e1..160d1ae7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,6 +60,7 @@ common-steps: run: name: Build debian package command: | + source .venv/bin/activate export VERSION_CODENAME=$(~/project/scripts/codename) export PKG_PATH=~/packaging/$PKG_NAME/ export PKG_VERSION=$VERSION_TO_BUILD diff --git a/Makefile b/Makefile index ea15ebca..815ac85a 100644 --- a/Makefile +++ b/Makefile @@ -46,17 +46,15 @@ requirements: ## Creates requirements files for the Python projects ./scripts/update-requirements .PHONY: build-wheels -build-wheels: ## Builds the wheels and adds them to the localwheels directory +build-wheels: ## Builds the wheels and adds them to the wheels subdirectory ./scripts/verify-sha256sum-signature $$(basename ${PKG_DIR}) ./scripts/build-sync-wheels ./scripts/sync-sha256sums $$(basename ${PKG_DIR}) - @printf "Done! Now please follow the instructions in\n" - @printf "https://github.com/freedomofpress/securedrop-debian-packaging-guide/" - @printf "to push these changes to the FPF PyPI index\n" + @echo Done! .PHONY: test test: ## Run simple test suite (skips reproducibility checks) - pytest -v tests/test_update_requirements.py tests/test_deb_package.py + pytest -v tests/test_update_requirements.py tests/test_deb_package.py tests/test_utils.py .PHONY: reprotest reprotest: ## Runs only reproducibility tests, for .deb and .whl files diff --git a/README.md b/README.md index 0b78b934..7bd5f230 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # securedrop-builder -`securedrop-builder` is the tool we use to package Python projects into Debian packages for the [SecureDrop Workstation](https://github.com/freedomofpress/securedrop-workstation). +`securedrop-builder` is the tool we use to package Python projects into Debian packages for the [SecureDrop Workstation](https://github.com/freedomofpress/securedrop-workstation). This repository also holds copies of reproducibly built wheels included with some Debian packages, in the `wheels` subdirectory of the package. * For instructions on how to build [SecureDrop](https://github.com/freedomofpress/securedrop) Debian packages, see https://developers.securedrop.org/en/latest/release_management.html. @@ -100,8 +100,9 @@ the requirements files which are used for build of these packages (`build-requir using `make requirements` are kept up to date in latest `main` of those repositories. If new dependencies were added in the `build-requirements.txt` of that -repo that are not in the FPF PyPI mirror (`./localwheels/` in this repository), then the maintainer needs -to do the following (we are taking `securedrop-client` project as example): +repo that are not in the `wheels` subdirectory for the package in this repository, +then the maintainer needs to do the following (we are taking `securedrop-client` project +as an example): ### 0. Enable the virtualenv @@ -133,8 +134,7 @@ pytest==3.10.1 Please build the wheel by using the following command. PKG_DIR=/home/user/code/securedrop-client make build-wheels -Then add the newly built wheels and sources to ./localwheels/. -Also update the index HTML files accordingly commit your changes. +Then add the newly built wheels and sources to the `wheels` subdirectory for the package. After these steps, please rerun the command again. ``` @@ -153,12 +153,13 @@ This above command will let you know about any new wheels + sources. It will build/download sources from PyPI (by verifying it against the sha256sums from the `requirements.txt` of the project). -### 3. Commit changes to the localwheels directory (if only any update of wheels) +### 3. Commit changes to the wheels directory (if only any update of wheels) -Now add these built artifacts to version control: +Now add these built artifacts to version control, from the relevant package +directory: ```shell -git add localwheels/ +git add wheels/ git commit ``` diff --git a/scripts/build-sync-wheels b/scripts/build-sync-wheels index bf974f2f..ef4507da 100755 --- a/scripts/build-sync-wheels +++ b/scripts/build-sync-wheels @@ -19,9 +19,8 @@ if os.geteuid() == 0: # sys.exit(1) if "VIRTUAL_ENV" not in os.environ: print( - "This script should be run in a virtualenv: " - "`source .venv/bin/activate`", - file=sys.stderr + "This script should be run in a virtualenv: `source .venv/bin/activate`", + file=sys.stderr, ) sys.exit(1) @@ -43,17 +42,18 @@ WHEEL_BUILD_DIR = "/tmp/pip-wheel-build" def main(): - if "PKG_DIR" in os.environ and \ - not any(arg.startswith("--pkg-dir") for arg in sys.argv): + if "PKG_DIR" in os.environ and not any( + arg.startswith("--pkg-dir") for arg in sys.argv + ): sys.argv.extend(["--pkg-dir", os.environ["PKG_DIR"]]) - parser = argparse.ArgumentParser( - description="Builds and stores sources and wheels" - ) + parser = argparse.ArgumentParser(description="Builds and stores sources and wheels") parser.add_argument("--pkg-dir", help="Package directory", required=True) parser.add_argument("--project", help="Project name to update") parser.add_argument( - "--clobber", action="store_true", default=False, + "--clobber", + action="store_true", + default=False, help="Whether to overwrite wheels and source tarballs", ) parser.add_argument( @@ -79,12 +79,25 @@ def main(): if args.project is not None: project_name = args.project else: - project_name = utils.project_name(Path(args.pkg_dir)) + project_name = utils.get_project_name(Path(args.pkg_dir)) local_wheels = os.path.join(project_name, "wheels") + req_path = os.path.join(args.pkg_dir, args.requirements, "requirements.txt") + poetry_lock_path = os.path.join(args.pkg_dir, "poetry.lock") + + use_poetry = False + # Check if requirements.txt exists, if not check for poetry.lock and create a temporary requirements.txt if not os.path.exists(req_path): - print("requirements.txt missing at {0}.".format(req_path)) - sys.exit(3) + if os.path.exists(poetry_lock_path): + use_poetry = True + print( + f"requirements.txt was not found at {req_path}, but poetry.lock was found at {poetry_lock_path}, using." + ) + else: + print( + f"requirements.txt not found at {req_path} and poetry.lock not found at {poetry_lock_path}." + ) + sys.exit(3) if os.path.exists(WHEEL_BUILD_DIR): shutil.rmtree(WHEEL_BUILD_DIR) @@ -93,9 +106,22 @@ def main(): os.mkdir(WHEEL_BUILD_DIR) with tempfile.TemporaryDirectory() as tmpdir: + if use_poetry: + poetry_reqs = utils.get_requirements_from_poetry(Path(args.pkg_dir) / 'poetry.lock', Path(args.pkg_dir) / 'pyproject.toml') + req_path = os.path.join(tmpdir, 'requirements.txt') + with open(req_path, 'w') as req_file: + req_file.write(poetry_reqs) + # The --require-hashes option will be used by default if there are # hashes in the requirements.txt file. We specify it anyway to guard # against use of a requirements.txt file without hashes. + # + # NOTE: Even with this invocation, pip may execute build steps, as + # part of its metadata collection process. Switching to + # manual downloading and hash verification may be preferable + # to make this process more resilient. + # + # See https://github.com/pypa/pip/issues/1884 for background. cmd = [ "pip3", "download", @@ -122,7 +148,16 @@ def main(): for project in project_names: print(f"Building {project}") source_path = os.path.join(WHEEL_BUILD_DIR, project) - cmd = ["python3", "-m", "build", "--wheel", source_path, "--no-isolation", "-o", tmpdir] + cmd = [ + "python3", + "-m", + "build", + "--wheel", + source_path, + "--no-isolation", + "-o", + tmpdir, + ] subprocess.check_call(cmd) print(f"build command used: {' '.join(cmd)}") diff --git a/scripts/sync-sha256sums b/scripts/sync-sha256sums index 1b847023..89e2e4f7 100755 --- a/scripts/sync-sha256sums +++ b/scripts/sync-sha256sums @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# A script to update the sha256sums from localwheels directory +# A script to update the sha256sums from local wheels directory import os import subprocess diff --git a/scripts/update-requirements b/scripts/update-requirements index e5e559d4..0bccd0ec 100755 --- a/scripts/update-requirements +++ b/scripts/update-requirements @@ -1,27 +1,34 @@ #!/usr/bin/env python3 -# To update the requirements files with sha256sums from our local PyPI. +# To update the requirements files with sha256sums from our local wheels. import argparse import os import sys import subprocess from pathlib import Path -from typing import List import utils +SKIP_PACKAGES = ["pyqt5", "sip"] + def parse_args(): # For backwards-compat - if "PKG_DIR" in os.environ and \ - not any(arg.startswith("--pkg-dir") for arg in sys.argv): + if "PKG_DIR" in os.environ and not any( + arg.startswith("--pkg-dir") for arg in sys.argv + ): sys.argv.extend(["--pkg-dir", os.environ["PKG_DIR"]]) - parser = argparse.ArgumentParser(description="Update requirements files with sha256sums from our wheels") + parser = argparse.ArgumentParser( + description="Update requirements files with sha256sums from our wheels" + ) parser.add_argument("--pkg-dir", help="Package directory", required=True) parser.add_argument("--project", help="Project to update") - parser.add_argument("--requirements", help="Directory that contains requirements.txt inside the package directory", - default="requirements") + parser.add_argument( + "--requirements", + help="Directory that contains requirements.txt inside the package directory", + default="requirements", + ) return parser.parse_args() @@ -31,20 +38,43 @@ def main(): if args.project is not None: project_name = args.project else: - project_name = utils.project_name(pkg_dir) + project_name = utils.get_project_name(pkg_dir) requirements_file = pkg_dir / args.requirements / "requirements.txt" + if not requirements_file.exists(): + poetry_lock = requirements_file.parent.parent / "poetry.lock" + pyproject_toml = requirements_file.parent.parent / "pyproject.toml" + if poetry_lock.exists() and pyproject_toml.exists(): + requirements_file = poetry_lock + else: + print("Could not find project requirements, checked:") + print(f"* {requirements_file.absolute()}") + print(f"* {poetry_lock.absolute()}") + sys.exit(1) + project = Path(__file__).parent.parent / project_name # First remove index line and any PyQt or sip dependency - cleaned_lines = cleanup(requirements_file) + + if requirements_file.name == "poetry.lock": + dependencies = utils.get_poetry_names_and_versions( + requirements_file, pyproject_toml + ) + else: + dependencies = utils.get_requirements_names_and_versions(requirements_file) + + dependencies = [dep for dep in dependencies if dep[0] not in SKIP_PACKAGES] verify_sha256sums_file(project) - build_requirements_file = pkg_dir / args.requirements / "build-requirements.txt" + if (pkg_dir / args.requirements).exists(): + build_requirements_file = pkg_dir / args.requirements / "build-requirements.txt" + else: + # If there is no 'requirements' directory, we look in the project root + build_requirements_file = pkg_dir / "build-requirements.txt" shasums_file = project / "sha256sums.txt" - # Now let us update the files along with the sha256sums from localwheels - add_sha256sums(build_requirements_file, cleaned_lines, shasums_file, pkg_dir) + # Now let us update the files along with the sha256sums from local wheels + add_sha256sums(build_requirements_file, dependencies, shasums_file, pkg_dir) def verify_sha256sums_file(project: Path): @@ -65,8 +95,9 @@ def verify_sha256sums_file(project: Path): subprocess.check_call(["./scripts/verify-sha256sum-signature", project.name]) -def add_sha256sums(path: Path, requirements_lines: List[str], - shasums_file: Path, pkg_dir: Path) -> None: +def add_sha256sums( + path: Path, dependencies: list[(str, str)], shasums_file: Path, pkg_dir: Path +) -> None: """Adds all the required sha256sums to the wheels""" files = [] @@ -85,18 +116,15 @@ def add_sha256sums(path: Path, requirements_lines: List[str], missing_wheels = [] # For each dependency in the requirements file - for mainline in requirements_lines: - package_name_and_version = mainline.strip().split()[0] - package_name = package_name_and_version.split('==')[0] - package_version = package_name_and_version.split('==')[1] - - wheel_name_prefix = '{}-{}'.format(package_name, package_version) - package_othername = '{}-{}'.format(package_name.replace("-", "_"), package_version) + for package_name, package_version in dependencies: + wheel_name_prefix = "{}-{}".format(package_name, package_version) + package_othername = "{}-{}".format( + package_name.replace("-", "_"), package_version + ) line = "" - for name in files: - lowername = name[1].lower() - digest = name[0] + for digest, name in files: + lowername = name.lower() # Now check if a wheel is already available if lowername.startswith(wheel_name_prefix) or lowername.startswith( @@ -104,16 +132,15 @@ def add_sha256sums(path: Path, requirements_lines: List[str], ): # Now add the hash to the line if line.find("--hash") == -1: - line = "{} --hash=sha256:{}".format( - package_name_and_version, digest) + line = f"{package_name}=={package_version} --hash=sha256:{digest}" else: # Means a second wheel hash - line += " --hash=sha256:{}".format(digest) + line += f" --hash=sha256:{digest}" line += "\n" newlines.append(line) if line.find("--hash") == -1: # Missing wheel - missing_wheels.append(package_name_and_version) + missing_wheels.append(f"{package_name}=={package_version}") # Do not update the file if missing wheels if missing_wheels: @@ -123,10 +150,8 @@ def add_sha256sums(path: Path, requirements_lines: List[str], print("\nPlease build the wheel by using the following command:\n") print(f"\tPKG_DIR={pkg_dir} make build-wheels\n") - print("Then add the newly built wheels and sources to ./localwheels/.") - print("Also update the index HTML files accordingly commit your changes.") - print("After these steps, please rerun the command again.") - + print("Then add the newly built wheels and sources to the wheels subdirectory") + print("for the project. After these steps, please rerun the command again.") sys.exit(1) # Now update the file @@ -134,49 +159,5 @@ def add_sha256sums(path: Path, requirements_lines: List[str], print(f"Updated {path}") -def cleanup(path: Path) -> List[str]: - """Cleans up requirement files - - :param path: The file to cleanup - :type path: str - :return: None - :rtype: None - """ - - lines = path.read_text().splitlines() - - finallines = [] - - for line in lines: - # Skip the PyPI index line so that when we - # install we use the FPF PyPI mirror. - if line.startswith("-i https://pypi.org/simple"): - continue - # We don't want to install pyqt5 from wheels. - # It will come as a debian package dependency. - elif line.startswith("pyqt5"): - continue - # We don't want to install sip from wheels. - # It will come as a debian package dependency. - elif line.startswith("sip"): - continue - # We want to ignore lines that are comments. - elif line.lstrip().startswith("#"): - continue - # We just want the package names, since we're - # using the hashes of the wheels (and we verified - # the integrity of those hashes by verifying the gpg - # signature of a textfile containing the list of hashes). - elif line.lstrip().startswith("--hash=sha256"): - continue - else: - # To strip any extra new line characters - line = line.strip("\ \n") - if line: - finallines.append(line) - - return finallines - - if __name__ == "__main__": main() diff --git a/scripts/utils.py b/scripts/utils.py index 70a0e848..269b3824 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -2,19 +2,179 @@ Shared functions between various scripts """ import re + +try: + import tomllib +except ModuleNotFoundError: + # pre-Python 3.11 compatibility + try: + import toml as tomllib + except ModuleNotFoundError: + print("WARNING: Could not find tomllib or toml, Poetry support unavailable.") + from pathlib import Path RE_NAME = re.compile(r'name="(.*?)"') -def project_name(path: Path) -> str: +def get_project_name(path: Path) -> str: """Extract the project name from setup.py""" setup_py = path / "setup.py" if not setup_py.exists(): - raise RuntimeError(f"No setup.py in {path}." - "If this isn't a Python project, use --project?") + raise RuntimeError( + f"No setup.py in {path}." "If this isn't a Python project, use --project?" + ) search = RE_NAME.search(setup_py.read_text()) if not search: - raise RuntimeError(f"Unable to parse name out of {setup_py}. " - "If this isn't a Python project, use --project?") + raise RuntimeError( + f"Unable to parse name out of {setup_py}. " + "If this isn't a Python project, use --project?" + ) return search.group(1) + + +def get_requirements_names_and_versions(path: Path) -> list[(str, str)]: + """ + Return a list of package names and versions for all dependencies declared in a + requirements.txt file. + """ + ret = [] + for line in path.read_text().splitlines(): + if line.startswith("#"): + continue + if "==" not in line: + continue + name, constraint = line.split("==", 1) + version = constraint.rstrip("\\").strip() + ret.append((name, version)) + return ret + + +def get_poetry_names_and_versions( + path_to_poetry_lock: Path, path_to_pyproject_toml +) -> list[(str, str)]: + """ + Return a list of package names and versions for all main (non-development) dependencies, + including transitive ones, defined via Poetry configuration files. + """ + data = tomllib.loads(path_to_poetry_lock.read_text()) + relevant_dependencies = get_relevant_poetry_dependencies( + path_to_pyproject_toml, path_to_poetry_lock + ) + ret = [] + for package in data["package"]: + if package["name"] not in relevant_dependencies: + continue + ret.append((package["name"], package["version"])) + + return ret + + +def get_relevant_poetry_dependencies( + path_to_pyproject_toml: Path, path_to_poetry_lock: Path +) -> list[str]: + """ + Identify main (non-development) requirements. poetry.lock does not preserve + the distinction between different dependency groups, so we have to parse + both files. + """ + pyproject_dict = tomllib.loads(path_to_pyproject_toml.read_text()) + + # Extract main dependencies + main_dependencies = pyproject_dict["tool"]["poetry"]["dependencies"] + + # Remove 'python' as it's not a package dependency + if "python" in main_dependencies: + del main_dependencies["python"] + + parsed_toml = tomllib.loads(path_to_poetry_lock.read_text()) + + # Create a set to keep track of main and transitive dependencies (set ensures + # no duplicates) + relevant_dependencies = set(main_dependencies) + + # Identify transitive dependencies (may be enumerated in lockfile + # before the dependency which declares them) + for package in parsed_toml.get("package", []): + package_name = package["name"] + if package_name in main_dependencies: + for sub_dependency in package.get("dependencies", {}): + relevant_dependencies.add(sub_dependency) + + return list(relevant_dependencies) + + +def get_poetry_hashes( + path_to_poetry_lock: Path, path_to_pyproject_toml: Path +) -> dict[str, list[str]]: + """ + Get a dictionary for all main (non-development) dependencies and their + valid hashes as defined in a set of requirements as defined in + pyproject.toml/poetry.lock. This includes transitive dependencies of + main dependencies. + """ + dependencies = {} + relevant_dependencies = get_relevant_poetry_dependencies( + path_to_pyproject_toml, path_to_poetry_lock + ) + parsed_toml = tomllib.loads(path_to_poetry_lock.read_text()) + + for package in parsed_toml.get("package", []): + package_name = package["name"] + if package_name in relevant_dependencies: + package_name_and_version = f"{package_name}=={package['version']}" + dependencies[package_name_and_version] = [ + file_dict["hash"].replace("sha256:", "") + for file_dict in package["files"] + ] + + return dependencies + + +def get_requirements_hashes(path_to_requirements_file: Path) -> dict[str, list[str]]: + """ + Return a dictionary of valid hashes for each dependency declared in a requirements + file. + """ + lines = path_to_requirements_file.read_text().splitlines() + + result_dict = {} + current_package = None + + for line in lines: + if line.startswith("#") or not line.strip(): + continue + + package_match = re.match(r"(\S+==\S+)", line) + hash_match = re.search(r"--hash=sha256:([\da-f]+)", line) + + if package_match: + current_package = package_match.group(1) + result_dict[current_package] = [] + elif hash_match and current_package: + result_dict[current_package].append(hash_match.group(1)) + + for package, hashes in result_dict.items(): + if not hashes: + raise ValueError(f"The package {package} does not have any hashes.") + + return result_dict + + +def get_requirements_from_poetry( + path_to_poetry_lock: Path, path_to_pyproject_toml: Path +) -> str: + """ + Returns a multiline string in requirements.txt format for a set of Poetry main dependencies. + """ + # Get the hashes along with the package names and versions + hashes_dict = get_poetry_hashes(path_to_poetry_lock, path_to_pyproject_toml) + + # Form the requirements.txt string + requirements = [] + for package_name_and_version, hashes in hashes_dict.items(): + hash_strings = [f"--hash=sha256:{h}" for h in hashes] + requirement_line = f"{package_name_and_version} {' '.join(hash_strings)}" + requirements.append(requirement_line) + + return "\n".join(requirements) diff --git a/scripts/verify-hashes b/scripts/verify-hashes index 7d015616..c6570f67 100755 --- a/scripts/verify-hashes +++ b/scripts/verify-hashes @@ -3,96 +3,68 @@ import os import sys -if len(sys.argv) != 2: - print("Usage: ./scripts/verify-hashes path/to/sha256sums.txt") - sys.exit(1) +from pathlib import Path +import utils -requirements_file = "requirements.txt" -if not os.path.exists(requirements_file): - requirements_file = "requirements/requirements.txt" - if not os.path.exists(requirements_file): - print("Cannot find requirements.txt") - sys.exit(1) - -# This is the already gpg signed and verified data -sha256sum_data = {} -with open(sys.argv[1]) as fobj: - data = fobj.readlines() - -for line in data: - line = line.strip() - if line.startswith('#'): - continue - words = line.split() - # just check that the sums are of correct length - if len(words[0]) != 64: - print("Wrong sha256sum {0}".format(words[0])) - sys.exit(3) - sha256sum_data[words[0]] = True - - -# Now read the requirements.txt file -lines = [] -with open(requirements_file) as fobj: - lines = fobj.readlines() - -# Now we want to verify that for each dependency in the project -# to be packaged, has a matching source tarball on FPF's PyPI. - -# Remove lines with comments. -uncommented_lines = [line for line in lines if not line.lstrip().startswith('#')] - -# The hashes for a given requirement will be distributed -# across multiple lines, e.g. -# -# atomicwrites==1.2.1 \ -# --hash=sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0 \ -# --hash=sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee +# For a requirements file in one of our projects, this script checks if we have at +# least one corresponding sha256sum (for the source tarball) for each dependency. +# The script should be run from the project directory whose requirements you +# want to parse. # -# Let's create a list with one list element per dependency. - -dependencies_with_hashes = ''.join(uncommented_lines).replace('\\\n', '').splitlines() - -# Now we'll construct a dict containing each dependency, -# and a list of its hashes, e.g.: -# -# { -# 'alembic': ['04bcb970ca8659c3607ddd8ffd86cc9d6a99661c9bc590955e8813c66bfa582b'] -# } -# -# Note that at this point the hashes can be of upstream wheels. - -dependencies = {} -for dependency_line in dependencies_with_hashes: - if not dependency_line: - continue - - package_name_and_version = dependency_line.split()[0] - - # If this fails, we are missing a hash in requirements.txt. - assert len(dependency_line.split()) >= 2 - - hashes = [] - for sha_256_hash in dependency_line.split()[1:]: - hashes.append(sha_256_hash.replace("--hash=sha256:", "")) - - dependencies.update({ - package_name_and_version: hashes - }) - -# Now check, for each dependency that there is at least one matching hash -# on FPF's PyPI (this will be the hash of the source tarball). -for dependency in dependencies.keys(): - - found_a_hash = False - for requirements_sha_256_hash in dependencies[dependency]: - if requirements_sha_256_hash in sha256sum_data: - found_a_hash = True - - # If we get here, it means we did not find a corresponding hash in our - # sha256sums data (representing the state of FPF's PyPI) - if not found_a_hash: - print("Missing sha256sum for package: {0}".format(dependency)) +# For now, both requirements.txt and poetry.lock/pyproject.toml are supported. +# Once Poetry support is stable across all components, requirements.txt support +# can be deprecated. + + +def main(): + """ " + Ensure that we have at least one signed SHA256 sum for each dependency declared + in a project's requirements file. This will be the source tarball. + """ + if len(sys.argv) != 2: + print("Usage: ./scripts/verify-hashes path/to/sha256sums.txt") sys.exit(1) -sys.exit(0) + sha256sum_data = {} + with open(sys.argv[1]) as fobj: + data = fobj.readlines() + for line in data: + line = line.strip() + if line.startswith("#"): + continue + words = line.split() + if len(words[0]) != 64: + print(f"'{words[0]}' does not appear to be a sha256sum - exiting.") + sys.exit(3) + sha256sum_data[words[0]] = True + + if os.path.exists("poetry.lock") and os.path.exists("pyproject.toml"): + dependencies = utils.get_poetry_hashes( + Path("poetry.lock"), Path("pyproject.toml") + ) + else: + requirements_file = "requirements.txt" + if not os.path.exists(requirements_file): + requirements_file = "requirements/requirements.txt" + if not os.path.exists(requirements_file): + print( + "Cannot find poetry.lock/pyproject.toml or requirements.txt or requirements/requirements.txt" + ) + sys.exit(1) + + dependencies = utils.get_requirements_hashes(Path(requirements_file)) + + for dependency in dependencies.keys(): + found_a_hash = False + for requirements_sha_256_hash in dependencies[dependency]: + if requirements_sha_256_hash in sha256sum_data: + found_a_hash = True + if not found_a_hash: + print(f"Missing sha256sum for package: {dependency} - exiting.") + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-sha256sum-signature b/scripts/verify-sha256sum-signature index dd2a58f9..97f9dc47 100755 --- a/scripts/verify-sha256sum-signature +++ b/scripts/verify-sha256sum-signature @@ -1,6 +1,6 @@ #!/bin/bash # Wrapper to verify that the checksums file, used to track the integrity -# of the assets in the PyPI mirror, has a valid signature. +# of the local wheels, has a valid signature. # # We expect a valid signature to correspond to a SecureDrop Maintainer, # so we create a temporary keyring in order to force gpg to use *only* diff --git a/tests/fixtures/poetry.lock b/tests/fixtures/poetry.lock new file mode 100644 index 00000000..134ea2e9 --- /dev/null +++ b/tests/fixtures/poetry.lock @@ -0,0 +1,142 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cowsay" +version = "6.0" +description = "The famous cowsay for GNU/Linux is now available for python" +optional = false +python-versions = "*" +files = [ + {file = "cowsay-6.0-py2.py3-none-any.whl", hash = "sha256:77b07c508af48aa300a90f3b3c5c013a12360a71fc5c87b1efb763fe2803a775"}, + {file = "cowsay-6.0-py3-none-any.whl", hash = "sha256:011c067841451ea49baf8ff49c355bd7f659d53fc538459e0a84c12ae2b4e027"}, + {file = "cowsay-6.0.tar.gz", hash = "sha256:47445cb273684618a1786db8e8d05ec9258455f7eb74893e5d0933daafeb44ba"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "7.4.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "d50a30a52449453ec493a0b572214f497fdeab8cdf117ce4765ac1bd850f7a2a" diff --git a/tests/fixtures/pyproject.toml b/tests/fixtures/pyproject.toml new file mode 100644 index 00000000..b8c97889 --- /dev/null +++ b/tests/fixtures/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "test" +version = "0.1.0" +description = "" +authors = ["SecureDrop Team "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.9" +colorama = "0.4.6" +cowsay = "^6.0" +beautifulsoup4 = "^4.12.2" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.2" diff --git a/tests/fixtures/requirements.txt b/tests/fixtures/requirements.txt new file mode 100644 index 00000000..709bc853 --- /dev/null +++ b/tests/fixtures/requirements.txt @@ -0,0 +1,13 @@ +beautifulsoup4==4.12.2 \ + --hash=sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da \ + --hash=sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +cowsay==6.0 \ + --hash=sha256:011c067841451ea49baf8ff49c355bd7f659d53fc538459e0a84c12ae2b4e027 \ + --hash=sha256:47445cb273684618a1786db8e8d05ec9258455f7eb74893e5d0933daafeb44ba \ + --hash=sha256:77b07c508af48aa300a90f3b3c5c013a12360a71fc5c87b1efb763fe2803a775 +soupsieve==2.5 \ + --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ + --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 diff --git a/tests/test_deb_package.py b/tests/test_deb_package.py index a76e2fcc..84c62970 100644 --- a/tests/test_deb_package.py +++ b/tests/test_deb_package.py @@ -1,4 +1,3 @@ -import os import subprocess import tempfile from pathlib import Path diff --git a/tests/test_update_requirements.py b/tests/test_update_requirements.py index d9bc2efd..1fa8bb5d 100644 --- a/tests/test_update_requirements.py +++ b/tests/test_update_requirements.py @@ -1,9 +1,9 @@ -import imp +from importlib.machinery import SourceFileLoader import os import sys import pytest from pathlib import Path - +import types # This below stanza is necessary because the scripts are not # structured as a module. @@ -11,12 +11,16 @@ os.path.dirname(os.path.abspath(__file__)), "../scripts/update-requirements" ) sys.path.append(os.path.dirname(path_to_script)) -update_requirements = imp.load_source("update-requirements", path_to_script) +update_requirements = types.ModuleType("update-requirements") +loader = SourceFileLoader("update-requirements", path_to_script) +loader.exec_module(update_requirements) + +TEST_DEPENDENCIES = [('pathlib2', '2.3.2')] TEST_SOURCE_HASH = "8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83" TEST_WHEEL_HASH = "8e276e2bf50a9a06c36e20f03b050e59b63dfe0678e37164333deb87af03b6ad" -TEST_SHASUM_LINES = ["{} pathlib2-2.3.2-py2.py3-none-any.whl".format(TEST_WHEEL_HASH), - "{} pathlib2-2.3.2.tar.gz".format(TEST_SOURCE_HASH)] +TEST_SHASUM_LINES = ["\n{} pathlib2-2.3.2-py2.py3-none-any.whl".format(TEST_WHEEL_HASH), + "\n{} pathlib2-2.3.2.tar.gz".format(TEST_SOURCE_HASH)] def test_build_fails_if_sha256_sums_absent(tmpdir, mocker): @@ -44,12 +48,9 @@ def test_shasums_skips_sources(tmpdir): with open(path_test_shasums, 'w') as f: f.writelines(TEST_SHASUM_LINES) - requirement_name = "pathlib2" - requirement_version = "2.3.2" - requirements_lines = ["{}=={}".format(requirement_name, requirement_version)] path_result = os.path.join(tmpdir, "test-req.txt") - update_requirements.add_sha256sums(Path(path_result), requirements_lines, Path(path_test_shasums), Path("foo")) + update_requirements.add_sha256sums(Path(path_result), TEST_DEPENDENCIES, Path(path_test_shasums), Path("foo")) with open(path_result, 'r') as f: result = f.read() @@ -63,13 +64,10 @@ def test_build_fails_if_missing_wheels(tmpdir): with open(path_test_shasums, 'w') as f: f.writelines([]) - requirement_name = "pathlib2" - requirement_version = "2.3.2" - requirements_lines = ["{}=={}".format(requirement_name, requirement_version)] path_result = os.path.join(tmpdir, "test-req.txt") with pytest.raises(SystemExit) as exc_info: - update_requirements.add_sha256sums(Path(path_result), requirements_lines, Path(path_test_shasums), Path("foo")) + update_requirements.add_sha256sums(Path(path_result), TEST_DEPENDENCIES, Path(path_test_shasums), Path("foo")) exit_code = exc_info.value.args[0] assert exit_code == 1 diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..84ea2889 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,86 @@ +import os +import pytest +import sys +from pathlib import Path + +# Adjusting the path to import utils module +path_to_script = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../scripts/utils.py" +) +sys.path.append(os.path.dirname(path_to_script)) + +from utils import ( + get_requirements_names_and_versions, + get_poetry_names_and_versions, + get_relevant_poetry_dependencies, + get_poetry_hashes, + get_requirements_hashes, + get_requirements_from_poetry, +) + +# These tests generally verify that our utility functions correctly parse the +# fixtures below. The Poetry fixture includes a development-only requirement +# (pytest) which should not be returned. +PYPROJECT_TOML_PATH = Path(__file__).parent / "fixtures/pyproject.toml" +POETRY_LOCK_PATH = Path(__file__).parent / "fixtures/poetry.lock" +REQUIREMENTS_TXT_PATH = Path(__file__).parent / "fixtures/requirements.txt" +EXPECTED_DEPENDENCIES = [ + ("beautifulsoup4", "4.12.2"), + ("colorama", "0.4.6"), + ("cowsay", "6.0"), + ("soupsieve", "2.5"), +] +EXPECTED_DEPENDENCY_NAMES = [name for name, _ in EXPECTED_DEPENDENCIES] +EXPECTED_KEYS = [f"{name}=={version}" for name, version in EXPECTED_DEPENDENCIES] + + +def test_get_requirements_names_and_versions(): + assert sorted(get_requirements_names_and_versions(REQUIREMENTS_TXT_PATH)) == sorted( + EXPECTED_DEPENDENCIES + ) + + +def test_get_poetry_names_and_versions(): + output = get_poetry_names_and_versions(POETRY_LOCK_PATH, PYPROJECT_TOML_PATH) + assert sorted(output) == sorted(EXPECTED_DEPENDENCIES) + + +def test_get_relevant_poetry_dependencies(): + output = get_relevant_poetry_dependencies(PYPROJECT_TOML_PATH, POETRY_LOCK_PATH) + # Ensure we correctly ignore development-only dependencies + assert "pytest" not in output + assert sorted(output) == sorted(EXPECTED_DEPENDENCY_NAMES) + + +def _check_hashes(output): + assert sorted(list(output)) == sorted(EXPECTED_KEYS) + for _, hashes in output.items(): + # We should have at least one hash per dependency + assert len(hashes) > 0 + # Hex-encoded SHA-256 hashes are 64 characters long + for hash in hashes: + assert len(hash) == 64 + + +def test_get_poetry_hashes(): + output = get_poetry_hashes(POETRY_LOCK_PATH, PYPROJECT_TOML_PATH) + _check_hashes(output) + + +def test_get_requirements_hashes(): + output = get_requirements_hashes(REQUIREMENTS_TXT_PATH) + _check_hashes(output) + + +def test_get_requirements_from_poetry(): + output = get_requirements_from_poetry(POETRY_LOCK_PATH, PYPROJECT_TOML_PATH) + output_lines = output.strip().split("\n") + assert len(output_lines) == len(EXPECTED_KEYS) + for line in output_lines: + package_name_and_version, *hashes = line.split() + assert package_name_and_version in EXPECTED_KEYS + assert all(hash_str.startswith("--hash=sha256:") for hash_str in hashes) + + +if __name__ == "__main__": + pytest.main([__file__])