From 4b40bc75f28752620a78371e6ee8266a8ca00548 Mon Sep 17 00:00:00 2001 From: Erik Moeller Date: Tue, 9 Apr 2024 10:16:41 -0700 Subject: [PATCH] Remove migrated CircleCI jobs and cleanup script --- .circleci/config.yml | 89 +-------------------------- scripts/clean-old-packages | 122 ------------------------------------- 2 files changed, 2 insertions(+), 209 deletions(-) delete mode 100755 scripts/clean-old-packages diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dfe65b4..10252839 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,14 +1,5 @@ --- common-steps: - - &persist - persist_to_workspace: - root: /tmp/workspace - paths: - - "*" - - - &attach - attach_workspace: - at: /tmp/workspace - &installdeps run: @@ -24,11 +15,6 @@ common-steps: apt-get install reprotest faketime -y --no-install-recommends .venv/bin/pip install -r test-requirements.txt - - &addsshkeys - add_ssh_keys: - fingerprints: - - "85:3a:62:40:e2:91:ab:00:a0:67:9d:c5:26:e1:09:9f" - version: 2.1 @@ -65,76 +51,11 @@ jobs: chown ci:root -R . sudo -u ci bash -c "source .venv/bin/activate && pytest -vvs tests/test_reproducible_wheels.py" - build-rpm: - parameters: - package: - type: string - docker: - - image: fedora:32 - environment: - PKG_NAME: << parameters.package >> - steps: - - checkout - - run: - name: Clone and install dependencies - command: | - dnf install git make -y - git clone https://github.com/freedomofpress/${PKG_NAME}.git - cd ${PKG_NAME} - make install-deps - - run: - name: Bump version and build rpm - command: | - cd ${PKG_NAME} - # Version format is "${VERSION}-0.YYYYMMDDHHMMSS.fXX", which sorts lower than "${VERSION}-1" - rpmdev-bumpspec --new="$(cat VERSION)-0.$(date +%Y%m%d%H%M%S)%{?dist}" rpm-build/SPECS/*.spec - make build-rpm - mkdir -p /tmp/workspace/ - cp -v rpm-build/RPMS/noarch/*.rpm /tmp/workspace/ - - *persist - - push-rpm: - docker: - - image: fedora:32 - steps: - - checkout - - *attach - - *addsshkeys - - run: - name: Commit and push - command: | - dnf install -y ca-certificates git git-lfs openssh-clients python3-rpm python3-debian rpmdevtools - git clone git@github.com:freedomofpress/securedrop-yum-test.git - cd securedrop-yum-test - git lfs install - git config user.email "securedrop@freedom.press" - git config user.name "sdcibot" - mkdir -p workstation/dom0/f32-nightlies - # Copy the new packages over and cleanup the old ones - cp -v /tmp/workspace/*.rpm workstation/dom0/f32-nightlies/ - ~/project/scripts/clean-old-packages workstation/dom0/f32-nightlies 7 - git add . - # If there are changes, diff-index will fail, so we commit - git diff-index --quiet HEAD || git commit -m "Automated SecureDrop workstation build" - # And clean up non-nightly packages too - ~/project/scripts/clean-old-packages workstation/dom0/f32 4 - git add . - git diff-index --quiet HEAD || git commit -m "Cleanup old packages" - - git push origin main - - workflows: - build-packages: + builder_ci: jobs: - lint-and-test - reprotest-wheels - - build-rpm: - matrix: - parameters: - package: &rpmpackages - - securedrop-updater - - securedrop-workstation nightly: triggers: @@ -145,10 +66,4 @@ workflows: only: - main jobs: - - build-rpm: - matrix: - parameters: - package: *rpmpackages - - push-rpm: - requires: - - build-rpm + - reprotest-wheels diff --git a/scripts/clean-old-packages b/scripts/clean-old-packages deleted file mode 100755 index 3e306674..00000000 --- a/scripts/clean-old-packages +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -""" -Clean up old packages, specifying how many to keep - -Example: - ./clean-old-packages securedrop-apt-test/workstation/buster-nightlies 7 - -""" -import argparse -import functools -import subprocess -from collections import defaultdict -from pathlib import Path -from typing import Tuple - -import rpm -from debian import debfile - - -def sort_deb_versions(one: Tuple[str, Path], two: Tuple[str, Path]): - """sort two Debian package versions""" - status = subprocess.run(['dpkg', '--compare-versions', one[0], 'lt', two[0]]) - if status.returncode == 1: - # false, one is bigger - return 1 - else: - # true, two is bigger - return -1 - - -def sort_rpm_versions(one: Tuple[str, Path], two: Tuple[str, Path]): - """sort two RPM package versions""" - status = subprocess.run(['rpmdev-vercmp', one[0], two[0]], stdout=subprocess.DEVNULL) - if status.returncode == 11: - # false, one is bigger - return 1 - else: # status.returncode == 12 - # true, two is bigger - return -1 - - -def fix_name(name: str) -> str: - """ - Linux packages embed the version in the name, so we'd never have multiple - packages meet the deletion threshold. Silly string manipulation to drop - the version. - E.g. "linux-image-5.15.26-grsec-securedrop" -> "linux-image-securedrop" - """ - if name.endswith(('-securedrop', '-workstation')): - suffix = name.split('-')[-1] - else: - return name - if name.startswith('linux-image-'): - return f'linux-image-{suffix}' - elif name.startswith('linux-headers-'): - return f'linux-headers-{suffix}' - return name - - -def rpm_info(path: Path) -> Tuple[str, str]: - """ - learned this incantation from - and help(headers) - """ - ts = rpm.ts() - with path.open() as f: - headers = ts.hdrFromFdno(f) - return headers[rpm.RPMTAG_NAME], headers[rpm.RPMTAG_VERSION] + '-' + headers[rpm.RPMTAG_RELEASE] - - -def cleanup(data, to_keep: int, sorter): - for name, versions in sorted(data.items()): - if len(versions) <= to_keep: - # Nothing to delete - continue - print(f'### {name}') - items = sorted(versions.items(), key=functools.cmp_to_key(sorter), reverse=True) - keeps = items[:to_keep] - print('Keeping:') - for _, keep in keeps: - print(f'* {keep.name}') - delete = items[to_keep:] - print('Deleting:') - for _, path in delete: - print(f'* {path.name}') - path.unlink() - - -def main(): - parser = argparse.ArgumentParser( - description="Cleans up old packages" - ) - parser.add_argument( - "directory", - type=Path, - help="Directory to clean up", - ) - parser.add_argument( - "keep", - type=int, - help="Number of packages to keep" - ) - args = parser.parse_args() - if not args.directory.is_dir(): - raise RuntimeError(f"Directory, {args.directory}, doesn't exist") - print(f'Only keeping the latest {args.keep} packages') - debs = defaultdict(dict) - rpms = defaultdict(dict) - for deb in args.directory.glob('*.deb'): - control = debfile.DebFile(deb).control.debcontrol() - name = fix_name(control['Package']) - debs[name][control['Version']] = deb - for rpm in args.directory.glob('*.rpm'): - name, version = rpm_info(rpm) - rpms[name][version] = rpm - - cleanup(debs, args.keep, sort_deb_versions) - cleanup(rpms, args.keep, sort_rpm_versions) - - -if __name__ == '__main__': - main()