From 9fb257ab0eaad89d51af645fd4d69f866bc86d6a Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Fri, 13 Dec 2024 13:20:04 -0500 Subject: [PATCH] refac(cross): #1378 deprecate internal builtins - Deprecate calculateCvss3 - Deprecate chunks - Deprecate escapeShellArg - Deprecate escapeShellArgs - Deprecate filterAttrs - Deprecate flatten - Deprecate fromToml - Deprecate makeDerivationParallel - Deprecate makeNomadEnvironment - Deprecate sortAscii - Deprecate sortAsciiCaseless - Adapt source code - Remove documentation - Remove CI jobs Signed-off-by: Daniel Salazar --- .github/workflows/dev.yml | 16 ---- .github/workflows/prod.yml | 20 ----- docs/src/api/extensions/format-conversion.md | 55 ------------ docs/src/api/extensions/others.md | 87 ------------------- makes/tests/calculateCvss3/main.nix | 5 -- makes/utils/makePythonLock/update.sh | 32 ------- src/args/agnostic.nix | 13 --- src/args/calculate-cvss-3/builder.sh | 7 -- src/args/calculate-cvss-3/cvss.py | 39 --------- src/args/calculate-cvss-3/default.nix | 18 ---- src/args/calculate-cvss-3/poetry.lock | 17 ---- src/args/calculate-cvss-3/pyproject.toml | 15 ---- src/args/chunks/default.nix | 13 --- src/args/make-derivation-parallel/default.nix | 3 - src/args/make-nomad-environment/default.nix | 8 -- src/args/project-path-ls-dirs/default.nix | 4 +- src/args/to-bash-array/default.nix | 4 +- src/args/to-bash-map/default.nix | 5 +- src/args/to-module-outputs/default.nix | 4 +- src/evaluator/modules/pipelines/default.nix | 6 +- 20 files changed, 12 insertions(+), 359 deletions(-) delete mode 100644 makes/tests/calculateCvss3/main.nix delete mode 100644 makes/utils/makePythonLock/update.sh delete mode 100644 src/args/calculate-cvss-3/builder.sh delete mode 100644 src/args/calculate-cvss-3/cvss.py delete mode 100644 src/args/calculate-cvss-3/default.nix delete mode 100644 src/args/calculate-cvss-3/poetry.lock delete mode 100644 src/args/calculate-cvss-3/pyproject.toml delete mode 100644 src/args/chunks/default.nix delete mode 100644 src/args/make-derivation-parallel/default.nix delete mode 100644 src/args/make-nomad-environment/default.nix diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c3d09fea..7e9edb3a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -308,22 +308,6 @@ jobs: - name: /testLicense run: nix-env -if . && m . /testLicense - linux_tests_calculateCvss3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 - - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845 - name: /tests/calculateCvss3 - with: - args: sh -c "chown -R root:root /github/workspace && nix-env -if . && m . /tests/calculateCvss3" - macos_tests_calculateCvss3: - runs-on: macos-latest - steps: - - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 - - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac - - name: /tests/calculateCvss3 - run: nix-env -if . && m . /tests/calculateCvss3 - linux_tests_makeSearchPaths: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 00ad6ada..3d809f8b 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -456,26 +456,6 @@ jobs: - name: /testLicense run: nix-env -if . && m . /testLicense - linux_tests_calculateCvss3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 - - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845 - name: /tests/calculateCvss3 - with: - args: sh -c "chown -R root:root /github/workspace && nix-env -if . && m . /tests/calculateCvss3" - env: - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - macos_tests_calculateCvss3: - runs-on: macos-latest - steps: - - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 - - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac - - name: /tests/calculateCvss3 - run: nix-env -if . && m . /tests/calculateCvss3 - env: - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - linux_tests_makeSearchPaths: runs-on: ubuntu-latest steps: diff --git a/docs/src/api/extensions/format-conversion.md b/docs/src/api/extensions/format-conversion.md index 4b25bf41..dc369383 100644 --- a/docs/src/api/extensions/format-conversion.md +++ b/docs/src/api/extensions/format-conversion.md @@ -54,61 +54,6 @@ Example: [INFO] Tickets is: 3 ``` -## fromToml - -Convert a TOML formatted string -to a Nix expression. - -Types: - -- fromToml (`function str -> anything`): - - (`str`): - TOML formatted string to convert. - -Example: - -=== "main.nix" - - ```nix - # /path/to/my/project/makes/example/main.nix - { - fromToml, - makeDerivation, - ... - }: - let - data = fromToml '' - [example] - name = "John" - lastName = "Doe" - tickets = 3 - ''; - in - makeDerivation { - env = { - envName = data.example.name; - envLastName = data.example.lastName; - envTickets = data.example.tickets; - }; - builder = '' - info "Name is: $envName" - info "Last name is: $envLastName" - info "Tickets is: $envTickets" - ''; - name = "example"; - } - ``` - -=== "Invocation" - - ```bash - $ m . /example - - [INFO] Name is: John - [INFO] Last name is: Doe - [INFO] Tickets is: 3 - ``` - ## fromYaml Convert a YAML formatted string diff --git a/docs/src/api/extensions/others.md b/docs/src/api/extensions/others.md index 2e249535..9c4aeca9 100644 --- a/docs/src/api/extensions/others.md +++ b/docs/src/api/extensions/others.md @@ -1,90 +1,3 @@ -## calculateCvss3 - -Calculate [CVSS3](https://www.first.org/cvss/v3.0/specification-document) -score and severity for a -[CVSS3 Vector String](https://www.first.org/cvss/v3.0/specification-document#Vector-String). - -Types: - -- calculateCvss3 (`function str -> package`): - - (`str`): - CVSS3 Vector String - to calculate. - -Example: - -=== "main.nix" - - ```nix - # /path/to/my/project/makes/example/main.nix - { - makeScript, - calculateCvss3, - ... - }: - makeScript { - replace = { - __argCalculate__ = calculateCvss3 - "CVSS:3.0/S:C/C:H/I:H/A:N/AV:P/AC:H/PR:H/UI:R/E:H/RL:O/RC:R/CR:H/IR:X/AR:X/MAC:H/MPR:X/MUI:X/MC:L/MA:X"; - }; - entrypoint = '' - cat "__argCalculate__" - ''; - name = "example"; - } - ``` - -=== "Invocation" - - ```bash - $ m . /example - - {"score": {"base": 6.5, "temporal": 6.0, "environmental": 5.3}, "severity": {"base": "Medium", "temporal": "Medium", "environmental": "Medium"}} - ``` - -## chunks - -Split a given list into N chunks -for workload distributed parallelization. - -Types: - -- chunks (`function list, ints.positive -> listOf (listOf Any)`): - - (`list`): - List to split into chunks. - - (`ints.positive`): - Number of chunks to create from list. - -Example: - -=== "main.nix" - - ```nix - { - chunks, - inputs, - makeDerivation, - makeDerivationParallel, - ... - }: let - numbers = [0 1 2 3 4 5 6 7 8 9]; - myChunks = chunks numbers 3; # [[0 1 2 3] [4 5 6] [7 8 9]] - - buildNumber = n: makeDerivation { - name = "build-number-${n}"; - env.envNumber = n; - builder = '' - echo "$envNumber" - touch "$out" - ''; - }; - in - makeDerivationParallel { - dependencies = builtins.map buildNumber (inputs.nixpkgs.lib.lists.elemAt myChunks 0); - name = "build-numbers-0"; - } - ``` - ## libGit A small template for doing git kung-fu. diff --git a/makes/tests/calculateCvss3/main.nix b/makes/tests/calculateCvss3/main.nix deleted file mode 100644 index c78b7aeb..00000000 --- a/makes/tests/calculateCvss3/main.nix +++ /dev/null @@ -1,5 +0,0 @@ -{ calculateCvss3, ... }: -calculateCvss3 (builtins.concatStringsSep "" [ - "CVSS:3.0/S:C/C:H/I:H/A:N/AV:P/AC:H/PR:H/UI:R/E:H/RL:O/RC:R/CR:H" - "/IR:X/AR:X/MAC:H/MPR:X/MUI:X/MC:L/MA:X" -]) diff --git a/makes/utils/makePythonLock/update.sh b/makes/utils/makePythonLock/update.sh deleted file mode 100644 index 7be80452..00000000 --- a/makes/utils/makePythonLock/update.sh +++ /dev/null @@ -1,32 +0,0 @@ -# shellcheck shell=bash - -function main { - local cmd=(just m . /utils/makePythonLock) - - "${cmd[@]}" 3.10 \ - "${PWD}/makes/cli/env/runtime/pypi/pypi-deps.yaml" \ - "${PWD}/makes/cli/env/runtime/pypi/pypi-sources.yaml" - - "${cmd[@]}" 3.10 \ - "${PWD}/makes/cli/env/test/pypi-deps.yaml" \ - "${PWD}/makes/cli/env/test/pypi-sources.yaml" - - "${cmd[@]}" 3.12 \ - "${PWD}/src/args/calculate-cvss-3/dependencies.yaml" \ - "${PWD}/src/args/calculate-cvss-3/sources.yaml" - - : && "${cmd[@]}" 3.9 \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/dependencies.yaml" \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/sources-39.yaml" \ - && "${cmd[@]}" 3.10 \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/dependencies.yaml" \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/sources-310.yaml" \ - && "${cmd[@]}" 3.11 \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/dependencies.yaml" \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/sources-311.yaml" \ - && "${cmd[@]}" 3.12 \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/dependencies.yaml" \ - "${PWD}/src/args/make-python-pypi-environment/sources/numpy-1.24.0/sources-312.yaml" -} - -main "${@}" diff --git a/src/args/agnostic.nix b/src/args/agnostic.nix index a78d8fd8..81e8cd6f 100644 --- a/src/args/agnostic.nix +++ b/src/args/agnostic.nix @@ -26,29 +26,22 @@ let attrsMapToList = __nixpkgs__.lib.mapAttrsToList; attrsMerge = builtins.foldl' __nixpkgs__.lib.recursiveUpdate { }; attrsOptional = __nixpkgs__.lib.optionalAttrs; - calculateCvss3 = import ./calculate-cvss-3/default.nix self; - chunks = import ./chunks/default.nix self; computeOnAwsBatch = import ./compute-on-aws-batch/default.nix self; deployContainer = import ./deploy-container/default.nix self; deployContainerManifest = import ./deploy-container-manifest/default.nix self; deployTerraform = import ./deploy-terraform/default.nix self; - inherit (__nixpkgs__.lib.strings) escapeShellArg; - inherit (__nixpkgs__.lib.strings) escapeShellArgs; inherit (__nixpkgs__.lib) fakeSha256; fetchArchive = import ./fetch-archive/default.nix self; fetchNixpkgs = import ./fetch-nixpkgs/default.nix self; fetchRubyGem = import ./fetch-rubygem/default.nix self; fetchUrl = import ./fetch-url/default.nix self; - inherit (__nixpkgs__.lib) filterAttrs; - inherit (__nixpkgs__.lib.lists) flatten; formatBash = import ./format-bash/default.nix self; formatNix = import ./format-nix/default.nix self; formatTerraform = import ./format-terraform/default.nix self; formatYaml = import ./format-yaml/default.nix self; fromJson = builtins.fromJSON; fromJsonFile = path: builtins.fromJSON (builtins.readFile path); - fromToml = builtins.fromTOML; fromYaml = import ./from-yaml/default.nix self; fromYamlFile = path: self.fromYaml (builtins.readFile path); gitlabCi = import ./gitlab-ci/default.nix; @@ -64,13 +57,10 @@ let inherit (__nixpkgs__.lib.filesystem) listFilesRecursive; makeContainerImage = import ./make-container-image/default.nix self; makeDerivation = import ./make-derivation/default.nix self; - makeDerivationParallel = - import ./make-derivation-parallel/default.nix self; makeDynamoDb = import ./make-dynamodb/default.nix self; makeEnvVars = import ./make-env-vars/default.nix self; makeEnvVarsForTerraform = import ./make-env-vars-for-terraform/default.nix self; - makeNomadEnvironment = import ./make-nomad-environment/default.nix self; makePythonEnvironment = import ./make-python-environment/default.nix self; makePythonPoetryEnvironment = import ./make-python-poetry-environment/default.nix self; @@ -106,9 +96,6 @@ let patchShebangs = import ./patch-shebangs/default.nix self; pythonOverrideUtils = import ./python-override-utils/default.nix; inherit (__nixpkgs__.lib) removePrefix; - sortAscii = builtins.sort (a: b: a < b); - sortAsciiCaseless = builtins.sort - (a: b: __nixpkgs__.lib.toLower a < __nixpkgs__.lib.toLower b); stringCapitalize = import ./string-capitalize/default.nix self; sublist = import ./sublist/default.nix self; testLicense = import ./test-license/default.nix self; diff --git a/src/args/calculate-cvss-3/builder.sh b/src/args/calculate-cvss-3/builder.sh deleted file mode 100644 index 94533813..00000000 --- a/src/args/calculate-cvss-3/builder.sh +++ /dev/null @@ -1,7 +0,0 @@ -# shellcheck shell=bash - -function main { - python "${envScriptCvss}" "${envTarget}" > "${out}" -} - -main "${@}" diff --git a/src/args/calculate-cvss-3/cvss.py b/src/args/calculate-cvss-3/cvss.py deleted file mode 100644 index 6c4b31dc..00000000 --- a/src/args/calculate-cvss-3/cvss.py +++ /dev/null @@ -1,39 +0,0 @@ -from cvss import ( - CVSS3, -) -import json -import sys -from typing import ( - Tuple, -) - - -def _str(scores: Tuple[float, float, float]) -> Tuple[str, str, str]: - return tuple(str(score) for score in scores) - - -def main(vector: str) -> None: - result: CVSS3 = CVSS3(vector) - scores: Tuple[str, str, str] = _str(result.scores()) - severities: Tuple[str, str, str] = result.severities() - - print( - json.dumps( - { - "score": { - "base": scores[0], - "temporal": scores[1], - "environmental": scores[2], - }, - "severity": { - "base": severities[0], - "temporal": severities[1], - "environmental": severities[2], - }, - } - ) - ) - - -if __name__ == "__main__": - main(sys.argv[1]) diff --git a/src/args/calculate-cvss-3/default.nix b/src/args/calculate-cvss-3/default.nix deleted file mode 100644 index b8f3f4ce..00000000 --- a/src/args/calculate-cvss-3/default.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ __nixpkgs__, makeDerivation, makePythonEnvironment, ... }: -envTarget: -makeDerivation { - env = { - inherit envTarget; - envScriptCvss = ./cvss.py; - }; - name = "calculate-cvss-3"; - searchPaths = { - source = [ - (makePythonEnvironment { - pythonProjectDir = ./.; - pythonVersion = "3.11"; - }) - ]; - }; - builder = ./builder.sh; -} diff --git a/src/args/calculate-cvss-3/poetry.lock b/src/args/calculate-cvss-3/poetry.lock deleted file mode 100644 index 72cb5fd9..00000000 --- a/src/args/calculate-cvss-3/poetry.lock +++ /dev/null @@ -1,17 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "cvss" -version = "2.3" -description = "CVSS2/3 library with interactive calculator for Python 2 and Python 3" -optional = false -python-versions = "*" -files = [ - {file = "cvss-2.3-py2.py3-none-any.whl", hash = "sha256:0b6b6b78e140f5abbaf75a48d77e970afe6da17763f568a2dca32b23e986a336"}, - {file = "cvss-2.3.tar.gz", hash = "sha256:fc22c9da2e7b93e0476cbea1279c95e7be08a3413e2d946686f0271501a24140"}, -] - -[metadata] -lock-version = "2.0" -python-versions = "^3.11" -content-hash = "2af356ad93cd66ac9fdbf6bf4c7a6a8134e175302a2613fa8dc5acfbb2120d41" diff --git a/src/args/calculate-cvss-3/pyproject.toml b/src/args/calculate-cvss-3/pyproject.toml deleted file mode 100644 index c56a4cbb..00000000 --- a/src/args/calculate-cvss-3/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[tool.poetry] -name = "calculate-cvss-3" -version = "0.1.0" -description = "" -authors = ["Your Name "] -readme = "README.md" - -[tool.poetry.dependencies] -python = "^3.11" -cvss = "2.3" - - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/src/args/chunks/default.nix b/src/args/chunks/default.nix deleted file mode 100644 index 42db766e..00000000 --- a/src/args/chunks/default.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ __nixpkgs__, sublist, ... }: -list: n: -let - inherit (__nixpkgs__) lib; - - len = lib.lists.length list; - div = len / n; - mod = lib.trivial.mod len n; - range = lib.lists.range 0 (n - 1); - inherit (lib.trivial) min; - startIndex = i: i * div + (min i mod); - endIndex = i: (i + 1) * div + (min (i + 1) mod); -in builtins.map (i: sublist list (startIndex i) (endIndex i)) range diff --git a/src/args/make-derivation-parallel/default.nix b/src/args/make-derivation-parallel/default.nix deleted file mode 100644 index efae9107..00000000 --- a/src/args/make-derivation-parallel/default.nix +++ /dev/null @@ -1,3 +0,0 @@ -{ toFileJson, ... }: -{ dependencies, name, }: -toFileJson "make-derivation-parallel-for-${name}" dependencies diff --git a/src/args/make-nomad-environment/default.nix b/src/args/make-nomad-environment/default.nix deleted file mode 100644 index fc06021b..00000000 --- a/src/args/make-nomad-environment/default.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ __nixpkgs__, makeSearchPaths, ... }: -{ version }: -let - nomad = { - "1.0" = __nixpkgs__.nomad_1_0; - "1.1" = __nixpkgs__.nomad_1_1; - }.${version}; -in makeSearchPaths { bin = [ nomad ]; } diff --git a/src/args/project-path-ls-dirs/default.nix b/src/args/project-path-ls-dirs/default.nix index 8b2adfce..63c07a05 100644 --- a/src/args/project-path-ls-dirs/default.nix +++ b/src/args/project-path-ls-dirs/default.nix @@ -1,8 +1,8 @@ -{ __nixpkgs__, filterAttrs, projectPath, ... }: +{ __nixpkgs__, projectPath, ... }: rel: let isDir = _: value: value == "directory"; ls = builtins.readDir (projectPath rel); - dirs = filterAttrs isDir ls; + dirs = __nixpkgs__.lib.filterAttrs isDir ls; dirNames = builtins.attrNames dirs; in builtins.map builtins.unsafeDiscardStringContext dirNames diff --git a/src/args/to-bash-array/default.nix b/src/args/to-bash-array/default.nix index 24bac7d2..9891a511 100644 --- a/src/args/to-bash-array/default.nix +++ b/src/args/to-bash-array/default.nix @@ -1,7 +1,7 @@ -{ escapeShellArgs, makeTemplate, ... }: +{ __nixpkgs__, makeTemplate, ... }: list: makeTemplate { - replace = { __argArray__ = escapeShellArgs list; }; + replace = { __argArray__ = __nixpkgs__.lib.strings.escapeShellArgs list; }; name = "to-bash-array"; template = ./template.sh; } diff --git a/src/args/to-bash-map/default.nix b/src/args/to-bash-map/default.nix index 5922a6c6..71e32f6e 100644 --- a/src/args/to-bash-map/default.nix +++ b/src/args/to-bash-map/default.nix @@ -1,6 +1,7 @@ -{ attrsMapToList, makeTemplate, escapeShellArg, ... }: +{ __nixpkgs__, attrsMapToList, makeTemplate, ... }: attrset: -makeTemplate { +let inherit (__nixpkgs__.lib.strings) escapeShellArg; +in makeTemplate { replace = { __argMap__ = builtins.toString (attrsMapToList (name: value: escapeShellArg "[${name}]=${escapeShellArg value}") diff --git a/src/args/to-module-outputs/default.nix b/src/args/to-module-outputs/default.nix index a8b97d15..31669def 100644 --- a/src/args/to-module-outputs/default.nix +++ b/src/args/to-module-outputs/default.nix @@ -1,5 +1,5 @@ -{ __nixpkgs__, flatten, attrsMapToList, ... }: +{ __nixpkgs__, attrsMapToList, ... }: makeOutput: set: (builtins.foldl' (all: one: all // (if one == { } then { } else { "${one.name}" = one.value; })) { } - (flatten (attrsMapToList makeOutput set))) + (__nixpkgs__.lib.lists.flatten (attrsMapToList makeOutput set))) diff --git a/src/evaluator/modules/pipelines/default.nix b/src/evaluator/modules/pipelines/default.nix index 018e5d57..c4307d29 100644 --- a/src/evaluator/modules/pipelines/default.nix +++ b/src/evaluator/modules/pipelines/default.nix @@ -1,5 +1,5 @@ -{ __nixpkgs__, __toModuleOutputs__, attrsMerge, attrsOptional, escapeShellArgs -, toBashArray, toBashMap, toFileYaml, makeScript, outputs, ... }: +{ __nixpkgs__, __toModuleOutputs__, attrsMerge, attrsOptional, toBashArray +, toBashMap, toFileYaml, makeScript, outputs, ... }: { config, lib, ... }: let toJobName = output: args: builtins.concatStringsSep "__" ([ output ] ++ args); @@ -72,7 +72,7 @@ let script = if args == [ ] then [ "m . ${output}" ] else - [ "m . ${output} ${escapeShellArgs args}" ]; + [ "m . ${output} ${__nixpkgs__.lib.strings.escapeShellArgs args}" ]; variables = { GIT_DEPTH = 3; MAKES_GIT_DEPTH = 3;