From d93c6832e06991feef72c0f64d46c12f4da50700 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 13 Oct 2023 14:35:37 -0400 Subject: [PATCH 1/6] feat(actions): adds support for rules in YAML format Removes CSV based rule examples Removes CSV automation for rules Adds action to auto-transform adhoc and when the YAML is edited Adds a concurrency group to make sure branch modifying worflow are not run that the same time Signed-off-by: Jennifer Power --- .github/workflows/autofix-cd.yml | 3 +- .github/workflows/linters.yml | 3 +- .github/workflows/manual-transform.yml | 11 +++++ .github/workflows/transform-on-push.yml | 18 ++++++++ .github/workflows/transform-rules.yml | 42 ++++++++++++++++++ .github/workflows/validate.yml | 12 +++-- .gitignore | 3 ++ Makefile | 13 ------ .../example/component-definition.json | 44 +++++++++++++++---- data/example.config | 7 --- data/example.csv | 3 -- rules/example/Example/rule_1.yml | 17 +++++++ scripts/csv_sanity_check.sh | 27 ------------ scripts/get_config_updates.sh | 22 ---------- 14 files changed, 135 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/manual-transform.yml create mode 100644 .github/workflows/transform-on-push.yml create mode 100644 .github/workflows/transform-rules.yml delete mode 100644 data/example.config delete mode 100644 data/example.csv create mode 100644 rules/example/Example/rule_1.yml delete mode 100644 scripts/csv_sanity_check.sh delete mode 100644 scripts/get_config_updates.sh diff --git a/.github/workflows/autofix-cd.yml b/.github/workflows/autofix-cd.yml index 78b1429..4792447 100644 --- a/.github/workflows/autofix-cd.yml +++ b/.github/workflows/autofix-cd.yml @@ -8,8 +8,9 @@ on: type: string +# Using concurrency to ensure any branch-modifying workflows are not run at the same time. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: sync-${{ inputs.branch }} cancel-in-progress: true jobs: diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index b50f9c1..7944713 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -4,7 +4,8 @@ name: MegaLinter on: push: pull_request: - branches: main + branches: + - main concurrency: group: ${{ github.ref }}-${{ github.workflow }} diff --git a/.github/workflows/manual-transform.yml b/.github/workflows/manual-transform.yml new file mode 100644 index 0000000..f0a21bd --- /dev/null +++ b/.github/workflows/manual-transform.yml @@ -0,0 +1,11 @@ +name: Run transform adhoc + +on: + workflow_dispatch: + +jobs: + call-transform: + uses: ./.github/workflows/transform-rules.yml + with: + branch: ${{ github.ref_name }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/transform-on-push.yml b/.github/workflows/transform-on-push.yml new file mode 100644 index 0000000..6164c89 --- /dev/null +++ b/.github/workflows/transform-on-push.yml @@ -0,0 +1,18 @@ +name: Transform Rules On Push + +# Only run on pushes to branches other than main. Pushes to main should be +# a result of a pull request merge. The transform rules operation is not idempotent +# and the OSCAL JSON will be modified each time it is run. +on: + push: + branches-ignore: + - main + paths: + - 'rules/**' + +jobs: + call-transform: + uses: ./.github/workflows/transform-rules.yml + with: + branch: ${{ github.ref_name }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/transform-rules.yml b/.github/workflows/transform-rules.yml new file mode 100644 index 0000000..419ae7e --- /dev/null +++ b/.github/workflows/transform-rules.yml @@ -0,0 +1,42 @@ +name: Transform Rules + +on: + workflow_call: + inputs: + branch: + required: true + type: string + +# Using concurrency to ensure any branch-modifying workflows are not run at the same time. +concurrency: + group: sync-${{ inputs.branch }} + cancel-in-progress: true + +jobs: + transform-rules: + name: Transform rules content + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Generate app token + uses: tibdex/github-app-token@v1 + id: get_installation_token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + permissions: >- + {"contents": "write"} + - name: Clone + uses: actions/checkout@v3 + with: + token: ${{ steps.get_installation_token.outputs.token }} + - name: Transform rules + id: transform + uses: jpower432/trestle-bot/actions/rules-transform@bug/single-rule-failure + with: + rules_view_path: "rules/" + file_pattern: "*.json,rules/*" + branch: ${{ inputs.branch }} + commit_user_name: "trestle-bot[bot]" + commit_user_email: "136850459+trestle-bot[bot]@users.noreply.github.com" \ No newline at end of file diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b6692b9..83f1b92 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,13 +1,12 @@ name: Evaluate component definitions on: pull_request: - types: [ opened, ready_for_review, reopened, synchronize ] branches: - main paths: - 'component-definitions/**' - 'markdown/components/**' - - 'data/**' + jobs: test: name: Evaluate components @@ -17,8 +16,6 @@ jobs: steps: - name: Clone uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - name: Check components id: check-components uses: RedHatProductSecurity/trestle-bot@v0.1.1 @@ -26,12 +23,13 @@ jobs: markdown_path: "markdown/components" oscal_model: "compdef" check_only: true - - name: CSV sanity check - run: make check-csv + # Only autofix if the test job fails and the PR is from the same repo call-autofix: needs: [test] - if: ${{ always() && contains(needs.*.result, 'failure') }} + if: | + always() && contains(needs.*.result, 'failure') + && github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url uses: ./.github/workflows/autofix-cd.yml with: branch: ${{ github.head_ref }} diff --git a/.gitignore b/.gitignore index 9653e8a..4bb12c2 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ cython_debug/ ## Trestle Specifc .trestle/cache/ + + +.vscode/ diff --git a/Makefile b/Makefile index f7faf0e..00a8f35 100644 --- a/Makefile +++ b/Makefile @@ -18,19 +18,6 @@ update-subtree: ## Component Definition Custom tasks ############################################################################ -# $1 - config path -define update-cd -echo $(1) -trestle task csv-to-oscal-cd -c $(1); -endef - -update-cds: - @source $(SCRIPTS_DIR)/trestle.sh && $(foreach f,$(CONFIGS),$(call update-cd,$(f))) -.PHONY: update-cd - -check-csv: - @bash scripts/csv_sanity_check.sh - trestlebot-install: @python3 -m pip install --upgrade pip setuptools && python3 -m pip install -r requirements.txt .PHONY: trestlebot-install \ No newline at end of file diff --git a/component-definitions/example/component-definition.json b/component-definitions/example/component-definition.json index ba455b6..b589c49 100644 --- a/component-definitions/example/component-definition.json +++ b/component-definitions/example/component-definition.json @@ -1,46 +1,72 @@ { "component-definition": { - "uuid": "3f0a6f01-e123-4c9e-8d17-267a831d4471", + "uuid": "86378b1f-7ba6-4cac-bdd4-3b52b13f5132", "metadata": { - "title": "Component definition for Example", - "last-modified": "2023-05-08T21:56:09.855332+00:00", + "title": "Component definition for example", + "last-modified": "2023-10-13T14:07:08.049847+00:00", "version": "1.0", "oscal-version": "1.0.4" }, "components": [ { - "uuid": "49e7cc5c-395a-4ea9-a8dd-dc678515ae7b", + "uuid": "5752e82b-8d71-4d35-898c-e8aa5c474ed7", "type": "service", "title": "Example", "description": "Example Application", "props": [ { "name": "Rule_Id", - "ns": "http://ibm.github.io/compliance-trestle/schemas/oscal/cd", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", "value": "Test-rule_001", "remarks": "rule_set_0" }, { "name": "Rule_Description", - "ns": "http://ibm.github.io/compliance-trestle/schemas/oscal/cd", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", "value": "Ensure all of the services are running these tests", "remarks": "rule_set_0" + }, + { + "name": "Parameter_Id", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", + "value": "prm_1", + "remarks": "rule_set_0" + }, + { + "name": "Parameter_Description", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", + "value": "prm_1 description", + "remarks": "rule_set_0" + }, + { + "name": "Parameter_Value_Alternatives", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", + "value": "{'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'}", + "remarks": "rule_set_0" } ], "control-implementations": [ { - "uuid": "66666ab1-f02c-4bb2-a64b-0f90f73910a5", + "uuid": "95f1cd02-e507-44b7-9ef7-22c5b4c557ad", "source": "profiles/fedramp_rev5_high/profile.json", "description": "FedRAMP REV5 High Baseline", + "set-parameters": [ + { + "param-id": "prm_1", + "values": [ + "5%" + ] + } + ], "implemented-requirements": [ { - "uuid": "cae26ab2-9172-450d-9525-351195bc5879", + "uuid": "ae9abfdf-ac4a-4dcb-bf53-ca91e3b86971", "control-id": "ac-1", "description": "", "props": [ { "name": "Rule_Id", - "ns": "http://ibm.github.io/compliance-trestle/schemas/oscal/cd", + "ns": "https://ibm.github.io/compliance-trestle/schemas/oscal", "value": "Test-rule_001" }, { diff --git a/data/example.config b/data/example.config deleted file mode 100644 index 6279eca..0000000 --- a/data/example.config +++ /dev/null @@ -1,7 +0,0 @@ -[task.csv-to-oscal-cd] - -title = Component definition for Example -version = 1.0 -csv-file = data/example.csv -output-dir = component-definitions/example -output-overwrite = true \ No newline at end of file diff --git a/data/example.csv b/data/example.csv deleted file mode 100644 index b2b5ac7..0000000 --- a/data/example.csv +++ /dev/null @@ -1,3 +0,0 @@ -$$Component_Title,$$Component_Description,$$Component_Type,$$Rule_Id,$$Rule_Description,$Parameter_Id,$Parameter_Description,$Parameter_Value_Alternatives,$Parameter_Value_Default,$$Profile_Source,$$Profile_Description,$$Control_Id_List,Check_Id,Check_Description,$$Namespace -A human readable name for the component.,A description of the component including information about its function.,A category describing the purpose of the component. ALLOWED VALUES interconnection:software:hardware:service:physical:process-procedure:plan:guidance:standard:validation,A textual label that uniquely identifies a policy (desired state) that can be used to reference it elsewhere in this or other documents.,A description of the policy (desired state) including information about its purpose and scope.,A textual label that uniquely identifies the parameter associated with that policy (desired state) or controls implemented by the policy (desired state).,A description of the parameter including the purpose and use of the parameter.,ONLY for the policy (desired state) parameters: A value or set of values the parameter can take. The catalog parameters values are defined in the catalog. ,"A value recommended by Compliance Team in this profile for the parameter of the control or policy (desired state). If a CIS-benchmark exists, the default default could be the CIS-benchmark recommended value.",A URL reference to the source catalog or profile for which this component is implementing controls for. A profile designates a selection and configuration of controls from one or more catalogs.,A description of the profile.,A list of textual labels that uniquely identify the controls or statements that the component implements.,A textual label that uniquely identifies a check of the policy (desired state) that can be used to reference it elsewhere in this or other documents.,A description of the check of the policy (desired state) including the method (interview or examine or test) and procedure details.,"A namespace qualifying the property name. This allows different organizations to associate distinct semantics with the same name. Used in conjunction with ""class"" as the ontology concept. " -Example,Example Application,service,Test-rule_001,Ensure all of the services are running these tests,,,,,profiles/fedramp_rev5_high/profile.json,FedRAMP REV5 High Baseline,ac-1,,,http://ibm.github.io/compliance-trestle/schemas/oscal/cd diff --git a/rules/example/Example/rule_1.yml b/rules/example/Example/rule_1.yml new file mode 100644 index 0000000..50b6de5 --- /dev/null +++ b/rules/example/Example/rule_1.yml @@ -0,0 +1,17 @@ +x-trestle-rule-info: + name: Test-rule_001 + description: Ensure all of the services are running these tests + parameter: + name: prm_1 + description: prm_1 description + alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default-value: '5%' + profile: + description: FedRAMP REV5 High Baseline + href: profiles/fedramp_rev5_high/profile.json + include-controls: + - id: ac-1 +x-trestle-component-info: + name: Example + description: Example Application + type: service \ No newline at end of file diff --git a/scripts/csv_sanity_check.sh b/scripts/csv_sanity_check.sh deleted file mode 100644 index bd18678..0000000 --- a/scripts/csv_sanity_check.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -CHANGES=$(git diff-tree --no-commit-id --name-only -r HEAD) - -json_changed=false -csv_changed=false - -json1=$"^component-definitions/" -json2=$"\.json$" - -csv1=$"^data/" -csv2=$"\.csv$" - -for val in "${CHANGES[@]}" ; do - if [[ $val =~ $json1 && $val =~ $json2 ]]; then - json_changed=true - fi - - if [[ $val =~ $csv1 && $val =~ $csv2 ]]; then - csv_changed=true - fi -done - -if [[ $csv_changed == true && $json_changed == false ]]; then - echo "CSV file(s) were changed, but no changes to JSON detected. Run make update-cds." - exit 1 -fi \ No newline at end of file diff --git a/scripts/get_config_updates.sh b/scripts/get_config_updates.sh deleted file mode 100644 index 4ca14d3..0000000 --- a/scripts/get_config_updates.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -CHANGES=$(git diff-tree --no-commit-id --name-only -r HEAD) - -csv1=$"^data/" -csv2=$"\.csv$" - -# Define the new file extension -new_extension=".config" - -declare -a changed_csvs - -for val in "${CHANGES[@]}" ; do - if [[ $val =~ $csv1 && $val =~ $csv2 ]]; then - file_name_without_ext="${val%.*}" - new_file_name="$file_name_without_ext$new_extension" - changed_csvs+=("$new_file_name") - fi -done - - -echo "${changed_csvs[@]}" \ No newline at end of file From 340dd80259ceb01bd14aed887de47b44102a9959 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 13 Oct 2023 14:46:24 -0400 Subject: [PATCH 2/6] fix: removes write permissions from transform rules workflow Signed-off-by: Jennifer Power --- .github/workflows/transform-rules.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/transform-rules.yml b/.github/workflows/transform-rules.yml index 419ae7e..a63aab4 100644 --- a/.github/workflows/transform-rules.yml +++ b/.github/workflows/transform-rules.yml @@ -16,8 +16,6 @@ jobs: transform-rules: name: Transform rules content runs-on: ubuntu-latest - permissions: - contents: write steps: - name: Generate app token uses: tibdex/github-app-token@v1 From eb41ea1212cbe357c039ecbe6ff7e2e3960d50d9 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 13 Oct 2023 15:08:13 -0400 Subject: [PATCH 3/6] chore: adds concurrency group for validate.yml Adds concurrency group to ensure checks are not run concurrently for the same branch Signed-off-by: Jennifer Power --- .github/workflows/validate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 83f1b92..d2eb2cc 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -7,6 +7,10 @@ on: - 'component-definitions/**' - 'markdown/components/**' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: test: name: Evaluate components From cbc9ce7b60c9f576b1987c81f357ce86a8b12928 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 13 Oct 2023 15:35:52 -0400 Subject: [PATCH 4/6] chore: adds branch to actions/checkout Signed-off-by: Jennifer Power --- .github/workflows/transform-rules.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/transform-rules.yml b/.github/workflows/transform-rules.yml index a63aab4..865a7c2 100644 --- a/.github/workflows/transform-rules.yml +++ b/.github/workflows/transform-rules.yml @@ -28,6 +28,7 @@ jobs: - name: Clone uses: actions/checkout@v3 with: + ref: ${{ inputs.branch }} token: ${{ steps.get_installation_token.outputs.token }} - name: Transform rules id: transform From 371172bf0a0d808bd36520a7847efe1855170619 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 13 Oct 2023 16:12:08 -0400 Subject: [PATCH 5/6] docs: adds design and purpose information on GitHub Actions workflows Signed-off-by: Jennifer Power --- docs/design.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/design.md diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..5aa40eb --- /dev/null +++ b/docs/design.md @@ -0,0 +1,50 @@ +# Design + +In the repository, GitHub Actions is used to manage the trestle workspace. This document describes the purpose and design of each workflow. + +## Workflows + +#### Create + +Associated Workflows +- [create-new.yml](../.github/workflows/create-new.yml) + +The `create-new` workflow is triggered manually by going to the Action tab. This creates a new OSCAL component definitions in the trestle workspace. A new branch is created and a pull request is opened. + +#### AutoFix + +Associated Workflows +- [autofix-cd.yml](../.github/workflows/autofix-cd.yml) +- [manual-autofix.yml](../.github/workflows/manual-autofix.yml) +- [validate.yml](../.github/workflows/validate.yml) + +The `validate` workflow is triggered when a pull request is created or updated with updates to component definitions. It validates the trestle workspace and automatically sync any difference between the OSCAL JSON files and trestle managed Markdown files. The same workflow can be triggered through the Action tab using the `Run autofix adhoc` workflow. The `autofix-cd.yml` has all common logic for both workflows. The `validate.yml` and `manual-autofix.yml` workflows adds customer triggers and logic. + +The `validate` workflow will run checks with read-only permissions when a pull request is opened from a from a fork, but the autofix workflow will not run. + +#### Transform + +Associated Workflows +- [transform-rules.yml](../.github/workflows/transform-rules.yml) +- [manual-transform.yml](../.github/workflows/manual-transform.yml) +- [transform-on-push.yml](../.github/workflows/transform-on-push.yml) + +The `transform-on-push` workflow when files under the `rules` directory are made. This will transform the rules managed in YAML into OSCAL JSON files. The `transform-rules.yml` workflow has all common logic for both workflows. The `manual-transform.yml` and `transform-on-push.yml` workflows adds customer triggers and logic. + +Concurrency groups are in place to ensure this does not run at the same time as any branch altering workflows to avoid overwrite to the branches. This can occur if a branch containing rules updates is modified while a pull request is open for the branch. + +#### Regenerate + +Associated Workflows +- [regenerate-cd.yml](../.github/workflows/regenerate-cd.yml) + +The regenerate workflow is triggered when changes to catalogs, profile, or component definitions in OSCAL JSON format are merged to main. This will regenerate the markdown and open a pull request if any changes are found. + +#### Update Profile + +Associated Workflows +- [update-profile.yml](../.github/workflows/update-profile.yml) + +The `update-profile` workflow is triggered manually to pull in updated profile information from upstream sources. This will update the profile information in the trestle workspace and open a pull request. + + From 34f69b781e346b95cf777cd2286767d66738bb7a Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 17 Oct 2023 18:48:00 -0400 Subject: [PATCH 6/6] chore(deps): bump trestlebot to v0.3.0 Signed-off-by: Jennifer Power --- .github/workflows/autofix-cd.yml | 2 +- .github/workflows/create-new.yml | 2 +- .github/workflows/regenerate-cd.yml | 2 +- .github/workflows/transform-rules.yml | 2 +- .github/workflows/validate.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/autofix-cd.yml b/.github/workflows/autofix-cd.yml index 4792447..3b03fc8 100644 --- a/.github/workflows/autofix-cd.yml +++ b/.github/workflows/autofix-cd.yml @@ -33,7 +33,7 @@ jobs: token: ${{ steps.get_installation_token.outputs.token }} - name: Autofix components id: autofix-component - uses: RedHatProductSecurity/trestle-bot@v0.1.1 + uses: RedHatProductSecurity/trestle-bot/actions/autosync@v0.3.0 with: markdown_path: "markdown/components" oscal_model: "compdef" diff --git a/.github/workflows/create-new.yml b/.github/workflows/create-new.yml index 35a964c..21f7801 100644 --- a/.github/workflows/create-new.yml +++ b/.github/workflows/create-new.yml @@ -51,7 +51,7 @@ jobs: COMP_DESCRIPTION: ${{ github.event.inputs.component_description }} - name: Generate and PR new component definition id: generate-cd - uses: RedHatProductSecurity/trestle-bot@v0.1.1 + uses: RedHatProductSecurity/trestle-bot/actions/autosync@v0.3.0 with: markdown_path: "markdown/components" oscal_model: "compdef" diff --git a/.github/workflows/regenerate-cd.yml b/.github/workflows/regenerate-cd.yml index fe5a3e4..671b966 100644 --- a/.github/workflows/regenerate-cd.yml +++ b/.github/workflows/regenerate-cd.yml @@ -30,7 +30,7 @@ jobs: token: ${{ steps.get_installation_token.outputs.token }} - name: Regenerate component definitions id: regenerate - uses: RedHatProductSecurity/trestle-bot@v0.1.1 + uses: RedHatProductSecurity/trestle-bot/actions/autosync@v0.3.0 with: markdown_path: "markdown/components" oscal_model: "compdef" diff --git a/.github/workflows/transform-rules.yml b/.github/workflows/transform-rules.yml index 865a7c2..8b48781 100644 --- a/.github/workflows/transform-rules.yml +++ b/.github/workflows/transform-rules.yml @@ -32,7 +32,7 @@ jobs: token: ${{ steps.get_installation_token.outputs.token }} - name: Transform rules id: transform - uses: jpower432/trestle-bot/actions/rules-transform@bug/single-rule-failure + uses: RedHatProductSecurity/trestle-bot/actions/rules-transform@v0.3.0 with: rules_view_path: "rules/" file_pattern: "*.json,rules/*" diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d2eb2cc..9516c71 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v3 - name: Check components id: check-components - uses: RedHatProductSecurity/trestle-bot@v0.1.1 + uses: RedHatProductSecurity/trestle-bot/actions/autosync@v0.3.0 with: markdown_path: "markdown/components" oscal_model: "compdef"