From 9e9d317074f2778cc0e706f8dc5f37b36c107d70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:14:08 -0700 Subject: [PATCH 1/6] Bump golang from `adee809` to `a7f2fc9` (#488) Bumps golang from `adee809` to `a7f2fc9`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4aa6ca38..0d8c862e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.23.2@sha256:adee809c2d0009a4199a11a1b2618990b244c6515149fe609e2788ddf164bd10 as builder +FROM golang:1.23.2@sha256:a7f2fc9834049c1f5df787690026a53738e55fc097cd8a4a93faa3e06c67ee32 as builder ENV APP_ROOT=/opt/app-root ENV GOPATH=$APP_ROOT @@ -29,7 +29,7 @@ RUN go build ./cmd/verifier RUN CGO_ENABLED=0 go build -gcflags "all=-N -l" -o verifier_debug ./cmd/verifier # Multi-Stage build -FROM golang:1.23.2@sha256:adee809c2d0009a4199a11a1b2618990b244c6515149fe609e2788ddf164bd10 as deploy +FROM golang:1.23.2@sha256:a7f2fc9834049c1f5df787690026a53738e55fc097cd8a4a93faa3e06c67ee32 as deploy # Retrieve the binary from the previous stage COPY --from=builder /opt/app-root/src/verifier /usr/local/bin/verifier From 042205fe4d94bd1b854eed46878bdb067625ef82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:14:53 -0700 Subject: [PATCH 2/6] Bump github/codeql-action from 3.26.11 to 3.26.12 (#487) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.11 to 3.26.12. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea...c36620d31ac7c881962c3d9dd939c40ec9434f2b) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql_analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index 2c6bd4d5..85f04a2b 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -42,12 +42,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 + uses: github/codeql-action/init@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 + uses: github/codeql-action/autobuild@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 + uses: github/codeql-action/analyze@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 From fef1a5ab90307d7d22e0392596cf33677fa7aa18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:15:06 -0700 Subject: [PATCH 3/6] Bump actions/upload-artifact from 4.4.0 to 4.4.3 (#486) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.4.0 to 4.4.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/50769540e7f4bd5e21e526ee35c689e35e0d6874...b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/reusable_monitoring.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_monitoring.yml b/.github/workflows/reusable_monitoring.yml index 20ad3cfd..359700ea 100644 --- a/.github/workflows/reusable_monitoring.yml +++ b/.github/workflows/reusable_monitoring.yml @@ -78,7 +78,7 @@ jobs: continue-on-error: true - run: go run ./cmd/verifier --file ${{ env.LOG_FILE }} --once --monitored-values "${{ inputs.identities }}" --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}" - name: Upload checkpoint - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: ${{ env.UPLOADED_LOG_NAME }} path: ${{ env.LOG_FILE }} From c7c2c948a00ab4d556bd762f3741ec6c60708a07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:15:17 -0700 Subject: [PATCH 4/6] Bump actions/checkout from 4.2.0 to 4.2.1 (#485) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/d632683dd7b4114ad314bca15554477dd762a938...eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql_analysis.yml | 2 +- .github/workflows/dependency_review.yml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/reusable_monitoring.yml | 6 +++--- .github/workflows/verify.yml | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index 85f04a2b..7906904d 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -38,7 +38,7 @@ jobs: language: [ 'go' ] steps: - name: Checkout repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dependency_review.yml b/.github/workflows/dependency_review.yml index 4cef79f5..dfcbae2e 100644 --- a/.github/workflows/dependency_review.yml +++ b/.github/workflows/dependency_review.yml @@ -23,6 +23,6 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: 'Dependency Review' uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 053f35fb..dd34d677 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: OS: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds - uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 with: diff --git a/.github/workflows/reusable_monitoring.yml b/.github/workflows/reusable_monitoring.yml index 359700ea..d4dbcf3d 100644 --- a/.github/workflows/reusable_monitoring.yml +++ b/.github/workflows/reusable_monitoring.yml @@ -59,7 +59,7 @@ jobs: runs-on: ubuntu-latest needs: [detect-workflow] steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: repository: ${{ needs.detect-workflow.outputs.repository }} ref: "${{ needs.detect-workflow.outputs.ref }}" @@ -96,7 +96,7 @@ jobs: ISSUE_REPOSITORY: ${{ github.repository }} if: ${{ needs.monitor.result == 'success' && inputs.file_issue }} steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: repository: ${{ needs.detect-workflow.outputs.repository }} ref: "${{ needs.detect-workflow.outputs.ref }}" @@ -114,7 +114,7 @@ jobs: ISSUE_REPOSITORY: ${{ github.repository }} if: ${{ always() && needs.monitor.result == 'failure' && inputs.file_issue }} steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: repository: ${{ needs.detect-workflow.outputs.repository }} ref: "${{ needs.detect-workflow.outputs.ref }}" diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 13887da7..6c655b6a 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -25,7 +25,7 @@ jobs: name: license boilerplate check runs-on: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: './go.mod' @@ -41,7 +41,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: './go.mod' From 9a536d54192fbbffb360065d0d0bcae94b5f6ec1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:15:32 -0700 Subject: [PATCH 5/6] Bump actions/cache from 4.1.0 to 4.1.1 (#484) Bumps [actions/cache](https://github.com/actions/cache) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2...3624ceb22c1c5a301c8db4169662070a689d9ea8) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd34d677..d10a3cb3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds - - uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 with: # In order: # * Module download cache From 0f9811aa0a16cf3b5ec2bd8098f4d6411283f837 Mon Sep 17 00:00:00 2001 From: Linus Sun <73363182+linus-sun@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:05:42 -0700 Subject: [PATCH 6/6] Refactor reusable workflow consistency verification + add e2e testing into PR CI (#482) * refactor helper verification methods out, move consistency check into verification pkg Signed-off-by: linus-sun * add e2e test to ci Signed-off-by: linus-sun * add TODO to string/flag consistency check Signed-off-by: linus-sun --------- Signed-off-by: linus-sun --- .github/workflows/main.yml | 28 +++++ cmd/verifier/main.go | 106 +---------------- pkg/rekor/verifier.go | 126 +++++++++++++++++++++ {cmd/verifier => pkg/test/e2e}/e2e_test.go | 10 +- {cmd/verifier => pkg/test/e2e}/e2e_test.sh | 2 +- 5 files changed, 162 insertions(+), 110 deletions(-) rename {cmd/verifier => pkg/test/e2e}/e2e_test.go (95%) rename {cmd/verifier => pkg/test/e2e}/e2e_test.sh (97%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d10a3cb3..204fa324 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,3 +63,31 @@ jobs: - name: Run Go tests w/ `-race` if: ${{ runner.os == 'Linux' }} run: go test -race $(go list ./... | grep -v third_party/) + + e2e-tests: + name: Run end-to-end tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + - uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: './go.mod' + check-latest: true + - name: run e2e test + run: ./pkg/test/e2e/e2e_test.sh + diff --git a/cmd/verifier/main.go b/cmd/verifier/main.go index 6c128d3d..f9ddcaf2 100644 --- a/cmd/verifier/main.go +++ b/cmd/verifier/main.go @@ -17,23 +17,16 @@ package main import ( "context" - "encoding/hex" "flag" "fmt" "log" - "os" "runtime" "strings" "time" "github.com/sigstore/rekor-monitor/pkg/identity" "github.com/sigstore/rekor-monitor/pkg/rekor" - "github.com/sigstore/rekor-monitor/pkg/util/file" "github.com/sigstore/rekor/pkg/client" - gclient "github.com/sigstore/rekor/pkg/generated/client" - "github.com/sigstore/rekor/pkg/util" - "github.com/sigstore/rekor/pkg/verify" - "github.com/sigstore/sigstore/pkg/signature" "gopkg.in/yaml.v3" "sigs.k8s.io/release-utils/version" @@ -46,103 +39,6 @@ const ( outputIdentitiesFileName = "identities.txt" ) -// runConsistencyCheck periodically verifies the root hash consistency of a Rekor log. -func RunConsistencyCheck(interval *time.Duration, rekorClient *gclient.Rekor, verifier signature.Verifier, logInfoFile *string, mvs identity.MonitoredValues, outputIdentitiesFile *string, once *bool) error { - ticker := time.NewTicker(*interval) - defer ticker.Stop() - - // Loop will: - // 1. Fetch latest checkpoint and verify - // 2. If old checkpoint is present, verify consistency proof - // 3. Write latest checkpoint to file - - // To get an immediate first tick - for ; ; <-ticker.C { - logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient) - if err != nil { - return fmt.Errorf("getting log info: %v", err) - } - checkpoint := &util.SignedCheckpoint{} - if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil { - return fmt.Errorf("unmarshalling logInfo.SignedTreeHead to Checkpoint: %v", err) - } - if !checkpoint.Verify(verifier) { - return fmt.Errorf("verifying checkpoint (size %d, hash %s) failed", checkpoint.Size, hex.EncodeToString(checkpoint.Hash)) - } - - fi, err := os.Stat(*logInfoFile) - var prevCheckpoint *util.SignedCheckpoint - if err == nil && fi.Size() != 0 { - // File containing previous checkpoints exists - prevCheckpoint, err = file.ReadLatestCheckpoint(*logInfoFile) - if err != nil { - return fmt.Errorf("reading checkpoint log: %v", err) - } - if !prevCheckpoint.Verify(verifier) { - return fmt.Errorf("verifying checkpoint (size %d, hash %s) failed", checkpoint.Size, hex.EncodeToString(checkpoint.Hash)) - } - } - if prevCheckpoint != nil { - if err := verify.ProveConsistency(context.Background(), rekorClient, prevCheckpoint, checkpoint, *logInfo.TreeID); err != nil { - return fmt.Errorf("failed to verify log consistency: %v", err) - } - fmt.Fprintf(os.Stderr, "Root hash consistency verified - Current Size: %d Root Hash: %s - Previous Size: %d Root Hash %s\n", - checkpoint.Size, hex.EncodeToString(checkpoint.Hash), prevCheckpoint.Size, hex.EncodeToString(prevCheckpoint.Hash)) - } - - // Write if there was no stored checkpoint or the sizes differ - if prevCheckpoint == nil || prevCheckpoint.Size != checkpoint.Size { - if err := file.WriteCheckpoint(checkpoint, *logInfoFile); err != nil { - return fmt.Errorf("failed to write checkpoint: %v", err) - } - } - - // TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need - // to persist the last checkpoint. - // Delete old checkpoints to avoid the log growing indefinitely - if err := file.DeleteOldCheckpoints(*logInfoFile); err != nil { - return fmt.Errorf("failed to delete old checkpoints: %v", err) - } - - // Look for identities if there was a previous, different checkpoint - if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size { - // Get log size of inactive shards - totalSize := 0 - for _, s := range logInfo.InactiveShards { - totalSize += int(*s.TreeSize) - } - startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow - endIndex := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 - - // Search for identities in the log range - if len(mvs.CertificateIdentities) > 0 || len(mvs.Fingerprints) > 0 || len(mvs.Subjects) > 0 { - entries, err := rekor.GetEntriesByIndexRange(context.Background(), rekorClient, startIndex, endIndex) - if err != nil { - return fmt.Errorf("error getting entries by index range: %v", err) - } - idEntries, err := rekor.MatchedIndices(entries, mvs) - if err != nil { - return fmt.Errorf("error finding log indices: %v", err) - } - - if len(idEntries) > 0 { - for _, idEntry := range idEntries { - fmt.Fprintf(os.Stderr, "Found %s\n", idEntry.String()) - - if err := file.WriteIdentity(*outputIdentitiesFile, idEntry); err != nil { - return fmt.Errorf("failed to write entry: %v", err) - } - } - } - } - } - - if *once { - return nil - } - } -} - // This main function performs a periodic root hash consistency check. // Upon starting, any existing latest snapshot data is loaded and the function runs // indefinitely to perform consistency check for every time interval that was specified. @@ -187,7 +83,7 @@ func main() { log.Fatal(err) } - err = RunConsistencyCheck(interval, rekorClient, verifier, logInfoFile, monitoredVals, outputIdentitiesFile, once) + err = rekor.RunConsistencyCheck(interval, rekorClient, verifier, logInfoFile, monitoredVals, outputIdentitiesFile, once) if err != nil { log.Fatalf("%v", err) } diff --git a/pkg/rekor/verifier.go b/pkg/rekor/verifier.go index 2ab014f3..222a3a47 100644 --- a/pkg/rekor/verifier.go +++ b/pkg/rekor/verifier.go @@ -17,8 +17,17 @@ package rekor import ( "context" "crypto" + "encoding/hex" + "fmt" + "os" + "time" + "github.com/sigstore/rekor-monitor/pkg/identity" + "github.com/sigstore/rekor-monitor/pkg/util/file" "github.com/sigstore/rekor/pkg/generated/client" + "github.com/sigstore/rekor/pkg/generated/models" + "github.com/sigstore/rekor/pkg/util" + "github.com/sigstore/rekor/pkg/verify" "github.com/sigstore/sigstore/pkg/cryptoutils" "github.com/sigstore/sigstore/pkg/signature" ) @@ -40,3 +49,120 @@ func GetLogVerifier(ctx context.Context, rekorClient *client.Rekor) (signature.V } return verifier, nil } + +// verifyLatestCheckpoint fetches and verifies the signature of the latest checkpoint from log info fetched from Rekor. +// If it successfully verifies the checkpoint's signature, it returns the checkpoint; otherwise, it returns an error. +func verifyLatestCheckpointSignature(logInfo *models.LogInfo, verifier signature.Verifier) (*util.SignedCheckpoint, error) { + checkpoint := &util.SignedCheckpoint{} + if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil { + return nil, fmt.Errorf("unmarshalling logInfo.SignedTreeHead to Checkpoint: %v", err) + } + if !checkpoint.Verify(verifier) { + return nil, fmt.Errorf("verifying checkpoint (size %d, hash %s) failed", checkpoint.Size, hex.EncodeToString(checkpoint.Hash)) + } + return checkpoint, nil +} + +// verifyCheckpointConsistency reads and verifies the consistency of the previous latest checkpoint from a log info file against the current up-to-date checkpoint. +// If it successfully fetches and verifies the consistency between these two checkpoints, it returns the previous checkpoint; otherwise, it returns an error. +func verifyCheckpointConsistency(logInfoFile *string, checkpoint *util.SignedCheckpoint, treeID string, rekorClient *client.Rekor, verifier signature.Verifier) (*util.SignedCheckpoint, error) { + var prevCheckpoint *util.SignedCheckpoint + prevCheckpoint, err := file.ReadLatestCheckpoint(*logInfoFile) + if err != nil { + return nil, fmt.Errorf("reading checkpoint log: %v", err) + } + if !prevCheckpoint.Verify(verifier) { + return nil, fmt.Errorf("verifying checkpoint (size %d, hash %s) failed", checkpoint.Size, hex.EncodeToString(checkpoint.Hash)) + } + if err := verify.ProveConsistency(context.Background(), rekorClient, prevCheckpoint, checkpoint, treeID); err != nil { + return nil, fmt.Errorf("failed to verify log consistency: %v", err) + } + fmt.Fprintf(os.Stderr, "Root hash consistency verified - Current Size: %d Root Hash: %s - Previous Size: %d Root Hash %s\n", + checkpoint.Size, hex.EncodeToString(checkpoint.Hash), prevCheckpoint.Size, hex.EncodeToString(prevCheckpoint.Hash)) + return prevCheckpoint, nil +} + +// RunConsistencyCheck periodically verifies the root hash consistency of a Rekor log. +// TODO: RunConsistencyCheck should take in string/bool flags directly instead of pointers and check that flags are being set correctly. +func RunConsistencyCheck(interval *time.Duration, rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile *string, mvs identity.MonitoredValues, outputIdentitiesFile *string, once *bool) error { + ticker := time.NewTicker(*interval) + defer ticker.Stop() + + // Loop will: + // 1. Fetch latest checkpoint and verify + // 2. If old checkpoint is present, verify consistency proof + // 3. Write latest checkpoint to file + + // To get an immediate first tick + for ; ; <-ticker.C { + logInfo, err := GetLogInfo(context.Background(), rekorClient) + if err != nil { + return fmt.Errorf("failed to get log info: %v", err) + } + checkpoint, err := verifyLatestCheckpointSignature(logInfo, verifier) + if err != nil { + return fmt.Errorf("failed to verify signature of latest checkpoint: %v", err) + } + + fi, err := os.Stat(*logInfoFile) + // File containing previous checkpoints exists + var prevCheckpoint *util.SignedCheckpoint + if err == nil && fi.Size() != 0 { + prevCheckpoint, err = verifyCheckpointConsistency(logInfoFile, checkpoint, *logInfo.TreeID, rekorClient, verifier) + if err != nil { + return fmt.Errorf("failed to verify previous checkpoint: %v", err) + } + + } + + // Write if there was no stored checkpoint or the sizes differ + if prevCheckpoint == nil || prevCheckpoint.Size != checkpoint.Size { + if err := file.WriteCheckpoint(checkpoint, *logInfoFile); err != nil { + return fmt.Errorf("failed to write checkpoint: %v", err) + } + } + + if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size { + // Get log size of inactive shards + totalSize := 0 + for _, s := range logInfo.InactiveShards { + totalSize += int(*s.TreeSize) + } + startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow + endIndex := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 + + // Search for identities in the log range + if len(mvs.CertificateIdentities) > 0 || len(mvs.Fingerprints) > 0 || len(mvs.Subjects) > 0 { + entries, err := GetEntriesByIndexRange(context.Background(), rekorClient, startIndex, endIndex) + if err != nil { + return fmt.Errorf("error getting entries by index range: %v", err) + } + idEntries, err := MatchedIndices(entries, mvs) + if err != nil { + return fmt.Errorf("error finding log indices: %v", err) + } + + if len(idEntries) > 0 { + for _, idEntry := range idEntries { + fmt.Fprintf(os.Stderr, "Found %s\n", idEntry.String()) + + if err := file.WriteIdentity(*outputIdentitiesFile, idEntry); err != nil { + return fmt.Errorf("failed to write entry: %v", err) + } + } + } + } + } + + // TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need + // to persist the last checkpoint. + // Delete old checkpoints to avoid the log growing indefinitely + if err := file.DeleteOldCheckpoints(*logInfoFile); err != nil { + return fmt.Errorf("failed to delete old checkpoints: %v", err) + } + + if *once { + return nil + } + } +} diff --git a/cmd/verifier/e2e_test.go b/pkg/test/e2e/e2e_test.go similarity index 95% rename from cmd/verifier/e2e_test.go rename to pkg/test/e2e/e2e_test.go index dbe327a7..78d0a5c8 100755 --- a/cmd/verifier/e2e_test.go +++ b/pkg/test/e2e/e2e_test.go @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +//go:build e2e +// +build e2e + +package e2e import ( "bytes" @@ -56,7 +59,6 @@ const ( // Check that Rekor-monitor reusable monitoring workflow successfully verifies consistency of the log checkpoint // and is able to find a monitored identity within the checkpoint indices and write it to file. func TestRunConsistencyCheck(t *testing.T) { - t.Skip("skipping test outside of being run from e2e_test.sh") rekorClient, err := client.GetRekorClient(rekorURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) if err != nil { log.Fatalf("getting Rekor client: %v", err) @@ -168,7 +170,7 @@ func TestRunConsistencyCheck(t *testing.T) { } once := true - err = RunConsistencyCheck(&interval, rekorClient, verifier, &tempLogInfoFileName, monitoredVals, &tempOutputIdentitiesFileName, &once) + err = rekor.RunConsistencyCheck(&interval, rekorClient, verifier, &tempLogInfoFileName, monitoredVals, &tempOutputIdentitiesFileName, &once) if err != nil { t.Errorf("first consistency check failed: %v", err) } @@ -208,7 +210,7 @@ func TestRunConsistencyCheck(t *testing.T) { t.Errorf("expected checkpoint size of 2, received size %d", checkpoint.Size) } - err = RunConsistencyCheck(&interval, rekorClient, verifier, &tempLogInfoFileName, monitoredVals, &tempOutputIdentitiesFileName, &once) + err = rekor.RunConsistencyCheck(&interval, rekorClient, verifier, &tempLogInfoFileName, monitoredVals, &tempOutputIdentitiesFileName, &once) if err != nil { t.Errorf("second consistency check failed: %v", err) } diff --git a/cmd/verifier/e2e_test.sh b/pkg/test/e2e/e2e_test.sh similarity index 97% rename from cmd/verifier/e2e_test.sh rename to pkg/test/e2e/e2e_test.sh index d4879d14..46e86b13 100755 --- a/cmd/verifier/e2e_test.sh +++ b/pkg/test/e2e/e2e_test.sh @@ -66,4 +66,4 @@ echo echo "running tests" popd -go test -tags=e2e -v -race ./cmd/verifier/... \ No newline at end of file +go test -tags=e2e -v -race ./pkg/test/e2e/... \ No newline at end of file