From 75bc2ca5788652fd59bad8ee8d76d50a6e370793 Mon Sep 17 00:00:00 2001 From: Kris Coleman Date: Tue, 9 Apr 2024 22:58:32 -0400 Subject: [PATCH] chore(archivista): archivista client struct (#1) * Initial commit * chore: Configure SAST in `.gitlab-ci.yml`, creating this file if it does not already exist * chore: initial commit of judge monorepo core projects * chore: Set up our new home for product and development planning (#7) chore: Set up our new home for product development and onboarding Fixes #1 Fixes #2 removed spire k8s * fix(web): friendly errors fixes #19 * chore: Added issue templates for additional issue types closes #16 a bit further, had some idle time * chore: Added issue templates for epic and feature * chore: update Judge monorepo with latest judge-api change delta * story: As a user I want a simple and intuitive sign-in flow fixes #20 Now when the user registers with their github, they are automatically signed in * docs(web): added todo to move sorting logic that shouldn't be in frontend * fix(web): the dashboard attestation results should belong to the repo they are associated with fixes #18 * chore(web): remove of .old web ui * fix(web): tests all pass fixes #31 * chore: implemented prepush-if-changed so that we can only run tests for things on push if we actually changed them * chore: implemented prepush test githooks for all go subprojectsg * docs: wrote docs for our githooks * chore: sorted our root package.json scripts * feat: add proxies to production services for netlify deployments (#34) Sets up proxies to archivista, judge-api, kratos, and login from netlify deployments to our production instances of these. Eventually we may need to break these out to short lived versions of the back end services. Fixes #33 * fix: adds missing wildcards to netlify redirects Netlify was only doing redirects for exact matches of `/archivista` or `/login`, where we want all paths to be redirected to those services. Fixes #33 * fix: tell kratos to return us to our current judge web instance * chore: implemented conventioanal-commits with commitlint * docs: updated githooks documentation for conventional commits * chore: implemented git-conventional-commits this helps enable generating our versions and changelog from our git tree helps close #9 * fix: githooks should be frictionless subtasks: fix: prepush-if-changed shouldn't run node tests if node files aren't touched chore: update commitlint to allow for bypassing with 'wip' fix: prepushrc for node and go prepush test hook chore: updates to commitlint for longer messages fixes #40 if a ops developer contributed code but had not run `npm run start:web` on the web project to produce the graphql cache, then linting would fail. Rather than ask ops devs to make sure they start the web app every now and then, we have addressed this by making the graphql cache optional. * chore: updated subtree scripts * Squashed 'subtrees/witness/' content from commit cc2478e git-subtree-dir: subtrees/witness git-subtree-split: cc2478e854c6dd91cc6b20ed3baa6ba47dbadb3f * Squashed 'subtrees/archivista/' content from commit 21ab99d git-subtree-dir: subtrees/archivista git-subtree-split: 21ab99d5c42737eb9526ac43e51e460f74436685 * Squashed 'subtrees/go-witness/' content from commit 31e6790 git-subtree-dir: subtrees/go-witness git-subtree-split: 31e67909220f336fd5771b6f5c2916c799da2496 * chore: move subtrees to subtrees/ subfolder closes #27 updated dev environment and githooks to work with new subtrees subfolder updated docs on subtrees * chore: upgrade kratos to v13.0 (#55) chore: upgrade kratos * fix: tests fail if you run npm build instead of npm start before testing fixes #56 * chore(web): implement node workflow to github workflows closes #60 allows us to build and test our web project from the monorepo as on changes to our trunk (`main`) Signed-off-by: Kris Coleman * feat: implemented metadata webhook in kratos for updating tenant metadata closes #28 * feat: upgrade kratos ui * Squashed 'subtrees/witness-run-action/' content from commit bdd8272 git-subtree-dir: subtrees/witness-run-action git-subtree-split: bdd82729b316d071606007cc9eecae326429caaf * chore: updated conventional-commit to support subtree type * subtree(witness-run-action): onboarded the witness-run-action to the monorepo closes #70 * chore: updated download-compress-witness.sh to support macos * chore: updated how we push subtrees to a smarter approach now we don't push directly to main... we should just make a new branch off our trunk before we push using the provided scripts. this will push a clean branch of just the subtree changes to the subtree. * chore(web): update the yo generators to work with monorepo OOTB also improved cli wordage * feat(web): implemented RepoCard RepoCard provides a reusable component for showing a repo with a Card view * feat(web): implemented CommitLink component provides a reusable commit for displaying a Commit * story: as a user, anytime I click on a git sha, I want to copy it the clipboard * ops(go): implement go ci workflow to github workflows closes #62 * chore: dogfood the witness-run-action on the monorepo closes #22 * subtree(witness-run-action): updated docs to help users with generating attestations in monorepos * subtree(witness): updated `run` docs to help users with generating attestations in monorepos * subtree(witness): updated `run` docs to help users with generating attestations with npm * ops: renamed web.yaml to node.yml * chore(judge-api): use logrus directly * chore(archivista): use logrus directly * fix(archivista): update archivista's usage of updated go-witness function * feat(monorepo): add go workspace file to monorepo * chore(judge-api): rename go module * chore(archivista): use logrus directly * fix(monorepo): update witness-run-action version used * fix(archivista): update archivista's usage of updated go-witness function * chore(deps): bump yaml in /subtrees/witness-run-action Bumps [yaml](https://github.com/eemeli/yaml) from 2.2.1 to 2.2.2. - [Release notes](https://github.com/eemeli/yaml/releases) - [Commits](https://github.com/eemeli/yaml/compare/v2.2.1...v2.2.2) --- updated-dependencies: - dependency-name: yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] * chore(judge-api): support multiple database providers * fix(witness): witness should not error on an empty git repo with no commits Fixes #121 * chore: make skaffold config modular * chore: add skaffold README * chore(policy): implemented struct representing a decision on a policy * feat(policy): judge-api has a endpoint for submitting witness verify results and stores them * refactor(witness): use generic registry for attestor options * refactor(go-witness): use generic registry for attestor options * feat(go-witness): add signer provider registry * refactor: add SetOptions helper to registry, fix return values in some signer's option funcs * feat(witness): use signer registry to setup signers for CLI flags * docs(witness): regenerate docs for new cli flags * fix: use go mod download in install scripts for go modules * feat(go-witness): add time.Duration option for registries * feat(go-witness): add vault pki signer provider * refactor(witness): create helper function to add options from registries * docs(witness): regenerate docs for new cli flags * fix(witness): re-enable verify tests As part of the signer registry refactor I commented these tests out to work on some other tests. Turns out I forgot to uncomment and fix these before merging the refactor... Ooops. * chore: update go.work.sum * fix: fix remotes:add:all script * Squashed 'subtrees/go-witness/' changes from 31e6790..6e5c872 6e5c872 fix: update github actions to use new fulcio url 7332f62 fix: use witness-run-action instead of testifysec-run-action git-subtree-dir: subtrees/go-witness git-subtree-split: 6e5c872cb5f282059bc01bda200b0354f88bacd9 * Squashed 'subtrees/witness/' changes from cc2478e..8a53d68 8a53d68 fix: update github actions to use new fulcio url 2dc9401 fix: update goreleaser action to use go 1.20.x 9bac7df chore(deps): bump github.com/cloudflare/circl from 1.3.2 to 1.3.3 57b29fd feat: publish ko built images to ghcr bde414e fix: use witness-run-action instead of testifysec-run-action 58b3f59 fix: update scorecard version to fix invalid key error git-subtree-dir: subtrees/witness git-subtree-split: 8a53d681de06d9a447210841c1e96e03b6c2edfb * Squashed 'subtrees/go-witness/' changes from 6e5c872..cd0c222 cd0c222 feat(signer): add vault pki signer provider 47e1475 feat(registry): add time.Duration option for registries 6e4df6c refactor(registry): add SetOptions helper to registry, fix return values in some signer's option funcs ce479bf feat(signer): add signer provider registry 8f0f4b9 refactor(attestor): use generic registry for attestor options 2357401 fix(witness): witness should not error on an empty git repo with no commits Fixes #121 git-subtree-dir: subtrees/go-witness git-subtree-split: cd0c222058a8830a8e190b840e466098b25a3c41 * Squashed 'subtrees/witness/' changes from 8a53d68..be20100 be20100 fix: dev/Dockerfile.go-builder to reduce vulnerabilities aa35c1f fix: update changed signer flags in tests f7d7e96 fix: use the pflag.FlagSet.Set function to set values from config files 03ab65f fix: re-enable verify tests 5bf31d7 docs: regenerate docs for new cli flags d713711 refactor: create helper function to add options from registries 88a8d93 docs: regenerate docs for new cli flags 4a41144 feat: use signer registry to setup signers for CLI flags 0c7a4e5 refactor: use generic registry for attestor options git-subtree-dir: subtrees/witness git-subtree-split: be20100af602c780deeef50c54f5338662ce917c * Squashed 'subtrees/go-witness/' changes from cd0c222..0b28c0f 0b28c0f Adding support for using timestamp authority and CA certificates for verifying policy (#124) 43a586f Adding support for supplying POM on Maven Attestor (#129) 61576e0 Adding function to add a single attestor (#128) 404b654 chore: bump actions/download-artifact from 4.1.0 to 4.1.1 (#127) 8937af7 chore: bump actions/upload-artifact from 4.0.0 to 4.1.0 (#126) a54b4c0 fix: added oidc redirect url option for fulcio (#76) 0aaf29b chore: bump github/codeql-action from 3.22.12 to 3.23.0 (#122) 4354822 chore: bump actions/dependency-review-action from 3.1.4 to 3.1.5 (#123) 90c26c3 chore: bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 (#121) 9875fcc Update SECURITY-INSIGHTS.yml with additional information (#108) 3088442 chore: bump k8s.io/apimachinery from 0.26.11 to 0.26.12 (#116) 6ab0399 chore: bump actions/download-artifact from 4.0.0 to 4.1.0 (#117) c5246d4 chore: bump github/codeql-action from 3.22.11 to 3.22.12 (#118) a39d484 chore: bump github.com/go-git/go-git/v5 from 5.5.2 to 5.11.0 (#119) c28d93f chore: bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#115) 603cfa9 chore: bump actions/upload-artifact from 3.1.3 to 4.0.0 (#111) 84bdf2a chore: bump actions/download-artifact from 3.0.2 to 4.0.0 (#112) 9465ff4 chore: bump github/codeql-action from 2.22.9 to 3.22.11 (#110) cfee7c9 Create SECURITY.md (#107) 6094e21 Point to v0.2.0 of archivista (#105) 00081b0 Fixing bug introduced in logs - warning and debug logs not printing (#103) 5b5647c WIP: Migrating Go module to in-toto (#101) c555ac6 Adding go test command to Makefile (#96) 737eed8 Updating README (#97) 70efbcf Improving `--signer-fulcio-token` flag to accept both path and raw token string (#82) b11e25f chore: bump github/codeql-action from 2.22.8 to 2.22.9 (#90) 1ec7071 chore: bump actions/dependency-review-action from 2.5.1 to 3.1.4 (#91) 765aa2b chore: bump actions/checkout from 3.6.0 to 4.1.1 (#92) 9243257 chore: bump actions/setup-go from 4.1.0 to 5.0.0 (#93) 19d2725 chore: bump ossf/scorecard-action from 2.0.6 to 2.3.1 (#89) a10252c Don't run FOSSA Scan on PR from fork (#95) bec608e Changes to improve CLOMonitor Score (#88) 3328596 Fix pre-commit violations (#87) eac781c [StepSecurity] Apply security best practices (#86) 1d30fe2 Refactoring error messages to use %w formatting directive and fix logging issue (#85) 7ec4004 Update README.md 2bdd1c6 chore: bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.1 3e86283 chore: bump github.com/open-policy-agent/opa from 0.49.1 to 0.49.2 86c8967 chore: bump github.com/mattn/go-isatty from 0.0.17 to 0.0.20 5f74d75 chore: bump k8s.io/apimachinery from 0.26.10 to 0.26.11 9a2cff0 chore: bump github.com/stretchr/testify from 1.8.2 to 1.8.4 40c7ed5 chore: bump github.com/sigstore/sigstore from 1.5.1 to 1.5.2 4ddd1b4 chore: bump k8s.io/apimachinery from 0.26.1 to 0.26.10 e927252 chore: bump go.step.sm/crypto from 0.25.0 to 0.25.2 4273fcf chore: bump github.com/spiffe/go-spiffe/v2 from 2.1.2 to 2.1.6 c5bac1b chore: bump github.com/aws/aws-sdk-go from 1.44.207 to 1.44.334 044ab95 chore: bump actions/setup-go from 2 to 4 46ff412 chore: bump actions/checkout from 2 to 4 78ca945 Improve DigestSet logic and JSON marshalling c487391 Changed to pointer receiver when both were mixed 08d1c37 Add dependabot config, reusable witness workflow, and update pipeline 5c92286 Add maintainers file (#64) 69cb3ee chore(deps): bump google.golang.org/grpc from 1.53.0 to 1.56.3 (#60) 5e567f0 chore(deps): bump golang.org/x/net from 0.7.0 to 0.17.0 (#54) 03cf3f0 chore(deps): bump github.com/cloudflare/circl from 1.3.2 to 1.3.3 (#44) 4f01b96 Add support for controller-gen deepcopy of policy package structs. This will be required for archivista data provider controllers (#53) git-subtree-dir: subtrees/go-witness git-subtree-split: 0b28c0f52c40cdac5e3a15151ca360d965929086 * Squashed 'subtrees/witness/' changes from be20100..06031da 06031da Checking attestors for duplicates (#361) 1a9b5a2 Initial attempt at PR and Issue templates (#351) 83ca942 chore: bump actions/download-artifact from 4.1.0 to 4.1.1 (#358) 63cc5d8 chore: bump github/codeql-action from 3.22.12 to 3.23.0 (#357) 70e0b09 chore: bump actions/upload-artifact from 4.0.0 to 4.1.0 (#356) d2471e6 chore: bump actions/cache from 3.3.2 to 3.3.3 (#355) f2e2a6f Update cloudflare/circl due to dependabot failure (#352) abce18b Add cosign install 15d9014 Add signing to goreleaser and Best Practices badge to readme. 93768db Pin dependencies and restrict permissions 494d44a Add Security MD files an add FOSSA scan badge b9e38d5 Add FOSSA license scanning 617e15a chore: bump actions/dependency-review-action from 3.1.4 to 3.1.5 (#349) 2c590bb Update go-git to resolve vulnerability (#346) 88881fa chore: bump actions/download-artifact from 4.0.0 to 4.1.0 (#342) ea67d31 chore: bump github/codeql-action from 3.22.11 to 3.22.12 (#343) b8f36d6 chore: bump actions/upload-artifact from 3.1.3 to 4.0.0 (#337) 34563ab chore: bump github/codeql-action from 2.22.9 to 3.22.11 (#336) 46b168d chore: bump actions/download-artifact from 3.0.2 to 4.0.0 (#335) b36c96d Bumping Go version for goreleaser (#333) c06555d Migrating to the use of in-toto/go-witness module (#331) c0f5843 Migrating go module (#328) 937eab8 Adding the contributing.md from archivista (#327) f0c8f43 Adding help to Makefile and updating `make test` target (#325) 71856fd chore: bump actions/dependency-review-action from 2.5.1 to 3.1.4 (#324) 709ad35 chore: bump github/codeql-action from 2.22.8 to 2.22.9 (#323) 684fd6a chore: bump actions/setup-go from 4.1.0 to 5.0.0 (#322) a823f58 chore: bump actions/checkout from 3.6.0 to 4.1.1 (#321) 862d8c4 chore: bump actions/upload-artifact from 3.0.0 to 3.1.3 (#320) b19afc8 Fix initial pre-commit violations (#319) a56715e Refactoring error messages to use `%w` formatting directive and fix logging issue (#314) 0bca967 feat: add algo hash list for digest calc in config (#292) 81bdfce Improve gha (#318) f65b232 [StepSecurity] Apply security best practices (#316) bcf7ecf Update README.md - fixing quickstart url 8dde14c docs: correct sign policy file command in README.md 752b9e0 chore: bump github/codeql-action from 2.22.7 to 2.22.8 15bec9e chore: bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.1 0363ee3 chore: bump actions/setup-go from 2 to 4 a412c18 chore: bump actions/cache from 2 to 3 e7a6f44 chore: bump github/codeql-action from 2.22.6 to 2.22.7 932ff1e chore: bump actions/checkout from 2 to 4 (#301) 5e56558 chore: bump github.com/stretchr/testify from 1.8.1 to 1.8.4 (#305) f49ff8e chore: bump github.com/sirupsen/logrus from 1.9.0 to 1.9.3 (#304) 873f55c chore: bump golangci/golangci-lint-action from 2 to 3 (#303) 1880baa chore: bump ossf/scorecard-action from 2.1.3 to 2.3.1 (#302) 9380cbe chore: bump github/codeql-action from 1.0.26 to 2.22.6 (#300) 21cb944 chore: bump docker/login-action from 2 to 3 (#299) 2219a76 fix: updating urls to `in-toto` from `testifysec` and `-L` to the curl for version (#297) b3d7207 Add dependabot config and add reusable workflow for calling witness (#298) 5beb113 Add maintainers file 602dc48 chore(deps): bump google.golang.org/grpc from 1.53.0 to 1.56.3 edef808 docs: Update key to signer-file-key-path in getting starter .witness.yaml 8e9d798 fix: dev/Dockerfile.go-builder to reduce vulnerabilities 27f68b9 chore(deps): bump golang.org/x/net from 0.7.0 to 0.17.0 git-subtree-dir: subtrees/witness git-subtree-split: 06031da4459ee4aea13ee83c59f9dee8171133ff * chore(archivista): archivista client struct - created `httpclient` package for HTTP client structure with methods attached. - created `httpclient_test` for HTTP client structure. * Squashed 'subtrees/go-witness/' changes from 0b28c0f5..8fbc70b1 8fbc70b1 chore: bump github.com/aws/aws-sdk-go from 1.50.30 to 1.50.38 (#196) 289e9b23 chore: bump k8s.io/apimachinery from 0.29.2 to 0.29.3 (#195) 5429db56 chore: bump cloud.google.com/go/kms from 1.15.7 to 1.15.8 (#194) 7f6ea51e chore: bump github.com/aws/aws-sdk-go-v2/config from 1.27.8 to 1.27.9 (#193) 53610c1b chore: bump actions/dependency-review-action from 4.1.3 to 4.2.4 (#192) 29296244 chore: bump softprops/action-gh-release from 2.0.3 to 2.0.4 (#191) ed8ae371 chore: bump github/codeql-action from 3.24.8 to 3.24.9 (#190) fe836545 chore: bump github.com/aws/aws-sdk-go-v2/config from 1.27.4 to 1.27.8 (#189) 27a5a540 chore: bump actions/checkout from 4.1.1 to 4.1.2 (#188) e12f3e75 chore: bump github/codeql-action from 3.24.6 to 3.24.8 (#187) 981d0fbd chore: bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#186) 629e83ac chore: bump github.com/aws/aws-sdk-go-v2/service/kms from 1.29.1 to 1.29.2 (#183) 22975635 chore: bump google.golang.org/grpc from 1.62.0 to 1.62.1 (#182) c479bdb4 chore: bump softprops/action-gh-release from 1 to 2 (#181) af6cb4b7 chore: bump gopkg.in/go-jose/go-jose.v2 from 2.6.2 to 2.6.3 (#179) f0cd5b11 chore: bump github.com/go-jose/go-jose/v3 from 3.0.2 to 3.0.3 (#180) 31febef5 chore: bump github.com/aws/aws-sdk-go from 1.50.27 to 1.50.30 (#177) 61776d68 chore: bump github.com/sigstore/sigstore from 1.8.1 to 1.8.2 (#178) 2aac8f3f chore: bump actions/download-artifact from 4.1.2 to 4.1.4 (#176) 73f387a9 chore: bump github/codeql-action from 3.24.5 to 3.24.6 (#175) 2604d61e 168 support all fulcio cert extensions (#174) b6f3a568 fix: reset verifier each iteration while loading pub keys from policy (#173) 0d420e02 chore: bump google.golang.org/grpc from 1.61.0 to 1.61.1 (#171) a7362f41 chore: bump actions/dependency-review-action from 4.1.1 to 4.1.3 (#170) f0456d7c chore: bump github/codeql-action from 3.24.3 to 3.24.5 (#169) 98357187 chore: bump github/codeql-action from 3.24.0 to 3.24.3 (#162) ac085845 chore: bump k8s.io/apimachinery from 0.26.13 to 0.26.14 (#161) c2607968 chore: bump github.com/aws/aws-sdk-go-v2/config from 1.18.14 to 1.18.45 (#160) 7dbbdbdb chore: bump cloud.google.com/go/kms from 1.15.2 to 1.15.7 (#158) 8278d008 chore: bump github.com/aws/aws-sdk-go-v2/service/kms from 1.20.4 to 1.20.12 (#157) 856f1a04 chore: bump actions/download-artifact from 4.1.1 to 4.1.2 (#163) 7732ec53 chore: bump fossas/fossa-action from 1.3.1 to 1.3.3 (#164) 4dcce5ec chore: bump actions/dependency-review-action from 4.0.0 to 4.1.1 (#165) 5e54141d chore: bump testifysec/witness-run-action from 0.1.3 to 0.1.5 (#166) 57ca28ab Add Tom as an official maintainer (#156) f7a1037f KMS Support (#120) c5816df0 fix: vault warnings are an array, not a string (#153) 94153c72 chore: bump actions/checkout from 3.6.0 to 4.1.1 (#151) 5fe9d92f chore: bump step-security/harden-runner from 2.6.1 to 2.7.0 (#152) ed3767b5 chore: bump actions/upload-artifact from 4.3.0 to 4.3.1 (#154) 94717f0e chore: bump golangci/golangci-lint-action from 3.7.0 to 4.0.0 (#155) 95cf785f chore: bump github/codeql-action from 3.23.2 to 3.24.0 (#150) f07f03c9 [StepSecurity] ci: Harden GitHub Actions (#148) 86f50965 Checking policy signature against cert constraints (#144) 3ce1385b Adding workaround due to failing workflows (#145) 4f2a630a RunAttestors refactor (#131) be75142a fixing error in github actions workflow (#147) 77a9f42e Adding job to auto cut releases (#141) fa5d2caa chore: bump github/codeql-action from 3.23.1 to 3.23.2 (#143) b8734c70 chore: bump actions/upload-artifact from 4.2.0 to 4.3.0 (#142) 027b47d0 refactor: move gitoid code to cyrptoutil, use digestvalue everywhere (#139) 2cb096b7 Adding policy intermediates option to verify function (#138) 33998ffe Improved the search to be concurrent (#62) cfcb7cc5 Moving the timestamper interfaces to the timestamp directory (#132) dd59a2ba chore: bump actions/dependency-review-action from 3.1.5 to 4.0.0 (#137) 45992877 chore: bump github/codeql-action from 3.23.0 to 3.23.1 (#136) 6c44e5b1 chore: bump actions/upload-artifact from 4.1.0 to 4.2.0 (#135) 884637a8 chore: bump k8s.io/apimachinery from 0.26.12 to 0.26.13 (#134) acaefcf0 chore: bump github.com/spiffe/go-spiffe/v2 from 2.1.6 to 2.1.7 (#133) 07128d24 Included tests for GitHub attestations (#61) 3e7ddccb Included Tests for memory.go LoadEnvelope and Search (#59) git-subtree-dir: subtrees/go-witness git-subtree-split: 8fbc70b1d7db128d88f2aba60e16c97ff267d583 * Squashed 'subtrees/witness/' changes from 06031da4..74f6c3dc 74f6c3dc chore: bump the all-go-mod group with 1 update (#425) bed18639 Update GHA triggers to fine tune for code changes vs other updates (#406) 6f7d4a80 Adding ability to list attestors (#384) 1fbdaa9b chore: bump the all-gha group with 1 update (#426) 90cb5acb Update dependabot.yml (#405) c86b46dc small typo fix (#424) f5deef58 chore: bump express from 4.18.3 to 4.19.2 in /docs-website (#423) 6bec1817 chore: bump actions/cache from 4.0.1 to 4.0.2 (#421) 78f1a7b6 chore: bump actions/dependency-review-action from 4.1.3 to 4.2.4 (#420) 2b4213f1 chore: bump github/codeql-action from 3.24.8 to 3.24.9 (#419) fe61acd7 chore: bump webpack-dev-middleware from 5.3.3 to 5.3.4 in /docs-website (#417) 917e13b5 chore: bump docker/login-action from 3.0.0 to 3.1.0 (#413) b1ee6814 chore: bump github/codeql-action from 3.24.6 to 3.24.8 (#415) 10f895d4 chore: bump actions/checkout from 4.1.1 to 4.1.2 (#412) 1844b269 chore: bump k8s.io/apimachinery from 0.29.2 to 0.29.3 (#411) 7528df2d chore: bump follow-redirects from 1.15.5 to 1.15.6 in /docs-website (#410) 3fc10e4e chore: bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#409) bb8b3c07 chore: bump the go_modules group group with 2 updates (#408) 910d630e Witness website netlify (#394) a4c40293 doc: fix example in signers kms doc (#403) 09f8cbb2 fix: run e2e test script as part of workflows (#397) e54d8be3 chore: bump actions/download-artifact from 4.1.2 to 4.1.4 (#399) feac3aa7 chore: bump github/codeql-action from 3.24.5 to 3.24.6 (#400) 3c8d14d6 chore: bump actions/cache from 4.0.0 to 4.0.1 (#401) 997af3b5 Bump to go-witness v0.3.1 (#398) dcac011c chore: bump github/codeql-action from 3.24.3 to 3.24.5 (#396) c211bfee chore: bump actions/dependency-review-action from 4.1.1 to 4.1.3 (#395) 0df242bb chore: bump actions/dependency-review-action from 4.0.0 to 4.1.1 (#392) db7a2664 chore: bump github/codeql-action from 3.24.0 to 3.24.3 (#391) 161286db chore: bump fossas/fossa-action from 1.3.1 to 1.3.3 (#390) f772f2db chore: bump golangci/golangci-lint-action from 3.7.0 to 4.0.0 (#387) 001a113b chore: bump k8s.io/apimachinery from 0.26.13 to 0.26.14 (#386) e438568f chore: bump testifysec/witness-run-action from 0.1.3 to 0.1.5 (#389) 17bdb4ed Add Tom as a Witness maintainer (#385) c27a4f56 KMS Support (#376) be37eeee chore: bump actions/upload-artifact from 4.3.0 to 4.3.1 (#383) 58fe0939 chore: bump actions/download-artifact from 4.1.1 to 4.1.2 (#382) 1144fa56 chore: bump sigstore/cosign-installer from 3.3.0 to 3.4.0 (#380) 3195add2 chore: bump step-security/harden-runner from 2.6.1 to 2.7.0 (#379) 2923f967 chore: bump github/codeql-action from 3.23.2 to 3.24.0 (#378) 0e7dda92 Add back license scanning badge (#377) dfd64fe7 Updated witness to use changes made to `cryptoutil.DigestValue` implemented in go-witness (#371) 58d5516f chore: bump github/codeql-action from 3.23.1 to 3.23.2 (#370) cd18d5eb chore: bump actions/upload-artifact from 4.2.0 to 4.3.0 (#369) 1bbd0e84 Updating timestamper (#367) df179e2e Fixing mistakes in the readme (#368) b90f41ba README and docs restructure (#362) 2b872a34 chore: bump actions/dependency-review-action from 3.1.5 to 4.0.0 (#366) 9247c817 chore: bump github/codeql-action from 3.23.0 to 3.23.1 (#365) 55418b54 chore: bump actions/upload-artifact from 4.1.0 to 4.2.0 (#363) 272e492b chore: bump actions/cache from 3.3.3 to 4.0.0 (#364) git-subtree-dir: subtrees/witness git-subtree-split: 74f6c3dcb07ad6b6c2e67eede125bca3ef302793 * chore: merge-conflict with Archivista 0.4.0 * chore(go-witness): update go-witness to use new store with http client methods --------- Signed-off-by: Kris Coleman Signed-off-by: dependabot[bot] Co-authored-by: Mikhail Swift Co-authored-by: Mikhail Swift Co-authored-by: David Lake Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cole Co-authored-by: Nick Kane --- Makefile | 2 +- cmd/archivistactl/cmd/e2e_test.go | 2 - cmd/archivistactl/cmd/retrieve.go | 50 +---- cmd/archivistactl/cmd/search.go | 116 ++++------- cmd/archivistactl/cmd/store.go | 2 +- pkg/api/download.go | 45 ++++- pkg/api/graphql.go | 67 +++++-- pkg/api/graphql_test.go | 14 +- pkg/api/structs.go | 76 +++++++ pkg/api/upload.go | 20 +- pkg/http-client/client.go | 181 +++++++++++++++++ pkg/http-client/client_test.go | 320 ++++++++++++++++++++++++++++++ 12 files changed, 726 insertions(+), 169 deletions(-) create mode 100644 pkg/api/structs.go create mode 100644 pkg/http-client/client.go create mode 100644 pkg/http-client/client_test.go diff --git a/Makefile b/Makefile index 5add6a1b..e88dab67 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ clean: ## Clean up the dev server .PHONY: test test: ## Run tests - @go test ./... -covermode atomic -coverprofile=cover.out -v + @go test ./... -covermode atomic -coverprofile=cover.out -v -failfast .PHONY: coverage coverage: ## Show html coverage diff --git a/cmd/archivistactl/cmd/e2e_test.go b/cmd/archivistactl/cmd/e2e_test.go index d54320cd..79b00c80 100644 --- a/cmd/archivistactl/cmd/e2e_test.go +++ b/cmd/archivistactl/cmd/e2e_test.go @@ -53,7 +53,6 @@ func (e2e *E2EStoreSuite) TearDownTest() { // Run the E2E tests func (e2e *E2EStoreSuite) Test_E2E() { - // Define tests for supported dbs testDBCases := []string{"mysql", "pgsql"} @@ -208,5 +207,4 @@ func (e2e *E2EStoreSuite) Test_E2E() { } } } - } diff --git a/cmd/archivistactl/cmd/retrieve.go b/cmd/archivistactl/cmd/retrieve.go index 3c52ec7b..b5770907 100644 --- a/cmd/archivistactl/cmd/retrieve.go +++ b/cmd/archivistactl/cmd/retrieve.go @@ -49,7 +49,6 @@ var ( defer file.Close() out = file } - return api.DownloadWithWriter(cmd.Context(), archivistaUrl, args[0], out) }, } @@ -60,7 +59,12 @@ var ( SilenceUsage: true, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - results, err := api.GraphQlQuery[retrieveSubjectResults](cmd.Context(), archivistaUrl, retrieveSubjectsQuery, retrieveSubjectVars{Gitoid: args[0]}) + results, err := api.GraphQlQuery[api.RetrieveSubjectResults]( + cmd.Context(), + archivistaUrl, + api.RetrieveSubjectsQuery, + api.RetrieveSubjectVars{Gitoid: args[0]}, + ) if err != nil { return err } @@ -78,7 +82,7 @@ func init() { envelopeCmd.Flags().StringVarP(&outFile, "out", "o", "", "File to write the envelope out to. Defaults to stdout") } -func printSubjects(results retrieveSubjectResults) { +func printSubjects(results api.RetrieveSubjectResults) { for _, edge := range results.Subjects.Edges { digestStrings := make([]string, 0, len(edge.Node.SubjectDigests)) for _, digest := range edge.Node.SubjectDigests { @@ -88,43 +92,3 @@ func printSubjects(results retrieveSubjectResults) { rootCmd.Printf("Name: %s\nDigests: %s\n", edge.Node.Name, strings.Join(digestStrings, ", ")) } } - -type retrieveSubjectVars struct { - Gitoid string `json:"gitoid"` -} - -type retrieveSubjectResults struct { - Subjects struct { - Edges []struct { - Node struct { - Name string `json:"name"` - SubjectDigests []struct { - Algorithm string `json:"algorithm"` - Value string `json:"value"` - } `json:"subjectDigests"` - } `json:"node"` - } `json:"edges"` - } `json:"subjects"` -} - -const retrieveSubjectsQuery = `query($gitoid: String!) { - subjects( - where: { - hasStatementWith:{ - hasDsseWith:{ - gitoidSha256: $gitoid - } - } - } - ) { - edges { - node{ - name - subjectDigests{ - algorithm - value - } - } - } - } -}` diff --git a/cmd/archivistactl/cmd/search.go b/cmd/archivistactl/cmd/search.go index e41306fd..698ef7e0 100644 --- a/cmd/archivistactl/cmd/search.go +++ b/cmd/archivistactl/cmd/search.go @@ -22,42 +22,46 @@ import ( "github.com/spf13/cobra" ) -var ( - searchCmd = &cobra.Command{ - Use: "search", - Short: "Searches the archivista instance for an attestation matching a query", - SilenceUsage: true, - Long: `Searches the archivista instance for an envelope with a specified subject digest. +var searchCmd = &cobra.Command{ + Use: "search", + Short: "Searches the archivista instance for an attestation matching a query", + SilenceUsage: true, + Long: `Searches the archivista instance for an envelope with a specified subject digest. Optionally a collection name can be provided to further constrain results. Digests are expected to be in the form algorithm:digest, for instance: sha256:456c0c9a7c05e2a7f84c139bbacedbe3e8e88f9c`, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return errors.New("expected exactly 1 argument") - } + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("expected exactly 1 argument") + } - if _, _, err := validateDigestString(args[0]); err != nil { - return err - } + if _, _, err := validateDigestString(args[0]); err != nil { + return err + } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - algo, digest, err := validateDigestString(args[0]) - if err != nil { - return err - } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + algo, digest, err := validateDigestString(args[0]) + if err != nil { + return err + } - results, err := api.GraphQlQuery[searchResults](cmd.Context(), archivistaUrl, searchQuery, searchVars{Algorithm: algo, Digest: digest}) - if err != nil { - return err - } + results, err := api.GraphQlQuery[api.SearchResults]( + cmd.Context(), + archivistaUrl, + api.SearchQuery, + api.SearchVars{Algorithm: algo, Digest: digest}, + ) + if err != nil { + return err + } - printResults(results) - return nil - }, - } -) + // TODO(nick): e2e test are failing here because of output, this is kind of terrible + printResults(results) + return nil + }, +} func init() { rootCmd.AddCommand(searchCmd) @@ -72,7 +76,7 @@ func validateDigestString(ds string) (algo, digest string, err error) { return algo, digest, nil } -func printResults(results searchResults) { +func printResults(results api.SearchResults) { for _, edge := range results.Dsses.Edges { rootCmd.Printf("Gitoid: %s\n", edge.Node.GitoidSha256) rootCmd.Printf("Collection name: %s\n", edge.Node.Statement.AttestationCollection.Name) @@ -84,55 +88,3 @@ func printResults(results searchResults) { rootCmd.Printf("Attestations: %s\n\n", strings.Join(types, ", ")) } } - -type searchVars struct { - Algorithm string `json:"algo"` - Digest string `json:"digest"` -} - -type searchResults struct { - Dsses struct { - Edges []struct { - Node struct { - GitoidSha256 string `json:"gitoidSha256"` - Statement struct { - AttestationCollection struct { - Name string `json:"name"` - Attestations []struct { - Type string `json:"type"` - } `json:"attestations"` - } `json:"attestationCollections"` - } `json:"statement"` - } `json:"node"` - } `json:"edges"` - } `json:"dsses"` -} - -const searchQuery = `query($algo: String!, $digest: String!) { - dsses( - where: { - hasStatementWith: { - hasSubjectsWith: { - hasSubjectDigestsWith: { - value: $digest, - algorithm: $algo - } - } - } - } - ) { - edges { - node { - gitoidSha256 - statement { - attestationCollections { - name - attestations { - type - } - } - } - } - } - } -}` diff --git a/cmd/archivistactl/cmd/store.go b/cmd/archivistactl/cmd/store.go index 87b3e59d..ef7e76a2 100644 --- a/cmd/archivistactl/cmd/store.go +++ b/cmd/archivistactl/cmd/store.go @@ -54,7 +54,7 @@ func storeAttestationByPath(ctx context.Context, baseUrl, path string) (string, } defer file.Close() - resp, err := api.UploadWithReader(ctx, baseUrl, file) + resp, err := api.StoreWithReader(ctx, baseUrl, file) if err != nil { return "", err } diff --git a/pkg/api/download.go b/pkg/api/download.go index 06c67883..21487acf 100644 --- a/pkg/api/download.go +++ b/pkg/api/download.go @@ -26,9 +26,40 @@ import ( "github.com/in-toto/go-witness/dsse" ) -func Download(ctx context.Context, baseUrl string, gitoid string) (dsse.Envelope, error) { +func DownloadReadCloser(ctx context.Context, baseURL string, gitoid string) (io.ReadCloser, error) { + return DownloadReadCloserWithHTTPClient(ctx, &http.Client{}, baseURL, gitoid) +} + +func DownloadReadCloserWithHTTPClient(ctx context.Context, client *http.Client, baseURL string, gitoid string) (io.ReadCloser, error) { + downloadURL, err := url.JoinPath(baseURL, "download", gitoid) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + // NOTE: attempt to read body on error and + // only close if an error occurs + defer resp.Body.Close() + errMsg, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return nil, errors.New(string(errMsg)) + } + return resp.Body, nil +} + +func Download(ctx context.Context, baseURL string, gitoid string) (dsse.Envelope, error) { buf := &bytes.Buffer{} - if err := DownloadWithWriter(ctx, baseUrl, gitoid, buf); err != nil { + if err := DownloadWithWriter(ctx, baseURL, gitoid, buf); err != nil { return dsse.Envelope{}, err } @@ -41,13 +72,17 @@ func Download(ctx context.Context, baseUrl string, gitoid string) (dsse.Envelope return env, nil } -func DownloadWithWriter(ctx context.Context, baseUrl, gitoid string, dst io.Writer) error { - downloadUrl, err := url.JoinPath(baseUrl, "download", gitoid) +func DownloadWithWriter(ctx context.Context, baseURL string, gitoid string, dst io.Writer) error { + return DownloadWithWriterWithHTTPClient(ctx, &http.Client{}, baseURL, gitoid, dst) +} + +func DownloadWithWriterWithHTTPClient(ctx context.Context, client *http.Client, baseURL string, gitoid string, dst io.Writer) error { + downloadUrl, err := url.JoinPath(baseURL, "download", gitoid) if err != nil { return err } - req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadUrl, nil) if err != nil { return err } diff --git a/pkg/api/graphql.go b/pkg/api/graphql.go index 73dbfb4e..246b8e2b 100644 --- a/pkg/api/graphql.go +++ b/pkg/api/graphql.go @@ -25,19 +25,56 @@ import ( "net/url" ) -type graphQLError struct { - Message string `json:"message"` -} - -type graphQLResponse[T any] struct { - Data T `json:"data,omitempty"` - Errors []graphQLError `json:"errors,omitempty"` -} +const RetrieveSubjectsQuery = `query($gitoid: String!) { + subjects( + where: { + hasStatementWith:{ + hasDsseWith:{ + gitoidSha256: $gitoid + } + } + } + ) { + edges { + node{ + name + subjectDigests{ + algorithm + value + } + } + } + } +}` -type graphQLRequestBody[TVars any] struct { - Query string `json:"query"` - Variables TVars `json:"variables,omitempty"` -} +const SearchQuery = `query($algo: String!, $digest: String!) { + dsses( + where: { + hasStatementWith: { + hasSubjectsWith: { + hasSubjectDigestsWith: { + value: $digest, + algorithm: $algo + } + } + } + } + ) { + edges { + node { + gitoidSha256 + statement { + attestationCollections { + name + attestations { + type + } + } + } + } + } + } +}` func GraphQlQuery[TRes any, TVars any](ctx context.Context, baseUrl, query string, vars TVars) (TRes, error) { var response TRes @@ -46,7 +83,7 @@ func GraphQlQuery[TRes any, TVars any](ctx context.Context, baseUrl, query strin return response, err } - requestBody := graphQLRequestBody[TVars]{ + requestBody := GraphQLRequestBodyGeneric[TVars]{ Query: query, Variables: vars, } @@ -56,7 +93,7 @@ func GraphQlQuery[TRes any, TVars any](ctx context.Context, baseUrl, query strin return response, err } - req, err := http.NewRequestWithContext(ctx, "POST", queryUrl, bytes.NewReader(reqBody)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, queryUrl, bytes.NewReader(reqBody)) if err != nil { return response, err } @@ -79,7 +116,7 @@ func GraphQlQuery[TRes any, TVars any](ctx context.Context, baseUrl, query strin } dec := json.NewDecoder(res.Body) - gqlRes := graphQLResponse[TRes]{} + gqlRes := GraphQLResponseGeneric[TRes]{} if err := dec.Decode(&gqlRes); err != nil { return response, err } diff --git a/pkg/api/graphql_test.go b/pkg/api/graphql_test.go index 30948db2..e936a426 100644 --- a/pkg/api/graphql_test.go +++ b/pkg/api/graphql_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/suite" ) -// Test Suite: UT APIStore +// Test Suite: UT APIGraphQL type UTAPIGraphQLSuite struct { suite.Suite } @@ -34,7 +34,6 @@ func TestAPIGraphQLSuite(t *testing.T) { } func (ut *UTAPIGraphQLSuite) Test_Store() { - testServer := httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { @@ -62,7 +61,6 @@ func (ut *UTAPIGraphQLSuite) Test_Store() { } func (ut *UTAPIGraphQLSuite) Test_Store_NoServer() { - ctx := context.TODO() type testSubjectVar struct { @@ -72,13 +70,17 @@ func (ut *UTAPIGraphQLSuite) Test_Store_NoServer() { type testSubjectResult struct { Data string `json:"data"` } - result, err := api.GraphQlQuery[testSubjectResult](ctx, "http://invalid-archivista", `query`, testSubjectVar{Gitoid: "test_Gitoid"}) + result, err := api.GraphQlQuery[testSubjectResult]( + ctx, + "http://invalid-archivista", + `query`, + testSubjectVar{Gitoid: "test_Gitoid"}, + ) ut.Error(err) ut.Equal(testSubjectResult{Data: ""}, result) } func (ut *UTAPIGraphQLSuite) Test_Store_BadStatusCode_NoMsg() { - testServer := httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { @@ -102,7 +104,6 @@ func (ut *UTAPIGraphQLSuite) Test_Store_BadStatusCode_NoMsg() { } func (ut *UTAPIGraphQLSuite) Test_Store_InvalidData() { - testServer := httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { @@ -130,7 +131,6 @@ func (ut *UTAPIGraphQLSuite) Test_Store_InvalidData() { } func (ut *UTAPIGraphQLSuite) Test_Store_QLReponseWithErrors() { - testServer := httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/api/structs.go b/pkg/api/structs.go new file mode 100644 index 00000000..9cdfff01 --- /dev/null +++ b/pkg/api/structs.go @@ -0,0 +1,76 @@ +package api + +type GraphQLError struct { + Message string `json:"message"` +} + +type GraphQLResponseGeneric[T any] struct { + Data T `json:"data,omitempty"` + Errors []GraphQLError `json:"errors,omitempty"` +} + +type GraphQLRequestBodyGeneric[TVars any] struct { + Query string `json:"query"` + Variables TVars `json:"variables,omitempty"` +} + +type RetrieveSubjectVars struct { + Gitoid string `json:"gitoid"` +} + +type SearchVars struct { + Algorithm string `json:"algo"` + Digest string `json:"digest"` +} + +type RetrieveSubjectResults struct { + Subjects Subjects `json:"subjects"` +} + +type Subjects struct { + Edges []SubjectEdge `json:"edges"` +} + +type SubjectEdge struct { + Node SubjectNode `json:"node"` +} + +type SubjectNode struct { + Name string `json:"name"` + SubjectDigests []SubjectDigest `json:"subjectDigests"` +} + +type SubjectDigest struct { + Algorithm string `json:"algorithm"` + Value string `json:"value"` +} + +type SearchResults struct { + Dsses DSSES `json:"dsses"` +} + +type DSSES struct { + Edges []SearchEdge `json:"edges"` +} + +type SearchEdge struct { + Node SearchNode `json:"node"` +} + +type SearchNode struct { + GitoidSha256 string `json:"gitoidSha256"` + Statement Statement `json:"statement"` +} + +type Statement struct { + AttestationCollection AttestationCollection `json:"attestationCollections"` +} + +type AttestationCollection struct { + Name string `json:"name"` + Attestations []Attestation `json:"attestations"` +} + +type Attestation struct { + Type string `json:"type"` +} diff --git a/pkg/api/upload.go b/pkg/api/upload.go index 94504412..3d42e772 100644 --- a/pkg/api/upload.go +++ b/pkg/api/upload.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Witness Contributors +// Copyright 2023 The Archivista Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,28 +33,22 @@ type UploadResponse struct { // Deprecated: Use UploadResponse instead. It will be removed in version >= v0.6.0 type StoreResponse = UploadResponse -// Deprecated: Use Upload instead. It will be removed in version >= v0.6.0 -func Store(ctx context.Context, baseUrl string, envelope dsse.Envelope) (StoreResponse, error) { - return Upload(ctx, baseUrl, envelope) -} - -func Upload(ctx context.Context, baseUrl string, envelope dsse.Envelope) (StoreResponse, error) { +func Store(ctx context.Context, baseURL string, envelope dsse.Envelope) (StoreResponse, error) { buf := &bytes.Buffer{} enc := json.NewEncoder(buf) if err := enc.Encode(envelope); err != nil { return StoreResponse{}, err } - return UploadWithReader(ctx, baseUrl, buf) + return StoreWithReader(ctx, baseURL, buf) } -// Deprecated: Use UploadWithReader instead. It will be removed in version >= v0.6.0 -func StoreWithReader(ctx context.Context, baseUrl string, r io.Reader) (StoreResponse, error) { - return UploadWithReader(ctx, baseUrl, r) +func StoreWithReader(ctx context.Context, baseURL string, r io.Reader) (StoreResponse, error) { + return StoreWithReaderWithHTTPClient(ctx, &http.Client{}, baseURL, r) } -func UploadWithReader(ctx context.Context, baseUrl string, r io.Reader) (StoreResponse, error) { - uploadPath, err := url.JoinPath(baseUrl, "upload") +func StoreWithReaderWithHTTPClient(ctx context.Context, client *http.Client, baseURL string, r io.Reader) (StoreResponse, error) { + uploadPath, err := url.JoinPath(baseURL, "upload") if err != nil { return UploadResponse{}, err } diff --git a/pkg/http-client/client.go b/pkg/http-client/client.go new file mode 100644 index 00000000..61f18bef --- /dev/null +++ b/pkg/http-client/client.go @@ -0,0 +1,181 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpclient + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/in-toto/archivista/pkg/api" + "github.com/in-toto/go-witness/dsse" +) + +type ArchivistaClient struct { + BaseURL string + GraphQLURL string + *http.Client +} + +func CreateArchivistaClient(httpClient *http.Client, baseURL string) (*ArchivistaClient, error) { + client := ArchivistaClient{ + BaseURL: baseURL, + Client: http.DefaultClient, + } + if httpClient != nil { + client.Client = httpClient + } + var err error + client.GraphQLURL, err = url.JoinPath(client.BaseURL, "query") + if err != nil { + return nil, err + } + return &client, nil +} + +func (ac *ArchivistaClient) DownloadDSSE(ctx context.Context, gitoid string) (dsse.Envelope, error) { + reader, err := api.DownloadReadCloserWithHTTPClient(ctx, ac.Client, ac.BaseURL, gitoid) + if err != nil { + return dsse.Envelope{}, err + } + env := dsse.Envelope{} + if err := json.NewDecoder(reader).Decode(&env); err != nil { + return dsse.Envelope{}, err + } + return env, nil +} + +func (ac *ArchivistaClient) DownloadReadCloser(ctx context.Context, gitoid string) (io.ReadCloser, error) { + return api.DownloadReadCloserWithHTTPClient(ctx, ac.Client, ac.BaseURL, gitoid) +} + +func (ac *ArchivistaClient) DownloadWithWriter(ctx context.Context, gitoid string, dst io.Writer) error { + return api.DownloadWithWriterWithHTTPClient(ctx, ac.Client, ac.BaseURL, gitoid, dst) +} + +func (ac *ArchivistaClient) Store(ctx context.Context, envelope dsse.Envelope) (api.StoreResponse, error) { + return api.Store(ctx, ac.BaseURL, envelope) +} + +func (ac *ArchivistaClient) StoreWithReader(ctx context.Context, r io.Reader) (api.StoreResponse, error) { + return api.StoreWithReader(ctx, ac.BaseURL, r) +} + +type GraphQLRequestBodyInterface struct { + Query string `json:"query"` + Variables interface{} `json:"variables,omitempty"` +} + +type GraphQLResponseInterface struct { + Data interface{} + Errors []api.GraphQLError `json:"errors,omitempty"` +} + +func (ac *ArchivistaClient) GraphQLRetrieveSubjectResults( + ctx context.Context, + gitoid string, +) (api.RetrieveSubjectResults, error) { + return api.GraphQlQuery[api.RetrieveSubjectResults]( + ctx, + ac.BaseURL, + api.RetrieveSubjectsQuery, + api.RetrieveSubjectVars{Gitoid: gitoid}, + ) +} + +func (ac *ArchivistaClient) GraphQLRetrieveSearchResults( + ctx context.Context, + algo string, + digest string, +) (api.SearchResults, error) { + return api.GraphQlQuery[api.SearchResults]( + ctx, + ac.BaseURL, + api.SearchQuery, + api.SearchVars{Algorithm: algo, Digest: digest}, + ) +} + +func (ac *ArchivistaClient) GraphQLQueryIface( + ctx context.Context, + query string, + variables interface{}, +) (*GraphQLResponseInterface, error) { + reader, err := ac.GraphQLQueryReadCloser(ctx, query, variables) + if err != nil { + return nil, err + } + defer reader.Close() + gqlRes := GraphQLResponseInterface{} + dec := json.NewDecoder(reader) + if err := dec.Decode(&gqlRes); err != nil { + return nil, err + } + if len(gqlRes.Errors) > 0 { + return nil, fmt.Errorf("graph ql query failed: %v", gqlRes.Errors) + } + return &gqlRes, nil +} + +func (ac *ArchivistaClient) GraphQLQueryToDst(ctx context.Context, query string, variables interface{}, dst interface{}) error { + reader, err := ac.GraphQLQueryReadCloser(ctx, query, variables) + if err != nil { + return err + } + defer reader.Close() + dec := json.NewDecoder(reader) + if err := dec.Decode(&dst); err != nil { + return err + } + return nil +} + +func (ac *ArchivistaClient) GraphQLQueryReadCloser( + ctx context.Context, + query string, + variables interface{}, +) (io.ReadCloser, error) { + requestBodyMap := GraphQLRequestBodyInterface{ + Query: query, + Variables: variables, + } + requestBodyJSON, err := json.Marshal(requestBodyMap) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, ac.GraphQLURL, bytes.NewReader(requestBodyJSON)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + res, err := ac.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + defer res.Body.Close() + errMsg, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, errors.New(string(errMsg)) + } + return res.Body, nil +} diff --git a/pkg/http-client/client_test.go b/pkg/http-client/client_test.go new file mode 100644 index 00000000..e2de972d --- /dev/null +++ b/pkg/http-client/client_test.go @@ -0,0 +1,320 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpclient_test + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/in-toto/archivista/pkg/api" + httpclient "github.com/in-toto/archivista/pkg/http-client" + "github.com/in-toto/go-witness/dsse" + "github.com/stretchr/testify/suite" +) + +// Test Suite: UT HTTPClientDownloadSuite +type UTHTTPClientDownloadSuite struct { + suite.Suite +} + +func TestHTTPClientAPIDownloadSuite(t *testing.T) { + suite.Run(t, new(UTHTTPClientDownloadSuite)) +} + +func (ut *UTHTTPClientDownloadSuite) Test_DownloadDSSE() { + testEnvelope, err := os.ReadFile("../../test/package.attestation.json") + if err != nil { + ut.FailNow(err.Error()) + } + expectedEnvelop := dsse.Envelope{} + err = json.Unmarshal(testEnvelope, &expectedEnvelop) + if err != nil { + ut.FailNow(err.Error()) + } + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err = w.Write(testEnvelope) + if err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + resp, err := client.DownloadDSSE(ctx, "gitoid_test") + if err != nil { + ut.FailNow(err.Error()) + } + ut.Equal(expectedEnvelop, resp) +} + +func (ut *UTHTTPClientDownloadSuite) Test_DownloadReadCloser() { + testEnvelope, err := os.ReadFile("../../test/package.attestation.json") + if err != nil { + ut.FailNow(err.Error()) + } + expectedEnvelop := dsse.Envelope{} + err = json.Unmarshal(testEnvelope, &expectedEnvelop) + if err != nil { + ut.FailNow(err.Error()) + } + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err = w.Write(testEnvelope) + if err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + readCloser, err := client.DownloadReadCloser(ctx, "gitoid_test") + if err != nil { + ut.FailNow(err.Error()) + } + env := dsse.Envelope{} + if err := json.NewDecoder(readCloser).Decode(&env); err != nil { + ut.FailNow(err.Error()) + } + ut.Equal(expectedEnvelop, env) +} + +func (ut *UTHTTPClientDownloadSuite) Test_DownloadWithWriter() { + testEnvelope, err := os.ReadFile("../../test/package.attestation.json") + if err != nil { + ut.FailNow(err.Error()) + } + expectedEnvelop := dsse.Envelope{} + err = json.Unmarshal(testEnvelope, &expectedEnvelop) + if err != nil { + ut.FailNow(err.Error()) + } + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err = w.Write(testEnvelope) + if err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + buf := bytes.NewBuffer(nil) + if err := client.DownloadWithWriter(ctx, "gitoid_test", buf); err != nil { + ut.FailNow(err.Error()) + } + env := dsse.Envelope{} + if err := json.NewDecoder(buf).Decode(&env); err != nil { + ut.FailNow(err.Error()) + } + ut.Equal(expectedEnvelop, env) +} + +// Test Suite: UT HTTPClientStore +type UTHTTPClientStoreSuite struct { + suite.Suite +} + +func TestAPIStoreSuite(t *testing.T) { + suite.Run(t, new(UTHTTPClientStoreSuite)) +} + +func (ut *UTHTTPClientStoreSuite) Test_Store() { + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"gitoid":"test"}`)) + if err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + attFile, err := os.ReadFile("../../test/package.attestation.json") + if err != nil { + ut.FailNow(err.Error()) + } + attEnvelop := dsse.Envelope{} + err = json.Unmarshal(attFile, &attEnvelop) + if err != nil { + ut.FailNow(err.Error()) + } + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + resp, err := client.Store(ctx, attEnvelop) + if err != nil { + ut.FailNow(err.Error()) + } + ut.Equal(resp, api.StoreResponse{Gitoid: "test"}) +} + +func (ut *UTHTTPClientStoreSuite) Test_StoreWithReader() { + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"gitoid":"test"}`)) + if err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + attIo, err := os.Open("../../test/package.attestation.json") + if err != nil { + ut.FailNow(err.Error()) + } + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + resp, err := client.StoreWithReader(ctx, attIo) + if err != nil { + ut.FailNow(err.Error()) + } + ut.Equal(resp, api.StoreResponse{Gitoid: "test"}) +} + +// Test Suite: UT HTTPClientStore +type UTHTTPClientGraphQLSuite struct { + suite.Suite +} + +func TestAPIGraphQLSuite(t *testing.T) { + suite.Run(t, new(UTHTTPClientGraphQLSuite)) +} + +func (ut *UTHTTPClientGraphQLSuite) Test_GraphQLRetrieveSubjectResults() { + expected := api.GraphQLResponseGeneric[api.RetrieveSubjectResults]{ + Data: api.RetrieveSubjectResults{ + Subjects: api.Subjects{ + Edges: []api.SubjectEdge{ + { + Node: api.SubjectNode{ + Name: "test_Gitoid", + SubjectDigests: []api.SubjectDigest{ + { + Algorithm: "test_Gitoid", + Value: "test_Gitoid", + }, + }, + }, + }, + }, + }, + }, + Errors: []api.GraphQLError{}, + } + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(expected); err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + actual, err := client.GraphQLRetrieveSubjectResults(ctx, "test_Gitoid") + ut.NoError(err) + ut.Equal(expected.Data, actual) +} + +func (ut *UTHTTPClientGraphQLSuite) Test_GraphQLSearchResults() { + expected := api.GraphQLResponseGeneric[api.SearchResults]{ + Data: api.SearchResults{ + Dsses: api.DSSES{ + Edges: []api.SearchEdge{ + { + Node: api.SearchNode{ + GitoidSha256: "test_Gitoid", + Statement: api.Statement{ + AttestationCollection: api.AttestationCollection{ + Name: "test_Gitoid", + Attestations: []api.Attestation{ + { + Type: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + Errors: []api.GraphQLError{}, + } + testServer := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(expected); err != nil { + ut.FailNow(err.Error()) + } + }, + ), + ) + defer testServer.Close() + ctx := context.TODO() + client, err := httpclient.CreateArchivistaClient(http.DefaultClient, testServer.URL) + if err != nil { + ut.FailNow(err.Error()) + } + actual, err := client.GraphQLRetrieveSearchResults(ctx, "test_Gitoid", "test_Gitoid") + ut.NoError(err) + ut.Equal(expected.Data, actual) +}