generated from ipfs/ipfs-repository-template
-
Notifications
You must be signed in to change notification settings - Fork 98
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
gateway: remote car gateway backend (#587)
- Loading branch information
Showing
33 changed files
with
3,956 additions
and
267 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,17 +1,23 @@ | ||
| name: Gateway Conformance | ||
| # This workflow runs https://github.com/ipfs/gateway-conformance | ||
| # against different backend implementations of boxo/gateway | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| pull_request: | ||
| workflow_dispatch: | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| gateway-conformance: | ||
| # This test uses a static CAR file as a local blockstore, | ||
| # allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go) | ||
| # which is used by implementations like Kubo | ||
| local-block-backend: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| # 1. Download the gateway-conformance fixtures | ||
|
|
@@ -21,49 +27,185 @@ jobs: | |
| output: fixtures | ||
| merged: true | ||
|
|
||
| # 2. Build the car-gateway | ||
| # 2. Build the gateway binary | ||
| - name: Checkout boxo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: boxo | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: 'boxo/examples/go.mod' | ||
| cache-dependency-path: "boxo/**/*.sum" | ||
| - name: Build test-gateway | ||
| run: go build -o test-gateway | ||
| working-directory: boxo/examples/gateway/car-file | ||
|
|
||
| # 3. Start the gateway binary | ||
| - name: Start test-gateway | ||
| run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 & | ||
|
|
||
| # 4. Run the gateway-conformance tests | ||
| - name: Run gateway-conformance tests | ||
| uses: ipfs/gateway-conformance/.github/actions/[email protected] | ||
| with: | ||
| gateway-url: http://127.0.0.1:8040 | ||
| json: output.json | ||
| xml: output.xml | ||
| html: output.html | ||
| markdown: output.md | ||
| subdomain-url: http://example.net | ||
| specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway | ||
|
|
||
| # 5. Upload the results | ||
| - name: Upload MD summary | ||
| if: failure() || success() | ||
| run: cat output.md >> $GITHUB_STEP_SUMMARY | ||
| - name: Upload HTML report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: gateway-conformance_local-block-backend.html | ||
| path: output.html | ||
| - name: Upload JSON report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: gateway-conformance_local-block-backend.json | ||
| path: output.json | ||
|
|
||
| # This test uses remote block gateway (?format=raw) as a remote blockstore, | ||
| # allowing us to test conformance against RemoteBlocksBackend | ||
| # (gateway/backend_blocks.go) which is used by implementations like | ||
| # rainbow configured to use with remote block backend | ||
| # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw | ||
| remote-block-backend: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| # 1. Download the gateway-conformance fixtures | ||
| - name: Download gateway-conformance fixtures | ||
| uses: ipfs/gateway-conformance/.github/actions/[email protected] | ||
| with: | ||
| output: fixtures | ||
| merged: true | ||
|
|
||
| # 2. Build the gateway binaries | ||
| - name: Checkout boxo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: boxo | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v4 | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: 'boxo/examples/go.mod' | ||
| cache-dependency-path: "boxo/**/*.sum" | ||
| - name: Build remote-block-backend # it will act as a trustless CAR gateway | ||
| run: go build -o remote-block-backend | ||
| working-directory: boxo/examples/gateway/car-file | ||
| - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend | ||
| run: go build -o test-gateway | ||
| working-directory: boxo/examples/gateway/proxy-blocks | ||
|
|
||
| # 3. Start the gateway binaries | ||
| - name: Start remote HTTP backend that serves application/vnd.ipld.raw | ||
| run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests | ||
| - name: Start gateway that uses the remote block backend | ||
| run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 & | ||
|
|
||
| # 4. Run the gateway-conformance tests | ||
| - name: Run gateway-conformance tests | ||
| uses: ipfs/gateway-conformance/.github/actions/[email protected] | ||
| with: | ||
| gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway | ||
| json: output.json | ||
| xml: output.xml | ||
| html: output.html | ||
| markdown: output.md | ||
| subdomain-url: http://example.net | ||
| specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway | ||
| args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore' | ||
|
|
||
| # 5. Upload the results | ||
| - name: Upload MD summary | ||
| if: failure() || success() | ||
| run: cat output.md >> $GITHUB_STEP_SUMMARY | ||
| - name: Upload HTML report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| go-version: 1.21.x | ||
| name: gateway-conformance_remote-block-backend.html | ||
| path: output.html | ||
| - name: Upload JSON report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: gateway-conformance_remote-block-backend.json | ||
| path: output.json | ||
|
|
||
| # This test uses remote CAR gateway (?format=car, IPIP-402) | ||
| # allowing us to test conformance against remote CarFetcher backend. | ||
| # (gateway/backend_car_fetcher.go) which is used by implementations like | ||
| # rainbow configured to use with remote car backend | ||
| # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car | ||
| remote-car-backend: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| # 1. Download the gateway-conformance fixtures | ||
| - name: Download gateway-conformance fixtures | ||
| uses: ipfs/gateway-conformance/.github/actions/[email protected] | ||
| with: | ||
| output: fixtures | ||
| merged: true | ||
|
|
||
| # 2. Build the gateway binaries | ||
| - name: Checkout boxo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: boxo | ||
| - name: Build car-gateway | ||
| run: go build -o car-gateway | ||
| working-directory: boxo/examples/gateway/car | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: 'boxo/examples/go.mod' | ||
| cache-dependency-path: "boxo/**/*.sum" | ||
| - name: Build remote-car-backend # it will act as a trustless CAR gateway | ||
| run: go build -o remote-car-backend | ||
| working-directory: boxo/examples/gateway/car-file | ||
| - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend | ||
| run: go build -o test-gateway | ||
| working-directory: boxo/examples/gateway/proxy-car | ||
|
|
||
| # 3. Start the car-gateway | ||
| - name: Start car-gateway | ||
| run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 & | ||
| # 3. Start the gateway binaries | ||
| - name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402) | ||
| run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests | ||
| - name: Start gateway that uses the remote CAR backend | ||
| run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 & | ||
|
|
||
| # 4. Run the gateway-conformance tests | ||
| - name: Run gateway-conformance tests | ||
| uses: ipfs/gateway-conformance/.github/actions/[email protected] | ||
| with: | ||
| gateway-url: http://127.0.0.1:8040 | ||
| gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway | ||
| json: output.json | ||
| xml: output.xml | ||
| html: output.html | ||
| markdown: output.md | ||
| subdomain-url: http://example.net | ||
| specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway | ||
| args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' | ||
| args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore' | ||
|
|
||
| # 5. Upload the results | ||
| - name: Upload MD summary | ||
| if: failure() || success() | ||
| run: cat output.md >> $GITHUB_STEP_SUMMARY | ||
| - name: Upload HTML report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v3 | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: gateway-conformance.html | ||
| name: gateway-conformance_remote-car-backend.html | ||
| path: output.html | ||
| - name: Upload JSON report | ||
| if: failure() || success() | ||
| uses: actions/upload-artifact@v3 | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: gateway-conformance.json | ||
| name: gateway-conformance_remote-car-backend.json | ||
| path: output.json | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| # Gateway as Proxy for Trustless CAR Remote Backend | ||
|
|
||
| This is an example of building a "verifying proxy" Gateway that has no | ||
| local on-disk blockstore, but instead, uses `application/vnd.ipld.car` and | ||
| `application/vnd.ipfs.ipns-record` responses from a remote HTTP server that | ||
| implements CAR support from [Trustless Gateway | ||
| Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). | ||
|
|
||
| **NOTE:** the remote CAR backend MUST implement [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) | ||
|
|
||
| ## Build | ||
|
|
||
| ```bash | ||
| > go build -o gateway | ||
| ``` | ||
|
|
||
| ## Usage | ||
|
|
||
| First, you need a compliant gateway that supports both [CAR requests](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) and IPNS Record response | ||
| types. Once you have it, run the proxy gateway with its address as the host parameter: | ||
|
|
||
| ``` | ||
| ./gateway -g https://trustless-gateway.link -p 8040 | ||
| ``` | ||
|
|
||
| ### Subdomain gateway | ||
|
|
||
| Now you can access the gateway in [`localhost:8040`](http://localhost:8040/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze). It will | ||
| behave like a regular [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway), | ||
| except for the fact that it runs no libp2p, and has no local blockstore. | ||
| All data is provided by a remote trustless gateway, fetched as CAR files and IPNS Records, and verified locally. | ||
|
|
||
| ### Path gateway | ||
|
|
||
| If you don't need Origin isolation and only care about hosting flat files, | ||
| a plain [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at | ||
| [`127.0.0.1:8040`](http://127.0.0.1:8040/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi) | ||
| may suffice. | ||
|
|
||
| ### DNSLink gateway | ||
|
|
||
| Gateway supports hosting of [DNSLink](https://dnslink.dev/) websites. All you need is to pass `Host` header with FQDN that has DNSLink set up: | ||
|
|
||
| ```console | ||
| $ curl -sH 'Host: en.wikipedia-on-ipfs.org' 'http://127.0.0.1:8080/wiki/' | head -3 | ||
| <!DOCTYPE html><html class="client-js"><head> | ||
| <meta charset="UTF-8"> | ||
| <title>Wikipedia, the free encyclopedia</title> | ||
| ``` | ||
|
|
||
| Put it behind a reverse proxy terminating TLS (like Nginx) and voila! |
Oops, something went wrong.