From f7ec272af3cb6b8ac666679d974aab42f1dcbc8a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 6 Oct 2023 12:11:23 +0700 Subject: [PATCH] try accessing Backblaze --- .github/workflows/interop.yml | 293 ++-------------------------------- implementations.json | 75 --------- 2 files changed, 13 insertions(+), 355 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index c5851c40..b75ba122 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -1,248 +1,15 @@ name: interop on: - schedule: - - cron: "0 */8 * * *" # every 8h - -# Cache key for caching the Wireshark build. -# To trigger a rebuild of Wireshark increment this value. -# The rebuild will then build the current master of Wireshark and save it under the new key. -env: - WIRESHARK_CACHEKEY: 6 + push: + branches: ["s3-setup"] jobs: - wireshark: - runs-on: ubuntu-latest - steps: - - name: Restore from cache - id: restore-cache - uses: actions/cache@v3 - env: - VERSION: ${{ env.WIRESHARK_CACHEKEY }} - with: - key: wireshark-${{ env.VERSION }} - path: tshark.tar.gz - - name: Show tshark version information - if: steps.restore-cache.outputs.cache-hit == 'true' - run: | - tar xfz tshark.tar.gz - ./tshark -v - - uses: actions/checkout@v3 - if: steps.restore-cache.outputs.cache-hit != 'true' - with: - repository: the-tcpdump-group/libpcap - - name: Build libpcap - if: steps.restore-cache.outputs.cache-hit != 'true' - run: | - ./autogen.sh - ./configure --disable-dbus --disable-rdma - sudo make install - - uses: actions/checkout@v3 - if: steps.restore-cache.outputs.cache-hit != 'true' - with: - repository: wireshark/wireshark - - name: Install dependencies - if: steps.restore-cache.outputs.cache-hit != 'true' - run: sudo apt-get install -y cmake libglib2.0-dev libc-ares-dev libgcrypt20-dev flex bison byacc ninja-build - - name: Build Wireshark - if: steps.restore-cache.outputs.cache-hit != 'true' - run: | - cmake -GNinja -DBUILD_wireshark=0 -DBUILD_qtshark=0 -DBUILD_editcap=0 -DBUILD_capinfos=0 -DBUILD_text2pcap=0 -DBUILD_rawshark=0 -DBUILD_sdjournal=0 -DBUILD_sshdump=0 -DBUILD_ciscodump=0 -DBUILD_sharkd=0 -DENABLE_STATIC=1 -DENABLE_PLUGINS=0 -DENABLE_LIBXML2=0 -DENABLE_BROTLI=0 -DUSE_STATIC=1 -DENABLE_GNUTLS=1 . - ninja - - run: run/tshark -v - if: steps.restore-cache.outputs.cache-hit != 'true' - - name: Compress - if: steps.restore-cache.outputs.cache-hit != 'true' - run: tar -czvf tshark.tar.gz -C run/ tshark - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: wireshark - path: tshark.tar.gz - config: - runs-on: ubuntu-latest - outputs: - logname: ${{ steps.set-logname.outputs.logname }} - starttime: ${{ steps.set-starttime.outputs.starttime }} - servers: ${{ steps.set-servers.outputs.servers }} - clients: ${{ steps.set-clients.outputs.clients }} - images: ${{ steps.set-images.outputs.images }} - steps: - - name: Set log name - id: set-logname - run: | - LOGNAME=$(date -u +"%Y-%m-%dT%H:%M") - echo $LOGNAME - echo "logname=$LOGNAME" >> $GITHUB_OUTPUT - - name: Save start time - id: set-starttime - run: | - STARTTIME=$(date +%s) - echo $STARTTIME - echo "starttime=$STARTTIME" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Determine servers - id: set-servers - run: | - SERVERS=$(jq -c 'with_entries(select(.value.role == "server" or .value.role == "both")) | keys_unsorted' implementations.json) - echo $SERVERS - echo "servers=$SERVERS" >> $GITHUB_OUTPUT - - name: Determine clients - id: set-clients - run: | - CLIENTS=$(jq -c 'with_entries(select(.value.role == "client" or .value.role == "both")) | keys_unsorted' implementations.json) - echo $CLIENTS - echo "clients=$CLIENTS" >> $GITHUB_OUTPUT - - name: Determine Docker images - id: set-images - run: | - IMAGES=$(jq -c 'keys_unsorted' implementations.json) - echo $IMAGES - echo "images=$IMAGES" >> $GITHUB_OUTPUT - docker-pull-tools: - runs-on: ubuntu-latest - strategy: - matrix: - image: [ 'quic-network-simulator', 'quic-interop-iperf-endpoint' ] - steps: - - uses: actions/checkout@v3 - - name: Pull - run: | - URL="martenseemann/${{ matrix.image }}" - docker pull $URL - echo "URL=$URL" >> $GITHUB_ENV - - name: Docker inspect - run: docker image inspect $URL - - name: Save Docker image - run: | - docker save $URL | gzip --best > ${{ matrix.image }}.tar.gz - du -sh ${{ matrix.image }}.tar.gz - - name: Upload result - uses: actions/upload-artifact@v3 - with: - name: images-tools - path: ${{ matrix.image }}.tar.gz - if-no-files-found: error - docker-pull-images: - needs: [ config ] - runs-on: ubuntu-latest - strategy: - matrix: - image: ${{ fromJson(needs.config.outputs.images) }} - name: Pull ${{ matrix.image }} - steps: - - uses: actions/checkout@v3 - - name: Run docker pull - run: | - URL=$(jq -r '.["${{ matrix.image }}"].image' implementations.json) - echo $URL - docker pull $URL - echo "URL=$URL" >> $GITHUB_ENV - - name: Docker inspect - run: docker image inspect $URL - - name: Save Docker image - run: | - docker save $URL | gzip --best > ${{ matrix.image }}.tar.gz - du -sh ${{ matrix.image }}.tar.gz - - name: Upload result - uses: actions/upload-artifact@v3 - with: - name: image-${{ matrix.image }} - path: ${{ matrix.image }}.tar.gz - if-no-files-found: error - tests: - needs: [ wireshark, config, docker-pull-tools, docker-pull-images ] - runs-on: ubuntu-latest - continue-on-error: true - strategy: - fail-fast: false - matrix: - server: ${{ fromJson(needs.config.outputs.servers) }} - client: ${{ fromJson(needs.config.outputs.clients) }} - name: (${{ matrix.server }} - ${{ matrix.client }}) - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Enable IPv6 support - run: sudo modprobe ip6table_filter - - run: docker image ls - - name: Download tools Docker images - uses: actions/download-artifact@v3 - with: - name: images-tools - - name: Download ${{ matrix.server }} Docker image - uses: actions/download-artifact@v3 - with: - name: image-${{ matrix.server }} - - name: Download ${{ matrix.client }} Docker image - if: ${{ matrix.server != matrix.client }} - uses: actions/download-artifact@v3 - with: - name: image-${{ matrix.client }} - - name: Load docker images - run: | - docker load --input quic-network-simulator.tar.gz - docker load --input quic-interop-iperf-endpoint.tar.gz - docker load --input ${{ matrix.server }}.tar.gz - docker load --input ${{ matrix.client }}.tar.gz - - run: docker image ls - - name: Download Wireshark - uses: actions/download-artifact@v3 - with: - name: wireshark - path: wireshark - - name: Install Wireshark - run: | - cd wireshark - tar xfz tshark.tar.gz - sudo mv tshark /usr/local/bin - cd .. && rm -r wireshark - - name: Install Python packages - run: | - pip install -U pip - pip install -r requirements.txt - - name: Run tests - env: - CRON: "true" - run: | - (python run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --log-dir logs --json ${{ matrix.server }}_${{ matrix.client }}_results.json -t onlyTests || true) | tee output.txt - mkdir -p logs/${{ matrix.server }}_${{ matrix.client }} - mv output.txt logs/${{ matrix.server }}_${{ matrix.client }}/ - - name: Run measurements - env: - CRON: "true" - run: | - python run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --log-dir logs_measurement --json ${{ matrix.server }}_${{ matrix.client }}_measurements.json -t onlyMeasurements || true - if [ ! -d "logs_measurement" ]; then exit 0; fi - find logs_measurement -depth -name "sim" -type d -exec rm -r "{}" \; - find logs_measurement -depth -name "client" -type d -exec rm -r "{}" \; - find logs_measurement -depth -name "server" -type d -exec rm -r "{}" \; - mv logs_measurement/${{ matrix.server }}_${{ matrix.client }}/* logs/${{ matrix.server }}_${{ matrix.client }}/ - - name: Upload logs - if: ${{ github.event_name == 'schedule' }} - uses: appleboy/scp-action@master - with: - host: interop.seemann.io - username: ${{ secrets.INTEROP_SEEMANN_IO_USER }} - key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }} - source: logs/${{ matrix.server }}_${{ matrix.client }} - target: /mnt/logs/${{ needs.config.outputs.logname }} - strip_components: 1 - - name: Upload result - uses: actions/upload-artifact@v3 - with: - name: results - if-no-files-found: error - path: | - ${{ matrix.server }}_${{ matrix.client }}_results.json - ${{ matrix.server }}_${{ matrix.client }}_measurements.json - aggregate: - needs: [ config, tests ] + backblaze: runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read env: LOGNAME: ${{ needs.config.outputs.logname }} steps: @@ -250,44 +17,10 @@ jobs: - uses: actions/setup-python@v4 with: python-version: 3.8 - - name: Download results - uses: actions/download-artifact@v3 - with: - name: results - - name: Aggregate results - run: | - python .github/workflows/aggregate.py \ - --start-time ${{ needs.config.outputs.starttime }} \ - --server ${{ join(fromJson(needs.config.outputs.servers), ',') }} \ - --client ${{ join(fromJson(needs.config.outputs.clients), ',') }} \ - --log-dir=$LOGNAME \ - --output result.json - - name: Print result - run: jq '.' result.json - - name: Upload result to artifacts - uses: actions/upload-artifact@v3 - with: - name: results - path: result.json - - name: Upload result to interop.seemann.io - if: ${{ github.event_name == 'schedule' }} - uses: appleboy/scp-action@master - with: - host: interop.seemann.io - username: ${{ secrets.INTEROP_SEEMANN_IO_USER }} - key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }} - source: result.json - target: /mnt/logs/${{ needs.config.outputs.logname }} - - name: Publish result - if: ${{ github.event_name == 'schedule' }} - uses: appleboy/ssh-action@master + - run: echo "foobar" > result.json + - name: Configure S3 credentials + uses: aws-actions/configure-aws-credentials@v4 with: - host: interop.seemann.io - username: ${{ secrets.INTEROP_SEEMANN_IO_USER }} - key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }} - envs: LOGNAME - script: | - cd /mnt/logs - jq '. += [ "${{ needs.config.outputs.logname }}" ]' logs.json | sponge logs.json - rm latest || true - ln -s $LOGNAME latest + aws-access-key-id: ${{ secrets.BACKBLAZE_KEY_ID }} + aws-secret-access-key: ${{ secrets.BACKBLAZE_APPLICATION_KEY }} + - run: aws s3 ls --endpoint-url=https://s3.us-west-000.backblazeb2.com diff --git a/implementations.json b/implementations.json index 7b2654d9..3c07d79a 100644 --- a/implementations.json +++ b/implementations.json @@ -4,84 +4,9 @@ "url": "https://github.com/lucas-clemente/quic-go", "role": "both" }, - "ngtcp2": { - "image": "ghcr.io/ngtcp2/ngtcp2-interop:latest", - "url": "https://github.com/ngtcp2/ngtcp2", - "role": "both" - }, - "quant": { - "image": "ntap/quant:interop", - "url": "https://github.com/NTAP/quant", - "role": "both" - }, - "mvfst": { - "image": "lnicco/mvfst-qns:latest", - "url": "https://github.com/facebookincubator/mvfst", - "role": "both" - }, - "quiche": { - "image": "cloudflare/quiche-qns:latest", - "url": "https://github.com/cloudflare/quiche", - "role": "both" - }, - "kwik": { - "image": "peterdoornbosch/kwik_n_flupke-interop", - "url": "https://github.com/ptrd/kwik", - "role": "both" - }, - "picoquic": { - "image": "privateoctopus/picoquic:latest", - "url": "https://github.com/private-octopus/picoquic", - "role": "both" - }, - "aioquic": { - "image": "aiortc/aioquic-qns:latest", - "url": "https://github.com/aiortc/aioquic", - "role": "both" - }, - "neqo": { - "image": "neqoquic/neqo-qns:latest", - "url": "https://github.com/mozilla/neqo", - "role": "both" - }, - "nginx": { - "image": "public.ecr.aws/nginx/nginx-quic-qns:latest", - "url": "https://quic.nginx.org/", - "role": "server" - }, "msquic": { "image": "ghcr.io/microsoft/msquic/qns:main", "url": "https://github.com/microsoft/msquic", "role": "both" - }, - "chrome": { - "image": "martenseemann/chrome-quic-interop-runner", - "url": "https://github.com/marten-seemann/chrome-quic-interop-runner", - "role": "client" - }, - "xquic": { - "image": "ghcr.io/alibaba/xquic/xquic-interop:latest", - "url": "https://github.com/alibaba/xquic", - "role": "both" - }, - "lsquic": { - "image": "litespeedtech/lsquic-qir:latest", - "url": "https://github.com/litespeedtech/lsquic", - "role": "both" - }, - "haproxy": { - "image": "haproxytech/haproxy-qns:latest", - "url": "https://github.com/haproxy/haproxy", - "role": "server" - }, - "quinn": { - "image": "stammw/quinn-interop:latest", - "url": "https://github.com/quinn-rs/quinn", - "role": "both" - }, - "s2n-quic": { - "image": "public.ecr.aws/s2n/s2n-quic-qns:latest", - "url": "https://github.com/aws/s2n-quic", - "role": "both" } }