Merge remote-tracking branch 'upstream/master' into 6.0/stage

PR URL: https://www.github.com/delphix/bcc/pull/13

.dockerignore

@@ -1,3 +1,4 @@
 Dockerfile*
 build
 .*.swp
+docker/Dockerfile*

.github/actions/build-container/action.yml

@@ -0,0 +1,47 @@
+name: "Build/Push container"
+description: "Build a BCC CI container and push it when not a pull-request."
+
+inputs:
+  os_distro:
+    description: "OS Disctribution. Ex: ubuntu"
+    required: true
+  os_version:
+    description: "Version of the OS. Ex: 20.04"
+    required: true
+  os_nick:
+    description: "Nickname of the OS. Ex: focal"
+    required: true
+  registry:
+    description: "Registry where to push images"
+    default: ghcr.io
+  password:
+    description: "Password used to log into the docker registry."
+  push:
+    description: "Whether or not to push the build image"
+    type: boolean
+    default: false
+
+runs:
+  using: "composite"
+  steps:
+    # Login against registry except on PR
+    # https://github.com/docker/login-action
+    - name: Log into registry ${{ inputs.registry }}
+      if: ${{ inputs.push == 'true' && github.event_name != 'pull_request' }}
+
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.registry }}
+        username: ${{ github.actor }}
+        password: ${{ inputs.password }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v3
+      with:
+        push: ${{ inputs.push == 'true' && github.event_name != 'pull_request' }}
+        build-args: |
+          VERSION=${{ inputs.os_version }}
+          SHORTNAME=${{ inputs.os_nick }}
+        file: docker/build/Dockerfile.${{ inputs.os_distro }}
+        tags: ${{ inputs.registry }}/${{ github.repository }}:${{ inputs.os_distro }}-${{ inputs.os_version }}
+

.github/disabled-workflows/publish.yml

@@ -1,68 +1,15 @@
 name: Publish Build Artifacts
 
-on: push
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
-jobs:
-  publish_images:
-    # Optionally publish container images, guarded by the GitHub secret
-    # QUAY_PUBLISH.
-    # To set this up, sign up for quay.io (you can connect it to your github)
-    # then create a robot user with write access user called "bcc_buildbot",
-    # and add the secret token for it to GitHub secrets as:
-    #   - QUAY_TOKEN = <token from quay.io>
-    name: Publish to quay.io
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        env:
-        - NAME: bionic-release
-          OS_RELEASE: 18.04
-        - NAME: focal-release
-          OS_RELEASE: 20.04
-    steps:
-
-    - uses: actions/checkout@v1
-
-    - name: Initialize workflow variables
-      id: vars
-      shell: bash
-      run: |
-          if [ -n "${QUAY_TOKEN}" ];then
-            echo "Quay token is set, will push an image"
-            echo ::set-output name=QUAY_PUBLISH::true
-          else
-            echo "Quay token not set, skipping"
-          fi
-
-      env:
-        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
-
-    - name: Authenticate with quay.io docker registry
-      if: >
-        steps.vars.outputs.QUAY_PUBLISH
-      env:
-        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
-      run: ./scripts/docker/auth.sh ${{ github.repository }}
-
-    - name: Package docker image and push to quay.io
-      if: >
-        steps.vars.outputs.QUAY_PUBLISH
-      run: >
-        ./scripts/docker/push.sh
-        ${{ github.repository }}
-        ${{ github.ref }}
-        ${{ github.sha }}
-        ${{ matrix.env['NAME'] }}
-        ${{ matrix.env['OS_RELEASE'] }}
-
-    # Uploads the packages built in docker to the github build as an artifact for convenience
-    - uses: actions/upload-artifact@v1
-      if: >
-        steps.vars.outputs.QUAY_PUBLISH
-      with:
-        name: ${{ matrix.env['NAME'] }}
-        path: output
+permissions:
+  contents: read # to fetch code (actions/checkout)
 
+jobs:
   # Optionally publish container images to custom docker repository,
   # guarded by presence of all required github secrets.
   # GitHub secrets can be configured as follows:

.github/workflows/bcc-test.yml

@@ -1,27 +1,63 @@
 name: BCC Build and tests
 
-on: push
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+  pull-requests: read # to read pull requests (dorny/paths-filter)
 
 jobs:
   test_bcc:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        os: [ubuntu-18.04] # 18.04.3 release has 5.0.0 kernel                                        
+        os: [{distro: "ubuntu", version: "18.04", nick: bionic}, {distro: "ubuntu", version: "20.04", nick: focal}]
         env:
         - TYPE: Debug
           PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: ON
+        - TYPE: Debug
+          PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: OFF
         - TYPE: Release
           PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: ON
     steps:
     - uses: actions/checkout@v2
+    - uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          docker:
+            - 'docker/build/**'
     - name: System info
       run: |
         uname -a
         ip addr
-    - name: Build docker container with all deps
+    - name: Pull docker container
+      if: steps.changes.outputs.docker == 'false'
+      run: |
+        docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }}
+    - name: Build docker container
+      if: steps.changes.outputs.docker == 'true'
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+    - name: Tag docker container
       run: |
-        docker build -t bcc-docker -f Dockerfile.tests .
+        docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }} bcc-docker
     - name: Run bcc build
       env: ${{ matrix.env }}
       run: |
@@ -36,7 +72,7 @@ jobs:
                    bcc-docker \
                    /bin/bash -c \
                    'mkdir -p /bcc/build && cd /bcc/build && \
-                    cmake -DCMAKE_BUILD_TYPE=${TYPE} .. && make -j9'"
+                    cmake -DCMAKE_BUILD_TYPE=${TYPE} -DENABLE_LLVM_NATIVECODEGEN=${RW_ENGINE_ENABLED} .. && make -j9'"
     - name: Run bcc's cc tests
       env: ${{ matrix.env }}
       # tests are wrapped with `script` as a hack to get a TTY as github actions doesn't provide this
@@ -84,9 +120,121 @@ jobs:
 
     - uses: actions/upload-artifact@v1
       with:
-        name: critical-tests-${{ matrix.env['TYPE'] }}-${{ matrix.os }}
+        name: critical-tests-${{ matrix.env['TYPE'] }}-${{ matrix.os.version }}
         path: tests/python/critical.log
 
+  test_bcc_fedora:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        os: [{distro: "fedora", version: "34", nick: "f34"}, {distro: "fedora", version: "36", nick: "f36"}]
+        env:
+        - TYPE: Debug
+          PYTHON_TEST_LOGFILE: critical.log
+        - TYPE: Release
+          PYTHON_TEST_LOGFILE: critical.log
+    steps:
+    - uses: actions/checkout@v2
+    - uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          docker:
+            - 'docker/build/**'
+    - name: System info
+      run: |
+        uname -a
+        ip addr
+    - name: Pull docker container
+      if: steps.changes.outputs.docker == 'false'
+      run: |
+        docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }}
+    - name: Build docker container
+      if: steps.changes.outputs.docker == 'true'
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+    - name: Tag docker container
+      run: |
+        docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }} bcc-docker
+    - name: Run bcc build
+      env: ${{ matrix.env }}
+      run: |
+        /bin/bash -c \
+                   "docker run --privileged \
+                   --pid=host \
+                   -v $(pwd):/bcc \
+                   -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                   -v /lib/modules:/lib/modules:ro \
+                   -v /usr/src:/usr/src:ro \
+                   -v /usr/include/linux:/usr/include/linux:ro \
+                   bcc-docker \
+                   /bin/bash -c \
+                   'mkdir -p /bcc/build && cd /bcc/build && \
+                    cmake -DCMAKE_BUILD_TYPE=${TYPE} -DENABLE_LLVM_SHARED=ON -DRUN_LUA_TESTS=OFF .. && make -j9'"
+    - name: Run libbpf-tools build
+      env: ${{ matrix.env }}
+      run: |
+        /bin/bash -c \
+                   "docker run --privileged \
+                   --pid=host \
+                   -v $(pwd):/bcc \
+                   -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                   -v /lib/modules:/lib/modules:ro \
+                   -v /usr/src:/usr/src:ro \
+                   -v /usr/include/linux:/usr/include/linux:ro \
+                   bcc-docker \
+                   /bin/bash -c \
+                   'cd /bcc/libbpf-tools && make -j9'"
+
+    - name: Run bcc's cc tests
+      env: ${{ matrix.env }}
+      # tests are wrapped with `script` as a hack to get a TTY as github actions doesn't provide this
+      # see https://github.com/actions/runner/issues/241
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    '/bcc/build/tests/wrapper.sh \
+                        c_test_all sudo /bcc/build/tests/cc/test_libbcc'"
+
+    - name: Run all tests
+      env: ${{ matrix.env }}
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    'cd /bcc/build && \
+                     make test PYTHON_TEST_LOGFILE=$PYTHON_TEST_LOGFILE ARGS=-V'"
+
+    - name: Check critical tests
+      env: ${{ matrix.env }}
+      run: |
+        critical_count=$(grep @mayFail tests/python/critical.log | wc -l)
+        echo "There were $critical_count critical tests skipped with @mayFail:"
+        grep -A2 @mayFail tests/python/critical.log
+
+
 # To debug weird issues, you can add this step to be able to SSH to the test environment
 #     https://github.com/marketplace/actions/debugging-with-tmate
 #    - name: Setup tmate session

.github/workflows/publish-build-containers.yml

@@ -0,0 +1,44 @@
+name: Publish Build Containers
+
+on:
+  # We want to update this image regularly and when updating master
+  schedule:
+    - cron: '00 18 * * *'
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - 'docker/build/**'
+
+permissions: {}
+
+jobs:
+
+  publish_ghcr:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+      packages: write # to push container
+    name: Publish To GitHub Container Registry
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [
+          {distro: "ubuntu", version: "18.04", nick: bionic},
+          {distro: "ubuntu", version: "20.04", nick: focal},
+          {distro: "fedora", version: "34", nick: "f34"},
+          {distro: "fedora", version: "36", nick: "f36"}
+        ]
+
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: Build and push
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+        push: true

.gitignore

@@ -16,3 +16,6 @@ examples/cgroupid/cgroupid
 # Output from docker builds
 scripts/docker/output/
 /output/
+
+# UAPI header generated for libbpf package-based builds
+src/cc/compat/linux/bpf.h

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "src/cc/libbpf"]
 	path = src/cc/libbpf
 	url = https://github.com/libbpf/libbpf.git
+[submodule "libbpf-tools/bpftool"]
+	path = libbpf-tools/bpftool
+	url = https://github.com/libbpf/bpftool
+[submodule "libbpf-tools/blazesym"]
+	path = libbpf-tools/blazesym
+	url = https://github.com/libbpf/blazesym