Merge branch 'master' into dependabot/pip/build/stf-run-ci/requests-2…

….31.0

.github/workflows/main.yml

@@ -38,6 +38,31 @@ jobs:
         run: ${HOME}/.local/bin/ansible-lint .
         working-directory: ./build/stf-run-ci
 
+  generate-bundle-diff-check:
+    name: Check if generating the bundle would result in local changes
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: v0.19.4
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get operator-sdk image 0.19.4
+        run: curl --output operator-sdk -JL https://github.com/operator-framework/operator-sdk/releases/download/$RELEASE_VERSION/operator-sdk-$RELEASE_VERSION-x86_64-linux-gnu
+
+      - name: Make operator-sdk executable
+        run: chmod +x operator-sdk
+
+      - name: Move operator-sdk binary
+        run: sudo mv operator-sdk /usr/local/bin
+
+      - name: Generate bundle locally
+        run: operator-sdk generate bundle --manifests --metadata --default-channel unstable --channels unstable
+
+      - name: Check if bundle generation results in local changes
+        run: git diff --exit-code
+
   build-operator-check:
     name: Build Operator check
     runs-on: ubuntu-20.04

.zuul.yaml

@@ -1,6 +1,41 @@
 ---
+- job:
+    name: stf-base
+    # defined in: https://review.rdoproject.org/cgit/config/tree/zuul.d/_jobs-crc.yaml
+    parent: base-simple-crc
+    abstract: true
+    description: |
+      Run the stf-run-ci role, and then test stf
+    roles:  # adds in dependent roles i.e. put it in the role path
+      - zuul: github.com/openstack-k8s-operators/ci-framework
+    # These are the additional repos that zuul will clone
+    required-projects:
+      - name: openstack-k8s-operators/ci-framework
+        override-checkout: main
+    pre-run:
+      - ci/prepare.yml
+    run:
+      - ci/deploy_stf.yml
+      - ci/test_stf.yml
+    post-run:
+      - ci/post-collect_logs.yml
+    nodeset: centos-9-crc-xxl
+    # The default (~30 minutes) is not enough to run through all the job stages
+    timeout: 3600
+    vars:
+      # Pass vars to crc cli https://review.rdoproject.org/cgit/config/tree/playbooks/crc/simple-start.yaml#n30
+      crc_parameters: '--memory 16000 --disk-size 80 --cpus 6'  # Increase from 14336
+
+- job:
+    name: stf-crc-latest-local_build
+    parent: stf-base
+    description: |
+      Build images locally and deploy STF
+    vars:
+      scenario: "local_build"
+
 - project:
     name: infrawatch/service-telemetry-operator
     github-check:
       jobs:
-        - noop
+        - stf-crc-latest-local_build

Jenkinsfile

@@ -36,6 +36,7 @@ spec:
           strategy: ephemeral
   transports:
     qdr:
+      auth: none
       enabled: true
       deploymentSize: 1
       web:

build/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/openshift/origin-ansible-operator:4.10
+FROM quay.io/openshift/origin-ansible-operator:4.12
 
 USER 0
 # Upstream CI builds need the additional EPEL sources for python3-passlib and python3-bcrypt but have no working repos to install epel-release

build/stf-run-ci/README.md

@@ -51,8 +51,8 @@ choose to override:
 | `__service_telemetry_trap_oid_prefix`                                | <oid_prefix>                                                | 1.3.6.1.4.1.50495.15                                  | The OID prefix for trap variable bindings.                                                                                                |
 | `__service_telemetry_trap_default_oid`                               | <default_oid>                                               | 1.3.6.1.4.1.50495.15.1.2.1                            | The trap OID if none is found in the Prometheus alert labels.                                                                             |
 | `__service_telemetry_trap_default_severity`                          | <default_severity>                                          | <undefined>                                           | The trap severity if none is found in the Prometheus alert labels.                                                                        |
-| `__service_telemetry_logs_enabled`                                   | {true,false}                                                | false                                                 | Whether to enable logs support in ServiceTelemetry                                                                                        |
 | `__service_telemetry_observability_strategy`                         | <observability_strategy>                                    | `use_redhat`                                          | Which observability strategy to use for deployment. Default is 'use_redhat'. Also supported are 'use_hybrid', 'use_community', and 'none' |
+| `__service_telemetry_transports_qdr_auth`                            | {'none', 'basic'}                                           | `none`                                                | Which auth method to use for QDR. Can be 'none' or 'basic'. Note: 'basic' is not yet supported in smoketests.                             |
 | `__service_telemetry_transports_certificates_endpoint_cert_duration` | [ParseDuration](https://golang.org/pkg/time/#ParseDuration) | 70080h                                                | Lifetime of the QDR endpoint certificate (minimum duration is 1h)                                                                         |
 | `__service_telemetry_transports_certificates_ca_cert_duration`       | [ParseDuration](https://golang.org/pkg/time/#ParseDuration) | 70080h                                                | Lifetime of the QDR CA certificate (minimum duration is 1h)                                                                               |
 | `__internal_registry_path`                                           | <registry_path>                                             | image-registry.openshift-image-registry.svc:5000      | Path to internal registry for image path                                                                                                  |

build/stf-run-ci/defaults/main.yml

@@ -26,7 +26,6 @@ __service_telemetry_snmptraps_alert_oid_label: "oid"
 __service_telemetry_snmptraps_trap_oid_prefix: "1.3.6.1.4.1.50495.15"
 __service_telemetry_snmptraps_trap_default_oid: "1.3.6.1.4.1.50495.15.1.2.1"
 __service_telemetry_snmptraps_trap_default_severity: ""
-__service_telemetry_logs_enabled: false
 __service_telemetry_observability_strategy: use_redhat
 __service_telemetry_transports_certificates_endpoint_cert_duration: 70080h
 __service_telemetry_transports_certificates_ca_cert_duration: 70080h

build/stf-run-ci/tasks/deploy_stf.yml

@@ -43,19 +43,15 @@
                 persistent:
                   storageClass: {{ __service_telemetry_storage_persistent_storage_class }}
       {% endif %}
-          logs:
-            loki:
-              enabled: false
-              replicationFactor: 1
-              flavor: 1x.extra-small
-              storage:
-                objectStorageSecret: test
       {% if __service_telemetry_storage_persistent_storage_class is defined %}
                 storageClass: {{ __service_telemetry_storage_persistent_storage_class }}
       {% endif %}
         transports:
           qdr:
             enabled: true
+      {% if __service_telemetry_transports_qdr_auth is defined %}
+            auth: "{{ __service_telemetry_transports_qdr_auth }}"
+      {% endif %}
             certificates:
               endpointCertDuration: {{ __service_telemetry_transports_certificates_endpoint_cert_duration }}
               caCertDuration: {{ __service_telemetry_transports_certificates_ca_cert_duration }}

build/stf-run-ci/tasks/main.yml

@@ -5,7 +5,7 @@
 - name: Setup default values
   ansible.builtin.set_fact:
     branch: "{{ working_branch | default('master') }}"
-    namespace: "{{ working_namespace | default('service-telemetry') }}"
+    namespace: "{{ namespace if namespace is defined else (working_namespace | default('service-telemetry'))}}"
 
 - name: Set default image paths for local builds
   ansible.builtin.set_fact:
@@ -205,7 +205,7 @@
 
   - name: Validate system is operational
     ansible.builtin.shell: |
-      OCP_PROJECT="{{ namespace }}" VALIDATION_SCOPE="{{ __service_telemetry_observability_strategy }}" timeout 600 "{{ base_dir }}/validate_deployment.sh" >> {{ logfile_dir }}/validate_deployment.log 2>&1
+      OCP_PROJECT="{{ namespace }}" VALIDATION_SCOPE="{{ __service_telemetry_observability_strategy }}" timeout 1200 "{{ base_dir }}/validate_deployment.sh" >> {{ logfile_dir }}/validate_deployment.log 2>&1
     args:
       executable: /bin/bash
     register: validate_deployment

ci/deploy_stf.yml

@@ -0,0 +1,25 @@
+---
+- name: "Deploy STF"
+  hosts: controller
+  tasks:
+    - name: "Set the sto_dir if it isn't already set"
+      ansible.builtin.set_fact:
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+      when: sto_dir | default('') | length == 0
+
+    - name: "Get vars common to all jobs"
+      ansible.builtin.include_vars:
+        file: "vars-zuul-common.yml"
+
+    - name: "Get scenario-specific vars"
+      ansible.builtin.include_vars:
+        file: "vars-{{ scenario }}.yml"
+
+    - name: "Log into the cluster"
+      ansible.builtin.import_role:
+        name: rhol_crc
+        tasks_from: add_crc_creds.yml
+
+    - name: "Deploy STF using stf-run-ci"
+      ansible.builtin.import_role:
+        name: '../build/stf-run-ci'

ci/post-collect_logs.yml

@@ -0,0 +1,112 @@
+---
+# Based on https://raw.githubusercontent.com/openstack-k8s-operators/nova-operator/bc10c4f579f8538899ac7bc5f87bfdb62d7042a4/ci/nova-operator-base/playbooks/collect-logs.yaml
+- hosts: all
+  name: Create zuul-output log dir
+  gather_facts: false
+  tasks:
+    - name: Create log dir
+      ansible.builtin.file:
+        path: "{{ ansible_user_dir }}/zuul-output/logs"
+        state: directory
+        mode: "0755"
+
+- hosts: controller
+  name: Collect logs on the controller
+  gather_facts: false
+  tasks:
+    - name: "Set the sto_dir if it isn't already set"
+      ansible.builtin.set_fact:
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+      when: sto_dir | default('') | length == 0
+
+    - name: "Get vars common to all jobs"
+      ansible.builtin.include_vars:
+        file: "vars-zuul-common.yml"
+
+    - name: "Get scenario-specific vars"
+      ansible.builtin.include_vars:
+        file: "vars-{{ scenario }}.yml"
+
+    - name: "Create log dir"
+      ansible.builtin.file:
+        path: "{{ logfile_dir }}"
+        state: directory
+        mode: "0755"
+
+    - name: "Log into the cluster"
+      ansible.builtin.import_role:
+        name: rhol_crc
+        tasks_from: add_crc_creds.yml
+
+    - name: "Get builds"
+      ansible.builtin.shell:
+        cmd: |
+          echo "*** [INFO] Showing oc get builds" > {{ logfile_dir }}/post_oc_get_builds.log 2>&1
+          oc get builds -oyaml >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1
+          echo "*** [INFO] Showing oc get builds -oyaml" >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1
+          oc get builds -oyaml >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1
+          cat {{ logfile_dir }}/post_oc_get_builds.log
+      ignore_errors: true
+      changed_when: false
+
+    - name: "Get subscription details"
+      ansible.builtin.shell:
+        cmd: |
+          oc get subscriptions > {{ logfile_dir }}/post_oc_get_subscriptions.log 2>&1
+          oc describe subscription service-telemetry-operator >> {{ logfile_dir }}/post_oc_get_subscriptions.log 2>&1
+          cat {{ logfile_dir}}/post_oc_get_subscriptions.log
+      ignore_errors: true
+
+    - name: "Get image infos"
+      ansible.builtin.shell:
+        cmd: |
+          echo "[INFO] oc get images" > {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          oc get images >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          echo "[INFO] oc get imagestreams" >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          oc get imagestream >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          #echo "[INFO] oc get images -oyaml" >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          #oc get images -oyaml >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          echo "[INFO] oc get imagestream -oyaml" >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          oc get imagestream -oyaml >> {{ logfile_dir }}/post_oc_get_images.log 2>&1
+          cat {{ logfile_dir }}/post_oc_get_images.log 2>&1
+      register: post_oc_get_images
+      ignore_errors: true
+      retries: 3
+      delay: 10
+
+    - name: "Get STO info"
+      ansible.builtin.shell:
+        cmd: |
+          oc describe pod $(oc get pod -l name=service-telemetry-operator -ojsonpath='{ .items[].metadata.name }') >> {{ logfile_dir }}/describe_sto.log 2>&1
+      ignore_errors: true
+      retries: 3
+      delay: 10
+
+    - name: "Question the deployment"
+      ansible.builtin.shell:
+        cmd: |
+          echo "What images were created in the internal registry?" > {{ logfile_dir }}/post_question_deployment.log 2>&1
+          oc get images | grep $(oc registry info --internal) >> {{ logfile_dir }}/post_question_deployment.log 2>&1
+          echo "What state is the STO csv in?" >> {{ logfile_dir }}/post_question_deployment.log 2>&1
+          oc get csv -n {{ namespace }} | grep service-telemetry-operator >> {{ logfile_dir }}/post_question_deployment.log 2>&1
+          oc get csv -n {{ namespace }} service-telemetry-operator -oyaml >> {{ logfile_dir }}/post_question_deployment.log 2>&1
+      register: output
+      ignore_errors: true
+      retries: 3
+      delay: 10
+
+    - name: "Copy generated logs"
+      ansible.builtin.shell: |
+        cp {{ ansible_env.HOME }}/*.log .
+      args:
+        chdir: "{{ ansible_user_dir }}/zuul-output/logs/controller"
+      changed_when: true
+      ignore_errors: true
+
+- hosts: all
+  name: Copy files from controller on node
+  gather_facts: false
+  tasks:
+    - name: Copy files from controller on node
+      ansible.builtin.include_role:
+        name: fetch-output

ci/prepare.yml

@@ -0,0 +1,50 @@
+---
+- name: "Prepare the environment for running stf"
+  hosts: controller
+  tasks:
+    - name: "Set the sto_dir if it isn't already set"
+      ansible.builtin.set_fact:
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+      when: sto_dir | default('') | length == 0
+
+    - name: "Get vars common to all jobs"
+      ansible.builtin.include_vars:
+        file: "vars-zuul-common.yml"
+
+    - name: "Get scenario-specific vars"
+      ansible.builtin.include_vars:
+        file: "vars-{{ scenario }}.yml"
+
+    - name: "Update pip"
+      ansible.builtin.pip:
+        name: pip
+        state: latest
+        extra_args: "-U"
+
+    - name: "Install pre-reqs from pip"
+      ansible.builtin.pip:
+        requirements: "build/stf-run-ci/requirements.txt"
+        chdir: "{{ sto_dir }}"
+        state: present
+
+    - name: "Install ansible collections"
+      community.general.ansible_galaxy_install:
+        type: collection
+        name: "{{ item }}"
+      with_items:
+        - "kubernetes.core:2.3.2"
+        - "community.general:6.2.0"
+
+    - name: "Log into the cluster"
+      ansible.builtin.import_role:
+        name: rhol_crc
+        tasks_from: add_crc_creds.yml
+
+    - name: "Create the service-telemetry project"
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: "{{ namespace }}"
+        state: present
+      retries: 3
+      delay: 30

ci/test_stf.yml

@@ -0,0 +1,28 @@
+---
+- name: "Run tests to verify that STF runs as expected"
+  hosts: controller
+  tasks:
+    - name: "Set the sto_dir if it isn't already set"
+      ansible.builtin.set_fact:
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+      when: sto_dir | default('') | length == 0
+
+    - name: "Get vars common to all jobs"
+      ansible.builtin.include_vars:
+        file: "vars-zuul-common.yml"
+
+    - name: "Get scenario-specific vars"
+      ansible.builtin.include_vars:
+        file: "vars-{{ scenario }}.yml"
+
+    - name: "Log into the cluster"
+      ansible.builtin.import_role:
+        name: rhol_crc
+        tasks_from: add_crc_creds.yml
+
+    - name: "Run STF smoketests"
+      ansible.builtin.shell:
+        cmd: |
+          OCP_PROJECT={{ namespace }} CLEANUP=false ./tests/smoketest/smoketest.sh > {{ logfile_dir }}/smoketest.log 2>&1
+        chdir: "{{ sto_dir }}"
+      changed_when: false

ci/vars-local_build.yml

@@ -0,0 +1,5 @@
+---
+__deploy_stf: true
+__local_build_enabled: true
+__service_telemetry_snmptraps_enabled: true
+__service_telemetry_storage_ephemeral_enabled: true

ci/vars-zuul-common.yml

@@ -0,0 +1,7 @@
+---
+namespace: "service-telemetry"
+setup_bundle_registry_tls_ca: false
+setup_bundle_registry_auth: false
+__service_telemetry_transports_qdr_auth: none
+base_dir: "{{ sto_dir }}/build"
+logfile_dir: "{{ ansible_user_dir }}/zuul-output/logs/controller"