[allow_skip_clone] Allow skipping of the cloning stages (#512)

* [allow_skip_clone] Use <repo>_dir instead of hardcoding all directories relative to base_dir

This will allow configuration of the repo clone destination, so we can
use pre-cloned dirs instead of explicitly cloning the dirs each time.

This is essential for CI systems like zuul, that set-up the repos with
particular versions/branches prior to running the test scripts.

* [zuul] List the other infrawatch repos as required for the job

* [zuul] Set the {sgo,sg-bridge,sg-core,prometheus-webhook-snmp}_dir vars

Add in the repo dir locations where the repos should be pre-cloned by
zuul

* Replace base_dir with sto_dir

* set sto_dir relative to base_dir is it isn't already set

* [ci] use absolute dir for requirements.txt

* [ci] Update sto_dir using explicit reference

zuul.project.src_dir refers to the current project dir. When using the jobs
in another infrawatch project, this becomes invalid.
Instead, sto_dir is explicitly set using
zuul.projects[<project_name>].src_dir, the same way that the other repo dirs
are set in vars-zuul-common

---------

Co-authored-by: Chris Sibbitt <[email protected]>

.zuul.yaml

@@ -12,6 +12,11 @@
     required-projects:
       - name: openstack-k8s-operators/ci-framework
         override-checkout: main
+      - name: github.com/infrawatch/smart-gateway-operator
+      - name: github.com/infrawatch/sg-bridge
+      - name: github.com/infrawatch/sg-core
+      - name: github.com/infrawatch/prometheus-webhook-snmp
+
     pre-run:
       - ci/prepare.yml
     run:

build/stf-run-ci/defaults/main.yml

@@ -67,5 +67,6 @@ prometheus_webhook_snmp_repository: https://github.com/infrawatch/prometheus-web
 
 base_dir: ''
 
+clone_repos: true
 setup_bundle_registry_auth: true
 setup_bundle_registry_tls_ca: true

build/stf-run-ci/tasks/clone_repos.yml

@@ -3,60 +3,84 @@
 # NOTE: since you can't loop against blocks (and we're using them for failure #
 #       recovery when the request branch doesn't exist) we have to define each
 #       of these separately rather than using a loop.
+
+- name: Check if the {{ sgo_dir }} already exists
+  ansible.builtin.stat:
+    path: "{{ sgo_dir }}"
+  register: check_sgo_dir
+
 - name: Get Smart Gateway Operator
+  when: not check_sgo_dir.stat.exists
   block:
     - name: Try cloning same-named branch or override branch from SGO repository
       ansible.builtin.git:
         repo: "{{ sgo_repository }}"
-        dest: "{{ base_dir }}/working/smart-gateway-operator"
-        version: "{{ sgo_branch | default(branch, true) }}"
-        force: true
+        dest: "{{ sgo_dir }}"
+        version: "{{ version_branches.sgo | default(branch, true) }}"
   rescue:
     - name: "Get {{ version_branches.sgo }} upstream branch because specified branch or repository doesn't exist"
       ansible.builtin.git:
         repo: https://github.com/infrawatch/smart-gateway-operator
-        dest: "{{ base_dir }}/working/smart-gateway-operator"
+        dest: "{{ sgo_dir }}"
         version: "{{ version_branches.sgo }}"
 
+- name: Check if the {{ sg_core_dir }} already exists
+  ansible.builtin.stat:
+    path: "{{ sg_core_dir }}"
+  register: check_sg_core_dir
+
 - name: Get sg-core
+  when: not check_sg_core_dir.stat.exists
   block:
     - name: Try cloning same-named branch or override branch from sg-core repository
       ansible.builtin.git:
         repo: "{{ sg_core_repository }}"
-        dest: "{{ base_dir }}/working/sg-core"
-        version: "{{ sg_core_branch | default(branch, true) }}"
+        dest: "{{ sg_core_dir }}"
+        version: "{{ version_branches.sg_core | default(branch, true) }}"
   rescue:
     - name: "Get {{ version_branches.sg_core }} upstream branch because specified branch or repository doesn't exist"
       ansible.builtin.git:
         repo: https://github.com/infrawatch/sg-core
-        dest: "{{ base_dir }}/working/sg-core"
+        dest: "{{ sg_core_dir }}"
         version: "{{ version_branches.sg_core }}"
 
+- name: Check if the {{ sg_bridge_dir }} already exists
+  ansible.builtin.stat:
+    path: "{{ sg_bridge_dir }}"
+  register: check_sg_bridge_dir
+
 - name: Get sg-bridge
+  when: not check_sg_bridge_dir.stat.exists
   block:
     - name: Try cloning same-named branch or override branch from sg-bridge repository
       ansible.builtin.git:
         repo: "{{ sg_bridge_repository }}"
-        dest: "{{ base_dir }}/working/sg-bridge"
-        version: "{{ sg_bridge_branch | default(branch, true) }}"
+        dest: "{{ sg_bridge_dir }}"
+        version: "{{ version_branches.sg_bridge | default(branch, true) }}"
   rescue:
     - name: "Get {{ version_branches.sg_bridge }} upstream branch because specified branch or repository doesn't exist"
       ansible.builtin.git:
         repo: https://github.com/infrawatch/sg-bridge
-        dest: "{{ base_dir }}/working/sg-bridge"
+        dest: "{{ sg_bridge_dir }}"
         version: "{{ version_branches.sg_bridge }}"
 
+- name: Check if the {{ prometheus_webhook_snmp_dir }} already exists
+  ansible.builtin.stat:
+    path: "{{ prometheus_webhook_snmp_dir }}"
+  register: check_prometheus_webhook_snmp_dir
+
 - name: Get prometheus-webhook-snmp
+  when: not check_prometheus_webhook_snmp_dir.stat.exists
   block:
     - name: Try cloning same-named branch or override branch from prometheus-webhook-snmp repository
       ansible.builtin.git:
         repo: "{{ prometheus_webhook_snmp_repository }}"
-        dest: "{{ base_dir }}/working/prometheus-webhook-snmp"
-        version: "{{ prometheus_webhook_snmp_branch | default(branch, true) }}"
+        dest: "{{ prometheus_webhook_snmp_dir }}"
+        version: "{{ version_branches.prometheus_webhook_snmp | default(branch, true) }}"
   rescue:
     - name: "Get {{ version_branches.prometheus_webhook_snmp }} upstream branch because specified branch or repository doesn't exist"
       ansible.builtin.git:
         repo: https://github.com/infrawatch/prometheus-webhook-snmp
-        dest: "{{ base_dir }}/working/prometheus-webhook-snmp"
+        dest: "{{ prometheus_webhook_snmp_dir }}"
         version: "{{ version_branches.prometheus_webhook_snmp }}"
 

build/stf-run-ci/tasks/main.yml

@@ -4,6 +4,7 @@
 # -- initial setup
 - name: Setup default values
   ansible.builtin.set_fact:
+    # The branch should be removed, we should assume that everything is checked out to the right place.
     branch: "{{ working_branch | default('master') }}"
     namespace: "{{ namespace if namespace is defined else (working_namespace | default('service-telemetry'))}}"
 
@@ -68,6 +69,14 @@
     base_dir: "{{ playbook_dir }}"
   when: base_dir | length == 0
 
+- name: Set the repo destination dirs, if not provided
+  ansible.builtin.set_fact:
+    sto_dir: "{{ sto_dir if sto_dir is defined else base_dir + '/..' }}"
+    sgo_dir: "{{ sgo_dir if sgo_dir is defined else base_dir + '/working/smart-gateway-operator' }}"
+    sg_core_dir: "{{ sg_core_dir if sg_core_dir is defined else base_dir + '/working/sg-core' }}"
+    sg_bridge_dir: "{{ sg_bridge_dir if sg_bridge_dir is defined else base_dir + '/working/sg-bridge'}}"
+    prometheus_webhook_snmp_dir: "{{ prometheus_webhook_snmp_dir if prometheus_webhook_snmp_dir is defined else base_dir + '/working/prometheus-webhook-snmp' }}"
+
 - name: Get operator_sdk_v0 (build bundles)
   ansible.builtin.command:
     cmd: "./get_operator_sdk.sh {{ operator_sdk_v0 }}"
@@ -97,18 +106,19 @@
     - create_builds
   block:
   - name: Setup supporting repositories
+    when: clone_repos | bool
     ansible.builtin.include_tasks: clone_repos.yml
     tags:
       - clone
 
   - name: Create base build list
     ansible.builtin.set_fact:
       build_list:
-        - {name: service-telemetry-operator, dockerfile_path: build/Dockerfile, image_reference_name: sto_image_path, working_build_dir: "{{ base_dir }}/../"}
-        - {name: smart-gateway-operator, dockerfile_path: build/Dockerfile, image_reference_name: sgo_image_path, working_build_dir: "{{ base_dir }}/working/smart-gateway-operator"}
-        - {name: sg-core, dockerfile_path: build/Dockerfile, image_reference_name: sg_core_image_path, working_build_dir: "{{ base_dir }}/working/sg-core"}
-        - {name: sg-bridge, dockerfile_path: build/Dockerfile, image_reference_name: sg_bridge_image_path, working_build_dir: "{{ base_dir }}/working/sg-bridge"}
-        - {name: prometheus-webhook-snmp, dockerfile_path: Dockerfile, image_reference_name: prometheus_webhook_snmp_image_path, working_build_dir: "{{ base_dir }}/working/prometheus-webhook-snmp"}
+        - {name: service-telemetry-operator, dockerfile_path: build/Dockerfile, image_reference_name: sto_image_path, working_build_dir: "{{ sto_dir }}"}
+        - {name: smart-gateway-operator, dockerfile_path: build/Dockerfile, image_reference_name: sgo_image_path, working_build_dir: "{{ sgo_dir }}"}
+        - {name: sg-core, dockerfile_path: build/Dockerfile, image_reference_name: sg_core_image_path, working_build_dir: "{{ sg_core_dir }}"}
+        - {name: sg-bridge, dockerfile_path: build/Dockerfile, image_reference_name: sg_bridge_image_path, working_build_dir: "{{ sg_bridge_dir }}"}
+        - {name: prometheus-webhook-snmp, dockerfile_path: Dockerfile, image_reference_name: prometheus_webhook_snmp_image_path, working_build_dir: "{{ prometheus_webhook_snmp_dir }}"}
 
   - ansible.builtin.debug:
       var: build_list

build/stf-run-ci/tasks/setup_stf_local_build.yml

@@ -6,7 +6,7 @@
 - block:
   - name: Generate Smart Gateway Operator CSV
     ansible.builtin.shell:
-      chdir: "{{ base_dir }}/working/smart-gateway-operator/build"
+      chdir: "{{ sgo_dir }}/build"
       cmd: |
         LOGFILE="{{ logfile_dir }}/sgo_gen_bundle.log" \
         OPERATOR_SDK="{{ base_dir }}/working/operator-sdk-{{ operator_sdk_v0 }}" \
@@ -28,7 +28,7 @@
 
 - name: Replace namespace in SGO role binding
   ansible.builtin.replace:
-    path: "{{ base_dir }}/working/smart-gateway-operator/deploy/role_binding.yaml"
+    path: "{{ sgo_dir }}/deploy/role_binding.yaml"
     regexp: 'placeholder'
     replace: '{{ namespace }}'
 
@@ -42,7 +42,8 @@
   block:
   - name: Load Smart Gateway Operator RBAC
     ansible.builtin.command:
-      cmd: oc apply -f {{ base_dir }}/working/smart-gateway-operator/deploy/{{ item }} -n "{{ namespace }}"
+      cmd: oc apply -f ./deploy/{{ item }} -n "{{ namespace }}"
+      chdir: "{{ sgo_dir }}"
     loop:
       - service_account.yaml
       - role.yaml
@@ -57,7 +58,7 @@
 - block:
   - name: Generate Service Telemetry Operator CSV
     ansible.builtin.shell:
-      chdir: "{{ base_dir }}"
+      chdir: "{{ sto_dir }}/build"
       cmd: |
         LOGFILE="{{ logfile_dir }}/sto_gen_bundle.log" \
         OPERATOR_SDK="{{ base_dir }}/working/operator-sdk-{{ operator_sdk_v0 }}" \
@@ -76,7 +77,7 @@
 
 - name: Replace namespace in STO role binding
   ansible.builtin.replace:
-    path: "{{ base_dir }}/../deploy/role_binding.yaml"
+    path: "{{ sto_dir }}/deploy/role_binding.yaml"
     regexp: 'placeholder'
     replace: '{{ namespace }}'
 
@@ -90,8 +91,8 @@
   block:
   - name: Load Service Telemetry Operator RBAC
     ansible.builtin.command:
-      cmd: oc apply -f ../deploy/{{ item }} -n "{{ namespace }}"
-      chdir: "{{ base_dir }}"
+      cmd: oc apply -f ./deploy/{{ item }} -n "{{ namespace }}"
+      chdir: "{{ sto_dir }}"
     loop:
       - service_account.yaml
       - role.yaml
@@ -105,5 +106,5 @@
 # cleanup
 - name: Revert local change to role_binding.yaml
   ansible.builtin.shell:
-    cmd: git checkout -- "{{ base_dir }}/../deploy/role_binding.yaml"
-    chdir: "{{ base_dir }}"
+    cmd: git checkout -- deploy/role_binding.yaml
+    chdir: "{{ sto_dir }}"

ci/deploy_stf.yml

@@ -4,7 +4,7 @@
   tasks:
     - name: "Set the sto_dir if it isn't already set"
       ansible.builtin.set_fact:
-        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.projects["github.com/infrawatch/service-telemetry-operator"].src_dir }}'
       when: sto_dir | default('') | length == 0
 
     - name: "Get vars common to all jobs"

ci/post-collect_logs.yml

@@ -16,7 +16,7 @@
   tasks:
     - name: "Set the sto_dir if it isn't already set"
       ansible.builtin.set_fact:
-        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.projects["github.com/infrawatch/service-telemetry-operator"].src_dir }}'
       when: sto_dir | default('') | length == 0
 
     - name: "Get vars common to all jobs"

ci/prepare.yml

@@ -4,7 +4,7 @@
   tasks:
     - name: "Set the sto_dir if it isn't already set"
       ansible.builtin.set_fact:
-        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.projects["github.com/infrawatch/service-telemetry-operator"].src_dir }}'
       when: sto_dir | default('') | length == 0
 
     - name: "Get vars common to all jobs"
@@ -23,7 +23,7 @@
 
     - name: "Install pre-reqs from pip"
       ansible.builtin.pip:
-        requirements: "build/stf-run-ci/requirements.txt"
+        requirements: "{{ sto_dir }}/build/stf-run-ci/requirements.txt"
         chdir: "{{ sto_dir }}"
         state: present
 

ci/test_stf.yml

@@ -4,7 +4,7 @@
   tasks:
     - name: "Set the sto_dir if it isn't already set"
       ansible.builtin.set_fact:
-        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}'
+        sto_dir: '{{ ansible_env.HOME }}/{{ zuul.projects["github.com/infrawatch/service-telemetry-operator"].src_dir }}'
       when: sto_dir | default('') | length == 0
 
     - name: "Get vars common to all jobs"

ci/vars-zuul-common.yml

@@ -5,3 +5,8 @@ setup_bundle_registry_auth: false
 __service_telemetry_transports_qdr_auth: none
 base_dir: "{{ sto_dir }}/build"
 logfile_dir: "{{ ansible_user_dir }}/zuul-output/logs/controller"
+clone_repos: false
+sgo_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/smart-gateway-operator'].src_dir }}"
+sg_core_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/sg-core'].src_dir }}"
+sg_bridge_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/sg-bridge'].src_dir }}"
+prometheus_webhook_snmp_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/prometheus-webhook-snmp'].src_dir }}"