From d08de5af248decec0fe4c1927af94504bab42f74 Mon Sep 17 00:00:00 2001 From: garciagenrique Date: Fri, 17 Nov 2023 16:16:13 +0100 Subject: [PATCH] update repo with rucio noise cronjobs, the correct manifests and the list of working rses --- .../docker-build-rucio-client-test.yaml | 92 +++++----- containers/rucio-noise/rses.txt | 2 + .../flux/rucio-dev/rucio-dev-cronjobs.yaml | 161 ++++++++++++++---- 3 files changed, 172 insertions(+), 83 deletions(-) diff --git a/.github/workflows/docker-build-rucio-client-test.yaml b/.github/workflows/docker-build-rucio-client-test.yaml index bb9d03b..a31207c 100644 --- a/.github/workflows/docker-build-rucio-client-test.yaml +++ b/.github/workflows/docker-build-rucio-client-test.yaml @@ -1,53 +1,53 @@ -name: Docker auto build - VRE DEV +# name: Docker auto build - VRE DEV -on: - push: - path: - containers/** +# on: +# push: +# path: +# containers/** -env: - REGISTRY: ghcr.io +# env: +# REGISTRY: ghcr.io -jobs: - build-and-push: - runs-on: ubuntu-latest - strategy: - matrix: - context: [ - 'rucio-client-testing', - ] - fail-fast: false +# jobs: +# build-and-push: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# context: [ +# 'rucio-client-testing', +# ] +# fail-fast: false - steps: - - name: Checkout repository - uses: actions/checkout@v3 +# steps: +# - name: Checkout repository +# uses: actions/checkout@v3 - - name: Log in to the GH Container Registry - ${{ env.REGISTRY }} - id: login - uses: docker/login-action@v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} +# - name: Log in to the GH Container Registry - ${{ env.REGISTRY }} +# id: login +# uses: docker/login-action@v2.1.0 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata for ${{ matrix.context }} - id: meta - uses: docker/metadata-action@v4.3.0 - with: - flavor: | - latest=true - tags: | - type=sha - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.context }} +# - name: Extract metadata for ${{ matrix.context }} +# id: meta +# uses: docker/metadata-action@v4.3.0 +# with: +# flavor: | +# latest=true +# tags: | +# type=sha +# images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.context }} - - name: Build and push ${{ matrix.context }} - id: build-and-push - uses: docker/build-push-action@v4.0.0 - with: - context: ./containers/${{ matrix.context }} - file: containers/${{ matrix.context }}/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - TAG=32.0.0 +# - name: Build and push ${{ matrix.context }} +# id: build-and-push +# uses: docker/build-push-action@v4.0.0 +# with: +# context: ./containers/${{ matrix.context }} +# file: containers/${{ matrix.context }}/Dockerfile +# push: true +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} +# build-args: | +# TAG=32.0.0 diff --git a/containers/rucio-noise/rses.txt b/containers/rucio-noise/rses.txt index e69de29..1953147 100644 --- a/containers/rucio-noise/rses.txt +++ b/containers/rucio-noise/rses.txt @@ -0,0 +1,2 @@ +DEV-CERN-EOS +DEV-CESNET-S3 \ No newline at end of file diff --git a/infrastructure/cluster/flux/rucio-dev/rucio-dev-cronjobs.yaml b/infrastructure/cluster/flux/rucio-dev/rucio-dev-cronjobs.yaml index 24da3ae..6193268 100644 --- a/infrastructure/cluster/flux/rucio-dev/rucio-dev-cronjobs.yaml +++ b/infrastructure/cluster/flux/rucio-dev/rucio-dev-cronjobs.yaml @@ -7,6 +7,7 @@ spec: containers: - name: rucio-client-dev image: ghcr.io/vre-hub/rucio-client:sha-ae6ac2c + # To make this pod run, ssh into it and run `j2 /opt/user/rucio.cfg.j2 > /opt/rucio/etc/rucio.cfg` if `rucio whoami` is not working imagePullPolicy: Always env: - name: RUCIO_CFG_RUCIO_HOST @@ -36,42 +37,128 @@ spec: cpu: 100m memory: 50Mi --- -# apiVersion: batch/v1 -# kind: CronJob +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rucio-noise + namespace: rucio-vre +spec: + schedule: "0 */2 * * *" # Every 2 hours + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: rucio-noise + image: ghcr.io/vre-hub/rucio-noise:sha-e101d52 + env: + - name: RUCIO_CFG_CLIENT_RUCIO_HOST + value: "https://vre-dev-rucio.cern.ch" + - name: RUCIO_CFG_CLIENT_AUTH_HOST + value: "https://vre-dev-auth-rucio.cern.ch" + - name: RUCIO_CFG_CLIENT_ACCOUNT + value: "root" + - name: RUCIO_CFG_CLIENT_AUTH_TYPE + value: "userpass" + - name: RUCIO_CFG_CLIENT_USERNAME + valueFrom: + secretKeyRef: + name: rucio-root-account + key: root-username + - name: RUCIO_CFG_CLIENT_PASSWORD + valueFrom: + secretKeyRef: + name: rucio-root-account + key: root-password + # volumeMounts: + # - name: rucio-cfg + # mountPath: /opt/rucio/etc/ + # - name: prod-rucio-x509up + # mountPath: /tmp/ + tty: true + imagePullPolicy: Always + command: + - /bin/sh + - -c + - date; ls -l /etc/pki/tls/certs/; ls -l /tmp/; cd /opt/rucio/etc/; pwd; echo Hello from the DEV rucio-noise container; + export RUCIO_CONFIG=/opt/rucio/etc/rucio.cfg; echo Exported config; cat /opt/rucio/etc/rucio.cfg; rucio -vvv whoami; + cd /home; export FSIZE=10M; FILE_SIZE=${FSIZE} /bin/bash produce_noise.sh; echo "Rucio noise cronjob ${FSIZE} Done!" + # volumes: + # - name: rucio-cfg + # secret: + # secretName: escape-service-account + # defaultMode: 0400 + # - name: prod-rucio-x509up + # secret: + # secretName: prod-rucio-x509up +--- +# +# This container is missing all the packaged installed in the escape one - to be done when RUCIO creates the corresponding release. +# +# apiVersion: v1 +# kind: Pod # metadata: -# name: rucio-noise -# namespace: rucio-vre +# name: rucio-client-test-root-user +# namespace: rucio-dev # spec: -# schedule: "*/10 * * * *" -# concurrencyPolicy: Forbid -# successfulJobsHistoryLimit: 1 -# jobTemplate: -# spec: -# template: -# spec: -# restartPolicy: OnFailure -# containers: -# - name: rucio-noise -# image: ghcr.io/vre-hub/vre-rucio-noise:v1.0.0-rc0-10-be00405 -# volumeMounts: -# - name: rucio-cfg -# mountPath: /opt/rucio/etc/ -# - name: prod-rucio-x509up -# mountPath: /tmp/ -# tty: true -# imagePullPolicy: Always -# command: -# - /bin/sh -# - -c -# - date; ls -l /etc/pki/tls/certs/; ls -l /tmp/; cd /opt/rucio/etc/; pwd; echo Hello from rucio-noise container; -# export RUCIO_CONFIG=/opt/rucio/etc/rucio.cfg; echo Exported config; cat /opt/rucio/etc/rucio.cfg; rucio -vvv whoami; -# cd /home; export FSIZE=10M; FILE_SIZE=${FSIZE} /bin/bash produce_noise.sh; echo "Rucio noise cronjob ${FSIZE} Done!" -# volumes: -# - name: rucio-cfg -# secret: -# secretName: escape-service-account -# defaultMode: 0400 -# - name: prod-rucio-x509up -# secret: -# secretName: prod-rucio-x509up -#--- +# containers: +# - name: rucio-client-test-root-user +# image: ghcr.io/vre-hub/rucio-client-testing:latest +# env: +# - name: RUCIO_CFG_CLIENT_RUCIO_HOST +# value: "https://vre-dev-rucio.cern.ch" +# - name: RUCIO_CFG_CLIENT_AUTH_HOST +# value: "https://vre-dev-auth-rucio.cern.ch" +# - name: RUCIO_CFG_CLIENT_ACCOUNT +# value: "root" +# - name: RUCIO_CFG_CLIENT_AUTH_TYPE +# value: "userpass" +# - name: RUCIO_CFG_CLIENT_USERNAME +# valueFrom: +# secretKeyRef: +# name: rucio-root-account +# key: root-username +# - name: RUCIO_CFG_CLIENT_PASSWORD +# valueFrom: +# secretKeyRef: +# name: rucio-root-account +# key: root-password + # From here i'm just testing that the VARs are acorreclty passed to the image. + # - name: RUCIO_CFG_CLIENT_CA_CERT + # value: "/etc/pki/tls/certs/CERN-bundle.pem" + # - name: RUCIO_CFG_CLIENT_CLIENT_CERT + # value: "/opt/rucio/etc/usercert.pem" + # - name: RUCIO_CFG_CLIENT_CLIENT_KEY + # value: "/opt/rucio/etc/userkey.pem" + # - name: RUCIO_CFG_CLIENT_REQUEST_RETRIES + # value: "3" + # - name: RUCIO_CFG_CLIENT_VO + # value: "escape" + # - name: RUCIO_CFG_CLIENT_X509_PROXY + # value: "/tmp/x509up_u0" + # - name: RUCIO_CFG_COMMON_MULTI_VO + # value: "test_value" + # - name: RUCIO_CFG_POLICY_LFN2PFN_ALGORITHM_DEFAULT + # value: "hash" + # - name: RUCIO_CFG_POLICY_PACKAGE + # value: "escape" + # - name: RUCIO_CFG_POLICY_PERMISSION + # value: "escape" + # - name: RUCIO_CFG_POLICY_SCHEMA + # value: "escape" + # - name: RUCIO_CFG_POLICY_SUPPORT + # value: "https://github.com/rucio/rucio/issues/" + # - name: RUCIO_CFG_POLICY_SUPPORT_RUCIO + # value: "https://github.com/rucio/rucio/issues/" +# command: ["sleep","3600"] +# resources: +# limits: +# cpu: 100m +# memory: 50Mi +# requests: +# cpu: 100m +# memory: 50Mi +# --- \ No newline at end of file