Skip to content

Commit

Permalink
Enable remote tasks to be run in cluster
Browse files Browse the repository at this point in the history 
By default, we should run builds matching the local architecture
in-cluster to reduce the overhead of provisioning platforms. This will
enable a fully matrixed build for all images using only the remote
builds. This change will require the multi-platform controller to set
the /ssh/host to localhost in order for the builds to run in-cluster.

Signed-off-by: arewm <[email protected]>
  • Loading branch information
@arewm
arewm committed Aug 13, 2024
1 parent 3984e5f commit fc9c1d0
Show file tree
Hide file tree
Showing 5 changed files with 380 additions and 289 deletions.
101 changes: 59 additions & 42 deletions task-generator/remote/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,13 @@ func convertToSsh(task *tektonapi.Task, taskVersion string) {
// to prevent them from getting garbage collected before generating the image index.
// We can simplify this process, preventing the need for users to manually specify
// the image by auto-appending the architecture from the PLATFORM parameter. For
// example, this will append -arm64 if PLATFORM is linux/arm64 if not present. Since
// we cannot modify the parameter itself, this replacement needs to happen in any task
// example, this will append -arm64 if PLATFORM is linux/arm64 if not present. If PLATFORM
// does not contain a "/" then the entire platform will be appended to IMAGE (i.e. "localhost").
// Since we cannot modify the parameter itself, this replacement needs to happen in any task
// step where the IMAGE parameter is used.
// If a user defines the IMAGE parameter with an -arm64 suffix, the arm64 suffix will
// not be appended again based on the PLATFORM.
adjustRemoteImage := `if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
adjustRemoteImage := `if [ -n "${APPEND_PLATFORM_ARCH}" ] && [ "${IMAGE##*-}" != "${PLATFORM##*/}" ]; then
export IMAGE="${IMAGE}-${PLATFORM##*/}"
fi
`
Expand Down Expand Up @@ -142,55 +143,65 @@ if [ -e "/ssh/error" ]; then
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
[ "$SSH_HOST" == "localhost" ] && IS_LOCALHOST=true
if [[ $IS_LOCALHOST ]]; then
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
`
if taskVersion != "0.1" {
ret += adjustRemoteImage
}
env := "$PODMAN_PORT_FORWARD \\\n"

// disable podman subscription-manager integration
env += " --tmpfs /run/secrets \\\n"
env += " --tmpfs /run/secrets \\\n"

// Before the build we sync the contents of the workspace to the remote host
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
ret += "\n rsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
}
// Also sync the volume mounts from the template
for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
for _, volume := range step.VolumeMounts {
if syncVolumes[volume.Name] {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
}
ret += "\nrsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\nrsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/tekton-results/\""
podmanArgs += " -v \"$BUILD_DIR/tekton-results/:/tekton/results:Z\" \\\n"
ret += "\n rsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\n rsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/results/\""
podmanArgs += " -v \"$BUILD_DIR/results/:/tekton/results:Z\" \\\n"
ret += "\nfi\n"

if taskVersion != "0.1" {
ret += adjustRemoteImage
}

script := "scripts/script-" + step.Name + ".sh"

Expand All @@ -214,33 +225,36 @@ fi
ret += step.Script
ret += "\nbuildah push \"$IMAGE\" oci:rhtap-final-image"
ret += "\nREMOTESSHEOF"
ret += "\nchmod +x " + script
ret += "\nchmod +x " + script + "\n"

if task.Spec.StepTemplate != nil {
for _, e := range task.Spec.StepTemplate.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
}
ret += "\nrsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
ret += "\n rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "/script/script-" + step.Name + ".sh"
for _, e := range step.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\nssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + "--user=0 --rm \"$BUILDER_IMAGE\" " + containerScript
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\n ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + " --user=0 --rm \"$BUILDER_IMAGE\" " + containerScript

// Sync the contents of the workspaces back so subsequent tasks can use them
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
}

for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
}
//sync back results
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/tekton-results/\" \"/tekton/results/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/results/\" \"/tekton/results/\""

ret += "\nbuildah pull oci:rhtap-final-image"
ret += "\n buildah pull oci:rhtap-final-image"
ret += "\nelse\n bash " + containerScript
ret += "\nfi"
ret += "\nbuildah images"
ret += "\nbuildah tag localhost/rhtap-final-image \"$IMAGE\""
ret += "\ncontainer=$(buildah from --pull-never \"$IMAGE\")\nbuildah mount \"$container\" | tee /shared/container_path\necho $container > /shared/container_name"
Expand Down Expand Up @@ -276,5 +290,8 @@ fi
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "BUILDER_IMAGE", Value: builderImage})
if taskVersion != "0.1" {
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "PLATFORM", Value: "$(params.PLATFORM)"})

task.Spec.Params = append(task.Spec.Params, tektonapi.ParamSpec{Name: "APPEND_PLATFORM_ARCH", Type: tektonapi.ParamTypeString, Description: "Whether the platform architecture should be appended to the IMAGE tag", Default: &tektonapi.ParamValue{StringVal: "", Type: tektonapi.ParamTypeString}})
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "APPEND_PLATFORM_ARCH", Value: "$(params.APPEND_PLATFORM_ARCH)"})
}
}
131 changes: 73 additions & 58 deletions task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,35 +220,45 @@ spec:
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
[ "$SSH_HOST" == "localhost" ] && IS_LOCALHOST=true
if [[ $IS_LOCALHOST ]]; then
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/results/"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/tekton-results/"
cat >scripts/script-build.sh <<'REMOTESSHEOF'
#!/bin/bash
set -o verbose
Expand Down Expand Up @@ -431,42 +441,47 @@ spec:
buildah push "$IMAGE" oci:rhtap-final-image
REMOTESSHEOF
chmod +x scripts/script-build.sh
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/tekton-results/:/tekton/results:Z" \
-v $BUILD_DIR/scripts:/script:Z \
--user=0 --rm "$BUILDER_IMAGE" /script/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/tekton-results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
if ! [[ $IS_LOCALHOST ]]; then
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/results/:/tekton/results:Z" \
-v $BUILD_DIR/scripts:/script:Z \
--user=0 --rm "$BUILDER_IMAGE" /script/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
else
bash /script/script-build.sh
fi
buildah images
buildah tag localhost/rhtap-final-image "$IMAGE"
container=$(buildah from --pull-never "$IMAGE")
Expand Down
Loading

0 comments on commit fc9c1d0

Please sign in to comment.