Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable remote tasks to be run in cluster #1216

Merged
merged 2 commits into from
Aug 15, 2024
Merged

Enable remote tasks to be run in cluster #1216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4aa1fe
Enable remote tasks to be run in cluster
arewm Jul 29, 2024
49c8395
unify on a common buildah image for all tasks
arewm Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 67 additions & 51 deletions task-generator/remote/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,16 @@ func convertToSsh(task *tektonapi.Task, taskVersion string) {
// The images produced in multi-platform builds need to have unique tags in order
// to prevent them from getting garbage collected before generating the image index.
// We can simplify this process, preventing the need for users to manually specify
// the image by auto-appending the architecture from the PLATFORM parameter. For
// example, this will append -arm64 if PLATFORM is linux/arm64 if not present. Since
// we cannot modify the parameter itself, this replacement needs to happen in any task
// step where the IMAGE parameter is used.
// If a user defines the IMAGE parameter with an -arm64 suffix, the arm64 suffix will
// not be appended again based on the PLATFORM.
adjustRemoteImage := `if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
export IMAGE="${IMAGE}-${PLATFORM##*/}"
// the image by auto-appending the a sanitized PLATFORM parameter. For example, this
// will append linux-arm64 if PLATFORM is linux/arm64 and IMAGE_APPEND_PLATFORM is true.
// Many special characters are not allowed in tags so we will replace anything that
// isn't alphanumeric with a "-" to be safe. Since we cannot modify the parameter itself,
mmorhun marked this conversation as resolved.
Show resolved Hide resolved
// this replacement needs to happen in any task step where the IMAGE parameter is used.
// IMAGE_APPEND_PLATFORM will be set to "false" by default so appending the platform is
// and explicit opt-in.
adjustRemoteImage := `if [ "${IMAGE_APPEND_PLATFORM}" == "true" ]; then
IMAGE="${IMAGE}-${PLATFORM//[^a-zA-Z0-9]/-}"
export IMAGE
fi
`

Expand Down Expand Up @@ -142,55 +144,65 @@ if [ -e "/ssh/error" ]; then
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)

if [ "$SSH_HOST" == "localhost" ] ; then
IS_LOCALHOST=true
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"

mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"

PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi

if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"

PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
`
if taskVersion != "0.1" {
ret += adjustRemoteImage
}
env := "$PODMAN_PORT_FORWARD \\\n"

// disable podman subscription-manager integration
env += " --tmpfs /run/secrets \\\n"
env += " --tmpfs /run/secrets \\\n"
mmorhun marked this conversation as resolved.
Show resolved Hide resolved

// Before the build we sync the contents of the workspace to the remote host
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
ret += "\n rsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
}
// Also sync the volume mounts from the template
for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
for _, volume := range step.VolumeMounts {
if syncVolumes[volume.Name] {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
}
ret += "\nrsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\nrsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/tekton-results/\""
podmanArgs += " -v \"$BUILD_DIR/tekton-results/:/tekton/results:Z\" \\\n"
ret += "\n rsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\n rsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/results/\""
podmanArgs += " -v \"$BUILD_DIR/results/:/tekton/results:Z\" \\\n"
ret += "\nfi\n"

if taskVersion != "0.1" {
ret += adjustRemoteImage
}

script := "scripts/script-" + step.Name + ".sh"

arewm marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -212,37 +224,39 @@ fi
ret += "cd " + step.WorkingDir + "\n"
}
ret += step.Script
ret += "\nbuildah push \"$IMAGE\" oci:rhtap-final-image"
ret += "\nbuildah push \"$IMAGE\" \"oci:konflux-final-image:$IMAGE\""
ret += "\nREMOTESSHEOF"
ret += "\nchmod +x " + script
ret += "\nchmod +x " + script + "\n"

if task.Spec.StepTemplate != nil {
for _, e := range task.Spec.StepTemplate.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
}
ret += "\nrsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "/script/script-" + step.Name + ".sh"
ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
ret += "\n rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "scripts/script-" + step.Name + ".sh"
for _, e := range step.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\nssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + "--user=0 --rm \"$BUILDER_IMAGE\" " + containerScript
podmanArgs += " -v \"$BUILD_DIR/scripts:/scripts:Z\" \\\n"
ret += "\n ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + " --user=0 --rm \"$BUILDER_IMAGE\" /" + containerScript

// Sync the contents of the workspaces back so subsequent tasks can use them
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
}

for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
}
//sync back results
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/tekton-results/\" \"/tekton/results/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/results/\" \"/tekton/results/\""

ret += "\nbuildah pull oci:rhtap-final-image"
ret += "\n buildah pull \"oci:konflux-final-image:$IMAGE\""
ret += "\nelse\n bash " + containerScript
ret += "\nfi"
ret += "\nbuildah images"
ret += "\nbuildah tag localhost/rhtap-final-image \"$IMAGE\""
ret += "\ncontainer=$(buildah from --pull-never \"$IMAGE\")\nbuildah mount \"$container\" | tee /shared/container_path\necho $container > /shared/container_name"

for _, i := range strings.Split(ret, "\n") {
Expand All @@ -252,7 +266,6 @@ fi
}
step.Script = ret
builderImage = step.Image
step.Image = "quay.io/redhat-appstudio/multi-platform-runner:01c7670e81d5120347cf0ad13372742489985e5f@sha256:246adeaaba600e207131d63a7f706cffdcdc37d8f600c56187123ec62823ff44"
step.VolumeMounts = append(step.VolumeMounts, v1.VolumeMount{
Name: "ssh",
ReadOnly: true,
Expand All @@ -276,5 +289,8 @@ fi
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "BUILDER_IMAGE", Value: builderImage})
if taskVersion != "0.1" {
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "PLATFORM", Value: "$(params.PLATFORM)"})

task.Spec.Params = append(task.Spec.Params, tektonapi.ParamSpec{Name: "IMAGE_APPEND_PLATFORM", Type: tektonapi.ParamTypeString, Description: "Whether to append a sanitized platform architecture on the IMAGE tag", Default: &tektonapi.ParamValue{StringVal: "false", Type: tektonapi.ParamTypeString}})
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "IMAGE_APPEND_PLATFORM", Value: "$(params.IMAGE_APPEND_PLATFORM)"})
}
}
2 changes: 1 addition & 1 deletion task/build-image-manifest/0.1/build-image-manifest.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ spec:
- name: COMMIT_SHA
value: $(params.COMMIT_SHA)
steps:
- image: quay.io/konflux-ci/buildah:latest@sha256:7cb5a35b7fe44e397fbf3b834f3bd8dcd9403a7c0a0b51469e6ec75b107d0846
- image: quay.io/konflux-ci/buildah-task:latest@sha256:5d933087a49a6e0b959b9a9fa5a91d545380217e565d7be7cc74a9588f64c314
# per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
# the cluster will set imagePullPolicy to IfNotPresent
name: build
Expand Down
4 changes: 2 additions & 2 deletions task/buildah-oci-ta/0.1/buildah-oci-ta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ spec:
- $(params.SOURCE_ARTIFACT)=/var/workdir/source
- $(params.CACHI2_ARTIFACT)=/var/workdir/cachi2
- name: build
image: quay.io/konflux-ci/buildah:latest@sha256:7cb5a35b7fe44e397fbf3b834f3bd8dcd9403a7c0a0b51469e6ec75b107d0846
image: quay.io/konflux-ci/buildah-task:latest@sha256:5d933087a49a6e0b959b9a9fa5a91d545380217e565d7be7cc74a9588f64c314
args:
- $(params.BUILD_ARGS[*])
workingDir: /var/workdir
Expand Down Expand Up @@ -528,7 +528,7 @@ spec:
securityContext:
runAsUser: 0
- name: inject-sbom-and-push
image: quay.io/konflux-ci/buildah:latest@sha256:7cb5a35b7fe44e397fbf3b834f3bd8dcd9403a7c0a0b51469e6ec75b107d0846
image: quay.io/konflux-ci/buildah-task:latest@sha256:5d933087a49a6e0b959b9a9fa5a91d545380217e565d7be7cc74a9588f64c314
workingDir: /var/workdir
volumeMounts:
- mountPath: /var/lib/containers
Expand Down
4 changes: 2 additions & 2 deletions task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ spec:
- $(params.SOURCE_ARTIFACT)=/var/workdir/source
- $(params.CACHI2_ARTIFACT)=/var/workdir/cachi2
- name: build
image: quay.io/konflux-ci/buildah:latest@sha256:7cb5a35b7fe44e397fbf3b834f3bd8dcd9403a7c0a0b51469e6ec75b107d0846
image: quay.io/konflux-ci/buildah-task:latest@sha256:5d933087a49a6e0b959b9a9fa5a91d545380217e565d7be7cc74a9588f64c314
args:
- $(params.BUILD_ARGS[*])
workingDir: /var/workdir
Expand Down Expand Up @@ -506,7 +506,7 @@ spec:
securityContext:
runAsUser: 0
- name: inject-sbom-and-push
image: quay.io/konflux-ci/buildah:latest@sha256:7cb5a35b7fe44e397fbf3b834f3bd8dcd9403a7c0a0b51469e6ec75b107d0846
image: quay.io/konflux-ci/buildah-task:latest@sha256:5d933087a49a6e0b959b9a9fa5a91d545380217e565d7be7cc74a9588f64c314
workingDir: /var/workdir
volumeMounts:
- mountPath: /var/lib/containers
Expand Down
Loading
Loading