From cc6790f716c8c4aca7d5e74bb823380ab95d566f Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 2 Jun 2020 11:59:58 -0700 Subject: [PATCH 01/32] bug 1597598 - use taskgraph --- .taskcluster.yml | 446 ++++++++++-------- docker.d/generate_version_json.sh | 2 +- docker.d/push_image.sh | 9 +- taskcluster/ci/config.yml | 48 ++ taskcluster/ci/docker-image/kind.yml | 41 ++ taskcluster/ci/k8s-image/kind.yml | 53 +++ taskcluster/ci/tox/kind.yml | 100 ++++ taskcluster/docker/REGISTRY | 1 + taskcluster/docker/k8s-image/Dockerfile | 27 ++ .../docker/k8s-image/build_and_push.sh | 21 + taskcluster/docker/pushapkscript/Dockerfile | 7 + .../docker/pushflatpakscript/Dockerfile | 8 + taskcluster/docker/pushsnapscript/Dockerfile | 9 + taskcluster/docker/python/Dockerfile | 27 ++ .../scriptworker_taskgraph/__init__.py | 22 + .../transforms/__init__.py | 0 .../transforms/cached.py | 95 ++++ .../transforms/k8s_image.py | 92 ++++ .../transforms/python_version.py | 60 +++ tox.ini | 83 ++-- 20 files changed, 906 insertions(+), 245 deletions(-) create mode 100644 taskcluster/ci/config.yml create mode 100644 taskcluster/ci/docker-image/kind.yml create mode 100644 taskcluster/ci/k8s-image/kind.yml create mode 100644 taskcluster/ci/tox/kind.yml create mode 100644 taskcluster/docker/REGISTRY create mode 100644 taskcluster/docker/k8s-image/Dockerfile create mode 100755 taskcluster/docker/k8s-image/build_and_push.sh create mode 100644 taskcluster/docker/pushapkscript/Dockerfile create mode 100644 taskcluster/docker/pushflatpakscript/Dockerfile create mode 100644 taskcluster/docker/pushsnapscript/Dockerfile create mode 100644 taskcluster/docker/python/Dockerfile create mode 100644 taskcluster/scriptworker_taskgraph/__init__.py create mode 100644 taskcluster/scriptworker_taskgraph/transforms/__init__.py create mode 100644 taskcluster/scriptworker_taskgraph/transforms/cached.py create mode 100644 taskcluster/scriptworker_taskgraph/transforms/k8s_image.py create mode 100644 taskcluster/scriptworker_taskgraph/transforms/python_version.py diff --git a/.taskcluster.yml b/.taskcluster.yml index c678e8073..0933c6251 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,210 +1,254 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- version: 1 +reporting: checks-v1 policy: - pullRequests: public + pullRequests: collaborators tasks: - $let: - # ------------------------------------------------------------------------- - # -- This is where you add new projects ----------------------------------- - # ------------------------------------------------------------------------- - PROJECTS: - $let: - setup_pushapkscript: 'apt-get update && apt-get install -y default-jdk &&' - setup_pushsnapscript: 'apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release &&' - setup_pushflatpakscript: 'apt-get update && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev &&' + - $let: + taskgraph: + branch: taskgraph + revision: ec220e56cf107ea279d7c08706fc370374869550 + trustDomain: scriptworker in: - # [ , , , ] - - ['client', '37', '', ''] - - ['client', '38', '', ''] - - ['configloader', '37', '', ''] - - ['configloader', '38', '', ''] - - ['iscript', '37', '', ''] - - ['iscript', '38', '', ''] - - ['addonscript', '37', '', ''] - - ['addonscript', '38', '', 'mozilla/releng-addonscript'] - - ['balrogscript', '37', '', ''] - - ['balrogscript', '38', '', 'mozilla/releng-balrogscript'] - - ['beetmoverscript', '37', '', ''] - - ['beetmoverscript', '38', '', 'mozilla/releng-beetmoverscript'] - - ['bouncerscript', '37', '', ''] - - ['bouncerscript', '38', '', 'mozilla/releng-bouncerscript'] - - ['notarization_poller', '37', '', ''] - - ['notarization_poller', '38', '', ''] - - ['pushapkscript', '37', '${setup_pushapkscript}', ''] - - ['pushapkscript', '38', '${setup_pushapkscript}', 'mozilla/releng-pushapkscript'] - - ['pushflatpakscript', '37', '${setup_pushflatpakscript}', ''] - - ['pushflatpakscript', '38', '${setup_pushflatpakscript}', 'mozilla/releng-pushflatpakscript'] - - ['pushsnapscript', '37', '${setup_pushsnapscript}', ''] - - ['pushsnapscript', '38', '${setup_pushsnapscript}', 'mozilla/releng-pushsnapscript'] - - ['shipitscript', '37', '', ''] - - ['shipitscript', '38', '', 'mozilla/releng-shipitscript'] - - ['signingscript', '37', '', ''] - - ['signingscript', '38', '', 'mozilla/releng-signingscript'] - - ['treescript', '37', '', ''] - - ['treescript', '38', '', 'mozilla/releng-treescript'] - # ------------------------------------------------------------------------- + $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]' + then: + $let: + # Github events have this stuff in different places... + ownerEmail: + $if: 'tasks_for == "github-push"' + then: '${event.pusher.email}' + # Assume Pull Request + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.user.login}@users.noreply.github.com' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${tasks_for}@noreply.mozilla.org' + baseRepoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.base.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + repoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + project: + $if: 'tasks_for == "github-push"' + then: '${event.repository.name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.name}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.project}' + head_branch: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.ref} + else: + $if: 'tasks_for == "github-push"' + then: ${event.ref} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.branch}' + head_sha: + $if: 'tasks_for == "github-push"' + then: '${event.after}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.sha}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.revision}' + ownTaskId: + $if: '"github" in tasks_for' + then: {$eval: as_slugid("decision_task")} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${ownTaskId}' + in: + $let: + level: + $if: 'tasks_for in ["github-push", "cron", "action"] && repoUrl == "https://github.com/mozilla-releng/scriptworker-scripts"' + then: 3 + else: 1 + in: + taskId: + $if: 'tasks_for != "action"' + then: '${ownTaskId}' + taskGroupId: + $if: 'tasks_for == "action"' + then: + '${action.taskGroupId}' + else: + '${ownTaskId}' # same as taskId; this is how automation identifies a decision task + schedulerId: '${trustDomain}-level-${level}' + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors + metadata: + $merge: + - owner: "${ownerEmail}" + source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + name: "Decision Task" + description: 'The task that creates all of the other tasks in the task graph' + else: + $if: 'tasks_for == "action"' + then: + name: "Action: ${action.title}" + description: '${action.description}' + else: + name: "Decision Task for cron job ${cron.job_name}" + description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})' + provisionerId: "${trustDomain}-${level}" + workerType: "decision" + tags: + $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + kind: decision-task + else: + $if: 'tasks_for == "action"' + then: + kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task + routes: + $flatten: + - checks + - $if: 'tasks_for == "github-push"' + then: + - "index.${trustDomain}.v2.${project}.revision.${head_sha}.taskgraph.decision" + else: [] + scopes: + # `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol. + $if: 'tasks_for == "github-push"' + then: + $let: + short_head_branch: + $if: 'head_branch[:10] == "refs/tags/"' + then: {$eval: 'head_branch[10:]'} + else: + $if: 'head_branch[:11] == "refs/heads/"' + then: {$eval: 'head_branch[11:]'} + else: ${head_branch} + in: + - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' - HEAD_REV: - $if: 'tasks_for == "github-pull-request"' - then: '${event.pull_request.head.sha}' - else: - $if: 'tasks_for == "github-push"' - then: '${event.after}' - else: '${event.release.tag_name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: + - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' + else: + $if: 'tasks_for == "action"' + then: + # when all actions are hooks, we can calculate this directly rather than using a variable + - '${action.repo_scope}' + else: + - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' - REPO_URL: - $if: 'tasks_for == "github-pull-request"' - then: '${event.pull_request.head.repo.html_url}' - else: '${event.repository.html_url}' + requires: all-completed + priority: lowest + retries: 5 - OWNER: '${event.sender.login}@users.noreply.github.com' + payload: + env: + # run-task uses these to check out the source; the inputs + # to `mach taskgraph decision` are all on the command line. + $merge: + - SCRIPTWORKER_BASE_REPOSITORY: '${baseRepoUrl}' + SCRIPTWORKER_HEAD_REPOSITORY: '${repoUrl}' + SCRIPTWORKER_HEAD_REF: '${head_branch}' + SCRIPTWORKER_HEAD_REV: '${head_sha}' + SCRIPTWORKER_REPOSITORY_TYPE: git + TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph + TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch} + TASKGRAPH_HEAD_REV: '${taskgraph.revision}' + TASKGRAPH_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {scriptworker: "Scriptworker-scripts", taskgraph: "Taskgraph"}} + HG_STORE_PATH: /builds/worker/checkouts/hg-store + - $if: 'tasks_for in ["github-pull-request"]' + then: + SCRIPTWORKER_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + - $if: 'tasks_for == "action"' + then: + ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task + ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) + ACTION_INPUT: {$json: {$eval: 'input'}} + ACTION_CALLBACK: '${action.cb_name}' + features: + taskclusterProxy: true + chainOfTrust: true + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the hash + image: mozillareleases/taskgraph:decision-21bef1bc0f11e62c7a23384584f9f8f0d96e95eef192e5bb599fc82ba55c81a7@sha256:d29af306b09cd00a63f982e8caeb53dbec7efd7bb3ae64ce3cef9cd889b750e6 + maxRunTime: 1800 - BRANCH_NAME: - $if: 'tasks_for == "github-pull-request"' - then: 'pull-request' - else: - $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"' - then: '${event.ref[11:]}' - else: 'unknown' - in: - $flatten: - $map: { "$eval": "PROJECTS" } - each(x,i): - $let: - project_name: { "$eval": "x[0]" } - python_version: { "$eval": "x[1]" } - setup_command: { "$eval": "x[2]" } - dockerhub_repo: { "$eval": "x[3]" } - run_tests: - $if: 'tasks_for == "github-pull-request"' - then: '1' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '1' - else: '0' - push_docker_image: - $if: 'tasks_for == "github-pull-request"' - then: '0' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '1' - else: '0' - worker_level: - $if: 'tasks_for == "github-pull-request"' - then: 't' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '3' - else: 't' - number_prefix: - $if: 'i + 1 < 10' - then: '0' - else: '' - docker_tag: - $if: 'tasks_for == "github-pull-request"' - then: 'pull-request' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '${event.ref[11:-(len(x[0]) + 1)]}' - else: - $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"' - then: '${event.ref[11:]}' - else: 'unknown' - in: - $match: - # Run code linting and unit tests for each project - 'run_tests == "1"': - taskId: '${as_slugid(project_name + python_version)}' - provisionerId: 'releng-t' - workerType: 'linux' - created: { $fromNow: '' } - deadline: { $fromNow: '4 hours' } - payload: - maxRunTime: 3600 - image: 'python:${python_version[0]}.${python_version[1]}' - command: - - sh - - -lxce - - >- - cd /tmp && - wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz && - tar zxf ${HEAD_REV}.tar.gz && - mv scriptworker-scripts-${HEAD_REV} /src && - cd /src && ${setup_command} - pip install tox && - tox -e ${project_name}-py${python_version} - metadata: - name: - $let: - test_task_number: - $if: 'dockerhub_repo != ""' - then: '${i+1}.1' - else: '${i+1}' - in: - '${number_prefix}${test_task_number}. ${project_name}-py${python_version}: Run tox [on ${BRANCH_NAME}]' - description: 'Code linting and unit tests for ${project_name} on python ${python_version[0]}.${python_version[1]}' - owner: '${OWNER}' - source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' - # Build docker image and (optionally) push to docker hub - 'run_tests == "1" && dockerhub_repo != ""': - taskId: '${as_slugid(project_name + "docker_build_and_push")}' - dependencies: - - '${as_slugid(project_name + python_version)}' - provisionerId: 'releng-${worker_level}' - workerType: 'linux' - created: { $fromNow: '' } - deadline: { $fromNow: '24 hours' } - payload: - features: - dind: true - taskclusterProxy: true - maxRunTime: 3600 - # we need to run really old docker version because taskcluster is using - # really old version in their setup - # image: docker:stable - image: 'docker:1.6.2' - env: + command: + - /usr/local/bin/run-task + - '--scriptworker-checkout=/builds/worker/checkouts/src' + - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph' + - '--task-cwd=/builds/worker/checkouts/src' + - '--' + - bash + - -cx + - $let: + extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + in: + $if: 'tasks_for == "action"' + then: > + PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph action-callback + else: > + PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --message="" + --owner='${ownerEmail}' + --level='${level}' + --base-repository="$SCRIPTWORKER_BASE_REPOSITORY" + --head-repository="$SCRIPTWORKER_HEAD_REPOSITORY" + --head-ref="$SCRIPTWORKER_HEAD_REF" + --head-rev="$SCRIPTWORKER_HEAD_REV" + --repository-type="$SCRIPTWORKER_REPOSITORY_TYPE" + --tasks-for='${tasks_for}' + ${extraArgs} + + artifacts: + 'public': + type: 'directory' + path: '/builds/worker/artifacts' + expires: {$fromNow: '1 year'} - DOCKERHUB_EMAIL: 'release+dockerhub+services@mozilla.com' - DOCKERHUB_USER: 'mozillarelengservices' - DOCKER_REPO: '${dockerhub_repo}' - DOCKER_TAG: '${docker_tag}' - GIT_HEAD_REV: '${HEAD_REV}' - PROJECT_NAME: '${project_name}' - PUSH_DOCKER_IMAGE: '${push_docker_image}' - REPO_URL: '${REPO_URL}' - SECRET_URL: 'http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy' - command: - - sh - - -lxce - - >- - cd /tmp && - wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz && - tar zxf ${HEAD_REV}.tar.gz && - mv scriptworker-scripts-${HEAD_REV} /src && - cd /src/ && - cp ${project_name}/docker.d/* docker.d/ && - cp ${project_name}/Dockerfile . && - ./docker.d/generate_version_json.sh && - ./docker.d/build_image.sh && - if [ "$PUSH_DOCKER_IMAGE" == "1" ]; then - ./docker.d/push_image.sh - fi - scopes: - $if: 'push_docker_image == "0"' - then: [] - else: - - 'secrets:get:project/releng/scriptworker-scripts/deploy' - metadata: - $let: - description: - $if: 'push_docker_image == "0"' - then: '${project_name}-py${python_version}: build docker image [on ${BRANCH_NAME}]' - else: '${project_name}-py${python_version}: build and push docker image [on ${BRANCH_NAME}]' - in: - name: '${number_prefix}${i+1}.2. ${description}' - description: '${description}' - owner: '${OWNER}' - source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' + extra: + $merge: + - $if: 'tasks_for == "action"' + then: + parent: '${action.taskGroupId}' + action: + name: '${action.name}' + context: + taskGroupId: '${action.taskGroupId}' + taskId: {$eval: 'taskId'} + input: {$eval: 'input'} + - $if: 'tasks_for == "cron"' + then: + cron: {$json: {$eval: 'cron'}} + - tasks_for: '${tasks_for}' diff --git a/docker.d/generate_version_json.sh b/docker.d/generate_version_json.sh index 680fd0b6e..5a45fa5e2 100755 --- a/docker.d/generate_version_json.sh +++ b/docker.d/generate_version_json.sh @@ -1,7 +1,7 @@ #/bin/sh set -e -test $GIT_HEAD_REV +test $HEAD_REV test $TASK_ID test $TASKCLUSTER_ROOT_URL test $REPO_URL diff --git a/docker.d/push_image.sh b/docker.d/push_image.sh index c64be0c49..7d99a054e 100755 --- a/docker.d/push_image.sh +++ b/docker.d/push_image.sh @@ -5,7 +5,8 @@ test $DOCKERHUB_EMAIL test $DOCKERHUB_USER test $DOCKER_REPO test $DOCKER_TAG -test $GIT_HEAD_REV +test $HEAD_REV +test $HOME test $PUSH_DOCKER_IMAGE test $SECRET_URL test $PROJECT_NAME @@ -18,12 +19,12 @@ docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$DOCKER_ARCHIVE_TAG echo "=== Generating dockercfg ===" # docker login stopped working in Taskcluster for some reason -wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > /root/.dockercfg -chmod 600 /root/.dockercfg +wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > $HOME/.dockercfg +chmod 600 $HOME/.dockercfg echo "=== Pushing to docker hub ===" docker push $DOCKER_REPO:$DOCKER_TAG docker push $DOCKER_REPO:$DOCKER_ARCHIVE_TAG echo "=== Clean up ===" -rm -f /root/.dockercfg +rm -f $HOME/.dockercfg diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 000000000..17e267276 --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,48 @@ +--- +trust-domain: scriptworker + +task-priority: high + +taskgraph: + register: scriptworker_taskgraph:register + repositories: + scriptworker: + name: "Scriptworker-scripts" + project-regex: scriptworker-scripts$ + default-repository: https://github.com/mozilla-releng/scriptworker-scripts + default-ref: master + type: git + +workers: + aliases: + b-linux: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'b-linux' + images: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'images' + +docker: + email: "release+dockerhub+services@mozilla.com" + user: mozillarelengservices + +notify: + email: + # TODO by-project/level + - aki+scriptworker-scripts@mozilla.com + # TODO by-project/level + prefix: "[scriptworker-scripts stage]" + +release-promotion: + flavors: + promote: + target-tasks-method: promote_scriptworker + +private-artifact-prefix: releng/scriptworker + +scriptworker: + scope-prefix: project:scriptworker:releng diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 000000000..c6098e0f0 --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +jobs: + k8s-image: + definition: k8s-image + python38: + definition: python + args: + PYTHON_VERSION: "3.8" + python37: + definition: python + args: + PYTHON_VERSION: "3.7" + pushapkscript-py38: + definition: pushapkscript + parent: python38 + pushapkscript-py37: + definition: pushapkscript + parent: python37 + pushsnapscript-py38: + definition: pushsnapscript + parent: python38 + pushsnapscript-py37: + definition: pushsnapscript + parent: python37 + pushflatpakscript-py38: + definition: pushflatpakscript + parent: python38 + pushflatpakscript-py37: + definition: pushflatpakscript + parent: python37 diff --git a/taskcluster/ci/k8s-image/kind.yml b/taskcluster/ci/k8s-image/kind.yml new file mode 100644 index 000000000..9712b35a1 --- /dev/null +++ b/taskcluster/ci/k8s-image/kind.yml @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - tox + +transforms: + - scriptworker_taskgraph.transforms.python_version:transforms + - scriptworker_taskgraph.transforms.k8s_image:transforms + - scriptworker_taskgraph.transforms.cached:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: "{name} k8s image builder py{python_version}" + label: "{name}-k8s-image-builder-python{python_version}" + run-on-tasks-for: ["action", "github-pull-request", "github-push"] + worker-type: images + worker: + docker-image: {in-tree: 'k8s-image'} + max-run-time: 3600 + docker-in-docker: true + taskcluster-proxy: true + chain-of-trust: true + artifacts: + - type: directory + name: public/ + path: /builds/worker/artifacts/ + command: + - sh + - -lxce + - >- + sh /usr/local/bin/build_and_push.sh + python-versions: + - 38 + deploy-secret-url: http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy + docker-repo: mozilla/releng-{name} + # TODO copy image to artifacts + +jobs: + addonscript: {} + balrogscript: {} + beetmoverscript: {} + bouncerscript: {} + pushapkscript: {} + pushflatpakscript: {} + pushsnapscript: {} + shipitscript: {} + signingscript: {} + treescript: {} diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml new file mode 100644 index 000000000..1d9d937dd --- /dev/null +++ b/taskcluster/ci/tox/kind.yml @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - docker-image + +transforms: + - scriptworker_taskgraph.transforms.python_version:transforms + - scriptworker_taskgraph.transforms.cached:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: "{name} tox-py{python_version}" + label: "tox-{name}-py{python_version}" + run-on-tasks-for: ["action", "github-pull-request", "github-push"] + worker-type: b-linux + worker: + docker-image: {in-tree: 'python{python_version}'} + max-run-time: 1800 + artifacts: + - type: directory + name: public/ + path: /builds/worker/artifacts/ + python-versions: + - 38 + - 37 + run: + using: run-task + cache-dotcache: false + checkout: + scriptworker: {} + # sparse-profile: basestring/none + # workdir: + cwd: '{checkout}' + command: + - sh + - -lxce + - >- + tox -e {name}-py{python_version} + +jobs: + addonscript: + digest-directories: + - addonscript + balrogscript: + digest-directories: + - balrogscript + - scriptworker_client + beetmoverscript: + digest-directories: + - beetmoverscript + bouncerscript: + digest-directories: + - bouncerscript + configloader: + digest-directories: + - configloader + iscript: + digest-directories: + - iscript + - scriptworker_client + - vendored/mozbuild + notarization_poller: + digest-directories: + - notarization_poller + - scriptworker_client + pushapkscript: + worker: + docker-image: {in-tree: 'pushapkscript-py{python_version}'} + digest-directories: + - pushapkscript + pushflatpakscript: + worker: + docker-image: {in-tree: 'pushflatpakscript-py{python_version}'} + digest-directories: + - pushflatpakscript + pushsnapscript: + worker: + docker-image: {in-tree: 'pushsnapscript-py{python_version}'} + digest-directories: + - pushsnapscript + scriptworker_client: + digest-directories: + - scriptworker_client + shipitscript: + digest-directories: + - shipitscript + signingscript: + digest-directories: + - signingscript + - vendored/mozbuild + treescript: + digest-directories: + - treescript + - scriptworker_client diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY new file mode 100644 index 000000000..8c26d4c91 --- /dev/null +++ b/taskcluster/docker/REGISTRY @@ -0,0 +1 @@ +mozilla-releng diff --git a/taskcluster/docker/k8s-image/Dockerfile b/taskcluster/docker/k8s-image/Dockerfile new file mode 100644 index 000000000..af42ac99b --- /dev/null +++ b/taskcluster/docker/k8s-image/Dockerfile @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# We need to run a really old docker version because taskcluster is using a +# really old version in their setup +FROM docker:1.6.2 + +RUN apk add --update xz +RUN mkdir -p /usr/local/bin +COPY build_and_push.sh /usr/local/bin/build_and_push.sh + +# Add worker user +RUN mkdir /builds && \ + adduser -D -h /builds/worker -g worker -u 1000 worker && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +ENV SHELL=/bin/sh \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# Set a default command useful for debugging +CMD ["/bin/sh"] diff --git a/taskcluster/docker/k8s-image/build_and_push.sh b/taskcluster/docker/k8s-image/build_and_push.sh new file mode 100755 index 000000000..9e55a4677 --- /dev/null +++ b/taskcluster/docker/k8s-image/build_and_push.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# TODO: move docker.d/ files into the image? +set -e +test REPO_URL +test HEAD_REV +test PROJECT_NAME +test PUSH_DOCKER_IMAGE + +mkdir -p /builds/worker/checkouts +cd /builds/worker/checkouts +wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz +tar zxf ${HEAD_REV}.tar.gz +mv *-${HEAD_REV} src +cd src +cp ${PROJECT_NAME}/docker.d/* docker.d/ +cp ${PROJECT_NAME}/Dockerfile . +sh ./docker.d/generate_version_json.sh +sh ./docker.d/build_image.sh +if [ "${PUSH_DOCKER_IMAGE}" == "1" ]; then + ./docker.d/push_image.sh +fi diff --git a/taskcluster/docker/pushapkscript/Dockerfile b/taskcluster/docker/pushapkscript/Dockerfile new file mode 100644 index 000000000..4eaaf8054 --- /dev/null +++ b/taskcluster/docker/pushapkscript/Dockerfile @@ -0,0 +1,7 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +RUN apt-get update && apt-get install -y default-jdk diff --git a/taskcluster/docker/pushflatpakscript/Dockerfile b/taskcluster/docker/pushflatpakscript/Dockerfile new file mode 100644 index 000000000..feb94f208 --- /dev/null +++ b/taskcluster/docker/pushflatpakscript/Dockerfile @@ -0,0 +1,8 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +RUN apt-get update \ + && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev diff --git a/taskcluster/docker/pushsnapscript/Dockerfile b/taskcluster/docker/pushsnapscript/Dockerfile new file mode 100644 index 000000000..92ffc25c7 --- /dev/null +++ b/taskcluster/docker/pushsnapscript/Dockerfile @@ -0,0 +1,9 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +RUN apt-get update \ + && apt-get install -y libsodium-dev +RUN truncate -s 0 /etc/os-release diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile new file mode 100644 index 000000000..b9ee4cd65 --- /dev/null +++ b/taskcluster/docker/python/Dockerfile @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# %ARG PYTHON_VERSION +FROM python:$PYTHON_VERSION + +# Add worker user +RUN mkdir /builds && \ + groupadd -g 1000 -o worker && \ + useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +RUN pip install tox + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py new file mode 100644 index 000000000..ea8861a74 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/__init__.py @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from importlib import import_module + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules([ + "transforms", + ]) + + +def _import_modules(modules): + for module in modules: + import_module(".{}".format(module), package=__name__) diff --git a/taskcluster/scriptworker_taskgraph/transforms/__init__.py b/taskcluster/scriptworker_taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py new file mode 100644 index 000000000..ac393ee0c --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import hashlib +import json +import os +import subprocess + +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.hash import hash_path +from taskgraph.util.memoize import memoize + +transforms = TransformSequence() + +BASE_DIR = os.getcwd() + +# Directory names we ignore, anywhere in the tree +# We won't recurse into these directories. +IGNORE_DIRS = (".git", "docs", "maintenance") + +# File extensions we ignore, anywhere in the tree +IGNORE_EXTENSIONS = (".pyc", ".swp") + +ADDITIONAL_FILES = tuple([]) + + +@memoize +def list_files(path): + files = [] + for dir_name, subdir_list, file_list in os.walk(path): + for dir_ in subdir_list: + if dir_ in IGNORE_DIRS: + subdir_list.remove(dir_) + continue + for file_ in file_list: + (_, ext) = os.path.splitext(file_) + if ext in IGNORE_EXTENSIONS: + continue + files.append( + os.path.relpath(os.path.join(BASE_DIR, dir_name, file_), BASE_DIR) + ) + return set(files) + + +@transforms.add +def add_digest_directories(config, tasks): + # TODO `s,digest-directories,digest-files`, where we check isdir(file) ? + for task in tasks: + digest_directories = task.pop("digest-directories", None) + if digest_directories: + task.setdefault("attributes", {})["digest-directories"] = digest_directories + yield task + + +@transforms.add +def build_cache(config, tasks): + repo_name = subprocess.check_output(["git", "remote", "get-url", "origin"]).rstrip() + repo_name = repo_name.replace(".git", "").rstrip("/") + repo_name = repo_name.split("/")[-1] + + for task in tasks: + if task.get("cache", True) and not taskgraph.fast: + digest_data = [] + h = hashlib.sha256() + h.update( + json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) + ) + directories = task.get("attributes", {}).get("digest-directories", []) + files = set([]) + for d in directories: + directory = os.path.join(BASE_DIR, d) + files.update(list_files(directory)) + # files.update(list_files(os.path.join(BASE_DIR, "taskcluster"))) + for path in ADDITIONAL_FILES: + if os.path.exists(path): + files.update({path}) + for path in sorted(list(files)): + h.update( + "{} {}\n".format( + hash_path(os.path.realpath(os.path.join(BASE_DIR, path))), path + ) + ) + task.setdefault("attributes", {}).setdefault("cached_task", {}) + cache_name = task["label"].replace(":", "-") + task["cache"] = { + "type": "{}.v2".format(repo_name), + "name": cache_name, + "digest-data": [h.hexdigest()], + } + + yield task diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py new file mode 100644 index 000000000..29b506e48 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Kubernetes docker image builds. +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy +import time + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + """Add dependencies that match python-version and script-name. + + Also copy the digest-directories attribute, and fail if there are + unexpected discrepancies in upstream deps. + + """ + for job in jobs: + attributes = job["attributes"] + dependencies = job.setdefault("dependencies", {}) + digest_directories = None + for dep_task in config.kind_dependencies_tasks: + dep_attrs = dep_task.attributes + dep_kind = dep_task.kind + if dep_attrs["python-version"] == attributes["python-version"] and \ + dep_attrs["script-name"] == attributes["script-name"]: + if dependencies.get(dep_kind): + raise Exception("Duplicate kind {kind} dependencies: {existing_label}, {new_label}".format( + kind=dep_kind, + existing_label=dependencies[dep_kind]["label"], + new_label=dep_task.label, + )) + dependencies[dep_kind] = dep_task.label + if dep_attrs.get("digest-directories"): + if digest_directories and digest_directories != dep_attrs["digest-directories"]: + raise Exception("Conflicting digest_directories: {existing_digest} {new_digest}".format( + existing_digest=digest_directories, + new_digest=dep_attrs["digest-directories"], + )) + digest_directories = dep_attrs["digest-directories"] + if digest_directories: + attributes["digest-directories"] = digest_directories + yield job + + +@transforms.add +def set_environment(config, jobs): + """Set the environment variables for the docker hub task.""" + for job in jobs: + project_name = job["attributes"]["script-name"] + secret_url = job.pop("deploy-secret-url") + tasks_for = config.params['tasks_for'] + scopes = job.setdefault("scopes", []) + attributes = job["attributes"] + env = job["worker"].setdefault("env", {}) + env["HEAD_REV"] = config.params['head_rev'] + env["REPO_URL"] = config.params['head_repository'] + env["PROJECT_NAME"] = project_name + env["TASKCLUSTER_ROOT_URL"] = "$TASKCLUSTER_ROOT_URL" + env["DOCKER_TAG"] = "unknown" + env["DOCKER_REPO"] = job.pop("docker-repo") + force_push_docker_image = False + if tasks_for == 'github-pull-request': + env["DOCKER_TAG"] = "pull-request" + elif tasks_for == 'github-push': + for ref_name in ("dev", "production"): + if config.params['head_ref'] == "refs/heads/{}-{}".format(ref_name, project_name): + env["DOCKER_TAG"] = ref_name + force_push_docker_image = True + break + else: + if config.params['head_ref'].startswith('refs/heads/'): + env["DOCKER_TAG"] = config.params['head_ref'].replace('refs/heads/', '') + if env["DOCKER_TAG"] in ("production", "dev") and config.params["level"] == "3": + env["SECRET_URL"] = secret_url + env["PUSH_DOCKER_IMAGE"] = "1" + env["DOCKERHUB_EMAIL"] = config.graph_config["docker"]["email"] + env["DOCKERHUB_USER"] = config.graph_config["docker"]["user"] + scopes.append('secrets:get:project/releng/scriptworker-scripts/deploy') + if force_push_docker_image: + attributes.setdefault("digest-extra", {}).setdefault("force_run", time.time()) + else: + env["PUSH_DOCKER_IMAGE"] = "0" + yield job diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py new file mode 100644 index 000000000..57ddfc3b3 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Create a task per job python-version +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + +def _replace_string(obj, repl_dict): + if isinstance(obj, dict): + for k in obj.keys(): + obj[k] = obj[k].format(**repl_dict) + elif isinstance(obj, list): + for c in range(0, len(obj)): + obj[c] = obj[c].format(**repl_dict) + else: + obj = obj.format(**repl_dict) + return obj + + +@transforms.add +def tasks_per_python_version(config, jobs): + for job in jobs: + for python_version in job.pop("python-versions"): + task = deepcopy(job) + repl_dict = {"name": job["name"], "python_version": python_version} + task["label"] = _replace_string(task["label"], repl_dict) + task['worker']['docker-image'] = _replace_string(task['worker']['docker-image'], repl_dict) + task['description'] = _replace_string(task['description'], repl_dict) + if task.get('run', {}).get('command'): + task['run']['command'] = _replace_string(task['run']['command'], repl_dict) + if task['worker'].get('command'): + task['worker']['command'] = _replace_string(task['worker']['command'], repl_dict) + if task.get('docker-repo'): + task['docker-repo'] = _replace_string(task['docker-repo'], repl_dict) + task.setdefault("attributes", {}).update({ + "script-name": job["name"], + "python-version": python_version, + }) + yield task + + +@transforms.add +def skip_on_project_specific_branches(config, jobs): + """Skip if the branch is project-specific for a different project.""" + project_specific_prefixes = ("refs/heads/dev-", "refs/heads/production-") + for job in jobs: + script_name = job["attributes"]["script-name"] + project_specific_branches = ["{}{}".format(prefix, script_name) for prefix in project_specific_prefixes] + if config.params['head_ref'].startswith(project_specific_prefixes) and \ + config.params['head_ref'] not in project_specific_branches: + continue + yield job diff --git a/tox.ini b/tox.ini index 2e12b3c38..f71d3bff7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,5 @@ [tox] envlist = - client-py37 - client-py3py38 - iscript-py37 - iscript-py38 addonscript-py37 addonscript-py38 balrogscript-py37 @@ -12,6 +8,10 @@ envlist = beetmoverscript-py38 bouncerscript-py37 bouncerscript-py38 + configloader-py37 + configloader-py38 + iscript-py37 + iscript-py38 notarization_poller-py37 notarization_poller-py38 pushapkscript-py37 @@ -20,14 +20,14 @@ envlist = pushflatpakscript-py38 pushsnapscript-py37 pushsnapscript-py38 + scriptworker_client-py37 + scriptworker_client-py3py38 shipitscript-py37 shipitscript-py38 signingscript-py37 signingscript-py38 treescript-py37 treescript-py38 - configloader-py37 - configloader-py38 skipsdist = true @@ -43,28 +43,6 @@ passenv = TRAVIS_JOB_ID TRAVIS_BRANCH -[testenv:client-py38] -changedir = {toxinidir}/scriptworker_client -commands = - tox -e py38,mypy - - tox -e coveralls - -[testenv:client-py37] -changedir = {toxinidir}/scriptworker_client -commands = - tox -e py37,mypy - -[testenv:iscript-py38] -changedir = {toxinidir}/iscript -commands = - tox -e py38 - -[testenv:iscript-py37] -changedir = {toxinidir}/iscript -commands = - tox -e py37,check - - tox -e coveralls - [testenv:addonscript-py38] changedir = {toxinidir}/addonscript commands = @@ -109,6 +87,28 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py37 +[testenv:configloader-py38] +changedir = {toxinidir}/configloader +commands = + tox -e py38,check + - tox -e coveralls + +[testenv:configloader-py37] +changedir = {toxinidir}/configloader +commands = + tox -e py37 + +[testenv:iscript-py38] +changedir = {toxinidir}/iscript +commands = + tox -e py38 + +[testenv:iscript-py37] +changedir = {toxinidir}/iscript +commands = + tox -e py37,check + - tox -e coveralls + [testenv:notarization_poller-py38] changedir = {toxinidir}/notarization_poller commands = @@ -153,6 +153,22 @@ changedir = {toxinidir}/pushsnapscript commands = tox -e py37 +[testenv:scriptworker_client-py38] +changedir = {toxinidir}/scriptworker_client +commands = + tox -e py38,mypy + - tox -e coveralls + +[testenv:scriptworker_client-py37] +changedir = {toxinidir}/scriptworker_client +commands = + tox -e py37,mypy + +[testenv:shipitscript-py38] +changedir = {toxinidir}/shipitscript +commands = + tox -e py38 + [testenv:shipitscript-py37] changedir = {toxinidir}/shipitscript commands = @@ -179,14 +195,3 @@ commands = changedir = {toxinidir}/treescript commands = tox -e py37 - -[testenv:configloader-py38] -changedir = {toxinidir}/configloader -commands = - tox -e py38,check - - tox -e coveralls - -[testenv:configloader-py37] -changedir = {toxinidir}/configloader -commands = - tox -e py37 From 181f03b2d93ab98758fbc6d6f3f4668f7da40791 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 5 Jun 2020 14:01:32 -0700 Subject: [PATCH 02/32] add pr kind --- taskcluster/ci/k8s-image/kind.yml | 2 ++ taskcluster/ci/pr/kind.yml | 18 ++++++++++++++++++ taskcluster/ci/tox/kind.yml | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 taskcluster/ci/pr/kind.yml diff --git a/taskcluster/ci/k8s-image/kind.yml b/taskcluster/ci/k8s-image/kind.yml index 9712b35a1..708c8ff51 100644 --- a/taskcluster/ci/k8s-image/kind.yml +++ b/taskcluster/ci/k8s-image/kind.yml @@ -18,6 +18,8 @@ job-defaults: description: "{name} k8s image builder py{python_version}" label: "{name}-k8s-image-builder-python{python_version}" run-on-tasks-for: ["action", "github-pull-request", "github-push"] + attributes: + code-review: true worker-type: images worker: docker-image: {in-tree: 'k8s-image'} diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml new file mode 100644 index 000000000..443a2d897 --- /dev/null +++ b/taskcluster/ci/pr/kind.yml @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - k8s-image + +transforms: + - taskgraph.transforms.code_review:transforms + - taskgraph.transforms.task:transforms + +jobs: + complete: + description: PR Summary Task + run-on-tasks-for: [github-pull-request] + worker-type: succeed diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index 1d9d937dd..9c108f2d8 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -18,6 +18,8 @@ job-defaults: description: "{name} tox-py{python_version}" label: "tox-{name}-py{python_version}" run-on-tasks-for: ["action", "github-pull-request", "github-push"] + attributes: + code-review: true worker-type: b-linux worker: docker-image: {in-tree: 'python{python_version}'} From d87c28ce9b989e0ed17d0e280aa8c3feaae9cc38 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 11:52:26 -0700 Subject: [PATCH 03/32] address review comments --- taskcluster/ci/config.yml | 6 ++---- taskcluster/scriptworker_taskgraph/transforms/cached.py | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 17e267276..a2b5f1f94 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -32,10 +32,8 @@ docker: notify: email: - # TODO by-project/level - - aki+scriptworker-scripts@mozilla.com - # TODO by-project/level - prefix: "[scriptworker-scripts stage]" + - release+scriptworker-scripts@mozilla.com + prefix: "[scriptworker-scripts]" release-promotion: flavors: diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index ac393ee0c..af019d68a 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -1,6 +1,9 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build the cached_task digest to prevent rerunning tasks if the code hasn't changed. +""" from __future__ import absolute_import, print_function, unicode_literals From ca84d5d4fa99177c1afc8f7a5ce18ccac54387e0 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 12:32:16 -0700 Subject: [PATCH 04/32] add tox to pr kind-dependencies --- taskcluster/ci/pr/kind.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml index 443a2d897..95e0a171f 100644 --- a/taskcluster/ci/pr/kind.yml +++ b/taskcluster/ci/pr/kind.yml @@ -5,6 +5,7 @@ loader: taskgraph.loader.transform:loader kind-dependencies: + - tox - k8s-image transforms: From 8b762c81c1416c20227b751737ec858bebc41e58 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 12:38:00 -0700 Subject: [PATCH 05/32] dict comprehension --- .../scriptworker_taskgraph/transforms/python_version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py index 57ddfc3b3..66a133543 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/python_version.py +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -15,8 +15,7 @@ def _replace_string(obj, repl_dict): if isinstance(obj, dict): - for k in obj.keys(): - obj[k] = obj[k].format(**repl_dict) + return {k: v.format(**repl_dict) for k, v in obj.items()} elif isinstance(obj, list): for c in range(0, len(obj)): obj[c] = obj[c].format(**repl_dict) From fef184bb7fbb286a21b4b60a6130cceebe56b7a9 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 14:26:07 -0700 Subject: [PATCH 06/32] don't pre-hash digest-extra; don't set empty `cached_task` attribute --- taskcluster/scriptworker_taskgraph/transforms/cached.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index af019d68a..a8dea1cca 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -68,8 +68,7 @@ def build_cache(config, tasks): for task in tasks: if task.get("cache", True) and not taskgraph.fast: digest_data = [] - h = hashlib.sha256() - h.update( + digest_data.append( json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) ) directories = task.get("attributes", {}).get("digest-directories", []) @@ -81,13 +80,13 @@ def build_cache(config, tasks): for path in ADDITIONAL_FILES: if os.path.exists(path): files.update({path}) + h = hashlib.sha256() for path in sorted(list(files)): h.update( "{} {}\n".format( hash_path(os.path.realpath(os.path.join(BASE_DIR, path))), path ) ) - task.setdefault("attributes", {}).setdefault("cached_task", {}) cache_name = task["label"].replace(":", "-") task["cache"] = { "type": "{}.v2".format(repo_name), From 043b93efb7e3e7235162dcce1434e1d2afffbf42 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 14:53:41 -0700 Subject: [PATCH 07/32] digest-paths, using hash_paths --- taskcluster/ci/tox/kind.yml | 28 ++++----- .../transforms/cached.py | 60 +++---------------- .../transforms/k8s_image.py | 20 +++---- 3 files changed, 33 insertions(+), 75 deletions(-) diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index 9c108f2d8..57288cb96 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -47,56 +47,56 @@ job-defaults: jobs: addonscript: - digest-directories: + digest-paths: - addonscript balrogscript: - digest-directories: + digest-paths: - balrogscript - scriptworker_client beetmoverscript: - digest-directories: + digest-paths: - beetmoverscript bouncerscript: - digest-directories: + digest-paths: - bouncerscript configloader: - digest-directories: + digest-paths: - configloader iscript: - digest-directories: + digest-paths: - iscript - scriptworker_client - vendored/mozbuild notarization_poller: - digest-directories: + digest-paths: - notarization_poller - scriptworker_client pushapkscript: worker: docker-image: {in-tree: 'pushapkscript-py{python_version}'} - digest-directories: + digest-paths: - pushapkscript pushflatpakscript: worker: docker-image: {in-tree: 'pushflatpakscript-py{python_version}'} - digest-directories: + digest-paths: - pushflatpakscript pushsnapscript: worker: docker-image: {in-tree: 'pushsnapscript-py{python_version}'} - digest-directories: + digest-paths: - pushsnapscript scriptworker_client: - digest-directories: + digest-paths: - scriptworker_client shipitscript: - digest-directories: + digest-paths: - shipitscript signingscript: - digest-directories: + digest-paths: - signingscript - vendored/mozbuild treescript: - digest-directories: + digest-paths: - treescript - scriptworker_client diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index a8dea1cca..b09b3ea75 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -14,48 +14,19 @@ import taskgraph from taskgraph.transforms.base import TransformSequence -from taskgraph.util.hash import hash_path +from taskgraph.util.hash import hash_paths from taskgraph.util.memoize import memoize transforms = TransformSequence() BASE_DIR = os.getcwd() -# Directory names we ignore, anywhere in the tree -# We won't recurse into these directories. -IGNORE_DIRS = (".git", "docs", "maintenance") - -# File extensions we ignore, anywhere in the tree -IGNORE_EXTENSIONS = (".pyc", ".swp") - -ADDITIONAL_FILES = tuple([]) - - -@memoize -def list_files(path): - files = [] - for dir_name, subdir_list, file_list in os.walk(path): - for dir_ in subdir_list: - if dir_ in IGNORE_DIRS: - subdir_list.remove(dir_) - continue - for file_ in file_list: - (_, ext) = os.path.splitext(file_) - if ext in IGNORE_EXTENSIONS: - continue - files.append( - os.path.relpath(os.path.join(BASE_DIR, dir_name, file_), BASE_DIR) - ) - return set(files) - - @transforms.add -def add_digest_directories(config, tasks): - # TODO `s,digest-directories,digest-files`, where we check isdir(file) ? +def add_digest_paths(config, tasks): for task in tasks: - digest_directories = task.pop("digest-directories", None) - if digest_directories: - task.setdefault("attributes", {})["digest-directories"] = digest_directories + digest_paths = task.pop("digest-paths", None) + if digest_paths: + task.setdefault("attributes", {})["digest-paths"] = digest_paths yield task @@ -71,27 +42,14 @@ def build_cache(config, tasks): digest_data.append( json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) ) - directories = task.get("attributes", {}).get("digest-directories", []) - files = set([]) - for d in directories: - directory = os.path.join(BASE_DIR, d) - files.update(list_files(directory)) - # files.update(list_files(os.path.join(BASE_DIR, "taskcluster"))) - for path in ADDITIONAL_FILES: - if os.path.exists(path): - files.update({path}) - h = hashlib.sha256() - for path in sorted(list(files)): - h.update( - "{} {}\n".format( - hash_path(os.path.realpath(os.path.join(BASE_DIR, path))), path - ) - ) + paths = task.get("attributes", {}).get("digest-paths", []) + for path in paths: + digest_data.append(hash_paths(os.path.join(BASE_DIR, path), [''])) cache_name = task["label"].replace(":", "-") task["cache"] = { "type": "{}.v2".format(repo_name), "name": cache_name, - "digest-data": [h.hexdigest()], + "digest-data": digest_data, } yield task diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py index 29b506e48..40c6b2380 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -19,14 +19,14 @@ def add_dependencies(config, jobs): """Add dependencies that match python-version and script-name. - Also copy the digest-directories attribute, and fail if there are + Also copy the digest-paths attribute, and fail if there are unexpected discrepancies in upstream deps. """ for job in jobs: attributes = job["attributes"] dependencies = job.setdefault("dependencies", {}) - digest_directories = None + digest_paths = None for dep_task in config.kind_dependencies_tasks: dep_attrs = dep_task.attributes dep_kind = dep_task.kind @@ -39,15 +39,15 @@ def add_dependencies(config, jobs): new_label=dep_task.label, )) dependencies[dep_kind] = dep_task.label - if dep_attrs.get("digest-directories"): - if digest_directories and digest_directories != dep_attrs["digest-directories"]: - raise Exception("Conflicting digest_directories: {existing_digest} {new_digest}".format( - existing_digest=digest_directories, - new_digest=dep_attrs["digest-directories"], + if dep_attrs.get("digest-paths"): + if digest_paths and digest_paths != dep_attrs["digest-paths"]: + raise Exception("Conflicting digest_paths: {existing_digest} {new_digest}".format( + existing_digest=digest_paths, + new_digest=dep_attrs["digest-paths"], )) - digest_directories = dep_attrs["digest-directories"] - if digest_directories: - attributes["digest-directories"] = digest_directories + digest_paths = dep_attrs["digest-paths"] + if digest_paths: + attributes["digest-paths"] = digest_paths yield job From 32e55a0e667d16b6ee7b8b2e5e173cf5eb85e384 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 15:19:56 -0700 Subject: [PATCH 08/32] digest-paths -> resources --- taskcluster/ci/tox/kind.yml | 28 +++++++++---------- .../transforms/cached.py | 14 +++++----- .../transforms/k8s_image.py | 22 +++++++-------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index 57288cb96..0e15213d3 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -47,56 +47,56 @@ job-defaults: jobs: addonscript: - digest-paths: + resources: - addonscript balrogscript: - digest-paths: + resources: - balrogscript - scriptworker_client beetmoverscript: - digest-paths: + resources: - beetmoverscript bouncerscript: - digest-paths: + resources: - bouncerscript configloader: - digest-paths: + resources: - configloader iscript: - digest-paths: + resources: - iscript - scriptworker_client - vendored/mozbuild notarization_poller: - digest-paths: + resources: - notarization_poller - scriptworker_client pushapkscript: worker: docker-image: {in-tree: 'pushapkscript-py{python_version}'} - digest-paths: + resources: - pushapkscript pushflatpakscript: worker: docker-image: {in-tree: 'pushflatpakscript-py{python_version}'} - digest-paths: + resources: - pushflatpakscript pushsnapscript: worker: docker-image: {in-tree: 'pushsnapscript-py{python_version}'} - digest-paths: + resources: - pushsnapscript scriptworker_client: - digest-paths: + resources: - scriptworker_client shipitscript: - digest-paths: + resources: - shipitscript signingscript: - digest-paths: + resources: - signingscript - vendored/mozbuild treescript: - digest-paths: + resources: - treescript - scriptworker_client diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index b09b3ea75..7b1b825b1 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -22,11 +22,11 @@ BASE_DIR = os.getcwd() @transforms.add -def add_digest_paths(config, tasks): +def add_resources(config, tasks): for task in tasks: - digest_paths = task.pop("digest-paths", None) - if digest_paths: - task.setdefault("attributes", {})["digest-paths"] = digest_paths + resources = task.pop("resources", None) + if resources: + task.setdefault("attributes", {})["resources"] = resources yield task @@ -42,9 +42,9 @@ def build_cache(config, tasks): digest_data.append( json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) ) - paths = task.get("attributes", {}).get("digest-paths", []) - for path in paths: - digest_data.append(hash_paths(os.path.join(BASE_DIR, path), [''])) + resources = task.get("attributes", {}).get("resources", []) + for resource in resources: + digest_data.append(hash_paths(os.path.join(BASE_DIR, resource), [''])) cache_name = task["label"].replace(":", "-") task["cache"] = { "type": "{}.v2".format(repo_name), diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py index 40c6b2380..9d302e1bf 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -19,14 +19,14 @@ def add_dependencies(config, jobs): """Add dependencies that match python-version and script-name. - Also copy the digest-paths attribute, and fail if there are - unexpected discrepancies in upstream deps. + Also copy the resources attribute, and fail if there are unexpected + discrepancies in upstream deps. """ for job in jobs: attributes = job["attributes"] dependencies = job.setdefault("dependencies", {}) - digest_paths = None + resources = None for dep_task in config.kind_dependencies_tasks: dep_attrs = dep_task.attributes dep_kind = dep_task.kind @@ -39,15 +39,15 @@ def add_dependencies(config, jobs): new_label=dep_task.label, )) dependencies[dep_kind] = dep_task.label - if dep_attrs.get("digest-paths"): - if digest_paths and digest_paths != dep_attrs["digest-paths"]: - raise Exception("Conflicting digest_paths: {existing_digest} {new_digest}".format( - existing_digest=digest_paths, - new_digest=dep_attrs["digest-paths"], + if dep_attrs.get("resources"): + if resources and resources != dep_attrs["resources"]: + raise Exception("Conflicting resources: {existing_digest} {new_digest}".format( + existing_digest=resources, + new_digest=dep_attrs["resources"], )) - digest_paths = dep_attrs["digest-paths"] - if digest_paths: - attributes["digest-paths"] = digest_paths + resources = dep_attrs["resources"] + if resources: + attributes["resources"] = resources yield job From 846eab30fe5a128d0082c37fedcd2aeda420ed0c Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 16:08:44 -0700 Subject: [PATCH 09/32] script_name transform --- .../transforms/python_version.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py index 66a133543..a4402f81c 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/python_version.py +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -24,6 +24,15 @@ def _replace_string(obj, repl_dict): return obj +@transforms.add +def set_script_name(config, jobs): + for job in jobs: + job.setdefault("attributes", {}).update({ + "script-name": job["name"], + }) + yield job + + @transforms.add def tasks_per_python_version(config, jobs): for job in jobs: @@ -40,7 +49,6 @@ def tasks_per_python_version(config, jobs): if task.get('docker-repo'): task['docker-repo'] = _replace_string(task['docker-repo'], repl_dict) task.setdefault("attributes", {}).update({ - "script-name": job["name"], "python-version": python_version, }) yield task From bf2a4581bcc6cb08d720945180391ceb2b106005 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 16:35:21 -0700 Subject: [PATCH 10/32] pushsnapscript - shared setup script --- pushsnapscript/Dockerfile | 15 +++++---------- pushsnapscript/Dockerfile.test | 5 +++-- pushsnapscript/docker.d/image_setup.sh | 12 ++++++++++++ taskcluster/docker/pushsnapscript/Dockerfile | 8 +++++--- 4 files changed, 25 insertions(+), 15 deletions(-) create mode 100755 pushsnapscript/docker.d/image_setup.sh diff --git a/pushsnapscript/Dockerfile b/pushsnapscript/Dockerfile index ad56549ef..87275f33c 100644 --- a/pushsnapscript/Dockerfile +++ b/pushsnapscript/Dockerfile @@ -3,20 +3,15 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y libsodium-dev \ - && apt-get install -y squashfs-tools \ - && apt-get install -y xdelta3 \ - && apt-get clean \ - # XXX Avoid snapcraft from loading useless libs when running on Ubuntu - && truncate -s 0 /etc/os-release \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app - COPY . /app +RUN /app/docker.d/image_setup.sh libsodium-dev squashfs-tools xdelta3 + +USER app + RUN python -m venv /app \ && cd pushsnapscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushsnapscript/Dockerfile.test b/pushsnapscript/Dockerfile.test index fe71ff840..72aafb15a 100644 --- a/pushsnapscript/Dockerfile.test +++ b/pushsnapscript/Dockerfile.test @@ -2,10 +2,11 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release - WORKDIR /app +COPY docker.d/image_setup.sh /app/ +RUN /app/image_setup.sh + COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ diff --git a/pushsnapscript/docker.d/image_setup.sh b/pushsnapscript/docker.d/image_setup.sh new file mode 100755 index 000000000..3f43cdd57 --- /dev/null +++ b/pushsnapscript/docker.d/image_setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -o errexit -o pipefail + +PKGS="${@:-libsodium-dev}" + +apt-get update +for pkg in $PKGS; do + apt-get install -y $pkg +done +apt-get clean +# XXX Avoid snapcraft from loading useless libs when running on Ubuntu +truncate -s 0 /etc/os-release diff --git a/taskcluster/docker/pushsnapscript/Dockerfile b/taskcluster/docker/pushsnapscript/Dockerfile index 92ffc25c7..973b30925 100644 --- a/taskcluster/docker/pushsnapscript/Dockerfile +++ b/taskcluster/docker/pushsnapscript/Dockerfile @@ -4,6 +4,8 @@ FROM $DOCKER_IMAGE_PARENT VOLUME /builds/worker/checkouts VOLUME /builds/worker/.cache -RUN apt-get update \ - && apt-get install -y libsodium-dev -RUN truncate -s 0 /etc/os-release +# %include pushsnapscript/docker.d/image_setup.sh +COPY topsrcdir/pushsnapscript/docker.d/image_setup.sh /usr/local/bin/ +RUN chmod a+x /usr/local/bin/image_setup.sh + +RUN image_setup.sh From e5fbab176a16a8443fcc8b8394c0c53155426a06 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 16:41:56 -0700 Subject: [PATCH 11/32] env.update --- .../scriptworker_taskgraph/transforms/k8s_image.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py index 9d302e1bf..27c81ead3 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -61,12 +61,14 @@ def set_environment(config, jobs): scopes = job.setdefault("scopes", []) attributes = job["attributes"] env = job["worker"].setdefault("env", {}) - env["HEAD_REV"] = config.params['head_rev'] - env["REPO_URL"] = config.params['head_repository'] - env["PROJECT_NAME"] = project_name - env["TASKCLUSTER_ROOT_URL"] = "$TASKCLUSTER_ROOT_URL" - env["DOCKER_TAG"] = "unknown" - env["DOCKER_REPO"] = job.pop("docker-repo") + env.update({ + "HEAD_REV": config.params['head_rev'], + "REPO_URL": config.params['head_repository'], + "PROJECT_NAME": project_name, + "TASKCLUSTER_ROOT_URL": "$TASKCLUSTER_ROOT_URL", + "DOCKER_TAG": "unknown", + "DOCKER_REPO": job.pop("docker-repo"), + }) force_push_docker_image = False if tasks_for == 'github-pull-request': env["DOCKER_TAG"] = "pull-request" From fad9aed7fae1693f3d64521d23a10001d43674ee Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 17:42:39 -0700 Subject: [PATCH 12/32] pushflatpakscript - shared setup script --- pushflatpakscript/Dockerfile | 11 +++++------ pushflatpakscript/Dockerfile.test | 7 ++++--- pushflatpakscript/docker.d/image_setup.sh | 6 ++++++ taskcluster/docker/pushflatpakscript/Dockerfile | 5 +++-- 4 files changed, 18 insertions(+), 11 deletions(-) create mode 100755 pushflatpakscript/docker.d/image_setup.sh diff --git a/pushflatpakscript/Dockerfile b/pushflatpakscript/Dockerfile index a37a2e390..11e731851 100644 --- a/pushflatpakscript/Dockerfile +++ b/pushflatpakscript/Dockerfile @@ -3,16 +3,15 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev \ - && apt-get clean \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app - COPY . /app +RUN /app/docker.d/image_setup.sh + +USER app + RUN python -m venv /app \ && cd pushflatpakscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushflatpakscript/Dockerfile.test b/pushflatpakscript/Dockerfile.test index c77050ba0..7cd0596b9 100644 --- a/pushflatpakscript/Dockerfile.test +++ b/pushflatpakscript/Dockerfile.test @@ -2,11 +2,12 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update \ - && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev - WORKDIR /app +COPY docker.d/image_setup.sh /app/ + +RUN /app/image_setup.sh + COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ diff --git a/pushflatpakscript/docker.d/image_setup.sh b/pushflatpakscript/docker.d/image_setup.sh new file mode 100755 index 000000000..b44b975ce --- /dev/null +++ b/pushflatpakscript/docker.d/image_setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -o errexit -o pipefail + +apt-get update +apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev +apt-get clean diff --git a/taskcluster/docker/pushflatpakscript/Dockerfile b/taskcluster/docker/pushflatpakscript/Dockerfile index feb94f208..3d7555ed8 100644 --- a/taskcluster/docker/pushflatpakscript/Dockerfile +++ b/taskcluster/docker/pushflatpakscript/Dockerfile @@ -4,5 +4,6 @@ FROM $DOCKER_IMAGE_PARENT VOLUME /builds/worker/checkouts VOLUME /builds/worker/.cache -RUN apt-get update \ - && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev +# %include pushflatpakscript/docker.d/image_setup.sh +COPY topsrcdir/pushflatpakscript/docker.d/image_setup.sh /usr/local/bin/ +RUN image_setup.sh From 8a1d21e273bd8aab5a3214461a3b93b790b7f2ff Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 18:31:41 -0700 Subject: [PATCH 13/32] pushapkscript - shared setup script --- pushapkscript/Dockerfile | 10 +++++----- pushapkscript/Dockerfile.test | 4 ++-- pushapkscript/docker.d/image_setup.sh | 6 ++++++ taskcluster/docker/pushapkscript/Dockerfile | 5 ++++- 4 files changed, 17 insertions(+), 8 deletions(-) create mode 100755 pushapkscript/docker.d/image_setup.sh diff --git a/pushapkscript/Dockerfile b/pushapkscript/Dockerfile index c85429e0b..078c30b11 100644 --- a/pushapkscript/Dockerfile +++ b/pushapkscript/Dockerfile @@ -3,16 +3,16 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y default-jdk \ - && apt-get clean \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app COPY . /app +RUN /app/docker.d/image_setup.sh + +USER app + RUN python -m venv /app \ && cd pushapkscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushapkscript/Dockerfile.test b/pushapkscript/Dockerfile.test index 5598cfbce..7bfe5c17e 100644 --- a/pushapkscript/Dockerfile.test +++ b/pushapkscript/Dockerfile.test @@ -2,12 +2,12 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update && apt-get install -y default-jdk && apt-get clean - WORKDIR /app COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ +COPY docker.d/image_setup.sh /app/ +RUN /app/image_setup.sh RUN pip install -r requirements/local.txt diff --git a/pushapkscript/docker.d/image_setup.sh b/pushapkscript/docker.d/image_setup.sh new file mode 100755 index 000000000..dbc4425ca --- /dev/null +++ b/pushapkscript/docker.d/image_setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -o errexit -o pipefail + +apt-get update +apt-get install -y default-jdk +apt-get clean diff --git a/taskcluster/docker/pushapkscript/Dockerfile b/taskcluster/docker/pushapkscript/Dockerfile index 4eaaf8054..8d0d5d2d3 100644 --- a/taskcluster/docker/pushapkscript/Dockerfile +++ b/taskcluster/docker/pushapkscript/Dockerfile @@ -4,4 +4,7 @@ FROM $DOCKER_IMAGE_PARENT VOLUME /builds/worker/checkouts VOLUME /builds/worker/.cache -RUN apt-get update && apt-get install -y default-jdk +# %include pushapkscript/docker.d/image_setup.sh +COPY topsrcdir/pushapkscript/docker.d/image_setup.sh /usr/local/bin/ + +RUN image_setup.sh From 379a31e3c41947ba751ff19072a9d6bebe60a022 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 18:52:12 -0700 Subject: [PATCH 14/32] _resolve_replace_string --- .../transforms/python_version.py | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py index a4402f81c..10c93bc31 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/python_version.py +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -13,17 +13,36 @@ transforms = TransformSequence() -def _replace_string(obj, repl_dict): +def _replace_string(obj, subs): if isinstance(obj, dict): - return {k: v.format(**repl_dict) for k, v in obj.items()} + return {k: v.format(**subs) for k, v in obj.items()} elif isinstance(obj, list): for c in range(0, len(obj)): - obj[c] = obj[c].format(**repl_dict) + obj[c] = obj[c].format(**subs) else: - obj = obj.format(**repl_dict) + obj = obj.format(**subs) return obj +def _resolve_replace_string(item, field, subs): + # largely from resolve_keyed_by + container, subfield = item, field + while '.' in subfield: + f, subfield = subfield.split('.', 1) + if f not in container: + return item + container = container[f] + if not isinstance(container, dict): + return item + + if subfield not in container: + return item + + container[subfield] = _replace_string(container[subfield], subs) + return item + + + @transforms.add def set_script_name(config, jobs): for job in jobs: @@ -35,22 +54,21 @@ def set_script_name(config, jobs): @transforms.add def tasks_per_python_version(config, jobs): + fields = [ + "description", + "docker-repo", + "label", + "run.command", + "worker.command", + "worker.docker-image", + ] for job in jobs: for python_version in job.pop("python-versions"): task = deepcopy(job) - repl_dict = {"name": job["name"], "python_version": python_version} - task["label"] = _replace_string(task["label"], repl_dict) - task['worker']['docker-image'] = _replace_string(task['worker']['docker-image'], repl_dict) - task['description'] = _replace_string(task['description'], repl_dict) - if task.get('run', {}).get('command'): - task['run']['command'] = _replace_string(task['run']['command'], repl_dict) - if task['worker'].get('command'): - task['worker']['command'] = _replace_string(task['worker']['command'], repl_dict) - if task.get('docker-repo'): - task['docker-repo'] = _replace_string(task['docker-repo'], repl_dict) - task.setdefault("attributes", {}).update({ - "python-version": python_version, - }) + subs = {"name": job["name"], "python_version": python_version} + for field in fields: + _resolve_replace_string(task, field, subs) + task["attributes"]["python-version"] = python_version yield task From 4a2d89e1d294f7c0671721cc9184be08382d94ba Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 15 Jun 2020 19:41:50 -0700 Subject: [PATCH 15/32] add parameters. still need to add target tasks method --- taskcluster/ci/config.yml | 1 + .../scriptworker_taskgraph/__init__.py | 1 + .../scriptworker_taskgraph/parameters.py | 35 +++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 taskcluster/scriptworker_taskgraph/parameters.py diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index a2b5f1f94..cb40260d9 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -5,6 +5,7 @@ task-priority: high taskgraph: register: scriptworker_taskgraph:register + decision-parameters: 'scriptworker_taskgraph.parameters:get_decision_parameters' repositories: scriptworker: name: "Scriptworker-scripts" diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py index ea8861a74..28dc815c6 100644 --- a/taskcluster/scriptworker_taskgraph/__init__.py +++ b/taskcluster/scriptworker_taskgraph/__init__.py @@ -13,6 +13,7 @@ def register(graph_config): the process. """ _import_modules([ + "parameters", "transforms", ]) diff --git a/taskcluster/scriptworker_taskgraph/parameters.py b/taskcluster/scriptworker_taskgraph/parameters.py new file mode 100644 index 000000000..296b68639 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/parameters.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.parameters import extend_parameters_schema +from voluptuous import ( + Any, + Optional, + Required, +) + +PROJECT_SPECIFIC_PREFIXES = ("refs/heads/dev-", "refs/heads/production-") + +scriptworker_schema = { + Optional('script_name'): Any(basestring, None), + Optional('script_revision'): Any(basestring, None), + Optional('shipping_phase'): Any("build", "promote", None), +} + +extend_parameters_schema(scriptworker_schema) + + +def get_decision_parameters(graph_config, parameters): + """Add repo-specific decision parameters. + + If we're on a production- or dev- branch, detect and set the `script_name`. + + """ + for prefix in PROJECT_SPECIFIC_PREFIXES: + if parameters["head_ref"].startswith(prefix): + parameters["script_name"] = parameters["head_ref"].replace(prefix, "") From 4c67bec47784e5193a9da8959f216477487dd6fe Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 15:16:42 -0700 Subject: [PATCH 16/32] cache type --- taskcluster/scriptworker_taskgraph/transforms/cached.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index 7b1b825b1..58f3c4499 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -45,9 +45,9 @@ def build_cache(config, tasks): resources = task.get("attributes", {}).get("resources", []) for resource in resources: digest_data.append(hash_paths(os.path.join(BASE_DIR, resource), [''])) - cache_name = task["label"].replace(":", "-") + cache_name = task["name"].replace(":", "-") task["cache"] = { - "type": "{}.v2".format(repo_name), + "type": "scriptworker-scripts.v1.{}".format(config.kind), "name": cache_name, "digest-data": digest_data, } From dfdd56b7622ae7f4042137298346d537e3689e32 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 15:01:45 -0700 Subject: [PATCH 17/32] label --- taskcluster/ci/k8s-image/kind.yml | 2 +- taskcluster/ci/tox/kind.yml | 2 +- taskcluster/scriptworker_taskgraph/transforms/cached.py | 8 ++++++++ .../scriptworker_taskgraph/transforms/python_version.py | 8 +++++++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/taskcluster/ci/k8s-image/kind.yml b/taskcluster/ci/k8s-image/kind.yml index 708c8ff51..c7d773133 100644 --- a/taskcluster/ci/k8s-image/kind.yml +++ b/taskcluster/ci/k8s-image/kind.yml @@ -16,7 +16,7 @@ transforms: job-defaults: description: "{name} k8s image builder py{python_version}" - label: "{name}-k8s-image-builder-python{python_version}" + name: "{name}-python{python_version}" run-on-tasks-for: ["action", "github-pull-request", "github-push"] attributes: code-review: true diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index 0e15213d3..74961dd7a 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -16,7 +16,7 @@ transforms: job-defaults: description: "{name} tox-py{python_version}" - label: "tox-{name}-py{python_version}" + name: "{name}-py{python_version}" run-on-tasks-for: ["action", "github-pull-request", "github-push"] attributes: code-review: true diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index 58f3c4499..14a1af34f 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -53,3 +53,11 @@ def build_cache(config, tasks): } yield task + + +@transforms.add +def set_label(config, jobs): + """Set the job label, which the `cached_tasks` transform needs""" + for job in jobs: + job["label"] = "{}-{}".format(config.kind, job.pop("name")) + yield job diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py index 10c93bc31..7a5be83bd 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/python_version.py +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -57,7 +57,6 @@ def tasks_per_python_version(config, jobs): fields = [ "description", "docker-repo", - "label", "run.command", "worker.command", "worker.docker-image", @@ -83,3 +82,10 @@ def skip_on_project_specific_branches(config, jobs): config.params['head_ref'] not in project_specific_branches: continue yield job + + +@transforms.add +def update_name_with_python_version(config, jobs): + for job in jobs: + job["name"] = "{}-python{}".format(job["name"], job["attributes"]["python-version"]) + yield job From 71ae5e9dd85b6b1a9a8c12f8ceb2824a8abf2d58 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 15:21:09 -0700 Subject: [PATCH 18/32] remove tox artifacts dir for now --- taskcluster/ci/tox/kind.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index 74961dd7a..fff463ce3 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -24,10 +24,6 @@ job-defaults: worker: docker-image: {in-tree: 'python{python_version}'} max-run-time: 1800 - artifacts: - - type: directory - name: public/ - path: /builds/worker/artifacts/ python-versions: - 38 - 37 From 8b0f04c6bab586128c182654b6d2aec7edf490bc Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 16:04:16 -0700 Subject: [PATCH 19/32] explicit docker-image dep for tox --- taskcluster/ci/tox/kind.yml | 1 + .../scriptworker_taskgraph/transforms/tox.py | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 taskcluster/scriptworker_taskgraph/transforms/tox.py diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index fff463ce3..d67333ca4 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -9,6 +9,7 @@ kind-dependencies: transforms: - scriptworker_taskgraph.transforms.python_version:transforms + - scriptworker_taskgraph.transforms.tox:transforms - scriptworker_taskgraph.transforms.cached:transforms - taskgraph.transforms.cached_tasks:transforms - taskgraph.transforms.job:transforms diff --git a/taskcluster/scriptworker_taskgraph/transforms/tox.py b/taskcluster/scriptworker_taskgraph/transforms/tox.py new file mode 100644 index 000000000..f3386cd69 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/tox.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Tox-specific transforms +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy +import time + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + """Explicitly add the docker-image task as a dependency. + + This needs to be done before the `cached_tasks` transform, so we can't + wait until the `build_docker_worker_payload` transform. + + From `build_docker_worker_payload`. + + """ + for job in jobs: + image = job['worker']['docker-image'] + if isinstance(image, dict): + if 'in-tree' in image: + name = image['in-tree'] + docker_image_task = 'build-docker-image-' + image['in-tree'] + job.setdefault('dependencies', {})['docker-image'] = docker_image_task + yield job From 0d85a8444c9620a7625b450b97b581998540143b Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 16:20:10 -0700 Subject: [PATCH 20/32] target tasks --- .../scriptworker_taskgraph/__init__.py | 1 + .../scriptworker_taskgraph/target_tasks.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 taskcluster/scriptworker_taskgraph/target_tasks.py diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py index 28dc815c6..b2c7d31b4 100644 --- a/taskcluster/scriptworker_taskgraph/__init__.py +++ b/taskcluster/scriptworker_taskgraph/__init__.py @@ -14,6 +14,7 @@ def register(graph_config): """ _import_modules([ "parameters", + "target_tasks", "transforms", ]) diff --git a/taskcluster/scriptworker_taskgraph/target_tasks.py b/taskcluster/scriptworker_taskgraph/target_tasks.py new file mode 100644 index 000000000..ec8426f42 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/target_tasks.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.target_tasks import _target_task, filter_for_tasks_for + + +@_target_task('default') +def target_tasks_default(full_task_graph, parameters, graph_config): + """Filter by `run_on_tasks_for` and `script-name`.""" + + def filter(task, parameters): + if not filter_for_tasks_for(task, parameters): + return False + if parameters.get("script_name"): + return task.attributes.get("script-name") == parameters["script_name"] + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] From a69a77cb033bbdee5c02fe5f6465ef6cf01431e5 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 16:36:13 -0700 Subject: [PATCH 21/32] parameters['docker_tag'] --- .../scriptworker_taskgraph/parameters.py | 18 ++++++++++--- .../transforms/k8s_image.py | 27 +++++++------------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/parameters.py b/taskcluster/scriptworker_taskgraph/parameters.py index 296b68639..8497f26ff 100644 --- a/taskcluster/scriptworker_taskgraph/parameters.py +++ b/taskcluster/scriptworker_taskgraph/parameters.py @@ -13,9 +13,13 @@ Required, ) -PROJECT_SPECIFIC_PREFIXES = ("refs/heads/dev-", "refs/heads/production-") +PROJECT_SPECIFIC_PREFIXES = { + "refs/heads/dev-": "dev", + "refs/heads/production-": "production", +} scriptworker_schema = { + Optional('docker_tag'): Any(basestring, None), Optional('script_name'): Any(basestring, None), Optional('script_revision'): Any(basestring, None), Optional('shipping_phase'): Any("build", "promote", None), @@ -30,6 +34,12 @@ def get_decision_parameters(graph_config, parameters): If we're on a production- or dev- branch, detect and set the `script_name`. """ - for prefix in PROJECT_SPECIFIC_PREFIXES: - if parameters["head_ref"].startswith(prefix): - parameters["script_name"] = parameters["head_ref"].replace(prefix, "") + if parameters["tasks_for"] == "github-pull-request": + parameters["docker_tag" ] = "github-pull-request" + elif parameters["head_ref"].startswith("refs/heads/"): + parameters["docker_tag"] = parameters["head_ref"].replace("refs/heads/", "") + for prefix, tag in PROJECT_SPECIFIC_PREFIXES.items(): + if parameters["head_ref"].startswith(prefix): + parameters["script_name"] = parameters["head_ref"].replace(prefix, "") + parameters["docker_tag" ] = tag + break diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py index 27c81ead3..7f588b28e 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -63,29 +63,20 @@ def set_environment(config, jobs): env = job["worker"].setdefault("env", {}) env.update({ "HEAD_REV": config.params['head_rev'], - "REPO_URL": config.params['head_repository'], + "DOCKER_REPO": job.pop("docker-repo"), + "DOCKER_TAG": config.params.get('docker_tag', 'unknown'), "PROJECT_NAME": project_name, + "REPO_URL": config.params['head_repository'], "TASKCLUSTER_ROOT_URL": "$TASKCLUSTER_ROOT_URL", - "DOCKER_TAG": "unknown", - "DOCKER_REPO": job.pop("docker-repo"), }) force_push_docker_image = False - if tasks_for == 'github-pull-request': - env["DOCKER_TAG"] = "pull-request" - elif tasks_for == 'github-push': - for ref_name in ("dev", "production"): - if config.params['head_ref'] == "refs/heads/{}-{}".format(ref_name, project_name): - env["DOCKER_TAG"] = ref_name - force_push_docker_image = True - break - else: - if config.params['head_ref'].startswith('refs/heads/'): - env["DOCKER_TAG"] = config.params['head_ref'].replace('refs/heads/', '') if env["DOCKER_TAG"] in ("production", "dev") and config.params["level"] == "3": - env["SECRET_URL"] = secret_url - env["PUSH_DOCKER_IMAGE"] = "1" - env["DOCKERHUB_EMAIL"] = config.graph_config["docker"]["email"] - env["DOCKERHUB_USER"] = config.graph_config["docker"]["user"] + env.update({ + "SECRET_URL": secret_url, + "PUSH_DOCKER_IMAGE": "1", + "DOCKERHUB_EMAIL": config.graph_config["docker"]["email"], + "DOCKERHUB_USER": config.graph_config["docker"]["user"], + }) scopes.append('secrets:get:project/releng/scriptworker-scripts/deploy') if force_push_docker_image: attributes.setdefault("digest-extra", {}).setdefault("force_run", time.time()) From 21d07e295e03d48fdf13f601d4c8c401c041c58d Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 16:47:01 -0700 Subject: [PATCH 22/32] docker-image py3{7,8} -> python3{7,8} --- taskcluster/ci/docker-image/kind.yml | 12 ++++++------ taskcluster/ci/tox/kind.yml | 7 +++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index c6098e0f0..6746cde87 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -21,21 +21,21 @@ jobs: definition: python args: PYTHON_VERSION: "3.7" - pushapkscript-py38: + pushapkscript-python38: definition: pushapkscript parent: python38 - pushapkscript-py37: + pushapkscript-python37: definition: pushapkscript parent: python37 - pushsnapscript-py38: + pushsnapscript-python38: definition: pushsnapscript parent: python38 - pushsnapscript-py37: + pushsnapscript-python37: definition: pushsnapscript parent: python37 - pushflatpakscript-py38: + pushflatpakscript-python38: definition: pushflatpakscript parent: python38 - pushflatpakscript-py37: + pushflatpakscript-python37: definition: pushflatpakscript parent: python37 diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml index d67333ca4..413023016 100644 --- a/taskcluster/ci/tox/kind.yml +++ b/taskcluster/ci/tox/kind.yml @@ -17,7 +17,6 @@ transforms: job-defaults: description: "{name} tox-py{python_version}" - name: "{name}-py{python_version}" run-on-tasks-for: ["action", "github-pull-request", "github-push"] attributes: code-review: true @@ -70,17 +69,17 @@ jobs: - scriptworker_client pushapkscript: worker: - docker-image: {in-tree: 'pushapkscript-py{python_version}'} + docker-image: {in-tree: 'pushapkscript-python{python_version}'} resources: - pushapkscript pushflatpakscript: worker: - docker-image: {in-tree: 'pushflatpakscript-py{python_version}'} + docker-image: {in-tree: 'pushflatpakscript-python{python_version}'} resources: - pushflatpakscript pushsnapscript: worker: - docker-image: {in-tree: 'pushsnapscript-py{python_version}'} + docker-image: {in-tree: 'pushsnapscript-python{python_version}'} resources: - pushsnapscript scriptworker_client: From fae03144a1e13d3e0b3adf29ea3917273aa7359f Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 16 Jun 2020 17:27:20 -0700 Subject: [PATCH 23/32] push_docker_image param --- taskcluster/scriptworker_taskgraph/parameters.py | 10 ++++++++++ .../scriptworker_taskgraph/transforms/k8s_image.py | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/parameters.py b/taskcluster/scriptworker_taskgraph/parameters.py index 8497f26ff..4fede5d06 100644 --- a/taskcluster/scriptworker_taskgraph/parameters.py +++ b/taskcluster/scriptworker_taskgraph/parameters.py @@ -18,8 +18,11 @@ "refs/heads/production-": "production", } +PUSH_TAGS = ("dev", "production") + scriptworker_schema = { Optional('docker_tag'): Any(basestring, None), + Optional('push_docker_image'): Any(True, "force", False, None), Optional('script_name'): Any(basestring, None), Optional('script_revision'): Any(basestring, None), Optional('shipping_phase'): Any("build", "promote", None), @@ -38,8 +41,15 @@ def get_decision_parameters(graph_config, parameters): parameters["docker_tag" ] = "github-pull-request" elif parameters["head_ref"].startswith("refs/heads/"): parameters["docker_tag"] = parameters["head_ref"].replace("refs/heads/", "") + force_push = False for prefix, tag in PROJECT_SPECIFIC_PREFIXES.items(): if parameters["head_ref"].startswith(prefix): parameters["script_name"] = parameters["head_ref"].replace(prefix, "") parameters["docker_tag" ] = tag + force_push = True break + if parameters["docker_tag"] in PUSH_TAGS and parameters["level"] == "3": + if force_push: + parameters["push_docker_image"] = "force" + else: + parameters["push_docker_image"] = True diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py index 7f588b28e..d42df7bda 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -69,8 +69,8 @@ def set_environment(config, jobs): "REPO_URL": config.params['head_repository'], "TASKCLUSTER_ROOT_URL": "$TASKCLUSTER_ROOT_URL", }) - force_push_docker_image = False - if env["DOCKER_TAG"] in ("production", "dev") and config.params["level"] == "3": + push_docker_image = config.params.get("push_docker_image") + if push_docker_image: env.update({ "SECRET_URL": secret_url, "PUSH_DOCKER_IMAGE": "1", @@ -78,7 +78,7 @@ def set_environment(config, jobs): "DOCKERHUB_USER": config.graph_config["docker"]["user"], }) scopes.append('secrets:get:project/releng/scriptworker-scripts/deploy') - if force_push_docker_image: + if push_docker_image == "force": attributes.setdefault("digest-extra", {}).setdefault("force_run", time.time()) else: env["PUSH_DOCKER_IMAGE"] = "0" From fcfbafe772157aef6f60a6451fed68d012e5d593 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:41:04 -0700 Subject: [PATCH 24/32] Update taskcluster/docker/k8s-image/Dockerfile Co-authored-by: Johan Lorenzo --- taskcluster/docker/k8s-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/k8s-image/Dockerfile b/taskcluster/docker/k8s-image/Dockerfile index af42ac99b..25441584d 100644 --- a/taskcluster/docker/k8s-image/Dockerfile +++ b/taskcluster/docker/k8s-image/Dockerfile @@ -4,7 +4,7 @@ # We need to run a really old docker version because taskcluster is using a # really old version in their setup -FROM docker:1.6.2 +FROM docker:1.6.2 RUN apk add --update xz RUN mkdir -p /usr/local/bin From da9140801f995690159276b176c3f69c50a4d56a Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:36:24 -0700 Subject: [PATCH 25/32] add comment to pushsnapscript/Dockerfile --- pushsnapscript/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/pushsnapscript/Dockerfile b/pushsnapscript/Dockerfile index 87275f33c..0f905a435 100644 --- a/pushsnapscript/Dockerfile +++ b/pushsnapscript/Dockerfile @@ -8,6 +8,7 @@ RUN ln -s /app/docker.d/healthcheck /bin/healthcheck WORKDIR /app COPY . /app +# Production image requires extra packages RUN /app/docker.d/image_setup.sh libsodium-dev squashfs-tools xdelta3 USER app From e9eaf62e477388caa5c736e81a754b8a112a918d Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:42:44 -0700 Subject: [PATCH 26/32] rm chmod from pushsnapscript tox Dockerfile --- taskcluster/docker/pushsnapscript/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/taskcluster/docker/pushsnapscript/Dockerfile b/taskcluster/docker/pushsnapscript/Dockerfile index 973b30925..1831be31d 100644 --- a/taskcluster/docker/pushsnapscript/Dockerfile +++ b/taskcluster/docker/pushsnapscript/Dockerfile @@ -6,6 +6,5 @@ VOLUME /builds/worker/.cache # %include pushsnapscript/docker.d/image_setup.sh COPY topsrcdir/pushsnapscript/docker.d/image_setup.sh /usr/local/bin/ -RUN chmod a+x /usr/local/bin/image_setup.sh RUN image_setup.sh From 82815d3081aa60c60d62486fc9f630419affd8f5 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:43:23 -0700 Subject: [PATCH 27/32] Update taskcluster/scriptworker_taskgraph/parameters.py Co-authored-by: Johan Lorenzo --- taskcluster/scriptworker_taskgraph/parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/scriptworker_taskgraph/parameters.py b/taskcluster/scriptworker_taskgraph/parameters.py index 4fede5d06..e10862da2 100644 --- a/taskcluster/scriptworker_taskgraph/parameters.py +++ b/taskcluster/scriptworker_taskgraph/parameters.py @@ -38,7 +38,7 @@ def get_decision_parameters(graph_config, parameters): """ if parameters["tasks_for"] == "github-pull-request": - parameters["docker_tag" ] = "github-pull-request" + parameters["docker_tag"] = "github-pull-request" elif parameters["head_ref"].startswith("refs/heads/"): parameters["docker_tag"] = parameters["head_ref"].replace("refs/heads/", "") force_push = False From e094802c86c203f0d843e08c5b06c5718aee543a Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:45:00 -0700 Subject: [PATCH 28/32] remove unused repo_name --- taskcluster/scriptworker_taskgraph/transforms/cached.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index 14a1af34f..b86ec2240 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -32,10 +32,6 @@ def add_resources(config, tasks): @transforms.add def build_cache(config, tasks): - repo_name = subprocess.check_output(["git", "remote", "get-url", "origin"]).rstrip() - repo_name = repo_name.replace(".git", "").rstrip("/") - repo_name = repo_name.split("/")[-1] - for task in tasks: if task.get("cache", True) and not taskgraph.fast: digest_data = [] From aaa9d7eda47cbcc63a42184589e0bef89f2842ef Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 18:56:15 -0700 Subject: [PATCH 29/32] address resources review comments --- .../scriptworker_taskgraph/transforms/cached.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index b86ec2240..967a87a83 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -24,9 +24,15 @@ @transforms.add def add_resources(config, tasks): for task in tasks: - resources = task.pop("resources", None) - if resources: - task.setdefault("attributes", {})["resources"] = resources + resources = task.pop("resources", []) + attributes = task.setdefault("attributes", {}) + if attributes.get("resources") is not None and attributes["resources"] != resources: + raise Exception( + "{} {} task.attributes.resources is already set to {}!".format( + config.kind, task.get("name"), attributes["resources"], + ) + ) + attributes["resources"] = resources yield task @@ -38,7 +44,7 @@ def build_cache(config, tasks): digest_data.append( json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) ) - resources = task.get("attributes", {}).get("resources", []) + resources = task["attributes"]["resources"] for resource in resources: digest_data.append(hash_paths(os.path.join(BASE_DIR, resource), [''])) cache_name = task["name"].replace(":", "-") From bb294947498c60b2e811a3a30eebadef6746cf36 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 19:01:02 -0700 Subject: [PATCH 30/32] fix --- taskcluster/scriptworker_taskgraph/transforms/cached.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index 967a87a83..c0bf2dd71 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -28,8 +28,8 @@ def add_resources(config, tasks): attributes = task.setdefault("attributes", {}) if attributes.get("resources") is not None and attributes["resources"] != resources: raise Exception( - "{} {} task.attributes.resources is already set to {}!".format( - config.kind, task.get("name"), attributes["resources"], + "setting {} {} task.attributes.resources to {}: it's already set to {}!".format( + config.kind, task.get("name"), resources, attributes["resources"], ) ) attributes["resources"] = resources From f5dbbe785dc928ab6854a1fa9cfa615c883e3156 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 19:04:13 -0700 Subject: [PATCH 31/32] adjust checks --- .../scriptworker_taskgraph/transforms/cached.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py index c0bf2dd71..b56477004 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/cached.py +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -26,12 +26,13 @@ def add_resources(config, tasks): for task in tasks: resources = task.pop("resources", []) attributes = task.setdefault("attributes", {}) - if attributes.get("resources") is not None and attributes["resources"] != resources: - raise Exception( - "setting {} {} task.attributes.resources to {}: it's already set to {}!".format( - config.kind, task.get("name"), resources, attributes["resources"], + if attributes.get("resources") is not None: + if resources and attributes["resources"] != resources: + raise Exception( + "setting {} {} task.attributes.resources to {}: it's already set to {}!".format( + config.kind, task.get("name"), resources, attributes["resources"], + ) ) - ) attributes["resources"] = resources yield task From babf19bc99654ae4ec7acb0f2a708339e78a920e Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Wed, 17 Jun 2020 19:09:41 -0700 Subject: [PATCH 32/32] remove skip_on_project_specific_branches --- .../transforms/python_version.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py index 7a5be83bd..9f93c5f1b 100644 --- a/taskcluster/scriptworker_taskgraph/transforms/python_version.py +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -71,19 +71,6 @@ def tasks_per_python_version(config, jobs): yield task -@transforms.add -def skip_on_project_specific_branches(config, jobs): - """Skip if the branch is project-specific for a different project.""" - project_specific_prefixes = ("refs/heads/dev-", "refs/heads/production-") - for job in jobs: - script_name = job["attributes"]["script-name"] - project_specific_branches = ["{}{}".format(prefix, script_name) for prefix in project_specific_prefixes] - if config.params['head_ref'].startswith(project_specific_prefixes) and \ - config.params['head_ref'] not in project_specific_branches: - continue - yield job - - @transforms.add def update_name_with_python_version(config, jobs): for job in jobs: