diff --git a/.taskcluster.yml b/.taskcluster.yml index c678e8073..0933c6251 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,210 +1,254 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- version: 1 +reporting: checks-v1 policy: - pullRequests: public + pullRequests: collaborators tasks: - $let: - # ------------------------------------------------------------------------- - # -- This is where you add new projects ----------------------------------- - # ------------------------------------------------------------------------- - PROJECTS: - $let: - setup_pushapkscript: 'apt-get update && apt-get install -y default-jdk &&' - setup_pushsnapscript: 'apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release &&' - setup_pushflatpakscript: 'apt-get update && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev &&' + - $let: + taskgraph: + branch: taskgraph + revision: ec220e56cf107ea279d7c08706fc370374869550 + trustDomain: scriptworker in: - # [ , , , ] - - ['client', '37', '', ''] - - ['client', '38', '', ''] - - ['configloader', '37', '', ''] - - ['configloader', '38', '', ''] - - ['iscript', '37', '', ''] - - ['iscript', '38', '', ''] - - ['addonscript', '37', '', ''] - - ['addonscript', '38', '', 'mozilla/releng-addonscript'] - - ['balrogscript', '37', '', ''] - - ['balrogscript', '38', '', 'mozilla/releng-balrogscript'] - - ['beetmoverscript', '37', '', ''] - - ['beetmoverscript', '38', '', 'mozilla/releng-beetmoverscript'] - - ['bouncerscript', '37', '', ''] - - ['bouncerscript', '38', '', 'mozilla/releng-bouncerscript'] - - ['notarization_poller', '37', '', ''] - - ['notarization_poller', '38', '', ''] - - ['pushapkscript', '37', '${setup_pushapkscript}', ''] - - ['pushapkscript', '38', '${setup_pushapkscript}', 'mozilla/releng-pushapkscript'] - - ['pushflatpakscript', '37', '${setup_pushflatpakscript}', ''] - - ['pushflatpakscript', '38', '${setup_pushflatpakscript}', 'mozilla/releng-pushflatpakscript'] - - ['pushsnapscript', '37', '${setup_pushsnapscript}', ''] - - ['pushsnapscript', '38', '${setup_pushsnapscript}', 'mozilla/releng-pushsnapscript'] - - ['shipitscript', '37', '', ''] - - ['shipitscript', '38', '', 'mozilla/releng-shipitscript'] - - ['signingscript', '37', '', ''] - - ['signingscript', '38', '', 'mozilla/releng-signingscript'] - - ['treescript', '37', '', ''] - - ['treescript', '38', '', 'mozilla/releng-treescript'] - # ------------------------------------------------------------------------- + $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]' + then: + $let: + # Github events have this stuff in different places... + ownerEmail: + $if: 'tasks_for == "github-push"' + then: '${event.pusher.email}' + # Assume Pull Request + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.user.login}@users.noreply.github.com' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${tasks_for}@noreply.mozilla.org' + baseRepoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.base.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + repoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + project: + $if: 'tasks_for == "github-push"' + then: '${event.repository.name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.name}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.project}' + head_branch: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.ref} + else: + $if: 'tasks_for == "github-push"' + then: ${event.ref} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.branch}' + head_sha: + $if: 'tasks_for == "github-push"' + then: '${event.after}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.sha}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.revision}' + ownTaskId: + $if: '"github" in tasks_for' + then: {$eval: as_slugid("decision_task")} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${ownTaskId}' + in: + $let: + level: + $if: 'tasks_for in ["github-push", "cron", "action"] && repoUrl == "https://github.com/mozilla-releng/scriptworker-scripts"' + then: 3 + else: 1 + in: + taskId: + $if: 'tasks_for != "action"' + then: '${ownTaskId}' + taskGroupId: + $if: 'tasks_for == "action"' + then: + '${action.taskGroupId}' + else: + '${ownTaskId}' # same as taskId; this is how automation identifies a decision task + schedulerId: '${trustDomain}-level-${level}' + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors + metadata: + $merge: + - owner: "${ownerEmail}" + source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + name: "Decision Task" + description: 'The task that creates all of the other tasks in the task graph' + else: + $if: 'tasks_for == "action"' + then: + name: "Action: ${action.title}" + description: '${action.description}' + else: + name: "Decision Task for cron job ${cron.job_name}" + description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})' + provisionerId: "${trustDomain}-${level}" + workerType: "decision" + tags: + $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + kind: decision-task + else: + $if: 'tasks_for == "action"' + then: + kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task + routes: + $flatten: + - checks + - $if: 'tasks_for == "github-push"' + then: + - "index.${trustDomain}.v2.${project}.revision.${head_sha}.taskgraph.decision" + else: [] + scopes: + # `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol. + $if: 'tasks_for == "github-push"' + then: + $let: + short_head_branch: + $if: 'head_branch[:10] == "refs/tags/"' + then: {$eval: 'head_branch[10:]'} + else: + $if: 'head_branch[:11] == "refs/heads/"' + then: {$eval: 'head_branch[11:]'} + else: ${head_branch} + in: + - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' - HEAD_REV: - $if: 'tasks_for == "github-pull-request"' - then: '${event.pull_request.head.sha}' - else: - $if: 'tasks_for == "github-push"' - then: '${event.after}' - else: '${event.release.tag_name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: + - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' + else: + $if: 'tasks_for == "action"' + then: + # when all actions are hooks, we can calculate this directly rather than using a variable + - '${action.repo_scope}' + else: + - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' - REPO_URL: - $if: 'tasks_for == "github-pull-request"' - then: '${event.pull_request.head.repo.html_url}' - else: '${event.repository.html_url}' + requires: all-completed + priority: lowest + retries: 5 - OWNER: '${event.sender.login}@users.noreply.github.com' + payload: + env: + # run-task uses these to check out the source; the inputs + # to `mach taskgraph decision` are all on the command line. + $merge: + - SCRIPTWORKER_BASE_REPOSITORY: '${baseRepoUrl}' + SCRIPTWORKER_HEAD_REPOSITORY: '${repoUrl}' + SCRIPTWORKER_HEAD_REF: '${head_branch}' + SCRIPTWORKER_HEAD_REV: '${head_sha}' + SCRIPTWORKER_REPOSITORY_TYPE: git + TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph + TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch} + TASKGRAPH_HEAD_REV: '${taskgraph.revision}' + TASKGRAPH_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {scriptworker: "Scriptworker-scripts", taskgraph: "Taskgraph"}} + HG_STORE_PATH: /builds/worker/checkouts/hg-store + - $if: 'tasks_for in ["github-pull-request"]' + then: + SCRIPTWORKER_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + - $if: 'tasks_for == "action"' + then: + ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task + ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) + ACTION_INPUT: {$json: {$eval: 'input'}} + ACTION_CALLBACK: '${action.cb_name}' + features: + taskclusterProxy: true + chainOfTrust: true + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the hash + image: mozillareleases/taskgraph:decision-21bef1bc0f11e62c7a23384584f9f8f0d96e95eef192e5bb599fc82ba55c81a7@sha256:d29af306b09cd00a63f982e8caeb53dbec7efd7bb3ae64ce3cef9cd889b750e6 + maxRunTime: 1800 - BRANCH_NAME: - $if: 'tasks_for == "github-pull-request"' - then: 'pull-request' - else: - $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"' - then: '${event.ref[11:]}' - else: 'unknown' - in: - $flatten: - $map: { "$eval": "PROJECTS" } - each(x,i): - $let: - project_name: { "$eval": "x[0]" } - python_version: { "$eval": "x[1]" } - setup_command: { "$eval": "x[2]" } - dockerhub_repo: { "$eval": "x[3]" } - run_tests: - $if: 'tasks_for == "github-pull-request"' - then: '1' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '1' - else: '0' - push_docker_image: - $if: 'tasks_for == "github-pull-request"' - then: '0' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '1' - else: '0' - worker_level: - $if: 'tasks_for == "github-pull-request"' - then: 't' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev","refs/heads/production","refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '3' - else: 't' - number_prefix: - $if: 'i + 1 < 10' - then: '0' - else: '' - docker_tag: - $if: 'tasks_for == "github-pull-request"' - then: 'pull-request' - else: - $if: 'tasks_for == "github-push" && event.ref in ["refs/heads/dev-" + x[0],"refs/heads/production-" + x[0]]' - then: '${event.ref[11:-(len(x[0]) + 1)]}' - else: - $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"' - then: '${event.ref[11:]}' - else: 'unknown' - in: - $match: - # Run code linting and unit tests for each project - 'run_tests == "1"': - taskId: '${as_slugid(project_name + python_version)}' - provisionerId: 'releng-t' - workerType: 'linux' - created: { $fromNow: '' } - deadline: { $fromNow: '4 hours' } - payload: - maxRunTime: 3600 - image: 'python:${python_version[0]}.${python_version[1]}' - command: - - sh - - -lxce - - >- - cd /tmp && - wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz && - tar zxf ${HEAD_REV}.tar.gz && - mv scriptworker-scripts-${HEAD_REV} /src && - cd /src && ${setup_command} - pip install tox && - tox -e ${project_name}-py${python_version} - metadata: - name: - $let: - test_task_number: - $if: 'dockerhub_repo != ""' - then: '${i+1}.1' - else: '${i+1}' - in: - '${number_prefix}${test_task_number}. ${project_name}-py${python_version}: Run tox [on ${BRANCH_NAME}]' - description: 'Code linting and unit tests for ${project_name} on python ${python_version[0]}.${python_version[1]}' - owner: '${OWNER}' - source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' - # Build docker image and (optionally) push to docker hub - 'run_tests == "1" && dockerhub_repo != ""': - taskId: '${as_slugid(project_name + "docker_build_and_push")}' - dependencies: - - '${as_slugid(project_name + python_version)}' - provisionerId: 'releng-${worker_level}' - workerType: 'linux' - created: { $fromNow: '' } - deadline: { $fromNow: '24 hours' } - payload: - features: - dind: true - taskclusterProxy: true - maxRunTime: 3600 - # we need to run really old docker version because taskcluster is using - # really old version in their setup - # image: docker:stable - image: 'docker:1.6.2' - env: + command: + - /usr/local/bin/run-task + - '--scriptworker-checkout=/builds/worker/checkouts/src' + - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph' + - '--task-cwd=/builds/worker/checkouts/src' + - '--' + - bash + - -cx + - $let: + extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + in: + $if: 'tasks_for == "action"' + then: > + PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph action-callback + else: > + PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --message="" + --owner='${ownerEmail}' + --level='${level}' + --base-repository="$SCRIPTWORKER_BASE_REPOSITORY" + --head-repository="$SCRIPTWORKER_HEAD_REPOSITORY" + --head-ref="$SCRIPTWORKER_HEAD_REF" + --head-rev="$SCRIPTWORKER_HEAD_REV" + --repository-type="$SCRIPTWORKER_REPOSITORY_TYPE" + --tasks-for='${tasks_for}' + ${extraArgs} + + artifacts: + 'public': + type: 'directory' + path: '/builds/worker/artifacts' + expires: {$fromNow: '1 year'} - DOCKERHUB_EMAIL: 'release+dockerhub+services@mozilla.com' - DOCKERHUB_USER: 'mozillarelengservices' - DOCKER_REPO: '${dockerhub_repo}' - DOCKER_TAG: '${docker_tag}' - GIT_HEAD_REV: '${HEAD_REV}' - PROJECT_NAME: '${project_name}' - PUSH_DOCKER_IMAGE: '${push_docker_image}' - REPO_URL: '${REPO_URL}' - SECRET_URL: 'http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy' - command: - - sh - - -lxce - - >- - cd /tmp && - wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz && - tar zxf ${HEAD_REV}.tar.gz && - mv scriptworker-scripts-${HEAD_REV} /src && - cd /src/ && - cp ${project_name}/docker.d/* docker.d/ && - cp ${project_name}/Dockerfile . && - ./docker.d/generate_version_json.sh && - ./docker.d/build_image.sh && - if [ "$PUSH_DOCKER_IMAGE" == "1" ]; then - ./docker.d/push_image.sh - fi - scopes: - $if: 'push_docker_image == "0"' - then: [] - else: - - 'secrets:get:project/releng/scriptworker-scripts/deploy' - metadata: - $let: - description: - $if: 'push_docker_image == "0"' - then: '${project_name}-py${python_version}: build docker image [on ${BRANCH_NAME}]' - else: '${project_name}-py${python_version}: build and push docker image [on ${BRANCH_NAME}]' - in: - name: '${number_prefix}${i+1}.2. ${description}' - description: '${description}' - owner: '${OWNER}' - source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' + extra: + $merge: + - $if: 'tasks_for == "action"' + then: + parent: '${action.taskGroupId}' + action: + name: '${action.name}' + context: + taskGroupId: '${action.taskGroupId}' + taskId: {$eval: 'taskId'} + input: {$eval: 'input'} + - $if: 'tasks_for == "cron"' + then: + cron: {$json: {$eval: 'cron'}} + - tasks_for: '${tasks_for}' diff --git a/docker.d/generate_version_json.sh b/docker.d/generate_version_json.sh index 680fd0b6e..5a45fa5e2 100755 --- a/docker.d/generate_version_json.sh +++ b/docker.d/generate_version_json.sh @@ -1,7 +1,7 @@ #/bin/sh set -e -test $GIT_HEAD_REV +test $HEAD_REV test $TASK_ID test $TASKCLUSTER_ROOT_URL test $REPO_URL diff --git a/docker.d/push_image.sh b/docker.d/push_image.sh index c64be0c49..7d99a054e 100755 --- a/docker.d/push_image.sh +++ b/docker.d/push_image.sh @@ -5,7 +5,8 @@ test $DOCKERHUB_EMAIL test $DOCKERHUB_USER test $DOCKER_REPO test $DOCKER_TAG -test $GIT_HEAD_REV +test $HEAD_REV +test $HOME test $PUSH_DOCKER_IMAGE test $SECRET_URL test $PROJECT_NAME @@ -18,12 +19,12 @@ docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$DOCKER_ARCHIVE_TAG echo "=== Generating dockercfg ===" # docker login stopped working in Taskcluster for some reason -wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > /root/.dockercfg -chmod 600 /root/.dockercfg +wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > $HOME/.dockercfg +chmod 600 $HOME/.dockercfg echo "=== Pushing to docker hub ===" docker push $DOCKER_REPO:$DOCKER_TAG docker push $DOCKER_REPO:$DOCKER_ARCHIVE_TAG echo "=== Clean up ===" -rm -f /root/.dockercfg +rm -f $HOME/.dockercfg diff --git a/pushapkscript/Dockerfile b/pushapkscript/Dockerfile index c85429e0b..078c30b11 100644 --- a/pushapkscript/Dockerfile +++ b/pushapkscript/Dockerfile @@ -3,16 +3,16 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y default-jdk \ - && apt-get clean \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app COPY . /app +RUN /app/docker.d/image_setup.sh + +USER app + RUN python -m venv /app \ && cd pushapkscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushapkscript/Dockerfile.test b/pushapkscript/Dockerfile.test index 5598cfbce..7bfe5c17e 100644 --- a/pushapkscript/Dockerfile.test +++ b/pushapkscript/Dockerfile.test @@ -2,12 +2,12 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update && apt-get install -y default-jdk && apt-get clean - WORKDIR /app COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ +COPY docker.d/image_setup.sh /app/ +RUN /app/image_setup.sh RUN pip install -r requirements/local.txt diff --git a/pushapkscript/docker.d/image_setup.sh b/pushapkscript/docker.d/image_setup.sh new file mode 100755 index 000000000..dbc4425ca --- /dev/null +++ b/pushapkscript/docker.d/image_setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -o errexit -o pipefail + +apt-get update +apt-get install -y default-jdk +apt-get clean diff --git a/pushflatpakscript/Dockerfile b/pushflatpakscript/Dockerfile index a37a2e390..11e731851 100644 --- a/pushflatpakscript/Dockerfile +++ b/pushflatpakscript/Dockerfile @@ -3,16 +3,15 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev \ - && apt-get clean \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app - COPY . /app +RUN /app/docker.d/image_setup.sh + +USER app + RUN python -m venv /app \ && cd pushflatpakscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushflatpakscript/Dockerfile.test b/pushflatpakscript/Dockerfile.test index c77050ba0..7cd0596b9 100644 --- a/pushflatpakscript/Dockerfile.test +++ b/pushflatpakscript/Dockerfile.test @@ -2,11 +2,12 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update \ - && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev - WORKDIR /app +COPY docker.d/image_setup.sh /app/ + +RUN /app/image_setup.sh + COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ diff --git a/pushflatpakscript/docker.d/image_setup.sh b/pushflatpakscript/docker.d/image_setup.sh new file mode 100755 index 000000000..b44b975ce --- /dev/null +++ b/pushflatpakscript/docker.d/image_setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -o errexit -o pipefail + +apt-get update +apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev +apt-get clean diff --git a/pushsnapscript/Dockerfile b/pushsnapscript/Dockerfile index ad56549ef..0f905a435 100644 --- a/pushsnapscript/Dockerfile +++ b/pushsnapscript/Dockerfile @@ -3,20 +3,16 @@ FROM python:3.8 RUN groupadd --gid 10001 app && \ useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app -RUN apt-get update \ - && apt-get install -y libsodium-dev \ - && apt-get install -y squashfs-tools \ - && apt-get install -y xdelta3 \ - && apt-get clean \ - # XXX Avoid snapcraft from loading useless libs when running on Ubuntu - && truncate -s 0 /etc/os-release \ - && ln -s /app/docker.d/healthcheck /bin/healthcheck +RUN ln -s /app/docker.d/healthcheck /bin/healthcheck -USER app WORKDIR /app - COPY . /app +# Production image requires extra packages +RUN /app/docker.d/image_setup.sh libsodium-dev squashfs-tools xdelta3 + +USER app + RUN python -m venv /app \ && cd pushsnapscript \ && /app/bin/pip install -r requirements/base.txt \ diff --git a/pushsnapscript/Dockerfile.test b/pushsnapscript/Dockerfile.test index fe71ff840..72aafb15a 100644 --- a/pushsnapscript/Dockerfile.test +++ b/pushsnapscript/Dockerfile.test @@ -2,10 +2,11 @@ ARG PYTHON_VERSION FROM python:${PYTHON_VERSION} -RUN apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release - WORKDIR /app +COPY docker.d/image_setup.sh /app/ +RUN /app/image_setup.sh + COPY MANIFEST.in setup.py tox.ini /app/ COPY requirements/ /app/requirements/ diff --git a/pushsnapscript/docker.d/image_setup.sh b/pushsnapscript/docker.d/image_setup.sh new file mode 100755 index 000000000..3f43cdd57 --- /dev/null +++ b/pushsnapscript/docker.d/image_setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -o errexit -o pipefail + +PKGS="${@:-libsodium-dev}" + +apt-get update +for pkg in $PKGS; do + apt-get install -y $pkg +done +apt-get clean +# XXX Avoid snapcraft from loading useless libs when running on Ubuntu +truncate -s 0 /etc/os-release diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 000000000..cb40260d9 --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,47 @@ +--- +trust-domain: scriptworker + +task-priority: high + +taskgraph: + register: scriptworker_taskgraph:register + decision-parameters: 'scriptworker_taskgraph.parameters:get_decision_parameters' + repositories: + scriptworker: + name: "Scriptworker-scripts" + project-regex: scriptworker-scripts$ + default-repository: https://github.com/mozilla-releng/scriptworker-scripts + default-ref: master + type: git + +workers: + aliases: + b-linux: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'b-linux' + images: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'images' + +docker: + email: "release+dockerhub+services@mozilla.com" + user: mozillarelengservices + +notify: + email: + - release+scriptworker-scripts@mozilla.com + prefix: "[scriptworker-scripts]" + +release-promotion: + flavors: + promote: + target-tasks-method: promote_scriptworker + +private-artifact-prefix: releng/scriptworker + +scriptworker: + scope-prefix: project:scriptworker:releng diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 000000000..6746cde87 --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +jobs: + k8s-image: + definition: k8s-image + python38: + definition: python + args: + PYTHON_VERSION: "3.8" + python37: + definition: python + args: + PYTHON_VERSION: "3.7" + pushapkscript-python38: + definition: pushapkscript + parent: python38 + pushapkscript-python37: + definition: pushapkscript + parent: python37 + pushsnapscript-python38: + definition: pushsnapscript + parent: python38 + pushsnapscript-python37: + definition: pushsnapscript + parent: python37 + pushflatpakscript-python38: + definition: pushflatpakscript + parent: python38 + pushflatpakscript-python37: + definition: pushflatpakscript + parent: python37 diff --git a/taskcluster/ci/k8s-image/kind.yml b/taskcluster/ci/k8s-image/kind.yml new file mode 100644 index 000000000..c7d773133 --- /dev/null +++ b/taskcluster/ci/k8s-image/kind.yml @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - tox + +transforms: + - scriptworker_taskgraph.transforms.python_version:transforms + - scriptworker_taskgraph.transforms.k8s_image:transforms + - scriptworker_taskgraph.transforms.cached:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: "{name} k8s image builder py{python_version}" + name: "{name}-python{python_version}" + run-on-tasks-for: ["action", "github-pull-request", "github-push"] + attributes: + code-review: true + worker-type: images + worker: + docker-image: {in-tree: 'k8s-image'} + max-run-time: 3600 + docker-in-docker: true + taskcluster-proxy: true + chain-of-trust: true + artifacts: + - type: directory + name: public/ + path: /builds/worker/artifacts/ + command: + - sh + - -lxce + - >- + sh /usr/local/bin/build_and_push.sh + python-versions: + - 38 + deploy-secret-url: http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy + docker-repo: mozilla/releng-{name} + # TODO copy image to artifacts + +jobs: + addonscript: {} + balrogscript: {} + beetmoverscript: {} + bouncerscript: {} + pushapkscript: {} + pushflatpakscript: {} + pushsnapscript: {} + shipitscript: {} + signingscript: {} + treescript: {} diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml new file mode 100644 index 000000000..95e0a171f --- /dev/null +++ b/taskcluster/ci/pr/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - tox + - k8s-image + +transforms: + - taskgraph.transforms.code_review:transforms + - taskgraph.transforms.task:transforms + +jobs: + complete: + description: PR Summary Task + run-on-tasks-for: [github-pull-request] + worker-type: succeed diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml new file mode 100644 index 000000000..413023016 --- /dev/null +++ b/taskcluster/ci/tox/kind.yml @@ -0,0 +1,98 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - docker-image + +transforms: + - scriptworker_taskgraph.transforms.python_version:transforms + - scriptworker_taskgraph.transforms.tox:transforms + - scriptworker_taskgraph.transforms.cached:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: "{name} tox-py{python_version}" + run-on-tasks-for: ["action", "github-pull-request", "github-push"] + attributes: + code-review: true + worker-type: b-linux + worker: + docker-image: {in-tree: 'python{python_version}'} + max-run-time: 1800 + python-versions: + - 38 + - 37 + run: + using: run-task + cache-dotcache: false + checkout: + scriptworker: {} + # sparse-profile: basestring/none + # workdir: + cwd: '{checkout}' + command: + - sh + - -lxce + - >- + tox -e {name}-py{python_version} + +jobs: + addonscript: + resources: + - addonscript + balrogscript: + resources: + - balrogscript + - scriptworker_client + beetmoverscript: + resources: + - beetmoverscript + bouncerscript: + resources: + - bouncerscript + configloader: + resources: + - configloader + iscript: + resources: + - iscript + - scriptworker_client + - vendored/mozbuild + notarization_poller: + resources: + - notarization_poller + - scriptworker_client + pushapkscript: + worker: + docker-image: {in-tree: 'pushapkscript-python{python_version}'} + resources: + - pushapkscript + pushflatpakscript: + worker: + docker-image: {in-tree: 'pushflatpakscript-python{python_version}'} + resources: + - pushflatpakscript + pushsnapscript: + worker: + docker-image: {in-tree: 'pushsnapscript-python{python_version}'} + resources: + - pushsnapscript + scriptworker_client: + resources: + - scriptworker_client + shipitscript: + resources: + - shipitscript + signingscript: + resources: + - signingscript + - vendored/mozbuild + treescript: + resources: + - treescript + - scriptworker_client diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY new file mode 100644 index 000000000..8c26d4c91 --- /dev/null +++ b/taskcluster/docker/REGISTRY @@ -0,0 +1 @@ +mozilla-releng diff --git a/taskcluster/docker/k8s-image/Dockerfile b/taskcluster/docker/k8s-image/Dockerfile new file mode 100644 index 000000000..25441584d --- /dev/null +++ b/taskcluster/docker/k8s-image/Dockerfile @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# We need to run a really old docker version because taskcluster is using a +# really old version in their setup +FROM docker:1.6.2 + +RUN apk add --update xz +RUN mkdir -p /usr/local/bin +COPY build_and_push.sh /usr/local/bin/build_and_push.sh + +# Add worker user +RUN mkdir /builds && \ + adduser -D -h /builds/worker -g worker -u 1000 worker && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +ENV SHELL=/bin/sh \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# Set a default command useful for debugging +CMD ["/bin/sh"] diff --git a/taskcluster/docker/k8s-image/build_and_push.sh b/taskcluster/docker/k8s-image/build_and_push.sh new file mode 100755 index 000000000..9e55a4677 --- /dev/null +++ b/taskcluster/docker/k8s-image/build_and_push.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# TODO: move docker.d/ files into the image? +set -e +test REPO_URL +test HEAD_REV +test PROJECT_NAME +test PUSH_DOCKER_IMAGE + +mkdir -p /builds/worker/checkouts +cd /builds/worker/checkouts +wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz +tar zxf ${HEAD_REV}.tar.gz +mv *-${HEAD_REV} src +cd src +cp ${PROJECT_NAME}/docker.d/* docker.d/ +cp ${PROJECT_NAME}/Dockerfile . +sh ./docker.d/generate_version_json.sh +sh ./docker.d/build_image.sh +if [ "${PUSH_DOCKER_IMAGE}" == "1" ]; then + ./docker.d/push_image.sh +fi diff --git a/taskcluster/docker/pushapkscript/Dockerfile b/taskcluster/docker/pushapkscript/Dockerfile new file mode 100644 index 000000000..8d0d5d2d3 --- /dev/null +++ b/taskcluster/docker/pushapkscript/Dockerfile @@ -0,0 +1,10 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# %include pushapkscript/docker.d/image_setup.sh +COPY topsrcdir/pushapkscript/docker.d/image_setup.sh /usr/local/bin/ + +RUN image_setup.sh diff --git a/taskcluster/docker/pushflatpakscript/Dockerfile b/taskcluster/docker/pushflatpakscript/Dockerfile new file mode 100644 index 000000000..3d7555ed8 --- /dev/null +++ b/taskcluster/docker/pushflatpakscript/Dockerfile @@ -0,0 +1,9 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# %include pushflatpakscript/docker.d/image_setup.sh +COPY topsrcdir/pushflatpakscript/docker.d/image_setup.sh /usr/local/bin/ +RUN image_setup.sh diff --git a/taskcluster/docker/pushsnapscript/Dockerfile b/taskcluster/docker/pushsnapscript/Dockerfile new file mode 100644 index 000000000..1831be31d --- /dev/null +++ b/taskcluster/docker/pushsnapscript/Dockerfile @@ -0,0 +1,10 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# %include pushsnapscript/docker.d/image_setup.sh +COPY topsrcdir/pushsnapscript/docker.d/image_setup.sh /usr/local/bin/ + +RUN image_setup.sh diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile new file mode 100644 index 000000000..b9ee4cd65 --- /dev/null +++ b/taskcluster/docker/python/Dockerfile @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# %ARG PYTHON_VERSION +FROM python:$PYTHON_VERSION + +# Add worker user +RUN mkdir /builds && \ + groupadd -g 1000 -o worker && \ + useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +RUN pip install tox + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py new file mode 100644 index 000000000..b2c7d31b4 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/__init__.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from importlib import import_module + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules([ + "parameters", + "target_tasks", + "transforms", + ]) + + +def _import_modules(modules): + for module in modules: + import_module(".{}".format(module), package=__name__) diff --git a/taskcluster/scriptworker_taskgraph/parameters.py b/taskcluster/scriptworker_taskgraph/parameters.py new file mode 100644 index 000000000..e10862da2 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/parameters.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.parameters import extend_parameters_schema +from voluptuous import ( + Any, + Optional, + Required, +) + +PROJECT_SPECIFIC_PREFIXES = { + "refs/heads/dev-": "dev", + "refs/heads/production-": "production", +} + +PUSH_TAGS = ("dev", "production") + +scriptworker_schema = { + Optional('docker_tag'): Any(basestring, None), + Optional('push_docker_image'): Any(True, "force", False, None), + Optional('script_name'): Any(basestring, None), + Optional('script_revision'): Any(basestring, None), + Optional('shipping_phase'): Any("build", "promote", None), +} + +extend_parameters_schema(scriptworker_schema) + + +def get_decision_parameters(graph_config, parameters): + """Add repo-specific decision parameters. + + If we're on a production- or dev- branch, detect and set the `script_name`. + + """ + if parameters["tasks_for"] == "github-pull-request": + parameters["docker_tag"] = "github-pull-request" + elif parameters["head_ref"].startswith("refs/heads/"): + parameters["docker_tag"] = parameters["head_ref"].replace("refs/heads/", "") + force_push = False + for prefix, tag in PROJECT_SPECIFIC_PREFIXES.items(): + if parameters["head_ref"].startswith(prefix): + parameters["script_name"] = parameters["head_ref"].replace(prefix, "") + parameters["docker_tag" ] = tag + force_push = True + break + if parameters["docker_tag"] in PUSH_TAGS and parameters["level"] == "3": + if force_push: + parameters["push_docker_image"] = "force" + else: + parameters["push_docker_image"] = True diff --git a/taskcluster/scriptworker_taskgraph/target_tasks.py b/taskcluster/scriptworker_taskgraph/target_tasks.py new file mode 100644 index 000000000..ec8426f42 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/target_tasks.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.target_tasks import _target_task, filter_for_tasks_for + + +@_target_task('default') +def target_tasks_default(full_task_graph, parameters, graph_config): + """Filter by `run_on_tasks_for` and `script-name`.""" + + def filter(task, parameters): + if not filter_for_tasks_for(task, parameters): + return False + if parameters.get("script_name"): + return task.attributes.get("script-name") == parameters["script_name"] + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] diff --git a/taskcluster/scriptworker_taskgraph/transforms/__init__.py b/taskcluster/scriptworker_taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/scriptworker_taskgraph/transforms/cached.py b/taskcluster/scriptworker_taskgraph/transforms/cached.py new file mode 100644 index 000000000..b56477004 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/cached.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build the cached_task digest to prevent rerunning tasks if the code hasn't changed. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import hashlib +import json +import os +import subprocess + +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.hash import hash_paths +from taskgraph.util.memoize import memoize + +transforms = TransformSequence() + +BASE_DIR = os.getcwd() + +@transforms.add +def add_resources(config, tasks): + for task in tasks: + resources = task.pop("resources", []) + attributes = task.setdefault("attributes", {}) + if attributes.get("resources") is not None: + if resources and attributes["resources"] != resources: + raise Exception( + "setting {} {} task.attributes.resources to {}: it's already set to {}!".format( + config.kind, task.get("name"), resources, attributes["resources"], + ) + ) + attributes["resources"] = resources + yield task + + +@transforms.add +def build_cache(config, tasks): + for task in tasks: + if task.get("cache", True) and not taskgraph.fast: + digest_data = [] + digest_data.append( + json.dumps(task.get("attributes", {}).get("digest-extra", {}), indent=2, sort_keys=True) + ) + resources = task["attributes"]["resources"] + for resource in resources: + digest_data.append(hash_paths(os.path.join(BASE_DIR, resource), [''])) + cache_name = task["name"].replace(":", "-") + task["cache"] = { + "type": "scriptworker-scripts.v1.{}".format(config.kind), + "name": cache_name, + "digest-data": digest_data, + } + + yield task + + +@transforms.add +def set_label(config, jobs): + """Set the job label, which the `cached_tasks` transform needs""" + for job in jobs: + job["label"] = "{}-{}".format(config.kind, job.pop("name")) + yield job diff --git a/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py new file mode 100644 index 000000000..d42df7bda --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/k8s_image.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Kubernetes docker image builds. +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy +import time + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + """Add dependencies that match python-version and script-name. + + Also copy the resources attribute, and fail if there are unexpected + discrepancies in upstream deps. + + """ + for job in jobs: + attributes = job["attributes"] + dependencies = job.setdefault("dependencies", {}) + resources = None + for dep_task in config.kind_dependencies_tasks: + dep_attrs = dep_task.attributes + dep_kind = dep_task.kind + if dep_attrs["python-version"] == attributes["python-version"] and \ + dep_attrs["script-name"] == attributes["script-name"]: + if dependencies.get(dep_kind): + raise Exception("Duplicate kind {kind} dependencies: {existing_label}, {new_label}".format( + kind=dep_kind, + existing_label=dependencies[dep_kind]["label"], + new_label=dep_task.label, + )) + dependencies[dep_kind] = dep_task.label + if dep_attrs.get("resources"): + if resources and resources != dep_attrs["resources"]: + raise Exception("Conflicting resources: {existing_digest} {new_digest}".format( + existing_digest=resources, + new_digest=dep_attrs["resources"], + )) + resources = dep_attrs["resources"] + if resources: + attributes["resources"] = resources + yield job + + +@transforms.add +def set_environment(config, jobs): + """Set the environment variables for the docker hub task.""" + for job in jobs: + project_name = job["attributes"]["script-name"] + secret_url = job.pop("deploy-secret-url") + tasks_for = config.params['tasks_for'] + scopes = job.setdefault("scopes", []) + attributes = job["attributes"] + env = job["worker"].setdefault("env", {}) + env.update({ + "HEAD_REV": config.params['head_rev'], + "DOCKER_REPO": job.pop("docker-repo"), + "DOCKER_TAG": config.params.get('docker_tag', 'unknown'), + "PROJECT_NAME": project_name, + "REPO_URL": config.params['head_repository'], + "TASKCLUSTER_ROOT_URL": "$TASKCLUSTER_ROOT_URL", + }) + push_docker_image = config.params.get("push_docker_image") + if push_docker_image: + env.update({ + "SECRET_URL": secret_url, + "PUSH_DOCKER_IMAGE": "1", + "DOCKERHUB_EMAIL": config.graph_config["docker"]["email"], + "DOCKERHUB_USER": config.graph_config["docker"]["user"], + }) + scopes.append('secrets:get:project/releng/scriptworker-scripts/deploy') + if push_docker_image == "force": + attributes.setdefault("digest-extra", {}).setdefault("force_run", time.time()) + else: + env["PUSH_DOCKER_IMAGE"] = "0" + yield job diff --git a/taskcluster/scriptworker_taskgraph/transforms/python_version.py b/taskcluster/scriptworker_taskgraph/transforms/python_version.py new file mode 100644 index 000000000..9f93c5f1b --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/python_version.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Create a task per job python-version +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + +def _replace_string(obj, subs): + if isinstance(obj, dict): + return {k: v.format(**subs) for k, v in obj.items()} + elif isinstance(obj, list): + for c in range(0, len(obj)): + obj[c] = obj[c].format(**subs) + else: + obj = obj.format(**subs) + return obj + + +def _resolve_replace_string(item, field, subs): + # largely from resolve_keyed_by + container, subfield = item, field + while '.' in subfield: + f, subfield = subfield.split('.', 1) + if f not in container: + return item + container = container[f] + if not isinstance(container, dict): + return item + + if subfield not in container: + return item + + container[subfield] = _replace_string(container[subfield], subs) + return item + + + +@transforms.add +def set_script_name(config, jobs): + for job in jobs: + job.setdefault("attributes", {}).update({ + "script-name": job["name"], + }) + yield job + + +@transforms.add +def tasks_per_python_version(config, jobs): + fields = [ + "description", + "docker-repo", + "run.command", + "worker.command", + "worker.docker-image", + ] + for job in jobs: + for python_version in job.pop("python-versions"): + task = deepcopy(job) + subs = {"name": job["name"], "python_version": python_version} + for field in fields: + _resolve_replace_string(task, field, subs) + task["attributes"]["python-version"] = python_version + yield task + + +@transforms.add +def update_name_with_python_version(config, jobs): + for job in jobs: + job["name"] = "{}-python{}".format(job["name"], job["attributes"]["python-version"]) + yield job diff --git a/taskcluster/scriptworker_taskgraph/transforms/tox.py b/taskcluster/scriptworker_taskgraph/transforms/tox.py new file mode 100644 index 000000000..f3386cd69 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/tox.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Tox-specific transforms +""" + +from __future__ import absolute_import, print_function, unicode_literals +from copy import deepcopy +import time + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + """Explicitly add the docker-image task as a dependency. + + This needs to be done before the `cached_tasks` transform, so we can't + wait until the `build_docker_worker_payload` transform. + + From `build_docker_worker_payload`. + + """ + for job in jobs: + image = job['worker']['docker-image'] + if isinstance(image, dict): + if 'in-tree' in image: + name = image['in-tree'] + docker_image_task = 'build-docker-image-' + image['in-tree'] + job.setdefault('dependencies', {})['docker-image'] = docker_image_task + yield job diff --git a/tox.ini b/tox.ini index 2e12b3c38..f71d3bff7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,5 @@ [tox] envlist = - client-py37 - client-py3py38 - iscript-py37 - iscript-py38 addonscript-py37 addonscript-py38 balrogscript-py37 @@ -12,6 +8,10 @@ envlist = beetmoverscript-py38 bouncerscript-py37 bouncerscript-py38 + configloader-py37 + configloader-py38 + iscript-py37 + iscript-py38 notarization_poller-py37 notarization_poller-py38 pushapkscript-py37 @@ -20,14 +20,14 @@ envlist = pushflatpakscript-py38 pushsnapscript-py37 pushsnapscript-py38 + scriptworker_client-py37 + scriptworker_client-py3py38 shipitscript-py37 shipitscript-py38 signingscript-py37 signingscript-py38 treescript-py37 treescript-py38 - configloader-py37 - configloader-py38 skipsdist = true @@ -43,28 +43,6 @@ passenv = TRAVIS_JOB_ID TRAVIS_BRANCH -[testenv:client-py38] -changedir = {toxinidir}/scriptworker_client -commands = - tox -e py38,mypy - - tox -e coveralls - -[testenv:client-py37] -changedir = {toxinidir}/scriptworker_client -commands = - tox -e py37,mypy - -[testenv:iscript-py38] -changedir = {toxinidir}/iscript -commands = - tox -e py38 - -[testenv:iscript-py37] -changedir = {toxinidir}/iscript -commands = - tox -e py37,check - - tox -e coveralls - [testenv:addonscript-py38] changedir = {toxinidir}/addonscript commands = @@ -109,6 +87,28 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py37 +[testenv:configloader-py38] +changedir = {toxinidir}/configloader +commands = + tox -e py38,check + - tox -e coveralls + +[testenv:configloader-py37] +changedir = {toxinidir}/configloader +commands = + tox -e py37 + +[testenv:iscript-py38] +changedir = {toxinidir}/iscript +commands = + tox -e py38 + +[testenv:iscript-py37] +changedir = {toxinidir}/iscript +commands = + tox -e py37,check + - tox -e coveralls + [testenv:notarization_poller-py38] changedir = {toxinidir}/notarization_poller commands = @@ -153,6 +153,22 @@ changedir = {toxinidir}/pushsnapscript commands = tox -e py37 +[testenv:scriptworker_client-py38] +changedir = {toxinidir}/scriptworker_client +commands = + tox -e py38,mypy + - tox -e coveralls + +[testenv:scriptworker_client-py37] +changedir = {toxinidir}/scriptworker_client +commands = + tox -e py37,mypy + +[testenv:shipitscript-py38] +changedir = {toxinidir}/shipitscript +commands = + tox -e py38 + [testenv:shipitscript-py37] changedir = {toxinidir}/shipitscript commands = @@ -179,14 +195,3 @@ commands = changedir = {toxinidir}/treescript commands = tox -e py37 - -[testenv:configloader-py38] -changedir = {toxinidir}/configloader -commands = - tox -e py38,check - - tox -e coveralls - -[testenv:configloader-py37] -changedir = {toxinidir}/configloader -commands = - tox -e py37