bug 1597598 - use taskgraph (#226)

* bug 1597598 - use taskgraph

* add pr kind

* address review comments

* add tox to pr kind-dependencies

* dict comprehension

* don't pre-hash digest-extra; don't set empty `cached_task` attribute

* digest-paths, using hash_paths

* digest-paths -> resources

* script_name transform

* pushsnapscript - shared setup script

* env.update

* pushflatpakscript - shared setup script

* pushapkscript - shared setup script

* _resolve_replace_string

* add parameters. still need to add target tasks method

* cache type

* label

* remove tox artifacts dir for now

* explicit docker-image dep for tox

* target tasks

* parameters['docker_tag']

* docker-image py3{7,8} -> python3{7,8}

* push_docker_image param

* Update taskcluster/docker/k8s-image/Dockerfile

Co-authored-by: Johan Lorenzo <[email protected]>

* add comment to pushsnapscript/Dockerfile

* rm chmod from pushsnapscript tox Dockerfile

* Update taskcluster/scriptworker_taskgraph/parameters.py

Co-authored-by: Johan Lorenzo <[email protected]>

* remove unused repo_name

* address resources review comments

* fix

* adjust checks

* remove skip_on_project_specific_branches

Co-authored-by: Johan Lorenzo <[email protected]>

docker.d/generate_version_json.sh

@@ -1,7 +1,7 @@
 #/bin/sh
 set -e
 
-test $GIT_HEAD_REV
+test $HEAD_REV
 test $TASK_ID
 test $TASKCLUSTER_ROOT_URL
 test $REPO_URL

docker.d/push_image.sh

@@ -5,7 +5,8 @@ test $DOCKERHUB_EMAIL
 test $DOCKERHUB_USER
 test $DOCKER_REPO
 test $DOCKER_TAG
-test $GIT_HEAD_REV
+test $HEAD_REV
+test $HOME
 test $PUSH_DOCKER_IMAGE
 test $SECRET_URL
 test $PROJECT_NAME
@@ -18,12 +19,12 @@ docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Generating dockercfg ==="
 # docker login stopped working in Taskcluster for some reason
-wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > /root/.dockercfg
-chmod 600 /root/.dockercfg
+wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > $HOME/.dockercfg
+chmod 600 $HOME/.dockercfg
 
 echo "=== Pushing to docker hub ==="
 docker push $DOCKER_REPO:$DOCKER_TAG
 docker push $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Clean up ==="
-rm -f /root/.dockercfg
+rm -f $HOME/.dockercfg

pushapkscript/Dockerfile

@@ -3,16 +3,16 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y default-jdk \
- && apt-get clean \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
 
 COPY . /app
 
+RUN /app/docker.d/image_setup.sh
+
+USER app
+
 RUN python -m venv /app \
  && cd pushapkscript \
  && /app/bin/pip install -r requirements/base.txt \

pushapkscript/Dockerfile.test

@@ -2,12 +2,12 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
-RUN apt-get update && apt-get install -y default-jdk && apt-get clean
-
 WORKDIR /app
 
 COPY MANIFEST.in setup.py tox.ini /app/
 COPY requirements/ /app/requirements/
+COPY docker.d/image_setup.sh /app/
+RUN /app/image_setup.sh
 
 RUN pip install -r requirements/local.txt
 

pushapkscript/docker.d/image_setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+apt-get update
+apt-get install -y default-jdk
+apt-get clean

pushflatpakscript/Dockerfile

@@ -3,16 +3,15 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev \
- && apt-get clean \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
-
 COPY . /app
 
+RUN /app/docker.d/image_setup.sh
+
+USER app
+
 RUN python -m venv /app \
  && cd pushflatpakscript \
  && /app/bin/pip install -r requirements/base.txt \

pushflatpakscript/Dockerfile.test

@@ -2,11 +2,12 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
-RUN apt-get update \
- && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev
-
 WORKDIR /app
 
+COPY docker.d/image_setup.sh /app/
+
+RUN /app/image_setup.sh
+
 COPY MANIFEST.in setup.py tox.ini /app/
 COPY requirements/ /app/requirements/
 

pushflatpakscript/docker.d/image_setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+apt-get update
+apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev
+apt-get clean

pushsnapscript/Dockerfile

@@ -3,20 +3,16 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y libsodium-dev \
- && apt-get install -y squashfs-tools \
- && apt-get install -y xdelta3 \
- && apt-get clean \
- # XXX Avoid snapcraft from loading useless libs when running on Ubuntu
- && truncate -s 0 /etc/os-release \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
-
 COPY . /app
 
+# Production image requires extra packages
+RUN /app/docker.d/image_setup.sh libsodium-dev squashfs-tools xdelta3
+
+USER app
+
 RUN python -m venv /app \
  && cd pushsnapscript \
  && /app/bin/pip install -r requirements/base.txt \

pushsnapscript/Dockerfile.test

@@ -2,10 +2,11 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
-RUN apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release
-
 WORKDIR /app
 
+COPY docker.d/image_setup.sh /app/
+RUN /app/image_setup.sh
+
 COPY MANIFEST.in setup.py tox.ini /app/
 COPY requirements/ /app/requirements/
 

pushsnapscript/docker.d/image_setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+PKGS="${@:-libsodium-dev}"
+
+apt-get update
+for pkg in $PKGS; do
+    apt-get install -y $pkg
+done
+apt-get clean
+# XXX Avoid snapcraft from loading useless libs when running on Ubuntu
+truncate -s 0 /etc/os-release

taskcluster/ci/config.yml

@@ -0,0 +1,47 @@
+---
+trust-domain: scriptworker
+
+task-priority: high
+
+taskgraph:
+    register: scriptworker_taskgraph:register
+    decision-parameters: 'scriptworker_taskgraph.parameters:get_decision_parameters'
+    repositories:
+        scriptworker:
+            name: "Scriptworker-scripts"
+            project-regex: scriptworker-scripts$
+            default-repository: https://github.com/mozilla-releng/scriptworker-scripts
+            default-ref: master
+            type: git
+
+workers:
+    aliases:
+        b-linux:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'b-linux'
+        images:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'images'
+
+docker:
+    email: "[email protected]"
+    user: mozillarelengservices
+
+notify:
+    email:
+        - [email protected]
+    prefix: "[scriptworker-scripts]"
+
+release-promotion:
+    flavors:
+        promote:
+            target-tasks-method: promote_scriptworker
+
+private-artifact-prefix: releng/scriptworker
+
+scriptworker:
+    scope-prefix: project:scriptworker:releng

taskcluster/ci/docker-image/kind.yml

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    k8s-image:
+        definition: k8s-image
+    python38:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.8"
+    python37:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.7"
+    pushapkscript-python38:
+        definition: pushapkscript
+        parent: python38
+    pushapkscript-python37:
+        definition: pushapkscript
+        parent: python37
+    pushsnapscript-python38:
+        definition: pushsnapscript
+        parent: python38
+    pushsnapscript-python37:
+        definition: pushsnapscript
+        parent: python37
+    pushflatpakscript-python38:
+        definition: pushflatpakscript
+        parent: python38
+    pushflatpakscript-python37:
+        definition: pushflatpakscript
+        parent: python37

taskcluster/ci/k8s-image/kind.yml

@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - tox
+
+transforms:
+    - scriptworker_taskgraph.transforms.python_version:transforms
+    - scriptworker_taskgraph.transforms.k8s_image:transforms
+    - scriptworker_taskgraph.transforms.cached:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: "{name} k8s image builder py{python_version}"
+    name: "{name}-python{python_version}"
+    run-on-tasks-for: ["action", "github-pull-request", "github-push"]
+    attributes:
+        code-review: true
+    worker-type: images
+    worker:
+        docker-image: {in-tree: 'k8s-image'}
+        max-run-time: 3600
+        docker-in-docker: true
+        taskcluster-proxy: true
+        chain-of-trust: true
+        artifacts:
+            - type: directory
+              name: public/
+              path: /builds/worker/artifacts/
+        command:
+            - sh
+            - -lxce
+            - >-
+              sh /usr/local/bin/build_and_push.sh
+    python-versions:
+        - 38
+    deploy-secret-url: http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy
+    docker-repo: mozilla/releng-{name}
+    # TODO copy image to artifacts
+
+jobs:
+    addonscript: {}
+    balrogscript: {}
+    beetmoverscript: {}
+    bouncerscript: {}
+    pushapkscript: {}
+    pushflatpakscript: {}
+    pushsnapscript: {}
+    shipitscript: {}
+    signingscript: {}
+    treescript: {}

taskcluster/ci/pr/kind.yml

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - tox
+    - k8s-image
+
+transforms:
+    - taskgraph.transforms.code_review:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    complete:
+        description: PR Summary Task
+        run-on-tasks-for: [github-pull-request]
+        worker-type: succeed