Implement pypiscript.

.taskcluster.yml

@@ -14,6 +14,7 @@ tasks:
         setup_pushapkscript:  'apt-get update && apt-get install -y default-jdk &&'
         setup_pushsnapscript: 'apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release &&'
         setup_pushflatpakscript: 'apt-get update && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev &&'
+        setup_rust: 'rustup component add clippy rustfmt &&'
       in:
         # [ <PROJECT NAME>,   <PYTHON VERSION>, <SETUP COMMAND>,           <DOCKERHUB REPO>]
         - ['client',          '37',             '',                        '']
@@ -44,6 +45,8 @@ tasks:
         - ['signingscript',   '38',             '',                        'mozilla/releng-signingscript']
         - ['treescript',      '37',             '',                        '']
         - ['treescript',      '38',             '',                        'mozilla/releng-treescript']
+        - ['pypiscript',      '38',             '',                        'mozilla/releng-pypiscript']
+        - ['rust',            'rust',           '${setup_rust}',           '']
     # -------------------------------------------------------------------------
 
     HEAD_REV:
@@ -68,6 +71,8 @@ tasks:
         $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"'
         then: '${event.ref[11:]}'
         else: 'unknown'
+
+    rust_version: 1.43
   in:
     $flatten:
       $map: { "$eval": "PROJECTS" }
@@ -115,7 +120,7 @@ tasks:
         in:
           $match:
             # Run code linting and unit tests for each project
-            'run_tests == "1"':
+            'run_tests == "1" && project_name != "rust"':
               taskId: '${as_slugid(project_name + python_version)}'
               provisionerId: 'releng-t'
               workerType: 'linux'
@@ -147,6 +152,37 @@ tasks:
                 description: 'Code linting and unit tests for ${project_name} on python ${python_version[0]}.${python_version[1]}'
                 owner: '${OWNER}'
                 source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml'
+            'run_tests == "1" && project_name == "rust"':
+              taskId: '${as_slugid("rust")}'
+              provisionerId: 'releng-t'
+              workerType: 'linux'
+              created: { $fromNow: '' }
+              deadline: { $fromNow: '4 hours' }
+              payload:
+                maxRunTime: 3600
+                image: 'rust:${rust_version}'
+                command:
+                  - sh
+                  - -xce
+                  - >-
+                    cd /tmp &&
+                    wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz &&
+                    tar zxf ${HEAD_REV}.tar.gz &&
+                    mv scriptworker-scripts-${HEAD_REV} /src &&
+                    cd /src && ${setup_command}
+                    cargo test && cargo clippy && cargo fmt -- --check
+              metadata:
+                name:
+                  $let:
+                    test_task_number:
+                      $if: 'dockerhub_repo != ""'
+                      then: '${i+1}.1'
+                      else: '${i+1}'
+                  in:
+                    '${number_prefix}${test_task_number}. ${project_name}: Run rust checks [on ${BRANCH_NAME}]'
+                description: 'Code linting and unit tests for rust code on rust ${rust_version}'
+                owner: '${OWNER}'
+                source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml'
             # Build docker image and (optionally) push to docker hub
             'run_tests == "1" && dockerhub_repo != ""':
               taskId: '${as_slugid(project_name + "docker_build_and_push")}'

Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+
+members = [
+  "pypiscript",
+  "script",
+]

docker.d/init.sh

@@ -74,6 +74,9 @@ case $COT_PRODUCT in
   adhoc)
     export TRUST_DOMAIN=adhoc
     ;;
+  releng)
+    export TRUST_DOMAIN=releng
+    ;;
   *)
     echo "Unknown COT_PRODUCT $COT_PRODUCT"
     exit 1
@@ -114,6 +117,7 @@ export WORK_DIR=/app/workdir
 export WORKER_TYPE="${TRUST_DOMAIN}-${TRUST_LEVEL}-${PROJECT_NAME}${WORKER_SUFFIX}"
 export WORKER_GROUP=${WORKER_TYPE}
 export WORKER_ID_PREFIX="${WORKER_TYPE}-"
+export PASS_WORK_DIR=false
 
 #
 # ensure configuration folder exists we can write to it

docker.d/scriptworker.yml

@@ -14,6 +14,7 @@ task_max_timeout: 3600
 task_script:
   - { "$eval": "TASK_SCRIPT" }
   - { "$eval": "TASK_CONFIG" }
+  - { "$if": "PASS_WORK_DIR == 'true'", then: "${WORK_DIR}" }
 verbose: { "$eval": "VERBOSE == 'true'" }
 verify_chain_of_trust: { "$eval": "VERIFY_CHAIN_OF_TRUST == 'true'" }
 sign_chain_of_trust: { "$eval": "SIGN_CHAIN_OF_TRUST == 'true'" }
@@ -30,3 +31,39 @@ scriptworker_provisioners:
     - { "$eval": "PROVISIONER_ID" }
     - scriptworker-prov-v1  # keep for mac-v3-signing
 ed25519_private_key_path: { "$eval": "ED25519_PRIVKEY_PATH" }
+
+valid_decision_worker_pools:
+    by-cot-product:
+        releng:
+            - releng-1/linux
+            - releng-3/linux
+valid_docker_image_worker_pools:
+    by-cot-product:
+        releng:
+            - releng-1/linux
+            - releng-3/linux
+trusted_vcs_rules:
+    by-cot-product:
+        releng: []
+valid_tasks_for:
+    by-cot-product:
+        releng:
+            - "github-pull-request"
+            - "github-push"
+            - "github-release"
+official_github_repos_owner:
+    by-cot-product:
+        releng: mozilla-releng
+cot_restricted_scopes:
+    by-cot-product:
+        releng: []
+cot_restricted_trees:
+    by-cot-product:
+        releng: []
+prebuilt_docker_image_task_types:
+    - decision
+    - action
+    - docker-image
+source_env_prefix: RELENG
+
+project_configuration_url: https://hg.mozilla.org/ci/ci-configuration-try/raw-file/65ca84a9c40e1d7764dce2ed8469165191f6764a/projects.yml

pypiscript/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "pypiscript"
+version = "0.1.0"
+authors = ["Tom Prince <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+serde = "1.0.99"
+serde_derive = "1.0.99"
+scriptworker_script = { path = "../script" }

pypiscript/Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.8
+
+RUN groupadd --gid 10001 app && \
+    useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
+
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.42.0
+
+RUN set -eux; \
+    url="https://static.rust-lang.org/rustup/archive/1.21.1/x86_64-unknown-linux-gnu/rustup-init"; \
+    wget "$url"; \
+    echo "ad1f8b5199b3b9e231472ed7aa08d2e5d1d539198a15c5b1e53c746aad81d27b *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+COPY . /app/
+COPY pypiscript/docker.d/* /app/docker.d/
+RUN chown -R app:app /app
+
+USER app
+WORKDIR /app
+
+RUN python -m venv /app \
+ && cd pypiscript \
+ && /app/bin/pip install -r requirements/base.txt \
+ && python -m venv /app/configloader_venv \
+ && cd /app/configloader \
+ && /app/configloader_venv/bin/pip install -r requirements/base.txt \
+ && /app/configloader_venv/bin/pip install . \
+ && cd /app \
+ && cargo install --path pypiscript --root . \
+ && mkdir /app/configs
+
+RUN python -m pip install twine
+
+
+CMD ["/app/docker.d/init.sh"]

pypiscript/config-example.json

@@ -0,0 +1,4 @@
+{
+	"taskcluster_scope_prefix": "project:releng",
+	"project_config_file": "/Depot/Mozilla/scriptworker-scripts/pypiscript/passwords.yml"
+}

pypiscript/docker.d/init_worker.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e errexit -o pipefail
+
+test_var_set() {
+  local varname=$1
+
+  if [[ -z "${!varname}" ]]; then
+    echo "error: ${varname} is not set"
+    exit 1
+  fi
+}
+
+#
+# Check that all required variables exist
+#
+test_var_set 'CONFIG_DIR'
+test_var_set 'CONFIG_LOADER'
+test_var_set 'COT_PRODUCT'
+test_var_set 'PROJECT_NAME'
+test_var_set 'TEMPLATE_DIR'
+
+export PASS_WORK_DIR=true
+
+export PASSWORDS_PATH=$CONFIG_DIR/passwords.json
+
+if [[ ! -f $PASSWORDS_PATH ]]; then
+  echo "error: ${PASSWORDS_PATH} is missing"
+fi

pypiscript/docker.d/worker.yml

@@ -0,0 +1,9 @@
+#artifact_dir: { "$eval": "ARTIFACTS_DIR" }
+project_config_file: "${PASSWORDS_PATH}"
+taskcluster_scope_prefix:
+    $let:
+        value:
+            $match:
+                'COT_PRODUCT == "releng"':
+                    'project:releng'
+    in: '${value[0]}'

pypiscript/requirements/base.in

@@ -0,0 +1 @@
+scriptworker

pypiscript/requirements/base.txt

@@ -0,0 +1,45 @@
+# SHA1:c829b627d519a44c3705088241a3460302f21bf5
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+aiohttp==3.6.2            # via scriptworker, taskcluster
+aiomemoizettl==0.0.3      # via scriptworker
+arrow==0.15.5             # via scriptworker
+async-timeout==3.0.1      # via aiohttp, taskcluster
+attrs==19.3.0             # via aiohttp, jsonschema
+certifi==2019.11.28       # via requests
+cffi==1.14.0              # via cryptography
+chardet==3.0.4            # via aiohttp, requests
+cryptography==2.8         # via jwcrypto, scriptworker
+dictdiffer==0.8.1         # via scriptworker
+github3.py==1.3.0         # via scriptworker
+idna-ssl==1.1.0           # via aiohttp
+idna==2.9                 # via idna-ssl, requests, yarl
+immutabledict==1.0.0      # via scriptworker
+importlib-metadata==1.5.2  # via jsonschema
+json-e==4.0.1             # via scriptworker
+jsonschema==3.2.0         # via scriptworker
+jwcrypto==0.7             # via github3.py
+mohawk==1.1.0             # via taskcluster
+multidict==4.7.5          # via aiohttp, yarl
+pycparser==2.20           # via cffi
+pyrsistent==0.16.0        # via jsonschema
+python-dateutil==2.8.1    # via arrow, github3.py
+pyyaml==5.3.1             # via scriptworker
+requests==2.23.0          # via github3.py, taskcluster
+scriptworker==32.2.1      # via -r pypiscript/requirements/base.in
+six==1.14.0               # via cryptography, jsonschema, mohawk, pyrsistent, python-dateutil, taskcluster
+slugid==2.0.0             # via taskcluster
+taskcluster-urls==12.1.0  # via taskcluster
+taskcluster==28.1.0       # via scriptworker
+typing-extensions==3.7.4.1  # via aiohttp
+uritemplate==3.0.1        # via github3.py
+urllib3==1.25.8           # via requests
+yarl==1.4.2               # via aiohttp
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools