From 093aadf6aa1c5ab2606d52e6960503f0e95203bd Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sun, 10 May 2020 16:36:42 -0600 Subject: [PATCH] Implement a PyPI script in rust. --- .taskcluster.yml | 37 ++++++- Cargo.lock | 172 +++++++++++++++++++++++++++++++++ Cargo.toml | 6 ++ pypiscript/Cargo.toml | 10 ++ pypiscript/config-example.json | 4 + pypiscript/src/main.rs | 137 ++++++++++++++++++++++++++ script/Cargo.toml | 14 +++ script/macros/Cargo.toml | 12 +++ script/macros/src/lib.rs | 25 +++++ script/src/error.rs | 51 ++++++++++ script/src/lib.rs | 76 +++++++++++++++ script/src/task.rs | 136 ++++++++++++++++++++++++++ 12 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 pypiscript/Cargo.toml create mode 100644 pypiscript/config-example.json create mode 100644 pypiscript/src/main.rs create mode 100644 script/Cargo.toml create mode 100644 script/macros/Cargo.toml create mode 100644 script/macros/src/lib.rs create mode 100644 script/src/error.rs create mode 100644 script/src/lib.rs create mode 100644 script/src/task.rs diff --git a/.taskcluster.yml b/.taskcluster.yml index c678e8073..182fa8428 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -14,6 +14,7 @@ tasks: setup_pushapkscript: 'apt-get update && apt-get install -y default-jdk &&' setup_pushsnapscript: 'apt-get update && apt-get install -y libsodium-dev && truncate -s 0 /etc/os-release &&' setup_pushflatpakscript: 'apt-get update && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev &&' + setup_rust: 'rustup component add clippy rustfmt &&' in: # [ , , , ] - ['client', '37', '', ''] @@ -44,6 +45,7 @@ tasks: - ['signingscript', '38', '', 'mozilla/releng-signingscript'] - ['treescript', '37', '', ''] - ['treescript', '38', '', 'mozilla/releng-treescript'] + - ['rust', 'rust', '${setup_rust}', ''] # ------------------------------------------------------------------------- HEAD_REV: @@ -68,6 +70,8 @@ tasks: $if: 'tasks_for == "github-push" && event.ref[0:11] == "refs/heads/"' then: '${event.ref[11:]}' else: 'unknown' + + rust_version: 1.43 in: $flatten: $map: { "$eval": "PROJECTS" } @@ -115,7 +119,7 @@ tasks: in: $match: # Run code linting and unit tests for each project - 'run_tests == "1"': + 'run_tests == "1" && project_name != "rust"': taskId: '${as_slugid(project_name + python_version)}' provisionerId: 'releng-t' workerType: 'linux' @@ -147,6 +151,37 @@ tasks: description: 'Code linting and unit tests for ${project_name} on python ${python_version[0]}.${python_version[1]}' owner: '${OWNER}' source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' + 'run_tests == "1" && project_name == "rust"': + taskId: '${as_slugid("rust")}' + provisionerId: 'releng-t' + workerType: 'linux' + created: { $fromNow: '' } + deadline: { $fromNow: '4 hours' } + payload: + maxRunTime: 3600 + image: 'rust:${rust_version}' + command: + - sh + - -xce + - >- + cd /tmp && + wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz && + tar zxf ${HEAD_REV}.tar.gz && + mv scriptworker-scripts-${HEAD_REV} /src && + cd /src && ${setup_command} + cargo test && cargo clippy && cargo fmt -- --check + metadata: + name: + $let: + test_task_number: + $if: 'dockerhub_repo != ""' + then: '${i+1}.1' + else: '${i+1}' + in: + '${number_prefix}${test_task_number}. ${project_name}: Run rust checks [on ${BRANCH_NAME}]' + description: 'Code linting and unit tests for rust code on rust ${rust_version}' + owner: '${OWNER}' + source: '${REPO_URL}/raw/${HEAD_REV}/.taskcluster.yml' # Build docker image and (optionally) push to docker hub 'run_tests == "1" && dockerhub_repo != ""': taskId: '${as_slugid(project_name + "docker_build_and_push")}' diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..0c2c6c1a1 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,172 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dtoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pypiscript" +version = "0.1.0" +dependencies = [ + "scriptworker_script 0.1.0", + "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scriptworker_script" +version = "0.1.0" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "scriptworker_script_macros 0.1.0", + "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scriptworker_script_macros" +version = "0.1.0" +dependencies = [ + "quote 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.110" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.110" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_yaml" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", + "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum dtoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" +"checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +"checksum linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +"checksum proc-macro2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)" = "8872cf6f48eee44265156c111456a700ab3483686b3f96df4cf5481c89157319" +"checksum quote 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4c1f4b0efa5fc5e8ceb705136bfee52cfdb6a4e3509f770b478cd6ed434232a7" +"checksum ryu 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +"checksum serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)" = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +"checksum serde_derive 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)" = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +"checksum serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)" = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" +"checksum serde_yaml 0.8.12 (registry+https://github.com/rust-lang/crates.io-index)" = "16c7a592a1ec97c9c1c68d75b6e537dcbf60c7618e038e7841e00af1d9ccf0c4" +"checksum syn 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "e8e5aa70697bb26ee62214ae3288465ecec0000f05182f039b477001f08f5ae7" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..918e30cb1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "pypiscript", + "script", +] diff --git a/pypiscript/Cargo.toml b/pypiscript/Cargo.toml new file mode 100644 index 000000000..40d15ea26 --- /dev/null +++ b/pypiscript/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "pypiscript" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +serde = "1.0.99" +serde_derive = "1.0.99" +scriptworker_script = { path = "../script" } diff --git a/pypiscript/config-example.json b/pypiscript/config-example.json new file mode 100644 index 000000000..d2a519927 --- /dev/null +++ b/pypiscript/config-example.json @@ -0,0 +1,4 @@ +{ + "taskcluster_scope_prefix": "project:releng", + "project_config_file": "/Depot/Mozilla/scriptworker-scripts/pypiscript/passwords.yml" +} diff --git a/pypiscript/src/main.rs b/pypiscript/src/main.rs new file mode 100644 index 000000000..fcec5a19d --- /dev/null +++ b/pypiscript/src/main.rs @@ -0,0 +1,137 @@ +use scriptworker_script::{Context, Error, Task}; +use serde_derive::Deserialize; +use std::collections::HashMap; +use std::os::unix::process::ExitStatusExt; +use std::process::Command; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Config { + #[serde( + alias = "project_config_file", + deserialize_with = "scriptworker_script::load_secrets" + )] + projects: HashMap, + #[serde(alias = "taskcluster_scope_prefix")] + scope_prefix: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Project { + api_token: String, + repository_url: String, +} + +#[derive(Deserialize, Debug)] +struct Attr { + project: String, +} + +#[derive(Deserialize, Debug)] +struct Extra { + action: String, +} + +fn verify_payload(config: &Config, _: &Context, task: &Task) -> Result<(), Error> { + if task.payload.extra.action != "upload" { + return Err(Error::MalformedPayload(format!( + "Unsupported action: {}", + task.payload.extra.action + ))); + } + + task.require_scopes(task.payload.upstream_artifacts.iter().map(|upstream| { + let project_name = &upstream.attributes.project; + format!("{}:pypi:project:{}", config.scope_prefix, project_name) + })) +} + +fn run_command(mut command: Command, action: &dyn Fn() -> String) -> Result<(), Error> { + println!("Running: {:?}", command); + match command.status() { + Ok(result) => { + if !result.success() { + println!( + "Failed to {}: {}", + action(), + match (result.code(), result.signal()) { + (Some(code), _) => format!("exit code {}", code), + (_, Some(signal)) => format!("exited with signal {}", signal), + (None, None) => "unknown exit reason".to_string(), + } + ); + return Err(Error::Failure); + } + Ok(()) + } + Err(err) => { + println!("Failed to start command: {:?}", err); + Err(Error::Failure) + } + } +} + +impl Config { + fn get_project(&self, project_name: &str) -> Result<&Project, Error> { + self.projects.get(project_name).ok_or_else(|| { + Error::MalformedPayload(format!("Unknown pypi project {}", project_name)) + }) + } +} + +#[scriptworker_script::main] +fn do_work(config: Config, context: &Context, task: Task) -> Result<(), Error> { + verify_payload(&config, &context, &task)?; + + task.payload + .upstream_artifacts + .iter() + .map(|upstream| -> Result<(), Error> { + let project_name = &upstream.attributes.project; + // Ensure project exists + config.get_project(project_name)?; + + let mut command = Command::new("twine"); + command.arg("check"); + for artifact in &upstream.paths { + command.arg(artifact.file_path(context)); + } + run_command(command, &|| format!("upload files for {}", project_name)) + }) + .fold(Ok(()), Result::or)?; + + for upstream in &task.payload.upstream_artifacts { + let project_name = &upstream.attributes.project; + let project = config.get_project(project_name)?; + + println!( + "Uploading {} from task {} to {} for project {}", + &upstream + .paths + .iter() + .map(|p| p.task_path().to_string_lossy()) + .collect::>() + .join(", "), + &upstream.task_id, + project.repository_url, + project_name + ); + + // To use tokens with PyPI, set the username to `__token__` and the password to the token. + // See https://pypi.org/help/#apitoken + let mut command = Command::new("twine"); + command + .arg("upload") + .arg("--user") + .arg("__token__") + .arg("--repository-url") + .arg(&project.repository_url); + for artifact in &upstream.paths { + command.arg(artifact.file_path(context)); + } + command.env("TWINE_PASSWORD", &project.api_token); + run_command(command, &|| format!("upload files for {}", project_name))?; + } + Ok(()) +} diff --git a/script/Cargo.toml b/script/Cargo.toml new file mode 100644 index 000000000..f644b3e8d --- /dev/null +++ b/script/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "scriptworker_script" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +# We disable features for clap, since it isn't used interactively. +clap = {version = "2.33.0", default-features = false} +serde_yaml = "0.8.9" +serde_json = "1.0.40" +serde = "1.0.99" +serde_derive = "1.0.99" +scriptworker_script_macros = { path = "macros" } diff --git a/script/macros/Cargo.toml b/script/macros/Cargo.toml new file mode 100644 index 000000000..5a755d4bf --- /dev/null +++ b/script/macros/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "scriptworker_script_macros" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +syn = {version = "1.0.5", features=["full"]} +quote = "1.0.2" + +[lib] +proc-macro = true diff --git a/script/macros/src/lib.rs b/script/macros/src/lib.rs new file mode 100644 index 000000000..6e05ba9db --- /dev/null +++ b/script/macros/src/lib.rs @@ -0,0 +1,25 @@ +#![cfg(not(test))] // Work around for rust-lang/rust#62127 +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; + +#[proc_macro_attribute] +pub fn main(args: TokenStream, item: TokenStream) -> TokenStream { + let input = syn::parse_macro_input!(item as syn::ItemFn); + let args = syn::parse_macro_input!(args as syn::AttributeArgs); + + if !args.is_empty() { + panic!("scriptworker_script main function should not have arguments.") + } + + let name = &input.sig.ident; + + let result = quote! { + #input + fn main() { + ::scriptworker_script::scriptworker_main(#name) + } + }; + result.into() +} diff --git a/script/src/error.rs b/script/src/error.rs new file mode 100644 index 000000000..3dc58b7ee --- /dev/null +++ b/script/src/error.rs @@ -0,0 +1,51 @@ +use std::convert::From; + +#[derive(Clone)] +pub enum Error { + Failure, + WorkerShutdown, + MalformedPayload(String), + ResourceUnavailable, + InternalError(String), + Superseded, + IntermittentTask, +} + +impl From for Error { + fn from(err: std::io::Error) -> Error { + Error::InternalError(format!("{}", err)) + } +} + +impl From for Error { + fn from(err: serde_yaml::Error) -> Error { + Error::InternalError(format!("{}", err)) + } +} + +impl Error { + pub(crate) fn exit_code(self) -> i32 { + match self { + Self::Failure => 1, + Self::WorkerShutdown => 2, + Self::MalformedPayload(_) => 3, + Self::ResourceUnavailable => 4, + Self::InternalError(_) => 5, + Self::Superseded => 6, + Self::IntermittentTask => 7, + } + } + + #[allow(dead_code)] + pub(crate) fn description(self) -> &'static str { + match self { + Self::Failure => "failure", + Self::WorkerShutdown => "worker-shutdown", + Self::MalformedPayload(_) => "malformed-payload", + Self::ResourceUnavailable => "resource-unavailable", + Self::InternalError(_) => "internal-error", + Self::Superseded => "superseded", + Self::IntermittentTask => "intermittent-task", + } + } +} diff --git a/script/src/lib.rs b/script/src/lib.rs new file mode 100644 index 000000000..e4c0cfa48 --- /dev/null +++ b/script/src/lib.rs @@ -0,0 +1,76 @@ +use serde::de::DeserializeOwned; +use std::path::{Path, PathBuf}; + +use clap::{App, Arg}; + +mod error; +pub use error::Error; + +pub mod task; +pub use task::Task; + +pub struct Context { + work_dir: PathBuf, +} + +fn init_config() -> Result<(T, PathBuf), Error> +where + T: DeserializeOwned, +{ + let matches = App::new("scriptworker") + .arg(Arg::with_name("CONFIG_FILE").index(1).required(true)) + .arg(Arg::with_name("WORK_DIR").index(2).required(true)) + .get_matches(); + + let config_file = matches.value_of_os("CONFIG_FILE").unwrap(); + let work_dir = Path::new(matches.value_of_os("WORK_DIR").unwrap()); + Ok(( + serde_yaml::from_reader(std::fs::File::open(config_file)?)?, + work_dir.into(), + )) +} + +pub fn load_secrets<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: DeserializeOwned, +{ + let secret_file_path: String = serde::Deserialize::deserialize(deserializer)?; + let secret_file = std::fs::File::open(secret_file_path) + .map_err(|_| serde::de::Error::custom("Could not open secret file."))?; + Ok(serde_yaml::from_reader(secret_file) + .map_err(|_| serde::de::Error::custom("Could not parse secrets file."))?) +} + +pub fn scriptworker_main( + do_work: impl FnOnce(Config, &Context, Task) -> Result<(), Error>, +) where + Config: DeserializeOwned, + A: DeserializeOwned, + E: DeserializeOwned, +{ + let result = (|| { + let (config, work_dir) = init_config::()?; + // TODO: Setup rust logging + let task_filename = work_dir.join("task.json"); + let task = Task::::load(&task_filename)?; + + do_work(config, &Context { work_dir }, task) + })(); + match result { + Ok(()) => std::process::exit(0), + Err(err) => { + if let Error::MalformedPayload(message) = &err { + std::println!("{}", &message) + } + if let Error::InternalError(message) = &err { + std::println!("{}", &message) + } + std::process::exit(err.exit_code()) + } + } +} + +#[cfg(not(test))] +// Work around for rust-lang/rust#62127 +pub use scriptworker_script_macros::main; diff --git a/script/src/task.rs b/script/src/task.rs new file mode 100644 index 000000000..45ff9cb9d --- /dev/null +++ b/script/src/task.rs @@ -0,0 +1,136 @@ +use std::path::{Path, PathBuf}; + +use serde::de::DeserializeOwned; +use serde_derive::Deserialize; + +use crate::error::Error; +use crate::Context; + +#[derive(Deserialize, Debug)] +pub struct Empty {} + +#[derive(Debug)] +pub struct TaskArtifacts { + pub task_type: String, + pub task_id: String, + // TODO: Figure out how to thread work dir here + // so we don't need to pass the context to get it. + pub paths: Vec, + pub attributes: A, +} + +impl<'de, A> serde::Deserialize<'de> for TaskArtifacts +where + A: serde::Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RawArtifacts { + pub task_type: String, + pub task_id: String, + // TODO: Path + pub paths: Vec, + #[serde(flatten)] + pub attributes: A, + } + let raw: RawArtifacts = serde::Deserialize::deserialize(deserializer)?; + let task_id = raw.task_id.clone(); + let paths = raw + .paths + .into_iter() + .map(|path| { + if path.is_absolute() { + Err(serde::de::Error::custom( + "Cannot sepecify absolute path in upstreamArtifacts.", + )) + } else { + Ok(ArtifactPath { + task_id: task_id.clone(), + path, + }) + } + }) + .collect::>()?; + Ok(TaskArtifacts:: { + task_type: raw.task_type, + task_id: raw.task_id, + paths, + attributes: raw.attributes, + }) + } +} + +#[derive(Debug)] +pub struct ArtifactPath { + task_id: String, + path: PathBuf, +} + +impl ArtifactPath { + pub fn task_path(&self) -> &PathBuf { + &self.path + } + pub fn file_path(&self, context: &Context) -> PathBuf { + context + .work_dir + .join("cot") + .join(&self.task_id) + .join(&self.path) + } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct TaskPayload { + pub upstream_artifacts: Vec>, + #[serde(flatten)] + pub extra: E, +} + +#[derive(Deserialize, Debug)] +pub struct Task { + pub dependencies: Vec, + pub scopes: Vec, + pub payload: TaskPayload, +} + +impl Task { + pub(crate) fn load(filename: &Path) -> Result, Error> + where + A: DeserializeOwned, + E: DeserializeOwned, + { + let file = std::fs::File::open(filename) + .map_err(|_| Error::InternalError("Could not open task definition.".to_string()))?; + Ok(serde_json::from_reader(file).map_err(|err| { + Error::MalformedPayload(format!("Could not parse task payload: {}", err)) + })?) + } + + pub fn require_scope(&self, scope: &str) -> Result<(), Error> { + if self.scopes.iter().any(|x| x == scope) { + Ok(()) + } else { + Err(Error::MalformedPayload(format!("missing scope {}", scope))) + } + } + + pub fn require_scopes(&self, scopes: impl IntoIterator) -> Result<(), Error> { + let missing_scopes: Vec<_> = scopes + .into_iter() + .filter(|scope| self.scopes.iter().all(|x| x != scope)) + .collect(); + if missing_scopes.is_empty() { + Ok(()) + } else { + Err(Error::MalformedPayload(format!( + "missing scopes: {:?}", + missing_scopes + ))) + } + } +}