From e7badcc6381a1a857ecb5146d4e61b7c38487721 Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:25:07 +0200 Subject: [PATCH] Basic support for cloning public projects (#10) - Clone a project given its id, namespace/slug or full url - Write a `.renku/config.toml` file containing the identifying information --- .github/CODEOWNERS | 1 + .github/workflows/ci.yml | 3 + Cargo.lock | 155 ++++++++++++++++++++++++- Cargo.toml | 4 + flake.nix | 4 +- src/cli.rs | 9 +- src/cli/cmd.rs | 43 ++++--- src/cli/cmd/project.rs | 3 +- src/cli/cmd/project/clone.rs | 186 ++++++++++++++++++++++++++++-- src/cli/cmd/userdoc.rs | 11 +- src/cli/cmd/version.rs | 29 +++-- src/cli/opts.rs | 14 ++- src/cli/sink.rs | 24 +++- src/data.rs | 9 ++ src/data/project_id.rs | 77 +++++++++++++ src/data/renku_url.rs | 69 ++++++++++++ src/data/simple_message.rs | 14 +++ src/httpclient.rs | 212 ++++++++++++++++++++++++++++++----- src/httpclient/data.rs | 49 ++++++++ src/httpclient/proxy.rs | 1 + src/lib.rs | 2 + src/project_config.rs | 112 ++++++++++++++++++ src/util.rs | 6 + src/util/file.rs | 1 + src/util/mod.rs | 1 - 25 files changed, 960 insertions(+), 79 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 src/data.rs create mode 100644 src/data/project_id.rs create mode 100644 src/data/renku_url.rs create mode 100644 src/data/simple_message.rs create mode 100644 src/project_config.rs create mode 100644 src/util.rs delete mode 100644 src/util/mod.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..dc6e8af --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @SwissDataScienceCenter/renku-python-maintainers @SwissDataScienceCenter/renku-graph-maintainers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02e54c0..587f57a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,9 +61,12 @@ jobs: command: run args: --release --features user-doc -- user-doc ./docs + # check installer on main only, as we reach the github api limit too + # quickly check-installer: name: "check installer" runs-on: ${{ matrix.os }} + if: github.ref == 'refs/heads/main' strategy: matrix: os: [macos-latest, ubuntu-latest] diff --git a/Cargo.lock b/Cargo.lock index 3932ef9..7e79f02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,6 +219,11 @@ name = "cc" version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -638,6 +643,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152dd546e5bdac80844ce6befabb9af5784ce375cb6cea554aed99fe2d1fb169" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -655,6 +669,21 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "git2" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +dependencies = [ + "bitflags 2.6.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -830,12 +859,32 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "iso8601-timestamp" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d4e5d712dd664b11e778d1cfc06c79ba2700d6bc1771e44fb7b6a4656b487d" +dependencies = [ + "generic-array", + "serde", + "time", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -857,6 +906,46 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libgit2-sys" +version = "0.17.0+1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1412,6 +1501,8 @@ dependencies = [ "console", "env_logger", "futures", + "git2", + "iso8601-timestamp", "log", "openssl", "predicates", @@ -1421,6 +1512,8 @@ dependencies = [ "serde_json", "snafu", "tokio", + "toml", + "url", "vergen", ] @@ -1490,9 +1583,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring", "rustls-pki-types", @@ -1598,6 +1691,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1886,6 +1988,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1956,6 +2092,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicase" version = "2.7.0" @@ -2315,6 +2457,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 24cd0d6..99ae3b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,10 @@ snafu = { version = "0.8.4" } tokio = { version = "1", features = ["full"] } futures = { version = "0.3" } regex = { version = "1.10.5" } +iso8601-timestamp = { version = "0.2.17" } +toml = { version = "0.8.12" } +git2 = "0.19.0" +url = { version = "2.5.1" } comrak = { version = "0.24.1", optional = true } [features] diff --git a/flake.nix b/flake.nix index 8b99c29..eca7f26 100644 --- a/flake.nix +++ b/flake.nix @@ -173,7 +173,7 @@ # Additional dev-shell environment variables can be set directly # MY_CUSTOM_DEVELOPMENT_VAR = "something else"; - RENKU_CLI_RENKU_URL = "https://ci-renku-3668.dev.renku.ch"; + RENKU_CLI_RENKU_URL = "https://ci-renku-3689.dev.renku.ch"; # Enable mold https://github.com/rui314/mold CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.clang}/bin/clang"; @@ -186,6 +186,8 @@ # Extra inputs can be added here; cargo and rustc are provided by default. packages = with pkgs; [ cargo-edit + cargo-expand + tokio-console fenixToolChain.rust-analyzer fenixToolChain.rustfmt ]; diff --git a/src/cli.rs b/src/cli.rs index 6fa9a4c..9a8aedc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,6 +2,7 @@ pub mod cmd; pub mod opts; pub mod sink; +use self::cmd::project::Error as ProjectError; use self::cmd::{CmdError, Context}; use self::opts::{MainOpts, SubCommand}; use clap::CommandFactory; @@ -18,10 +19,14 @@ pub async fn execute_cmd(opts: MainOpts) -> Result<(), CmdError> { let mut app = MainOpts::command(); input.print_completions(&mut app).await; } - SubCommand::Project(input) => input.exec(&ctx).await?, + SubCommand::Project(input) => input.exec(ctx).await?, + SubCommand::Clone(input) => input + .exec(ctx) + .await + .map_err(|source| ProjectError::Clone { source })?, #[cfg(feature = "user-doc")] - SubCommand::UserDoc(input) => input.exec(&ctx).await?, + SubCommand::UserDoc(input) => input.exec(ctx).await?, }; Ok(()) } diff --git a/src/cli/cmd.rs b/src/cli/cmd.rs index 1d04ec8..6dccf51 100644 --- a/src/cli/cmd.rs +++ b/src/cli/cmd.rs @@ -6,51 +6,64 @@ pub mod version; use super::sink::{Error as SinkError, Sink}; use crate::cli::opts::{CommonOpts, ProxySetting}; +use crate::data::renku_url::RenkuUrl; use crate::httpclient::{self, proxy, Client}; use serde::Serialize; use snafu::{ResultExt, Snafu}; const RENKULAB_IO: &str = "https://renkulab.io"; -pub struct Context<'a> { - pub opts: &'a CommonOpts, +pub struct Context { + pub opts: CommonOpts, pub client: Client, - pub renku_url: String, } -impl Context<'_> { +impl Context { pub fn new(opts: &CommonOpts) -> Result { - let base_url = get_renku_url(opts); - let client = Client::new(&base_url, proxy_settings(opts), &None, false) - .context(ContextCreateSnafu)?; + let base_url = get_renku_url(opts)?; + let client = + Client::new(base_url, proxy_settings(opts), None, false).context(ContextCreateSnafu)?; Ok(Context { - opts, + opts: opts.clone(), client, - renku_url: base_url, }) } - /// A short hand for `Sink::write(self.format(), value)` + pub fn renku_url(&self) -> &RenkuUrl { + self.client.base_url() + } + + /// A short hand for `Sink::write_out(self.format(), value)` async fn write_result(&self, value: &A) -> Result<(), SinkError> { let fmt = self.opts.format; - Sink::write(&fmt, value) + Sink::write_out(&fmt, value) + } + + /// A short hand for `Sink::write_err(self.format(), value)` + async fn write_err(&self, value: &A) -> Result<(), SinkError> { + let fmt = self.opts.format; + Sink::write_err(&fmt, value) } } -fn get_renku_url(opts: &CommonOpts) -> String { +fn get_renku_url(opts: &CommonOpts) -> Result { match &opts.renku_url { Some(u) => { log::debug!("Use renku url from arguments: {}", u); - u.clone() + Ok(u.clone()) } None => match std::env::var("RENKU_CLI_RENKU_URL").ok() { Some(u) => { log::debug!("Use renku url from env RENKU_CLI_RENKU_URL: {}", u); - u + RenkuUrl::parse(&u).map_err(|e| CmdError::ContextCreate { + source: httpclient::Error::UrlParse { source: e }, + }) } None => { log::debug!("Use renku url: https://renkulab.io"); - RENKULAB_IO.to_string() + RenkuUrl::parse(RENKULAB_IO).map_err(|e| CmdError::ContextCreate { + source: httpclient::Error::UrlParse { source: e }, + }) } }, } diff --git a/src/cli/cmd/project.rs b/src/cli/cmd/project.rs index 6a26532..74a82e9 100644 --- a/src/cli/cmd/project.rs +++ b/src/cli/cmd/project.rs @@ -6,6 +6,7 @@ use snafu::{ResultExt, Snafu}; #[derive(Debug, Snafu)] pub enum Error { + #[snafu(display("Error cloning project: {}", source))] Clone { source: clone::Error }, } @@ -17,7 +18,7 @@ pub struct Input { } impl Input { - pub async fn exec<'a>(&self, ctx: &Context<'a>) -> Result<(), Error> { + pub async fn exec(&self, ctx: Context) -> Result<(), Error> { match &self.subcmd { ProjectCommand::Clone(input) => input.exec(ctx).await.context(CloneSnafu), } diff --git a/src/cli/cmd/project/clone.rs b/src/cli/cmd/project/clone.rs index 8d748d4..a844917 100644 --- a/src/cli/cmd/project/clone.rs +++ b/src/cli/cmd/project/clone.rs @@ -1,22 +1,192 @@ +use crate::httpclient::data::ProjectDetails; +use crate::project_config::{ProjectConfigError, ProjectInfo, RenkuProjectConfig}; + use super::Context; +use crate::cli::sink::Error as SinkError; +use crate::data::project_id::{ProjectId, ProjectIdParseError}; +use crate::data::simple_message::SimpleMessage; +use crate::httpclient::Error as HttpError; +use std::sync::Arc; use clap::Parser; -use snafu::Snafu; +use git2::{Error as GitError, Repository}; +use snafu::{ResultExt, Snafu}; +use std::path::{Path, PathBuf}; +use tokio::task::{JoinError, JoinSet}; -/// Clone a project +/// Clone a project. +/// +/// Clones a renku project by creating a directory with the project +/// slug and cloning each code repository into it. #[derive(Parser, Debug)] pub struct Input { - /// The project slug - pub slug: String, + /// The project to clone, identified by either its id, the + /// namespace/slug identifier or the complete url. If a complete + /// url is given, it will override any renku-url that might have + /// been given otherwise. + #[arg()] + pub project_ref: ProjectId, + + /// Optional target directory to create the project in. By default + /// the current working directory is used. + #[arg()] + pub target_dir: Option, +} + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("An http error occurred: {}", source))] + HttpClient { source: HttpError }, + + #[snafu(display("Error writing data: {}", source))] + WriteResult { source: SinkError }, + + #[snafu(display("Error reading project id: {}", source))] + ProjectIdParse { source: ProjectIdParseError }, + + #[snafu(display("Error getting current directory: {}", source))] + CurrentDir { source: std::io::Error }, + + #[snafu(display("Error creating directory: {}", source))] + CreateDir { source: std::io::Error }, + + #[snafu(display("Error cloning project: {}", source))] + GitClone { source: GitError }, + + #[snafu(display("Error in task: {}", source))] + TaskJoin { source: JoinError }, + + #[snafu(display("Error creating config file: {}", source))] + RenkuConfig { source: ProjectConfigError }, + + #[snafu(display("The project name is missing: {}", repo_url))] + MissingProjectName { repo_url: String }, } impl Input { - pub async fn exec<'a>(&self, _ctx: &Context<'a>) -> Result<(), Error> { + pub async fn exec(&self, ctx: Context) -> Result<(), Error> { + let opt_details = ctx + .client + .get_project(&self.project_ref, ctx.opts.verbose > 1) + .await + .context(HttpClientSnafu)?; + if let Some(details) = opt_details { + let target = self.target_dir()?.join(&details.slug); + let renku_project_cfg = RenkuProjectConfig::new( + ctx.renku_url().clone(), + ProjectInfo { + id: details.id.clone(), + namespace: details.namespace.clone(), + slug: details.slug.clone(), + }, + ); + + ctx.write_err(&SimpleMessage { + message: format!( + "Cloning {} ({}) into {}...", + details.slug, + details.id, + &target.display() + ), + }) + .await + .context(WriteResultSnafu)?; + + write_config(renku_project_cfg, &target).await?; + + let ctx = clone_project(ctx, &details, target).await?; + ctx.write_result(&details).await.context(WriteResultSnafu)?; + } else { + ctx.write_err(&SimpleMessage { + message: format!("Project '{}' doesn't exist.", &self.project_ref), + }) + .await + .context(WriteResultSnafu)?; + } Ok(()) } + + fn target_dir(&self) -> Result { + match &self.target_dir { + Some(dir) => Ok(std::path::PathBuf::from(dir)), + None => std::env::current_dir().context(CurrentDirSnafu), + } + } } -#[derive(Debug, Snafu)] -pub enum Error { - Dummy, +async fn clone_project<'a>( + ctx: Context, + project: &ProjectDetails, + target: PathBuf, +) -> Result { + tokio::fs::create_dir_all(&target) + .await + .context(CreateDirSnafu)?; + + let mut tasks = JoinSet::new(); + let cc = Arc::new(ctx); + let tt = Arc::new(target); + for repo in project.repositories.iter() { + let cc = cc.clone(); + let tt = tt.clone(); + let rr = repo.to_string(); + tasks.spawn(clone_repository(cc, rr, tt)); + } + + while let Some(res) = tasks.join_next().await { + res.context(TaskJoinSnafu)??; + } + Ok(Arc::into_inner(cc).unwrap()) +} + +async fn clone_repository( + ctx: Arc, + repo_url: String, + dir: Arc, +) -> Result<(), Error> { + let name = match repo_url.rsplit_once('/') { + Some((_, n)) => Ok(n), + None => Err(Error::MissingProjectName { + repo_url: repo_url.clone(), + }), + }?; + let local_path = dir.join(name); + if local_path.exists() { + ctx.write_err(&SimpleMessage { + message: format!("The repository {} already exists", name), + }) + .await + .context(WriteResultSnafu)?; + } else { + // TODO use the repository builder to access more options, + // show clone progress and provide credentials + let (repo, repo_url, local_path) = tokio::task::spawn_blocking(|| { + let r = Repository::clone(&repo_url, &local_path).context(GitCloneSnafu); + (r, repo_url, local_path) + }) + .await + .context(TaskJoinSnafu)?; + let git_repo = repo?; + if ctx.opts.verbose > 1 { + let head = git_repo + .head() + .ok() + .and_then(|r| r.name().map(str::to_string)); + log::debug!("Checked out ref {:?} for repo {}", head, repo_url); + } + + ctx.write_err(&SimpleMessage { + message: format!("Cloned: {} to {}", repo_url, local_path.display()), + }) + .await + .context(WriteResultSnafu)?; + } + Ok(()) +} + +async fn write_config(data: RenkuProjectConfig, local_dir: &Path) -> Result<(), Error> { + let target = local_dir.join(".renku").join("config.toml"); + tokio::task::spawn_blocking(move || data.write(&target).context(RenkuConfigSnafu)) + .await + .context(TaskJoinSnafu)? } diff --git a/src/cli/cmd/userdoc.rs b/src/cli/cmd/userdoc.rs index 4dec975..dc2fc99 100644 --- a/src/cli/cmd/userdoc.rs +++ b/src/cli/cmd/userdoc.rs @@ -132,13 +132,15 @@ pub enum Error { } impl Input { - pub async fn exec<'a>(&self, ctx: &Context<'a>) -> Result<(), Error> { + pub async fn exec(&self, ctx: Context) -> Result<(), Error> { let md_regex: &Regex = &self.filter_regex; let myself = std::env::current_exe().context(GetBinarySnafu)?; let bin = match &self.renku_cli { Some(p) => p.as_path(), None => myself.as_path(), }; + + let fmt = ctx.opts.format; let walk = file_util::visit_entries(self.files.iter()) .try_filter(|p| future::ready(Self::path_match(&p.entry, md_regex))); walk.map_err(|source| Error::ListDir { source }) @@ -146,7 +148,7 @@ impl Input { let result = process_markdown_file(&entry.entry, bin, &self.result_marker).await?; match self.get_output() { OutputOption::Stdout => { - if ctx.opts.format != Format::Json { + if fmt != Format::Json { println!("{}", result); } } @@ -161,7 +163,7 @@ impl Input { entry, output: result, }; - ctx.write_result(&res).await.context(WriteResultSnafu)?; + Sink::write_out(&fmt, &res).context(WriteResultSnafu)?; Ok(()) }) .await?; @@ -262,6 +264,7 @@ fn run_cli_command(cli: &Path, line: &str) -> Result { let mut args = line.split_whitespace(); args.next(); // skip first word which is the binary name let remain: Vec<&str> = args.collect(); + // TODO use tokio::process instead let cmd = Command::new(cli) .args(remain) .output() @@ -317,8 +320,6 @@ fn parse_fence_info(info: &str) -> Option { parts.next().and_then(|s| FenceModifier::from_str(s).ok()) } -impl Sink for PathEntry {} - #[derive(Debug, Serialize)] struct Processed { pub entry: PathEntry, diff --git a/src/cli/cmd/version.rs b/src/cli/cmd/version.rs index fd7763b..0bf0771 100644 --- a/src/cli/cmd/version.rs +++ b/src/cli/cmd/version.rs @@ -14,7 +14,12 @@ use std::fmt; /// Queries the server for its version information and prints more /// version details about this client. #[derive(Parser, Debug, PartialEq)] -pub struct Input {} +pub struct Input { + /// Only show the client version and don't request server side + /// version information. + #[arg(long, default_value_t = false)] + pub client_only: bool, +} #[derive(Debug, Snafu)] pub enum Error { @@ -26,14 +31,20 @@ pub enum Error { } impl Input { - pub async fn exec<'a>(&self, ctx: &Context<'a>) -> Result<(), Error> { - let result = ctx - .client - .version(ctx.opts.verbose > 1) - .await - .context(HttpClientSnafu)?; - let vinfo = Versions::create(result, &ctx.renku_url); - ctx.write_result(&vinfo).await.context(WriteResultSnafu)?; + pub async fn exec(&self, ctx: &Context) -> Result<(), Error> { + if self.client_only { + let vinfo = BuildInfo::default(); + ctx.write_result(&vinfo).await.context(WriteResultSnafu)?; + } else { + let result = ctx + .client + .version(ctx.opts.verbose > 1) + .await + .context(HttpClientSnafu)?; + let urlstr = ctx.renku_url().as_str(); + let vinfo = Versions::create(result, urlstr); + ctx.write_result(&vinfo).await.context(WriteResultSnafu)?; + } Ok(()) } } diff --git a/src/cli/opts.rs b/src/cli/opts.rs index 5f888de..0f461c7 100644 --- a/src/cli/opts.rs +++ b/src/cli/opts.rs @@ -1,3 +1,5 @@ +use crate::data::renku_url::RenkuUrl; + use super::cmd::*; use clap::{ArgAction, Parser, ValueEnum, ValueHint}; use serde::{Deserialize, Serialize}; @@ -5,7 +7,7 @@ use std::str::FromStr; /// Main options are available to all commands. They must appear /// before a sub-command. -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command()] pub struct CommonOpts { /// Be more verbose when logging. Verbosity increases with each @@ -23,7 +25,7 @@ pub struct CommonOpts { /// The (base) URL to Renku. It can be given as environment /// variable RENKU_CLI_RENKU_URL. #[arg(long, value_hint = ValueHint::Url)] - pub renku_url: Option, + pub renku_url: Option, /// Set a proxy to use for doing http requests. By default, the /// system proxy will be used. Can be either `none` or . If @@ -52,6 +54,10 @@ pub enum SubCommand { #[command()] Project(project::Input), + /// Clone a project. (Shortcut for 'project clone') + #[command()] + Clone(project::clone::Input), + #[cfg(feature = "user-doc")] UserDoc(userdoc::Input), } @@ -61,8 +67,8 @@ pub enum SubCommand { /// them. Each sub command has its own set of flags/options and /// arguments. /// -/// Repository: https://github.com/SwissDataScienceCenter/renku-cli -/// Issue tracker: https://github.com/SwissDataScienceCenter/renku-cli/issues +/// Repository: +/// Issue tracker: #[derive(Parser, Debug)] #[command(name = "rnk", version)] pub struct MainOpts { diff --git a/src/cli/sink.rs b/src/cli/sink.rs index 17ac3ed..16cdb78 100644 --- a/src/cli/sink.rs +++ b/src/cli/sink.rs @@ -1,8 +1,13 @@ use crate::cli::opts::Format; +use crate::data::simple_message::SimpleMessage; +use crate::httpclient::data::*; +use crate::util::file::PathEntry; use serde::Serialize; use snafu::Snafu; use std::fmt::Display; +use super::BuildInfo; + #[derive(Debug, Snafu)] pub enum Error { Json { source: serde_json::Error }, @@ -12,7 +17,7 @@ pub trait Sink where Self: Serialize + Display, { - fn write(format: &Format, value: &Self) -> Result<(), Error> { + fn write_out(format: &Format, value: &Self) -> Result<(), Error> { match format { Format::Json => { serde_json::to_writer(std::io::stdout(), value)?; @@ -24,6 +29,18 @@ where } } } + fn write_err(format: &Format, value: &Self) -> Result<(), Error> { + match format { + Format::Json => { + serde_json::to_writer(std::io::stderr(), value)?; + Ok(()) + } + Format::Default => { + eprintln!("{}", value); + Ok(()) + } + } + } } impl From for Error { @@ -31,3 +48,8 @@ impl From for Error { Error::Json { source: e } } } + +impl Sink for ProjectDetails {} +impl Sink for SimpleMessage {} +impl Sink for BuildInfo {} +impl Sink for PathEntry {} diff --git a/src/data.rs b/src/data.rs new file mode 100644 index 0000000..5674cd0 --- /dev/null +++ b/src/data.rs @@ -0,0 +1,9 @@ +/*! + +Data types used across the cli + +*/ + +pub mod project_id; +pub mod renku_url; +pub mod simple_message; diff --git a/src/data/project_id.rs b/src/data/project_id.rs new file mode 100644 index 0000000..8b596ea --- /dev/null +++ b/src/data/project_id.rs @@ -0,0 +1,77 @@ +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; +use url::ParseError as UrlParseError; + +use super::renku_url::RenkuUrl; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum ProjectId { + NamespaceSlug { namespace: String, slug: String }, + Id(String), + FullUrl(RenkuUrl), +} + +#[derive(Debug, PartialEq, Snafu)] +pub enum ProjectIdParseError { + UrlParse { source: UrlParseError }, +} + +impl ProjectId { + pub fn parse(s: &str) -> Result { + s.parse::() + } +} + +impl FromStr for ProjectId { + type Err = ProjectIdParseError; + + fn from_str(s: &str) -> Result { + if s.starts_with("http") { + let u = RenkuUrl::parse(s).context(UrlParseSnafu)?; + Ok(ProjectId::FullUrl(u)) + } else { + match s.split_once('/') { + Some((pre, suf)) => Ok(ProjectId::NamespaceSlug { + namespace: pre.into(), + slug: suf.into(), + }), + None => Ok(ProjectId::Id(s.to_string())), + } + } + } +} + +impl fmt::Display for ProjectId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + ProjectId::NamespaceSlug { namespace, slug } => { + format!("{}/{}", namespace, slug) + } + ProjectId::Id(id) => id.to_string(), + ProjectId::FullUrl(url) => url.to_string(), + } + ) + } +} + +#[test] +fn read_to_string() { + let id1 = ProjectId::NamespaceSlug { + namespace: "n1".into(), + slug: "s1".into(), + }; + let id2 = ProjectId::Id("pr-id-42".into()); + let id3 = ProjectId::FullUrl(RenkuUrl::parse("http://localhost/project/1").unwrap()); + + for id in vec![id1, id2, id3] { + let id_str = format!("{}", id); + let id_parsed = ProjectId::parse(&id_str).unwrap(); + assert_eq!(id, id_parsed); + } +} diff --git a/src/data/renku_url.rs b/src/data/renku_url.rs new file mode 100644 index 0000000..7a23bc8 --- /dev/null +++ b/src/data/renku_url.rs @@ -0,0 +1,69 @@ +use std::{fmt::Display, str::FromStr}; + +use reqwest::Url; +use serde::{Deserialize, Serialize}; +use url::ParseError; + +// Need this newtype to be able to implement Serialize and Deserialize + +#[derive(Debug, PartialEq, Clone)] +pub struct RenkuUrl(Url); + +impl RenkuUrl { + pub fn new(url: Url) -> RenkuUrl { + RenkuUrl(url) + } + + pub fn parse(s: &str) -> Result { + s.parse::() + } + + pub fn as_url(&self) -> &Url { + let RenkuUrl(u) = self; + u + } + + pub fn as_str(&self) -> &str { + let RenkuUrl(u) = self; + u.as_str() + } + + pub fn join(&self, seg: &str) -> Result { + let RenkuUrl(u) = self; + u.join(seg).map(RenkuUrl) + } +} + +impl<'de> Deserialize<'de> for RenkuUrl { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let u = RenkuUrl::from_str(&s).map_err(serde::de::Error::custom)?; + Ok(u) + } +} + +impl Serialize for RenkuUrl { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let RenkuUrl(u) = self; + serializer.serialize_str(u.as_str()) + } +} +impl Display for RenkuUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl FromStr for RenkuUrl { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Url::parse(s).map(RenkuUrl) + } +} diff --git a/src/data/simple_message.rs b/src/data/simple_message.rs new file mode 100644 index 0000000..3e4a7db --- /dev/null +++ b/src/data/simple_message.rs @@ -0,0 +1,14 @@ +use std::fmt; + +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub struct SimpleMessage { + pub message: String, +} + +impl fmt::Display for SimpleMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} diff --git a/src/httpclient.rs b/src/httpclient.rs index a51137d..dd3add2 100644 --- a/src/httpclient.rs +++ b/src/httpclient.rs @@ -6,10 +6,12 @@ //! //! ```rust //! use rnk::httpclient; +//! use rnk::data::renku_url::RenkuUrl; +//! //! let client = httpclient::Client::new( -//! "https://renkulab.io", +//! RenkuUrl::parse("https://renkulab.io").unwrap(), //! httpclient::proxy::ProxySetting::System, -//! &None, +//! None, //! false //! ).unwrap(); //! async { @@ -24,9 +26,14 @@ pub mod data; pub mod proxy; +use crate::data::project_id::ProjectId; +use crate::data::renku_url::RenkuUrl; + use self::data::*; use reqwest::Certificate; use reqwest::ClientBuilder; +use reqwest::IntoUrl; +use reqwest::Url; use serde::de::DeserializeOwned; use snafu::{ResultExt, Snafu}; use std::path::PathBuf; @@ -52,6 +59,9 @@ pub enum Error { #[snafu(display("An error occured reading the response: {}", source))] DeserializeJson { source: serde_json::Error }, + + #[snafu(display("Error reading url: {}", source))] + UrlParse { source: url::ParseError }, } /// The renku http client. @@ -60,21 +70,28 @@ pub enum Error { /// endpoints. pub struct Client { client: reqwest::Client, - base_url: String, + settings: Settings, +} + +#[derive(Debug)] +struct Settings { + proxy: proxy::ProxySetting, + trusted_certificate: Option, + accept_invalid_certs: bool, + base_url: RenkuUrl, } impl Client { - pub fn new>( - renku_url: S, + pub fn new( + renku_url: RenkuUrl, proxy: proxy::ProxySetting, - trusted_certificate: &Option, + trusted_certificate: Option, accept_invalid_certs: bool, ) -> Result { - let url = renku_url.into(); - log::debug!("Create renku client for: {}", url); + log::debug!("Create renku client for: {}", renku_url); let mut client_builder = ClientBuilder::new().user_agent(USER_AGENT); client_builder = proxy.set(client_builder).context(ClientCreateSnafu)?; - match trusted_certificate { + match &trusted_certificate { Some(cert_file) => { log::debug!( "Adding extra certificate from file: {}", @@ -101,37 +118,81 @@ impl Client { let client = client_builder.build().context(ClientCreateSnafu)?; Ok(Client { client, - base_url: url, + settings: Settings { + proxy, + trusted_certificate, + accept_invalid_certs, + base_url: renku_url, + }, }) } + pub fn base_url(&self) -> &RenkuUrl { + &self.settings.base_url + } + + fn make_url(&self, path: &str) -> Result { + self.settings + .base_url + .as_url() + .join(path) + .context(UrlParseSnafu) + } + /// Runs a GET request to the given url. When `debug` is true, the /// response is first decoded into utf8 chars and logged at debug /// level. Otherwise bytes are directly decoded from JSON into the /// expected structure. async fn json_get(&self, path: &str, debug: bool) -> Result { - let url = &format!("{}{}", self.base_url, path); + let url = self.make_url(path)?; + log::debug!("JSON GET: {}", url); + let resp = self + .client + .get(url.clone()) + .send() + .await + .context(HttpSnafu { url: url.clone() })?; + if debug { + let body = resp.text().await.context(DeserializeRespSnafu)?; + log::debug!("GET {} -> {}", url, body); + serde_json::from_str::(&body).context(DeserializeJsonSnafu) + } else { + resp.json::().await.context(DeserializeRespSnafu) + } + } + + /// Runs a GET request to the given url. When `debug` is true, the + /// response is first decoded into utf8 chars and logged at debug + /// level. Otherwise bytes are directly decoded from JSON into the + /// expected structure. + async fn json_get_option( + &self, + path: &str, + debug: bool, + ) -> Result, Error> { + let url = self.make_url(path)?; + let resp = self + .client + .get(url.clone()) + .send() + .await + .context(HttpSnafu { url: url.clone() })?; + if debug { - let resp = self - .client - .get(url) - .send() - .await - .context(HttpSnafu { url })? - .text() - .await - .context(DeserializeRespSnafu)?; - log::debug!("GET {} -> {}", url, resp); - serde_json::from_str::(&resp).context(DeserializeJsonSnafu) + if resp.status() == reqwest::StatusCode::NOT_FOUND { + log::debug!("GET {} -> NotFound", &url); + Ok(None) + } else { + let body = &resp.text().await.context(DeserializeRespSnafu)?; + log::debug!("GET {} -> {}", &url, body); + let r = serde_json::from_str::(body).context(DeserializeJsonSnafu)?; + Ok(Some(r)) + } + } else if resp.status() == reqwest::StatusCode::NOT_FOUND { + Ok(None) } else { - self.client - .get(url) - .send() - .await - .context(HttpSnafu { url })? - .json::() - .await - .context(DeserializeRespSnafu) + let r = resp.json::().await.context(DeserializeRespSnafu)?; + Ok(Some(r)) } } @@ -145,4 +206,95 @@ impl Client { .await?; Ok(VersionInfo { search, data }) } + + pub async fn get_project( + &self, + id: &ProjectId, + debug: bool, + ) -> Result, Error> { + match id { + ProjectId::NamespaceSlug { namespace, slug } => { + self.get_project_by_slug(namespace, slug, debug).await + } + ProjectId::Id(pid) => self.get_project_by_id(pid, debug).await, + + ProjectId::FullUrl(url) => self.get_project_by_url(url.as_url().clone(), debug).await, + } + } + + /// Get project details given the namespace and slug. + pub async fn get_project_by_slug( + &self, + namespace: &str, + slug: &str, + debug: bool, + ) -> Result, Error> { + log::debug!("Get project by namespace/slug: {}/{}", namespace, slug); + let path = format!("/api/data/projects/{}/{}", namespace, slug); + let details = self.json_get_option::(&path, debug).await?; + Ok(details) + } + + /// Get project details by project id. + pub async fn get_project_by_id( + &self, + id: &str, + debug: bool, + ) -> Result, Error> { + log::debug!("Get project by id: {}", id); + let path = format!("/api/data/projects/{}", id); + let details = self.json_get_option::(&path, debug).await?; + Ok(details) + } + + pub async fn get_project_by_url( + &self, + url: U, + debug: bool, + ) -> Result, Error> { + let urlstr = url.as_str().to_string(); + let url = url.into_url().context(HttpSnafu { url: urlstr })?; + log::debug!("Get project by url: {}", &url); + // there are different urls identifying the project + // /api/data/projects/ + // /api/data/projects// + // /v2/projects/ (ui) + // /v2/projects// (ui) + // the api is only the first two. Try to replace `v2` with `api/data` + // note the ui urls are currently not stable + + let path = match url.path_segments() { + Some(it) => { + let mut seen = false; + it.flat_map(|s| { + if s == "v2" && !seen { + seen = true; + vec!["api", "data"] + } else { + vec![s] + } + }) + .fold(String::new(), |a, b| a + b + "/") + } + None => url.path().to_string(), + }; + + log::debug!("Transformed path {} to: {}", url.path(), &path); + let mut base = url.clone(); + base.set_path(""); + let base_url = RenkuUrl::new(base); + + log::debug!("Create temporary client for {}", &base_url); + let client = Client::new( + base_url, + self.settings.proxy.clone(), + self.settings.trusted_certificate.clone(), + self.settings.accept_invalid_certs, + )?; + + let details = client + .json_get_option::(&path, debug) + .await?; + Ok(details) + } } diff --git a/src/httpclient/data.rs b/src/httpclient/data.rs index 6569a74..febf42a 100644 --- a/src/httpclient/data.rs +++ b/src/httpclient/data.rs @@ -1,7 +1,29 @@ //! Defines data structures for requests and responses and their //! `De/Serialize` instances. +use iso8601_timestamp::Timestamp; use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Debug, Serialize, Deserialize)] +pub enum Visibility { + #[serde(alias = "public")] + Public, + #[serde(alias = "private")] + Private, +} +impl fmt::Display for Visibility { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + Visibility::Private => "private", + Visibility::Public => "public", + } + ) + } +} #[derive(Debug, Serialize, Deserialize)] pub struct SearchServiceVersion { @@ -22,3 +44,30 @@ pub struct VersionInfo { pub search: SearchServiceVersion, pub data: SimpleVersion, } + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProjectDetails { + pub id: String, + pub name: String, + pub namespace: String, + pub slug: String, + pub visibility: Visibility, + pub etag: Option, + pub repositories: Vec, + pub description: Option, + pub keywords: Vec, + pub creation_date: Timestamp, +} +impl fmt::Display for ProjectDetails { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let lines = self + .repositories + .iter() + .fold(String::new(), |a, b| a + "\n - " + b); + write!( + f, + "Id: {}\nNamespace/Slug: {}/{}\nVisibility: {}\nCreated At: {}\nRepositories:{}", + self.id, self.namespace, self.slug, self.visibility, self.creation_date, lines + ) + } +} diff --git a/src/httpclient/proxy.rs b/src/httpclient/proxy.rs index 0951929..e4fb329 100644 --- a/src/httpclient/proxy.rs +++ b/src/httpclient/proxy.rs @@ -1,6 +1,7 @@ use reqwest::ClientBuilder; use reqwest::{Proxy, Result}; +#[derive(Debug, Clone)] pub enum ProxySetting { System, None, diff --git a/src/lib.rs b/src/lib.rs index 8cb26b9..7f7b908 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,8 @@ pub mod cli; +pub mod data; pub mod error; pub mod httpclient; +pub mod project_config; pub mod util; pub use cli::execute_cmd; diff --git a/src/project_config.rs b/src/project_config.rs new file mode 100644 index 0000000..f6fcb43 --- /dev/null +++ b/src/project_config.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; +use snafu::Snafu; +use std::path::{Path, PathBuf}; + +use crate::data::renku_url::RenkuUrl; + +#[derive(Debug, Snafu)] +pub enum ProjectConfigError { + #[snafu(display("Unable to read config file {}: {}", path.display(), source))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("Unable to write config file {}: {}", path.display(), source))] + WriteFile { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("Unable to parse file {}: {}", path.display(), source))] + ParseFile { + source: toml::de::Error, + path: PathBuf, + }, + #[snafu(display("The config file could not be serialized"))] + WriteToml { + source: toml::ser::Error, + path: PathBuf, + }, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub struct RenkuProjectConfig { + /// The version of this config file. + version: u16, + + /// The base url to the renku platform. + pub renku_url: RenkuUrl, + + /// Information about the project + pub project: ProjectInfo, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub struct ProjectInfo { + pub id: String, + pub namespace: String, + pub slug: String, +} + +impl RenkuProjectConfig { + pub fn new(renku_url: RenkuUrl, project: ProjectInfo) -> RenkuProjectConfig { + RenkuProjectConfig { + version: 1, + renku_url, + project, + } + } + + pub fn read(file: &Path) -> Result { + let cnt = std::fs::read_to_string(file).map_err(|e| ProjectConfigError::ReadFile { + source: e, + path: file.to_path_buf(), + }); + cnt.and_then(|c| { + toml::from_str(&c).map_err(|e| ProjectConfigError::ParseFile { + source: e, + path: file.to_path_buf(), + }) + }) + } + + pub fn write(&self, file: &Path) -> Result<(), ProjectConfigError> { + if !file.exists() { + if let Some(dir) = file.parent() { + std::fs::create_dir_all(dir).map_err(|e| ProjectConfigError::WriteFile { + source: e, + path: file.to_path_buf(), + })?; + } + } + let cnt = toml::to_string(self).map_err(|e| ProjectConfigError::WriteToml { + source: e, + path: file.to_path_buf(), + }); + + cnt.and_then(|c| { + std::fs::write(file, c).map_err(|e| ProjectConfigError::WriteFile { + source: e, + path: file.to_path_buf(), + }) + }) + } +} + +#[test] +fn write_and_read_config() { + let data = RenkuProjectConfig { + version: 1, + renku_url: RenkuUrl::parse("http://renkulab.io").unwrap(), + project: ProjectInfo { + id: "abc123".into(), + namespace: "my-ns".into(), + slug: "projecta".into(), + }, + }; + let tmp = std::env::temp_dir(); + let target = tmp.join("test.conf"); + data.write(&target).unwrap(); + let from_file = RenkuProjectConfig::read(&target).unwrap(); + std::fs::remove_file(&target).unwrap(); + assert_eq!(data, from_file); +} diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..8ffd2ca --- /dev/null +++ b/src/util.rs @@ -0,0 +1,6 @@ +/*! + +Utility functions. + + */ +pub mod file; diff --git a/src/util/file.rs b/src/util/file.rs index eafc868..7872627 100644 --- a/src/util/file.rs +++ b/src/util/file.rs @@ -57,6 +57,7 @@ where .flatten() } +//TODO look at tokio-stream crate, that provides wrappers for implementing Stream for tokio::ReadDir /// Visits all entries of the given paths recursively using tokios async read_dir. pub fn visit_all(paths: I) -> impl Stream> + Send + 'static where diff --git a/src/util/mod.rs b/src/util/mod.rs deleted file mode 100644 index 2e172cd..0000000 --- a/src/util/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod file;