diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30e1fd5..4ca8033 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,6 @@ jobs: build: - aarch64 - aarch64-musl - # - i686 - amd64-musl - amd64 include: @@ -28,11 +27,6 @@ jobs: target: aarch64-unknown-linux-musl use-cross: true features: "--no-default-features --features rustls" - # - build: i686 - # os: ubuntu-latest - # target: i686-unknown-linux-gnu - # use-cross: true - # features: "--no-default-features --features rustls" - build: amd64 os: ubuntu-latest target: x86_64-unknown-linux-gnu diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d50aa59..f0725e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,14 @@ cargo run -- Run it without any options after the `--` to see a quick help. +## Run ci checks + +All checks that are run by ci can be run locally with: +``` +nix flake check +``` + + ## Nix Setup Description The `flake.nix` defines how to build and test this package by @@ -130,3 +138,13 @@ Alternatively, edit `.envrc` to read `use flake .#`. - in `cli.rs` run the command, most likely analogous to the existing ones - in `cmd.rs` add another `From` impl for the error (if necessary) + +### Adding integration test + +The crate [assert-cmd](https://docs.rs/assert_cmd/latest/assert_cmd/) +is used to create integration tests. By default, integration tests are +looked up in the `tests/` folder. + +To get started, look into the existing tests. The `common` module +provides some convience functions, like the `mk_cmd` which creates the +command and adds some global options for testing. diff --git a/README.md b/README.md index f046695..20e7a73 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [x] for testing an all platforms - [x] contributing.md (check mold) - [x] add tokio and async all the things -- [ ] user documentation +- [x] user documentation - the `--help` is good for a quick and to-the-point documentation of the cli, but there must also be something more elaborate - this documentation should contain example runs of the cli and it's @@ -19,8 +19,9 @@ - silent code blocks - [ ] getting started guide (what to download, how to run etc) - [ ] rename binary to `rnk` -- [ ] test on mac and windows -- [ ] check to create a mac package +- [ ] test on mac +- [x] test on windows +- [ ] check how to create a proper mac package - [ ] nice to have: clone a project / repo - [ ] think better what is `pub` and what is not, thinking about providing a rust library alongside the cli maybe diff --git a/docs/index.md b/docs/index.md index e7c260b..fdf1868 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,13 +3,48 @@ This is the documentation for `renku-cli` the command line interface to the Renku platform. -To get quick help, use the cli like this: -``` :renku-cli -renku-cli --help +## Installation + +The binary name for the renku-cli is `renku-cli`. + +### Manual Download + +You can download the binary for your platform from the [release +page](https://github.com/SwissDataScienceCenter/renku-cli/releases/latest). + +If you run on MacOS, download the `*-darwin` binary. If you run some +form of linux, try `*-amd64` or `*-aarch64`. Last for Windows use the +`*-windows` binary. + +### Nix User + +If you are a nix user and have flakes enabled, you can install renku-cli +from this repository: + +``` +nix profile install github:SwissDatascienceCenter/renku-cli ``` +If you want to try it out without installing: +``` +nix run github:SwissDatascienceCenter/renku-cli +``` + +### Debian/Ubuntu User + +TODO + +### Mac Homebrew -## Content +TODO -- [Clone a project](./project/clone) +## Getting started + +The renku cli accepts commands to interact with the renku platform. To +get an overview of possible commands, run the binary without any +options or adding `--help`. + +``` bash renku-cli +renku-cli --help +``` diff --git a/docs/project/clone.md b/docs/project/clone.md deleted file mode 100644 index a16ac67..0000000 --- a/docs/project/clone.md +++ /dev/null @@ -1,8 +0,0 @@ -# Clone a project - -The `clone` sub command allows to clone a project to your local hard drive. - -Available options are: -``` :renku-cli -renku-cli project clone --help -``` diff --git a/src/cli/cmd.rs b/src/cli/cmd.rs index f9f2e36..1d04ec8 100644 --- a/src/cli/cmd.rs +++ b/src/cli/cmd.rs @@ -5,7 +5,7 @@ pub mod userdoc; pub mod version; use super::sink::{Error as SinkError, Sink}; -use crate::cli::opts::{CommonOpts, Format, ProxySetting}; +use crate::cli::opts::{CommonOpts, ProxySetting}; use crate::httpclient::{self, proxy, Client}; use serde::Serialize; use snafu::{ResultExt, Snafu}; @@ -31,13 +31,9 @@ impl Context<'_> { } /// A short hand for `Sink::write(self.format(), value)` - async fn write_result(&self, value: A) -> Result<(), SinkError> { - let fmt = self.format(); - Sink::write(fmt, &value) - } - - fn format(&self) -> Format { - self.opts.format.unwrap_or(Format::Default) + async fn write_result(&self, value: &A) -> Result<(), SinkError> { + let fmt = self.opts.format; + Sink::write(&fmt, value) } } diff --git a/src/cli/cmd/userdoc.rs b/src/cli/cmd/userdoc.rs index ee7982c..08e16b2 100644 --- a/src/cli/cmd/userdoc.rs +++ b/src/cli/cmd/userdoc.rs @@ -1,5 +1,6 @@ use super::Context; -use crate::cli::sink::Error as SinkError; +use crate::cli::opts::Format; +use crate::cli::sink::{Error as SinkError, Sink}; use crate::util::file as file_util; use crate::util::file::PathEntry; use clap::{Parser, ValueHint}; @@ -8,16 +9,22 @@ use comrak::{Arena, Options}; use futures::future; use futures::stream::TryStreamExt; use regex::Regex; +use serde::Serialize; use snafu::{ResultExt, Snafu}; use std::cell::RefCell; +use std::fmt; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; +use std::str::FromStr; /// Reads markdown files and processes renku-cli code blocks. /// -/// Each code block marked with `:renku-cli` is run against this -/// binary and the result is added below the command code-block. +/// Each code block marked with `renku-cli` or `rnk` is run against +/// this binary and the result is added below the command code-block. +/// +/// If you use `renku-cli:silent` or `rnk:silent` the command will be +/// run, but the output is ignored. #[derive(Parser, Debug)] pub struct Input { /// The markdown file(s) to process. If a directory is given, it @@ -49,16 +56,15 @@ pub struct Input { #[arg(long, default_value_t = false)] pub overwrite: bool, - /// The code block marker to use for detecting which code blocks - /// to extract. - #[arg(long, default_value = ":renku-cli")] - pub code_marker: String, - /// The code block marker to use for annotating the result code blocks /// that are inserted into the document. - #[arg(long, default_value = ":renku-cli-output")] + #[arg(long, default_value = "renku-cli-output")] pub result_marker: String, + /// A regex for filtering files when traversing directories. By + /// default only markdown (*.md) files are picked up. The regex is + /// matched against the simple file name (not the absolute one + /// including the full path). #[arg(long, default_value = "^.*\\.md$")] pub filter_regex: Regex, } @@ -126,7 +132,7 @@ pub enum Error { } impl Input { - pub async fn exec<'a>(&self, _ctx: &Context<'a>) -> Result<(), Error> { + pub async fn exec<'a>(&self, ctx: &Context<'a>) -> Result<(), Error> { let md_regex: &Regex = &self.filter_regex; let myself = std::env::current_exe().context(GetBinarySnafu)?; let bin = match &self.renku_cli { @@ -137,25 +143,25 @@ impl Input { .try_filter(|p| future::ready(Self::path_match(&p.entry, md_regex))); walk.map_err(|source| Error::ListDir { source }) .try_for_each_concurrent(10, |entry| async move { - eprintln!("Processing {} …", entry); - let result = process_markdown_file( - &entry.entry, - bin, - &self.result_marker, - &self.code_marker, - ) - .await?; + let result = process_markdown_file(&entry.entry, bin, &self.result_marker).await?; match self.get_output() { OutputOption::Stdout => { - println!("{}", result); + if ctx.opts.format != Format::Json { + println!("{}", result); + } } OutputOption::OutFile(f) => { - write_to_file(f, &result, self.overwrite)?; + write_to_file(f, &result, self.overwrite, true)?; } OutputOption::OutDir(f) => { write_to_dir(&entry, f, &result, self.overwrite)?; } } + let res = Processed { + entry, + output: result, + }; + ctx.write_result(&res).await.context(WriteResultSnafu)?; Ok(()) }) .await?; @@ -182,16 +188,17 @@ fn write_to_dir( log::debug!("Ensuring directory: {}", p.display()); std::fs::create_dir_all(p).context(CreateDirSnafu)? } - write_to_file(&out, content, overwrite)?; + write_to_file(&out, content, overwrite, false)?; Ok(()) } -fn write_to_file(file: &Path, content: &str, overwrite: bool) -> Result<(), Error> { +fn write_to_file(file: &Path, content: &str, overwrite: bool, append: bool) -> Result<(), Error> { let mut out = std::fs::File::options() .write(true) - .truncate(overwrite) + .truncate(overwrite && !append) .create(overwrite) .create_new(!overwrite) + .append(append) .open(file) .context(WriteFileSnafu)?; @@ -213,7 +220,6 @@ async fn process_markdown_file( file: &PathBuf, cli_binary: &Path, result_marker: &str, - code_marker: &str, ) -> Result { let src_md = std::fs::read_to_string(file).context(ReadFileSnafu { path: file })?; let src_nodes = Arena::new(); @@ -221,15 +227,25 @@ async fn process_markdown_file( for node in root.descendants() { let node_data = node.data.borrow(); if let NodeValue::CodeBlock(ref cc) = node_data.value { - let code_info = &cc.info; let command = &cc.literal; - if code_info == code_marker { - let cli_out = run_cli_command(cli_binary, command)?; - let nn = src_nodes.alloc(AstNode::new(RefCell::new(Ast::new( - make_code_block(result_marker, cli_out), - node_data.sourcepos.end, - )))); - node.insert_after(nn); + log::debug!("Process code block: {}", &cc.info); + match parse_fence_info(&cc.info) { + None => { + log::debug!("Code block not processed: {}", &cc.info); + } + Some(FenceModifier::Default) => { + log::debug!("Run code block and insert result for: {}", &cc.info); + let cli_out = run_cli_command(cli_binary, command)?; + let nn = src_nodes.alloc(AstNode::new(RefCell::new(Ast::new( + make_code_block(result_marker, cli_out), + node_data.sourcepos.end, + )))); + node.insert_after(nn); + } + Some(FenceModifier::Silent) => { + log::debug!("Run code block and ignore result for: {}", &cc.info); + run_cli_command(cli_binary, command)?; + } } } } @@ -241,6 +257,7 @@ async fn process_markdown_file( /// Run the given command line using the given binary. fn run_cli_command(cli: &Path, line: &str) -> Result { + log::debug!("Run: {} {}", cli.display(), line); // TODO: instead of running itself as a new process, just call main let mut args = line.split_whitespace(); args.next(); // skip first word which is the binary name @@ -250,6 +267,7 @@ fn run_cli_command(cli: &Path, line: &str) -> Result { .output() .context(ExecuteCliSnafu)?; if cmd.status.success() { + log::debug!("Command ran successful"); let out = String::from_utf8(cmd.stdout).context(Utf8DecodeSnafu)?; Ok(out) } else { @@ -272,3 +290,45 @@ fn make_code_block(marker: &str, content: String) -> NodeValue { literal: content, }) } + +enum FenceModifier { + Default, + Silent, +} + +impl FromStr for FenceModifier { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "renku-cli" => Ok(FenceModifier::Default), + "rnk" => Ok(FenceModifier::Default), + "renku-cli:silent" => Ok(FenceModifier::Silent), + "rnk:silent" => Ok(FenceModifier::Silent), + &_ => Err(format!("Invalid modifier: {}", s)), + } + } +} + +fn parse_fence_info(info: &str) -> Option { + log::debug!("Read fence info: {}", info); + let mut parts = info.split_whitespace(); + parts.next(); // skip language definition + parts.next().and_then(|s| FenceModifier::from_str(s).ok()) +} + +impl Sink for PathEntry {} + +#[derive(Debug, Serialize)] +struct Processed { + pub entry: PathEntry, + pub output: String, +} + +impl fmt::Display for Processed { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Processed {} ...", self.entry.entry.display()) + } +} + +impl Sink for Processed {} diff --git a/src/cli/cmd/version.rs b/src/cli/cmd/version.rs index 77452db..fd7763b 100644 --- a/src/cli/cmd/version.rs +++ b/src/cli/cmd/version.rs @@ -33,7 +33,7 @@ impl Input { .await .context(HttpClientSnafu)?; let vinfo = Versions::create(result, &ctx.renku_url); - ctx.write_result(vinfo).await.context(WriteResultSnafu)?; + ctx.write_result(&vinfo).await.context(WriteResultSnafu)?; Ok(()) } } diff --git a/src/cli/opts.rs b/src/cli/opts.rs index c4c7384..1db9bb7 100644 --- a/src/cli/opts.rs +++ b/src/cli/opts.rs @@ -17,8 +17,8 @@ pub struct CommonOpts { /// may choose to not show every detail for better readability. /// The json output format can be used to always show all details /// in a structured form. - #[arg(short, long, value_enum)] - pub format: Option, + #[arg(short, long, value_enum, default_value_t = Format::Default)] + pub format: Format, /// The (base) URL to Renku. It can be given as environment /// variable RENKU_CLI_RENKU_URL. @@ -74,7 +74,7 @@ pub struct MainOpts { } /// The format for presenting the results. -#[derive(ValueEnum, Debug, Copy, Clone, Serialize, Deserialize)] +#[derive(ValueEnum, Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] pub enum Format { Json, Default, diff --git a/src/cli/sink.rs b/src/cli/sink.rs index d7369e6..17ac3ed 100644 --- a/src/cli/sink.rs +++ b/src/cli/sink.rs @@ -12,14 +12,14 @@ pub trait Sink where Self: Serialize + Display, { - fn write(format: Format, value: &Self) -> Result<(), Error> { + fn write(format: &Format, value: &Self) -> Result<(), Error> { match format { Format::Json => { - serde_json::to_writer(std::io::stdout(), &value)?; + serde_json::to_writer(std::io::stdout(), value)?; Ok(()) } Format::Default => { - println!("{}", &value); + println!("{}", value); Ok(()) } } diff --git a/src/util/file.rs b/src/util/file.rs index 2d5412e..eafc868 100644 --- a/src/util/file.rs +++ b/src/util/file.rs @@ -1,5 +1,6 @@ use futures::TryStreamExt; use futures::{stream, Stream, StreamExt}; +use serde::Serialize; use std::fmt; use std::io; use std::path::{Path, PathBuf, StripPrefixError}; @@ -19,6 +20,7 @@ pub fn splice_name(fname: &str, suffix: &i32) -> String { } } +#[derive(Debug, Serialize)] pub struct PathEntry { pub root: PathBuf, pub entry: PathBuf,