Some cleanup for userdoc cmd

.github/workflows/publish.yml

@@ -14,7 +14,6 @@ jobs:
         build:
         - aarch64
         - aarch64-musl
-        # - i686
         - amd64-musl
         - amd64
         include:
@@ -28,11 +27,6 @@ jobs:
           target: aarch64-unknown-linux-musl
           use-cross: true
           features: "--no-default-features --features rustls"
-        # - build: i686
-        #   os: ubuntu-latest
-        #   target: i686-unknown-linux-gnu
-        #   use-cross: true
-        #   features: "--no-default-features --features rustls"
         - build: amd64
           os: ubuntu-latest
           target: x86_64-unknown-linux-gnu

CONTRIBUTING.md

@@ -68,6 +68,14 @@ cargo run -- <options to the cli tool here>
 
 Run it without any options after the `--` to see a quick help.
 
+## Run ci checks
+
+All checks that are run by ci can be run locally with:
+```
+nix flake check
+```
+
+
 ## Nix Setup Description
 
 The `flake.nix` defines how to build and test this package by
@@ -130,3 +138,13 @@ Alternatively, edit `.envrc` to read `use flake .#<your-devshell-attribute>`.
    - in `cli.rs` run the command, most likely analogous to the
      existing ones
    - in `cmd.rs` add another `From` impl for the error (if necessary)
+
+### Adding integration test
+
+The crate [assert-cmd](https://docs.rs/assert_cmd/latest/assert_cmd/)
+is used to create integration tests. By default, integration tests are
+looked up in the `tests/` folder.
+
+To get started, look into the existing tests. The `common` module
+provides some convience functions, like the `mk_cmd` which creates the
+command and adds some global options for testing.

README.md

@@ -10,7 +10,7 @@
   - [x] for testing an all platforms
 - [x] contributing.md (check mold)
 - [x] add tokio and async all the things
-- [ ] user documentation
+- [x] user documentation
   - the `--help` is good for a quick and to-the-point documentation of
     the cli, but there must also be something more elaborate
   - this documentation should contain example runs of the cli and it's
@@ -19,8 +19,9 @@
   - silent code blocks
 - [ ] getting started guide (what to download, how to run etc)
 - [ ] rename binary to `rnk`
-- [ ] test on mac and windows
-- [ ] check to create a mac package
+- [ ] test on mac 
+- [x] test on windows
+- [ ] check how to create a proper mac package
 - [ ] nice to have: clone a project / repo
 - [ ] think better what is `pub` and what is not, thinking about
       providing a rust library alongside the cli maybe

docs/index.md

@@ -3,13 +3,48 @@
 This is the documentation for `renku-cli` the command line interface
 to the Renku platform.
 
-To get quick help, use the cli like this:
 
-``` :renku-cli
-renku-cli --help
+## Installation
+
+The binary name for the renku-cli is `renku-cli`.
+
+### Manual Download
+
+You can download the binary for your platform from the [release
+page](https://github.com/SwissDataScienceCenter/renku-cli/releases/latest).
+
+If you run on MacOS, download the `*-darwin` binary. If you run some
+form of linux, try `*-amd64` or `*-aarch64`. Last for Windows use the
+`*-windows` binary.
+
+### Nix User
+
+If you are a nix user and have flakes enabled, you can install renku-cli
+from this repository:
+
+```
+nix profile install github:SwissDatascienceCenter/renku-cli
 ```
 
+If you want to try it out without installing:
+```
+nix run github:SwissDatascienceCenter/renku-cli
+```
+
+### Debian/Ubuntu User
+
+TODO
+
+### Mac Homebrew
 
-## Content
+TODO
 
-- [Clone a project](./project/clone)
+## Getting started
+
+The renku cli accepts commands to interact with the renku platform. To
+get an overview of possible commands, run the binary without any
+options or adding `--help`.
+
+``` bash renku-cli
+renku-cli --help
+```

src/cli/cmd.rs

@@ -5,7 +5,7 @@ pub mod userdoc;
 pub mod version;
 
 use super::sink::{Error as SinkError, Sink};
-use crate::cli::opts::{CommonOpts, Format, ProxySetting};
+use crate::cli::opts::{CommonOpts, ProxySetting};
 use crate::httpclient::{self, proxy, Client};
 use serde::Serialize;
 use snafu::{ResultExt, Snafu};
@@ -31,13 +31,9 @@ impl Context<'_> {
     }
 
     /// A short hand for `Sink::write(self.format(), value)`
-    async fn write_result<A: Sink + Serialize>(&self, value: A) -> Result<(), SinkError> {
-        let fmt = self.format();
-        Sink::write(fmt, &value)
-    }
-
-    fn format(&self) -> Format {
-        self.opts.format.unwrap_or(Format::Default)
+    async fn write_result<A: Sink + Serialize>(&self, value: &A) -> Result<(), SinkError> {
+        let fmt = self.opts.format;
+        Sink::write(&fmt, value)
     }
 }
 

src/cli/cmd/userdoc.rs

@@ -1,5 +1,6 @@
 use super::Context;
-use crate::cli::sink::Error as SinkError;
+use crate::cli::opts::Format;
+use crate::cli::sink::{Error as SinkError, Sink};
 use crate::util::file as file_util;
 use crate::util::file::PathEntry;
 use clap::{Parser, ValueHint};
@@ -8,16 +9,22 @@ use comrak::{Arena, Options};
 use futures::future;
 use futures::stream::TryStreamExt;
 use regex::Regex;
+use serde::Serialize;
 use snafu::{ResultExt, Snafu};
 use std::cell::RefCell;
+use std::fmt;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
+use std::str::FromStr;
 
 /// Reads markdown files and processes renku-cli code blocks.
 ///
-/// Each code block marked with `:renku-cli` is run against this
-/// binary and the result is added below the command code-block.
+/// Each code block marked with `renku-cli` or `rnk` is run against
+/// this binary and the result is added below the command code-block.
+///
+/// If you use `renku-cli:silent` or `rnk:silent` the command will be
+/// run, but the output is ignored.
 #[derive(Parser, Debug)]
 pub struct Input {
     /// The markdown file(s) to process. If a directory is given, it
@@ -49,16 +56,15 @@ pub struct Input {
     #[arg(long, default_value_t = false)]
     pub overwrite: bool,
 
-    /// The code block marker to use for detecting which code blocks
-    /// to extract.
-    #[arg(long, default_value = ":renku-cli")]
-    pub code_marker: String,
-
     /// The code block marker to use for annotating the result code blocks
     /// that are inserted into the document.
-    #[arg(long, default_value = ":renku-cli-output")]
+    #[arg(long, default_value = "renku-cli-output")]
     pub result_marker: String,
 
+    /// A regex for filtering files when traversing directories. By
+    /// default only markdown (*.md) files are picked up. The regex is
+    /// matched against the simple file name (not the absolute one
+    /// including the full path).
     #[arg(long, default_value = "^.*\\.md$")]
     pub filter_regex: Regex,
 }
@@ -126,7 +132,7 @@ pub enum Error {
 }
 
 impl Input {
-    pub async fn exec<'a>(&self, _ctx: &Context<'a>) -> Result<(), Error> {
+    pub async fn exec<'a>(&self, ctx: &Context<'a>) -> Result<(), Error> {
         let md_regex: &Regex = &self.filter_regex;
         let myself = std::env::current_exe().context(GetBinarySnafu)?;
         let bin = match &self.renku_cli {
@@ -137,25 +143,25 @@ impl Input {
             .try_filter(|p| future::ready(Self::path_match(&p.entry, md_regex)));
         walk.map_err(|source| Error::ListDir { source })
             .try_for_each_concurrent(10, |entry| async move {
-                eprintln!("Processing {} …", entry);
-                let result = process_markdown_file(
-                    &entry.entry,
-                    bin,
-                    &self.result_marker,
-                    &self.code_marker,
-                )
-                .await?;
+                let result = process_markdown_file(&entry.entry, bin, &self.result_marker).await?;
                 match self.get_output() {
                     OutputOption::Stdout => {
-                        println!("{}", result);
+                        if ctx.opts.format != Format::Json {
+                            println!("{}", result);
+                        }
                     }
                     OutputOption::OutFile(f) => {
-                        write_to_file(f, &result, self.overwrite)?;
+                        write_to_file(f, &result, self.overwrite, true)?;
                     }
                     OutputOption::OutDir(f) => {
                         write_to_dir(&entry, f, &result, self.overwrite)?;
                     }
                 }
+                let res = Processed {
+                    entry,
+                    output: result,
+                };
+                ctx.write_result(&res).await.context(WriteResultSnafu)?;
                 Ok(())
             })
             .await?;
@@ -182,16 +188,17 @@ fn write_to_dir(
         log::debug!("Ensuring directory: {}", p.display());
         std::fs::create_dir_all(p).context(CreateDirSnafu)?
     }
-    write_to_file(&out, content, overwrite)?;
+    write_to_file(&out, content, overwrite, false)?;
     Ok(())
 }
 
-fn write_to_file(file: &Path, content: &str, overwrite: bool) -> Result<(), Error> {
+fn write_to_file(file: &Path, content: &str, overwrite: bool, append: bool) -> Result<(), Error> {
     let mut out = std::fs::File::options()
         .write(true)
-        .truncate(overwrite)
+        .truncate(overwrite && !append)
         .create(overwrite)
         .create_new(!overwrite)
+        .append(append)
         .open(file)
         .context(WriteFileSnafu)?;
 
@@ -213,23 +220,32 @@ async fn process_markdown_file(
     file: &PathBuf,
     cli_binary: &Path,
     result_marker: &str,
-    code_marker: &str,
 ) -> Result<String, Error> {
     let src_md = std::fs::read_to_string(file).context(ReadFileSnafu { path: file })?;
     let src_nodes = Arena::new();
     let root = comrak::parse_document(&src_nodes, src_md.as_str(), &Options::default());
     for node in root.descendants() {
         let node_data = node.data.borrow();
         if let NodeValue::CodeBlock(ref cc) = node_data.value {
-            let code_info = &cc.info;
             let command = &cc.literal;
-            if code_info == code_marker {
-                let cli_out = run_cli_command(cli_binary, command)?;
-                let nn = src_nodes.alloc(AstNode::new(RefCell::new(Ast::new(
-                    make_code_block(result_marker, cli_out),
-                    node_data.sourcepos.end,
-                ))));
-                node.insert_after(nn);
+            log::debug!("Process code block: {}", &cc.info);
+            match parse_fence_info(&cc.info) {
+                None => {
+                    log::debug!("Code block not processed: {}", &cc.info);
+                }
+                Some(FenceModifier::Default) => {
+                    log::debug!("Run code block and insert result for: {}", &cc.info);
+                    let cli_out = run_cli_command(cli_binary, command)?;
+                    let nn = src_nodes.alloc(AstNode::new(RefCell::new(Ast::new(
+                        make_code_block(result_marker, cli_out),
+                        node_data.sourcepos.end,
+                    ))));
+                    node.insert_after(nn);
+                }
+                Some(FenceModifier::Silent) => {
+                    log::debug!("Run code block and ignore result for: {}", &cc.info);
+                    run_cli_command(cli_binary, command)?;
+                }
             }
         }
     }
@@ -241,6 +257,7 @@ async fn process_markdown_file(
 
 /// Run the given command line using the given binary.
 fn run_cli_command(cli: &Path, line: &str) -> Result<String, Error> {
+    log::debug!("Run: {} {}", cli.display(), line);
     // TODO: instead of running itself as a new process, just call main
     let mut args = line.split_whitespace();
     args.next(); // skip first word which is the binary name
@@ -250,6 +267,7 @@ fn run_cli_command(cli: &Path, line: &str) -> Result<String, Error> {
         .output()
         .context(ExecuteCliSnafu)?;
     if cmd.status.success() {
+        log::debug!("Command ran successful");
         let out = String::from_utf8(cmd.stdout).context(Utf8DecodeSnafu)?;
         Ok(out)
     } else {
@@ -272,3 +290,45 @@ fn make_code_block(marker: &str, content: String) -> NodeValue {
         literal: content,
     })
 }
+
+enum FenceModifier {
+    Default,
+    Silent,
+}
+
+impl FromStr for FenceModifier {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "renku-cli" => Ok(FenceModifier::Default),
+            "rnk" => Ok(FenceModifier::Default),
+            "renku-cli:silent" => Ok(FenceModifier::Silent),
+            "rnk:silent" => Ok(FenceModifier::Silent),
+            &_ => Err(format!("Invalid modifier: {}", s)),
+        }
+    }
+}
+
+fn parse_fence_info(info: &str) -> Option<FenceModifier> {
+    log::debug!("Read fence info: {}", info);
+    let mut parts = info.split_whitespace();
+    parts.next(); // skip language definition
+    parts.next().and_then(|s| FenceModifier::from_str(s).ok())
+}
+
+impl Sink for PathEntry {}
+
+#[derive(Debug, Serialize)]
+struct Processed {
+    pub entry: PathEntry,
+    pub output: String,
+}
+
+impl fmt::Display for Processed {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Processed {} ...", self.entry.entry.display())
+    }
+}
+
+impl Sink for Processed {}

src/cli/cmd/version.rs

@@ -33,7 +33,7 @@ impl Input {
             .await
             .context(HttpClientSnafu)?;
         let vinfo = Versions::create(result, &ctx.renku_url);
-        ctx.write_result(vinfo).await.context(WriteResultSnafu)?;
+        ctx.write_result(&vinfo).await.context(WriteResultSnafu)?;
         Ok(())
     }
 }