base argmaps and various updates (#29)

CHANGELOG.md

@@ -1,9 +1,20 @@
 # Change Log
 
-## [3.4.1] - 2024-11-12
+## [3.5.0] - 2024-11-13
+
+### Changed
+- `--arg, --arg-map, and --arg-map` are now `--target-argmap, and --target-argmap`
+- Default commands directory from `monorail` to `monorail/cmd`
+
+### Added
+- Optional per-target base argmaps
+- Default argmaps directory of `monorail/argmap`
 
 ### Changed
 
+## [3.4.1] - 2024-11-12
+
+### Changed
 - `--arg, --arg-map, and --arg-map` may all be specified simultaneously, and are processed with a precendece order
 - Support for multiple `--arg-map` and `--arg-map-file` arguments and merging of them
 

Cargo.toml

@@ -4,7 +4,7 @@ description = "A tool for effective polyglot, multi-project monorepo development
 license = "MIT"
 homepage = "https://github.com/pnordahl/monorail"
 repository = "https://github.com/pnordahl/monorail"
-version = "3.4.1"
+version = "3.5.0"
 authors = ["Patrick Nordahl <[email protected]>"]
 edition = "2021"
 keywords = ["monorail", "monorepo", "build", "cli", "build-tool"]

Monorail.reference.js

@@ -59,6 +59,30 @@ const reference = {
 				"path/within/repository"
 			],
 
+			/*
+			Configuration and overrides for argument mappings.
+			*/
+			"argmaps": {
+				/*
+				Default location within this target path containing argmap 
+				definition files. This is used when locating argmaps for this
+				target automatically when it is changed.
+
+				Optional, default: "monorail/argmap"
+				*/
+				"path": "path/within/this/target",
+
+				/*
+				Path to the default argmap to load for this target. This argmap
+				is loaded prior to any arguments supplied by use of 
+				--arg, --target-argmap, and --target-argmap-file switches on `monorail run`.
+
+				Optional, default: "base.json"
+				*/
+				"base": "path/within/target/argmaps/path"
+			}
+
+
 			/*
 			Configuration and overrides for this targets commands.
 			Optional.
@@ -69,7 +93,7 @@ const reference = {
 				executables. This is used when locating executables for a
 				command when no definition path for that command is specified.
 
-				Optional, default: "monorail"
+				Optional, default: "monorail/cmd"
 				*/
 				"path": "path/within/this/target"
 

README.md

@@ -299,37 +299,58 @@ This will run `build` first for `dep1`, and then `target1` and `target2` in para
 
 #### Arguments
 
-Commands can be provided with runtime positional arguments by providing the `--arg (-a)`, `--arg-map (-m)`, and `--arg-map-file (-f)` switches to `monorail run`. For In your command executables, capture these positional arguments as you would in any program in that language. For example:
+Commands can be provided with positional arguments at runtime. There are two mechanism for doing this; the base argmap, and as flags provided to `monorail run`. In your command executables, capture these positional arguments as you would in any program in that language.
+
+##### Base argmap
+The base argmap is an optional file containing argument mappings, which, if provided, is automatically loaded before any other argument mappings provided to `monorail run`. This is useful for specifying static parameterization for commands, especially when a command executable is generic and reused among multiple targets. An example of this useful pattern is shown in https://github.com/pnordahl/monorail-example in the rust crate targets.
+
+For example, here `target1` has a base argmap in the default location:
+
+`target1/monorail/argmap/base.json`
+```json
+{
+  "build": [
+    "--all"
+  ]
+}
+```
+
+You can change both the argmap search path and the base argmap file by specifying them in `Monorail.json`. Refer to the `argmaps` section of `Monorail.reference.js` for more details.
+
+Finally, while base argmaps are often required by commands that are built to expect them, you can disable base argmaps during a `monorail run` with the `--no-base-argmaps` switch.
+
+##### Runtime flags
+
+In addition to the base argmap, `monorail run` accepts `--arg (-a)`, `--target-argmap (-m)`, and `--target-argmap-file (-f)` flags for providing additional argument mappings. 
 
 ```sh
 monorail run -c build -t target1 --arg '--release' --arg '-v'
 ```
 
-This will provide `--release` and `-v` as the first and second positional arguments to the `build` command for `target1`.
+This will first provide `-all` from the base argmap as the first positional argument, and then `--release` and `-v` as the second and third positional arguments to the `build` command for `target1`.
 
-Note that when using `--arg`, you must specify exactly one command and target. For more flexibility, use `--arg-map` and/or `--arg-map-file`, which allow for specifying argument arrays for specific command-target combinations. For example:
+Note that when using `--arg`, you must specify exactly one command and target. For more flexibility, use `--target-argmap` and/or `--target-argmap-file`, which allow for specifying argument arrays for specific command-target combinations. For example:
 
 ```sh
-monorail run -c build test --arg-map '{"target1":{"build":["--release"],"test":["--release"]}}'
+monorail run -c build test --target-argmap '{"target1":{"build":["--release"],"test":["--release"]}}'
 ```
 
-This will provide the specified arguments to the appropriate command-target combinations. Multiple `--arg-map` flags may be provided, and if so keys that appear in both have their arrays appended to each other in the order specified. For example:
+This will provide the specified arguments to the appropriate command-target combinations. Multiple `--target-argmap` flags may be provided, and if so keys that appear in both have their arrays appended to each other in the order specified. For example:
 
 ```sh
-monorail run -c build test --arg-map '{"target1":{"build":["--release"],"test":["--release"]}}' --arg-map '{"target1":{"build":["-v"]}}'
+monorail run -c build test --target-argmap '{"target1":{"build":["--release"],"test":["--release"]}}' --target-argmap '{"target1":{"build":["-v"]}}'
 ```
 
 In this case, `build` appears twice for `target1` so the arrays are combined in order, and the final arguments array for `target1` is `[--release, -v]`.
 
-Additionally, you can provide the `--arg-map-file` switch zero or more times with a filesystem path to a JSON file containing an argument mapping. The structure of this file is identical to the one described above for `--arg-map`. As with that switch, files provided are merged from left to right. For example
+Additionally, you can provide the `--target-argmap-file` flag zero or more times with a filesystem path to a JSON file containing an argument mapping. The structure of this file is identical to the one described above for `--target-argmap`. As with that flag, files provided are merged from left to right. For example
 
 ```sh
-monorail run -c build -f target1/argmap1.json -f target1/argmap2.json
+monorail run -c build -f target1/monorail/argmap/foo.json -f target1/monorail/argmap/bar.json
 ```
 
 Would merge the keys and values from each file in turn in order.
 
-
 #### Sequences
 
 A sequence is an array of commands, and is specified in `Monorail.json`. It's useful for bundling together related commands into a convenient alias, and when used with `run` simply expands into the list of commands it references. For example:

TUTORIAL.md

@@ -103,7 +103,7 @@ monorail checkpoint update
 These commands will make a rust workspace with two member projects with tests:
 
 ```sh
-mkdir -p rust/monorail
+mkdir -p rust/monorail/cmd
 pushd rust
 cargo init --lib app1
 cargo init --lib app2
@@ -121,7 +121,7 @@ popd
 ### Python
 These commands will make a simple python project with a virtualenv and a test:
 ```sh
-mkdir -p python/app3/monorail
+mkdir -p python/app3/monorail/cmd
 pushd python/app3
 
 python3 -m venv "venv"
@@ -153,7 +153,7 @@ popd
 ```
 ### Protobuf
 ```sh
-mkdir -p proto/monorail
+mkdir -p proto/monorail/cmd
 pushd proto
 touch README.md
 popd
@@ -164,13 +164,13 @@ popd
 Commands will be covered in more depth later in the tutorial (along with logging), but now that we have a valid `Monorail.json` we can execute a command and view logs right away. Run the following to create an executable (in this case, a `bash` script) for the `rust` target:
 
 ```sh
-cat <<EOF > rust/monorail/hello.sh
+cat <<EOF > rust/monorail/cmd/hello.sh
 #!/bin/bash
 
 echo 'Hello, world!'
 echo 'An error message' >&2 
 EOF
-chmod +x rust/monorail/hello.sh
+chmod +x rust/monorail/cmd/hello.sh
 ```
 
 Now execute it:
@@ -312,7 +312,7 @@ monorail analyze --changes | jq
       "path": "rust/app2/src/lib.rs"
     },
     {
-      "path": "rust/monorail/hello.sh",
+      "path": "rust/monorail/cmd/hello.sh",
     }
   ],
   "targets": [
@@ -504,7 +504,7 @@ monorail log tail --stderr --stdout
 This has started a server that will receive logs. For the rest of this tutorial, leave that running in a separate window. Now, let's update our existing command to demonstrate tailing:
 
 ```sh
-cat <<EOF > rust/monorail/hello.sh
+cat <<EOF > rust/monorail/cmd/hello.sh
 #!/bin/bash
 
 for ((i=0; i<20; i++)); do
@@ -604,30 +604,30 @@ cargo test -- --nocapture
 
 ### Defining a command
 
-By default, `monorail` will use the `commands_path` (default: a `monorail` directory in the target path) field of a target as a search path for commands, and by default look for a file with a stem of `{{command}}`, e.g. `{{command}}.sh`. The command we defined earlier in the tutorial, `rust/monorail/hello.sh`, used these defaults; While customizing these defaults is possible via `Monorail.json`, it's not necessary for this tutorial. Let's define two new executables, this time in Python and Awk:
+By default, `monorail` will use the `commands_path` (default: a `monorail` directory in the target path) field of a target as a search path for commands, and by default look for a file with a stem of `{{command}}`, e.g. `{{command}}.sh`. The command we defined earlier in the tutorial, `rust/monorail/cmd/hello.sh`, used these defaults; While customizing these defaults is possible via `Monorail.json`, it's not necessary for this tutorial. Let's define two new executables, this time in Python and Awk:
 
 ```sh
-cat <<EOF > python/app3/monorail/hello.py
+cat <<EOF > python/app3/monorail/cmd/hello.py
 #!venv/bin/python3
 import sys
 
 print("Hello, from python/app3 and virtualenv python!")
 print("An error occurred", file=sys.stderr)
 EOF
-chmod +x python/app3/monorail/hello.py
+chmod +x python/app3/monorail/cmd/hello.py
 ```
 
 NOTE: While we're able to use the venv python3 executable directly in this trivial way, we wouldn't be able to access things that are installed in the environment without first activating the venv in a shell. So, in practice Python projects generally require a command written in a shell script that activates the env and then calls the script.
 
 ```sh
-cat <<EOF > proto/monorail/hello.awk
+cat <<EOF > proto/monorail/cmd/hello.awk
 #!/usr/bin/awk -f
 
 BEGIN {
     print "Hello, from proto and awk!"
 }
 EOF
-chmod +x proto/monorail/hello.awk
+chmod +x proto/monorail/cmd/hello.awk
 ```
 
 As mentioned earlier, commands can be written in any language, and need only be executable. We're using hashbangs to avoid cluttering the tutorial with compilation steps, but commands could be compiled to machine code, stored as something like `hello`, and executed just the same (though this approach wouldn't be portable across OS/architectures). Before we run this command, let's look at the output of analyze:
@@ -774,7 +774,7 @@ monorail target show --commands | jq
       "commands": {
         "hello": {
           "name": "hello",
-          "path": "/private/tmp/monorail-tutorial/rust/monorail/hello.sh",
+          "path": "/private/tmp/monorail-tutorial/rust/monorail/cmd/hello.sh",
           "args": null,
           "is_executable": true
         }
@@ -788,7 +788,7 @@ monorail target show --commands | jq
       "commands": {
         "hello": {
           "name": "hello",
-          "path": "/private/tmp/monorail-tutorial/python/app3/monorail/hello.py",
+          "path": "/private/tmp/monorail-tutorial/python/app3/monorail/cmd/hello.py",
           "args": null,
           "is_executable": true
         }
@@ -802,7 +802,7 @@ monorail target show --commands | jq
       "commands": {
         "hello": {
           "name": "hello",
-          "path": "/private/tmp/monorail-tutorial/proto/monorail/hello.awk",
+          "path": "/private/tmp/monorail-tutorial/proto/monorail/cmd/hello.awk",
           "args": null,
           "is_executable": true
         }
@@ -847,19 +847,19 @@ monorail checkpoint update --pending | jq
   "checkpoint": {
     "id": "4b5c5a4ce18a05b0175c1db6a14fb69bf1ca30d3",
     "pending": {
-      "python/app3/monorail/hello.py": "fad357d1f5adadb0e270dfcf1029c6ed76e2565e62c811613eb315de10143ceb",
+      "python/app3/monorail/cmd/hello.py": "fad357d1f5adadb0e270dfcf1029c6ed76e2565e62c811613eb315de10143ceb",
       "rust/app1/Cargo.toml": "044de847669ad2d9681ba25c4c71e584b5f12d836b9a49e71b4c8d68119e5592",
       "proto/LICENSE.md": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
       "Monorail.json": "b02cc0db02ef8c35ba8c285336748810c590950c263b5555f1781ac80f49a6da",
       "proto/README.md": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
       "rust/app2/Cargo.toml": "111f4cf0fd1b6ce6f690a5f938599be41963905db7d1169ec02684b00494e383",
       ".gitignore": "c1cf4f9ff4b1419420b8508426051e8925e2888b94b0d830e27b9071989e8e7d",
-      "rust/monorail/hello.sh": "c1b9355995507cd3e90727bc79a0d6716b3f921a29b003f9d7834882218e2020",
+      "rust/monorail/cmd/hello.sh": "c1b9355995507cd3e90727bc79a0d6716b3f921a29b003f9d7834882218e2020",
       "python/app3/hello.py": "3639634f2916441a55e4b9be3497673f110014d0ce3b241c93a9794ffcf2c910",
       "python/app3/tests/test_hello.py": "72b3668ed95f4f246150f5f618e71f6cdbd397af785cd6f1137ee87524566948",
       "rust/Cargo.toml": "a35f77bcdb163b0880db4c5efeb666f96496bcb409b4cd52ba6df517fb4d625b",
       "rust/app2/src/lib.rs": "536215b9277326854bd1c31401224ddf8f2d7758065c9076182b37621ad68bd9",
-      "proto/monorail/hello.awk": "5af404fedc153710aec00c8bf788d8f71b00c733c506d4c28fda1b7d618e4af6",
+      "proto/monorail/cmd/hello.awk": "5af404fedc153710aec00c8bf788d8f71b00c733c506d4c28fda1b7d618e4af6",
       "rust/app1/src/lib.rs": "536215b9277326854bd1c31401224ddf8f2d7758065c9076182b37621ad68bd9"
     }
   }

src/api/cli.rs

@@ -43,9 +43,10 @@ pub const ARG_STDOUT: &str = "stdout";
 pub const ARG_ID: &str = "id";
 pub const ARG_DEPS: &str = "deps";
 pub const ARG_ARG: &str = "arg";
-pub const ARG_ARG_MAP: &str = "arg-map";
-pub const ARG_ARG_MAP_FILE: &str = "arg-map-file";
+pub const ARG_ARG_MAP: &str = "argmap";
+pub const ARG_ARG_MAP_FILE: &str = "argmap-file";
 pub const ARG_FAIL_ON_UNDEFINED: &str = "fail-on-undefined";
+pub const ARG_NO_BASE_ARGMAPS: &str = "no-base-argmaps";
 
 pub const VAL_JSON: &str = "json";
 
@@ -213,11 +214,10 @@ pub fn build() -> clap::Command {
     .subcommand(Command::new(CMD_RUN)
         .about("Run target-defined commands.")
         .after_help(r#"
-When --arg-map-file, --arg-map, and/or --arg are provided, keys that appear multiple 
-times will have their respective arrays concatenated in the following order:
+When --target-argmap-file, --target-argmap, and/or --arg are provided, keys that appear multiple times will have their respective arrays concatenated in the following order, after any base argmaps for targets involved in the run:
 
-    1. Each --arg-map-file, in the order provided
-    2. Each --arg-map literal, in the order provided
+    1. Each --target-argmap-file, in the order provided
+    2. Each --target-argmap literal, in the order provided
     3. Each --arg, in the order provided
 
 Refer to --help for more information on these options.
@@ -274,6 +274,12 @@ Refer to --help for more information on these options.
                 .long_help("Fail commands that are undefined by targets.")
                 .action(ArgAction::SetTrue),
         )
+        .arg(
+            Arg::new(ARG_NO_BASE_ARGMAPS)
+                .long(ARG_NO_BASE_ARGMAPS)
+                .long_help("Disable loading base argmaps for run targets. By default, all base argmaps are loaded.")
+                .action(ArgAction::SetTrue),
+        )
         .arg(
             Arg::new(ARG_ARG)
                 .short('a')
@@ -282,7 +288,7 @@ Refer to --help for more information on these options.
                 .required(false)
                 .action(ArgAction::Append)
                 .help("One or more runtime argument(s) to be provided when executing a command for a single target.")
-                .long_help("This is a shorthand form of the more expressive '--arg-map' and '--arg-map-file', designed for single command + single target use. Providing this flag without specifying exactly one command and one target will result in an error.")
+                .long_help("This is a shorthand form of the more expressive '--target-argmap' and '--target-argmap-file', designed for single command + single target use. Providing this flag without specifying exactly one command and one target will result in an error.")
         )
         .arg(
             Arg::new(ARG_ARG_MAP)
@@ -771,6 +777,7 @@ impl<'a> TryFrom<&'a clap::ArgMatches> for app::run::HandleRunInput<'a> {
                 .collect(),
             include_deps: cmd.get_flag(ARG_DEPS),
             fail_on_undefined: cmd.get_flag(ARG_FAIL_ON_UNDEFINED),
+            use_base_argmaps: !cmd.get_flag(ARG_NO_BASE_ARGMAPS),
         })
     }
 }