From e8385b7ccc4e774dbd60937c15aa6b7ab814acc6 Mon Sep 17 00:00:00 2001 From: Dimi Kot Date: Thu, 12 Dec 2024 21:39:19 -0800 Subject: [PATCH] Add hint input to action.yml (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## PRs in the Stack - ➡ #33 (The stack is managed by [git-grok](https://github.com/dimikot/git-grok).) --- .github/workflows/ci.yml | 5 ++++ README.md | 13 +++++++-- action.yml | 5 ++++ ci-storage | 27 +++++++++++-------- ...-local-meta-and-uses-them-at-store.test.sh | 1 + ...nts-with-patterns-expand-to-digest.test.sh | 12 +++++++++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e43cf00..a426c30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,10 @@ jobs: action: store storage-dir: ~/storage-dir sudo: false + hint: | + aaa + @dummy.txt + bbb run-before: | set -ex echo "run-before" > run-before.txt @@ -68,6 +72,7 @@ jobs: with: action: load storage-dir: ~/storage-dir + hint: aaa - name: Check that dummy.txt and run-before.txt were restored run: | set -e diff --git a/README.md b/README.md index de016fb..2ae0094 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ ones, so rsync can run efficiently. # Required. action: '' - # Storage host in the format [user@]host[:port]; it must allow password-free + # Storage host in the format [user@]host[:port]. It must allow password-free # SSH key based access. # Default: the content of ~/ci-storage-host file. storage-host: '' @@ -64,7 +64,7 @@ ones, so rsync can run efficiently. # Default: 14400 (4 hours). storage-max-age-sec: '' - # Id of the slot to store to or load from; use "*" to load a smart-random + # Id of the slot to store to or load from. Use "*" to load a smart-random # slot (e.g. most recent or best in terms of layer compatibility) and skip # if it does not exist. # Default: $GITHUB_RUN_ID (which is friendly to "Re-run failed jobs"). @@ -78,6 +78,15 @@ ones, so rsync can run efficiently. # Default: "." (current work directory). local-dir: '' + # Optional hints of the CI run to let slot-id="*" specifier find the best + # slot in the storage to load from. The leftmost matching hints have higher + # priority. If a line in multi-line hint starts with "@", then it expands + # to a digest of the content of all files matching the space-separated list + # of patterns on the same line after the "@". On "store" action, if --hint + # is not provided, the hints are derived from the previous "load" action. + # Default: empty. + hint: '' + # Newline separated exclude pattern(s) for rsync. # Default: empty. exclude: '' diff --git a/action.yml b/action.yml index c8aa44e..fbd6c66 100644 --- a/action.yml +++ b/action.yml @@ -22,6 +22,9 @@ inputs: local-dir: description: 'Local directory path to store from or load to. The value namespaces the data stored, so different local-dir values correspond to different storages. If the owner of the directory is different from the current user, then ci-storage tool is run with sudo, and the binary is used not from the action directory, but from /usr/bin/ci-storage. Default: "." (current work directory).' required: false + hint: + description: 'Optional hints of the CI run to let slot-id="*" specifier find the best slot in the storage to load from. The leftmost matching hints have higher priority. If a line in multi-line hint starts with "@", then it expands to a digest of the content of all files matching the space-separated list of patterns on the same line after the "@". On "store" action, if --hint is not provided, the hints are derived from the previous "load" action. Default: empty.' + required: false exclude: description: "Newline separated exclude pattern(s) for rsync. Default: empty." required: false @@ -57,6 +60,7 @@ runs: storage_max_age_sec="${{ inputs.storage-max-age-sec || '' }}" slot_id="${{ inputs.slot-id }}" local_dir="${{ inputs.local-dir || '.' }}" + hint="${{ inputs.hint || '' }}" exclude="${{ inputs.exclude || '' }}" layer_name="${{ inputs.layer-name || '' }}" layer_include="${{ inputs.layer-include || '' }}" @@ -101,6 +105,7 @@ runs: --storage-max-age-sec="$storage_max_age_sec" --slot-id="$slot_id" --local-dir="$local_dir" + --hint="$hint" --exclude="$exclude" --layer="$layer_include" $verbose diff --git a/ci-storage b/ci-storage index 4b83f43..b1156b0 100755 --- a/ci-storage +++ b/ci-storage @@ -56,27 +56,27 @@ def main(): "action", type=str, choices=["store", "load"], - help="action to run", + help="Action to run.", ) parser.add_argument( "--storage-host", type=str, required=False, - help="storage host in the format [user@]host[:port]; it must allow password-free SSH key based access; if omitted, uses the local filesystem (no SSH)", + help="Storage host in the format [user@]host[:port]. It must allow password-free SSH key based access. If omitted, uses the local filesystem (no SSH).", ) parser.add_argument( "--storage-dir", type=str, default=STORAGE_DIR_DEFAULT, required=False, - help=f"storage directory path on the storage host (will be created if it does not exist)", + help=f"Storage directory path on the storage host (will be created if it does not exist).", ) parser.add_argument( "--storage-max-age-sec", type=str, default=str(STORAGE_MAX_AGE_SEC_DEFAULT), required=False, - help="remove slots created earlier than this many seconds ago", + help="Remove slots created earlier than this many seconds ago.", ) parser.add_argument( "--slot-id", @@ -84,40 +84,40 @@ def main(): required=True, default=[], action="append", - help='id of the slot to store to or load from; use "*" to load a smart-random slot (e.g. most recent or best in terms of layer compatibility) and skip if it does not exist; when loading, you may provide multiple --slot-id options to try loading them in order', + help='Id of the slot to store to or load from. Use "*" to load a smart-random slot (e.g. most recent or best in terms of layer compatibility) and skip if it does not exist. When loading, you may provide multiple --slot-id options to try loading them in order.', ) parser.add_argument( "--local-dir", type=str, required=True, - help="local directory path", + help="Local directory path.", ) parser.add_argument( "--hint", type=str, default=[], action="append", - help='optional hints of the CI run to let slot-id="*" specifier find the best slot in the storage to load from; the leftmost matching hints have higher priority; on "store" action, if --hint is not provided, the hints are derived from the previous "load" action', + help='Optional hints of the CI run to let slot-id="*" specifier find the best slot in the storage to load from. The leftmost matching hints have higher priority. If a line in multi-line hint starts with "@", then it expands to a digest of the content of all files matching the space-separated list of patterns on the same line after the "@". On "store" action, if --hint is not provided, the hints are derived from the previous "load" action.', ) parser.add_argument( "--exclude", type=str, default=[], action="append", - help="exclude pattern(s) for rsync", + help="Exclude pattern(s) for rsync.", ) parser.add_argument( "--layer", type=str, default=[], action="append", - help="include pattern(s) for rsync; if set, only the matching files will be transferred; empty directories will be ignored; deletion will be turned off on load", + help="Include pattern(s) for rsync. If set, only the matching files will be transferred. Empty directories will be ignored. Deletion will be turned off on load.", ) parser.add_argument( "--verbose", default=False, action="store_true", - help="if set, prints the list of transferred files", + help="If set, prints the list of transferred files.", ) args = parser.parse_intermixed_args() @@ -133,7 +133,12 @@ def main(): ) slot_ids: list[str] = " ".join(args.slot_id).split() local_dir: str = re.sub(r"/+$", "", args.local_dir) - hints: list[str] = [hint for arg in args.hint for hint in expand_hint_arg(arg=arg)] + hints: list[str] = [ + hint + for arg in "\n".join(args.hint).splitlines() + if arg.strip() + for hint in expand_hint_arg(arg=arg.strip()) + ] exclude: list[str] = [ line for line in "\n".join(args.exclude).splitlines() if line.strip() ] diff --git a/tests/0035-load-star-saves-hints-to-local-meta-and-uses-them-at-store.test.sh b/tests/0035-load-star-saves-hints-to-local-meta-and-uses-them-at-store.test.sh index a4db269..ebe219f 100755 --- a/tests/0035-load-star-saves-hints-to-local-meta-and-uses-them-at-store.test.sh +++ b/tests/0035-load-star-saves-hints-to-local-meta-and-uses-them-at-store.test.sh @@ -10,6 +10,7 @@ grep -qF "hints=aaa" "$LOCAL_META_FILE" ci-storage \ --slot-id=myslot1 \ + --hint="" \ store grep -qF "hints=aaa" "$STORAGE_DIR/myslot1/.ci-storage.meta" diff --git a/tests/0045-hints-with-patterns-expand-to-digest.test.sh b/tests/0045-hints-with-patterns-expand-to-digest.test.sh index 58e667f..99d6589 100755 --- a/tests/0045-hints-with-patterns-expand-to-digest.test.sh +++ b/tests/0045-hints-with-patterns-expand-to-digest.test.sh @@ -4,6 +4,7 @@ source ./common.sh echo "a" > "$LOCAL_DIR/file-new-a" echo "b" > "$LOCAL_DIR/file-new-b" echo "c" > "$LOCAL_DIR/file-c" + ci-storage \ --slot-id=myslot1 \ --hint="aaa bbb" \ @@ -12,3 +13,14 @@ ci-storage \ store grep -qF "hints=aaa bbb @f3bba3f02fb45a2a ccc" "$STORAGE_DIR/myslot1/.ci-storage.meta" + +ci-storage \ + --slot-id=myslot2 \ + --hint=" + aaa bbb + @$LOCAL_DIR/file-new-* $LOCAL_DIR/file-c $LOCAL_DIR/file-absent? + ccc + " \ + store + +grep -qF "hints=aaa bbb @f3bba3f02fb45a2a ccc" "$STORAGE_DIR/myslot2/.ci-storage.meta"