Skip to content

Commit

Permalink
Straighten the logic of action_load()
Browse files Browse the repository at this point in the history
  • Loading branch information
dimikot committed Nov 30, 2024
1 parent 5c73ad8 commit 29baf4f
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 46 deletions.
67 changes: 34 additions & 33 deletions ci-storage
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ def main():
# enough" slot from the storage:
# - If we are loading a layer, then we try to use the slot id which "best
# matches" the full (non-layer) snapshot loaded in the past.
# - If we are loading a full snapshot,
# - If we are loading a full snapshot, then just use the most recent slot in the
# storage.
#
def action_load(
*,
Expand All @@ -197,65 +198,65 @@ def action_load(
layer: list[str],
verbose: bool,
):
full_snapshot_history = None
if layer:
os.makedirs(local_dir, exist_ok=True)
meta = SlotMeta.read_from(local_dir=local_dir)
full_snapshot_history = meta.full_snapshot_history
os.makedirs(local_dir, exist_ok=True)
meta = SlotMeta.read_from(local_dir=local_dir)
full_snapshot_history = meta.full_snapshot_history

slot_infos = list_slots(
storage_host=storage_host,
storage_dir=storage_dir,
storage_max_age_sec=storage_max_age_sec,
)

in_storage = "layer storage" if layer else "storage"
slot_id = ""
for id in map(normalize_slot_id, slot_ids):
prefix = f'Checking slot-id="{id}"...'
if id == "*":
if not slot_infos:
print(f"{prefix} storage has no slots, so exiting with a no-op")
print(f"{prefix} {in_storage} has no slots, so exiting with a no-op")
return
elif full_snapshot_history is None:
elif not layer:
slot_id = list(slot_infos.keys())[0]
print(
f'{prefix} using the most recent slot-id="{slot_id}" for the full (non-layer) load'
f'{prefix} loading the most recent full (non-layer) slot-id="{slot_id}"'
)
break
elif len(full_snapshot_history) > 0:
print(
f"{prefix} prioritizing slots from past full snapshot loading history..."
)
for id in full_snapshot_history:
prefix = f'Checking slot-id="{id}" from history...'
if id in slot_infos:
slot_id = id
print(f"{prefix} found in the storage, using it")
break
else:
print(f"{prefix} not found in the storage")
if not slot_id:
elif layer:
if full_snapshot_history:
print(
f"{prefix} prioritizing layer slots mentioned in the past full snapshot loading history..."
)
for id in full_snapshot_history:
prefix = f'Checking slot-id="{id}" from history...'
if id in slot_infos:
slot_id = id
print(f"{prefix} found in the {in_storage}, using it")
break
else:
print(f"{prefix} not found in the {in_storage}")
if not slot_id:
slot_id = list(slot_infos.keys())[0]
print(
f'No slots from past full snapshot loading history were found in the {in_storage}, so using just the most recent slot-id="{slot_id}"'
)
break
else:
slot_id = list(slot_infos.keys())[0]
print(
f'None slots from past full snapshot loading history were found in the storage, so using just the most recent slot-id="{slot_id}"'
f'{prefix} no past loading history, so using just the most recent layer slot-id="{slot_id}"'
)
break
else:
slot_id = list(slot_infos.keys())[0]
print(
f'{prefix} no past loading history, so using just the most recent slot-id="{slot_id}"'
)
break
break
elif id in slot_infos:
slot_id = id
print(f"{prefix} found in the storage, using it")
print(f"{prefix} found in the {in_storage}, using it")
break
else:
print(f"{prefix} not found in the storage")
print(f"{prefix} not found in the {in_storage}")

if not slot_id:
raise UserException(
f"none of the provided slot id(s) were found in the storage, aborting"
f"none of the provided slot id(s) were found in the {in_storage}, aborting"
)

host, port = parse_host_port(storage_host)
Expand Down
2 changes: 1 addition & 1 deletion tests/0015-load-then-store-dedups-with-loaded-slot.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ test "$(hardlink-count "$STORAGE_DIR/myslot3-late/file")" == 2
# Just in case, load from myslot3-late, and since it has been dedupped with
# myslot1-early, they should all dedup between each other.
ci-storage --slot-id="*" load
grep -qF 'Checking slot-id="*"... using the most recent slot-id="myslot3-late"' "$OUT"
grep -qF 'Checking slot-id="*"... loading the most recent full (non-layer) slot-id="myslot3-late"' "$OUT"

ci-storage --slot-id="myslot4-end" store
grep -qF -- '--link-dest=../myslot3-late/' "$OUT"
Expand Down
2 changes: 1 addition & 1 deletion tests/0020-load-absent-causes-error-1.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ ci-storage \
load || error=$?

test "$error" == 1
grep -qF 'Checking slot-id="absent"... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="absent"... not found' "$OUT"
grep -qF 'Error: none of the provided slot id(s) were found in the storage, aborting' "$OUT"
6 changes: 3 additions & 3 deletions tests/0020-load-absent-causes-error-2.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ ci-storage \
load || error=$?

test "$error" == 1
grep -qF 'Checking slot-id="absent1"... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="absent2"... not found in the storage' "$OUT"
grep -qF 'Error: none of the provided slot id(s) were found in the storage, aborting' "$OUT"
grep -qF 'Checking slot-id="absent1"... not found' "$OUT"
grep -qF 'Checking slot-id="absent2"... not found' "$OUT"
grep -qF 'Error: none of the provided slot id(s) were found' "$OUT"
2 changes: 1 addition & 1 deletion tests/0040-load-star-with-slots-in-storage.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ ci-storage \
load || error=$?

test "$error" == 0
grep -qF 'Checking slot-id="*"... using the most recent slot-id="myslot" for the full (non-layer) load' "$OUT"
grep -qF 'Checking slot-id="*"... loading the most recent full (non-layer) slot-id="myslot"' "$OUT"
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ ci-storage \
load || error=$?

test "$error" == 0
grep -qF 'Checking slot-id="absent1"... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="absent2"... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="*"... using the most recent slot-id="myslot" for the full (non-layer) load' "$OUT"
grep -qF 'Checking slot-id="absent1"... not found' "$OUT"
grep -qF 'Checking slot-id="absent2"... not found' "$OUT"
grep -qF 'Checking slot-id="*"... loading the most recent full (non-layer) slot-id="myslot"' "$OUT"
8 changes: 4 additions & 4 deletions tests/0110-load-layer-from-past-history.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ ci-storage \
load

test -f "$LOCAL_DIR/file-in-layer-1"
grep -qF 'Checking slot-id="slot_without_layer_2"... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="*"... prioritizing slots from past full snapshot loading history...' "$OUT"
grep -qF 'Checking slot-id="slot_without_layer_2" from history... not found in the storage' "$OUT"
grep -qF 'Checking slot-id="slot_with_layer_1" from history... found in the storage, using it' "$OUT"
grep -qF 'Checking slot-id="slot_without_layer_2"... not found' "$OUT"
grep -qF 'Checking slot-id="*"... prioritizing' "$OUT"
grep -qF 'Checking slot-id="slot_without_layer_2" from history... not found' "$OUT"
grep -qF 'Checking slot-id="slot_with_layer_1" from history... found' "$OUT"

ci-storage \
--slot-id=slot_with_layer_1 \
Expand Down

0 comments on commit 29baf4f

Please sign in to comment.