Skip to content

Commit

Permalink
Consider slots which will be garbage collected soon as non-existing (#32
Browse files Browse the repository at this point in the history
)

## PRs in the Stack
- ➡ #32

(The stack is managed by
[git-grok](https://github.com/dimikot/git-grok).)
  • Loading branch information
dimikot authored Nov 22, 2024
1 parent d52d36c commit 5c73ad8
Showing 1 changed file with 14 additions and 9 deletions.
23 changes: 14 additions & 9 deletions ci-storage
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def main():
action_load(
storage_host=storage_host,
storage_dir=storage_dir,
storage_max_age_sec=storage_max_age_sec,
slot_ids=slot_ids,
local_dir=local_dir,
exclude=exclude,
Expand All @@ -189,6 +190,7 @@ def action_load(
*,
storage_host: str | None,
storage_dir: str,
storage_max_age_sec: int,
slot_ids: list[str],
local_dir: str,
exclude: list[str],
Expand All @@ -204,6 +206,7 @@ def action_load(
slot_infos = list_slots(
storage_host=storage_host,
storage_dir=storage_dir,
storage_max_age_sec=storage_max_age_sec,
)

slot_id = ""
Expand Down Expand Up @@ -308,14 +311,11 @@ def action_store(
slot_infos = list_slots(
storage_host=storage_host,
storage_dir=storage_dir,
storage_max_age_sec=storage_max_age_sec,
)

slot_recent = None
if (
slot_id_we_used_to_load_from in slot_infos
and slot_infos[slot_id_we_used_to_load_from].age_sec
< storage_max_age_sec - STORAGE_MAX_AGE_SEC_BAK
):
if slot_id_we_used_to_load_from in slot_infos:
slot_recent = slot_infos[slot_id_we_used_to_load_from]
elif slot_infos:
slot_recent = list(slot_infos.values())[0]
Expand Down Expand Up @@ -385,12 +385,16 @@ def action_maintenance(
# Returns the list of existing slot ids and their ages in seconds, sorted by age
# (i.e. most recently created slots on top of the list). Also, as a side effect,
# touches the newest slot directory on the server (assuming it'll be accessed),
# so it will unlikely be garbage collected anytime soon.
# so it will unlikely be garbage collected anytime soon. Only the slots which
# are not about to be garbage collected within the next STORAGE_MAX_AGE_SEC_BAK
# will be returned (since returning the older ones is dangerous: they may start
# being deleted right at the moment when we load or store).
#
def list_slots(
*,
storage_host: str | None,
storage_dir: str,
storage_max_age_sec: int,
) -> collections.OrderedDict[str, SlotInfo]:
slot_infos = collections.OrderedDict[str, SlotInfo]()
lines = check_output_script(
Expand All @@ -401,14 +405,15 @@ def list_slots(
for line in lines.splitlines():
match = re.match(r"^(\S+) (\d+) (.*)$", line)
if match:
slot_id = match.group(1)
slot_infos[slot_id] = SlotInfo(
id=slot_id,
slot_info = SlotInfo(
id=match.group(1),
age_sec=int(match.group(2)),
meta=SlotMeta.deserialize(
match.group(3).encode().decode("unicode_escape")
),
)
if slot_info.age_sec < storage_max_age_sec - STORAGE_MAX_AGE_SEC_BAK:
slot_infos[slot_info.id] = slot_info
return slot_infos


Expand Down

0 comments on commit 5c73ad8

Please sign in to comment.