Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update for allocation 2024-2025 and add support for RGU #118

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update for allocation 2024-2025 and add support for RGU #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 30 additions & 20 deletions data/allocations.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,55 @@
start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size
start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, rgu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size
2017-04-01, 2018-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 100,
2017-04-01, 2018-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, 15TB,
2017-04-01, 2018-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, 15TB,

2018-04-01, 2019-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 100,
2018-04-01, 2019-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 70TB, , ,
2018-04-01, 2019-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 70TB, , ,

2019-04-01, 2020-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 190,
2019-04-01, 2020-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 90TB, 5e6, 90TB,
2019-04-01, 2020-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 90TB, 5e6, 90TB,

2020-04-01, 2021-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 130,
2020-04-01, 2021-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 30TB, 2e6, ,
2020-04-01, 2021-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 30TB, 2e6, ,
2020-04-01, 2021-04-01, beluga, beluga-compute, rrg-bengioy-ad, 219,
2020-04-01, 2021-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 200,
2020-04-01, 2021-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 70TB, 5e6, 80TB,
2020-04-01, 2021-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 70TB, 5e6, 80TB,

2021-04-01, 2022-04-01, graham, graham-gpu, rrg-bengioy-ad, , 60,
2021-04-01, 2022-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 40,
2021-04-01, 2022-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 30TB, 2e6, ,
2021-04-01, 2022-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 30TB, 2e6, ,
2021-04-01, 2022-04-01, beluga, beluga-compute, rrg-bengioy-ad, 185,
2021-04-01, 2022-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 150,
2021-04-01, 2022-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 70TB, 5e6, 80TB,
2021-04-01, 2022-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 70TB, 5e6, 80TB,
2021-04-01, 2022-04-01, narval, narval-compute, rrg-bengioy-ad, 185,
2021-04-01, 2022-04-01, narval, narval-gpu, rrg-bengioy-ad, , 75,

2022-04-01, 2023-04-01, graham, graham-compute, rrg-bengioy-ad, 34,
2022-04-01, 2023-04-01, cedar, cedar-compute, rrg-bengioy-ad, 34,
2022-04-01, 2023-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 138,
2022-04-01, 2023-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, ,
2022-04-01, 2023-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, ,
2022-04-01, 2023-04-01, beluga, beluga-compute, rrg-bengioy-ad, 238,
2022-04-01, 2023-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 77,
2022-04-01, 2023-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, 25TB,
2022-04-01, 2023-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, 25TB,
2022-04-01, 2023-04-01, narval, narval-compute, rrg-bengioy-ad, 34,
2022-04-01, 2023-04-01, narval, narval-gpu, rrg-bengioy-ad, , 185,
2022-04-01, 2023-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 25TB,
2022-04-01, 2023-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB,

2023-04-01, 2024-04-01, cedar, cedar-compute, rrg-bengioy-ad, 197,
2023-04-01, 2024-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 127,
2023-04-01, 2024-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 25TB,
2023-04-01, 2024-04-01, beluga, beluga-compute, rrg-bengioy-ad, 197,
2023-04-01, 2024-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 127,
2023-04-01, 2024-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 125TB,
2023-04-01, 2024-04-01, narval, narval-compute, rrg-bengioy-ad, 917,
2023-04-01, 2024-04-01, narval, narval-gpu, rrg-bengioy-ad, , 154,
2023-04-01, 2024-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , 150TB, 5e6, 75TB,
2023-04-01, 2024-04-03, cedar, cedar-compute, rrg-bengioy-ad, 197,
2023-04-01, 2024-04-03, cedar, cedar-gpu, rrg-bengioy-ad, , 127,
2023-04-01, 2024-04-03, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB,
2023-04-01, 2024-04-03, beluga, beluga-compute, rrg-bengioy-ad, 197,
2023-04-01, 2024-04-03, beluga, beluga-gpu, rrg-bengioy-ad, , 127,
2023-04-01, 2024-04-03, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 200TB, 5e6, 125TB,
2023-04-01, 2024-04-03, narval, narval-compute, rrg-bengioy-ad, 917,
2023-04-01, 2024-04-03, narval, narval-gpu, rrg-bengioy-ad, , 154,
2023-04-01, 2024-04-03, narval, narval-storage, rrg-bengioy-ad, , , , , , 150TB, 5e6, 75TB,

2024-04-03, 2025-04-03, cedar, cedar-compute, rrg-bengioy-ad, 125,
2024-04-03, 2025-04-03, cedar, cedar-gpu, rrg-bengioy-ad, , 118, 305,
2024-04-03, 2025-04-03, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB,
2024-04-03, 2025-04-03, beluga, beluga-compute, rrg-bengioy-ad, 125,
2024-04-03, 2025-04-03, beluga, beluga-gpu, rrg-bengioy-ad, , 111, 250,
2024-04-03, 2025-04-03, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 300TB, 5e6, 125TB,
2024-04-03, 2025-04-03, narval, narval-compute, rrg-bengioy-ad, 580,
2024-04-03, 2025-04-03, narval, narval-gpu, rrg-bengioy-ad, , 110, 440,
2024-04-03, 2025-04-03, narval, narval-storage, rrg-bengioy-ad, , , , , , 150TB, 5e6, 75TB,
5 changes: 4 additions & 1 deletion sarc/allocations/allocations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@
from pydantic import ByteSize, validator
from pydantic_mongo import AbstractRepository, ObjectIdField

from sarc.traces import trace_decorator
from sarc.config import BaseModel, config, validate_date


class AllocationCompute(BaseModel):
gpu_year: Optional[int] = 0
cpu_year: Optional[int] = 0
rgu_year: Optional[int] = 0
vcpu_year: Optional[int] = 0
vgpu_year: Optional[int] = 0

Expand Down Expand Up @@ -61,6 +63,7 @@ class AllocationsRepository(AbstractRepository[Allocation]):
class Meta:
collection_name = "allocations"

@trace_decorator()
def add(self, allocation: Allocation):
document = self.to_document(allocation)
query_attrs = ["cluster_name", "resource_name", "group_name", "start", "end"]
Expand Down Expand Up @@ -120,7 +123,7 @@ def allocation_key(allocation: Allocation):
for allocation in allocations:
key = allocation_key(allocation)
if key in summaries:
for field in ["cpu_year", "gpu_year", "vcpu_year", "vgpu_year"]:
for field in ["cpu_year", "gpu_year", "rgu_year", "vcpu_year", "vgpu_year"]:
setattr(
summaries[key].resources.compute,
field,
Expand Down
17 changes: 14 additions & 3 deletions sarc/cli/acquire/allocations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
from __future__ import annotations

import logging
import csv
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path

from sarc.traces import trace_decorator

from sarc.allocations.allocations import (
Allocation,
AllocationCompute,
Expand All @@ -13,7 +16,10 @@
get_allocations_collection,
)

logger = logging.getLogger(__name__)


@trace_decorator()
def convert_csv_row_to_allocation(
cluster_name: str,
resource_name: str,
Expand All @@ -22,6 +28,7 @@ def convert_csv_row_to_allocation(
end: datetime,
cpu_year: None | int = None,
gpu_year: None | int = None,
rgu_year: None | int = None,
vcpu_year: None | int = None,
vgpu_year: None | int = None,
project_size: None | str = None,
Expand All @@ -38,6 +45,7 @@ def convert_csv_row_to_allocation(
resources=AllocationRessources(
compute=AllocationCompute(
gpu_year=gpu_year,
rgu_year=rgu_year,
cpu_year=cpu_year,
vcpu_year=vcpu_year,
vgpu_year=vgpu_year,
Expand All @@ -58,6 +66,8 @@ def convert_csv_row_to_allocation(
@dataclass
class AcquireAllocations:
file: Path
# Do not actually insert the data into the database
dry: bool

def execute(self) -> int:
collection = get_allocations_collection()
Expand All @@ -76,10 +86,11 @@ def execute(self) -> int:
try:
allocation = convert_csv_row_to_allocation(**row)
except Exception as e: # pylint: disable=broad-exception-caught
print(f"Skipping row: {row}")
print(e)
logger.warning(f"Skipping row: {row}")
continue

collection.add(allocation)
logger.info(f"Adding allocation: {allocation}")
if not self.dry:
collection.add(allocation)

return 0
22 changes: 11 additions & 11 deletions tests/functional/allocations/allocations.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size
2017-04-01, 2018-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100,
2017-04-01, 2018-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 50TB, 5e6, 15TB,
start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, rgu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size
2017-04-01, 2018-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, 300,
2017-04-01, 2018-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 50TB, 5e6, 15TB,

2018-04-01, 2019-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100,
2018-04-01, 2019-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 70TB, , ,
2018-04-01, 2019-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, 300,
2018-04-01, 2019-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 70TB, , ,

2019-04-01, 2020-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 190,
2019-04-01, 2020-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , 90TB, 5e6, 90TB,
2019-04-01, 2020-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 190, 400,
2019-04-01, 2020-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , , 90TB, 5e6, 90TB,

2020-04-01, 2021-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 130,
2020-04-01, 2021-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 30TB, 5e6, ,
2020-04-01, 2021-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 130, 350,
2020-04-01, 2021-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 30TB, 5e6, ,
2020-04-01, 2021-04-01, patate, patate-compute, rrg-bonhomme-ad, 219,
2020-04-01, 2021-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 200,
2020-04-01, 2021-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , 70TB, 5e6, 80TB,
2020-04-01, 2021-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 200, 450,
2020-04-01, 2021-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , , 70TB, 5e6, 80TB,
11 changes: 11 additions & 0 deletions tests/functional/allocations/factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": 100,
"rgu_year": 400,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -37,6 +38,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -58,6 +60,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": 100,
"rpu_year": 400,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -79,6 +82,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -100,6 +104,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": 190,
"rgu_year": 190,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -121,6 +126,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -142,6 +148,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": 130,
"rgu_year": 450,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -163,6 +170,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -184,6 +192,7 @@ def create_allocations():
"compute": {
"cpu_year": 219,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -205,6 +214,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": 200,
"rgu_year": 500,
"vcpu_year": None,
"vgpu_year": None,
},
Expand All @@ -226,6 +236,7 @@ def create_allocations():
"compute": {
"cpu_year": None,
"gpu_year": None,
"rgu_year": None,
"vcpu_year": None,
"vgpu_year": None,
},
Expand Down
Loading