Skip to content

Commit

Permalink
Merge pull request opensciencegrid#3901 from matyasselmeci/pr/product…
Browse files Browse the repository at this point in the history
…ion-in-namespaces-json.SOFTWARE-5862

Add filters for production and itb in namespaces json (SOFTWARE-5862)
  • Loading branch information
matyasselmeci authored May 17, 2024
2 parents 76200bf + be694b0 commit 47e4d18
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 26 deletions.
9 changes: 6 additions & 3 deletions src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -513,9 +513,12 @@ base_path = /ospool/PROTECTED
### Namespaces JSON generation

The JSON file containing cache and namespace information for stashcp/OSDF is served at `/osdf/namespaces`.
The endpoint takes two optional parameters, `include_downed=1`, and `include_inactive=1`;
if they are set, caches in downtime or that are not marked as active, respectively, are also included in the result.
Otherwise, they are not included.
The endpoint takes some optional parameters for filtering:
- `include_downed=1` includes caches that are in downtime in the result; otherwise they are omitted
- `include_inactive=1` includes caches that are not marked as active in the result; otherwise they are omitted
- `production=1` includes resources in "production" (as opposed to ITB) in the result
- `itb=1` includes resources in "itb" in the result
if neither `production` nor `itb` are specified then both production and itb resources are included

The JSON contains an attribute `caches` that is a list of caches.
Each cache in the list contains the following attributes:
Expand Down
19 changes: 14 additions & 5 deletions src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,11 @@

from webapp import default_config
from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \
escape, cache_control_private, PreJSON, is_true
escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters
from webapp.flask_common import create_accepted_response
from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
from webapp.models import GlobalData
from webapp.topology import GRIDTYPE_1, GRIDTYPE_2
from webapp.oasis_managers import get_oasis_manager_endpoint_info
from webapp.github import create_file_pr, update_file_pr, GithubUser, GitHubAuth, GitHubRepoAPI, GithubRequestException, GithubReferenceExistsException, GithubNotFoundException

Expand Down Expand Up @@ -514,10 +513,20 @@ def scitokens():
def stashcache_namespaces_json():
if not stashcache:
return Response("Can't get scitokens config: stashcache module unavailable", status=503)
include_downed = is_true(request.args.get("include_downed", False))
include_inactive = is_true(request.args.get("include_inactive", False))
args = request.args
filters = NamespacesFilters()
filters.include_downed = is_true(args.get("include_downed", False))
filters.include_inactive = is_true(args.get("include_inactive", False))
if "production" not in args and "itb" not in args:
# default: include both production and itb
filters.production = True
filters.itb = True
else:
filters.production = is_true(request.args.get("production", False))
filters.itb = is_true(request.args.get("itb", False))

try:
return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, include_downed, include_inactive)),
return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, filters=filters)),
mimetype='application/json')
except ResourceNotRegistered as e:
return Response("# {}\n"
Expand Down
23 changes: 17 additions & 6 deletions src/stashcache.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from collections import defaultdict
from typing import Dict, List, Optional

from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER
from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters
from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
from webapp.models import GlobalData
from webapp.topology import Resource, ResourceGroup, Topology
Expand Down Expand Up @@ -538,7 +538,7 @@ def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]:
)


def get_namespaces_info(global_data: GlobalData, include_downed=False, include_inactive=False) -> PreJSON:
def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFilters] = None) -> PreJSON:
"""Return data for the /stashcache/namespaces JSON endpoint.
This includes a list of caches and origins, with some data about their endpoints,
Expand All @@ -547,6 +547,9 @@ def get_namespaces_info(global_data: GlobalData, include_downed=False, include_i
If `include_downed` is True, caches/origins in downtime are also included.
If `include_inactive` is True, caches/origins that are not marked as active are also included.
"""
if filters is None:
filters = NamespacesFilters()

# Helper functions

def _service_resource_dict(
Expand Down Expand Up @@ -645,10 +648,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
cache_resource_dicts = {} # type: Dict[str, Dict]

for group in resource_groups:
if group.production and not filters.production:
continue
if group.itb and not filters.itb:
continue
for resource in group.resources:
if (_resource_has_cache(resource)
and (include_inactive or resource.is_active)
and (include_downed or not _resource_has_downed_cache(resource, topology))
and (filters.include_inactive or resource.is_active)
and (filters.include_downed or not _resource_has_downed_cache(resource, topology))
):
cache_resource_objs[resource.name] = resource
cache_resource_dicts[resource.name] = _cache_resource_dict(resource)
Expand All @@ -659,10 +666,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
origin_resource_dicts = {} # type: Dict[str, Dict]

for group in resource_groups:
if group.production and not filters.production:
continue
if group.itb and not filters.itb:
continue
for resource in group.resources:
if (_resource_has_origin(resource)
and (include_inactive or resource.is_active)
and (include_downed or not _resource_has_downed_origin(resource, topology))
and (filters.include_inactive or resource.is_active)
and (filters.include_downed or not _resource_has_downed_origin(resource, topology))
):
origin_resource_objs[resource.name] = resource
origin_resource_dicts[resource.name] = _origin_resource_dict(resource)
Expand Down
36 changes: 33 additions & 3 deletions src/tests/test_stashcache.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from app import app, global_data
from webapp import models, topology, vos_data
from webapp.common import load_yaml_file
from webapp.common import load_yaml_file, NamespacesFilters
from webapp.data_federation import CredentialGeneration, StashCache
import stashcache

Expand Down Expand Up @@ -265,13 +265,35 @@ def caches(self, namespaces_json) -> List[Dict]:

@pytest.fixture
def caches_include_inactive(self, test_global_data) -> List[Dict]:
namespaces_json = stashcache.get_namespaces_info(test_global_data, include_inactive=True)
filters = NamespacesFilters()
filters.include_inactive = True
namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
assert "caches" in namespaces_json
return namespaces_json["caches"]

@pytest.fixture
def caches_include_downed(self, test_global_data) -> List[Dict]:
namespaces_json = stashcache.get_namespaces_info(test_global_data, include_downed=True)
filters = NamespacesFilters()
filters.include_downed = True
namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
assert "caches" in namespaces_json
return namespaces_json["caches"]

@pytest.fixture
def caches_production(self, test_global_data) -> List[Dict]:
filters = NamespacesFilters()
filters.production = True
filters.itb = False
namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
assert "caches" in namespaces_json
return namespaces_json["caches"]

@pytest.fixture
def caches_itb(self, test_global_data) -> List[Dict]:
filters = NamespacesFilters()
filters.production = False
filters.itb = True
namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
assert "caches" in namespaces_json
return namespaces_json["caches"]

Expand Down Expand Up @@ -401,6 +423,14 @@ def test_caches_include_downed_param(self, caches, caches_include_downed):
x["resource"] for x in caches_include_downed
), "Downed cache missing from namespaces JSON with ?include_downed=1"

def test_caches_production(self, caches_production, caches_itb):
assert "TEST_TIGER_CACHE" in (
x["resource"] for x in caches_production
), "Production cache not present in namespaces JSON with production filter"
assert "TEST_TIGER_CACHE" not in (
x["resource"] for x in caches_itb
), "Production cache wrongly present in namespaces JSON with itb filter"


if __name__ == '__main__':
pytest.main()
13 changes: 13 additions & 0 deletions src/webapp/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,17 @@ def populate_voown_name(self, vo_id_to_name: Dict):
self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id]


class NamespacesFilters:
"""
Filters for namespaces json
"""
def __init__(self):
self.include_inactive = False
self.include_downed = False
self.production = True
self.itb = True


def to_csv(data: list) -> str:
csv_string = StringIO()
writer = csv.writer(csv_string)
Expand Down Expand Up @@ -385,3 +396,5 @@ def wrapped():

XROOTD_CACHE_SERVER = "XRootD cache server"
XROOTD_ORIGIN_SERVER = "XRootD origin server"
GRIDTYPE_1 = "OSG Production Resource"
GRIDTYPE_2 = "OSG Integration Test Bed Resource"
19 changes: 10 additions & 9 deletions src/webapp/topology.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@

import icalendar

from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\
is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, gen_id_from_yaml
from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \
is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \
gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true
from .contacts_reader import ContactsData, User
from .exceptions import DataError

GRIDTYPE_1 = "OSG Production Resource"
GRIDTYPE_2 = "OSG Integration Test Bed Resource"

log = getLogger(__name__)


Expand Down Expand Up @@ -372,7 +370,7 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co
self.site = site
self.service_types = common_data.service_types
self.common_data = common_data
self.production = yaml_data.get("Production", "")
self.production = is_true(yaml_data.get("Production", ""))

scname = yaml_data["SupportCenter"]
scid = int(common_data.support_centers[scname]["ID"])
Expand All @@ -395,6 +393,10 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co
def resources(self):
return [self.resources_by_name[k] for k in sorted(self.resources_by_name)]

@property
def itb(self):
return not self.production

def get_tree(self, authorized=False, filters: Filters = None) -> Optional[OrderedDict]:
if filters is None:
filters = Filters()
Expand All @@ -404,7 +406,7 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere
(filters.rg_id, self.id)]:
if filter_list and attribute not in filter_list:
return
data_gridtype = GRIDTYPE_1 if self.data.get("Production", None) else GRIDTYPE_2
data_gridtype = GRIDTYPE_1 if self.production else GRIDTYPE_2
if filters.grid_type is not None and data_gridtype != filters.grid_type:
return

Expand Down Expand Up @@ -455,8 +457,7 @@ def _expand_rg(self) -> OrderedDict:
new_rg["GroupName"] = self.name
new_rg["SupportCenter"] = self.support_center
new_rg["IsCCStar"] = self.is_ccstar
production = new_rg.get("Production")
if production:
if self.production:
new_rg["GridType"] = GRIDTYPE_1
else:
new_rg["GridType"] = GRIDTYPE_2
Expand Down

0 comments on commit 47e4d18

Please sign in to comment.