Merge pull request #123 from ecmwf-projects/COPDS-1806-messages-by-site

Portal messages by site (not portal)

alembic/versions/18ba95eb351a_messages_by_site_header.py

@@ -0,0 +1,33 @@
+"""messages by SITE header.
+
+Revision ID: 18ba95eb351a
+Revises: 63827287c182
+Create Date: 2024-07-15 17:08:21.290858
+
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "18ba95eb351a"
+down_revision = "63827287c182"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("messages", "portal")
+    op.add_column(
+        "messages",
+        sa.Column("site", sa.String, index=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("messages", "site")
+    op.add_column(
+        "messages",
+        sa.Column("portal", sa.String, index=True),
+    )

cads_catalogue/database.py

@@ -142,7 +142,7 @@ class Message(BaseModel):
     message_id = sa.Column(sa.Integer, primary_key=True)
     message_uid = sa.Column(sa.String, index=True, unique=True, nullable=False)
     date = sa.Column(sa.DateTime, nullable=False)
-    portal = sa.Column(sa.String, index=True)
+    site = sa.Column(sa.String, index=True)
     summary = sa.Column(sa.String, nullable=True)
     url = sa.Column(sa.String)
     severity = sa.Column(

cads_catalogue/messages.py

@@ -46,7 +46,7 @@ def message_sync(
     """
     message = msg.copy()
     message_uid = message["message_uid"]
-    # NOTE: portal messages have always entries = []
+    # NOTE: site messages have always entries = []
     entries = message.pop("entries", [])
 
     # translate entries in resource_objs
@@ -165,31 +165,31 @@ def load_contents_messages(
     return loaded_messages
 
 
-def load_portal_messages(
-    root_msg_folder: str | pathlib.Path, portal_folder: str | pathlib.Path
+def load_site_messages(
+    root_msg_folder: str | pathlib.Path, site_folder: str | pathlib.Path
 ) -> List[dict[str, Any]]:
     """
-    Look for messages specific for a portal (i.e. 'global' messages).
+    Look for messages specific for a site (i.e. 'global' messages).
 
     Parameters
     ----------
     root_msg_folder: base root path (i.e. cads-messages root folder) (used to generate msg uids)
-    portal_folder: root path where to look for portal messages
+    site_folder: root path where to look for site messages
 
     Returns
     -------
     List of found messages parsed.
     """
     loaded_messages: List[dict[str, Any]] = []
-    portal = os.path.basename(portal_folder)
-    for current_root, dirs, files in os.walk(portal_folder):
+    site = os.path.basename(site_folder)
+    for current_root, dirs, files in os.walk(site_folder):
         for current_file in files:
             file_path = os.path.join(current_root, current_file)
             if os.path.splitext(current_file)[1].lower() == ".md":
                 msg_uid = os.path.relpath(file_path, root_msg_folder)
                 try:
                     msg_record = md2message_record(
-                        file_path, msg_uid, is_global=True, portal=portal
+                        file_path, msg_uid, is_global=True, site=site
                     )
                 except:  # noqa
                     logger.exception("error loading message %r" % file_path)
@@ -220,16 +220,16 @@ def load_messages(root_msg_folder: str | pathlib.Path) -> List[dict[str, Any]]:
     else:
         logger.warning("not found folder %r" % contents_folder)
 
-    # load 'portals' folder
-    portals_folder = os.path.join(root_msg_folder, "portals")
-    if os.path.isdir(portals_folder):
-        for portal in os.listdir(portals_folder):
-            portal_folder = os.path.join(portals_folder, portal)
-            if not os.path.isdir(portal_folder):
+    # load 'sites' folder
+    sites_folder = os.path.join(root_msg_folder, "sites")
+    if os.path.isdir(sites_folder):
+        for site in os.listdir(sites_folder):
+            site_folder = os.path.join(sites_folder, site)
+            if not os.path.isdir(site_folder):
                 continue
-            loaded_messages += load_portal_messages(root_msg_folder, portal_folder)
+            loaded_messages += load_site_messages(root_msg_folder, site_folder)
     else:
-        logger.warning("not found folder %r" % portals_folder)
+        logger.warning("not found folder %r" % sites_folder)
 
     return loaded_messages
 

tests/test_10_messages.py

@@ -24,8 +24,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": True,
-            "message_uid": "portals/c3s/2023/Jan/2021-01-example-of-info-active.md",
-            "portal": "c3s",
+            "message_uid": "sites/cds/2023/Jan/2021-01-example-of-info-active.md",
+            "site": "cds",
             "severity": "info",
             "summary": "a summary of the message",
         },
@@ -78,8 +78,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": False,
-            "message_uid": "portals/c3s/2023/Jan/2023-01-example-of-archived-critical.md",
-            "portal": "c3s",
+            "message_uid": "sites/cds/2023/Jan/2023-01-example-of-archived-critical.md",
+            "site": "cds",
             "severity": "critical",
             "summary": "A **brief description** of the message",
         },
@@ -91,8 +91,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": True,
-            "message_uid": "portals/c3s/2023/Jan/2023-01-example-warning-active.md",
-            "portal": "c3s",
+            "message_uid": "sites/cds/2023/Jan/2023-01-example-warning-active.md",
+            "site": "cds",
             "severity": "warning",
             "summary": "A **brief description** of the message",
         },
@@ -105,8 +105,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": True,
-            "message_uid": "portals/ads/2023/02/2021-02-example-of-info-active.md",
-            "portal": "ads",
+            "message_uid": "sites/ads/2023/02/2021-02-example-of-info-active.md",
+            "site": "ads",
             "severity": "info",
             "summary": "a summary of the message",
         },
@@ -118,8 +118,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": False,
-            "message_uid": "portals/ads/2023/02/2023-02-example-of-archived-critical.md",
-            "portal": "ads",
+            "message_uid": "sites/ads/2023/02/2023-02-example-of-archived-critical.md",
+            "site": "ads",
             "severity": "critical",
             "summary": "A **brief description** of the message",
         },
@@ -131,8 +131,8 @@ def test_load_messages() -> None:
             "entries": [],
             "is_global": True,
             "live": True,
-            "message_uid": "portals/ads/2023/02/2023-02-example-warning-active.md",
-            "portal": "ads",
+            "message_uid": "sites/ads/2023/02/2023-02-example-warning-active.md",
+            "site": "ads",
             "severity": "warning",
             "summary": "A **brief description** of the message",
         },
@@ -163,8 +163,8 @@ def test_message_sync(session_obj: sa.orm.sessionmaker) -> None:
         "entries": [],
         "is_global": True,
         "live": False,
-        "message_uid": "portals/c3s/2023/Jan/2023-01-example-of-archived-critical.md",
-        "portal": "portal2",
+        "message_uid": "sites/cds/2023/Jan/2023-01-example-of-archived-critical.md",
+        "site": "site2",
         "severity": "success",
         "summary": "A **brief description** of the message",
     }
@@ -228,7 +228,7 @@ def test_message_sync(session_obj: sa.orm.sessionmaker) -> None:
             assert getattr(db_message2, key) == value
         assert sorted([r.resource_id for r in db_message2.resources]) == [2, 3, 4]  # type: ignore
 
-        # adding portal message
+        # adding site message
         db_message3 = messages.message_sync(session, prt_message)
         session.commit()
         for key, value in prt_message.items():

tests/test_90_entry_points.py

@@ -404,8 +404,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": True,
-                "message_uid": "portals/c3s/2023/Jan/2021-01-example-of-info-active.md",
-                "portal": "c3s",
+                "message_uid": "sites/cds/2023/Jan/2021-01-example-of-info-active.md",
+                "site": "cds",
                 "severity": "info",
                 "summary": "a summary of the message",
             },
@@ -464,8 +464,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": False,
-                "message_uid": "portals/c3s/2023/Jan/2023-01-example-of-archived-critical.md",
-                "portal": "c3s",
+                "message_uid": "sites/cds/2023/Jan/2023-01-example-of-archived-critical.md",
+                "site": "cds",
                 "severity": "critical",
                 "summary": "A **brief description** of the message",
             },
@@ -477,8 +477,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": True,
-                "message_uid": "portals/c3s/2023/Jan/2023-01-example-warning-active.md",
-                "portal": "c3s",
+                "message_uid": "sites/cds/2023/Jan/2023-01-example-warning-active.md",
+                "site": "cds",
                 "severity": "warning",
                 "summary": "A **brief description** of the message",
             },
@@ -491,8 +491,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": True,
-                "message_uid": "portals/ads/2023/02/2021-02-example-of-info-active.md",
-                "portal": "ads",
+                "message_uid": "sites/ads/2023/02/2021-02-example-of-info-active.md",
+                "site": "ads",
                 "severity": "info",
                 "summary": "a summary of the message",
             },
@@ -504,8 +504,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": False,
-                "message_uid": "portals/ads/2023/02/2023-02-example-of-archived-critical.md",
-                "portal": "ads",
+                "message_uid": "sites/ads/2023/02/2023-02-example-of-archived-critical.md",
+                "site": "ads",
                 "severity": "critical",
                 "summary": "A **brief description** of the message",
             },
@@ -517,8 +517,8 @@ def test_update_catalogue(
                 "entries": [],
                 "is_global": True,
                 "live": True,
-                "message_uid": "portals/ads/2023/02/2023-02-example-warning-active.md",
-                "portal": "ads",
+                "message_uid": "sites/ads/2023/02/2023-02-example-warning-active.md",
+                "site": "ads",
                 "severity": "warning",
                 "summary": "A **brief description** of the message",
             },
@@ -719,7 +719,7 @@ def test_update_catalogue(
         session.execute(
             sa.text(
                 "update messages set live=false "
-                "where message_uid='portals/c3s/2023/Jan/2021-01-example-of-info-active.md'"
+                "where message_uid='sites/c3s/2023/Jan/2021-01-example-of-info-active.md'"
             )
         )
         session.commit()
@@ -778,7 +778,7 @@ def test_update_catalogue(
         assert session.execute(
             sa.text(
                 "select live from messages "
-                "where message_uid='portals/c3s/2023/Jan/2021-01-example-of-info-active.md'"
+                "where message_uid='sites/cds/2023/Jan/2021-01-example-of-info-active.md'"
             )
         ).all() == [(True,)]