Remove _v2 from metadata (#1354)

* Remove _v2 from metadata * fixup arcgis_feature_service to use updated md --------- Co-authored-by: Alex Richey <[email protected]>
NYCPlanning · Dec 31, 2024 · 47f2592 · 47f2592
1 parent 89f8722
commit 47f2592
Show file tree

Hide file tree

Showing 25 changed files with 405 additions and 840 deletions.
diff --git a/admin/ops/publish_json_schemas.py b/admin/ops/publish_json_schemas.py
@@ -1,7 +1,7 @@
 # type: ignore
 from dcpy.utils import s3
 from dcpy.models.product import metadata as product_metadata
-from dcpy.models.product.dataset import metadata_v2 as dataset_metadata
+from dcpy.models.product.dataset import metadata as dataset_metadata
 from dcpy.models.lifecycle import ingest as ingest_models
 
 import json

diff --git a/dcpy/connectors/esri/arcgis_feature_service.py b/dcpy/connectors/esri/arcgis_feature_service.py
@@ -191,48 +191,45 @@ def make_dcp_metadata(layer_url: str) -> models.Metadata:
     resp = requests.get(layer_url).json()
     esri_to_dcp = {
         "esriFieldTypeString": "text",
-        "esriFieldTypeDouble": "double",
+        "esriFieldTypeDouble": "decimal",
         "esriFieldTypeSmallInteger": "integer",
     }
 
     raw_cols = resp.get("fields")
     our_cols = [
-        models.Column(
-            name=c.get("name"),
-            display_name=c.get("alias"),
+        models.DatasetColumn(
+            id=c.get("name"),
+            name=c.get("alias"),
             description="",
-            data_type=esri_to_dcp[c.get("type")],
+            data_type=esri_to_dcp[c.get("type")],  # type: ignore
         )
         for c in raw_cols
         if c["name"] != "OBJECTID"
     ]
 
     return models.Metadata(
-        name=resp.get("name"),
-        display_name=models.FILL_ME_IN_PLACEHOLDER,
-        package=models.Package(
-            dataset_files=[
-                models.DatasetFile(
-                    name="primary_shapefile",
+        id=resp.get("name"),
+        attributes=models.DatasetAttributes(
+            display_name="FILL ME IN",
+            description=resp.get("description"),
+            tags=[],
+            each_row_is_a="",
+        ),
+        files=[
+            models.FileAndOverrides(
+                file=models.File(
+                    id="primary_shapefile",
                     filename="shapefile.zip",
                     type="shapefile",
-                    overrides=models.DatasetOverrides(),
                 )
-            ],
-            attachments=[],
-        ),
+            )
+        ],
         destinations=[
-            models.SocrataDestination(
+            models.DestinationWithFiles(
                 id="socrata_prod",
-                four_four="",
-                datasets=["primary_shapefile"],
-                omit_columns=[],
+                type="socrata",
             )
         ],
-        summary="",
-        description=resp.get("description"),
-        tags=[],
-        each_row_is_a="",
         columns=our_cols,
     )
 
@@ -257,7 +254,7 @@ def _export_metadata(
     md = make_dcp_metadata(layer_url)
     md_json = md.model_dump(exclude_none=True)
 
-    output_path = output_path or Path(f"{md.name}.yml")
+    output_path = output_path or Path(f"{md.id}.yml")
 
     logger.info(f"exporting metadata to {output_path}")
     with open(output_path, "w") as outfile:

diff --git a/dcpy/connectors/socrata/metadata.py b/dcpy/connectors/socrata/metadata.py
@@ -5,7 +5,7 @@
 from dcpy.connectors.socrata import publish as pub
 from dcpy.connectors.socrata import metadata
 
-import dcpy.models.product.dataset.metadata_v2 as md
+import dcpy.models.product.dataset.metadata as md
 from dcpy.utils.logging import logger
 
 soc_types_to_dcp_types = {

diff --git a/dcpy/connectors/socrata/publish.py b/dcpy/connectors/socrata/publish.py
@@ -22,7 +22,7 @@
 
 from dcpy.utils.logging import logger
 
-import dcpy.models.product.dataset.metadata_v2 as md
+import dcpy.models.product.dataset.metadata as md
 from .utils import SOCRATA_USER, SOCRATA_PASSWORD, _socrata_request
 
 SOCRATA_REVISION_APPLY_TIMEOUT_SECS = 10 * 60  # Ten Mins

diff --git a/dcpy/lifecycle/distribute/socrata.py b/dcpy/lifecycle/distribute/socrata.py
@@ -2,7 +2,7 @@
 import typer
 from typing import TypedDict, Unpack, NotRequired, Required
 
-import dcpy.models.product.dataset.metadata_v2 as m
+import dcpy.models.product.dataset.metadata as m
 from dcpy.utils.logging import logger
 import dcpy.connectors.edm.packaging as packaging
 import dcpy.connectors.socrata.publish as soc_pub

diff --git a/dcpy/lifecycle/package/abstract_doc.py b/dcpy/lifecycle/package/abstract_doc.py
@@ -2,7 +2,7 @@
 from typing import Any
 
 from dcpy.models.design import elements as de
-from dcpy.models.product.dataset.metadata_v2 import Dataset
+from dcpy.models.product.dataset.metadata import Dataset
 from dcpy.models.product.metadata import OrgMetadata
 from dcpy.models.product.artifacts import Artifact, ExcelTableComponentDefinition
 from dcpy.utils.logging import logger

diff --git a/dcpy/lifecycle/package/assemble.py b/dcpy/lifecycle/package/assemble.py
@@ -9,7 +9,7 @@
 from dcpy.lifecycle import WORKING_DIRECTORIES
 from dcpy.lifecycle.package import xlsx_writer
 from dcpy.lifecycle.package import assemble
-import dcpy.models.product.dataset.metadata_v2 as md
+import dcpy.models.product.dataset.metadata as md
 import dcpy.models.product.metadata as prod_md
 from dcpy.utils.logging import logger
 

diff --git a/dcpy/lifecycle/package/pdf_writer.py b/dcpy/lifecycle/package/pdf_writer.py
@@ -3,7 +3,7 @@
 import css_inline
 from bs4 import BeautifulSoup
 import subprocess
-from dcpy.models.product.dataset.metadata_v2 import Metadata
+from dcpy.models.product.dataset.metadata import Metadata
 from dcpy.utils.logging import logger
 
 from . import RESOURCES_PATH

diff --git a/dcpy/lifecycle/package/shapefiles.py b/dcpy/lifecycle/package/shapefiles.py
@@ -2,7 +2,7 @@
 import typer
 import xml.etree.ElementTree as ET
 
-from dcpy.models.product.dataset.metadata_v2 import (
+from dcpy.models.product.dataset.metadata import (
     Metadata,
     DatasetAttributes,
     DatasetColumn,

diff --git a/dcpy/lifecycle/package/validate.py b/dcpy/lifecycle/package/validate.py
@@ -9,7 +9,7 @@
 from shapely import wkb, wkt
 import typer
 
-import dcpy.models.product.dataset.metadata_v2 as dataset_md
+import dcpy.models.product.dataset.metadata as dataset_md
 from dcpy.utils.logging import logger
 
 

diff --git a/dcpy/models/product/artifacts.py b/dcpy/models/product/artifacts.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from .dataset.metadata_v2 import CustomizableBase
+from .dataset.metadata import CustomizableBase
 from dcpy.models.base import TemplatedYamlReader
 
 

diff --git a/dcpy/models/product/data_dictionary.py b/dcpy/models/product/data_dictionary.py
@@ -1,4 +1,4 @@
-from .dataset.metadata_v2 import CustomizableBase
+from .dataset.metadata import CustomizableBase
 from dcpy.models.base import TemplatedYamlReader