add FeatureServerLayer to ingest sources

dcpy/connectors/esri/arcgis_feature_service.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from pathlib import Path
 import requests
+import json
 import typer
 from typing import cast
 from rich.progress import (
@@ -34,6 +35,7 @@ def get_feature_server_layers(
     resp = get_feature_server_metadata(feature_server)
     return [
         FeatureServerLayer(
+            type="esri",
             server=feature_server.server,
             name=feature_server.name,
             layer_name=layer["name"],
@@ -94,6 +96,7 @@ def resolve_layer(
             assert layer_name is not None
             assert layer_id is not None
             layer = FeatureServerLayer(
+                type="esri",
                 server=feature_server.server,
                 name=feature_server.name,
                 layer_name=layer_name,
@@ -184,6 +187,15 @@ def _downcase_properties_keys(feat):
     return {"type": "FeatureCollection", "crs": crs, "features": features}
 
 
+def download_file(layer: FeatureServerLayer, crs: str, path: Path) -> None:
+    geojson = get_layer(
+        layer,
+        crs=int(crs.strip("EPSG:")),
+    )
+    with open(path, "w") as f:
+        json.dump(geojson, f)
+
+
 def make_dcp_metadata(layer_url: str) -> models.Metadata:
     if layer_url.endswith("FeatureServer/0"):
         layer_url = layer_url + "?f=pjson"

dcpy/lifecycle/ingest/configure.py

@@ -18,7 +18,7 @@
     Template,
     Config,
 )
-from dcpy.models.connectors import socrata, web as web_models
+from dcpy.models.connectors import socrata, esri, web as web_models
 from dcpy.models.connectors.edm.publishing import GisDataset
 from dcpy.utils import metadata
 from dcpy.utils.logging import logger
@@ -98,6 +98,8 @@ def get_filename(source: Source, ds_id: str) -> str:
             return os.path.basename(urlparse(source.url).path)
         case web_models.GenericApiSource():
             return f"{ds_id}.{source.format}"
+        case esri.FeatureServerLayer():
+            return f"{ds_id}.geojson"
         case socrata.Source():
             return f"{ds_id}.{source.extension}"
         case S3Source():

dcpy/lifecycle/ingest/extract.py

@@ -10,10 +10,13 @@
 )
 from dcpy.models.connectors import socrata, web as web_models
 from dcpy.models.connectors.edm.publishing import GisDataset
+from dcpy.models.connectors.esri import FeatureServerLayer
 from dcpy.utils import s3
 from dcpy.connectors.edm import publishing
 from dcpy.connectors.socrata import extract as extract_socrata
 from dcpy.connectors import web
+from dcpy.connectors.esri import arcgis_feature_service
+import json
 
 
 def download_file_from_source(
@@ -46,6 +49,8 @@ def download_file_from_source(
             web.download_file(source.url, path)
         case web_models.GenericApiSource():
             web.download_file(source.endpoint, path)
+        case FeatureServerLayer():
+            arcgis_feature_service.download_file(source, "EPSG:4326", path)
         case socrata.Source():
             extract_socrata.download(source, path)
         case _:

dcpy/models/connectors/esri.py

@@ -1,5 +1,6 @@
 from enum import StrEnum
 from pydantic import BaseModel
+from typing import Literal
 
 
 class Server(StrEnum):
@@ -31,6 +32,7 @@ def url(self) -> str:
 
 
 class FeatureServerLayer(BaseModel, extra="forbid"):
+    type: Literal["esri"]
     server: Server
     name: str
     layer_name: str

dcpy/models/lifecycle/ingest.py

@@ -6,7 +6,7 @@
 
 from dcpy.utils.metadata import RunDetails
 from dcpy.models.connectors.edm import recipes, publishing
-from dcpy.models.connectors import web, socrata
+from dcpy.models.connectors import web, esri, socrata
 from dcpy.models import file
 from dcpy.models.base import SortedSerializedBase
 from dcpy.models.dataset import Column as BaseColumn, COLUMN_TYPES
@@ -39,6 +39,7 @@ class DEPublished(BaseModel, extra="forbid"):
     LocalFileSource
     | web.FileDownloadSource
     | web.GenericApiSource
+    | esri.FeatureServerLayer
     | socrata.Source
     | publishing.GisDataset
     | DEPublished

dcpy/test/connectors/test_esri.py

@@ -19,6 +19,7 @@
 MULTIPLE_LAYER_DATASET = "National_Register_Building_Listings_Multiple"
 MULTIPLE_LAYER_FS = FeatureServer(server=Server.nys_parks, name=MULTIPLE_LAYER_DATASET)
 MULTIPLE_LAYER = FeatureServerLayer(
+    type="esri",
     server=Server.nys_parks,
     name=MULTIPLE_LAYER_DATASET,
     layer_name=LAYER_NAME,
@@ -73,6 +74,7 @@ def test_implicit_single(self, request_get):
         dataset = "National_Register_Building_Listings"
         fs = FeatureServer(server=Server.nys_parks, name=dataset)
         layer = FeatureServerLayer(
+            type="esri",
             server=Server.nys_parks,
             name=dataset,
             layer_name=LAYER_NAME,
@@ -123,7 +125,11 @@ def test_get_layer_metadata_error(self, get, post):
         with pytest.raises(Exception, match="Error fetching ESRI Server metadata"):
             arcfs.get_layer_metadata(
                 FeatureServerLayer(
-                    server=Server.nys_parks, name="error", layer_name="", layer_id=13
+                    type="esri",
+                    server=Server.nys_parks,
+                    name="error",
+                    layer_name="",
+                    layer_id=13,
                 )
             )
 

dcpy/test/lifecycle/ingest/__init__.py

@@ -2,7 +2,7 @@
 
 
 from dcpy.models.connectors.edm.publishing import GisDataset
-from dcpy.models.connectors import socrata, web
+from dcpy.models.connectors import socrata, web, esri
 from dcpy.models.lifecycle.ingest import (
     LocalFileSource,
     ScriptSource,
@@ -31,6 +31,13 @@ class Sources:
         endpoint="https://www.bklynlibrary.org/locations/json",
         format="json",
     )
+    esri = esri.FeatureServerLayer(
+        type="esri",
+        server="nys_parks",
+        name="National_Register_Building_Listings",
+        layer_name="National Register Building Listings",
+        layer_id="13",
+    )
     socrata = socrata.Source(
         type="socrata", org=socrata.Org.nyc, uid="w7w3-xahh", format="csv"
     )
@@ -45,6 +52,7 @@ class Sources:
     (Sources.gis, f"{TEST_DATASET_NAME}.zip"),
     (Sources.file_download, "pad_24a.zip"),
     (Sources.api, f"{TEST_DATASET_NAME}.json"),
+    (Sources.esri, f"{TEST_DATASET_NAME}.geoson"),
     (Sources.socrata, f"{TEST_DATASET_NAME}.csv"),
     (Sources.s3, "test.txt"),
 ]