From a041b06bf69d44ed11e288a1a1ab7de273395c66 Mon Sep 17 00:00:00 2001 From: sf-dcp <144725249+sf-dcp@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:45:49 -0500 Subject: [PATCH] Library to Ingest: weekly OpenData templates (#1289) * nypd_policeprecincts: library -> ingest * lpc_scenic_landmarks: library -> ingest * lpc_landmarks: library -> ingest Remove lon and lat columns from `lpc_landmarks` source in GFT - no longer exist at the source. Note, these columns are not used in GFT build. * lpc_historic_district_areas: library -> ingest * lpc_historic_districts: library -> ingest * fdny_firehouses: library -> ingest * dsny_garages: library -> ingest * dsny_frequencies: library -> ingest * dpr_schoolyard_to_playgrounds: library -> ingest * dpr_park_access_zone: library -> ingest * dpr_greenthumb: library -> ingest * dpr_parksproperties: library -> ingest * dpr_forever_wild: library -> ingest * doitt_buildingfootprints_historical: library -> ingest * doitt_buildingfootprints: library -> ingest * dot_projects_streets: library -> ingest * dot_projects_intersections: library -> ingest * dcp_edesignation_csv: library -> ingest * dpr_capitalprojects: remove it from weekly Open Data pull GHA (does not belong there) * dpr_capitalprojects: library -> ingest --- .github/workflows/ingest_open_data.yml | 1 - dcpy/library/script/dpr_capitalprojects.py | 37 ----------- .../templates/dcp_edesignation_csv.yml | 34 ---------- .../templates/doitt_buildingfootprints.yml | 34 ---------- .../doitt_buildingfootprints_historical.yml | 39 ------------ .../templates/dot_projects_intersections.yml | 32 ---------- .../templates/dot_projects_streets.yml | 32 ---------- .../library/templates/dpr_capitalprojects.yml | 32 ---------- dcpy/library/templates/dpr_forever_wild.yml | 34 ---------- dcpy/library/templates/dpr_greenthumb.yml | 28 --------- .../templates/dpr_park_access_zone.yml | 31 --------- .../library/templates/dpr_parksproperties.yml | 34 ---------- .../dpr_schoolyard_to_playgrounds.yml | 45 ------------- dcpy/library/templates/dsny_frequencies.yml | 32 ---------- dcpy/library/templates/dsny_garages.yml | 32 ---------- dcpy/library/templates/fdny_firehouses.yml | 33 ---------- .../templates/lpc_historic_district_areas.yml | 35 ----------- .../templates/lpc_historic_districts.yml | 37 ----------- dcpy/library/templates/lpc_landmarks.yml | 32 ---------- .../templates/lpc_scenic_landmarks.yml | 31 --------- .../templates/nypd_policeprecincts.yml | 31 --------- .../ingest/templates/dcp_edesignation_csv.yml | 62 ++++++++++++++++++ .../templates/doitt_buildingfootprints.yml | 47 ++++++++++++++ .../doitt_buildingfootprints_historical.yml | 36 +++++++++++ .../templates/dot_projects_intersections.yml | 38 +++++++++++ .../ingest/templates/dot_projects_streets.yml | 38 +++++++++++ .../ingest/templates/dpr_capitalprojects.yml | 63 +++++++++++++++++++ .../ingest/templates/dpr_forever_wild.yml | 38 +++++++++++ .../ingest/templates/dpr_greenthumb.yml | 32 ++++++++++ .../ingest/templates/dpr_park_access_zone.yml | 48 ++++++++++++++ .../ingest/templates/dpr_parksproperties.yml | 47 ++++++++++++++ .../dpr_schoolyard_to_playgrounds.yml | 55 ++++++++++++++++ .../ingest/templates/dsny_frequencies.yml | 28 +++++++++ .../ingest/templates/dsny_garages.yml | 43 +++++++++++++ .../ingest/templates/fdny_firehouses.yml | 52 +++++++++++++++ .../templates/lpc_historic_district_areas.yml | 36 +++++++++++ .../templates/lpc_historic_districts.yml | 33 ++++++++++ .../ingest/templates/lpc_landmarks.yml | 61 ++++++++++++++++++ .../ingest/templates/lpc_scenic_landmarks.yml | 36 +++++++++++ .../ingest/templates/nypd_policeprecincts.yml | 26 ++++++++ dcpy/test/library/test_ingest_script.py | 8 --- products/green_fast_track/models/_sources.yml | 5 +- 42 files changed, 821 insertions(+), 687 deletions(-) delete mode 100644 dcpy/library/script/dpr_capitalprojects.py delete mode 100644 dcpy/library/templates/dcp_edesignation_csv.yml delete mode 100644 dcpy/library/templates/doitt_buildingfootprints.yml delete mode 100644 dcpy/library/templates/doitt_buildingfootprints_historical.yml delete mode 100644 dcpy/library/templates/dot_projects_intersections.yml delete mode 100644 dcpy/library/templates/dot_projects_streets.yml delete mode 100644 dcpy/library/templates/dpr_capitalprojects.yml delete mode 100644 dcpy/library/templates/dpr_forever_wild.yml delete mode 100644 dcpy/library/templates/dpr_greenthumb.yml delete mode 100644 dcpy/library/templates/dpr_park_access_zone.yml delete mode 100644 dcpy/library/templates/dpr_parksproperties.yml delete mode 100644 dcpy/library/templates/dpr_schoolyard_to_playgrounds.yml delete mode 100644 dcpy/library/templates/dsny_frequencies.yml delete mode 100644 dcpy/library/templates/dsny_garages.yml delete mode 100644 dcpy/library/templates/fdny_firehouses.yml delete mode 100644 dcpy/library/templates/lpc_historic_district_areas.yml delete mode 100644 dcpy/library/templates/lpc_historic_districts.yml delete mode 100644 dcpy/library/templates/lpc_landmarks.yml delete mode 100644 dcpy/library/templates/lpc_scenic_landmarks.yml delete mode 100644 dcpy/library/templates/nypd_policeprecincts.yml create mode 100644 dcpy/lifecycle/ingest/templates/dcp_edesignation_csv.yml create mode 100644 dcpy/lifecycle/ingest/templates/doitt_buildingfootprints.yml create mode 100644 dcpy/lifecycle/ingest/templates/doitt_buildingfootprints_historical.yml create mode 100644 dcpy/lifecycle/ingest/templates/dot_projects_intersections.yml create mode 100644 dcpy/lifecycle/ingest/templates/dot_projects_streets.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_capitalprojects.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_forever_wild.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_greenthumb.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_park_access_zone.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_parksproperties.yml create mode 100644 dcpy/lifecycle/ingest/templates/dpr_schoolyard_to_playgrounds.yml create mode 100644 dcpy/lifecycle/ingest/templates/dsny_frequencies.yml create mode 100644 dcpy/lifecycle/ingest/templates/dsny_garages.yml create mode 100644 dcpy/lifecycle/ingest/templates/fdny_firehouses.yml create mode 100644 dcpy/lifecycle/ingest/templates/lpc_historic_district_areas.yml create mode 100644 dcpy/lifecycle/ingest/templates/lpc_historic_districts.yml create mode 100644 dcpy/lifecycle/ingest/templates/lpc_landmarks.yml create mode 100644 dcpy/lifecycle/ingest/templates/lpc_scenic_landmarks.yml create mode 100644 dcpy/lifecycle/ingest/templates/nypd_policeprecincts.yml diff --git a/.github/workflows/ingest_open_data.yml b/.github/workflows/ingest_open_data.yml index 1f361db76..27537f8bc 100644 --- a/.github/workflows/ingest_open_data.yml +++ b/.github/workflows/ingest_open_data.yml @@ -34,7 +34,6 @@ jobs: - doitt_buildingfootprints_historical - dot_projects_intersections - dot_projects_streets - - dpr_capitalprojects - dpr_forever_wild - dpr_greenthumb - dpr_park_access_zone diff --git a/dcpy/library/script/dpr_capitalprojects.py b/dcpy/library/script/dpr_capitalprojects.py deleted file mode 100644 index 76f04dd2e..000000000 --- a/dcpy/library/script/dpr_capitalprojects.py +++ /dev/null @@ -1,37 +0,0 @@ -import pandas as pd - -from . import get_json_content, df_to_tempfile -from .scriptor import ScriptorInterface - - -class Scriptor(ScriptorInterface): - def ingest(self) -> pd.DataFrame: - data = get_json_content(self.source["path"]) - - df = pd.DataFrame(data) - df = df[["TrackerID", "FMSID", "Title", "TotalFunding", "Locations"]] - df["Locations"] = df["Locations"].apply(lambda x: x.get("Location")) - df2 = df.drop(columns=["Locations"]).join(df["Locations"].explode().to_frame()) - horiz_exploded = pd.json_normalize(df2["Locations"].to_list()) - horiz_exploded.index = df2.index - df3 = pd.concat([df2, horiz_exploded], axis=1).drop(columns=["Locations"]) - df3 = df3.rename( - columns={ - "TrackerID": "proj_id", - "FMSID": "fmsid", - "Title": "desc", - "TotalFunding": "total_funding", - "ParkID": "park_id", - "Latitude": "lat", - "Longitude": "lon", - } - ) - df3 = df3[ - ["proj_id", "fmsid", "desc", "total_funding", "park_id", "lat", "lon"] - ] - return df3 - - def runner(self) -> str: - df = self.ingest() - local_path = df_to_tempfile(df) - return local_path diff --git a/dcpy/library/templates/dcp_edesignation_csv.yml b/dcpy/library/templates/dcp_edesignation_csv.yml deleted file mode 100644 index fbb6f4c31..000000000 --- a/dcpy/library/templates/dcp_edesignation_csv.yml +++ /dev/null @@ -1,34 +0,0 @@ -dataset: - name: dcp_edesignation_csv - acl: public-read - source: - socrata: - uid: hxm3-23vy - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - destination: - geometry: - SRS: null - type: NONE - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### NYC (E) Designations - (E) Designations are established to provide notice of the presence of an environmental - requirement pertaining to potential hazardous materials contamination, high ambient noise - levels or air emission concerns on a particular tax lot. This includes CEQR Environment - Requirements and CEQR Restrictive Declarations, in Appendix C of the - Zoning Resolution. - - Not all columns are currently included in shapefile, hence why we need this table. - This should be fixed sometime before 2024 Q3, and at that point this template can be archived - url: "https://www1.nyc.gov/assets/planning/download/pdf/data-maps/open-data/nyedes_metadata.pdf" - dependents: [] diff --git a/dcpy/library/templates/doitt_buildingfootprints.yml b/dcpy/library/templates/doitt_buildingfootprints.yml deleted file mode 100644 index ffdff4be2..000000000 --- a/dcpy/library/templates/doitt_buildingfootprints.yml +++ /dev/null @@ -1,34 +0,0 @@ -dataset: - name: doitt_buildingfootprints - acl: public-read - source: - socrata: - uid: qb5r-6dgf # uid of child view of overall dataset which has polygons and not centroids. - format: shapefile - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### Building Footprints - Shapefile of footprint outlines of buildings in New York City. - Please see the following link for additional documentation- - https://github.com/CityOfNewYork/nyc-geo-metadata/blob/master/Metadata/Metadata_BuildingFootprints.md - Previously posted versions of the data are retained to comply with Local Law 106 of 2015 and can be provided upon request made to Open Data. - url: "https://data.cityofnewyork.us/Housing-Development/Building-Footprints/nqwf-w8eh" - dependents: [] diff --git a/dcpy/library/templates/doitt_buildingfootprints_historical.yml b/dcpy/library/templates/doitt_buildingfootprints_historical.yml deleted file mode 100644 index 1fc392971..000000000 --- a/dcpy/library/templates/doitt_buildingfootprints_historical.yml +++ /dev/null @@ -1,39 +0,0 @@ -dataset: - name: doitt_buildingfootprints_historical - acl: public-read - source: - socrata: - uid: ipkp-snf6 - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### Building Footprints - Shapefile of historical footprint outlines of buildings in New York City. - Please see the following link for additional documentation- - https://github.com/CityOfNewYork/nyc-planimetrics/blob/master/Capture_Rules.md. - Previously posted versions of the data are retained to comply with Local Law 106 of 2015 and - can be provided upon request made to Open Data. - NOTE: The Socrata UID is not the same as the url (below). We want UID: ipkp-snf6 which is - the multipolygon geojson layer. There are two layers of data on the opendata portal: - one contains the multipolygon (building_historic) and the other contains points only (building_historic_p). - url: "https://data.cityofnewyork.us/Housing-Development/Building-Footprints-Historical-Shape/s5zg-yzea" - dependents: [] - \ No newline at end of file diff --git a/dcpy/library/templates/dot_projects_intersections.yml b/dcpy/library/templates/dot_projects_intersections.yml deleted file mode 100644 index 5820dc9c7..000000000 --- a/dcpy/library/templates/dot_projects_intersections.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: dot_projects_intersections - acl: public-read - source: - socrata: - uid: 97nd-ff3i - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: MULTIPOINT - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOINT - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### Street and Highway Capital Reconstruction Projects - Intersection - This data is a spatial representation of street construction projects. Street and Highway capital projects are major street reconstruction projects, ranging from general street resurfacing projects to full reconstruction of the roadbed, sidewalks, sewer and water pipes and other utilities. Capital projects are essential to keep the City's infrastructure in a state of good repair. - url: "https://data.cityofnewyork.us/Transportation/Street-and-Highway-Capital-Reconstruction-Projects/97nd-ff3i" - dependents: [] diff --git a/dcpy/library/templates/dot_projects_streets.yml b/dcpy/library/templates/dot_projects_streets.yml deleted file mode 100644 index 4e4cc2489..000000000 --- a/dcpy/library/templates/dot_projects_streets.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: dot_projects_streets - acl: public-read - source: - socrata: - uid: jvk9-k4re - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: MULTILINESTRING - - destination: - geometry: - SRS: EPSG:4326 - type: MULTILINESTRING - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### Street and Highway Capital Reconstruction Projects - Block - This data is a spatial representation of street construction projects. Street and Highway capital projects are major street reconstruction projects, ranging from general street resurfacing projects to full reconstruction of the roadbed, sidewalks, sewer and water pipes and other utilities. Capital projects are essential to keep the City's infrastructure in a state of good repair. - url: "https://data.cityofnewyork.us/Transportation/Street-and-Highway-Capital-Reconstruction-Projects/jvk9-k4re" - dependents: [] diff --git a/dcpy/library/templates/dpr_capitalprojects.yml b/dcpy/library/templates/dpr_capitalprojects.yml deleted file mode 100644 index 0a94d2adf..000000000 --- a/dcpy/library/templates/dpr_capitalprojects.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: dpr_capitalprojects - acl: public-read - source: - script: - path: https://www.nycgovparks.org/bigapps/DPR_CapitalProjectTracker_001.json - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - - "X_POSSIBLE_NAMES=longitude,Longitude,Lon,lon,x" - - "Y_POSSIBLE_NAMES=latitude,Latitude,Lat,lat,y" - geometry: - SRS: EPSG:4326 - type: POINT - - destination: - geometry: - SRS: EPSG:4326 - type: POINT - options: - - "OVERWRITE=YES" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### DPR Capital Projects - check [here] for metadata - url: "https://www.nycgovparks.org/bigapps" - dependents: [] diff --git a/dcpy/library/templates/dpr_forever_wild.yml b/dcpy/library/templates/dpr_forever_wild.yml deleted file mode 100644 index 8371b65ab..000000000 --- a/dcpy/library/templates/dpr_forever_wild.yml +++ /dev/null @@ -1,34 +0,0 @@ -dataset: - name: dpr_forever_wild - acl: public-read - source: - socrata: - uid: 48va-85tp - format: shapefile - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:2263 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - - info: - description: | - ### Evictions - The Forever Wild layer delineates the location of ecologically important natural resources within NYC Parks property. - Approximately 12,500 acres of NYC Parks' 30,000 acres of land is comprised of natural areas - - forests, salt marshes, open water, grasslands, streams, and freshwater wetlands. - The vast majority of these natural areas are accessible to residents and visitors. - These areas provide valuable habitat for wildlife and a variety of environmental services to all New Yorkers - including carbon sequestration, air quality improvement, cooling, and stormwater capture. - url: https://data.cityofnewyork.us/Environment/NYC-Parks-Forever-Wild/48va-85tp/about_data - dependents: [] diff --git a/dcpy/library/templates/dpr_greenthumb.yml b/dcpy/library/templates/dpr_greenthumb.yml deleted file mode 100644 index 6627a277a..000000000 --- a/dcpy/library/templates/dpr_greenthumb.yml +++ /dev/null @@ -1,28 +0,0 @@ -dataset: - name: dpr_greenthumb - acl: public-read - source: - socrata: - uid: p78i-pat6 - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - fields: [] - - info: - description: | - ### GreenThumb Garden Info - GreenThumb provides programming and material support to over 550 community gardens in New York City. - The data contains garden information and is part of the GreenThumb Gardens Data Collection. - url: https://data.cityofnewyork.us/dataset/GreenThumb-Garden-Info/p78i-pat6 diff --git a/dcpy/library/templates/dpr_park_access_zone.yml b/dcpy/library/templates/dpr_park_access_zone.yml deleted file mode 100644 index 504bd1082..000000000 --- a/dcpy/library/templates/dpr_park_access_zone.yml +++ /dev/null @@ -1,31 +0,0 @@ -dataset: - name: dpr_park_access_zone - acl: public-read - source: - socrata: - uid: 99ii-hwh9 - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "GEOMETRY=AS_WKT" - fields: [] - - info: - description: | - ## Walk-to-a-Park Service area - The Walk to a Park initiative focuses on increasing access to parks and open spaces, concentrating on areas of the city that are under-resourced and where residents are living further than a walk to a park. NYC Parks calculates the number of New Yorkers within walking distance of a park and reports on this as part of the Mayor’s Management Report. “Walking distance” is defined as a 1/4-mile or less for sites such as small playgrounds and sitting areas; or a 1/2-mile or less for larger parks that serve a wider region, typically over 8 acres or situated on the waterfront. Certain properties in NYC Parks' portfolio, such as cemeteries, community gardens, or sites with no recreational equipment were not included in this analysis. Similarly, some parks and open-space amenities not under the jurisdiction of NYC Parks were included in this analysis, as they provide recreational value. - This dataset includes two ESRI shapefiles of access points, one for sites that get a 1/4-mile buffer and one for sites that get a 1/2-mile buffer. - This dataset is one row - the combined polygon that is the output of this analysis. This polygon represents the total area within walking distance of a park. To generate the number of New Yorkers within that distance, the polygon is compared with data from the U.S. Census. - This information is only current as of the publication date. For more information about this analysis and the Walk to a Park Initiative, visit: https://www.nycgovparks.org/planning-and-building/planning/walk-to-a-park - url: https://data.cityofnewyork.us/Recreation/1_4MilePoints/99ii-hwh9/about_data diff --git a/dcpy/library/templates/dpr_parksproperties.yml b/dcpy/library/templates/dpr_parksproperties.yml deleted file mode 100644 index ea721a5eb..000000000 --- a/dcpy/library/templates/dpr_parksproperties.yml +++ /dev/null @@ -1,34 +0,0 @@ -dataset: - name: dpr_parksproperties - acl: public-read - source: - socrata: - uid: enfh-gkve - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "GEOMETRY=AS_WKT" - fields: [] - - info: - description: | - This dataset identifies property managed partially or solely by NYC Parks. - This data has been produced in whole or part using secondary data. - Data accuracy is limited by the scale and accuracy of the original sources. - Site-specific conditions should be field-verified. - Records are added as more land is designated under NYC Parks’ jurisdiction. - Each record represents an acquisition. - User Guide: https://docs.google.com/document/d/1NExNJF5YKID04oOopi0fHainRuGG3Pz_jKSrMujPsPk/edit?usp=sharing - Data Dictionary: https://docs.google.com/spreadsheets/d/1Q4DBWu7riNFxWvy1vnTJHoOI3r2L9oW6eCN56jCNyCw/edit?usp=sharing - url: https://data.cityofnewyork.us/Recreation/Parks-Properties/enfh-gkve diff --git a/dcpy/library/templates/dpr_schoolyard_to_playgrounds.yml b/dcpy/library/templates/dpr_schoolyard_to_playgrounds.yml deleted file mode 100644 index c1262c165..000000000 --- a/dcpy/library/templates/dpr_schoolyard_to_playgrounds.yml +++ /dev/null @@ -1,45 +0,0 @@ -dataset: - name: dpr_schoolyard_to_playgrounds - acl: public-read - source: - socrata: - uid: bbtf-6p3c - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:2263 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "GEOMETRY=AS_WKT" - fields: [] - - info: - description: | - The Schoolyards to Playgrounds initiative began with PlaNYC 2007 as a way to expand public use - at existing City facilities and improve access to open spaces during non-school hours which includes - weekends, holidays, and during the summer. - - NYC Parks, in collaboration with the Department of Education (DOE) and the nonprofit Trust for Public Land - have improved hundreds of DOE operated schoolyards through capital improvements, including play equipment, - painted sports surfaces, trees, and benches. - - The program is an important approach to reaching the city’s target to getting more New Yorkers within - walking distance to open-space. In many neighborhoods, these DOE schoolyards are one of the few resources that - can be improved and provide publicly accessible open space. - - Each record in this dataset represents a participating DOE Schoolyard at the time of the dataset's publication. - The schoolyard's boundaries have been drawn in whole or part using secondary data. Data accuracy is limited by - the scale and accuracy of the original sources. Site-specific conditions should be field-verified. - - For more information on the Schoolyard to Playgrounds program, visit: https://www.nycgovparks.org/greening/planyc/schoolyards - This dataset may be updated throughout the year as school's join or exit the program. - Data Dictionary: https://docs.google.com/spreadsheets/d/10yVRhmdIAS1g8c3L0jC4XxSTptdIf5wESL5ks1e6Gto/edit?usp=sharing - url: https://data.cityofnewyork.us/dataset/Schoolyard-to-Playgrounds/bbtf-6p3c/about_data diff --git a/dcpy/library/templates/dsny_frequencies.yml b/dcpy/library/templates/dsny_frequencies.yml deleted file mode 100644 index 77a065201..000000000 --- a/dcpy/library/templates/dsny_frequencies.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: dsny_frequencies - acl: public-read - source: - socrata: - uid: rv63-53db - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=multipolygon" - geometry: - SRS: EPSG:4326 - type: NONE - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### DSNY Frequencies - Citywide DSNY frequency boundaries for collection operation (refuse, recycling, organics, bulk items). - These boundaries are sub-divisions of DSNY sections. - url: "https://data.cityofnewyork.us/City-Government/DSNY-Frequencies/rv63-53db" - dependents: [] diff --git a/dcpy/library/templates/dsny_garages.yml b/dcpy/library/templates/dsny_garages.yml deleted file mode 100644 index 869e34b62..000000000 --- a/dcpy/library/templates/dsny_garages.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: dsny_garages - acl: public-read - source: - socrata: - uid: xw3j-2yxf - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=point" - geometry: - SRS: EPSG:4326 - type: POINT - - destination: - geometry: - SRS: EPSG:4326 - type: POINT - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### DSNY Garages - Location of DSNY garages. Essential to DSNY operations are our 59 garages across the five boroughs. This structure enables the Department’s uniformed personnel to efficiently perform regular duties – and quickly execute emergency response. For more information about DSNY garages, see: https://www1.nyc.gov/assets/dsny/site/about/dsny-garage-locations - url: "https://data.cityofnewyork.us/Environment/DSNY-Garages/xw3j-2yxf" - dependents: [] diff --git a/dcpy/library/templates/fdny_firehouses.yml b/dcpy/library/templates/fdny_firehouses.yml deleted file mode 100644 index ca9e37867..000000000 --- a/dcpy/library/templates/fdny_firehouses.yml +++ /dev/null @@ -1,33 +0,0 @@ -dataset: - name: fdny_firehouses - acl: public-read - source: - socrata: - uid: hc8x-tcnd - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "X_POSSIBLE_NAMES=longitude,Longitude" - - "Y_POSSIBLE_NAMES=latitude,Latitude" - geometry: - SRS: EPSG:4326 - type: POINT - - destination: - geometry: - SRS: EPSG:4326 - type: POINT - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### FDNY Firehouse Listing - Location of Firehouses and companies - url: "https://data.cityofnewyork.us/Public-Safety/FDNY-Firehouse-Listing/hc8x-tcnd" - dependents: [] diff --git a/dcpy/library/templates/lpc_historic_district_areas.yml b/dcpy/library/templates/lpc_historic_district_areas.yml deleted file mode 100644 index 3093b7c1a..000000000 --- a/dcpy/library/templates/lpc_historic_district_areas.yml +++ /dev/null @@ -1,35 +0,0 @@ -dataset: - name: lpc_historic_district_areas - acl: public-read - source: - socrata: - uid: skyk-mpzq - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: NONE - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### LPC Individual Landmark and Historic District Building Database - This dataset contains boundaries and associated attribute information for all - designated historic districts or areas under consideration for historic district - designation (i.e. calendared) by the New York City Landmarks Preservation - Commission (LPC), including items that may have been denied designation or - overturned. - url: "https://data.cityofnewyork.us/Housing-Development/Historic-Districts/xbvj-gfnw" - dependents: [] diff --git a/dcpy/library/templates/lpc_historic_districts.yml b/dcpy/library/templates/lpc_historic_districts.yml deleted file mode 100644 index 46877c311..000000000 --- a/dcpy/library/templates/lpc_historic_districts.yml +++ /dev/null @@ -1,37 +0,0 @@ -dataset: - name: lpc_historic_districts - acl: public-read - source: - socrata: - uid: gpmc-yuvp - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: NONE - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### LPC Individual Landmark and Historic District Building Database - The LPC’s Individual Landmark and Historic District Building Database - contains building-by-building information from the more than 50 years - of the Commission’s individual landmark and historic district designation - reports. The data set contains detailed information on close to 36,000 - buildings or sites, including the nearly 34,000 historic buildings - within the city’s 141 historic districts and the 1,408 individual landmarks - (note: some individual landmarks include multiple buildings). - url: "https://data.cityofnewyork.us/Housing-Development/LPC-Individual-Landmark-and-Historic-District-Buil/7mgd-s57w" - dependents: [] diff --git a/dcpy/library/templates/lpc_landmarks.yml b/dcpy/library/templates/lpc_landmarks.yml deleted file mode 100644 index 8f258d715..000000000 --- a/dcpy/library/templates/lpc_landmarks.yml +++ /dev/null @@ -1,32 +0,0 @@ -dataset: - name: lpc_landmarks - acl: public-read - source: - socrata: - uid: ncre-qhxs - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: NONE - - destination: - geometry: - SRS: EPSG:4326 - type: POINT - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: | - ### LPC Landmark Sites - This dataset contains site boundaries for all designated individual - landmarks as well as administrative information such as site boundary description and designation date. - url: "https://data.cityofnewyork.us/Housing-Development/Individual-Landmark-Sites/ncre-qhxs" - dependents: [] diff --git a/dcpy/library/templates/lpc_scenic_landmarks.yml b/dcpy/library/templates/lpc_scenic_landmarks.yml deleted file mode 100644 index 0029a3a63..000000000 --- a/dcpy/library/templates/lpc_scenic_landmarks.yml +++ /dev/null @@ -1,31 +0,0 @@ -dataset: - name: lpc_scenic_landmarks - acl: public-read - source: - socrata: - uid: qexa-qpj6 - format: csv - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - - "GEOM_POSSIBLE_NAMES=the_geom" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - fields: [] - sql: null - - info: - description: > - Provides boundary information regarding all scenic landmarks that have been calendared, - heard, or designated by the Landmarks Preservation Commission, or for which an LP number has been issued. - url: "https://data.cityofnewyork.us/Housing-Development/Scenic-Landmarks/qexa-qpj6/about_data" - dependents: [] diff --git a/dcpy/library/templates/nypd_policeprecincts.yml b/dcpy/library/templates/nypd_policeprecincts.yml deleted file mode 100644 index 706f626ee..000000000 --- a/dcpy/library/templates/nypd_policeprecincts.yml +++ /dev/null @@ -1,31 +0,0 @@ -dataset: - name: nypd_policeprecincts - acl: public-read - source: - socrata: - uid: 78dh-3ptz - format: geojson - options: - - "AUTODETECT_TYPE=NO" - - "EMPTY_STRING_AS_NULL=YES" - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - - destination: - geometry: - SRS: EPSG:4326 - type: MULTIPOLYGON - options: - - "OVERWRITE=YES" - - "PRECISION=NO" - - "GEOMETRY=AS_WKT" - fields: [] - sql: null - - info: - description: | - ### NYPD Police Precincts - GIS data: Boundaries of Police Precincts - url: "https://data.cityofnewyork.us/Public-Safety/Police-Precincts/78dh-3ptz" - dependents: [] diff --git a/dcpy/lifecycle/ingest/templates/dcp_edesignation_csv.yml b/dcpy/lifecycle/ingest/templates/dcp_edesignation_csv.yml new file mode 100644 index 000000000..3ec93a0fc --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dcp_edesignation_csv.yml @@ -0,0 +1,62 @@ +id: dcp_edesignation_csv +acl: public-read + +attributes: + name: NYC (E) Designations + description: | + (E) Designations are established to provide notice of the presence of an environmental + requirement pertaining to potential hazardous materials contamination, high ambient noise + levels or air emission concerns on a particular tax lot. This includes CEQR Environment + Requirements and CEQR Restrictive Declarations, in Appendix C of the + Zoning Resolution. + + Not all columns are currently included in shapefile, hence why we need this table. + This should be fixed sometime before 2024 Q3, and at that point this template can be archived + url: https://www1.nyc.gov/assets/planning/download/pdf/data-maps/open-data/nyedes_metadata.pdf + +ingestion: + source: + type: socrata + org: nyc + uid: hxm3-23vy + format: csv + file_format: + type: csv + dtype: str + processing_steps: + - name: clean_column_names + args: { "lower": True } + +columns: + - id: enumber + data_type: text + - id: e_effective_date + data_type: text + - id: borocode + data_type: text + - id: taxblock + data_type: text + - id: taxlot + data_type: text + - id: hazmat_code + data_type: text + - id: air_code + data_type: text + - id: noise_code + data_type: text + - id: hazmat_date + data_type: text + - id: air_date + data_type: text + - id: noise_date + data_type: text + - id: ceqr_num + data_type: text + - id: ulurp_num + data_type: text + - id: zoning_map + data_type: text + - id: description + data_type: text + - id: bbl + data_type: text diff --git a/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints.yml b/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints.yml new file mode 100644 index 000000000..d6cbea3ce --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints.yml @@ -0,0 +1,47 @@ +id: doitt_buildingfootprints +acl: public-read + +attributes: + name: Building Footprints + description: | + Shapefile of footprint outlines of buildings in New York City. + Please see the following link for additional documentation- + https://github.com/CityOfNewYork/nyc-geo-metadata/blob/master/Metadata/Metadata_BuildingFootprints.md + + Previously posted versions of the data are retained to comply with + Local Law 106 of 2015 and can be provided upon request made to Open Data. + url: https://data.cityofnewyork.us/Housing-Development/Building-Footprints/nqwf-w8eh + +ingestion: + source: + type: socrata + org: nyc + uid: qb5r-6dgf + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: coerce_column_types + args: + column_types: { + "bin": "integer", + "heightroof": "numeric", + "cnstrct_yr": "integer", + "feat_code": "numeric", + "groundelev": "numeric" + } + errors: coerce + - name: multi + +columns: +- id: base_bbl + data_type: text +- id: mpluto_bbl + data_type: text +- id: bin + data_type: integer +- id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints_historical.yml b/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints_historical.yml new file mode 100644 index 000000000..7c98c8127 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/doitt_buildingfootprints_historical.yml @@ -0,0 +1,36 @@ +id: doitt_buildingfootprints_historical +acl: public-read + +attributes: + name: Building Footprints Historical Shape + description: | + Shapefile of historical footprint outlines of buildings in New York City. + Please see the following link for additional documentation- + https://github.com/CityOfNewYork/nyc-planimetrics/blob/master/Capture_Rules.md. + + Previously posted versions of the data are retained to comply with Local Law 106 + of 2015 and can be provided upon request made to Open Data. + + NOTE: The Socrata UID is not the same as the url (below). We want UID: ipkp-snf6 which is the + multipolygon geojson layer. There are two layers of data on the opendata portal: + one contains the multipolygon (building_historic) and the other contains points only (building_historic_p). + url: https://data.cityofnewyork.us/Housing-Development/Building-Footprints-Historical-Shape/s5zg-yzea + +ingestion: + source: + type: socrata + org: nyc + uid: ipkp-snf6 + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: bin + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dot_projects_intersections.yml b/dcpy/lifecycle/ingest/templates/dot_projects_intersections.yml new file mode 100644 index 000000000..7c60e48e1 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dot_projects_intersections.yml @@ -0,0 +1,38 @@ +id: dot_projects_intersections +acl: public-read + +attributes: + name: Street and Highway Capital Reconstruction Projects - Intersection + description: | + This data is a spatial representation of street construction projects. Street and Highway + capital projects are major street reconstruction projects, ranging from general street + resurfacing projects to full reconstruction of the roadbed, sidewalks, sewer and water + pipes and other utilities. Capital projects are essential to keep the City's infrastructure in + a state of good repair. + + Note: this is a private dataset. + url: https://data.cityofnewyork.us/Transportation/Street-and-Highway-Capital-Reconstruction-Projects/97nd-ff3i + +ingestion: + source: + type: socrata + org: nyc + uid: 97nd-ff3i + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { + "fmsagencyi": "fmsagencyid", + "geom": "wkb_geometry" + } + +columns: +- id: fmsagencyid + data_type: text +- id: fmsid + data_type: text +- id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dot_projects_streets.yml b/dcpy/lifecycle/ingest/templates/dot_projects_streets.yml new file mode 100644 index 000000000..d7c41dc98 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dot_projects_streets.yml @@ -0,0 +1,38 @@ +id: dot_projects_streets +acl: public-read + +attributes: + name: Street and Highway Capital Reconstruction Projects - Block + description: | + This data is a spatial representation of street construction projects. Street and Highway + capital projects are major street reconstruction projects, ranging from general street + resurfacing projects to full reconstruction of the roadbed, sidewalks, sewer and water + pipes and other utilities. Capital projects are essential to keep the City's infrastructure in + a state of good repair. + + Note: this is a private dataset. + url: https://data.cityofnewyork.us/Transportation/Street-and-Highway-Capital-Reconstruction-Projects/jvk9-k4re + +ingestion: + source: + type: socrata + org: nyc + uid: jvk9-k4re + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { + "fmsagencyi": "fmsagencyid", + "geom": "wkb_geometry" + } + +columns: +- id: fmsagencyid + data_type: text +- id: fmsid + data_type: text +- id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_capitalprojects.yml b/dcpy/lifecycle/ingest/templates/dpr_capitalprojects.yml new file mode 100644 index 000000000..dceae8b0c --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_capitalprojects.yml @@ -0,0 +1,63 @@ +id: dpr_capitalprojects +acl: public-read + +attributes: + name: DPR Capital Projects + description: | + check [here](https://www.nycgovparks.org/bigapps/desc/DPR_CapitalProjectTracker_001.txt) for metadata + url: https://www.nycgovparks.org/bigapps + +ingestion: + source: + type: file_download + url: https://www.nycgovparks.org/bigapps/DPR_CapitalProjectTracker_001.json + + file_format: + type: json + json_read_fn: normalize + json_read_kwargs: + { + "record_path": ["Locations", "Location"], + "meta": ["TrackerID", "FMSID", "Title", "TotalFunding"], + } + geometry: + crs: EPSG:4326 + geom_column: + x: Longitude + y: Latitude + + processing_steps: + - name: rename_columns + args: + map: + { + "TrackerID": "proj_id", + "FMSID": "fmsid", + "Title": "desc", + "TotalFunding": "total_funding", + "ParkID": "park_id", + "Latitude": "lat", + "Longitude": "lon", + "geom": "wkb_geometry", + } + - name: coerce_column_types + args: + column_types: { "proj_id": "string" } + +columns: + - id: park_id + data_type: text + - id: lat + data_type: decimal + - id: lon + data_type: decimal + - id: proj_id + data_type: text + - id: fmsid + data_type: text + - id: desc + data_type: text + - id: total_funding + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_forever_wild.yml b/dcpy/lifecycle/ingest/templates/dpr_forever_wild.yml new file mode 100644 index 000000000..882fd52ba --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_forever_wild.yml @@ -0,0 +1,38 @@ +id: dpr_forever_wild +acl: public-read + +attributes: + name: NYC Parks Forever Wild + description: The Forever Wild layer delineates the location of ecologically + important natural resources within NYC Parks property. Approximately 12,500 + acres of NYC Parks' 30,000 acres of land is comprised of natural areas - + forests, salt marshes, open water, grasslands, streams, and freshwater wetlands. + The vast majority of these natural areas are accessible to residents and visitors. + These areas provide valuable habitat for wildlife and a variety of environmental + services to all New Yorkers including carbon sequestration, air quality improvement, + cooling, and stormwater capture. + url: https://data.cityofnewyork.us/Environment/NYC-Parks-Forever-Wild/48va-85tp/about_data + +ingestion: + target_crs: EPSG:2263 + source: + type: socrata + org: nyc + uid: 48va-85tp + format: shapefile + file_format: + type: shapefile + crs: EPSG:4326 + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: multi + +columns: + - id: gispropnum + data_type: text + - id: propertyna + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_greenthumb.yml b/dcpy/lifecycle/ingest/templates/dpr_greenthumb.yml new file mode 100644 index 000000000..50d461827 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_greenthumb.yml @@ -0,0 +1,32 @@ +id: dpr_greenthumb +acl: public-read + +attributes: + name: Department of Parks and Recreation – GreenThumb Garden Info + description: | + GreenThumb provides programming and material support to over 550 community gardens in New York City. + The data contains garden information and is part of the GreenThumb Gardens Data Collection. + url: https://data.cityofnewyork.us/dataset/GreenThumb-Garden-Info/p78i-pat6 + +ingestion: + source: + type: socrata + org: nyc + uid: p78i-pat6 + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: bbl + data_type: text + - id: gardenname + data_type: text + - id: borough + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_park_access_zone.yml b/dcpy/lifecycle/ingest/templates/dpr_park_access_zone.yml new file mode 100644 index 000000000..0b6529c09 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_park_access_zone.yml @@ -0,0 +1,48 @@ +id: dpr_park_access_zone +acl: public-read + +attributes: + name: Walk-to-a-Park Service area + description: | + The Walk to a Park initiative focuses on increasing access to parks and open + spaces, concentrating on areas of the city that are under-resourced and where + residents are living further than a walk to a park. NYC Parks calculates the + number of New Yorkers within walking distance of a park and reports on this as + part of the Mayor’s Management Report. “Walking distance” is defined as a 1/4-mile + or less for sites such as small playgrounds and sitting areas; or a 1/2-mile or + less for larger parks that serve a wider region, typically over 8 acres or situated on + the waterfront. Certain properties in NYC Parks' portfolio, such as cemeteries, + community gardens, or sites with no recreational equipment were not included in + this analysis. Similarly, some parks and open-space amenities not under the + jurisdiction of NYC Parks were included in this analysis, as they provide + recreational value. + + This dataset includes two ESRI shapefiles of access points, one for sites that get + a 1/4-mile buffer and one for sites that get a 1/2-mile buffer. This polygon represents + the total area within walking distance of a park. To generate the number of + New Yorkers within that distance, the polygon is compared with data from the + U.S. Census. + + This information is only current as of the publication date. For more information + about this analysis and the Walk to a Park Initiative, visit: + https://www.nycgovparks.org/planning-and-building/planning/walk-to-a-park + url: https://data.cityofnewyork.us/Recreation/1_4MilePoints/99ii-hwh9/about_data + +ingestion: + source: + type: socrata + org: nyc + uid: 99ii-hwh9 + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: id + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_parksproperties.yml b/dcpy/lifecycle/ingest/templates/dpr_parksproperties.yml new file mode 100644 index 000000000..86990764e --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_parksproperties.yml @@ -0,0 +1,47 @@ +id: dpr_parksproperties +acl: public-read + +attributes: + name: Parks Properties + description: | + This dataset identifies property managed partially or solely by NYC Parks. + This data has been produced in whole or part using secondary data. + Data accuracy is limited by the scale and accuracy of the original sources. + Site-specific conditions should be field-verified. + Records are added as more land is designated under NYC Parks’ jurisdiction. + Each record represents an acquisition. + User Guide: https://docs.google.com/document/d/1NExNJF5YKID04oOopi0fHainRuGG3Pz_jKSrMujPsPk/edit?usp=sharing + Data Dictionary: https://docs.google.com/spreadsheets/d/1Q4DBWu7riNFxWvy1vnTJHoOI3r2L9oW6eCN56jCNyCw/edit?usp=sharing + url: https://data.cityofnewyork.us/Recreation/Parks-Properties/enfh-gkve + +ingestion: + source: + type: socrata + org: nyc + uid: enfh-gkve + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: multi + +columns: + - id: zipcode + data_type: text + - id: typecategory + data_type: text + - id: gispropnum + data_type: text + - id: name311 + data_type: text + - id: signname + data_type: text + - id: address + data_type: text + - id: borough + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dpr_schoolyard_to_playgrounds.yml b/dcpy/lifecycle/ingest/templates/dpr_schoolyard_to_playgrounds.yml new file mode 100644 index 000000000..5b2c1ab6d --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dpr_schoolyard_to_playgrounds.yml @@ -0,0 +1,55 @@ +id: dpr_schoolyard_to_playgrounds +acl: public-read + +attributes: + name: Schoolyard to Playgrounds + description: | + The Schoolyards to Playgrounds initiative began with PlaNYC 2007 as + a way to expand public use at existing City facilities and improve access + to open spaces during non-school hours which includes weekends, holidays, + and during the summer. + + NYC Parks, in collaboration with the Department of Education (DOE) and the nonprofit + Trust for Public Land \nhave improved hundreds of DOE operated schoolyards through + capital improvements, including play equipment, painted sports surfaces, trees, and + benches. + + The program is an important approach to reaching the city’s target to getting more New + Yorkers within walking distance to open-space. In many neighborhoods, these DOE + schoolyards are one of the few resources that can be improved and provide publicly + accessible open space. + + Each record in this dataset represents a participating DOE Schoolyard at the time of the + dataset's publication. The schoolyard's boundaries have been drawn in whole or part + using secondary data. Data accuracy is limited by the scale and accuracy of the original + sources. Site-specific conditions should be field-verified. + + For more information on the Schoolyard to Playgrounds program, visit: + https://www.nycgovparks.org/greening/planyc/schoolyards + + This dataset may be updated throughout the year as school's join or exit the program. + + Data Dictionary: https://docs.google.com/spreadsheets/d/10yVRhmdIAS1g8c3L0jC4XxSTptdIf5wESL5ks1e6Gto/edit?usp=sharing\n" + url: https://data.cityofnewyork.us/dataset/Schoolyard-to-Playgrounds/bbtf-6p3c/about_data + +ingestion: + target_crs: EPSG:2263 + source: + type: socrata + org: nyc + uid: bbtf-6p3c + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: location + data_type: text + - id: gispropnum + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dsny_frequencies.yml b/dcpy/lifecycle/ingest/templates/dsny_frequencies.yml new file mode 100644 index 000000000..001691876 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dsny_frequencies.yml @@ -0,0 +1,28 @@ +id: dsny_frequencies +acl: public-read + +attributes: + name: DSNY Frequencies + description: | + Citywide DSNY frequency boundaries for collection operation (refuse, recycling, organics, bulk items). + These boundaries are sub-divisions of DSNY sections. + url: https://data.cityofnewyork.us/City-Government/DSNY-Frequencies/rv63-53db + +ingestion: + source: + type: socrata + org: nyc + uid: rv63-53db + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: schedulecode + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/dsny_garages.yml b/dcpy/lifecycle/ingest/templates/dsny_garages.yml new file mode 100644 index 000000000..af28d7055 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/dsny_garages.yml @@ -0,0 +1,43 @@ +id: dsny_garages +acl: public-read + +attributes: + name: DSNY Garages + description: | + Location of DSNY garages. Essential to DSNY operations are our 59 garages across the five boroughs. + This structure enables the Department’s uniformed personnel to efficiently perform regular duties – and + quickly execute emergency response. + url: https://data.cityofnewyork.us/Environment/DSNY-Garages/xw3j-2yxf + +ingestion: + source: + type: socrata + org: nyc + uid: xw3j-2yxf + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: city + data_type: text + - id: name + data_type: text + - id: zip + data_type: text + - id: bbl + data_type: text + - id: address + data_type: text + - id: boro + data_type: text + - id: bin + data_type: text + - id: type + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/fdny_firehouses.yml b/dcpy/lifecycle/ingest/templates/fdny_firehouses.yml new file mode 100644 index 000000000..c84870d9e --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/fdny_firehouses.yml @@ -0,0 +1,52 @@ +id: fdny_firehouses +acl: public-read + +attributes: + name: FDNY Firehouse Listing + description: Location of Firehouses and companies + url: https://data.cityofnewyork.us/Public-Safety/FDNY-Firehouse-Listing/hc8x-tcnd + +ingestion: + source: + type: socrata + org: nyc + uid: hc8x-tcnd + format: csv + file_format: + type: csv + dtype: str + geometry: + crs: EPSG:4326 + geom_column: + x: Longitude + y: Latitude + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: clean_column_names + args: { "lower": True } + - name: coerce_column_types + args: + column_types: { "longitude": "numeric", "latitude": "numeric" } + errors: coerce + +columns: + - id: facilityname + data_type: text + - id: facilityaddress + data_type: text + - id: borough + data_type: text + - id: postcode + data_type: text + - id: latitude + data_type: decimal + - id: longitude + data_type: decimal + - id: bin + data_type: text + - id: bbl + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/lpc_historic_district_areas.yml b/dcpy/lifecycle/ingest/templates/lpc_historic_district_areas.yml new file mode 100644 index 000000000..2fd557188 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/lpc_historic_district_areas.yml @@ -0,0 +1,36 @@ +id: lpc_historic_district_areas +acl: public-read + +attributes: + name: Historic Districts + description: | + Provides information regarding all historic districts that have calendared, + heard, or designated by the Landmarks Preservation Commission, or for which + an LP number has been issued, as well as many proposed districts in the study + area phase. + url: https://data.cityofnewyork.us/Housing-Development/Historic-Districts/skyk-mpzq + +ingestion: + source: + type: socrata + org: nyc + uid: skyk-mpzq + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: coerce_column_types + args: + column_types: { "caldate": "date", "desdate": "date" } + errors: coerce + +columns: + - id: lp_number + data_type: text + - id: area_name + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/lpc_historic_districts.yml b/dcpy/lifecycle/ingest/templates/lpc_historic_districts.yml new file mode 100644 index 000000000..3f60ad3a7 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/lpc_historic_districts.yml @@ -0,0 +1,33 @@ +id: lpc_historic_districts +acl: public-read + +attributes: + name: Landmarks Preservation Commission - Historic Districts + description: | + The LPC’s Individual Landmark and Historic District Building Database + contains building-by-building information from the more than 50 years + of the Commission’s individual landmark and historic district designation + reports. The data set contains detailed information on close to 36,000 + buildings or sites, including the nearly 34,000 historic buildings + within the city’s 141 historic districts and the 1,408 individual landmarks + (note: some individual landmarks include multiple buildings). + url: https://data.cityofnewyork.us/Housing-Development/Individual-Landmark-and-Historic-District-Building/gpmc-yuvp + +ingestion: + source: + type: socrata + org: nyc + uid: gpmc-yuvp + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: bbl + data_type: text + - id: hist_dist + data_type: text diff --git a/dcpy/lifecycle/ingest/templates/lpc_landmarks.yml b/dcpy/lifecycle/ingest/templates/lpc_landmarks.yml new file mode 100644 index 000000000..d27e59157 --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/lpc_landmarks.yml @@ -0,0 +1,61 @@ +id: lpc_landmarks +acl: public-read + +attributes: + name: Landmarks Preservation Commission - Individual Landmarks + description: | + Provides information on all items designated or under consideration for designation + (i.e. calendared) by the New York City Landmarks Preservation Commission (LPC). + + The dataset contains records for each individual, scenic, or interior landmark, as well as + properties or sites located within the boundaries of historic districts. + + Please note that points in this dataset represent individual buildings in addition to + non-building sites (such as vacant lots or monuments) regulated by LPC. It is possible + for a single property to have multiple designations (such as individual and interior + designations, or individual and historic district). For this reason, it is not uncommon + to see multiple points on a single tax lot and multiple records for a single property within + the database. + url: https://data.cityofnewyork.us/Housing-Development/Individual-Landmark-Sites/ncre-qhxs + +ingestion: + source: + type: socrata + org: nyc + uid: ncre-qhxs + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: coerce_column_types + args: + column_types: { "caldate": "date", "desdate": "date" } + errors: coerce + - name: pd_series_func + args: + column_name: desig_addr + function_name: str.replace + pat: "\r\n" + repl: "\n" + regex: True + +columns: + - id: last_actio + data_type: text + - id: bbl + data_type: text + - id: status + data_type: text + - id: lm_type + data_type: text + - id: most_curre + data_type: text + - id: boroughid + data_type: text + - id: lm_name + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/lpc_scenic_landmarks.yml b/dcpy/lifecycle/ingest/templates/lpc_scenic_landmarks.yml new file mode 100644 index 000000000..1a0feb4ba --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/lpc_scenic_landmarks.yml @@ -0,0 +1,36 @@ +id: lpc_scenic_landmarks +acl: public-read + +attributes: + name: LPC Scenic Landmarks + description: | + Provides boundary information regarding all scenic landmarks that + have been calendared, heard, or designated by the Landmarks Preservation Commission, + or for which an LP number has been issued. + url: https://data.cityofnewyork.us/Housing-Development/Scenic-Landmarks/qexa-qpj6/about_data + +ingestion: + source: + type: socrata + org: nyc + uid: qexa-qpj6 + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + - name: coerce_column_types + args: + column_types: { "caldate": "date", "desdate": "date" } + errors: coerce + - name: multi + +columns: + - id: lp_number + data_type: text + - id: scen_lm_na + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/lifecycle/ingest/templates/nypd_policeprecincts.yml b/dcpy/lifecycle/ingest/templates/nypd_policeprecincts.yml new file mode 100644 index 000000000..1fcde1d9a --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/nypd_policeprecincts.yml @@ -0,0 +1,26 @@ +id: nypd_policeprecincts +acl: public-read + +attributes: + name: NYPD Police Precincts + description: "GIS data: Boundaries of Police Precincts" + url: https://data.cityofnewyork.us/Public-Safety/Police-Precincts/78dh-3ptz + +ingestion: + source: + type: socrata + org: nyc + uid: 78dh-3ptz + format: geojson + file_format: + type: geojson + processing_steps: + - name: rename_columns + args: + map: { "geom": "wkb_geometry" } + +columns: + - id: precinct + data_type: text + - id: wkb_geometry + data_type: geometry diff --git a/dcpy/test/library/test_ingest_script.py b/dcpy/test/library/test_ingest_script.py index eaf0433bf..a80ea6f11 100644 --- a/dcpy/test/library/test_ingest_script.py +++ b/dcpy/test/library/test_ingest_script.py @@ -11,14 +11,6 @@ def test_bpl_libraries(): assert os.path.isfile(".library/datasets/bpl_libraries/test/bpl_libraries.csv") -def test_dpr_capitalprojects(): - ingestor = Ingestor() - ingestor.csv(f"{template_path}/dpr_capitalprojects.yml", version="test") - assert os.path.isfile( - ".library/datasets/dpr_capitalprojects/test/dpr_capitalprojects.csv" - ) - - def test_nypl_libraries(): ingestor = Ingestor() ingestor.csv(f"{template_path}/nypl_libraries.yml", version="test") diff --git a/products/green_fast_track/models/_sources.yml b/products/green_fast_track/models/_sources.yml index dd0b16458..018f0a5a7 100644 --- a/products/green_fast_track/models/_sources.yml +++ b/products/green_fast_track/models/_sources.yml @@ -126,12 +126,11 @@ sources: columns: - name: bbl - name: last_actio - - name: latitude - name: lm_name - name: lm_type - - name: longitude - name: most_curre - name: status + - name: wkb_geometry - name: lpc_scenic_landmarks columns: @@ -524,4 +523,4 @@ sources: - unique - name: coastal_hazard_area_flag - name: fresh_water_wetlands_flag - - name: tidal_coastal_wetlands_flag + - name: tidal_coastal_wetlands_flag