#21 add support to load your own JSON schema

etl2osm/api.py

@@ -12,9 +12,9 @@ def process(infile, config, outfile, **kwargs):
     :param ``format``: Data output format [shp, geojson, osm].
     """
 
-    data = Extract(infile)
-    data.transform(config)
-    data.save(outfile)
+    data = Extract(infile, **kwargs)
+    data.transform(config, **kwargs)
+    data.save(outfile, **kwargs)
     return data
 
 

etl2osm/cli.py

@@ -8,7 +8,10 @@
 @click.argument('infile')
 @click.option('--config', '-c', help='Config file for column transformation.')
 @click.option('--outfile', '-o', help='Out file path to save.')
-def cli(infile, config, outfile):
+@click.option('--suffix', help='Filepath for suffix attribute function.')
+@click.option('--direction', help='Filepath for direction attribute function.')
+@click.option('--title_except', help='Filepath for title except attribute function.')
+def cli(infile, config, outfile, **kwargs):
     """Command Line Interface for ETL2OSM"""
 
-    etl2osm.process(infile, config, outfile)
+    etl2osm.process(infile, config, outfile, **kwargs)

etl2osm/extract.py

@@ -96,7 +96,6 @@ def read_geojson(self, infile, **kwargs):
         """Reads a GeoJSON and gives the results in GeoJSON format"""
 
         logging.info('Reading GeoJSON: %s' % infile)
-
         with open(infile) as f:
             geojson = json.load(f)
 
@@ -163,7 +162,7 @@ def read_osm(self, infile, **kwargs):
         logging.info('Reading OSM: %s' % infile)
         raise ValueError('Reading OSM not implemented')
 
-    def transform(self, config={}):
+    def transform(self, config={}, **kwargs):
         """ Transform the data using the config file """
 
         self.config = read_config(config)
@@ -174,11 +173,11 @@ def transform(self, config={}):
         for x, feature in enumerate(self.features):
             # Reproject data to WGS84
             if not self.epsg == 'ESPG:4326':
-                feature = reproject(feature, self.crs, osr.SRS_WKT_WGS84)
+                feature = reproject(feature, self.crs, osr.SRS_WKT_WGS84, **kwargs)
 
             # Transform Columns
             if self.config:
-                feature = transform_columns(feature, self.config)
+                feature = transform_columns(feature, self.config, **kwargs)
 
             # Save feature to self
             self[x] = feature

etl2osm/load.py

@@ -10,7 +10,7 @@
 
 
 class Load(object):
-    def save(self, outfile):
+    def save(self, outfile, **kwargs):
         """ Saves file to path """
 
         extension = os.path.splitext(outfile)[1][1:]

etl2osm/models/direction.json

@@ -0,0 +1,10 @@
+{
+    "N": "North",
+    "S": "South",
+    "E": "East",
+    "W": "West",
+    "NE": "Northeast",
+    "NW": "Northwest",
+    "SE": "Southeast",
+    "SW": "Southwest"
+}

etl2osm/models.py → etl2osm/models/suffix.json

@@ -1,27 +1,4 @@
-# -*- coding: utf-8 -*-
-
-cap_except = [
-    "'s",
-    "the",
-    "in",
-    "a",
-    "CR",
-    "US",
-    "SR"
-]
-
-direction = {
-    "N": "North",
-    "S": "South",
-    "E": "East",
-    "W": "West",
-    "NE": "Northeast",
-    "NW": "Northwest",
-    "SE": "Southeast",
-    "SW": "Southwest"
-}
-
-suffix = {
+{
     "ALY": "Alley",
     "ANX": "Annex",
     "ARC": "Arcade",
@@ -165,4 +142,4 @@
     "WALK": "Walk",
     "WAY": "Way",
     "WLS": "Wells"
-}
+}

etl2osm/models/title_except.json

@@ -0,0 +1,9 @@
+[
+    "'s",
+    "the",
+    "in",
+    "a",
+    "CR",
+    "US",
+    "SR"
+]

etl2osm/transform.py

@@ -4,15 +4,37 @@
 import re
 import json
 import logging
+import etl2osm
 from six import string_types, binary_type
 from collections import OrderedDict
 from osgeo import osr, ogr
-from etl2osm.models import suffix, direction, cap_except
 
 
 true_list = ['True', 'true', '1', True, 1]
 
 
+def load_json(model, **kwargs):
+    if model in kwargs:
+        model = kwargs[model]
+
+    # Model is already in JSON format
+    if isinstance(model, (dict, list, tuple)):
+        return model
+    else:
+        # Find user defined file path
+        if os.path.exists(model):
+            with open(model) as f:
+                return json.load(f)
+
+        # Look inside etl2osm [models] folder for .json files
+        root = os.path.dirname(etl2osm.__file__)[:-len('etl2osm')]
+        path = os.path.join(root, 'etl2osm', 'models', model + '.json')
+
+        if os.path.exists(path):
+            with open(path) as f:
+                return json.load(f)
+
+
 def regex_strip(value):
     # ESRI Shapfiles have fields [SUB] blank fields at the end
     # Regex will search all characters but must END with a letter or number
@@ -81,7 +103,7 @@ def extract_epsg(crs):
     return crs
 
 
-def reproject(feature, crs_source, crs_target=4326):
+def reproject(feature, crs_source, crs_target=4326, **kwargs):
     # Source Projection
     p1 = get_coordinate_rerefence_system(extract_epsg(crs_source))
 
@@ -158,13 +180,13 @@ def read_config(config):
         return json.load(f, object_pairs_hook=OrderedDict)
 
 
-def titlecase_except(value, exceptions=cap_except):
+def titlecase_except(value, **kwargs):
     if isinstance(value, (string_types, binary_type)):
         word_list = re.split(' ', value)
         final = []
 
         for word in word_list:
-            if word in exceptions:
+            if word in load_json('title_except', **kwargs):
                 final.append(word)
             else:
                 final.append(word.capitalize())
@@ -173,7 +195,7 @@ def titlecase_except(value, exceptions=cap_except):
     return value
 
 
-def clean_field(properties, conform):
+def clean_field(properties, conform, **kwargs):
     if 'field' in conform:
         # STRING
         if isinstance(conform, dict):
@@ -231,6 +253,8 @@ def clean_field(properties, conform):
 
         # Replaces the abreviated suffix (AVE=Avenue)
         elif 'suffix' in conform['function']:
+            suffix = load_json('suffix', **kwargs)
+
             if 'field' not in conform:
                 raise ValueError('[field] is missing using the Suffix Attribute Function.')
             if properties[field]:
@@ -241,6 +265,8 @@ def clean_field(properties, conform):
 
         # Replaces the abreviated directions (NE=Northeast)
         elif 'direction' in conform['function']:
+            direction = load_json('direction', **kwargs)
+
             if 'field' not in conform:
                 raise ValueError('[field] is missing using the Direction Attribute Function.')
             if properties[field]:
@@ -291,7 +317,7 @@ def clean_field(properties, conform):
     return value
 
 
-def transform_fields(properties, conform):
+def transform_fields(properties, conform, **kwargs):
     fields = OrderedDict()
     for key in conform.keys():
         value = None
@@ -301,28 +327,28 @@ def transform_fields(properties, conform):
         if isinstance(conform[key], (string_types, binary_type)):
             if conform[key] in properties:
                 value = properties[conform[key]]
-                value = clean_field(properties, conform[key])
+                value = clean_field(properties, conform[key], **kwargs)
             fields.update(dict([(key, value)]))
 
         # DICT
         # Replace & join multiple fields together
         elif isinstance(conform[key], (OrderedDict, dict)):
-            value = clean_field(properties, conform[key])
+            value = clean_field(properties, conform[key], **kwargs)
             fields.update(dict([(key, value)]))
 
         # LIST
         # Join a values from a list
         elif isinstance(conform[key], (list, tuple)):
-            value = clean_field(properties, conform[key])
+            value = clean_field(properties, conform[key], **kwargs)
             fields.update(dict([(key, value)]))
 
     return fields
 
 
-def transform_columns(feature, config):
+def transform_columns(feature, config, **kwargs):
     config = read_config(config)
     conform = config['conform']
-    feature['properties'] = transform_fields(feature['properties'], conform)
+    feature['properties'] = transform_fields(feature['properties'], conform, **kwargs)
 
     return feature
 

tests/models/direction.json

@@ -0,0 +1,10 @@
+{
+    "N": "North",
+    "S": "South",
+    "E": "East",
+    "W": "West",
+    "NE": "Northeast",
+    "NW": "Northwest",
+    "SE": "Southeast",
+    "SW": "Southwest"
+}