Skip to content

Commit

Permalink
make dag_type_from_filepath more robust
Browse files Browse the repository at this point in the history
  • Loading branch information
7yl4r committed May 23, 2018
1 parent 102b19e commit 13d4bcd
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 12 deletions.
22 changes: 18 additions & 4 deletions util/DAGType.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,25 @@
A list of DAG "types" identified for organizational purposes as described
[here]( https://github.com/USF-IMARS/imars_dags/issues/49 )
"""
import os

class DAGType:
PROCESSING='proc'
INGEST='ingest'
FILE_TRIGGER='file_trigger'
# NOTE: these must match directories for dag_type_from_filepath() to work
PROCESSING = 'processing'
INGEST = 'ingest'
FILE_TRIGGER = 'file_triggers'

@staticmethod
def all():
return [DAGType.PROCESSING,DAGType.INGEST,DAGType.FILE_TRIGGER]
return [
DAGType.PROCESSING,
DAGType.INGEST,
DAGType.FILE_TRIGGER
]

def dag_type_from_filepath(dag_filepath):
for d_type in DAGType.all():
if os.path.join("",d_type,"") in dag_filepath:
return d_type
else:
raise ValueError("could not determine dag_type from filepath")
9 changes: 1 addition & 8 deletions util/get_dag_id.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from imars_dags.util.DAGType import DAGType
from imars_dags.util.DAGType import DAGType, dag_type_from_filepath

def get_dag_id(filepath=None, region=None, dag_type=None, dag_name=None):
"""
Expand Down Expand Up @@ -29,12 +29,5 @@ def get_dag_id(filepath=None, region=None, dag_type=None, dag_name=None):

return dag_id

def dag_type_from_filepath(dag_filepath):
for d_type in DAGType.all():
if d_type in dag_filepath:
return d_type
else:
raise ValueError("could not determine dag_type from filepath")

def dag_name_from_filepath(dag_filepath):
return os.path.basename(dag_filepath).replace('.py','')

0 comments on commit 13d4bcd

Please sign in to comment.