From 13d4bcdd7c39a28af3da9bfb40904d3715e011d6 Mon Sep 17 00:00:00 2001 From: Tylar Murray Date: Wed, 23 May 2018 20:17:43 +0000 Subject: [PATCH] make dag_type_from_filepath more robust --- util/DAGType.py | 22 ++++++++++++++++++---- util/get_dag_id.py | 9 +-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/util/DAGType.py b/util/DAGType.py index a4d95ab..d715f0a 100644 --- a/util/DAGType.py +++ b/util/DAGType.py @@ -2,11 +2,25 @@ A list of DAG "types" identified for organizational purposes as described [here]( https://github.com/USF-IMARS/imars_dags/issues/49 ) """ +import os + class DAGType: - PROCESSING='proc' - INGEST='ingest' - FILE_TRIGGER='file_trigger' + # NOTE: these must match directories for dag_type_from_filepath() to work + PROCESSING = 'processing' + INGEST = 'ingest' + FILE_TRIGGER = 'file_triggers' @staticmethod def all(): - return [DAGType.PROCESSING,DAGType.INGEST,DAGType.FILE_TRIGGER] + return [ + DAGType.PROCESSING, + DAGType.INGEST, + DAGType.FILE_TRIGGER + ] + +def dag_type_from_filepath(dag_filepath): + for d_type in DAGType.all(): + if os.path.join("",d_type,"") in dag_filepath: + return d_type + else: + raise ValueError("could not determine dag_type from filepath") diff --git a/util/get_dag_id.py b/util/get_dag_id.py index 763bb7c..eee2f4c 100644 --- a/util/get_dag_id.py +++ b/util/get_dag_id.py @@ -1,6 +1,6 @@ import os -from imars_dags.util.DAGType import DAGType +from imars_dags.util.DAGType import DAGType, dag_type_from_filepath def get_dag_id(filepath=None, region=None, dag_type=None, dag_name=None): """ @@ -29,12 +29,5 @@ def get_dag_id(filepath=None, region=None, dag_type=None, dag_name=None): return dag_id -def dag_type_from_filepath(dag_filepath): - for d_type in DAGType.all(): - if d_type in dag_filepath: - return d_type - else: - raise ValueError("could not determine dag_type from filepath") - def dag_name_from_filepath(dag_filepath): return os.path.basename(dag_filepath).replace('.py','')