Uploading major changes for 0.0.74

README.md

@@ -1,13 +1,12 @@
-# Multi-Data Lineage System
+# ProvLake Lib
 
- IBM Research Multi-Data Lineage System is a provenance data management system capable of capturing, integrating, and querying provenance data generated across multiple, distributed services, programs, databases, and computational workflows.
+This is part of ProvLake Project. See [ibm.biz/provlake](http://ibm.biz/provlake) for more details.
 
-**For more information on the project, including participants and publications, please see [ibm.biz/provlake](http://ibm.biz/provlake).**
+### A Python lib to capture multiworkflow provenance data from Python Scripts
 
-This repository contains the Python library that captures provenance data in Python applications and send to the Multi-Data Lineage Manager, which is responsible for integrating the data in a provenance database stored as a knowledge graph (semantic detabase),
-then allowing users to run queries over the data.
+Use this library for code instrumentation to collect provenance data of function calls in a script. Input arguments or output values from functions can come from distributed data storages, including file systems and database systems.
 
-It supports Python>=3.6
+Python 3.6
 
 
 ### Very simple utilization example
@@ -17,7 +16,7 @@ from provlake.prov_lake import ProvLake
 from provlake.prov_task import ProvTask
 
 """
-Very simple example to show how this library is used to instrument a simple python script for provenance data management.
+Very simple example to show how ProvLake is used to instrument a simple python script for provenance data management.
 """
 
 
@@ -41,4 +40,5 @@ with ProvTask(prov, "factorial_number", in_args) as prov_task:
     prov_task.output(out_args)
 
 prov.close()
+
 ```

setup.py

@@ -6,11 +6,11 @@
 
 
 setup(name='provlake',
-      version='0.0.72',
+      version='0.0.74',
       description='A Python lib to capture multiworkflow provenance data from Python Scripts',
-      url='http://ibm.biz/provlake',
+      url='https://github.ibm.com/provlake/ProvLakePy',
       author='IBM Research',
-      license='Apache 2.0',
+      license='Internal use only / IBM only',
       install_requires=requires,
       package_dir={'': 'src'},
       packages=find_packages(where='src'),

src/provlake/prov_lake.py

@@ -13,7 +13,7 @@ def __init__(self,
                  prospective_provenance_dict: dict = None,
                  storage_configuration_path: str=None,
                  storage_configuration_dict: dict=None,
-                 dataflow_name: str=None,
+                 workflow_name: str=None,
                  context: str=None,
                  insert_prospective=False,
                  with_validation: bool=False,
@@ -71,10 +71,10 @@ def __init__(self,
         self.last_task_id = 0
         self.wf_start_time = time()
 
-        self.df_name = dataflow_name or self.df_structure.get("dataflow_name", "NI")
+        self.df_name = workflow_name or self.df_structure.get("dataflow_name", "NI")
 
         self.tasks = dict()
-        self.wf_execution = "wfexec_" + str(self.wf_start_time)
+        self.wf_execution = self.wf_start_time
         self.wf_obj = {
             "wf_execution":  self.wf_execution,
             "startTime": self.wf_start_time
@@ -96,7 +96,6 @@ def __init__(self,
         if log_level == "NONE":
             log_level = "ERROR"
         log_lvl = getattr(logging, log_level.upper())
-        #logging.getLogger().setLevel(log_lvl)
         logger.setLevel(log_lvl)
 
         self.prov_persister = _ProvPersister(self.df_name, service_url=service_url, context=context, bag_size=bag_size,

src/provlake/stateless/cycle.py

@@ -1,7 +1,8 @@
 import traceback
 import logging
 from time import time
-from provlake import prov_utils
+from provlake.utils import prov_utils
+
 logger = logging.getLogger('PROV')
 
 

src/provlake/stateless/task.py

@@ -1,7 +1,8 @@
 import traceback
 import logging
 from time import time
-from provlake import prov_utils
+from provlake.utils import prov_utils
+
 logger = logging.getLogger('PROV')
 
 

src/provlake/stateless/workflow.py

@@ -1,5 +1,5 @@
 from time import time
-from provlake import prov_utils
+from provlake.utils import prov_utils
 import os
 
 

src/provlake/utils/args_handler.py

@@ -0,0 +1,36 @@
+def get_dict(_dict:dict) -> dict:
+    if _dict is None:
+        return None
+    if len(_dict) == 0:
+        return {}
+    return {
+        "type": "dict",
+        "values": _dict
+    }
+
+def get_list(_list:list) -> dict:
+    if _list is None:
+        return None
+    if len(_list) == 0:
+        return []
+    return {
+       "type": "list",
+        "values": _list
+    }
+
+def get_recursive_dicts(_dict:dict) -> dict:
+    if _dict is None:
+        return None
+    if len(_dict) == 0:
+        return {}
+    values = dict()
+    for k in _dict:
+        v = _dict[k]
+        if type(v) == dict:
+            values[k] = get_recursive_dicts(v)
+        else:
+            values[k] = v
+    return {
+        "type": "dict",
+        "values": values
+    }