CCDIDC-1504

prefect.yaml

@@ -879,4 +879,40 @@ deployments:
     work_pool:
       name: ccdi-curation-8gb
       work_queue_name: null
-      job_variables: {}  
+      job_variables: {}  
+
+  - name: ccdi-gdc-file-upload
+    version: null
+    tags: [ "TEST" ]
+    description: null
+    entrypoint: workflows/gdc_file_upload_gdc_client.py:runner
+    schedule: null
+
+    # flow-specific parameters
+    parameters:
+      bucket: "ccdi-validation"
+      project_id: "CCDI-MCI"
+      manifest_path: ""
+      gdc_client_path: ""
+      runner: ""
+      secret_key_name: "gdc-token"
+      upload_part_size_mb: 1
+      max_n_processes: 4
+
+
+    # pull action to overwrite from file pull action
+    pull:
+    - prefect.projects.steps.git_clone_project:
+        id: clone-step
+        repository: https://github.com/CBIIT/ChildhoodCancerDataInitiative-Prefect_Pipeline.git
+        branch: CCDIDC-1504_GDC_File_Upload
+    - prefect.projects.steps.pip_install_requirements:
+        requirements_file: requirements.txt
+        directory: "{{ clone-step.directory }}"
+        stream_output: False
+
+    # infra-specific fields
+    work_pool:
+      name: ccdi-curation-32gb
+      work_queue_name: null
+      job_variables: {}

src/utils.py

@@ -1382,3 +1382,17 @@ def ccdi_to_dcf_index(ccdi_manifest: str) -> tuple:
     del combined_df
 
     return output_filename, log_name
+
+def sanitize_return(err_string: str, remove_value_list: list):
+    """Sanitize a string of provided substrings to remove. 
+
+    Args:
+        err_string (str): String to remove any secret or sensitive substrings from
+        remove_value_list (list): List of substrings to remove
+
+    Returns:
+        str: Sanitized string
+    """
+    for remove_value in remove_value_list:
+        err_string = err_string.replace(remove_value, "")
+    return err_string