Merge pull request #11 from limnoliver/fix_wqp_pulls

Fix wqp pulls
USGS-R · Oct 9, 2019 · 02a423e · 02a423e
2 parents ee9d275 + 3e37453
commit 02a423e
Show file tree

Hide file tree

Showing 219 changed files with 563 additions and 483 deletions.
diff --git a/1_wqp_pull.yml b/1_wqp_pull.yml
@@ -17,13 +17,16 @@ sources:
 targets:
   1_wqp_pull:
     depends:
-      - 1_wqp_pull/log/1_wqp_pull_tasks.ind
+      - 1_wqp_pull/out/wqp_data.rds.ind
 
 # -- get a CONUS inventory of available data for download -- #
 
   wqp_pull_parameters:
     command: yaml.load_file('1_wqp_pull/cfg/wqp_pull_params.yml')
 
+  wqp_partition_cfg:
+    command: yaml.load_file('1_wqp_pull/cfg/wqp_partition_config.yml')
+
   # -- get inventory of observations available to download -- #
 
   # get inventory of samples available on WQP
@@ -44,26 +47,32 @@ targets:
       partitions_ind=target_name,
       wqp_pull_params = wqp_pull_parameters,
       inventory_ind='1_wqp_pull/inout/wqp_inventory.feather.ind',
-      wqp_partition_cfg=I('1_wqp_pull/cfg/wqp_partition_config.yml'))
+      wqp_partition_cfg=wqp_partition_cfg)
 
   1_wqp_pull/inout/wqp_pull_partitions.feather:
     command: gd_get('1_wqp_pull/inout/wqp_pull_partitions.feather.ind')
 
-# -- pull data! -- #
+  # -- pull data! -- #
 
   # prepare a remake-style plan for running each state as a separate
   # remake target in a separate remake file (1_wqp_wqp_tasks.yml)
   wqp_pull_plan:
     command: plan_wqp_pull(partitions_ind='1_wqp_pull/inout/wqp_pull_partitions.feather.ind')
 
   1_wqp_pull_tasks.yml:
-    command: create_wqp_pull_makefile(makefile=target_name, task_plan=wqp_pull_plan)
+    command: create_wqp_pull_makefile(makefile = target_name, 
+      task_plan = wqp_pull_plan,
+      final_targets = I('1_wqp_pull/out/wqp_data.rds.ind'))
 
   # run the data pulls
-  1_wqp_pull/log/1_wqp_pull_tasks.ind:
+  1_wqp_pull/out/wqp_data.rds.ind:
     command: loop_wqp_tasks(
       ind_file=target_name,
-      task_plan=wqp_pull_plan, task_makefile='1_wqp_pull_tasks.yml',
+      task_plan=wqp_pull_plan, 
+      task_makefile='1_wqp_pull_tasks.yml',
       num_tries=I(30), sleep_on_error=I(20))
-
-
+    depends:
+      - 1_wqp_pull/src/wqp_pull.R
+
+  1_wqp_pull/out/wqp_data.rds:
+    command: gd_get('1_wqp_pull/out/wqp_data.rds.ind')
diff --git a/1_wqp_pull/cfg/wqp_partition_config.yml b/1_wqp_pull/cfg/wqp_partition_config.yml
@@ -3,5 +3,5 @@
 target_inv_size: 1000
 
 # Approximate maximum number of records to pull in each call to WQP.
-# Recommended number is 500000 unless that causes exceedance of max URL size allowed by the WAF
+# Recommended number is 25000 unless that causes exceedance of max URL size allowed by the WAF
 target_pull_size: 250000
diff --git a/1_wqp_pull/inout/wqp_inventory.feather.ind b/1_wqp_pull/inout/wqp_inventory.feather.ind
@@ -1,2 +1,2 @@
-hash: c5dc6e02c065833fb32f72e2a9758ba8
+hash: c5f77841a450ad915d2cc3bb6019965f
 
diff --git a/1_wqp_pull/inout/wqp_pull_partitions.feather.ind b/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
@@ -1,2 +1,2 @@
-hash: c48e0661c9a0d542286d9153cf704ee1
+hash: 01c2b6436068c9d36a5be2995fd768d4
 
diff --git a/1_wqp_pull/out/190412172909_001.feather.ind b/1_wqp_pull/out/190412172909_001.feather.ind
diff --git a/1_wqp_pull/out/190412203340_001.feather.ind b/1_wqp_pull/out/190412203340_001.feather.ind
diff --git a/1_wqp_pull/out/190412203340_002.feather.ind b/1_wqp_pull/out/190412203340_002.feather.ind
diff --git a/1_wqp_pull/out/190412203340_003.feather.ind b/1_wqp_pull/out/190412203340_003.feather.ind
diff --git a/1_wqp_pull/out/190412203340_004.feather.ind b/1_wqp_pull/out/190412203340_004.feather.ind
diff --git a/1_wqp_pull/out/190412203340_005.feather.ind b/1_wqp_pull/out/190412203340_005.feather.ind
diff --git a/1_wqp_pull/out/190412203340_006.feather.ind b/1_wqp_pull/out/190412203340_006.feather.ind
diff --git a/1_wqp_pull/out/190412203340_007.feather.ind b/1_wqp_pull/out/190412203340_007.feather.ind
diff --git a/1_wqp_pull/out/190412203340_008.feather.ind b/1_wqp_pull/out/190412203340_008.feather.ind
diff --git a/1_wqp_pull/out/190412203340_009.feather.ind b/1_wqp_pull/out/190412203340_009.feather.ind
diff --git a/1_wqp_pull/out/190412203340_010.feather.ind b/1_wqp_pull/out/190412203340_010.feather.ind
diff --git a/1_wqp_pull/out/190412203340_011.feather.ind b/1_wqp_pull/out/190412203340_011.feather.ind
diff --git a/1_wqp_pull/out/190412203340_012.feather.ind b/1_wqp_pull/out/190412203340_012.feather.ind
diff --git a/1_wqp_pull/out/190412203340_013.feather.ind b/1_wqp_pull/out/190412203340_013.feather.ind
diff --git a/1_wqp_pull/out/190412203340_014.feather.ind b/1_wqp_pull/out/190412203340_014.feather.ind
diff --git a/1_wqp_pull/out/190412203340_015.feather.ind b/1_wqp_pull/out/190412203340_015.feather.ind
diff --git a/1_wqp_pull/out/190412203340_016.feather.ind b/1_wqp_pull/out/190412203340_016.feather.ind
diff --git a/1_wqp_pull/out/190412203340_017.feather.ind b/1_wqp_pull/out/190412203340_017.feather.ind
diff --git a/1_wqp_pull/out/190412203340_018.feather.ind b/1_wqp_pull/out/190412203340_018.feather.ind
diff --git a/1_wqp_pull/out/190412203340_019.feather.ind b/1_wqp_pull/out/190412203340_019.feather.ind
diff --git a/1_wqp_pull/out/190412203340_021.feather.ind b/1_wqp_pull/out/190412203340_021.feather.ind
diff --git a/1_wqp_pull/out/190412203340_022.feather.ind b/1_wqp_pull/out/190412203340_022.feather.ind
diff --git a/1_wqp_pull/out/190412203340_023.feather.ind b/1_wqp_pull/out/190412203340_023.feather.ind
diff --git a/1_wqp_pull/out/190412203340_024.feather.ind b/1_wqp_pull/out/190412203340_024.feather.ind
diff --git a/1_wqp_pull/out/190412203340_025.feather.ind b/1_wqp_pull/out/190412203340_025.feather.ind
diff --git a/1_wqp_pull/out/190412203340_026.feather.ind b/1_wqp_pull/out/190412203340_026.feather.ind
diff --git a/1_wqp_pull/out/190412203340_027.feather.ind b/1_wqp_pull/out/190412203340_027.feather.ind
diff --git a/1_wqp_pull/out/190412203340_028.feather.ind b/1_wqp_pull/out/190412203340_028.feather.ind
diff --git a/1_wqp_pull/out/190412203340_029.feather.ind b/1_wqp_pull/out/190412203340_029.feather.ind
diff --git a/1_wqp_pull/out/190412203340_030.feather.ind b/1_wqp_pull/out/190412203340_030.feather.ind
diff --git a/1_wqp_pull/out/190412203340_031.feather.ind b/1_wqp_pull/out/190412203340_031.feather.ind
diff --git a/1_wqp_pull/out/190412203340_032.feather.ind b/1_wqp_pull/out/190412203340_032.feather.ind
diff --git a/1_wqp_pull/out/190412203340_033.feather.ind b/1_wqp_pull/out/190412203340_033.feather.ind
diff --git a/1_wqp_pull/out/190412203340_034.feather.ind b/1_wqp_pull/out/190412203340_034.feather.ind
diff --git a/1_wqp_pull/out/190412203340_035.feather.ind b/1_wqp_pull/out/190412203340_035.feather.ind
diff --git a/1_wqp_pull/out/190412203340_036.feather.ind b/1_wqp_pull/out/190412203340_036.feather.ind
diff --git a/1_wqp_pull/out/190412203340_038.feather.ind b/1_wqp_pull/out/190412203340_038.feather.ind
diff --git a/1_wqp_pull/out/190412203340_039.feather.ind b/1_wqp_pull/out/190412203340_039.feather.ind
diff --git a/1_wqp_pull/out/190412203340_040.feather.ind b/1_wqp_pull/out/190412203340_040.feather.ind
diff --git a/1_wqp_pull/out/190412203340_041.feather.ind b/1_wqp_pull/out/190412203340_041.feather.ind
diff --git a/1_wqp_pull/out/190412203340_042.feather.ind b/1_wqp_pull/out/190412203340_042.feather.ind
diff --git a/1_wqp_pull/out/190412203340_043.feather.ind b/1_wqp_pull/out/190412203340_043.feather.ind
diff --git a/1_wqp_pull/out/190412203340_044.feather.ind b/1_wqp_pull/out/190412203340_044.feather.ind
diff --git a/1_wqp_pull/out/190412203340_045.feather.ind b/1_wqp_pull/out/190412203340_045.feather.ind
diff --git a/1_wqp_pull/out/190412203340_046.feather.ind b/1_wqp_pull/out/190412203340_046.feather.ind
diff --git a/1_wqp_pull/out/190412203340_047.feather.ind b/1_wqp_pull/out/190412203340_047.feather.ind
diff --git a/1_wqp_pull/out/190412203340_049.feather.ind b/1_wqp_pull/out/190412203340_049.feather.ind
diff --git a/1_wqp_pull/out/190412203340_050.feather.ind b/1_wqp_pull/out/190412203340_050.feather.ind
diff --git a/1_wqp_pull/out/190412203340_051.feather.ind b/1_wqp_pull/out/190412203340_051.feather.ind
diff --git a/1_wqp_pull/out/190412203340_052.feather.ind b/1_wqp_pull/out/190412203340_052.feather.ind
diff --git a/1_wqp_pull/out/190412203340_053.feather.ind b/1_wqp_pull/out/190412203340_053.feather.ind
diff --git a/1_wqp_pull/out/190412203340_054.feather.ind b/1_wqp_pull/out/190412203340_054.feather.ind
diff --git a/1_wqp_pull/out/190412203340_055.feather.ind b/1_wqp_pull/out/190412203340_055.feather.ind
diff --git a/1_wqp_pull/out/190412203340_056.feather.ind b/1_wqp_pull/out/190412203340_056.feather.ind
diff --git a/1_wqp_pull/out/190412203340_057.feather.ind b/1_wqp_pull/out/190412203340_057.feather.ind
diff --git a/1_wqp_pull/out/190412203340_058.feather.ind b/1_wqp_pull/out/190412203340_058.feather.ind
diff --git a/1_wqp_pull/out/190412203340_059.feather.ind b/1_wqp_pull/out/190412203340_059.feather.ind
diff --git a/1_wqp_pull/out/190412203340_060.feather.ind b/1_wqp_pull/out/190412203340_060.feather.ind
diff --git a/1_wqp_pull/out/190412203340_061.feather.ind b/1_wqp_pull/out/190412203340_061.feather.ind
diff --git a/1_wqp_pull/out/190412203340_062.feather.ind b/1_wqp_pull/out/190412203340_062.feather.ind
diff --git a/1_wqp_pull/out/190412203340_063.feather.ind b/1_wqp_pull/out/190412203340_063.feather.ind
diff --git a/1_wqp_pull/out/190412203340_064.feather.ind b/1_wqp_pull/out/190412203340_064.feather.ind
diff --git a/1_wqp_pull/out/190412203340_065.feather.ind b/1_wqp_pull/out/190412203340_065.feather.ind
diff --git a/1_wqp_pull/out/190412203340_066.feather.ind b/1_wqp_pull/out/190412203340_066.feather.ind
diff --git a/1_wqp_pull/out/190412203340_067.feather.ind b/1_wqp_pull/out/190412203340_067.feather.ind
diff --git a/1_wqp_pull/out/190412203340_068.feather.ind b/1_wqp_pull/out/190412203340_068.feather.ind
diff --git a/1_wqp_pull/out/190412203340_069.feather.ind b/1_wqp_pull/out/190412203340_069.feather.ind
diff --git a/1_wqp_pull/out/190412203340_070.feather.ind b/1_wqp_pull/out/190412203340_070.feather.ind
diff --git a/1_wqp_pull/out/190412203340_071.feather.ind b/1_wqp_pull/out/190412203340_071.feather.ind
diff --git a/1_wqp_pull/out/190412203340_072.feather.ind b/1_wqp_pull/out/190412203340_072.feather.ind
diff --git a/1_wqp_pull/out/190412203340_073.feather.ind b/1_wqp_pull/out/190412203340_073.feather.ind
diff --git a/1_wqp_pull/out/190412203340_074.feather.ind b/1_wqp_pull/out/190412203340_074.feather.ind
diff --git a/1_wqp_pull/out/190412203340_075.feather.ind b/1_wqp_pull/out/190412203340_075.feather.ind
diff --git a/1_wqp_pull/out/190412203340_076.feather.ind b/1_wqp_pull/out/190412203340_076.feather.ind
diff --git a/1_wqp_pull/out/190412203340_077.feather.ind b/1_wqp_pull/out/190412203340_077.feather.ind
diff --git a/1_wqp_pull/out/190412203340_079.feather.ind b/1_wqp_pull/out/190412203340_079.feather.ind
diff --git a/1_wqp_pull/out/190412203340_080.feather.ind b/1_wqp_pull/out/190412203340_080.feather.ind
diff --git a/1_wqp_pull/out/190412203340_081.feather.ind b/1_wqp_pull/out/190412203340_081.feather.ind
diff --git a/1_wqp_pull/out/190412203340_082.feather.ind b/1_wqp_pull/out/190412203340_082.feather.ind
diff --git a/1_wqp_pull/out/190412203340_083.feather.ind b/1_wqp_pull/out/190412203340_083.feather.ind
diff --git a/1_wqp_pull/out/190412203340_084.feather.ind b/1_wqp_pull/out/190412203340_084.feather.ind
diff --git a/1_wqp_pull/out/190412203340_085.feather.ind b/1_wqp_pull/out/190412203340_085.feather.ind
diff --git a/1_wqp_pull/out/190412203340_086.feather.ind b/1_wqp_pull/out/190412203340_086.feather.ind
diff --git a/1_wqp_pull/out/190412203340_087.feather.ind b/1_wqp_pull/out/190412203340_087.feather.ind
diff --git a/1_wqp_pull/out/190412203340_088.feather.ind b/1_wqp_pull/out/190412203340_088.feather.ind
diff --git a/1_wqp_pull/out/190412203340_089.feather.ind b/1_wqp_pull/out/190412203340_089.feather.ind
diff --git a/1_wqp_pull/out/190412203340_090.feather.ind b/1_wqp_pull/out/190412203340_090.feather.ind
diff --git a/1_wqp_pull/out/190412203340_091.feather.ind b/1_wqp_pull/out/190412203340_091.feather.ind
diff --git a/1_wqp_pull/out/190412203340_092.feather.ind b/1_wqp_pull/out/190412203340_092.feather.ind
diff --git a/1_wqp_pull/out/190412203340_093.feather.ind b/1_wqp_pull/out/190412203340_093.feather.ind
diff --git a/1_wqp_pull/out/190412203340_094.feather.ind b/1_wqp_pull/out/190412203340_094.feather.ind
diff --git a/1_wqp_pull/out/190412203340_095.feather.ind b/1_wqp_pull/out/190412203340_095.feather.ind
diff --git a/1_wqp_pull/out/190412203340_096.feather.ind b/1_wqp_pull/out/190412203340_096.feather.ind
diff --git a/1_wqp_pull/out/190412203340_097.feather.ind b/1_wqp_pull/out/190412203340_097.feather.ind
diff --git a/1_wqp_pull/out/190412203340_098.feather.ind b/1_wqp_pull/out/190412203340_098.feather.ind
diff --git a/1_wqp_pull/out/190412203340_099.feather.ind b/1_wqp_pull/out/190412203340_099.feather.ind
diff --git a/1_wqp_pull/out/190412203340_100.feather.ind b/1_wqp_pull/out/190412203340_100.feather.ind
diff --git a/1_wqp_pull/out/190412203340_101.feather.ind b/1_wqp_pull/out/190412203340_101.feather.ind
diff --git a/1_wqp_pull/out/190412203340_102.feather.ind b/1_wqp_pull/out/190412203340_102.feather.ind
diff --git a/1_wqp_pull/out/190412203340_103.feather.ind b/1_wqp_pull/out/190412203340_103.feather.ind
diff --git a/1_wqp_pull/out/190412203340_104.feather.ind b/1_wqp_pull/out/190412203340_104.feather.ind
diff --git a/1_wqp_pull/out/190412203340_105.feather.ind b/1_wqp_pull/out/190412203340_105.feather.ind
diff --git a/1_wqp_pull/out/190412203340_106.feather.ind b/1_wqp_pull/out/190412203340_106.feather.ind
diff --git a/1_wqp_pull/out/190412203340_107.feather.ind b/1_wqp_pull/out/190412203340_107.feather.ind
diff --git a/1_wqp_pull/out/190412203340_108.feather.ind b/1_wqp_pull/out/190412203340_108.feather.ind
diff --git a/1_wqp_pull/out/190412203340_109.feather.ind b/1_wqp_pull/out/190412203340_109.feather.ind
diff --git a/1_wqp_pull/out/190412203340_110.feather.ind b/1_wqp_pull/out/190412203340_110.feather.ind
diff --git a/1_wqp_pull/out/190412203340_111.feather.ind b/1_wqp_pull/out/190412203340_111.feather.ind
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@
		hash: c5dc6e02c065833fb32f72e2a9758ba8
		hash: c5f77841a450ad915d2cc3bb6019965f
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@
		hash: c48e0661c9a0d542286d9153cf704ee1
		hash: 01c2b6436068c9d36a5be2995fd768d4