From e73852c451ac522c2fdcb5f57b41f3fe94d6690d Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 27 Aug 2019 15:20:43 -0500
Subject: [PATCH 01/22] adding WQP POST script that Jordan wrote.

---
 1_wqp_pull/src/wqp_pull.R | 65 ++++++++++++++++++++++++++++-----------
 1 file changed, 47 insertions(+), 18 deletions(-)

diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 902c9f9..35b344d 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -111,35 +111,31 @@ filter_partitions <- function(partitions_ind, pull_task) {
 }
 
 # pull a batch of WQP observations, save locally, return .tind file
-get_wqp_data <- function(ind_file, partition, wqp_pull_params) {
-
+get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = FALSE) {
+  
   # prepare the arguments to pass to readWQPdata
-  wqp_args <- wqp_pull_params
-  wqp_args$characteristicName <- as.character(unlist(wqp_pull_params$characteristicName))
+  wqp_args <- list()
+  wqp_args$characteristicName <- wqp_pull_params
   wqp_args$siteid <- partition$MonitoringLocationIdentifier
-
   # do the data pull
   wqp_dat_time <- system.time({
-    wqp_dat <- do.call(dataRetrieval::readWQPdata, wqp_args[c('siteid', 'characteristicName')])
+    wqp_dat <- wqp_POST(wqp_args)
   })
-  message(sprintf(
-    'WQP pull for %s took %0.0f seconds and returned %d rows',
-    partition$PullTask[1],
-    wqp_dat_time['elapsed'],
-    nrow(wqp_dat)))
-
+  if (verbose){
+    message(sprintf(
+      'WQP pull for %s took %0.0f seconds and returned %d rows',
+      partition$PullTask[1],
+      wqp_dat_time['elapsed'],
+      nrow(wqp_dat)))
+  }
   # make wqp_dat a tibble, converting either from data.frame (the usual case) or
   # NULL (if there are no results)
   wqp_dat <- as_data_frame(wqp_dat)
-
-  # write the data and temporary indicator file. do this even if there were 0
+  
+  # write the data to rds file. do this even if there were 0
   # results because remake expects this function to always create the target
   # file
-  data_file <- scipiper::as_data_file(ind_file, ind_ext='tind')
   saveRDS(wqp_dat, data_file)
-  sc_indicate(ind_file, data_file=data_file)
-
-  invisible()
 }
 
 extract_post_wqp_data <- function(ind_file, wqp_ind) {
@@ -153,3 +149,36 @@ extract_post_wqp_data <- function(ind_file, wqp_ind) {
   feather::write_feather(wqp_dat, path=as_data_file(ind_file))
   gd_put(ind_file)
 }
+
+# hack around w/ https://github.com/USGS-R/dataRetrieval/issues/434
+wqp_POST <- function(wqp_args_list){
+  wqp_url <- "https://www.waterqualitydata.us/Result/search"
+  
+  
+  wqp_args_list$siteid <- wqp_args_list$siteid
+  post_body = jsonlite::toJSON(wqp_args_list, pretty = TRUE)
+  
+  
+  x <- POST(paste0(wqp_url,"?mimeType=csv"),
+            body = post_body,
+            content_type("application/json"),
+            accept("text/csv"))
+  
+  returnedDoc <- content(x,
+                         type="text",
+                         encoding = "UTF-8")
+  retval <- suppressWarnings(read_delim(returnedDoc,
+                                        col_types = cols(`ActivityStartTime/Time` = col_character(),
+                                                         `ActivityEndTime/Time` = col_character(),
+                                                         USGSPCode = col_character(),
+                                                         ResultCommentText=col_character(),
+                                                         `ActivityDepthHeightMeasure/MeasureValue` = col_number(),
+                                                         `DetectionQuantitationLimitMeasure/MeasureValue` = col_number(),
+                                                         ResultMeasureValue = col_number(),
+                                                         `WellDepthMeasure/MeasureValue` = col_number(),
+                                                         `WellHoleDepthMeasure/MeasureValue` = col_number(),
+                                                         `HUCEightDigitCode` = col_character(),
+                                                         `ActivityEndTime/TimeZoneCode` = col_character()),
+                                        quote = "", delim = ","))
+  return(retval)
+}

From 552a0cd993671e4a33c710b9aae881a07e98510d Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 27 Aug 2019 15:24:05 -0500
Subject: [PATCH 02/22] had an extra 0! Though most pulls were okay
 previously...so wondering if this is a conservative estimate.

---
 1_wqp_pull/cfg/wqp_partition_config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/1_wqp_pull/cfg/wqp_partition_config.yml b/1_wqp_pull/cfg/wqp_partition_config.yml
index d3a8b31..1351c48 100644
--- a/1_wqp_pull/cfg/wqp_partition_config.yml
+++ b/1_wqp_pull/cfg/wqp_partition_config.yml
@@ -3,5 +3,5 @@
 target_inv_size: 1000
 
 # Approximate maximum number of records to pull in each call to WQP.
-# Recommended number is 500000 unless that causes exceedance of max URL size allowed by the WAF
-target_pull_size: 250000
+# Recommended number is 25000 unless that causes exceedance of max URL size allowed by the WAF
+target_pull_size: 25000

From 9fb084c497956167490b015b77e3227512287ddc Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 27 Aug 2019 16:18:37 -0500
Subject: [PATCH 03/22] the table returned from whatWQPdata is now different -
 there is a column for lat/long. Also generalized the "bad orgs" that are
 tripping up calls to WQP

---
 1_wqp_pull/src/wqp_inventory.R | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/1_wqp_pull/src/wqp_inventory.R b/1_wqp_pull/src/wqp_inventory.R
index f88018b..54894b6 100644
--- a/1_wqp_pull/src/wqp_inventory.R
+++ b/1_wqp_pull/src/wqp_inventory.R
@@ -38,16 +38,12 @@ inventory_wqp <- function(inv_ind, wqp_pull_params) {
   # only filter using characteristic names (and huc for test)
   wqp_dat <- wqp_call(whatWQPdata, wqp_args['characteristicName'])
   
-  # extract coordinates
-  coords <- data.frame(matrix(unlist(wqp_dat$out$coordinates), nrow = length(wqp_dat$out$coordinates), byrow = T))
-  
   # keep columns of interest, put back lat/long
   # keeping lat/long here in case there was a situation where you'd want to 
   # take stock/map sites before pulling data
   dat_out <- wqp_dat$out %>%
-    mutate(latitude = coords$X2, longitude = coords$X1) %>%
     select(OrganizationIdentifier, MonitoringLocationIdentifier, ResolvedMonitoringLocationTypeName, 
-           StateName, CountyName, HUCEightDigitCode, latitude, longitude, resultCount)
+           StateName, CountyName, HUCEightDigitCode, latitude = lat, longitude = lon, resultCount)
   
   # spit out nrows and time it took to get inventory
   message(sprintf('sample inventory complete, required %0.2f hours to retrieve %d rows', wqp_dat$time/(60*60), wqp_dat$nrow))
@@ -87,7 +83,15 @@ partition_wqp_inventory <- function(partitions_ind, wqp_pull_params, inventory_i
   wqp_inventory <- wqp_inventory %>%
     filter(!(ResolvedMonitoringLocationTypeName %in% wqp_pull_params$DropLocationTypeName))
   
+  # filter out bad org names
+  bad_orgs <- grep(' |\\.|/', wqp_inventory$OrganizationIdentifier, value = TRUE)
+  bad_orgs_sites <- filter(wqp_inventory, OrganizationIdentifier %in% bad_orgs)
+  
+  if (nrow(bad_orgs_sites) > 0){
+    message(sprintf("**dropping %s sites and %s results due to bad MonIDs", nrow(bad_orgs_sites), sum(bad_orgs_sites$resultCount)))
+  }
   
+  wqp_inventory <- filter(wqp_inventory, !OrganizationIdentifier %in% bad_orgs)
   
   # Define the atomic_groups to use in setting up data pull partitions. An
   # atomic group is a combination of parameters that can't be reasonably split

From 76d318ad9097c3971d8e55f0c6bc9ca50b606717 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 28 Aug 2019 16:24:50 -0500
Subject: [PATCH 04/22] added a combiner to WQP, and now not indicating the
 downloaded WQP files.

---
 1_wqp_pull/cfg/wqp_partition_config.yml |  2 +-
 1_wqp_pull/src/wqp_pull.R               | 74 +++++++++++--------------
 2 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/1_wqp_pull/cfg/wqp_partition_config.yml b/1_wqp_pull/cfg/wqp_partition_config.yml
index 1351c48..52c3575 100644
--- a/1_wqp_pull/cfg/wqp_partition_config.yml
+++ b/1_wqp_pull/cfg/wqp_partition_config.yml
@@ -4,4 +4,4 @@ target_inv_size: 1000
 
 # Approximate maximum number of records to pull in each call to WQP.
 # Recommended number is 25000 unless that causes exceedance of max URL size allowed by the WAF
-target_pull_size: 25000
+target_pull_size: 200000
diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 35b344d..483fcfa 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -35,45 +35,28 @@ plan_wqp_pull <- function(partitions_ind) {
   download <- scipiper::create_task_step(
     step_name = 'download',
     target_name = function(task_name, step_name, ...) {
-      scipiper::as_ind_file(file.path(folders$tmp, sprintf('%s.rds', task_name)), ind_ext='tind')
+      file.path(folders$tmp, sprintf('%s.rds', task_name))
     },
     command = function(steps, ...) {
       paste(
         "get_wqp_data(",
-        "ind_file=target_name,",
+        "data_file=target_name,",
         sprintf("partition=%s,", steps$partition$target_name),
         "wqp_pull_params=wqp_pull_parameters)",
         sep="\n      ")
     }
   )
 
-  # extract and post the data (dropping the site info attribute), creating an
-  # .ind file that we will want to share because it represents the shared cache
-  extract_post_data <- scipiper::create_task_step(
-    step_name = 'extract_post_data',
-    target_name = function(task_name, step_name, ...) {
-      scipiper::as_ind_file(file.path(folders$out, sprintf('%s.feather', task_name)))
-    },
-    command = function(steps, ...) {
-      paste(
-        "extract_post_wqp_data(",
-        "ind_file=target_name,",
-        sprintf("wqp_ind='%s')", steps$download$target_name),
-        sep="\n      ")
-    }
-  )
-
   # put the steps together into a task plan
   task_plan <- scipiper::create_task_plan(
     task_names=sort(partitions$PullTask),
-    task_steps=list(partition, download, extract_post_data),
-    final_steps=c('extract_post_data'),
-    add_complete=TRUE,
-    ind_dir=folders$tmp)
+    task_steps=list(partition, download),
+    final_steps=c('download'),
+    add_complete=FALSE)
 
 }
 
-create_wqp_pull_makefile <- function(makefile, task_plan) {
+create_wqp_pull_makefile <- function(makefile, task_plan, final_targets) {
 
   # after all wanted data have been pulled, this function will be called but
   # doesn't need to create anything much, so just write an empty file
@@ -84,11 +67,13 @@ create_wqp_pull_makefile <- function(makefile, task_plan) {
   }
 
   create_task_makefile(
-    makefile=makefile, task_plan=task_plan,
-    include='1_wqp_pull.yml',
-    packages=c('dplyr', 'dataRetrieval', 'feather', 'scipiper', 'yaml'),
-    file_extensions=c('ind','tind','feather'),
-    ind_complete=TRUE, ind_dir='1_wqp_pull/log')
+    makefile = makefile, 
+    task_plan = task_plan,
+    include = '1_wqp_pull.yml',
+    packages = c('dplyr', 'dataRetrieval', 'feather', 'scipiper', 'yaml', 'httr', 'readr'),
+    file_extensions = c('ind','rds'),
+    final_targets = final_targets, 
+    finalize_funs = 'combine_wqp_dat')
 }
 
 loop_wqp_tasks <- function(ind_file, task_plan, ...) {
@@ -115,7 +100,7 @@ get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = FALSE)
   
   # prepare the arguments to pass to readWQPdata
   wqp_args <- list()
-  wqp_args$characteristicName <- wqp_pull_params
+  wqp_args$characteristicName <- wqp_pull_params$characteristicName$temperature
   wqp_args$siteid <- partition$MonitoringLocationIdentifier
   # do the data pull
   wqp_dat_time <- system.time({
@@ -137,19 +122,6 @@ get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = FALSE)
   # file
   saveRDS(wqp_dat, data_file)
 }
-
-extract_post_wqp_data <- function(ind_file, wqp_ind) {
-  # read in the WQP data pull results, which must have been produced on this
-  # computer (we roughly guarantee this by depending on ind_file and yet not
-  # git-committing that ind_file, so the WQP-pulling computer is the only one
-  # that could have the data file)
-  wqp_dat <- readRDS(scipiper::as_data_file(wqp_ind, ind_ext='tind'))
-
-  # write locally as feather, which strips the attributes, and post to drive
-  feather::write_feather(wqp_dat, path=as_data_file(ind_file))
-  gd_put(ind_file)
-}
-
 # hack around w/ https://github.com/USGS-R/dataRetrieval/issues/434
 wqp_POST <- function(wqp_args_list){
   wqp_url <- "https://www.waterqualitydata.us/Result/search"
@@ -182,3 +154,21 @@ wqp_POST <- function(wqp_args_list){
                                         quote = "", delim = ","))
   return(retval)
 }
+
+# extract and post the data (dropping the site info attribute), creating an
+# .ind file that we will want to share because it represents the shared cache
+combine_wqp_data <- function(ind_file, ...){
+  
+  rds_files <- c(...)
+  df_list <- list()
+  
+  for (i in seq_len(length(rds_files))){
+    df_list[[i]] <- readRDS(rds_files[i])
+  }
+  
+  wqp_df <- do.call("rbind", df_list)
+  
+  data_file <- scipiper::as_data_file(ind_file)
+  saveRDS(wqp_df, data_file)
+  gd_put(ind_file, data_file)
+}

From 15ace42e48aa49034de829145aa92c43bb4e9266 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 28 Aug 2019 16:27:26 -0500
Subject: [PATCH 05/22] modified the partitioning algorithm to take into
 account the 1000 site limit. Now, it filters partition to those with n sites
 < 1000 and then finds the partition with the smallest record count.
 Partitions with >1000 sites were tripping the POST call up.

---
 1_wqp_pull/src/wqp_inventory.R | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/1_wqp_pull/src/wqp_inventory.R b/1_wqp_pull/src/wqp_inventory.R
index 54894b6..fc599de 100644
--- a/1_wqp_pull/src/wqp_inventory.R
+++ b/1_wqp_pull/src/wqp_inventory.R
@@ -76,7 +76,7 @@ partition_wqp_inventory <- function(partitions_ind, wqp_pull_params, inventory_i
   
   # Read in the inventory & config
   wqp_inventory <- feather::read_feather(sc_retrieve(inventory_ind))
-  wqp_partition_config <- yaml::yaml.load_file(wqp_partition_cfg)
+  wqp_partition_config <- wqp_partition_cfg
   
   # filter out site types that are not of interest
   
@@ -116,23 +116,38 @@ partition_wqp_inventory <- function(partitions_ind, wqp_pull_params, inventory_i
   # each of these sites/lakes will get its own partition + (B) the number of
   # remaining observations divided by the target partition size.
   target_pull_size <- wqp_partition_config$target_pull_size
+  target_site_size <- wqp_partition_config$target_inv_size
   n_single_site_partitions <- filter(atomic_groups, resultCount >= target_pull_size) %>% nrow()
-  n_multi_site_partitions <- filter(atomic_groups, resultCount < target_pull_size) %>%
+  n_multi_site_partitions_byresult <- filter(atomic_groups, resultCount < target_pull_size) %>%
     pull(resultCount) %>%
     { ceiling(sum(.)/target_pull_size) }
+  
+  # take into account ~1000 sites per pull
+  n_multi_site_partitions_bysite <- nrow(filter(atomic_groups, resultCount < target_pull_size))/target_site_size
+  n_multi_site_partitions <- ceiling(ifelse(n_multi_site_partitions_bysite >= n_multi_site_partitions_byresult, n_multi_site_partitions_bysite, n_multi_site_partitions_byresult))
+  
   num_partitions <- n_single_site_partitions + n_multi_site_partitions
   
   # Assign each site to a partition. Sites with huge numbers
   # of observations will each get their own partition.
   partition_sizes <- rep(0, num_partitions)
-  assignments <- rep(NA, nrow(atomic_groups)) # use a vector rather than adding a col to atomic_groups b/c it'll be way faster
-  for(i in seq_len(nrow(atomic_groups))) {
-    smallest_partition <- which.min(partition_sizes)
+  partition_site_sizes <- rep(0, num_partitions)
+  assignments <- rep(NA, nrow(atomic_groups)) 
+  partition_index <- 1:num_partitions
+  
+  # use a vector rather than adding a col to atomic_groups b/c it'll be way faster
+  for(i in 1:nrow(atomic_groups)) {
+    indexing_true <- partition_index[partition_site_sizes < target_site_size]
+    smallest_partition_nonindexed <- which.min(partition_sizes[partition_site_sizes < target_site_size])
+    smallest_partition <- indexing_true[smallest_partition_nonindexed]
+    
     assignments[i] <- smallest_partition
     size_i <- atomic_groups[[i,"resultCount"]]
     partition_sizes[smallest_partition] <- partition_sizes[smallest_partition] + size_i
+    partition_site_sizes[smallest_partition] <- partition_site_sizes[smallest_partition] + 1
   }
   
+  
   # Prepare one data_frame containing info about each site, including
   # the pull, constituent, and task name (where task name will become the core
   # of the filename)

From b40546413fdc20e8735c903729f2627834a7fa43 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 30 Aug 2019 09:28:28 -0500
Subject: [PATCH 06/22] reduced pull size again in attempt to make POST work

---
 1_wqp_pull/cfg/wqp_partition_config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/1_wqp_pull/cfg/wqp_partition_config.yml b/1_wqp_pull/cfg/wqp_partition_config.yml
index 52c3575..1351c48 100644
--- a/1_wqp_pull/cfg/wqp_partition_config.yml
+++ b/1_wqp_pull/cfg/wqp_partition_config.yml
@@ -4,4 +4,4 @@ target_inv_size: 1000
 
 # Approximate maximum number of records to pull in each call to WQP.
 # Recommended number is 25000 unless that causes exceedance of max URL size allowed by the WAF
-target_pull_size: 200000
+target_pull_size: 25000

From c0e651c8912fd1f9dc2ba2352073581a4f13971d Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 30 Aug 2019 09:28:52 -0500
Subject: [PATCH 07/22] increase digits to accomodate >1000 pulls

---
 1_wqp_pull/src/wqp_inventory.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/1_wqp_pull/src/wqp_inventory.R b/1_wqp_pull/src/wqp_inventory.R
index fc599de..bac7c05 100644
--- a/1_wqp_pull/src/wqp_inventory.R
+++ b/1_wqp_pull/src/wqp_inventory.R
@@ -157,7 +157,7 @@ partition_wqp_inventory <- function(partitions_ind, wqp_pull_params, inventory_i
   partitions <- atomic_groups %>%
     mutate(
       PullDate = pull_time,
-      PullTask = sprintf('%s_%03d', pull_id, assignments)) %>%
+      PullTask = sprintf('%s_%04d', pull_id, assignments)) %>%
     left_join(select(wqp_inventory, MonitoringLocationIdentifier, SiteNumObs=resultCount), by='MonitoringLocationIdentifier') %>%
     select(MonitoringLocationIdentifier, SiteNumObs, PullTask, PullDate)
   

From 18204eb6a422aa777267e888cf5c7c2e0774ed72 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 30 Aug 2019 09:29:53 -0500
Subject: [PATCH 08/22] changed POST from csv to tsv

---
 1_wqp_pull/src/wqp_pull.R | 47 ++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 483fcfa..da5e54a 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -96,7 +96,7 @@ filter_partitions <- function(partitions_ind, pull_task) {
 }
 
 # pull a batch of WQP observations, save locally, return .tind file
-get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = FALSE) {
+get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = TRUE) {
   
   # prepare the arguments to pass to readWQPdata
   wqp_args <- list()
@@ -106,6 +106,9 @@ get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = FALSE)
   wqp_dat_time <- system.time({
     wqp_dat <- wqp_POST(wqp_args)
   })
+  
+  wqp_dr_dat_time <- system.time(dat_dr <- readWQPdata(siteid = wqp_args$siteid,
+                                                       characteristicName = wqp_args$characteristicName))
   if (verbose){
     message(sprintf(
       'WQP pull for %s took %0.0f seconds and returned %d rows',
@@ -130,28 +133,32 @@ wqp_POST <- function(wqp_args_list){
   wqp_args_list$siteid <- wqp_args_list$siteid
   post_body = jsonlite::toJSON(wqp_args_list, pretty = TRUE)
   
-  
-  x <- POST(paste0(wqp_url,"?mimeType=csv"),
+  temp <- tempfile()
+  temp <- paste0(temp,".zip")
+  x <- POST(paste0(wqp_url,"?mimeType=tsv&zip=yes"),
             body = post_body,
             content_type("application/json"),
-            accept("text/csv"))
+            accept("application/zip"),
+            httr::write_disk(temp))
   
-  returnedDoc <- content(x,
-                         type="text",
-                         encoding = "UTF-8")
-  retval <- suppressWarnings(read_delim(returnedDoc,
-                                        col_types = cols(`ActivityStartTime/Time` = col_character(),
-                                                         `ActivityEndTime/Time` = col_character(),
-                                                         USGSPCode = col_character(),
-                                                         ResultCommentText=col_character(),
-                                                         `ActivityDepthHeightMeasure/MeasureValue` = col_number(),
-                                                         `DetectionQuantitationLimitMeasure/MeasureValue` = col_number(),
-                                                         ResultMeasureValue = col_number(),
-                                                         `WellDepthMeasure/MeasureValue` = col_number(),
-                                                         `WellHoleDepthMeasure/MeasureValue` = col_number(),
-                                                         `HUCEightDigitCode` = col_character(),
-                                                         `ActivityEndTime/TimeZoneCode` = col_character()),
-                                        quote = "", delim = ","))
+  headerInfo <- httr::headers(x)
+  file1 <- tempdir()
+  doc <- utils::unzip(temp, exdir=file1)
+  unlink(temp)
+  retval1 <- suppressWarnings(read_delim(doc, 
+                                         col_types = cols(`ActivityStartTime/Time` = col_character(),
+                                                          `ActivityEndTime/Time` = col_character(),
+                                                          USGSPCode = col_character(),
+                                                          ResultCommentText=col_character(),
+                                                          `ActivityDepthHeightMeasure/MeasureValue` = col_number(),
+                                                          `DetectionQuantitationLimitMeasure/MeasureValue` = col_number(),
+                                                          ResultMeasureValue = col_number(),
+                                                          `WellDepthMeasure/MeasureValue` = col_number(),
+                                                          `WellHoleDepthMeasure/MeasureValue` = col_number(),
+                                                          `HUCEightDigitCode` = col_character(), 
+                                                          `ActivityEndTime/TimeZoneCode` = col_character()),
+                                         quote = "", delim = "\t"))
+  unlink(doc)
   return(retval)
 }
 

From 42490570c469b78440fa1e0dcf6d637a5f2d1fef Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Mon, 23 Sep 2019 15:54:32 -0500
Subject: [PATCH 09/22] wrote some fault tolerance in - first tries
 dataRetrieval, then uses POST to retrieve data if that fails.

---
 1_wqp_pull.yml                          | 17 +++++---
 1_wqp_pull/cfg/wqp_partition_config.yml |  2 +-
 1_wqp_pull/src/wqp_pull.R               | 54 ++++++++++++++++++++-----
 3 files changed, 56 insertions(+), 17 deletions(-)

diff --git a/1_wqp_pull.yml b/1_wqp_pull.yml
index c33cf24..9f39449 100644
--- a/1_wqp_pull.yml
+++ b/1_wqp_pull.yml
@@ -17,13 +17,15 @@ sources:
 targets:
   1_wqp_pull:
     depends:
-      - 1_wqp_pull/log/1_wqp_pull_tasks.ind
       
 # -- get a CONUS inventory of available data for download -- #
 
   wqp_pull_parameters:
     command: yaml.load_file('1_wqp_pull/cfg/wqp_pull_params.yml')
     
+  wqp_partition_cfg:
+    command: yaml.load_file('1_wqp_pull/cfg/wqp_partition_config.yml')
+    
   # -- get inventory of observations available to download -- #
 
   # get inventory of samples available on WQP
@@ -44,12 +46,12 @@ targets:
       partitions_ind=target_name,
       wqp_pull_params = wqp_pull_parameters,
       inventory_ind='1_wqp_pull/inout/wqp_inventory.feather.ind',
-      wqp_partition_cfg=I('1_wqp_pull/cfg/wqp_partition_config.yml'))
+      wqp_partition_cfg=wqp_partition_cfg)
   
   1_wqp_pull/inout/wqp_pull_partitions.feather:
     command: gd_get('1_wqp_pull/inout/wqp_pull_partitions.feather.ind')
     
-# -- pull data! -- #
+  # -- pull data! -- #
 
   # prepare a remake-style plan for running each state as a separate
   # remake target in a separate remake file (1_wqp_wqp_tasks.yml)
@@ -57,13 +59,16 @@ targets:
     command: plan_wqp_pull(partitions_ind='1_wqp_pull/inout/wqp_pull_partitions.feather.ind')
   
   1_wqp_pull_tasks.yml:
-    command: create_wqp_pull_makefile(makefile=target_name, task_plan=wqp_pull_plan)
+    command: create_wqp_pull_makefile(makefile = target_name, 
+      task_plan = wqp_pull_plan,
+      final_targets = I('1_wqp_pull/out/wqp_data.rds.ind'))
 
   # run the data pulls
-  1_wqp_pull/log/1_wqp_pull_tasks.ind:
+  1_wqp_pull/out/wqp_data.rds.ind:
     command: loop_wqp_tasks(
       ind_file=target_name,
-      task_plan=wqp_pull_plan, task_makefile='1_wqp_pull_tasks.yml',
+      task_plan=wqp_pull_plan, 
+      task_makefile='1_wqp_pull_tasks.yml',
       num_tries=I(30), sleep_on_error=I(20))
 
 
diff --git a/1_wqp_pull/cfg/wqp_partition_config.yml b/1_wqp_pull/cfg/wqp_partition_config.yml
index 1351c48..8e4002d 100644
--- a/1_wqp_pull/cfg/wqp_partition_config.yml
+++ b/1_wqp_pull/cfg/wqp_partition_config.yml
@@ -4,4 +4,4 @@ target_inv_size: 1000
 
 # Approximate maximum number of records to pull in each call to WQP.
 # Recommended number is 25000 unless that causes exceedance of max URL size allowed by the WAF
-target_pull_size: 25000
+target_pull_size: 250000
diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index da5e54a..93aa951 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -103,22 +103,56 @@ get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = TRUE)
   wqp_args$characteristicName <- wqp_pull_params$characteristicName$temperature
   wqp_args$siteid <- partition$MonitoringLocationIdentifier
   # do the data pull
-  wqp_dat_time <- system.time({
-    wqp_dat <- wqp_POST(wqp_args)
-  })
+  # first pull using readWQPdata, then if that fails, try POST
   
-  wqp_dr_dat_time <- system.time(dat_dr <- readWQPdata(siteid = wqp_args$siteid,
-                                                       characteristicName = wqp_args$characteristicName))
+ 
+  getWQP <- function(wqp_args) {
+    wqp_dat_time <- tryCatch(
+      {
+        # first try readWQPdata - as this seems to be the fastest
+        time_start <- Sys.time()
+        wqp_dat <- readWQPdata(siteid = wqp_args$siteid,
+                    characteristicName = wqp_args$characteristicName)
+        time_end <- Sys.time()
+        time_diff <- time_end - time_start
+        
+        return(list(time_diff, wqp_dat, 'readWQPdata'))
+      },
+      error = function(cond) {
+        message('Call to WQP using readWQPdata failed. Trying via POST. Originally error message:')
+        message(cond)
+        
+        time_start <- Sys.time()
+        wqp_dat <- wqp_POST(wqp_args)
+        time_end <- Sys.time()
+        time_diff <- time_end - time_start
+        
+        return(list(time_diff, wqp_dat, 'POST'))
+      }
+    )
+    return(wqp_dat_time)
+  }
+  
+  wqp_dat_time <- getWQP(wqp_args)
+  
+  # wqp_dat_time <- system.time({
+  #   wqp_dat <- wqp_POST(wqp_args)
+  # })
+  # 
+  # wqp_dr_dat_time <- system.time(dat_dr <- readWQPdata(siteid = wqp_args$siteid,
+  #                                                      characteristicName = wqp_args$characteristicName))
   if (verbose){
     message(sprintf(
-      'WQP pull for %s took %0.0f seconds and returned %d rows',
+      'WQP pull (using %s) for %s took %0.1f %s and returned %d rows',
+      wqp_dat_time[[3]],
       partition$PullTask[1],
-      wqp_dat_time['elapsed'],
-      nrow(wqp_dat)))
+      as.numeric(wqp_dat_time[[1]]),
+      attr(wqp_dat_time[[1]], 'units'),
+      nrow(wqp_dat_time[[2]])))
   }
   # make wqp_dat a tibble, converting either from data.frame (the usual case) or
   # NULL (if there are no results)
-  wqp_dat <- as_data_frame(wqp_dat)
+  wqp_dat <- as_tibble(wqp_dat_time[[2]])
   
   # write the data to rds file. do this even if there were 0
   # results because remake expects this function to always create the target
@@ -145,7 +179,7 @@ wqp_POST <- function(wqp_args_list){
   file1 <- tempdir()
   doc <- utils::unzip(temp, exdir=file1)
   unlink(temp)
-  retval1 <- suppressWarnings(read_delim(doc, 
+  retval <- suppressWarnings(read_delim(doc, 
                                          col_types = cols(`ActivityStartTime/Time` = col_character(),
                                                           `ActivityEndTime/Time` = col_character(),
                                                           USGSPCode = col_character(),

From 9005a8e1386bf759df4aabf30908849bace05793 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 24 Sep 2019 10:37:15 -0500
Subject: [PATCH 10/22] swapped order of try. First tries POST because it is
 much faster, then tries readWQPdata.

---
 1_wqp_pull.yml                                   |  4 +---
 1_wqp_pull/inout/wqp_inventory.feather.ind       |  2 +-
 1_wqp_pull/inout/wqp_pull_partitions.feather.ind |  2 +-
 1_wqp_pull/src/wqp_pull.R                        | 15 ++++++++-------
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/1_wqp_pull.yml b/1_wqp_pull.yml
index 9f39449..56261aa 100644
--- a/1_wqp_pull.yml
+++ b/1_wqp_pull.yml
@@ -69,6 +69,4 @@ targets:
       ind_file=target_name,
       task_plan=wqp_pull_plan, 
       task_makefile='1_wqp_pull_tasks.yml',
-      num_tries=I(30), sleep_on_error=I(20))
-
-
+      num_tries=I(30), sleep_on_error=I(20))
\ No newline at end of file
diff --git a/1_wqp_pull/inout/wqp_inventory.feather.ind b/1_wqp_pull/inout/wqp_inventory.feather.ind
index dea6009..8618870 100644
--- a/1_wqp_pull/inout/wqp_inventory.feather.ind
+++ b/1_wqp_pull/inout/wqp_inventory.feather.ind
@@ -1,2 +1,2 @@
-hash: c5dc6e02c065833fb32f72e2a9758ba8
+hash: c5f77841a450ad915d2cc3bb6019965f
 
diff --git a/1_wqp_pull/inout/wqp_pull_partitions.feather.ind b/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
index c2fab0e..ab8a8fc 100644
--- a/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
+++ b/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
@@ -1,2 +1,2 @@
-hash: c48e0661c9a0d542286d9153cf704ee1
+hash: f5f6e494489918022cb3397ac12bcfd8
 
diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 93aa951..2168c37 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -109,25 +109,26 @@ get_wqp_data <- function(data_file, partition, wqp_pull_params, verbose = TRUE)
   getWQP <- function(wqp_args) {
     wqp_dat_time <- tryCatch(
       {
-        # first try readWQPdata - as this seems to be the fastest
         time_start <- Sys.time()
-        wqp_dat <- readWQPdata(siteid = wqp_args$siteid,
-                    characteristicName = wqp_args$characteristicName)
+        wqp_dat <- wqp_POST(wqp_args)
         time_end <- Sys.time()
         time_diff <- time_end - time_start
         
-        return(list(time_diff, wqp_dat, 'readWQPdata'))
+        return(list(time_diff, wqp_dat, 'POST'))
+       
+        
       },
       error = function(cond) {
-        message('Call to WQP using readWQPdata failed. Trying via POST. Originally error message:')
+        message('Call to WQP using POST failed. Trying via readWQPdata. Original error message:')
         message(cond)
         
         time_start <- Sys.time()
-        wqp_dat <- wqp_POST(wqp_args)
+        wqp_dat <- readWQPdata(siteid = wqp_args$siteid,
+                               characteristicName = wqp_args$characteristicName)
         time_end <- Sys.time()
         time_diff <- time_end - time_start
         
-        return(list(time_diff, wqp_dat, 'POST'))
+        return(list(time_diff, wqp_dat, 'readWQPdata'))
       }
     )
     return(wqp_dat_time)

From 7f44798567dc3cc700080267da95936d85672120 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 24 Sep 2019 10:38:16 -0500
Subject: [PATCH 11/22] new builds of wq inventory and partition after building
 in fault tolerance and upping n obs allowed in single call.

---
 ...VsbC9pbm91dC93cXBfaW52ZW50b3J5LmZlYXRoZXIuaW5k.yml |  6 +++---
 ...91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml | 11 ++++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfaW52ZW50b3J5LmZlYXRoZXIuaW5k.yml b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfaW52ZW50b3J5LmZlYXRoZXIuaW5k.yml
index 670e371..6c43849 100644
--- a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfaW52ZW50b3J5LmZlYXRoZXIuaW5k.yml
+++ b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfaW52ZW50b3J5LmZlYXRoZXIuaW5k.yml
@@ -1,12 +1,12 @@
 version: 0.3.0
 name: 1_wqp_pull/inout/wqp_inventory.feather.ind
 type: file
-hash: 4888acde020edded037219ee8f802787
-time: 2019-04-12 20:28:33 UTC
+hash: 756985b1caa92e538e06dc55de5b67ec
+time: 2019-08-27 21:25:15 UTC
 depends:
   wqp_pull_parameters: 61e80f749c74948bf95bb67cb07412b0
 fixed: 771195029a2667a2a7b9465306769095
 code:
   functions:
-    inventory_wqp: a976c41a7ec3073b920041a87aff9f2c
+    inventory_wqp: 2c1d993a23e7f8033dfb5d684d093022
 
diff --git a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
index 23e7319..dbde90f 100644
--- a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
+++ b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
@@ -1,13 +1,14 @@
 version: 0.3.0
 name: 1_wqp_pull/inout/wqp_pull_partitions.feather.ind
 type: file
-hash: 600c28fc91d095f1aacb47d638009994
-time: 2019-04-12 20:33:59 UTC
+hash: d34136e22268f73af5da4a2632be42b2
+time: 2019-09-23 14:35:56 UTC
 depends:
   wqp_pull_parameters: 61e80f749c74948bf95bb67cb07412b0
-  1_wqp_pull/inout/wqp_inventory.feather.ind: 4888acde020edded037219ee8f802787
-fixed: 8c3e259c3d42393d6c8f7bd6e0467a9d
+  1_wqp_pull/inout/wqp_inventory.feather.ind: 756985b1caa92e538e06dc55de5b67ec
+  wqp_partition_cfg: dc9f1a6787144e2e96aafb48b1ee67ce
+fixed: f1ee901bf2a86f4bac0c08cf4480a175
 code:
   functions:
-    partition_wqp_inventory: b8867b50f1fbae7087dc59e7502f15d3
+    partition_wqp_inventory: b9fab76a65793640fa18115442ea01c1
 

From a143a1d273ea7726963a078c53e974d38886522e Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Thu, 26 Sep 2019 16:30:09 -0500
Subject: [PATCH 12/22] cleanup - no longer tracking intermediate data files

---
 1_wqp_pull/out/190412172909_001.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_001.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_002.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_003.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_004.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_005.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_006.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_007.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_008.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_009.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_010.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_011.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_012.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_013.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_014.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_015.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_016.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_017.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_018.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_019.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_021.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_022.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_023.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_024.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_025.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_026.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_027.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_028.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_029.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_030.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_031.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_032.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_033.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_034.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_035.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_036.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_038.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_039.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_040.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_041.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_042.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_043.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_044.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_045.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_046.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_047.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_049.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_050.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_051.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_052.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_053.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_054.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_055.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_056.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_057.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_058.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_059.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_060.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_061.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_062.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_063.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_064.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_065.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_066.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_067.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_068.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_069.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_070.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_071.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_072.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_073.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_074.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_075.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_076.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_077.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_079.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_080.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_081.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_082.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_083.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_084.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_085.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_086.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_087.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_088.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_089.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_090.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_091.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_092.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_093.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_094.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_095.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_096.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_097.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_098.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_099.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_100.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_101.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_102.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_103.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_104.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_105.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_106.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_107.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_108.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_109.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_110.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_111.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_112.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_113.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_115.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_116.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_117.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_118.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_119.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_120.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_121.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_122.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_123.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_124.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_125.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_126.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_127.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_128.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_129.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_130.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_131.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_132.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_133.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_134.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_135.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_136.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_137.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_138.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_139.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_140.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_141.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_142.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_143.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_144.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_145.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_147.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_148.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_149.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_150.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_151.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_152.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_153.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_154.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_155.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_156.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_157.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_158.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_159.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_160.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_162.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_163.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_164.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_165.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_166.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_167.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_168.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_169.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_171.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_172.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_173.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_174.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_175.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_176.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_177.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_178.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_179.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_180.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_181.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_182.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_183.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_184.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_185.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_186.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_187.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_188.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_189.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_190.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_191.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_192.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_193.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_194.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_196.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_197.feather.ind                 | 2 --
 1_wqp_pull/out/190412203340_198.feather.ind                 | 2 --
 ...bC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml | 6 +++---
 191 files changed, 3 insertions(+), 383 deletions(-)
 delete mode 100644 1_wqp_pull/out/190412172909_001.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_001.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_002.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_003.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_004.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_005.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_006.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_007.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_008.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_009.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_010.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_011.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_012.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_013.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_014.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_015.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_016.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_017.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_018.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_019.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_021.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_022.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_023.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_024.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_025.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_026.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_027.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_028.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_029.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_030.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_031.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_032.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_033.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_034.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_035.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_036.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_038.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_039.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_040.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_041.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_042.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_043.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_044.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_045.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_046.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_047.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_049.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_050.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_051.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_052.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_053.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_054.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_055.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_056.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_057.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_058.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_059.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_060.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_061.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_062.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_063.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_064.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_065.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_066.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_067.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_068.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_069.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_070.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_071.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_072.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_073.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_074.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_075.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_076.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_077.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_079.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_080.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_081.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_082.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_083.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_084.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_085.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_086.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_087.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_088.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_089.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_090.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_091.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_092.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_093.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_094.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_095.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_096.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_097.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_098.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_099.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_100.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_101.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_102.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_103.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_104.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_105.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_106.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_107.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_108.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_109.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_110.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_111.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_112.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_113.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_115.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_116.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_117.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_118.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_119.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_120.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_121.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_122.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_123.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_124.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_125.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_126.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_127.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_128.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_129.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_130.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_131.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_132.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_133.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_134.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_135.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_136.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_137.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_138.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_139.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_140.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_141.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_142.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_143.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_144.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_145.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_147.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_148.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_149.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_150.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_151.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_152.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_153.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_154.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_155.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_156.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_157.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_158.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_159.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_160.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_162.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_163.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_164.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_165.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_166.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_167.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_168.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_169.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_171.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_172.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_173.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_174.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_175.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_176.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_177.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_178.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_179.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_180.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_181.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_182.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_183.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_184.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_185.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_186.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_187.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_188.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_189.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_190.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_191.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_192.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_193.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_194.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_196.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_197.feather.ind
 delete mode 100644 1_wqp_pull/out/190412203340_198.feather.ind

diff --git a/1_wqp_pull/out/190412172909_001.feather.ind b/1_wqp_pull/out/190412172909_001.feather.ind
deleted file mode 100644
index ffc9e18..0000000
--- a/1_wqp_pull/out/190412172909_001.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 687db8e608a29c80f3d43b4847a1fee7
-
diff --git a/1_wqp_pull/out/190412203340_001.feather.ind b/1_wqp_pull/out/190412203340_001.feather.ind
deleted file mode 100644
index 6946861..0000000
--- a/1_wqp_pull/out/190412203340_001.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 241021bb56c59526f05a085f01aed2be
-
diff --git a/1_wqp_pull/out/190412203340_002.feather.ind b/1_wqp_pull/out/190412203340_002.feather.ind
deleted file mode 100644
index e80f689..0000000
--- a/1_wqp_pull/out/190412203340_002.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3ed17b8042fa608a35a4d5e2bc036e0e
-
diff --git a/1_wqp_pull/out/190412203340_003.feather.ind b/1_wqp_pull/out/190412203340_003.feather.ind
deleted file mode 100644
index dfdaef7..0000000
--- a/1_wqp_pull/out/190412203340_003.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: ef15e4b1d6c1abf51e672b0de830024e
-
diff --git a/1_wqp_pull/out/190412203340_004.feather.ind b/1_wqp_pull/out/190412203340_004.feather.ind
deleted file mode 100644
index 26b83cb..0000000
--- a/1_wqp_pull/out/190412203340_004.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6dc0ff29b0f8d5e59f6b3803cc3baa54
-
diff --git a/1_wqp_pull/out/190412203340_005.feather.ind b/1_wqp_pull/out/190412203340_005.feather.ind
deleted file mode 100644
index 3d31bd1..0000000
--- a/1_wqp_pull/out/190412203340_005.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c60ca0404bb5eb259f6e0439e96800eb
-
diff --git a/1_wqp_pull/out/190412203340_006.feather.ind b/1_wqp_pull/out/190412203340_006.feather.ind
deleted file mode 100644
index 958adb9..0000000
--- a/1_wqp_pull/out/190412203340_006.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c5ae0493e63ad42469fff3b2e8834cc1
-
diff --git a/1_wqp_pull/out/190412203340_007.feather.ind b/1_wqp_pull/out/190412203340_007.feather.ind
deleted file mode 100644
index f698aca..0000000
--- a/1_wqp_pull/out/190412203340_007.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: dea1ac1aab8186628d4a5375a88605c3
-
diff --git a/1_wqp_pull/out/190412203340_008.feather.ind b/1_wqp_pull/out/190412203340_008.feather.ind
deleted file mode 100644
index cd30dd4..0000000
--- a/1_wqp_pull/out/190412203340_008.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0a9f396aa9f97904dc0113e479a445a0
-
diff --git a/1_wqp_pull/out/190412203340_009.feather.ind b/1_wqp_pull/out/190412203340_009.feather.ind
deleted file mode 100644
index 0d93a9c..0000000
--- a/1_wqp_pull/out/190412203340_009.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 72def46ade1e859bf3fc49708568125a
-
diff --git a/1_wqp_pull/out/190412203340_010.feather.ind b/1_wqp_pull/out/190412203340_010.feather.ind
deleted file mode 100644
index 561a25a..0000000
--- a/1_wqp_pull/out/190412203340_010.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 93ae47439cd268f4304ad0f9008aedb1
-
diff --git a/1_wqp_pull/out/190412203340_011.feather.ind b/1_wqp_pull/out/190412203340_011.feather.ind
deleted file mode 100644
index ba627d8..0000000
--- a/1_wqp_pull/out/190412203340_011.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 25960e90a28aaf41ced187ea7a160e71
-
diff --git a/1_wqp_pull/out/190412203340_012.feather.ind b/1_wqp_pull/out/190412203340_012.feather.ind
deleted file mode 100644
index d259514..0000000
--- a/1_wqp_pull/out/190412203340_012.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c2931c383114eeab1d2a053356849bfd
-
diff --git a/1_wqp_pull/out/190412203340_013.feather.ind b/1_wqp_pull/out/190412203340_013.feather.ind
deleted file mode 100644
index 9b564ac..0000000
--- a/1_wqp_pull/out/190412203340_013.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f40b709eda8c3f5edce27340d28db52f
-
diff --git a/1_wqp_pull/out/190412203340_014.feather.ind b/1_wqp_pull/out/190412203340_014.feather.ind
deleted file mode 100644
index bf29462..0000000
--- a/1_wqp_pull/out/190412203340_014.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: bad8bdaac065dec099964f460cd584fa
-
diff --git a/1_wqp_pull/out/190412203340_015.feather.ind b/1_wqp_pull/out/190412203340_015.feather.ind
deleted file mode 100644
index ffcd91c..0000000
--- a/1_wqp_pull/out/190412203340_015.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8d987e6de8bbb0f87bc8ebdb23e6f14b
-
diff --git a/1_wqp_pull/out/190412203340_016.feather.ind b/1_wqp_pull/out/190412203340_016.feather.ind
deleted file mode 100644
index 7f93762..0000000
--- a/1_wqp_pull/out/190412203340_016.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 5e7d70101bdbb3213ed56091552a93f8
-
diff --git a/1_wqp_pull/out/190412203340_017.feather.ind b/1_wqp_pull/out/190412203340_017.feather.ind
deleted file mode 100644
index 3a0ae6c..0000000
--- a/1_wqp_pull/out/190412203340_017.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3ab6b2a3c4e35d44a8547e715b3f5bc6
-
diff --git a/1_wqp_pull/out/190412203340_018.feather.ind b/1_wqp_pull/out/190412203340_018.feather.ind
deleted file mode 100644
index 16f2213..0000000
--- a/1_wqp_pull/out/190412203340_018.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 18ecd7e8a0cf37d5ae1584089fde6587
-
diff --git a/1_wqp_pull/out/190412203340_019.feather.ind b/1_wqp_pull/out/190412203340_019.feather.ind
deleted file mode 100644
index dd9c3f3..0000000
--- a/1_wqp_pull/out/190412203340_019.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c7deff2723e84c805e6b0783a3b27e66
-
diff --git a/1_wqp_pull/out/190412203340_021.feather.ind b/1_wqp_pull/out/190412203340_021.feather.ind
deleted file mode 100644
index a514293..0000000
--- a/1_wqp_pull/out/190412203340_021.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8ca964d26b15e129815f9b84076eb01d
-
diff --git a/1_wqp_pull/out/190412203340_022.feather.ind b/1_wqp_pull/out/190412203340_022.feather.ind
deleted file mode 100644
index ecebdb6..0000000
--- a/1_wqp_pull/out/190412203340_022.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f7b24bae272c575069a23b482baf76aa
-
diff --git a/1_wqp_pull/out/190412203340_023.feather.ind b/1_wqp_pull/out/190412203340_023.feather.ind
deleted file mode 100644
index 5617ded..0000000
--- a/1_wqp_pull/out/190412203340_023.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 84b674b272cda22dd2561ce980e11d85
-
diff --git a/1_wqp_pull/out/190412203340_024.feather.ind b/1_wqp_pull/out/190412203340_024.feather.ind
deleted file mode 100644
index efe15af..0000000
--- a/1_wqp_pull/out/190412203340_024.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7022f447ab3f989d941554a95af59a7e
-
diff --git a/1_wqp_pull/out/190412203340_025.feather.ind b/1_wqp_pull/out/190412203340_025.feather.ind
deleted file mode 100644
index 55532ee..0000000
--- a/1_wqp_pull/out/190412203340_025.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c3dff86c5676bf18c0d023742f41fb20
-
diff --git a/1_wqp_pull/out/190412203340_026.feather.ind b/1_wqp_pull/out/190412203340_026.feather.ind
deleted file mode 100644
index e9c7581..0000000
--- a/1_wqp_pull/out/190412203340_026.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9bde4f6a3d01006ef6eeca50b560f8dd
-
diff --git a/1_wqp_pull/out/190412203340_027.feather.ind b/1_wqp_pull/out/190412203340_027.feather.ind
deleted file mode 100644
index 648b9ff..0000000
--- a/1_wqp_pull/out/190412203340_027.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9867b88bc2d11fcd7c3f08646037e08f
-
diff --git a/1_wqp_pull/out/190412203340_028.feather.ind b/1_wqp_pull/out/190412203340_028.feather.ind
deleted file mode 100644
index 0ef1bf7..0000000
--- a/1_wqp_pull/out/190412203340_028.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1caae04ea34a58037ffaea1596748b2a
-
diff --git a/1_wqp_pull/out/190412203340_029.feather.ind b/1_wqp_pull/out/190412203340_029.feather.ind
deleted file mode 100644
index c6a7f11..0000000
--- a/1_wqp_pull/out/190412203340_029.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 18dbd13824ba7412a87f7ffc515a7c07
-
diff --git a/1_wqp_pull/out/190412203340_030.feather.ind b/1_wqp_pull/out/190412203340_030.feather.ind
deleted file mode 100644
index a62f98b..0000000
--- a/1_wqp_pull/out/190412203340_030.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1d8e46fb01305f1e40654627495c0381
-
diff --git a/1_wqp_pull/out/190412203340_031.feather.ind b/1_wqp_pull/out/190412203340_031.feather.ind
deleted file mode 100644
index a7ea9fa..0000000
--- a/1_wqp_pull/out/190412203340_031.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e3f7613ee967d6e251bd7f53dc01adda
-
diff --git a/1_wqp_pull/out/190412203340_032.feather.ind b/1_wqp_pull/out/190412203340_032.feather.ind
deleted file mode 100644
index 896144b..0000000
--- a/1_wqp_pull/out/190412203340_032.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d74a519822809e08c8fdf16c9dcc2653
-
diff --git a/1_wqp_pull/out/190412203340_033.feather.ind b/1_wqp_pull/out/190412203340_033.feather.ind
deleted file mode 100644
index 9e1d5df..0000000
--- a/1_wqp_pull/out/190412203340_033.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 69dc48deec964bfc91dbd3a9a526d6e4
-
diff --git a/1_wqp_pull/out/190412203340_034.feather.ind b/1_wqp_pull/out/190412203340_034.feather.ind
deleted file mode 100644
index df7f191..0000000
--- a/1_wqp_pull/out/190412203340_034.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 5ef189e188869d0cb7b6b1992fa120e6
-
diff --git a/1_wqp_pull/out/190412203340_035.feather.ind b/1_wqp_pull/out/190412203340_035.feather.ind
deleted file mode 100644
index 3c6623b..0000000
--- a/1_wqp_pull/out/190412203340_035.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 627c79a8b96d0361d6ae88efcf94f79d
-
diff --git a/1_wqp_pull/out/190412203340_036.feather.ind b/1_wqp_pull/out/190412203340_036.feather.ind
deleted file mode 100644
index a915ff9..0000000
--- a/1_wqp_pull/out/190412203340_036.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1f893989a02568b28796b4d21dc0eb03
-
diff --git a/1_wqp_pull/out/190412203340_038.feather.ind b/1_wqp_pull/out/190412203340_038.feather.ind
deleted file mode 100644
index 5b7c95c..0000000
--- a/1_wqp_pull/out/190412203340_038.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6078cb88917c40aa5527acba0bc57da9
-
diff --git a/1_wqp_pull/out/190412203340_039.feather.ind b/1_wqp_pull/out/190412203340_039.feather.ind
deleted file mode 100644
index 2529625..0000000
--- a/1_wqp_pull/out/190412203340_039.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 253b09d95f86d60f76af41205d59e9c3
-
diff --git a/1_wqp_pull/out/190412203340_040.feather.ind b/1_wqp_pull/out/190412203340_040.feather.ind
deleted file mode 100644
index 2417365..0000000
--- a/1_wqp_pull/out/190412203340_040.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 4b9359b0e2216d10640ac4313f3ece48
-
diff --git a/1_wqp_pull/out/190412203340_041.feather.ind b/1_wqp_pull/out/190412203340_041.feather.ind
deleted file mode 100644
index 2d6bbe8..0000000
--- a/1_wqp_pull/out/190412203340_041.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: ba9cbc0238d3d44bb5979dfd7b65a984
-
diff --git a/1_wqp_pull/out/190412203340_042.feather.ind b/1_wqp_pull/out/190412203340_042.feather.ind
deleted file mode 100644
index 63bc8cf..0000000
--- a/1_wqp_pull/out/190412203340_042.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9083d03b6323c78958ec38f970298721
-
diff --git a/1_wqp_pull/out/190412203340_043.feather.ind b/1_wqp_pull/out/190412203340_043.feather.ind
deleted file mode 100644
index 9a8bf2e..0000000
--- a/1_wqp_pull/out/190412203340_043.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 26505ab686c10f3a658d57a9b28bd0ae
-
diff --git a/1_wqp_pull/out/190412203340_044.feather.ind b/1_wqp_pull/out/190412203340_044.feather.ind
deleted file mode 100644
index 43e46cf..0000000
--- a/1_wqp_pull/out/190412203340_044.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 454309537d8d2bb7a121f872a96af9e1
-
diff --git a/1_wqp_pull/out/190412203340_045.feather.ind b/1_wqp_pull/out/190412203340_045.feather.ind
deleted file mode 100644
index 80df189..0000000
--- a/1_wqp_pull/out/190412203340_045.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: bb37f010a229b93b4bb52b2d398eaba2
-
diff --git a/1_wqp_pull/out/190412203340_046.feather.ind b/1_wqp_pull/out/190412203340_046.feather.ind
deleted file mode 100644
index 091e1da..0000000
--- a/1_wqp_pull/out/190412203340_046.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0889a863ec262d4a4d1946ced4c008db
-
diff --git a/1_wqp_pull/out/190412203340_047.feather.ind b/1_wqp_pull/out/190412203340_047.feather.ind
deleted file mode 100644
index 729e2c2..0000000
--- a/1_wqp_pull/out/190412203340_047.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 03c57a06416903b63a65fca936d6394a
-
diff --git a/1_wqp_pull/out/190412203340_049.feather.ind b/1_wqp_pull/out/190412203340_049.feather.ind
deleted file mode 100644
index 3c9b105..0000000
--- a/1_wqp_pull/out/190412203340_049.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: fa17dcbc4424e56d1c2baff04e53e487
-
diff --git a/1_wqp_pull/out/190412203340_050.feather.ind b/1_wqp_pull/out/190412203340_050.feather.ind
deleted file mode 100644
index 5209148..0000000
--- a/1_wqp_pull/out/190412203340_050.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e0540dcb5413c948aa8b8a513645fede
-
diff --git a/1_wqp_pull/out/190412203340_051.feather.ind b/1_wqp_pull/out/190412203340_051.feather.ind
deleted file mode 100644
index 41eccf6..0000000
--- a/1_wqp_pull/out/190412203340_051.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9b41474a44c2401dae2ac65250b547c0
-
diff --git a/1_wqp_pull/out/190412203340_052.feather.ind b/1_wqp_pull/out/190412203340_052.feather.ind
deleted file mode 100644
index 33ad660..0000000
--- a/1_wqp_pull/out/190412203340_052.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e643d607bfbd67fa44f565035340bde0
-
diff --git a/1_wqp_pull/out/190412203340_053.feather.ind b/1_wqp_pull/out/190412203340_053.feather.ind
deleted file mode 100644
index fbb0028..0000000
--- a/1_wqp_pull/out/190412203340_053.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b40e24cad704a3c6982d8050be38f9ae
-
diff --git a/1_wqp_pull/out/190412203340_054.feather.ind b/1_wqp_pull/out/190412203340_054.feather.ind
deleted file mode 100644
index 79f6564..0000000
--- a/1_wqp_pull/out/190412203340_054.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1896e477398d772cc0e513cfb14e9381
-
diff --git a/1_wqp_pull/out/190412203340_055.feather.ind b/1_wqp_pull/out/190412203340_055.feather.ind
deleted file mode 100644
index 0f4ff8f..0000000
--- a/1_wqp_pull/out/190412203340_055.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: cdb113280983b4a3561499264b9f9f02
-
diff --git a/1_wqp_pull/out/190412203340_056.feather.ind b/1_wqp_pull/out/190412203340_056.feather.ind
deleted file mode 100644
index 4851510..0000000
--- a/1_wqp_pull/out/190412203340_056.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1c90e453973a81ef3ba7ed7fa920c8c6
-
diff --git a/1_wqp_pull/out/190412203340_057.feather.ind b/1_wqp_pull/out/190412203340_057.feather.ind
deleted file mode 100644
index f1c960b..0000000
--- a/1_wqp_pull/out/190412203340_057.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 4cebba62b56aa1f77485cc65a664f614
-
diff --git a/1_wqp_pull/out/190412203340_058.feather.ind b/1_wqp_pull/out/190412203340_058.feather.ind
deleted file mode 100644
index f42f727..0000000
--- a/1_wqp_pull/out/190412203340_058.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e7e6819cca037c920fb470de30829dc3
-
diff --git a/1_wqp_pull/out/190412203340_059.feather.ind b/1_wqp_pull/out/190412203340_059.feather.ind
deleted file mode 100644
index 72f5fd3..0000000
--- a/1_wqp_pull/out/190412203340_059.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d902dd8ccae48fbe8f49ffad8b5e2010
-
diff --git a/1_wqp_pull/out/190412203340_060.feather.ind b/1_wqp_pull/out/190412203340_060.feather.ind
deleted file mode 100644
index fffe36f..0000000
--- a/1_wqp_pull/out/190412203340_060.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: fed3c913003b66f2b21a79377b34746e
-
diff --git a/1_wqp_pull/out/190412203340_061.feather.ind b/1_wqp_pull/out/190412203340_061.feather.ind
deleted file mode 100644
index 31ca741..0000000
--- a/1_wqp_pull/out/190412203340_061.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 023b30b9fa1e93e5c817e42521b26ba6
-
diff --git a/1_wqp_pull/out/190412203340_062.feather.ind b/1_wqp_pull/out/190412203340_062.feather.ind
deleted file mode 100644
index 60788f3..0000000
--- a/1_wqp_pull/out/190412203340_062.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 485badfee5327c2e1ce7b22ef6d835e8
-
diff --git a/1_wqp_pull/out/190412203340_063.feather.ind b/1_wqp_pull/out/190412203340_063.feather.ind
deleted file mode 100644
index 08ba6d3..0000000
--- a/1_wqp_pull/out/190412203340_063.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9c94cdcf13d2f4ab1903f3582e43993f
-
diff --git a/1_wqp_pull/out/190412203340_064.feather.ind b/1_wqp_pull/out/190412203340_064.feather.ind
deleted file mode 100644
index 53c7871..0000000
--- a/1_wqp_pull/out/190412203340_064.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 056d92aa23b547dc180fcd9e3c01a90e
-
diff --git a/1_wqp_pull/out/190412203340_065.feather.ind b/1_wqp_pull/out/190412203340_065.feather.ind
deleted file mode 100644
index 1c6375e..0000000
--- a/1_wqp_pull/out/190412203340_065.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1c1f69b7745a62f608acd64daa8836d1
-
diff --git a/1_wqp_pull/out/190412203340_066.feather.ind b/1_wqp_pull/out/190412203340_066.feather.ind
deleted file mode 100644
index 30da486..0000000
--- a/1_wqp_pull/out/190412203340_066.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d8b44a1963f4d6245224f01fb92d5b31
-
diff --git a/1_wqp_pull/out/190412203340_067.feather.ind b/1_wqp_pull/out/190412203340_067.feather.ind
deleted file mode 100644
index c40d7df..0000000
--- a/1_wqp_pull/out/190412203340_067.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c5c1462b14f3c80814e99b419a8c3ffc
-
diff --git a/1_wqp_pull/out/190412203340_068.feather.ind b/1_wqp_pull/out/190412203340_068.feather.ind
deleted file mode 100644
index 3a6c1f2..0000000
--- a/1_wqp_pull/out/190412203340_068.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8022085315bdfa5b72d4bec0d979f12e
-
diff --git a/1_wqp_pull/out/190412203340_069.feather.ind b/1_wqp_pull/out/190412203340_069.feather.ind
deleted file mode 100644
index ef377c9..0000000
--- a/1_wqp_pull/out/190412203340_069.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1095f53dad4b5950869c46b63a2e9853
-
diff --git a/1_wqp_pull/out/190412203340_070.feather.ind b/1_wqp_pull/out/190412203340_070.feather.ind
deleted file mode 100644
index 238ef2c..0000000
--- a/1_wqp_pull/out/190412203340_070.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3ed053a6b11adad2a31f4dbc461619ec
-
diff --git a/1_wqp_pull/out/190412203340_071.feather.ind b/1_wqp_pull/out/190412203340_071.feather.ind
deleted file mode 100644
index 8de0796..0000000
--- a/1_wqp_pull/out/190412203340_071.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d3451bf1334d344d8d6ecb3354eb80e0
-
diff --git a/1_wqp_pull/out/190412203340_072.feather.ind b/1_wqp_pull/out/190412203340_072.feather.ind
deleted file mode 100644
index adab1f1..0000000
--- a/1_wqp_pull/out/190412203340_072.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e46d2c5a36e5fa7ee35d7fa7c5b5a08b
-
diff --git a/1_wqp_pull/out/190412203340_073.feather.ind b/1_wqp_pull/out/190412203340_073.feather.ind
deleted file mode 100644
index c7f1566..0000000
--- a/1_wqp_pull/out/190412203340_073.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6cdd14e36eef3854f0d54a69eced52ec
-
diff --git a/1_wqp_pull/out/190412203340_074.feather.ind b/1_wqp_pull/out/190412203340_074.feather.ind
deleted file mode 100644
index 6db4513..0000000
--- a/1_wqp_pull/out/190412203340_074.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e1bc11a4d5a74a35ed5675e6314c216d
-
diff --git a/1_wqp_pull/out/190412203340_075.feather.ind b/1_wqp_pull/out/190412203340_075.feather.ind
deleted file mode 100644
index c25aac3..0000000
--- a/1_wqp_pull/out/190412203340_075.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9546fd073f98b0b22a9ae8e08f88cd48
-
diff --git a/1_wqp_pull/out/190412203340_076.feather.ind b/1_wqp_pull/out/190412203340_076.feather.ind
deleted file mode 100644
index 687625e..0000000
--- a/1_wqp_pull/out/190412203340_076.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e28537e3cbff6a363c4813a3f693a5a1
-
diff --git a/1_wqp_pull/out/190412203340_077.feather.ind b/1_wqp_pull/out/190412203340_077.feather.ind
deleted file mode 100644
index 9d8aee1..0000000
--- a/1_wqp_pull/out/190412203340_077.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6495e2bfefe4f34f3603558445fefbf4
-
diff --git a/1_wqp_pull/out/190412203340_079.feather.ind b/1_wqp_pull/out/190412203340_079.feather.ind
deleted file mode 100644
index 38f75b5..0000000
--- a/1_wqp_pull/out/190412203340_079.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0179795fd48ee70aac0633de2afe7c91
-
diff --git a/1_wqp_pull/out/190412203340_080.feather.ind b/1_wqp_pull/out/190412203340_080.feather.ind
deleted file mode 100644
index f1383cc..0000000
--- a/1_wqp_pull/out/190412203340_080.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 4c121801138fe545d6ec4fb6b9cee702
-
diff --git a/1_wqp_pull/out/190412203340_081.feather.ind b/1_wqp_pull/out/190412203340_081.feather.ind
deleted file mode 100644
index e507e36..0000000
--- a/1_wqp_pull/out/190412203340_081.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e648f23533c6d6114e79610ac8d1e603
-
diff --git a/1_wqp_pull/out/190412203340_082.feather.ind b/1_wqp_pull/out/190412203340_082.feather.ind
deleted file mode 100644
index 5db330a..0000000
--- a/1_wqp_pull/out/190412203340_082.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2545bed22371a08cf37386704bdc6196
-
diff --git a/1_wqp_pull/out/190412203340_083.feather.ind b/1_wqp_pull/out/190412203340_083.feather.ind
deleted file mode 100644
index e940c1b..0000000
--- a/1_wqp_pull/out/190412203340_083.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: ef5af500c8ef02d3e024e0f822caf25d
-
diff --git a/1_wqp_pull/out/190412203340_084.feather.ind b/1_wqp_pull/out/190412203340_084.feather.ind
deleted file mode 100644
index 02182c8..0000000
--- a/1_wqp_pull/out/190412203340_084.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 4ad1f44aa91350eba01a5575dac84c3b
-
diff --git a/1_wqp_pull/out/190412203340_085.feather.ind b/1_wqp_pull/out/190412203340_085.feather.ind
deleted file mode 100644
index d936afd..0000000
--- a/1_wqp_pull/out/190412203340_085.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: e31591cae1278ace0c9392aad73597f3
-
diff --git a/1_wqp_pull/out/190412203340_086.feather.ind b/1_wqp_pull/out/190412203340_086.feather.ind
deleted file mode 100644
index ca089e0..0000000
--- a/1_wqp_pull/out/190412203340_086.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2d75a7b7c6401284af807e802b4dcd04
-
diff --git a/1_wqp_pull/out/190412203340_087.feather.ind b/1_wqp_pull/out/190412203340_087.feather.ind
deleted file mode 100644
index 3910495..0000000
--- a/1_wqp_pull/out/190412203340_087.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 975294f46a41fc4d3abfc5a7d11b7314
-
diff --git a/1_wqp_pull/out/190412203340_088.feather.ind b/1_wqp_pull/out/190412203340_088.feather.ind
deleted file mode 100644
index f91a869..0000000
--- a/1_wqp_pull/out/190412203340_088.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: ecd0fbffb0592ab92371a1780f28d380
-
diff --git a/1_wqp_pull/out/190412203340_089.feather.ind b/1_wqp_pull/out/190412203340_089.feather.ind
deleted file mode 100644
index 5850b03..0000000
--- a/1_wqp_pull/out/190412203340_089.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a33f5e1b79f04f60edac3274965467f5
-
diff --git a/1_wqp_pull/out/190412203340_090.feather.ind b/1_wqp_pull/out/190412203340_090.feather.ind
deleted file mode 100644
index c2b6f2b..0000000
--- a/1_wqp_pull/out/190412203340_090.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8578e7e04a2c330997b0a6261a6cc04c
-
diff --git a/1_wqp_pull/out/190412203340_091.feather.ind b/1_wqp_pull/out/190412203340_091.feather.ind
deleted file mode 100644
index 291ba01..0000000
--- a/1_wqp_pull/out/190412203340_091.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3b5944fe6c81ee97cbc493ce560ea99f
-
diff --git a/1_wqp_pull/out/190412203340_092.feather.ind b/1_wqp_pull/out/190412203340_092.feather.ind
deleted file mode 100644
index 7632938..0000000
--- a/1_wqp_pull/out/190412203340_092.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 42f991bffd3882e11789f04e2687e122
-
diff --git a/1_wqp_pull/out/190412203340_093.feather.ind b/1_wqp_pull/out/190412203340_093.feather.ind
deleted file mode 100644
index 067b71c..0000000
--- a/1_wqp_pull/out/190412203340_093.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 468a9ff83bf52fcc852ee3a24a3bebaf
-
diff --git a/1_wqp_pull/out/190412203340_094.feather.ind b/1_wqp_pull/out/190412203340_094.feather.ind
deleted file mode 100644
index 8e20312..0000000
--- a/1_wqp_pull/out/190412203340_094.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d18711742727c30054db295aaeb5d0ae
-
diff --git a/1_wqp_pull/out/190412203340_095.feather.ind b/1_wqp_pull/out/190412203340_095.feather.ind
deleted file mode 100644
index 8285f14..0000000
--- a/1_wqp_pull/out/190412203340_095.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f4e9a1050ae5a0005c97d44518a91042
-
diff --git a/1_wqp_pull/out/190412203340_096.feather.ind b/1_wqp_pull/out/190412203340_096.feather.ind
deleted file mode 100644
index 37358c6..0000000
--- a/1_wqp_pull/out/190412203340_096.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a07a5a32fac6622dcea6438fc1888c4f
-
diff --git a/1_wqp_pull/out/190412203340_097.feather.ind b/1_wqp_pull/out/190412203340_097.feather.ind
deleted file mode 100644
index 95116a7..0000000
--- a/1_wqp_pull/out/190412203340_097.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 36526f78d251f2da806054949213d8b0
-
diff --git a/1_wqp_pull/out/190412203340_098.feather.ind b/1_wqp_pull/out/190412203340_098.feather.ind
deleted file mode 100644
index 5353625..0000000
--- a/1_wqp_pull/out/190412203340_098.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 37714accb5fabfb2b0486fb0f92a3cf5
-
diff --git a/1_wqp_pull/out/190412203340_099.feather.ind b/1_wqp_pull/out/190412203340_099.feather.ind
deleted file mode 100644
index dcebbc6..0000000
--- a/1_wqp_pull/out/190412203340_099.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c341920d7581d190639162d210725752
-
diff --git a/1_wqp_pull/out/190412203340_100.feather.ind b/1_wqp_pull/out/190412203340_100.feather.ind
deleted file mode 100644
index d79763f..0000000
--- a/1_wqp_pull/out/190412203340_100.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f75ff68ffaa04ff33dab195475145932
-
diff --git a/1_wqp_pull/out/190412203340_101.feather.ind b/1_wqp_pull/out/190412203340_101.feather.ind
deleted file mode 100644
index 6f20dcf..0000000
--- a/1_wqp_pull/out/190412203340_101.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 580eedd93d7dd79b2b3011e450be302d
-
diff --git a/1_wqp_pull/out/190412203340_102.feather.ind b/1_wqp_pull/out/190412203340_102.feather.ind
deleted file mode 100644
index 82df6fa..0000000
--- a/1_wqp_pull/out/190412203340_102.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0962444f7b7e5c0c3f8546294cfc3860
-
diff --git a/1_wqp_pull/out/190412203340_103.feather.ind b/1_wqp_pull/out/190412203340_103.feather.ind
deleted file mode 100644
index 2f22ec6..0000000
--- a/1_wqp_pull/out/190412203340_103.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 828ab21e87a1a1480128d63db3fe338d
-
diff --git a/1_wqp_pull/out/190412203340_104.feather.ind b/1_wqp_pull/out/190412203340_104.feather.ind
deleted file mode 100644
index bcf452b..0000000
--- a/1_wqp_pull/out/190412203340_104.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a82339e938e722319e388f6db176f17e
-
diff --git a/1_wqp_pull/out/190412203340_105.feather.ind b/1_wqp_pull/out/190412203340_105.feather.ind
deleted file mode 100644
index 7756303..0000000
--- a/1_wqp_pull/out/190412203340_105.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1d554d410e01e564481ab7ced6b007ed
-
diff --git a/1_wqp_pull/out/190412203340_106.feather.ind b/1_wqp_pull/out/190412203340_106.feather.ind
deleted file mode 100644
index d391601..0000000
--- a/1_wqp_pull/out/190412203340_106.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0a8f1e3929ffffb0e692e44927539d61
-
diff --git a/1_wqp_pull/out/190412203340_107.feather.ind b/1_wqp_pull/out/190412203340_107.feather.ind
deleted file mode 100644
index 318c403..0000000
--- a/1_wqp_pull/out/190412203340_107.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 89b75783d92730332b36e61e1c88450a
-
diff --git a/1_wqp_pull/out/190412203340_108.feather.ind b/1_wqp_pull/out/190412203340_108.feather.ind
deleted file mode 100644
index 7a36efa..0000000
--- a/1_wqp_pull/out/190412203340_108.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9d99681b704a5140afae731bc652c97a
-
diff --git a/1_wqp_pull/out/190412203340_109.feather.ind b/1_wqp_pull/out/190412203340_109.feather.ind
deleted file mode 100644
index 0ca19ba..0000000
--- a/1_wqp_pull/out/190412203340_109.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1df274f1f73f2af0b7dff60be659b6c5
-
diff --git a/1_wqp_pull/out/190412203340_110.feather.ind b/1_wqp_pull/out/190412203340_110.feather.ind
deleted file mode 100644
index 76796a7..0000000
--- a/1_wqp_pull/out/190412203340_110.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d22cfe97bd25de88e90887fc6babc540
-
diff --git a/1_wqp_pull/out/190412203340_111.feather.ind b/1_wqp_pull/out/190412203340_111.feather.ind
deleted file mode 100644
index 7a1e5fa..0000000
--- a/1_wqp_pull/out/190412203340_111.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 1a7d90052c9a7a6b5b498cc1b0167f08
-
diff --git a/1_wqp_pull/out/190412203340_112.feather.ind b/1_wqp_pull/out/190412203340_112.feather.ind
deleted file mode 100644
index 3546ad8..0000000
--- a/1_wqp_pull/out/190412203340_112.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: df8147251c08720a2bf77a6d60b77793
-
diff --git a/1_wqp_pull/out/190412203340_113.feather.ind b/1_wqp_pull/out/190412203340_113.feather.ind
deleted file mode 100644
index bdb83ba..0000000
--- a/1_wqp_pull/out/190412203340_113.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6ec906c9337eea7c3ba3b8681d21c1a9
-
diff --git a/1_wqp_pull/out/190412203340_115.feather.ind b/1_wqp_pull/out/190412203340_115.feather.ind
deleted file mode 100644
index c4710c1..0000000
--- a/1_wqp_pull/out/190412203340_115.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: db88a8ef3a3554d4255026bd1f4e5de1
-
diff --git a/1_wqp_pull/out/190412203340_116.feather.ind b/1_wqp_pull/out/190412203340_116.feather.ind
deleted file mode 100644
index 5441d19..0000000
--- a/1_wqp_pull/out/190412203340_116.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 054176646387f1256d4c4a02c991cd85
-
diff --git a/1_wqp_pull/out/190412203340_117.feather.ind b/1_wqp_pull/out/190412203340_117.feather.ind
deleted file mode 100644
index eacaea3..0000000
--- a/1_wqp_pull/out/190412203340_117.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 844061891269740c0c43e40edd0e3f65
-
diff --git a/1_wqp_pull/out/190412203340_118.feather.ind b/1_wqp_pull/out/190412203340_118.feather.ind
deleted file mode 100644
index 5860f74..0000000
--- a/1_wqp_pull/out/190412203340_118.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 87783c2755aec85251334c3c655c8883
-
diff --git a/1_wqp_pull/out/190412203340_119.feather.ind b/1_wqp_pull/out/190412203340_119.feather.ind
deleted file mode 100644
index a8bb1a9..0000000
--- a/1_wqp_pull/out/190412203340_119.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a56f632234b499efdda4f8cbae45488d
-
diff --git a/1_wqp_pull/out/190412203340_120.feather.ind b/1_wqp_pull/out/190412203340_120.feather.ind
deleted file mode 100644
index e916d5f..0000000
--- a/1_wqp_pull/out/190412203340_120.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: cd48da9f8d7fd367b39eb7ac6ae0d3c2
-
diff --git a/1_wqp_pull/out/190412203340_121.feather.ind b/1_wqp_pull/out/190412203340_121.feather.ind
deleted file mode 100644
index fdf5e4f..0000000
--- a/1_wqp_pull/out/190412203340_121.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8507eff5b6646ea3e9d994ef395c8842
-
diff --git a/1_wqp_pull/out/190412203340_122.feather.ind b/1_wqp_pull/out/190412203340_122.feather.ind
deleted file mode 100644
index 05f9463..0000000
--- a/1_wqp_pull/out/190412203340_122.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9696f6841b87a765d502dd76dac2c5d5
-
diff --git a/1_wqp_pull/out/190412203340_123.feather.ind b/1_wqp_pull/out/190412203340_123.feather.ind
deleted file mode 100644
index a75c7a2..0000000
--- a/1_wqp_pull/out/190412203340_123.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a5a62935a8363b714050a44c2009a8cf
-
diff --git a/1_wqp_pull/out/190412203340_124.feather.ind b/1_wqp_pull/out/190412203340_124.feather.ind
deleted file mode 100644
index 6938c02..0000000
--- a/1_wqp_pull/out/190412203340_124.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 02c81c11e47ed08b01228a777dd62de8
-
diff --git a/1_wqp_pull/out/190412203340_125.feather.ind b/1_wqp_pull/out/190412203340_125.feather.ind
deleted file mode 100644
index 024b07f..0000000
--- a/1_wqp_pull/out/190412203340_125.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: fff49a1583e262335fba312d7763014b
-
diff --git a/1_wqp_pull/out/190412203340_126.feather.ind b/1_wqp_pull/out/190412203340_126.feather.ind
deleted file mode 100644
index 3b414b9..0000000
--- a/1_wqp_pull/out/190412203340_126.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d865fe8cb15914f46c15260934f574c3
-
diff --git a/1_wqp_pull/out/190412203340_127.feather.ind b/1_wqp_pull/out/190412203340_127.feather.ind
deleted file mode 100644
index 4de7b7c..0000000
--- a/1_wqp_pull/out/190412203340_127.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b0d3b9b7d6c00b568de17fe23a280f4e
-
diff --git a/1_wqp_pull/out/190412203340_128.feather.ind b/1_wqp_pull/out/190412203340_128.feather.ind
deleted file mode 100644
index d9ccd1e..0000000
--- a/1_wqp_pull/out/190412203340_128.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 01e7855ba45fa3b252577b32bc9564d8
-
diff --git a/1_wqp_pull/out/190412203340_129.feather.ind b/1_wqp_pull/out/190412203340_129.feather.ind
deleted file mode 100644
index 2883932..0000000
--- a/1_wqp_pull/out/190412203340_129.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 31ce155911fafa078e303d59cd8bd6a3
-
diff --git a/1_wqp_pull/out/190412203340_130.feather.ind b/1_wqp_pull/out/190412203340_130.feather.ind
deleted file mode 100644
index 49d228f..0000000
--- a/1_wqp_pull/out/190412203340_130.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7a6115e574f01de8b504c4e6b59af360
-
diff --git a/1_wqp_pull/out/190412203340_131.feather.ind b/1_wqp_pull/out/190412203340_131.feather.ind
deleted file mode 100644
index c2e25a0..0000000
--- a/1_wqp_pull/out/190412203340_131.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 74f0141ea3c55b3d7b519b3b53ea8009
-
diff --git a/1_wqp_pull/out/190412203340_132.feather.ind b/1_wqp_pull/out/190412203340_132.feather.ind
deleted file mode 100644
index 8e0303a..0000000
--- a/1_wqp_pull/out/190412203340_132.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d55ebdc91cbf1dff0c2fcf2f6242591f
-
diff --git a/1_wqp_pull/out/190412203340_133.feather.ind b/1_wqp_pull/out/190412203340_133.feather.ind
deleted file mode 100644
index 9bf6d23..0000000
--- a/1_wqp_pull/out/190412203340_133.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 4611fcb79ddb6831d2f4c34a22e7d0e8
-
diff --git a/1_wqp_pull/out/190412203340_134.feather.ind b/1_wqp_pull/out/190412203340_134.feather.ind
deleted file mode 100644
index 0aa2d41..0000000
--- a/1_wqp_pull/out/190412203340_134.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b7124642fac5f0c6d0783e1950ed5b24
-
diff --git a/1_wqp_pull/out/190412203340_135.feather.ind b/1_wqp_pull/out/190412203340_135.feather.ind
deleted file mode 100644
index 892fbda..0000000
--- a/1_wqp_pull/out/190412203340_135.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 6a00aa13cfc491b72baab20a306d0a9b
-
diff --git a/1_wqp_pull/out/190412203340_136.feather.ind b/1_wqp_pull/out/190412203340_136.feather.ind
deleted file mode 100644
index 3268d12..0000000
--- a/1_wqp_pull/out/190412203340_136.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b3c84b199dd303ce5fe9a540dda87943
-
diff --git a/1_wqp_pull/out/190412203340_137.feather.ind b/1_wqp_pull/out/190412203340_137.feather.ind
deleted file mode 100644
index 6fd0077..0000000
--- a/1_wqp_pull/out/190412203340_137.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 888a966d2569d1a2dc0ecee90d970a07
-
diff --git a/1_wqp_pull/out/190412203340_138.feather.ind b/1_wqp_pull/out/190412203340_138.feather.ind
deleted file mode 100644
index fde9d23..0000000
--- a/1_wqp_pull/out/190412203340_138.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d780c2b8a139a2a939b1ac0e14cdfe91
-
diff --git a/1_wqp_pull/out/190412203340_139.feather.ind b/1_wqp_pull/out/190412203340_139.feather.ind
deleted file mode 100644
index 5649f0f..0000000
--- a/1_wqp_pull/out/190412203340_139.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b66f31d471250ab34fb2a82cb1f85705
-
diff --git a/1_wqp_pull/out/190412203340_140.feather.ind b/1_wqp_pull/out/190412203340_140.feather.ind
deleted file mode 100644
index 187b76d..0000000
--- a/1_wqp_pull/out/190412203340_140.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c7bc55acdb0ddb4848ff9505072726f4
-
diff --git a/1_wqp_pull/out/190412203340_141.feather.ind b/1_wqp_pull/out/190412203340_141.feather.ind
deleted file mode 100644
index c13594f..0000000
--- a/1_wqp_pull/out/190412203340_141.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3cc3605a554406cd68c00ff6cca76089
-
diff --git a/1_wqp_pull/out/190412203340_142.feather.ind b/1_wqp_pull/out/190412203340_142.feather.ind
deleted file mode 100644
index 2595dc2..0000000
--- a/1_wqp_pull/out/190412203340_142.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 563a825154cfea4c8f8b7c666f10e067
-
diff --git a/1_wqp_pull/out/190412203340_143.feather.ind b/1_wqp_pull/out/190412203340_143.feather.ind
deleted file mode 100644
index f9884e8..0000000
--- a/1_wqp_pull/out/190412203340_143.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 5ce683b08304a9dadc1e1d79dd9387de
-
diff --git a/1_wqp_pull/out/190412203340_144.feather.ind b/1_wqp_pull/out/190412203340_144.feather.ind
deleted file mode 100644
index 4e6b7d2..0000000
--- a/1_wqp_pull/out/190412203340_144.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c3746151bda8cf57b0a9fb6734f9c463
-
diff --git a/1_wqp_pull/out/190412203340_145.feather.ind b/1_wqp_pull/out/190412203340_145.feather.ind
deleted file mode 100644
index 0881f00..0000000
--- a/1_wqp_pull/out/190412203340_145.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 55a9ebf57a98731abbd3c7aa79d91cf1
-
diff --git a/1_wqp_pull/out/190412203340_147.feather.ind b/1_wqp_pull/out/190412203340_147.feather.ind
deleted file mode 100644
index 458cea4..0000000
--- a/1_wqp_pull/out/190412203340_147.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 367c0c33ab20bdfd7ac00071cbb0bffa
-
diff --git a/1_wqp_pull/out/190412203340_148.feather.ind b/1_wqp_pull/out/190412203340_148.feather.ind
deleted file mode 100644
index 2033f56..0000000
--- a/1_wqp_pull/out/190412203340_148.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 09ceacec663bf14c03cbd47d5b759a31
-
diff --git a/1_wqp_pull/out/190412203340_149.feather.ind b/1_wqp_pull/out/190412203340_149.feather.ind
deleted file mode 100644
index b9d6407..0000000
--- a/1_wqp_pull/out/190412203340_149.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 56f8cd9f573efbca4ce1adc262983658
-
diff --git a/1_wqp_pull/out/190412203340_150.feather.ind b/1_wqp_pull/out/190412203340_150.feather.ind
deleted file mode 100644
index a857c90..0000000
--- a/1_wqp_pull/out/190412203340_150.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 9c639906be9e2c4c099442a4b9f8d8a7
-
diff --git a/1_wqp_pull/out/190412203340_151.feather.ind b/1_wqp_pull/out/190412203340_151.feather.ind
deleted file mode 100644
index ad8574c..0000000
--- a/1_wqp_pull/out/190412203340_151.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 5aa052c97d737fb897cd6574a4b5fc00
-
diff --git a/1_wqp_pull/out/190412203340_152.feather.ind b/1_wqp_pull/out/190412203340_152.feather.ind
deleted file mode 100644
index faebfba..0000000
--- a/1_wqp_pull/out/190412203340_152.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7d3041605cdb35f737b1d4e5f354a459
-
diff --git a/1_wqp_pull/out/190412203340_153.feather.ind b/1_wqp_pull/out/190412203340_153.feather.ind
deleted file mode 100644
index 42b9906..0000000
--- a/1_wqp_pull/out/190412203340_153.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 12fecd619586f9b4caeae5bfbf5e8ae9
-
diff --git a/1_wqp_pull/out/190412203340_154.feather.ind b/1_wqp_pull/out/190412203340_154.feather.ind
deleted file mode 100644
index ab7cb98..0000000
--- a/1_wqp_pull/out/190412203340_154.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: fe556607545623f7c6dfa18afbc9762d
-
diff --git a/1_wqp_pull/out/190412203340_155.feather.ind b/1_wqp_pull/out/190412203340_155.feather.ind
deleted file mode 100644
index 2793484..0000000
--- a/1_wqp_pull/out/190412203340_155.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f1053fa6d7165e26a0c17689cc689eca
-
diff --git a/1_wqp_pull/out/190412203340_156.feather.ind b/1_wqp_pull/out/190412203340_156.feather.ind
deleted file mode 100644
index 2df1acf..0000000
--- a/1_wqp_pull/out/190412203340_156.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7cd40ecce9fba8f8dbea25eb5a52fef0
-
diff --git a/1_wqp_pull/out/190412203340_157.feather.ind b/1_wqp_pull/out/190412203340_157.feather.ind
deleted file mode 100644
index 213dae3..0000000
--- a/1_wqp_pull/out/190412203340_157.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7bcec318f2fddb6f3c457cc0cac3a965
-
diff --git a/1_wqp_pull/out/190412203340_158.feather.ind b/1_wqp_pull/out/190412203340_158.feather.ind
deleted file mode 100644
index e212850..0000000
--- a/1_wqp_pull/out/190412203340_158.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c9a96afce340a28461ba975a05517171
-
diff --git a/1_wqp_pull/out/190412203340_159.feather.ind b/1_wqp_pull/out/190412203340_159.feather.ind
deleted file mode 100644
index 23b54c9..0000000
--- a/1_wqp_pull/out/190412203340_159.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: f673374c2f4233d0d6d8557cf4e7e599
-
diff --git a/1_wqp_pull/out/190412203340_160.feather.ind b/1_wqp_pull/out/190412203340_160.feather.ind
deleted file mode 100644
index 244743e..0000000
--- a/1_wqp_pull/out/190412203340_160.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 49a6c3fd3b0c08b6641c581d22490053
-
diff --git a/1_wqp_pull/out/190412203340_162.feather.ind b/1_wqp_pull/out/190412203340_162.feather.ind
deleted file mode 100644
index 626200a..0000000
--- a/1_wqp_pull/out/190412203340_162.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7dccd341898d4a17de110d1511abe72b
-
diff --git a/1_wqp_pull/out/190412203340_163.feather.ind b/1_wqp_pull/out/190412203340_163.feather.ind
deleted file mode 100644
index c4b164f..0000000
--- a/1_wqp_pull/out/190412203340_163.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2c25c89bd0569631ea2ac4c4fc9a5e53
-
diff --git a/1_wqp_pull/out/190412203340_164.feather.ind b/1_wqp_pull/out/190412203340_164.feather.ind
deleted file mode 100644
index 38e6ab5..0000000
--- a/1_wqp_pull/out/190412203340_164.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: dc0440f9900257ebcc0dbf17303443be
-
diff --git a/1_wqp_pull/out/190412203340_165.feather.ind b/1_wqp_pull/out/190412203340_165.feather.ind
deleted file mode 100644
index 591226a..0000000
--- a/1_wqp_pull/out/190412203340_165.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0bb78cc5b84f627fcca0017266e24f65
-
diff --git a/1_wqp_pull/out/190412203340_166.feather.ind b/1_wqp_pull/out/190412203340_166.feather.ind
deleted file mode 100644
index 5b14c2a..0000000
--- a/1_wqp_pull/out/190412203340_166.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 477e3f74bfc0dbe8d65ffb59dc8978f0
-
diff --git a/1_wqp_pull/out/190412203340_167.feather.ind b/1_wqp_pull/out/190412203340_167.feather.ind
deleted file mode 100644
index 3b673e6..0000000
--- a/1_wqp_pull/out/190412203340_167.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 3600093566780e353a2258c1590c6d81
-
diff --git a/1_wqp_pull/out/190412203340_168.feather.ind b/1_wqp_pull/out/190412203340_168.feather.ind
deleted file mode 100644
index 747228d..0000000
--- a/1_wqp_pull/out/190412203340_168.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: aa112a6b3a9b2d264e14a2f36ba9fa17
-
diff --git a/1_wqp_pull/out/190412203340_169.feather.ind b/1_wqp_pull/out/190412203340_169.feather.ind
deleted file mode 100644
index 92664c0..0000000
--- a/1_wqp_pull/out/190412203340_169.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 85e28e49bee2914eb6a06c7216f86b1a
-
diff --git a/1_wqp_pull/out/190412203340_171.feather.ind b/1_wqp_pull/out/190412203340_171.feather.ind
deleted file mode 100644
index f1c64f3..0000000
--- a/1_wqp_pull/out/190412203340_171.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: c1053261923356bd8bd257bd6f6e30bf
-
diff --git a/1_wqp_pull/out/190412203340_172.feather.ind b/1_wqp_pull/out/190412203340_172.feather.ind
deleted file mode 100644
index 6dc753d..0000000
--- a/1_wqp_pull/out/190412203340_172.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2e518b5ef7ee733e984e58ea79be3d7c
-
diff --git a/1_wqp_pull/out/190412203340_173.feather.ind b/1_wqp_pull/out/190412203340_173.feather.ind
deleted file mode 100644
index 7128569..0000000
--- a/1_wqp_pull/out/190412203340_173.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 69f2c15445ee7d3047719c8d561c3cbc
-
diff --git a/1_wqp_pull/out/190412203340_174.feather.ind b/1_wqp_pull/out/190412203340_174.feather.ind
deleted file mode 100644
index 10b61d8..0000000
--- a/1_wqp_pull/out/190412203340_174.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: b126d330890484d83dac78869c236cbf
-
diff --git a/1_wqp_pull/out/190412203340_175.feather.ind b/1_wqp_pull/out/190412203340_175.feather.ind
deleted file mode 100644
index be8fa12..0000000
--- a/1_wqp_pull/out/190412203340_175.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: a55a234e1e9b62dd2e0aeace460154db
-
diff --git a/1_wqp_pull/out/190412203340_176.feather.ind b/1_wqp_pull/out/190412203340_176.feather.ind
deleted file mode 100644
index 179e4fc..0000000
--- a/1_wqp_pull/out/190412203340_176.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: db76f7da813ca47ed206a7483adb31e8
-
diff --git a/1_wqp_pull/out/190412203340_177.feather.ind b/1_wqp_pull/out/190412203340_177.feather.ind
deleted file mode 100644
index a6f8ef3..0000000
--- a/1_wqp_pull/out/190412203340_177.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 56c30d86b48084d8b06b3f114bab8b7d
-
diff --git a/1_wqp_pull/out/190412203340_178.feather.ind b/1_wqp_pull/out/190412203340_178.feather.ind
deleted file mode 100644
index f31ac76..0000000
--- a/1_wqp_pull/out/190412203340_178.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 0591bc743a3c3b94a2f27d3e2d214840
-
diff --git a/1_wqp_pull/out/190412203340_179.feather.ind b/1_wqp_pull/out/190412203340_179.feather.ind
deleted file mode 100644
index 5db5624..0000000
--- a/1_wqp_pull/out/190412203340_179.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 8a45957ff32049b9e3be9203fd9a20ac
-
diff --git a/1_wqp_pull/out/190412203340_180.feather.ind b/1_wqp_pull/out/190412203340_180.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_180.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_181.feather.ind b/1_wqp_pull/out/190412203340_181.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_181.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_182.feather.ind b/1_wqp_pull/out/190412203340_182.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_182.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_183.feather.ind b/1_wqp_pull/out/190412203340_183.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_183.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_184.feather.ind b/1_wqp_pull/out/190412203340_184.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_184.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_185.feather.ind b/1_wqp_pull/out/190412203340_185.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_185.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_186.feather.ind b/1_wqp_pull/out/190412203340_186.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_186.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_187.feather.ind b/1_wqp_pull/out/190412203340_187.feather.ind
deleted file mode 100644
index 66212ff..0000000
--- a/1_wqp_pull/out/190412203340_187.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d54ef41f769cf1f67b73a54d7f33fe29
-
diff --git a/1_wqp_pull/out/190412203340_188.feather.ind b/1_wqp_pull/out/190412203340_188.feather.ind
deleted file mode 100644
index ca62f90..0000000
--- a/1_wqp_pull/out/190412203340_188.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 622a4568ee3ecdaa4b17fff4ba15e88e
-
diff --git a/1_wqp_pull/out/190412203340_189.feather.ind b/1_wqp_pull/out/190412203340_189.feather.ind
deleted file mode 100644
index 161d0a0..0000000
--- a/1_wqp_pull/out/190412203340_189.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2a8426883a31f340e3182bf465df4c98
-
diff --git a/1_wqp_pull/out/190412203340_190.feather.ind b/1_wqp_pull/out/190412203340_190.feather.ind
deleted file mode 100644
index 9ffa24a..0000000
--- a/1_wqp_pull/out/190412203340_190.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 682b7b6ebf33d2534f3fa5c75e6c5421
-
diff --git a/1_wqp_pull/out/190412203340_191.feather.ind b/1_wqp_pull/out/190412203340_191.feather.ind
deleted file mode 100644
index bc506f6..0000000
--- a/1_wqp_pull/out/190412203340_191.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: dc745117874e459ef2646c20cf46b3a8
-
diff --git a/1_wqp_pull/out/190412203340_192.feather.ind b/1_wqp_pull/out/190412203340_192.feather.ind
deleted file mode 100644
index f66e068..0000000
--- a/1_wqp_pull/out/190412203340_192.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 2c31a6d42e1aac0d9f45cd0d0cdd7969
-
diff --git a/1_wqp_pull/out/190412203340_193.feather.ind b/1_wqp_pull/out/190412203340_193.feather.ind
deleted file mode 100644
index 6d72fb6..0000000
--- a/1_wqp_pull/out/190412203340_193.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: d5defcce6ff3b5eb27a6560249237323
-
diff --git a/1_wqp_pull/out/190412203340_194.feather.ind b/1_wqp_pull/out/190412203340_194.feather.ind
deleted file mode 100644
index e9bd97a..0000000
--- a/1_wqp_pull/out/190412203340_194.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 772316d67097af62830cfaca94cf209a
-
diff --git a/1_wqp_pull/out/190412203340_196.feather.ind b/1_wqp_pull/out/190412203340_196.feather.ind
deleted file mode 100644
index 7a99513..0000000
--- a/1_wqp_pull/out/190412203340_196.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 83466239ed3f5d634e1b5525ed021390
-
diff --git a/1_wqp_pull/out/190412203340_197.feather.ind b/1_wqp_pull/out/190412203340_197.feather.ind
deleted file mode 100644
index a1dd21f..0000000
--- a/1_wqp_pull/out/190412203340_197.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: bcf0784badf8ee339f4a77ae2e2c1d68
-
diff --git a/1_wqp_pull/out/190412203340_198.feather.ind b/1_wqp_pull/out/190412203340_198.feather.ind
deleted file mode 100644
index 8cc9479..0000000
--- a/1_wqp_pull/out/190412203340_198.feather.ind
+++ /dev/null
@@ -1,2 +0,0 @@
-hash: 7192cfa7d55d8eecf9321bb2973e83d0
-
diff --git a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
index dbde90f..57ee059 100644
--- a/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
+++ b/build/status/MV93cXBfcHVsbC9pbm91dC93cXBfcHVsbF9wYXJ0aXRpb25zLmZlYXRoZXIuaW5k.yml
@@ -1,11 +1,11 @@
 version: 0.3.0
 name: 1_wqp_pull/inout/wqp_pull_partitions.feather.ind
 type: file
-hash: d34136e22268f73af5da4a2632be42b2
-time: 2019-09-23 14:35:56 UTC
+hash: dce7193f7c2d879e6b794e21cefea34d
+time: 2019-09-24 16:48:05 UTC
 depends:
   wqp_pull_parameters: 61e80f749c74948bf95bb67cb07412b0
-  1_wqp_pull/inout/wqp_inventory.feather.ind: 756985b1caa92e538e06dc55de5b67ec
+  1_wqp_pull/inout/wqp_inventory.feather.ind: b78394ef0d22bdd698afee8e912525c7
   wqp_partition_cfg: dc9f1a6787144e2e96aafb48b1ee67ce
 fixed: f1ee901bf2a86f4bac0c08cf4480a175
 code:

From 65573f00a4faa98a4a24ceabecc8833aced6689d Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Thu, 26 Sep 2019 16:54:53 -0500
Subject: [PATCH 13/22] fixes issues with memory allocation for combining all
 of the WQP pulls. Now WQP data in one file.

---
 1_wqp_pull.yml                                     |  8 +++++++-
 1_wqp_pull/inout/wqp_pull_partitions.feather.ind   |  2 +-
 1_wqp_pull/out/wqp_data.rds.ind                    |  2 ++
 1_wqp_pull/src/wqp_pull.R                          | 12 +++++++++---
 .../MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml | 14 ++++++++++++++
 5 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 1_wqp_pull/out/wqp_data.rds.ind
 create mode 100644 build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml

diff --git a/1_wqp_pull.yml b/1_wqp_pull.yml
index 56261aa..87decb8 100644
--- a/1_wqp_pull.yml
+++ b/1_wqp_pull.yml
@@ -17,6 +17,7 @@ sources:
 targets:
   1_wqp_pull:
     depends:
+      - 1_wqp_pull/out/wqp_data.rds.ind
       
 # -- get a CONUS inventory of available data for download -- #
 
@@ -69,4 +70,9 @@ targets:
       ind_file=target_name,
       task_plan=wqp_pull_plan, 
       task_makefile='1_wqp_pull_tasks.yml',
-      num_tries=I(30), sleep_on_error=I(20))
\ No newline at end of file
+      num_tries=I(30), sleep_on_error=I(20))
+    depends:
+      - 1_wqp_pull/src/wqp_pull.R
+      
+  1_wqp_pull/out/wqp_data.rds:
+    command: gd_get('1_wqp_pull/out/wqp_data.rds.ind')
\ No newline at end of file
diff --git a/1_wqp_pull/inout/wqp_pull_partitions.feather.ind b/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
index ab8a8fc..a011c65 100644
--- a/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
+++ b/1_wqp_pull/inout/wqp_pull_partitions.feather.ind
@@ -1,2 +1,2 @@
-hash: f5f6e494489918022cb3397ac12bcfd8
+hash: 01c2b6436068c9d36a5be2995fd768d4
 
diff --git a/1_wqp_pull/out/wqp_data.rds.ind b/1_wqp_pull/out/wqp_data.rds.ind
new file mode 100644
index 0000000..1399b58
--- /dev/null
+++ b/1_wqp_pull/out/wqp_data.rds.ind
@@ -0,0 +1,2 @@
+hash: 87936d6c0d201843cf869d170e474420
+
diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 2168c37..1e99837 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -199,16 +199,22 @@ wqp_POST <- function(wqp_args_list){
 
 # extract and post the data (dropping the site info attribute), creating an
 # .ind file that we will want to share because it represents the shared cache
-combine_wqp_data <- function(ind_file, ...){
+combine_wqp_dat <- function(ind_file, ...){
   
   rds_files <- c(...)
   df_list <- list()
   
+  # create a readRDS function that accomdates some column type issues
+  readRDS2 <- function(.) {
+    dat_mod <- readRDS(.) %>%
+      select_if(~!any(is.na(.)))
+  }
+  
   for (i in seq_len(length(rds_files))){
-    df_list[[i]] <- readRDS(rds_files[i])
+    df_list[[i]] <- readRDS2(rds_files[i])
   }
   
-  wqp_df <- do.call("rbind", df_list)
+  wqp_df <- do.call("bind_rows", df_list)
   
   data_file <- scipiper::as_data_file(ind_file)
   saveRDS(wqp_df, data_file)
diff --git a/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
new file mode 100644
index 0000000..0f009f9
--- /dev/null
+++ b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
@@ -0,0 +1,14 @@
+version: 0.3.0
+name: 1_wqp_pull/out/wqp_data.rds.ind
+type: file
+hash: 3991b857f64a708f000c56fbb7ba0ea9
+time: 2019-09-26 21:50:29 UTC
+depends:
+  wqp_pull_plan: 06fa9cd23fcab0ccbfd17bdad932f8e7
+  1_wqp_pull_tasks.yml: 8296cac3995482e0da9968ed30a57050
+  1_wqp_pull/src/wqp_pull.R: 9d6cf0c7a5551570efc472f7e31e695c
+fixed: b103af1551106aa1ebfea5ec48fe44e0
+code:
+  functions:
+    loop_wqp_tasks: 33f57e757da0dd017ed22a8462d6916e
+

From 659db523c84605e4ef44241b26e11f6cb4d036dc Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 27 Sep 2019 08:33:10 -0500
Subject: [PATCH 14/22] Oops, I wanted all, not any!

---
 1_wqp_pull/src/wqp_pull.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 1e99837..87dc1e8 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -207,7 +207,7 @@ combine_wqp_dat <- function(ind_file, ...){
   # create a readRDS function that accomdates some column type issues
   readRDS2 <- function(.) {
     dat_mod <- readRDS(.) %>%
-      select_if(~!any(is.na(.)))
+      select_if(~!all(is.na(.)))
   }
   
   for (i in seq_len(length(rds_files))){

From b3022cea834fc0560bc00fb807c1839bffebbc09 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 27 Sep 2019 10:11:31 -0500
Subject: [PATCH 15/22] Added some manual grabs of columns to reduce size.

---
 1_wqp_pull/out/wqp_data.rds.ind                   |  2 +-
 1_wqp_pull/src/wqp_pull.R                         | 15 ++++++++++++++-
 ...MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml |  6 +++---
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/1_wqp_pull/out/wqp_data.rds.ind b/1_wqp_pull/out/wqp_data.rds.ind
index 1399b58..1482170 100644
--- a/1_wqp_pull/out/wqp_data.rds.ind
+++ b/1_wqp_pull/out/wqp_data.rds.ind
@@ -1,2 +1,2 @@
-hash: 87936d6c0d201843cf869d170e474420
+hash: 56a9468ec55d588d9d2f960074fd5f8b
 
diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 87dc1e8..94a2bfb 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -207,13 +207,26 @@ combine_wqp_dat <- function(ind_file, ...){
   # create a readRDS function that accomdates some column type issues
   readRDS2 <- function(.) {
     dat_mod <- readRDS(.) %>%
+      filter(!is.na(ResultMeasureValue)) %>%
+      select(MonitoringLocationIdentifier, ActivityMediaName, ActivityMediaSubdivisionName, ActivityStartDate, `ActivityStartTime/Time`, `ActivityStartTime/TimeZoneCode`,
+             `ActivityDepthHeightMeasure/MeasureValue`, `ActivityDepthHeightMeasure/MeasureUnitCode`,   
+            `ActivityTopDepthHeightMeasure/MeasureValue`,`ActivityTopDepthHeightMeasure/MeasureUnitCode`, 
+            `ActivityBottomDepthHeightMeasure/MeasureValue`, `ActivityBottomDepthHeightMeasure/MeasureUnitCode`,
+            ActivityCommentText, `SampleCollectionMethod/MethodIdentifier`, `SampleCollectionMethod/MethodIdentifierContext`,
+            `SampleCollectionMethod/MethodName`,ResultDetectionConditionText, CharacteristicName, ResultMeasureValue, 
+            `ResultMeasure/MeasureUnitCode`, MeasureQualifierCode,
+            ResultStatusIdentifier, StatisticalBaseCode, ResultValueTypeName, PrecisionValue, 
+            ResultCommentText, USGSPCode, `ResultDepthHeightMeasure/MeasureValue`, `ResultDepthHeightMeasure/MeasureUnitCode`, ProviderName) %>%
+      mutate(PrecisionValue = as.numeric(PrecisionValue)) %>%
+      mutate_at(vars(contains('MeasureValue')), is.numeric) %>%
+      mutate_if(is.logical, as.character) %>%
       select_if(~!all(is.na(.)))
   }
   
   for (i in seq_len(length(rds_files))){
     df_list[[i]] <- readRDS2(rds_files[i])
   }
-  
+  message('Binding all WQP files.')
   wqp_df <- do.call("bind_rows", df_list)
   
   data_file <- scipiper::as_data_file(ind_file)
diff --git a/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
index 0f009f9..cdabf7e 100644
--- a/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
+++ b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
@@ -1,12 +1,12 @@
 version: 0.3.0
 name: 1_wqp_pull/out/wqp_data.rds.ind
 type: file
-hash: 3991b857f64a708f000c56fbb7ba0ea9
-time: 2019-09-26 21:50:29 UTC
+hash: 484e253c6c20d56c22c4279c523000c1
+time: 2019-09-27 15:08:22 UTC
 depends:
   wqp_pull_plan: 06fa9cd23fcab0ccbfd17bdad932f8e7
   1_wqp_pull_tasks.yml: 8296cac3995482e0da9968ed30a57050
-  1_wqp_pull/src/wqp_pull.R: 9d6cf0c7a5551570efc472f7e31e695c
+  1_wqp_pull/src/wqp_pull.R: c4e001b821f831b8a5fff263da5ff0f0
 fixed: b103af1551106aa1ebfea5ec48fe44e0
 code:
   functions:

From fc43845f76cf9d154363797f9d6c83a8c9dc8c2a Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Fri, 27 Sep 2019 10:40:32 -0500
Subject: [PATCH 16/22] as, not is!

---
 1_wqp_pull/src/wqp_pull.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 94a2bfb..01e442d 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -218,7 +218,7 @@ combine_wqp_dat <- function(ind_file, ...){
             ResultStatusIdentifier, StatisticalBaseCode, ResultValueTypeName, PrecisionValue, 
             ResultCommentText, USGSPCode, `ResultDepthHeightMeasure/MeasureValue`, `ResultDepthHeightMeasure/MeasureUnitCode`, ProviderName) %>%
       mutate(PrecisionValue = as.numeric(PrecisionValue)) %>%
-      mutate_at(vars(contains('MeasureValue')), is.numeric) %>%
+      mutate_at(vars(contains('MeasureValue')), as.numeric) %>%
       mutate_if(is.logical, as.character) %>%
       select_if(~!all(is.na(.)))
   }

From 865622d6d83fd385f9c244be07206381272a9ae9 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 1 Oct 2019 12:04:59 -0500
Subject: [PATCH 17/22] updates to NWIS data pulls because of function change.
 These files are out of date according to remake, but are essentially up to
 date and do not need a repull. Used sc_bless!

---
 ...Qvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml | 6 +++---
 ...1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml | 6 +++---
 ...aW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml | 6 +++---
 ...1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml | 6 +++---
 .../Ml9ud2lzX3B1bGwvb3V0L253aXNfZHZfZGF0YS5yZHMuaW5k.yml    | 4 ++--
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml
index f1e293f..e66439a 100644
--- a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml
+++ b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9pbnZlbnRvcnlfcmVkdWNlZC5mZWF0aGVyLmluZA.yml
@@ -1,10 +1,10 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind
 type: file
-hash: a71b4444498828d1a1b412b2a1b6b3f9
-time: 2019-08-23 21:53:29 UTC
+hash: e7ecb2537e9324d9f5e27eb04a15b7a1
+time: 2019-09-29 17:33:05 UTC
 depends:
-  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: 8d050e361ce28f7c85583f23a1b2efbd
+  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: e34392f510d42cc6e9999fde742933b7
   2_nwis_pull/inout/nwis_compare_services.feather.ind: 9ee130dfab31bb1e751a2620009af442
 fixed: 87462faeb4ae7586edd11b9687c2273a
 code:
diff --git a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
index 84191d5..c1355a2 100644
--- a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
+++ b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc191dl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
@@ -1,10 +1,10 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_uv_partition.feather.ind
 type: file
-hash: f356933890278b81a5e273cb5a9e41ff
-time: 2019-08-23 21:53:52 UTC
+hash: 5f803bd92f7d8939f6980cd51b28fcf8
+time: 2019-09-29 17:33:31 UTC
 depends:
-  2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind: a71b4444498828d1a1b412b2a1b6b3f9
+  2_nwis_pull/inout/nwis_uv_inventory_reduced.feather.ind: e7ecb2537e9324d9f5e27eb04a15b7a1
   nwis_pull_size: bf6a897122707e205d7ad26fc0132080
 fixed: ac3fcec4b396ccf004113a8dc5e03171
 code:
diff --git a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml
index f3d210c..bc7ab44 100644
--- a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml
+++ b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19jb21wYXJlX3NlcnZpY2VzLmZlYXRoZXIuaW5k.yml
@@ -2,10 +2,10 @@ version: 0.3.0
 name: 2_nwis_pull/inout/nwis_compare_services.feather.ind
 type: file
 hash: 9ee130dfab31bb1e751a2620009af442
-time: 2019-08-23 21:53:10 UTC
+time: 2019-09-28 03:56:57 UTC
 depends:
-  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: 1cb60dc59e4be29ed2f39d29637c175f
-  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: 8d050e361ce28f7c85583f23a1b2efbd
+  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: 40827d991e1f75e6629a1192dff520d9
+  2_nwis_pull/inout/nwis_uv_inventory.feather.ind: e34392f510d42cc6e9999fde742933b7
 fixed: da0ba4d054d65f999fa79e7d3e7fc0a5
 code:
   functions:
diff --git a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
index 378372c..534b8c1 100644
--- a/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
+++ b/build/status/Ml9ud2lzX3B1bGwvaW5vdXQvbndpc19kdl9wYXJ0aXRpb24uZmVhdGhlci5pbmQ.yml
@@ -1,10 +1,10 @@
 version: 0.3.0
 name: 2_nwis_pull/inout/nwis_dv_partition.feather.ind
 type: file
-hash: e0406ab80a05125f80bfaf0d42a9da40
-time: 2019-08-23 21:51:18 UTC
+hash: a3bf9000ed2f7659ebca2cf01e8aaf92
+time: 2019-09-03 12:49:35 UTC
 depends:
-  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: 1cb60dc59e4be29ed2f39d29637c175f
+  2_nwis_pull/inout/nwis_dv_inventory.feather.ind: 40827d991e1f75e6629a1192dff520d9
   nwis_pull_size: bf6a897122707e205d7ad26fc0132080
 fixed: 4679fb1ac3904ee445bbc34f4a62eb19
 code:
diff --git a/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfZHZfZGF0YS5yZHMuaW5k.yml b/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfZHZfZGF0YS5yZHMuaW5k.yml
index e1b3d82..109e52c 100644
--- a/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfZHZfZGF0YS5yZHMuaW5k.yml
+++ b/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfZHZfZGF0YS5yZHMuaW5k.yml
@@ -1,8 +1,8 @@
 version: 0.3.0
 name: 2_nwis_pull/out/nwis_dv_data.rds.ind
 type: file
-hash: dce8eee80f8d2fe564f53056a30067c7
-time: 2019-08-26 17:52:45 UTC
+hash: ccefaf5c0aae4cca121b9e7ffa529cb2
+time: 2019-09-25 02:17:01 UTC
 depends:
   nwis_dv_pull_plan: 14e61fe31283b987683451e3008c7dbd
   2_nwis_dv_pull_tasks.yml: 1d335280b01db38bcf44f27687a69bfc

From bf70f99a38d08e5dbfae8cb60fdd7b01fae9b1fd Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Tue, 1 Oct 2019 12:09:01 -0500
Subject: [PATCH 18/22] added first cut at munging down to daily timestep. Lots
 of work to be done yet, particularly for WQP, but this is first stab. End of
 5_munge step puts everything into single file, except ignores wqp with depth
 measures for now.

---
 1_wqp_pull/out/wqp_data.rds.ind               |   2 +-
 5_data_munge.yml                              |  76 +++++++++
 5_data_munge/out/daily_temperatures.rds.ind   |   2 +
 5_data_munge/out/nwis_dv_daily.rds.ind        |   2 +
 5_data_munge/out/nwis_uv_daily.rds.ind        |   2 +
 5_data_munge/out/wqp_daily_depths.rds.ind     |   2 +
 5_data_munge/out/wqp_daily_nodepths.rds.ind   |   2 +
 5_data_munge/src/munge_data_files.R           | 158 ++++++++++++++++++
 ...cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml |   6 +-
 ...B1bGwvb3V0L253aXNfdXZfZGF0YS5yZHMuaW5k.yml |  13 ++
 ...291dC93cXBfZGFpbHlfZGVwdGhzLnJkcy5pbmQ.yml |  16 ++
 ...dC93cXBfZGFpbHlfbm9kZXB0aHMucmRzLmluZA.yml |  16 ++
 ...dC9kYWlseV90ZW1wZXJhdHVyZXMucmRzLmluZA.yml |  14 ++
 ...mdlL291dC9ud2lzX2R2X2RhaWx5LnJkcy5pbmQ.yml |  15 ++
 ...mdlL291dC9ud2lzX3V2X2RhaWx5LnJkcy5pbmQ.yml |  15 ++
 15 files changed, 337 insertions(+), 4 deletions(-)
 create mode 100644 5_data_munge.yml
 create mode 100644 5_data_munge/out/daily_temperatures.rds.ind
 create mode 100644 5_data_munge/out/nwis_dv_daily.rds.ind
 create mode 100644 5_data_munge/out/nwis_uv_daily.rds.ind
 create mode 100644 5_data_munge/out/wqp_daily_depths.rds.ind
 create mode 100644 5_data_munge/out/wqp_daily_nodepths.rds.ind
 create mode 100644 5_data_munge/src/munge_data_files.R
 create mode 100644 build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfdXZfZGF0YS5yZHMuaW5k.yml
 create mode 100644 build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfZGVwdGhzLnJkcy5pbmQ.yml
 create mode 100644 build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfbm9kZXB0aHMucmRzLmluZA.yml
 create mode 100644 build/status/NV9kYXRhX211bmdlL291dC9kYWlseV90ZW1wZXJhdHVyZXMucmRzLmluZA.yml
 create mode 100644 build/status/NV9kYXRhX211bmdlL291dC9ud2lzX2R2X2RhaWx5LnJkcy5pbmQ.yml
 create mode 100644 build/status/NV9kYXRhX211bmdlL291dC9ud2lzX3V2X2RhaWx5LnJkcy5pbmQ.yml

diff --git a/1_wqp_pull/out/wqp_data.rds.ind b/1_wqp_pull/out/wqp_data.rds.ind
index 1482170..1b19908 100644
--- a/1_wqp_pull/out/wqp_data.rds.ind
+++ b/1_wqp_pull/out/wqp_data.rds.ind
@@ -1,2 +1,2 @@
-hash: 56a9468ec55d588d9d2f960074fd5f8b
+hash: 09ea2c704f6e411444cd0f1c3501d4be
 
diff --git a/5_data_munge.yml b/5_data_munge.yml
new file mode 100644
index 0000000..f5ea63b
--- /dev/null
+++ b/5_data_munge.yml
@@ -0,0 +1,76 @@
+target_default: 5_data_munge
+
+packages:
+  - scipiper
+  - dplyr
+  
+sources:
+  - 5_data_munge/src/munge_data_files.R
+  
+targets:
+  5_data_munge:
+    depends:
+      - 5_data_munge/out/wqp_daily_depths.rds.ind
+      - 5_data_munge/out/wqp_daily_nodepths.rds.ind
+    
+# -- For WQP and NWISuv, first do a coarse QA, then reduce to daily  -- #
+# -- For NWISdv, do the coarse QA  -- #
+
+  min_value:
+    command: I(0)
+    
+  max_value:
+    command: I(35)
+    
+  max_daily_range:
+    command: I(30)
+    
+  # first, wqp data # 
+  5_data_munge/out/wqp_daily_depths.rds.ind:
+    command: munge_wqp_withdepths(
+      in_ind = '1_wqp_pull/out/wqp_data.rds.ind',
+      min_value, max_value, max_daily_range, 
+      out_ind = target_name)
+   
+  5_data_munge/out/wqp_daily_depths.rds:
+    command: gd_get('5_data_munge/out/wqp_daily_depths.rds.ind')
+    
+  5_data_munge/out/wqp_daily_nodepths.rds.ind:
+    command: munge_wqp_withoutdepths(
+      in_ind = '1_wqp_pull/out/wqp_data.rds.ind',
+      min_value, max_value, max_daily_range, 
+      out_ind = target_name)
+   
+  5_data_munge/out/wqp_daily_nodepths.rds:
+    command: gd_get('5_data_munge/out/wqp_daily_nodepths.rds.ind')
+    
+  # NWIS uv data #
+  5_data_munge/out/nwis_uv_daily.rds.ind:
+    command: munge_nwis_uv(
+      in_ind = '2_nwis_pull/out/nwis_uv_data.rds.ind',
+      min_value, max_value, max_daily_range, 
+      out_ind = target_name)
+   
+  5_data_munge/out/nwis_uv_daily.rds:
+    command: gd_get('5_data_munge/out/nwis_uv_daily.rds.ind')
+    
+  # NWIS dv data #
+  5_data_munge/out/nwis_dv_daily.rds.ind:
+    command: munge_nwis_dv(
+      in_ind = '2_nwis_pull/out/nwis_dv_data.rds.ind',
+      min_value, max_value, max_daily_range, 
+      out_ind = target_name)
+   
+  5_data_munge/out/nwis_dv_daily.rds:
+    command: gd_get('5_data_munge/out/nwis_dv_daily.rds.ind')
+    
+  # bring it all together
+  5_data_munge/out/daily_temperatures.rds.ind:
+    command: combine_all_dat(
+      nwis_dv_ind  = '5_data_munge/out/nwis_dv_daily.rds.ind',
+      nwis_uv_ind = '5_data_munge/out/nwis_uv_daily.rds.ind',
+      wqp_ind = '5_data_munge/out/wqp_daily_nodepths.rds.ind',
+      out_ind = target_name)
+      
+  5_data_munge/out/daily_temperatures.rds:
+    command: gd_get('5_data_munge/out/daily_temperatures.rds.ind')
diff --git a/5_data_munge/out/daily_temperatures.rds.ind b/5_data_munge/out/daily_temperatures.rds.ind
new file mode 100644
index 0000000..e204a10
--- /dev/null
+++ b/5_data_munge/out/daily_temperatures.rds.ind
@@ -0,0 +1,2 @@
+hash: 2d87406e6c36ec034cf1b6eed78abc4e
+
diff --git a/5_data_munge/out/nwis_dv_daily.rds.ind b/5_data_munge/out/nwis_dv_daily.rds.ind
new file mode 100644
index 0000000..54eba31
--- /dev/null
+++ b/5_data_munge/out/nwis_dv_daily.rds.ind
@@ -0,0 +1,2 @@
+hash: 7ceae99d441a77a02f6d86d0a2f87c5f
+
diff --git a/5_data_munge/out/nwis_uv_daily.rds.ind b/5_data_munge/out/nwis_uv_daily.rds.ind
new file mode 100644
index 0000000..919fde9
--- /dev/null
+++ b/5_data_munge/out/nwis_uv_daily.rds.ind
@@ -0,0 +1,2 @@
+hash: 8c34d0141db45a61d5d16ac9f0a27fd2
+
diff --git a/5_data_munge/out/wqp_daily_depths.rds.ind b/5_data_munge/out/wqp_daily_depths.rds.ind
new file mode 100644
index 0000000..4c4d63f
--- /dev/null
+++ b/5_data_munge/out/wqp_daily_depths.rds.ind
@@ -0,0 +1,2 @@
+hash: c9a3fe38e9482e157c3d57e223c15405
+
diff --git a/5_data_munge/out/wqp_daily_nodepths.rds.ind b/5_data_munge/out/wqp_daily_nodepths.rds.ind
new file mode 100644
index 0000000..95074c9
--- /dev/null
+++ b/5_data_munge/out/wqp_daily_nodepths.rds.ind
@@ -0,0 +1,2 @@
+hash: 49e600e7b38cbf88986b55fe1fe165b5
+
diff --git a/5_data_munge/src/munge_data_files.R b/5_data_munge/src/munge_data_files.R
new file mode 100644
index 0000000..1794002
--- /dev/null
+++ b/5_data_munge/src/munge_data_files.R
@@ -0,0 +1,158 @@
+f_to_c <- function(f) {
+  c <- (f - 32) * (5/9)
+  return(c)
+}
+
+munge_wqp_withdepths <- function(in_ind, min_value, max_value, max_daily_range, out_ind) {
+  dat <- readRDS(sc_retrieve(in_ind)) %>%
+    filter(!is.na(`ResultMeasure/MeasureUnitCode`)) %>%
+    filter(ActivityMediaName %in% 'Water') %>%
+    filter(!is.na(`ActivityDepthHeightMeasure/MeasureValue`)|
+                         !is.na(`ResultDepthHeightMeasure/MeasureValue`)|
+                         !is.na(`ActivityTopDepthHeightMeasure/MeasureValue`)|
+                         !is.na(`ActivityBottomDepthHeightMeasure/MeasureValue`)) %>% 
+    filter(ActivityStartDate > as.Date('1800-01-01') & ActivityStartDate < as.Date('2020-01-01'))
+  
+  # create depth column
+  dat <- mutate(dat, sample_depth = case_when(
+    !is.na(`ActivityDepthHeightMeasure/MeasureValue`) ~ `ActivityDepthHeightMeasure/MeasureValue`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & !is.na(`ResultDepthHeightMeasure/MeasureValue`) ~ `ResultDepthHeightMeasure/MeasureValue`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & is.na(`ResultDepthHeightMeasure/MeasureValue`) & !is.na(`ActivityTopDepthHeightMeasure/MeasureValue`) ~ `ActivityTopDepthHeightMeasure/MeasureValue`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & is.na(`ResultDepthHeightMeasure/MeasureValue`) & is.na(`ActivityTopDepthHeightMeasure/MeasureValue`) ~ `ActivityBottomDepthHeightMeasure/MeasureValue`
+  ))
+  
+  dat <- mutate(dat, sample_depth_unit_code = case_when(
+    !is.na(`ActivityDepthHeightMeasure/MeasureValue`) ~ `ActivityDepthHeightMeasure/MeasureUnitCode`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & !is.na(`ResultDepthHeightMeasure/MeasureValue`) ~ `ResultDepthHeightMeasure/MeasureUnitCode`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & is.na(`ResultDepthHeightMeasure/MeasureValue`) & !is.na(`ActivityTopDepthHeightMeasure/MeasureValue`) ~ `ActivityTopDepthHeightMeasure/MeasureUnitCode`,
+    is.na(`ActivityDepthHeightMeasure/MeasureValue`) & is.na(`ResultDepthHeightMeasure/MeasureValue`) & is.na(`ActivityTopDepthHeightMeasure/MeasureValue`) ~ `ActivityBottomDepthHeightMeasure/MeasureUnitCode`
+  ))
+  
+
+  # now reduce to daily measures
+  dat_reduced <- dat %>%
+    filter(ResultMeasureValue > min_value) %>%
+    filter(grepl('deg C|deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE)) %>%
+    mutate(ResultMeasureValue = ifelse(grepl('deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE),
+                                       f_to_c(ResultMeasureValue), ResultMeasureValue)) %>%
+    mutate(`ResultMeasure/MeasureUnitCode` = 'deg C') %>%
+    filter(ResultMeasureValue < max_value)
+    
+  dat_daily <- group_by(dat_reduced, MonitoringLocationIdentifier, ActivityStartDate, sample_depth) %>%
+    summarize(temperature_mean_daily = mean(ResultMeasureValue),
+              activity_start_times = paste(`ActivityStartTime/Time`, collapse = ', '),
+              n_day = n())
+  
+  dat_daily_meta <- filter(dat_reduced, !is.na(`ResultMeasure/MeasureUnitCode`)) %>%
+    select(-ResultMeasureValue, -`ActivityStartTime/Time`) %>%
+    group_by(MonitoringLocationIdentifier, ActivityStartDate, sample_depth) %>%
+    summarize_all(first)
+  
+  # bring it together
+  dat_daily <- left_join(dat_daily, dat_daily_meta)
+  
+  # save
+  data_file <- scipiper::as_data_file(out_ind)
+  saveRDS(dat_daily, data_file)
+  gd_put(out_ind, data_file)
+    
+}
+
+munge_wqp_withoutdepths <- function(in_ind, min_value, max_value, max_daily_range, out_ind) {
+  dat <- readRDS(sc_retrieve(in_ind)) %>%
+    select(MonitoringLocationIdentifier, ActivityStartDate, ResultMeasureValue, 
+           `ResultMeasure/MeasureUnitCode`, `ActivityStartTime/Time`, ActivityMediaName, 
+           `ActivityDepthHeightMeasure/MeasureValue`, `ResultDepthHeightMeasure/MeasureValue`, 
+           `ActivityTopDepthHeightMeasure/MeasureValue`, `ActivityBottomDepthHeightMeasure/MeasureValue`) %>%
+    filter(!is.na(`ResultMeasure/MeasureUnitCode`)) %>%
+    filter(ActivityMediaName %in% 'Water') %>%
+    filter(is.na(`ActivityDepthHeightMeasure/MeasureValue`)&
+             is.na(`ResultDepthHeightMeasure/MeasureValue`)&
+             is.na(`ActivityTopDepthHeightMeasure/MeasureValue`)&
+             is.na(`ActivityBottomDepthHeightMeasure/MeasureValue`)) %>%
+    filter(ActivityStartDate > as.Date('1800-01-01') & ActivityStartDate < as.Date('2020-01-01'))
+  
+ 
+  # now reduce to daily measures
+  # first clean up obviously bad data
+  dat_reduced <- filter(dat, ResultMeasureValue > min_value) %>%
+    filter(grepl('deg C|deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE)) %>%
+    mutate(ResultMeasureValue = ifelse(grepl('deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE),
+                                       f_to_c(ResultMeasureValue), ResultMeasureValue)) %>%
+    mutate(`ResultMeasure/MeasureUnitCode` = 'deg C') %>%
+    filter(ResultMeasureValue < max_value)
+  
+  dat_daily <- group_by(dat_reduced, MonitoringLocationIdentifier, ActivityStartDate) %>%
+    summarize(temperature_mean_daily = mean(ResultMeasureValue),
+              activity_start_times = paste(`ActivityStartTime/Time`, collapse = ', '),
+              n_day = n())
+  
+  # save
+  data_file <- scipiper::as_data_file(out_ind)
+  saveRDS(dat_daily, data_file)
+  gd_put(out_ind, data_file)
+  
+}
+
+munge_nwis_uv <- function(in_ind, min_value, max_value, max_daily_range, out_ind) {
+  nwis_df <- readRDS(sc_retrieve(in_ind)) %>%
+    filter(!is.na(temp_value), !is.na(dateTime)) %>%
+    filter(grepl('A', cd_value, ignore.case = FALSE)) %>%
+    filter(temp_value > min_value,
+           temp_value < max_value) %>%
+    mutate(dateTime = as.Date(dateTime)) %>%
+    group_by(site_no, col_name, dateTime) %>%
+    summarize(temperature_mean_daily = round(mean(temp_value), 1), n_day = n())
+  
+  # save
+  data_file <- scipiper::as_data_file(out_ind)
+  saveRDS(nwis_df, data_file)
+  gd_put(out_ind, data_file)
+}
+
+munge_nwis_dv <- function(in_ind, min_value, max_value, max_daily_range, out_ind) {
+
+  dat <- readRDS(sc_retrieve(in_ind)) %>%
+    filter(grepl('A', cd_value, ignore.case = FALSE)) %>%
+    filter(temp_value > min_value,
+           temp_value < max_value)
+  
+  # save
+  data_file <- scipiper::as_data_file(out_ind)
+  saveRDS(dat, data_file)
+  gd_put(out_ind, data_file)
+  
+}
+
+combine_all_dat <- function(wqp_ind, nwis_dv_ind, nwis_uv_ind, out_ind) {
+  wqp <- readRDS(sc_retrieve(wqp_ind)) %>%
+    mutate(date = as.Date(ActivityStartDate), source = 'wqp') %>%
+    select(site_id = MonitoringLocationIdentifier, 
+           date,
+           temp_degC = temperature_mean_daily,
+           n_obs = n_day, source)
+  
+  nwis_dv <- readRDS(sc_retrieve(nwis_dv_ind)) %>%
+    mutate(date = as.Date(dateTime),
+           source = 'nwiw_dv') %>%
+    select(site_id = site_no, 
+           date, temp_degC = temp_value,
+           source)
+  
+  nwis_uv <- readRDS(sc_retrieve(nwis_uv_ind)) %>%
+    ungroup() %>%
+    mutate(date = as.Date(dateTime),
+           source = 'nwiw_uv') %>%
+    select(site_id = site_no, 
+           date, temp_degC = temperature_mean_daily,
+           n_obs = n_day, source)
+  
+  all_dat <- bind_rows(wqp, nwis_dv, nwis_uv)
+    
+  # save
+  data_file <- scipiper::as_data_file(out_ind)
+  saveRDS(all_dat, data_file)
+  gd_put(out_ind, data_file)
+  
+}
+  
diff --git a/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
index cdabf7e..80566f0 100644
--- a/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
+++ b/build/status/MV93cXBfcHVsbC9vdXQvd3FwX2RhdGEucmRzLmluZA.yml
@@ -1,12 +1,12 @@
 version: 0.3.0
 name: 1_wqp_pull/out/wqp_data.rds.ind
 type: file
-hash: 484e253c6c20d56c22c4279c523000c1
-time: 2019-09-27 15:08:22 UTC
+hash: 0c992c14505701c64f4fde7b24166b47
+time: 2019-09-27 15:59:27 UTC
 depends:
   wqp_pull_plan: 06fa9cd23fcab0ccbfd17bdad932f8e7
   1_wqp_pull_tasks.yml: 8296cac3995482e0da9968ed30a57050
-  1_wqp_pull/src/wqp_pull.R: c4e001b821f831b8a5fff263da5ff0f0
+  1_wqp_pull/src/wqp_pull.R: 62bf1f550e32f9936f1090fae07042ce
 fixed: b103af1551106aa1ebfea5ec48fe44e0
 code:
   functions:
diff --git a/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfdXZfZGF0YS5yZHMuaW5k.yml b/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfdXZfZGF0YS5yZHMuaW5k.yml
new file mode 100644
index 0000000..1837817
--- /dev/null
+++ b/build/status/Ml9ud2lzX3B1bGwvb3V0L253aXNfdXZfZGF0YS5yZHMuaW5k.yml
@@ -0,0 +1,13 @@
+version: 0.3.0
+name: 2_nwis_pull/out/nwis_uv_data.rds.ind
+type: file
+hash: 08b8d262ab8b9ed7b977c9a7adc75ba2
+time: 2019-09-28 03:56:10 UTC
+depends:
+  nwis_uv_pull_plan: 24771e25730387152ffa4b05b202c0df
+  2_nwis_uv_pull_tasks.yml: 1667c52afb98c298a1d5429736f55ae7
+  2_nwis_pull/src/nwis_pull.R: 30cc02bf27d1d6d627513c9eb0e1153c
+fixed: c5cfdfacf90cb942deb045710cb0842c
+code:
+  functions: {}
+
diff --git a/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfZGVwdGhzLnJkcy5pbmQ.yml b/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfZGVwdGhzLnJkcy5pbmQ.yml
new file mode 100644
index 0000000..0035cfe
--- /dev/null
+++ b/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfZGVwdGhzLnJkcy5pbmQ.yml
@@ -0,0 +1,16 @@
+version: 0.3.0
+name: 5_data_munge/out/wqp_daily_depths.rds.ind
+type: file
+hash: 08d55f16b068ed604c5868c05901d6b3
+time: 2019-09-28 03:30:45 UTC
+depends:
+  1_wqp_pull/out/wqp_data.rds.ind: 0c992c14505701c64f4fde7b24166b47
+  min_value: c96c32be843430ad3b4632edb49aee30
+  max_value: 8550904a2a45c13015a45b21ef719fb7
+  max_daily_range: afe39393dd5be2c095cd37fede5c5b89
+fixed: 91db974dfd34c0f1ff7b95f146c74c4f
+code:
+  functions:
+    f_to_c: a61909ceabd1801b610840dc86315ffe
+    munge_wqp_withdepths: 6813465a0b5223a60ba8b633e65b6517
+
diff --git a/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfbm9kZXB0aHMucmRzLmluZA.yml b/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfbm9kZXB0aHMucmRzLmluZA.yml
new file mode 100644
index 0000000..4a67cfb
--- /dev/null
+++ b/build/status/NV9kYXRhX211bmdlL291dC93cXBfZGFpbHlfbm9kZXB0aHMucmRzLmluZA.yml
@@ -0,0 +1,16 @@
+version: 0.3.0
+name: 5_data_munge/out/wqp_daily_nodepths.rds.ind
+type: file
+hash: e2d8227f1c21ad3dde6c2a6d10c13a55
+time: 2019-09-28 03:40:54 UTC
+depends:
+  1_wqp_pull/out/wqp_data.rds.ind: 0c992c14505701c64f4fde7b24166b47
+  min_value: c96c32be843430ad3b4632edb49aee30
+  max_value: 8550904a2a45c13015a45b21ef719fb7
+  max_daily_range: afe39393dd5be2c095cd37fede5c5b89
+fixed: c8da49989b1a3d05cbbfd15be13cfef9
+code:
+  functions:
+    f_to_c: a61909ceabd1801b610840dc86315ffe
+    munge_wqp_withoutdepths: 9998f1e94f9b6021f77d3835241923fd
+
diff --git a/build/status/NV9kYXRhX211bmdlL291dC9kYWlseV90ZW1wZXJhdHVyZXMucmRzLmluZA.yml b/build/status/NV9kYXRhX211bmdlL291dC9kYWlseV90ZW1wZXJhdHVyZXMucmRzLmluZA.yml
new file mode 100644
index 0000000..5a1d8e1
--- /dev/null
+++ b/build/status/NV9kYXRhX211bmdlL291dC9kYWlseV90ZW1wZXJhdHVyZXMucmRzLmluZA.yml
@@ -0,0 +1,14 @@
+version: 0.3.0
+name: 5_data_munge/out/daily_temperatures.rds.ind
+type: file
+hash: b3074146b6bfe0831465b2121798c197
+time: 2019-10-01 15:43:52 UTC
+depends:
+  5_data_munge/out/nwis_dv_daily.rds.ind: 4cff839dbd456ca82f03220d9f6a40a8
+  5_data_munge/out/nwis_uv_daily.rds.ind: 56e6962f557071a7e094875b915e3b89
+  5_data_munge/out/wqp_daily_nodepths.rds.ind: e2d8227f1c21ad3dde6c2a6d10c13a55
+fixed: 8f477b7e950fc4190a2f937e706494e7
+code:
+  functions:
+    combine_all_dat: f056c99b386a1e106fa8dcef993eb434
+
diff --git a/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX2R2X2RhaWx5LnJkcy5pbmQ.yml b/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX2R2X2RhaWx5LnJkcy5pbmQ.yml
new file mode 100644
index 0000000..af58ce6
--- /dev/null
+++ b/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX2R2X2RhaWx5LnJkcy5pbmQ.yml
@@ -0,0 +1,15 @@
+version: 0.3.0
+name: 5_data_munge/out/nwis_dv_daily.rds.ind
+type: file
+hash: 4cff839dbd456ca82f03220d9f6a40a8
+time: 2019-10-01 15:11:32 UTC
+depends:
+  2_nwis_pull/out/nwis_dv_data.rds.ind: ccefaf5c0aae4cca121b9e7ffa529cb2
+  min_value: c96c32be843430ad3b4632edb49aee30
+  max_value: 8550904a2a45c13015a45b21ef719fb7
+  max_daily_range: afe39393dd5be2c095cd37fede5c5b89
+fixed: 3e8032bfe30e3e4576bd57f818d90da0
+code:
+  functions:
+    munge_nwis_dv: 41ada4803645542d8ffdf296e656aeb7
+
diff --git a/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX3V2X2RhaWx5LnJkcy5pbmQ.yml b/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX3V2X2RhaWx5LnJkcy5pbmQ.yml
new file mode 100644
index 0000000..3f3f54a
--- /dev/null
+++ b/build/status/NV9kYXRhX211bmdlL291dC9ud2lzX3V2X2RhaWx5LnJkcy5pbmQ.yml
@@ -0,0 +1,15 @@
+version: 0.3.0
+name: 5_data_munge/out/nwis_uv_daily.rds.ind
+type: file
+hash: 56e6962f557071a7e094875b915e3b89
+time: 2019-10-01 15:24:31 UTC
+depends:
+  2_nwis_pull/out/nwis_uv_data.rds.ind: 08b8d262ab8b9ed7b977c9a7adc75ba2
+  min_value: c96c32be843430ad3b4632edb49aee30
+  max_value: 8550904a2a45c13015a45b21ef719fb7
+  max_daily_range: afe39393dd5be2c095cd37fede5c5b89
+fixed: c2d9321667f7c1219e64e155aaa09685
+code:
+  functions:
+    munge_nwis_uv: b12c273a0a979c572223a10bfae0c8b4
+

From 7c96a80f05b53ef12900da69644e830ac99a824b Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 9 Oct 2019 13:15:55 -0500
Subject: [PATCH 19/22] Changes based on Alison's PR review: move min/max
 filtering to after conversion, remove redundant is.na filtering, avoid
 rounding to too few sig figs, add "USGS" prefix to NWIS sites

---
 5_data_munge/src/munge_data_files.R | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/5_data_munge/src/munge_data_files.R b/5_data_munge/src/munge_data_files.R
index 1794002..6a2cef0 100644
--- a/5_data_munge/src/munge_data_files.R
+++ b/5_data_munge/src/munge_data_files.R
@@ -31,20 +31,19 @@ munge_wqp_withdepths <- function(in_ind, min_value, max_value, max_daily_range,
 
   # now reduce to daily measures
   dat_reduced <- dat %>%
-    filter(ResultMeasureValue > min_value) %>%
     filter(grepl('deg C|deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE)) %>%
     mutate(ResultMeasureValue = ifelse(grepl('deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE),
                                        f_to_c(ResultMeasureValue), ResultMeasureValue)) %>%
-    mutate(`ResultMeasure/MeasureUnitCode` = 'deg C') %>%
-    filter(ResultMeasureValue < max_value)
+    filter(ResultMeasureValue > min_value,
+           ResultMeasureValue < max_value) %>%
+    mutate(`ResultMeasure/MeasureUnitCode` = 'deg C')
     
   dat_daily <- group_by(dat_reduced, MonitoringLocationIdentifier, ActivityStartDate, sample_depth) %>%
     summarize(temperature_mean_daily = mean(ResultMeasureValue),
               activity_start_times = paste(`ActivityStartTime/Time`, collapse = ', '),
               n_day = n())
   
-  dat_daily_meta <- filter(dat_reduced, !is.na(`ResultMeasure/MeasureUnitCode`)) %>%
-    select(-ResultMeasureValue, -`ActivityStartTime/Time`) %>%
+  dat_daily_meta <- select(dat_reduced, -ResultMeasureValue, -`ActivityStartTime/Time`) %>%
     group_by(MonitoringLocationIdentifier, ActivityStartDate, sample_depth) %>%
     summarize_all(first)
   
@@ -75,12 +74,12 @@ munge_wqp_withoutdepths <- function(in_ind, min_value, max_value, max_daily_rang
  
   # now reduce to daily measures
   # first clean up obviously bad data
-  dat_reduced <- filter(dat, ResultMeasureValue > min_value) %>%
+  dat_reduced <- dat %>%
     filter(grepl('deg C|deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE)) %>%
     mutate(ResultMeasureValue = ifelse(grepl('deg F', `ResultMeasure/MeasureUnitCode`, ignore.case = TRUE),
                                        f_to_c(ResultMeasureValue), ResultMeasureValue)) %>%
     mutate(`ResultMeasure/MeasureUnitCode` = 'deg C') %>%
-    filter(ResultMeasureValue < max_value)
+    filter(ResultMeasureValue > min_value, ResultMeasureValue < max_value)
   
   dat_daily <- group_by(dat_reduced, MonitoringLocationIdentifier, ActivityStartDate) %>%
     summarize(temperature_mean_daily = mean(ResultMeasureValue),
@@ -102,7 +101,7 @@ munge_nwis_uv <- function(in_ind, min_value, max_value, max_daily_range, out_ind
            temp_value < max_value) %>%
     mutate(dateTime = as.Date(dateTime)) %>%
     group_by(site_no, col_name, dateTime) %>%
-    summarize(temperature_mean_daily = round(mean(temp_value), 1), n_day = n())
+    summarize(temperature_mean_daily = round(mean(temp_value), 3), n_day = n())
   
   # save
   data_file <- scipiper::as_data_file(out_ind)
@@ -134,18 +133,16 @@ combine_all_dat <- function(wqp_ind, nwis_dv_ind, nwis_uv_ind, out_ind) {
   
   nwis_dv <- readRDS(sc_retrieve(nwis_dv_ind)) %>%
     mutate(date = as.Date(dateTime),
-           source = 'nwiw_dv') %>%
-    select(site_id = site_no, 
-           date, temp_degC = temp_value,
-           source)
+           source = 'nwiw_dv',
+           site_id = paste0('USGS-', site_no)) %>%
+    select(site_id, date, temp_degC = temp_value, source)
   
   nwis_uv <- readRDS(sc_retrieve(nwis_uv_ind)) %>%
     ungroup() %>%
     mutate(date = as.Date(dateTime),
-           source = 'nwiw_uv') %>%
-    select(site_id = site_no, 
-           date, temp_degC = temperature_mean_daily,
-           n_obs = n_day, source)
+           source = 'nwiw_uv',
+           site_id = paste0('USGS-', site_no)) %>%
+    select(site_id, date, temp_degC = temperature_mean_daily, n_obs = n_day, source)
   
   all_dat <- bind_rows(wqp, nwis_dv, nwis_uv)
     

From 24b23188df09dda803b46acf24ac052144ec5e59 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 9 Oct 2019 13:40:23 -0500
Subject: [PATCH 20/22] changed formatting per Alison's suggestions in PR
 review. Also renamed objects in POST script to be descriptive.

---
 1_wqp_pull/src/wqp_pull.R | 54 ++++++++++++++++++++-------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/1_wqp_pull/src/wqp_pull.R b/1_wqp_pull/src/wqp_pull.R
index 01e442d..b69b775 100644
--- a/1_wqp_pull/src/wqp_pull.R
+++ b/1_wqp_pull/src/wqp_pull.R
@@ -168,33 +168,35 @@ wqp_POST <- function(wqp_args_list){
   wqp_args_list$siteid <- wqp_args_list$siteid
   post_body = jsonlite::toJSON(wqp_args_list, pretty = TRUE)
   
-  temp <- tempfile()
-  temp <- paste0(temp,".zip")
-  x <- POST(paste0(wqp_url,"?mimeType=tsv&zip=yes"),
-            body = post_body,
-            content_type("application/json"),
-            accept("application/zip"),
-            httr::write_disk(temp))
+  download_location <- tempfile()
+  download_location <- paste0(temp_location,".zip")
+  pull_metadata <- POST(paste0(wqp_url,"?mimeType=tsv&zip=yes"),
+                        body = post_body,
+                        content_type("application/json"),
+                        accept("application/zip"),
+                        httr::write_disk(download_location))
   
-  headerInfo <- httr::headers(x)
-  file1 <- tempdir()
-  doc <- utils::unzip(temp, exdir=file1)
-  unlink(temp)
-  retval <- suppressWarnings(read_delim(doc, 
-                                         col_types = cols(`ActivityStartTime/Time` = col_character(),
-                                                          `ActivityEndTime/Time` = col_character(),
-                                                          USGSPCode = col_character(),
-                                                          ResultCommentText=col_character(),
-                                                          `ActivityDepthHeightMeasure/MeasureValue` = col_number(),
-                                                          `DetectionQuantitationLimitMeasure/MeasureValue` = col_number(),
-                                                          ResultMeasureValue = col_number(),
-                                                          `WellDepthMeasure/MeasureValue` = col_number(),
-                                                          `WellHoleDepthMeasure/MeasureValue` = col_number(),
-                                                          `HUCEightDigitCode` = col_character(), 
-                                                          `ActivityEndTime/TimeZoneCode` = col_character()),
-                                         quote = "", delim = "\t"))
-  unlink(doc)
-  return(retval)
+  headerInfo <- httr::headers(pull_metadata)
+  unzip_location <- tempdir()
+  unzipped_filename <- utils::unzip(download_location, exdir=unzip_location)
+  unlink(download_location)
+  dat_out <- suppressWarnings(
+    read_delim(
+      unzipped_filename, 
+      col_types = cols(`ActivityStartTime/Time` = col_character(),
+                       `ActivityEndTime/Time` = col_character(),
+                       USGSPCode = col_character(),
+                       ResultCommentText=col_character(),
+                       `ActivityDepthHeightMeasure/MeasureValue` = col_number(),
+                       `DetectionQuantitationLimitMeasure/MeasureValue` = col_number(),
+                       ResultMeasureValue = col_number(),
+                       `WellDepthMeasure/MeasureValue` = col_number(),
+                       `WellHoleDepthMeasure/MeasureValue` = col_number(),
+                       `HUCEightDigitCode` = col_character(), 
+                       `ActivityEndTime/TimeZoneCode` = col_character()),
+      quote = "", delim = "\t"))
+  unlink(unzipped_filename)
+  return(dat_out)
 }
 
 # extract and post the data (dropping the site info attribute), creating an

From 6fedc4a0060123673bb03d19abd249e536dbf761 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 9 Oct 2019 13:40:44 -0500
Subject: [PATCH 21/22] described what errors these filters are accomodating.

---
 1_wqp_pull/src/wqp_inventory.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/1_wqp_pull/src/wqp_inventory.R b/1_wqp_pull/src/wqp_inventory.R
index bac7c05..3242c01 100644
--- a/1_wqp_pull/src/wqp_inventory.R
+++ b/1_wqp_pull/src/wqp_inventory.R
@@ -84,6 +84,7 @@ partition_wqp_inventory <- function(partitions_ind, wqp_pull_params, inventory_i
     filter(!(ResolvedMonitoringLocationTypeName %in% wqp_pull_params$DropLocationTypeName))
   
   # filter out bad org names
+  # these orgs cause "Frequest failed [400]" errors
   bad_orgs <- grep(' |\\.|/', wqp_inventory$OrganizationIdentifier, value = TRUE)
   bad_orgs_sites <- filter(wqp_inventory, OrganizationIdentifier %in% bad_orgs)
   

From 3e37453d8a2ce23efbfd4c5fb9792e9da7a52180 Mon Sep 17 00:00:00 2001
From: Sam Oliver <soliver@usgs.gov>
Date: Wed, 9 Oct 2019 13:41:06 -0500
Subject: [PATCH 22/22] add data munge step

---
 remake.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/remake.yml b/remake.yml
index 89c47b0..971cccb 100644
--- a/remake.yml
+++ b/remake.yml
@@ -3,10 +3,12 @@ target_default: everything
 include:
   - 1_wqp_pull.yml
   - 2_nwis_pull.yml
+  - 5_data_munge.yml
 
 targets:
   everything:
     depends:
       - 1_wqp_pull
       - 2_nwis_pull
+      - 5_data_munge