From 8f6db3c6de5e371778328ed8239b834afe185f74 Mon Sep 17 00:00:00 2001
From: rob <robcarver17@googlemail.com>
Date: Wed, 15 Jun 2022 08:33:44 +0100
Subject: [PATCH] add more logging to price cleaning

---
 sysdata/tools/cleaner.py                  | 24 ++++++++++++++++++-----
 sysproduction/update_historical_prices.py |  2 +-
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/sysdata/tools/cleaner.py b/sysdata/tools/cleaner.py
index 255b15ce6f..4dd7023754 100644
--- a/sysdata/tools/cleaner.py
+++ b/sysdata/tools/cleaner.py
@@ -31,28 +31,42 @@ def apply_price_cleaning(data: dataBlob,
     broker_prices = copy(broker_prices_raw)
 
     ## It's important that the data is in local time zone so that this works
+    price_length = len(broker_prices)
     if cleaning_config.ignore_future_prices:
-        log.msg("Ignoring prices with future timestamps")
         broker_prices = broker_prices.remove_future_data()
+        new_price_length = len(broker_prices)
+        if new_price_length<price_length:
+            log.msg("Ignoring %d prices with future timestamps" % (price_length - new_price_length))
+            price_length = new_price_length
     else:
         log.warn("*** NOT IGNORING PRICES WITH FUTURE TIMESTAMP - may lead to price gaps ***")
 
     if cleaning_config.ignore_prices_with_zero_volumes:
-        log.msg("Ignoring prices with zero volumes")
         broker_prices = broker_prices.remove_zero_volumes()
+        new_price_length = len(broker_prices)
+        if new_price_length<price_length:
+            log.msg("Ignoring %d prices with zero volumes" % (price_length - new_price_length))
+            price_length = new_price_length
+
     else:
         log.warn("**** NOT IGNORING PRICES WITH ZERO VOLUMES - data may be inaccurate")
 
     if cleaning_config.ignore_zero_prices:
-
-        log.msg("Ignoring prices with zero prices")
         broker_prices = broker_prices.remove_zero_prices()
+        new_price_length = len(broker_prices)
+        if new_price_length<price_length:
+            log.msg("Ignoring %d prices with zero prices" % (price_length - new_price_length))
+            price_length = new_price_length
+
     else:
         log.warn(("***** NOT IGNORING ZERO PRICES - COULD BE BAD DATA"))
 
     if cleaning_config.ignore_negative_prices:
-        log.warn("***** IGNORING NEGATIVE PRICES - might be really negative (think crude oil March 2020)")
         broker_prices = broker_prices.remove_negative_prices()
+        new_price_length = len(broker_prices)
+        if new_price_length<price_length:
+            log.warn("Ignoring %d prices with negative prices ****COULD BE REAL PRICES****" % (price_length - new_price_length))
+            price_length = new_price_length ## not used again but for tidiness
     else:
         log.msg("Keeping negative prices in data as could be real")
 
diff --git a/sysproduction/update_historical_prices.py b/sysproduction/update_historical_prices.py
index e25a5a50b3..a246b7398a 100644
--- a/sysproduction/update_historical_prices.py
+++ b/sysproduction/update_historical_prices.py
@@ -165,7 +165,7 @@ def get_and_add_prices_for_frequency(
                                                    )
     if error_or_rows_added is failure:
         return failure
-    
+
     data.log.msg(
         "Added %d rows at frequency %s for %s"
         % (error_or_rows_added, frequency, str(contract_object))