Fix: Change dataset files name and location

cd4ml/filenames.py

@@ -51,8 +51,8 @@ def _get_model_file_templates(model_results_dir):
 def _get_problem_file_templates(raw_problem_data_dir):
     file_names_problem = {
         'groceries': {
-            'raw_grocery_data': '%s/groceries.csv' % raw_problem_data_dir,
-            'grocery_data_shuffled': '%s/groceries_shuffled.csv' % raw_problem_data_dir
+            'raw_grocery_data': '%s/store47-2016.csv' % raw_problem_data_dir,
+            'grocery_data_shuffled': '%s/store47-2016_shuffled.csv' % raw_problem_data_dir
         },
         'houses': {
             'raw_house_data': '%s/house_sales.csv' % raw_problem_data_dir,

cd4ml/problems/groceries/download_data/download_data.py

@@ -1,9 +1,14 @@
 from cd4ml.filenames import get_problem_files
 from cd4ml.utils.utils import download_to_file_from_url, shuffle_csv_file
+import zipfile
+from pathlib import Path
+import logging
 
-download_params = {'key': 'store47-2016.csv',
-                   'gcs_bucket': 'continuous-intelligence',
-                   'base_url': 'https://storage.googleapis.com'}
+logger = logging.getLogger(__name__)
+
+download_params = {'key': 'store47-2016',
+                   'gcs_bucket': 'raw/master',
+                   'base_url': 'https://github.com/luizmachado/CDMLDataset'}
 
 
 def get_grocery_url_and_files(problem_name):
@@ -13,12 +18,17 @@ def get_grocery_url_and_files(problem_name):
     base_url = download_params['base_url']
 
     filename = file_names['raw_grocery_data']
-    url = "%s/%s/%s" % (base_url, gcs_bucket, key)
+    url = "%s/%s/%s.zip" % (base_url, gcs_bucket, key)
     filename_shuffled = file_names['grocery_data_shuffled']
     return url, filename, filename_shuffled
 
 
 def download(problem_name, use_cache=True):
     url, filename, filename_shuffled = get_grocery_url_and_files(problem_name)
-    download_to_file_from_url(url, filename, use_cache=use_cache)
+    zipname = f"{filename}.zip"
+    download_to_file_from_url(url, zipname, use_cache=use_cache)
+    target_dir = Path(filename).parent
+    logger.inf(f"Unzipping: {zipname} @ {target_dir}")
+    with zipfile.ZipFile(zipname, 'r') as zip_ref:
+        zip_ref.extractall(target_dir)
     shuffle_csv_file(filename, filename_shuffled)

cd4ml/problems/houses/download_data/download_data.py

@@ -1,9 +1,11 @@
 from cd4ml.filenames import get_problem_files
 from cd4ml.utils.utils import download_to_file_from_url
 
+baseUri = "https://github.com/luizmachado/CD4ML/tree/master/dataset/"
+
 download_params = {
-                    'url': "https://github.com/dave31415/house_price/raw/master/data/house_data_100000.csv",
-                    'url_lookup': "https://github.com/dave31415/house_price/raw/master/data/zip_lookup.csv"
+                    'url': "https://github.com/luizmachado/CD4ML/raw/master/dataset/house_data_100000.csv",
+                    'url_lookup': "https://github.com/luizmachado/CD4ML/raw/master/dataset/zip_lookup.csv"
                    }
 
 

cd4ml/problems/iris/download_data/download_data.py

@@ -2,8 +2,7 @@
 from cd4ml.utils.utils import download_to_file_from_url
 
 download_params = {
-                    'url': "https://gist.githubusercontent.com/curran/a08a1080b88344b0c8a7/"
-                           "raw/0e7a9b0a5d22642a06d3d5b9bcbad9890c8ee534/iris.csv"
+                    'url': "https://github.com/luizmachado/CD4ML/raw/master/dataset/iris.csv"
                    }
 
 

postgres/Dockerfile

@@ -1,6 +1,7 @@
 FROM postgres:12.2
-RUN apt-get update && apt-get install -y wget && \
+RUN apt-get update && apt-get install -y wget unzip && \
     mkdir -p /docker-entrypoint-initdb.d && \
-    wget https://storage.googleapis.com/continuous-intelligence/store47-2016.csv -O /docker-entrypoint-initdb.d/store47-2016.csv
+    wget  https://github.com/luizmachado/CD4ML/blob/master/dataset/store47-2016.zip -O /docker-entrypoint-initdb.d/store47-2016.zip && \
+    unzip /docker-entrypoint-initdb.d/store47-2016.zip -d /docker-entrypoint-initdb.d/
 
 COPY initialize.sql /docker-entrypoint-initdb.d/initialize.sql