Merge pull request #1 from dsfsi/master

Update to main
dsfsi · May 24, 2020 · 5e425db · 5e425db
2 parents 92a609f + c41363b
commit 5e425db
Show file tree

Hide file tree

Showing 18 changed files with 3,872 additions and 1,416 deletions.
diff --git a/.gitignore b/.gitignore
@@ -129,3 +129,13 @@ dmypy.json
 .pyre/
 
 data/Ghana_updates_moh/
+
+# WIP files
+.idea/
+act.exe
+actions-runner/
+data/testing/covid-testing-all-observations.csv
+twitter_developer_account_for_api
+
+# Other build files
+.vscode/
diff --git a/data/line_lists/line-list-egypt.csv b/data/line_lists/line-list-egypt.csv
diff --git a/data/time_series/africa_cdc/africa_cdc_daily_time_series_cases.csv b/data/time_series/africa_cdc/africa_cdc_daily_time_series_cases.csv
diff --git a/data/time_series/africa_cdc/africa_cdc_daily_time_series_deaths.csv b/data/time_series/africa_cdc/africa_cdc_daily_time_series_deaths.csv
diff --git a/data/time_series/africa_cdc/africa_cdc_daily_time_series_recovered.csv b/data/time_series/africa_cdc/africa_cdc_daily_time_series_recovered.csv
diff --git a/data/time_series/africa_cdc/africa_cdc_daily_time_series_tests.csv b/data/time_series/africa_cdc/africa_cdc_daily_time_series_tests.csv
diff --git a/data/time_series/africa_daily_time_series_cases.csv b/data/time_series/africa_daily_time_series_cases.csv
diff --git a/data/time_series/africa_daily_time_series_deaths.csv b/data/time_series/africa_daily_time_series_deaths.csv
diff --git a/data/time_series/africa_daily_time_series_recovered.csv b/data/time_series/africa_daily_time_series_recovered.csv
diff --git a/data/time_series/africa_daily_time_series_tests.csv b/data/time_series/africa_daily_time_series_tests.csv
diff --git a/data/time_series/africa_daily_time_series_unpivoted.csv b/data/time_series/africa_daily_time_series_unpivoted.csv
diff --git a/data/time_series/africa_daily_time_series_unpivoted_cases.csv b/data/time_series/africa_daily_time_series_unpivoted_cases.csv
diff --git a/data/time_series/africa_daily_time_series_unpivoted_deaths.csv b/data/time_series/africa_daily_time_series_unpivoted_deaths.csv
diff --git a/data/time_series/africa_daily_time_series_unpivoted_recovered.csv b/data/time_series/africa_daily_time_series_unpivoted_recovered.csv
diff --git a/data/time_series/africa_daily_time_series_unpivoted_tests.csv b/data/time_series/africa_daily_time_series_unpivoted_tests.csv
diff --git a/img/May_22_2020_6pm_EAT.jpg b/img/May_22_2020_6pm_EAT.jpg
diff --git a/img/May_23_2020_9am_EAT.jpg b/img/May_23_2020_9am_EAT.jpg
diff --git a/scripts/utils.py b/scripts/utils.py
@@ -20,7 +20,7 @@
 timeseries_path = "data/time_series/"
 # This is a hack for now, it will be replaced with similarity measures for more elegant accomodation of variants
 # Variants happen due to people mistyping or OCR artifacts
-freq_missed = {"ORC" : "DRC", "Cdte d'ivoire": "Cote d'ivoire", "Cdte d'Ivoire": "Cote d'ivoire", 
+freq_missed = {"Cape Verde":"Cabo Verde", "ORC" : "DRC", "Cdte d'ivoire": "Cote d'ivoire", "Cdte d'Ivoire": "Cote d'ivoire", 
                                 "Cate d'ivoire" : "Cote d'ivoire", "Cote d'Ivoire" : "Cote d'ivoire", "Cate d'Ivoire" : "Cote d'ivoire", 
                                 "Céte d'ivoire" : "Cote d'ivoire", "Céte d'Ivoire" : "Cote d'ivoire", "Cte d'Ivoire" : "Cote d'ivoire",
                                 "S20 Tome & Principe" : "Sao Tome & Principe", "Sa0 Tome & Principe" : "Sao Tome & Principe"}
@@ -72,9 +72,19 @@ def get_timeseries_filenames(files_path=timeseries_path, files_base="africa_dail
     return get_filenames(files_path, files_base)
 
 def read_time_series():
-    files = get_africa_cdc_filenames()[:3]
+    data_f, files = read_africa_cdc_time_series()
+    return data_f[:3], files[:3]
+
+def read_africa_cdc_time_series(use_country_asid=True):
+    files = get_africa_cdc_filenames()
     # read the files
-    data_f = [pd.read_csv(f, index_col='Country/Region', encoding = "ISO-8859-1") for f in files]
+    if use_country_asid:
+        data_f = [pd.read_csv(f, index_col='Country/Region', encoding = "ISO-8859-1", keep_default_na=False) for f in files]
+    else:
+        data_f = [pd.read_csv(f, encoding = "ISO-8859-1", keep_default_na=False) for f in files]
+    # df = data_f[0]
+    # print(df)
+    # print(df.loc[df['Country/Region'] == "Namibia"])
     # print(data_f)
     return data_f, files
 
@@ -109,12 +119,11 @@ def unpivot_timeseries():
     keys = ["Confirmed Cases", "Deaths", "Recovered Cases", "Tests"]
     df_unp = "unpivoted_dataframe"
     # First, get all the 4 files, unpivoted and sorted
-    #filenames = get_mixed_timeseries_filenames()
-    filenames = get_africa_cdc_filenames()
+    # filenames = get_mixed_timeseries_filenames()
+    # filenames = get_africa_cdc_filenames()
     # print(filenames)
-    dfs = [pd.read_csv(filenames[i], keep_default_na=False) for i in range(len(keys))]
-    # df = dfs[0]
-    # print(df.loc[df['Country/Region'] == "Namibia"])
+    dfs, filenames = read_africa_cdc_time_series(use_country_asid=False) #[pd.read_csv(filenames[i], keep_default_na=False) for i in range(len(keys))]
+
     data = {keys[i]:{"filename":filenames[i], \
                      "df": dfs[i], \
                       df_unp: dfs[i].melt(id_vars=["Country/Region", "iso2", "iso3", "Subregion", "Population-2020", "Lat", "Long"], var_name="Date", value_name="Values"), \