Reduce the size of the UC table

etsap-TIMES · Dec 26, 2024 · 9ddc7af · 9ddc7af
1 parent bcf2d09
commit 9ddc7af
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/xl2times/transforms.py b/xl2times/transforms.py
@@ -3223,6 +3223,18 @@ def apply_final_fixup(
 
     tables[Tag.fi_t] = df.reset_index(drop=True)
 
+    if Tag.uc_t in tables.keys():
+        df = tables[Tag.uc_t]
+        keep_cols.remove("year2")
+        keep_cols = keep_cols.union({"uc_n", "side"})
+        df.dropna(subset="value", inplace=True)
+        drop_cols = [
+            col for col in df.columns if col != "value" and col not in keep_cols
+        ]
+        df.drop(columns=drop_cols, inplace=True)
+        df = df.drop_duplicates(subset=list(keep_cols), keep="last")
+        tables[Tag.uc_t] = df.reset_index(drop=True)
+
     return tables