Add files via upload

transportenergy · Dec 9, 2023 · 6a76dbc · 6a76dbc
1 parent c86c8bd
commit 6a76dbc
Show file tree

Hide file tree

Showing 11 changed files with 5,002 additions and 1,127 deletions.
diff --git a/sandra/Output_data TAS-FRA-007(2).csv b/sandra/Output_data TAS-FRA-007(2).csv
diff --git a/sandra/Output_data TAS-PAG-005(3).csv b/sandra/Output_data TAS-PAG-005(3).csv
diff --git a/sandra/Output_data_TAS-VEP-017.csv b/sandra/Output_data_TAS-VEP-017.csv
diff --git a/sandra/T021.py b/sandra/T021.py
@@ -1,4 +1,3 @@
-
 import pandas as pd
 import yaml
 
@@ -41,6 +40,7 @@ def execute(self):
                 merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right')
                 merged_df.drop('merge_column', axis=1, inplace=True)
                 self.transformation_data(region_yaml_data, merged_df, applicable_rule)
+                merged_df['Data quality flag']='!'
                 merged_df.to_csv(self.file_name, index=False)
         print("Execution completed for sheet : "+ self.sheetname)
         """ ---------------------------------------------- """
@@ -187,3 +187,5 @@ def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data):
 """ ---------------------------------------------- """
 print("Starting execution for sheet : "+ item_transformer.sheetname)
 item_transformer.execute()
+
+
diff --git a/sandra/T021_TAS_PAG_005(3).ipynb b/sandra/T021_TAS_PAG_005(3).ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 74,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -57,6 +57,7 @@
     "                merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right')\n",
     "                merged_df.drop('merge_column', axis=1, inplace=True)\n",
     "                self.transformation_data(region_yaml_data, merged_df, applicable_rule)\n",
+    "                merged_df['Data quality flag']='!'\n",
     "                merged_df.to_csv(self.file_name, index=False)\n",
     "        print(\"Execution completed for sheet : \"+ self.sheetname)\n",
     "        \"\"\" ---------------------------------------------- \"\"\"\n",
@@ -102,7 +103,7 @@
     "    def transformation_data(self, yaml_data, merged_df, applicable_rule):\n",
     "\n",
     "        self.rename_and_reorder_columns(merged_df)\n",
-    "        unit = applicable_rule[\"Unit factor\"]\n",
+    "        unit = applicable_rule[\"Unit Factor\"]\n",
     "        for column_name in merged_df.columns:\n",
     "            no_spaces = column_name.replace(\" \", \"\")\n",
     "            try:\n",

diff --git a/sandra/T022.py b/sandra/T022.py
@@ -1,3 +1,4 @@
+
 import pandas as pd
 import os
 import os.path
@@ -62,7 +63,7 @@ def country_region_mapping(self, economy_code, regions):
     #     return rule_id, value, indicator_name
     # Function that returns ruleID, indicator_name, and dictionary
     def get_rule_id(self, rule_book: dict):
-        rule_id = "Txxx"
+        rule_id = "T022"
         valid_id_found = False
 
         for key, value in rule_book.items():
@@ -405,8 +406,8 @@ def process_input_data(self, workbook_file: str, master_file: str, regions_file:
 
 
 # Name and path of input files
-workbook_excel_file = r"C:/Users/magli/Desktop/item-internship/main/ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx"
-master_csv_file = r"C:/Users/magli/Desktop/item-internship/code/sandra/Output_data_TAS-VEP-017.csv"
+workbook_excel_file = r"ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx"
+master_csv_file = r"master dataset.csv"
 regions_file = r"regions.yaml"
 source_file = r"sources.yaml"
 

diff --git a/sandra/T022_TAS_VEP_017.ipynb b/sandra/T022_TAS_VEP_017.ipynb
diff --git a/sandra/T023.py b/sandra/T023.py
@@ -6,7 +6,7 @@
 import yaml # YAML is a human-readable data
 from yaml.loader import SafeLoader#used for safely loading YAML document
 #Safeloader:avoiding the execution of arbitrary code.
-import pycountry#it is a useful for dealing with country & currency information in a standardized way based on ISO codes.
+import pycountry #it is a useful for dealing with country & currency information in a standardized way based on ISO codes.
 import math
 
 class AtoWorkbook:
@@ -67,7 +67,7 @@ def country_region_mapping(self, economy_code, regions):
     # Function that returns ruleID, indicator_name and dictionary
             #rule_book:dictionary containing rule information.
     def get_rule_id(self, rule_book:dict):
-        rule_id = "Txxx"#default value 
+        rule_id = "T023"#default value 
         valid_id_found = False# boolean variable to track whether a valid rule ID is found.
 
         for key, value in rule_book.items():        
@@ -382,8 +382,8 @@ def process_input_data(self, workbook_file: str, master_file: str, regions_file:
         master_df_output.to_csv("Output_data "+ upper_part_attributes[5] + ".csv", index=False)
 
 # Name and path of input files
-workbook_excel_file = r"C:/Users/magli/Desktop/item-internship/main/ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx"
-master_csv_file = r"C:/Users/magli/Desktop/item-internship/code/sandra/Output_data TAS-FRA-007(2).csv"
+workbook_excel_file = r"ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx"
+master_csv_file = r"master dataset.csv"
 regions_file = r"regions.yaml"
 source_file = r"sources.yaml"