From c181faaa2a82a20c6a400ae382927faf6195218e Mon Sep 17 00:00:00 2001 From: jmartinkova Date: Fri, 8 Nov 2024 01:03:55 +0100 Subject: [PATCH] One handle seed function --- src/dsw_seed_maker/logic.py | 153 +++++++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 45 deletions(-) diff --git a/src/dsw_seed_maker/logic.py b/src/dsw_seed_maker/logic.py index ea18d42..2e1d727 100644 --- a/src/dsw_seed_maker/logic.py +++ b/src/dsw_seed_maker/logic.py @@ -1,7 +1,9 @@ import json import os +import re import uuid from datetime import datetime +from idlelib.iomenu import encoding from dotenv import load_dotenv import pathlib @@ -14,6 +16,12 @@ load_dotenv() +def connect_to_db_logic() -> Database: + return Database(name=os.getenv("DSW_DB_CONN_NAME"), dsn=os.getenv("DSW_DB_CONN_STR")) + +processed_resources = set() +db = connect_to_db_logic() +output_dir = "-" def example_logic(req_dto: ExampleRequestDTO) -> ExampleResponseDTO: return ExampleResponseDTO( @@ -21,9 +29,6 @@ def example_logic(req_dto: ExampleRequestDTO) -> ExampleResponseDTO: ) -def connect_to_db_logic() -> Database: - return Database(name=os.getenv("DSW_DB_CONN_NAME"), dsn=os.getenv("DSW_DB_CONN_STR")) - def connect_to_s3_logic() -> S3Storage: return S3Storage( @@ -37,7 +42,6 @@ def connect_to_s3_logic() -> S3Storage: def list_logic(resource_type: str) -> dict[str, list[dict[str, Any]]] | list[dict[str, Any]]: - db = connect_to_db_logic() if resource_type== "all": resource = {} for each in resource_attributes.keys(): @@ -75,16 +79,16 @@ def list_resource(resource_type, db, attributes): return parsed_resources -def download_file_logic(s3_path: str, target_dir: str, target_path) -> bool: +def download_file_logic(s3_path: str) -> bool: s3 = connect_to_s3_logic() s3.ensure_bucket() - target = pathlib.Path(target_dir + "/app/" + target_path) + target = pathlib.Path(output_dir + "/app/" + s3_path) downloaded_file = s3.download_file(s3_path, target) if downloaded_file: - print(f"File '{s3_path}' downloaded to '{target}'.") + print(f"File '{s3_path}' downloaded to '{target}'.") else: - print(f"File '{s3_path}' not found in bucket.") + print(f"File '{s3_path}' not found in bucket.") return downloaded_file @@ -101,7 +105,9 @@ def add_seed_file_to_recipe(recipe_path, db_file_name): with open(recipe_path, 'r') as recipe_file: recipe_data = json.load(recipe_file) - recipe_data["db"]["scripts"].append({"filename" : db_file_name}) + if not any(script.get("filename") == db_file_name for script in recipe_data["db"]["scripts"]): + # If not, append it to the scripts list + recipe_data["db"]["scripts"].append({"filename": db_file_name}) with open(recipe_path, 'w') as recipe_file: #recipe_file.write(str(recipe_data)) @@ -115,21 +121,21 @@ def create_seed_files_db(resource_type, output_dir): file = open(file_path, 'w', encoding='utf-8') return file -def process_input(data, output_dir): - db = connect_to_db_logic() +def process_input(data, output): + global output_dir + output_dir = output create_recipe_file(output_dir) for resource_type, items in data.items(): create_seed_files_db(resource_type, output_dir) - add_seed_file_to_recipe(output_dir + "/recipe.json", "add_" + resource_type + ".sql") for item in items: - handle_resource(resource_type, item[resource_idenificators[resource_type]], db, output_dir) + handle_resource(resource_type, item[resource_identificator[resource_type]]) def write_seed_files_db(output_dir, resource_type, query): - with open(os.path.join(output_dir, f"add_{resource_type}.sql"), 'a') as file: + with open(os.path.join(output_dir, f"add_{resource_type}.sql"), 'a', encoding='utf-8') as file: if file is None: - print("File not found") + print("File not found") file.write(query + "\n") - print("Query added to file" + resource_type) + # print("Query added to file" + resource_type) def format_for_sql(data_dict): # needs help a lot @@ -171,7 +177,7 @@ def handle_projects(resource_id, db, recipe_file, resource_type, output_dir): handle_knowledge_models(resource[0]['package_id'], db, recipe_file, "knowledge_models", output_dir) handle_document_templates(resource[0]['document_template_id'], db, recipe_file, "document_templates", output_dir) else: - print("Project not found in database") + print("Project not found in database") def handle_documents(resource_id, db, recipe_file, resource_type, output_dir): query = generate_select_query(resource_type, "uuid", resource_id) @@ -194,32 +200,76 @@ def handle_document_templates(resource_id, db, recipe_file, resource_type, outpu handle_document_templates_files(resource_id, db, recipe_file) handle_document_templates_assets(resource_id, db, recipe_file, output_dir) else: - print("User not found in database") + print("User not found in database") def return_fkey_dependency(resource_type, dependent_resource_type): - for dependency in resource_dependencies.get(resource_type, []): - if dependency == dependent_resource_type: - return dependency[dependent_resource_type] - -def handle_resource(resource_type, resource_id, db, output_dir): - query = generate_select_query(resource_type, resource_idenificators[resource_type], resource_id) - resource = db.execute_query(query) - # dependencies - for dependency in resource_dependencies.get(resource_type, []): - if isinstance(dependency, dict): - dep_type = list(dependency.keys())[0] - dep_id = return_fkey_dependency(resource_type, dep_type) - handle_resource(dep_type, dep_id, db, output_dir) - - # s3 objects - for s3_object in resource_s3_objects[resource_type]: - if resource_id != 'wizard:default:1.0.0': - download_file_logic(s3_object + resource_id, output_dir, s3_object + "/" + resource[0][resource_s3_objects_fileNames[resource_type]]) - - insert_query = generate_insert_query(resource[0], resource_tables[resource_type]) - write_seed_files_db(output_dir, resource_type, insert_query) + # Loop through the list of dependencies for the given resource type + for dependency in resource_dependencies_keys.get(resource_type, []): + # Check if the dependency is a dictionary with the dependent_resource_type as a key + if dependent_resource_type in dependency.keys(): + # Return the foreign key field name + #print("Dependency: " + dependent_resource_type + " key" + dependency[dependent_resource_type]) + return str(dependency[dependent_resource_type]) + # Return None or an appropriate default if dependency is not found + return None + +def is_resource_part_of_sql_file(resource_type, output_dir, resource_id): + # print("Checking if resource is part of sql file " + resource_tables[resource_type]) + pattern = rf"INSERT INTO\s+{re.escape(resource_tables[resource_type])}\s*\([^\)]*\)\s*VALUES\s*\(\s*['\"]?{re.escape(str(resource_id))}['\"]?\s*,.*\)" + # print(pattern) + try: + with open(os.path.join(output_dir, f"add_{resource_type}.sql"), 'r') as file: + for line in file: + if pattern in line: + if re.search(pattern, line): + return True + # print("Resource not part of sql file") + return False + except FileNotFoundError: + # print("File not found") + return False + +def handle_resource(resource_type, resource_id): + # print(str(resource_id) + " " + resource_type) + if resource_id in processed_resources: + # print(f"Resource {resource_type} with ID {resource_id} has already been processed.") + print( "already processed resources ") + print(processed_resources) + return + else: + processed_resources.add(resource_id) + print("Processing resource" + resource_type + " " + str(resource_id)) + query = generate_select_query(resource_type, resource_identificator[resource_type], resource_id) + resources = db.execute_query(query) + for resource in resources: + # dependencies + for dependency in resource_dependencies.get(resource_type, []): + dep_type = dependency + dep_id_key = return_fkey_dependency(resource_type, dep_type) + if dep_id_key in resource: + dep_id = resource[dep_id_key] + # print("Dependency: " + str(dependency) + " Dependent resource id: " + str(dep_id)) + handle_resource(dep_type, dep_id) + + # s3 objects + for s3_object in resource_s3_objects[resource_type]: + if resource_id != 'wizard:default:1.0.0': + download_file_logic(s3_object + str(resource_id)) + # print(resource) + add_seed_file_to_recipe(output_dir + "/recipe.json", "add_" + resource_type + ".sql") + insert_query = generate_insert_query(resource, resource_tables[resource_type]) + write_seed_files_db(output_dir, resource_type, insert_query) + + # call handle_resource on dependent resources + for dependent_resource_type in resources_part_of.get(resource_type,[]): #obrazek dependencies a dependent resource + # print("Dependent resource type: " + str(dependent_resource_type)) + dependent_resource_id_key = return_fkey_dependency(dependent_resource_type, resource_type) + query = generate_select_query(dependent_resource_type, dependent_resource_id_key, resource[resource_identificator[resource_type]]) + dependent_resources = db.execute_query(query) + for dependent_resource in dependent_resources: + handle_resource(dependent_resource_type, dependent_resource[resource_identificator[dependent_resource_type]]) + return - #TODO dependent resources - document_template assetes and files def handle_document_templates_assets(doc_temp_id, db, recipe_file, output_dir): query = generate_select_query("document_template_asset", "document_template_id", doc_temp_id) @@ -231,14 +281,27 @@ def handle_document_templates_assets(doc_temp_id, db, recipe_file, output_dir): if s3_assets: print("File downloaded") else: - print("File not found") + print("File not found") -def handle_document_templates_files(doc_temp_id, db, recipe_file): +def handle_document_templates_files(doc_temp_id, db, output_dir): query = generate_select_query("document_template_file", "document_template_id", doc_temp_id) resource = db.execute_query(query) for file in resource: insert_query = generate_insert_query(file, "document_template_file") - write_seed_files_db(recipe_file, insert_query) + write_seed_files_db(output_dir, "document_template_file", insert_query) + +resources_part_of = { + "users": [], + "projects": [], + "documents": [], + "project_importers": [], + "knowledge_models": [], + "locales": [], + "document_templates": ["document_template_asset", "document_template_file"], + "document_template_asset": [], + "document_template_file": [] +} + resource_dependencies = { "users": [], @@ -247,7 +310,7 @@ def handle_document_templates_files(doc_temp_id, db, recipe_file): "project_importers": [], "knowledge_models": ["knowledge_models"], "locales": [], - "document_templates": ["document_template_asset", "document_template_file"], + "document_templates": [], "document_template_asset": ["document_templates"], "document_template_file": ["document_templates"] } @@ -296,7 +359,7 @@ def handle_document_templates_files(doc_temp_id, db, recipe_file): "document_template_file": ["templates/"] } -resource_idenificators = { +resource_identificator= { "users": "uuid", "projects": "uuid", "documents": "uuid",