From dff174c72a7d5ea52aa604dc59336c5c9446ac54 Mon Sep 17 00:00:00 2001 From: ca20110820 Date: Wed, 29 May 2024 12:02:07 +0800 Subject: [PATCH 1/5] fix: resolved path issues using pathlib. fixes #12 --- app/app.py | 7 ++-- app/utils.py | 93 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/app/app.py b/app/app.py index d060138..c24560a 100644 --- a/app/app.py +++ b/app/app.py @@ -1,3 +1,4 @@ +from pathlib import Path import os.path import logging import shutil @@ -103,7 +104,7 @@ def serve_video(): Serve local/downloaded video file to view/template :return: Video file """ - video_path = f'{utils.get_vid_save_path()}{filename}' + video_path = str(Path(utils.get_vid_save_path(), f'{filename}').resolve()) return send_file(video_path) @@ -172,7 +173,8 @@ def upload_video(): if file: if not os.path.exists(f"{utils.get_vid_save_path()}"): os.makedirs(f"{utils.get_vid_save_path()}") - file.save(f"{utils.get_vid_save_path()}" + file.filename) + save_path = Path(utils.get_vid_save_path(), file.filename).resolve() + file.save(save_path) global filename filename = file.filename file_hash = utils.hash_video_file(filename) @@ -230,6 +232,7 @@ def update_settings(): def reset_settings(): print("Current working directory:", os.getcwd()) # Delete the existing config.ini file + os.chdir(str(utils.APP_DIR)) if os.path.exists('config.ini'): os.remove('config.ini') shutil.copy('config.example.ini', 'config.ini') diff --git a/app/utils.py b/app/utils.py index 76a8506..eaffd39 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,19 +1,23 @@ import hashlib import json import os.path +from pathlib import Path import shutil import subprocess import logging import time import cv2 from json import JSONDecodeError -from typing import Union, Optional +from typing import Union, Optional, Any, List import openai import pytesseract from pytube import YouTube from pytube.exceptions import RegexMatchError from configparser import ConfigParser +FILE_PATH = Path(__file__).resolve() # Absolute Path of utils.py +APP_DIR = FILE_PATH.parent +PROJ_ROOT = APP_DIR.parent def config(section: str = None, option: str = None) -> Union[ConfigParser, str]: """ @@ -24,11 +28,12 @@ def config(section: str = None, option: str = None) -> Union[ConfigParser, str]: :param option: [Optional] Key/option of value to retrieve :return: Return string or ConfigParser object """ + if (section is None) != (option is None): raise SyntaxError("section AND option parameters OR no parameters must be passed to function config()") parser = ConfigParser() - if not os.path.exists("config.ini"): - shutil.copy("config.example.ini", "config.ini") + if not (APP_DIR / 'config.ini').exists(): + shutil.copy(APP_DIR / 'config.example.ini', APP_DIR / 'config.ini') parser.read("config.ini") if parser.get("AppSettings", "openai_api_key") != "your_openai_api_key_here": openai.api_key = parser.get("AppSettings", "openai_api_key") @@ -49,7 +54,9 @@ def hash_video_file(filename: str) -> str: :return: Returns hex based md5 hash """ hash_md5 = hashlib.md5() - with open(f"{get_vid_save_path()}{filename}", "rb") as f: + # with open(f"{get_vid_save_path()}{filename}", "rb") as f: + video_file_path = Path(get_vid_save_path(), f'{filename}').resolve() + with video_file_path.open('rb') as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) return hash_md5.hexdigest() @@ -77,20 +84,21 @@ def format_timestamp(seconds: int) -> str: return f'{str(minutes).zfill(2)}:{str(remaining_seconds).zfill(2)}' -def read_user_data() -> json: +def read_user_data() -> Union[Any, None]: """ Reads the users data from json file :return: Returns user data as json """ - if not os.path.exists("data\\userdata.json"): - if not os.path.exists("data\\"): - os.makedirs("data\\") - with open("data\\userdata.json", "w") as user_data: + user_data_path = APP_DIR / 'data' / 'userdata.json' + if not user_data_path.exists(): + data_dir = APP_DIR / 'data' + if not data_dir.exists(): + data_dir.mkdir(parents=True, exist_ok=True) # data directory + with user_data_path.open('w') as user_data: user_data.write(json.dumps({"all_videos": []})) - pass return None try: - with open("data\\userdata.json", "r") as user_data_json: + with user_data_path.open('r') as user_data_json: data = json.load(user_data_json) return data except JSONDecodeError: @@ -104,18 +112,18 @@ def get_vid_save_path() -> str: :return: file path as string """ vid_download_path = config("UserSettings", "video_save_path") + # Set default output path for video download path if vid_download_path == "output_path": - default_path = os.path.dirname(os.getcwd()) + "\\out\\videos\\" - if not os.path.exists(default_path): - os.makedirs(default_path) + default_path = PROJ_ROOT / 'out' / 'videos' + if not default_path.exists(): + default_path.mkdir(parents=True, exist_ok=True) return default_path - # Check if the path ends with a backslash - if not vid_download_path.endswith("\\"): - # If it doesn't end with a backslash, append one - vid_download_path += "\\" - return vid_download_path + # if not vid_download_path.endswith(('/', '\\')): + # vid_download_path += '\\' if os.name == 'nt' else '/' + + return str(Path(vid_download_path).resolve()) def get_output_path() -> str: @@ -125,16 +133,17 @@ def get_output_path() -> str: """ output_path = config("UserSettings", "capture_output_path") # Set default output path for code files + if output_path == "output_path": - default_path = os.path.dirname(os.getcwd()) + "\\out\\" - if not os.path.exists(default_path): - os.makedirs(default_path) + default_path = PROJ_ROOT / 'out' + if not default_path.exists(): + default_path.mkdir(parents=True, exist_ok=True) return default_path - # Check if the path ends with a backslash - if not output_path.endswith("\\"): - # If it doesn't end with a backslash, append one - output_path += "\\" - return output_path + + # if not output_path.endswith(('/', '\\')): + # output_path += '\\' if os.name == 'nt' else '/' + + return str(Path(output_path).resolve()) def send_code_snippet_to_ide(filename: str, code_snippet: str) -> bool: @@ -266,7 +275,8 @@ def update_user_video_data(filename: str, progress: Optional[float] = None, capt record["progress"] = round(progress) if capture is not None: record["captures"].append(capture) - with open("data/userdata.json", "w") as json_data: + + with (APP_DIR / 'data' / 'userdata.json').open('w') as json_data: json.dump(user_data, json_data, indent=4) @@ -281,7 +291,10 @@ def add_video_to_user_data(filename: str, video_title: str, video_hash: str, you user_data = read_user_data() if user_data is None: return - video_capture = cv2.VideoCapture(f'{get_vid_save_path()}{filename}') + + # video_capture = cv2.VideoCapture(f'{get_vid_save_path()}{filename}') + video_path = str(Path(get_vid_save_path(), f'{filename}').resolve()) + video_capture = cv2.VideoCapture(video_path) if not video_capture.isOpened(): logging.error(f"Failed to open video capture for {filename}") return @@ -294,9 +307,13 @@ def add_video_to_user_data(filename: str, video_title: str, video_hash: str, you return thumbnail = str(int(time.time())) + ".png" # Check if img dir exists if not create - if not os.path.exists("static/img"): - os.makedirs("static/img") - cv2.imwrite(f"static/img/{thumbnail}", frame) + + static_dir = APP_DIR / 'static' + img_dir = static_dir / 'img' + if not img_dir.exists(): + img_dir.mkdir(parents=True, exist_ok=True) + + cv2.imwrite(str(img_dir / f'{thumbnail}'), frame) new_video = { "video_hash": video_hash, "filename": filename, @@ -310,7 +327,7 @@ def add_video_to_user_data(filename: str, video_title: str, video_hash: str, you new_video["youtube_url"] = youtube_url video_capture.release() user_data["all_videos"].append(new_video) - with open("data/userdata.json", "w") as json_data: + with (APP_DIR / 'data' / 'userdata.json').open('w') as json_data: json.dump(user_data, json_data, indent=4) @@ -329,7 +346,7 @@ def file_already_exists(video_hash: str) -> bool: return False -def get_setup_progress() -> [str]: +def get_setup_progress() -> List[str]: """ Gets users set up progress from config file :return: Returns array of string containing strings relating to settings that are already set up @@ -347,7 +364,7 @@ def get_setup_progress() -> [str]: return setup_progress -def parse_video_data() -> []: +def parse_video_data() -> List[list]: """ Gets all video data from userdata storage and parses all data for in progress videos :return: Array containing two arrays, 1 with all videos 1 with in progress videos @@ -444,7 +461,8 @@ def delete_video_from_userdata(filename: str) -> None: if current_video["filename"] == filename: all_videos.remove(current_video) break - with open("data/userdata.json", "w") as json_data: + + with (APP_DIR / 'data' / 'userdata.json').open('w') as json_data: json.dump(user_data, json_data, indent=4) @@ -464,8 +482,9 @@ def update_configuration(new_values_dict) -> None: if isinstance(value, bool) or isinstance(value, int): value = str(value) config_file.set(section, key, value) + # save the file - with open('config.ini', 'w') as config_file_save: + with (APP_DIR / 'config.ini').open('w') as config_file_save: config_file.write(config_file_save) From 8480fc375ca94d8a58032ed4c8bddc5cfb1348eb Mon Sep 17 00:00:00 2001 From: ca20110820 Date: Wed, 29 May 2024 20:35:22 +0800 Subject: [PATCH 2/5] fix(utils): fixed issues with video and out save paths removed current working directory and set default paths for out and videos --- app/utils.py | 35 ++++++++++++++++++++++++----------- tests/test_utils.py | 6 ++++-- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/app/utils.py b/app/utils.py index eaffd39..efb9e95 100644 --- a/app/utils.py +++ b/app/utils.py @@ -33,7 +33,9 @@ def config(section: str = None, option: str = None) -> Union[ConfigParser, str]: raise SyntaxError("section AND option parameters OR no parameters must be passed to function config()") parser = ConfigParser() if not (APP_DIR / 'config.ini').exists(): - shutil.copy(APP_DIR / 'config.example.ini', APP_DIR / 'config.ini') + src_path = str(APP_DIR / 'config.example.ini') + dst_path = str(APP_DIR / 'config.ini') + shutil.copy(src_path, dst_path) parser.read("config.ini") if parser.get("AppSettings", "openai_api_key") != "your_openai_api_key_here": openai.api_key = parser.get("AppSettings", "openai_api_key") @@ -54,7 +56,6 @@ def hash_video_file(filename: str) -> str: :return: Returns hex based md5 hash """ hash_md5 = hashlib.md5() - # with open(f"{get_vid_save_path()}{filename}", "rb") as f: video_file_path = Path(get_vid_save_path(), f'{filename}').resolve() with video_file_path.open('rb') as f: for chunk in iter(lambda: f.read(4096), b""): @@ -106,6 +107,17 @@ def read_user_data() -> Union[Any, None]: return None +def directory_append_slash(directory: str) -> str: + # Check if directory already have slash + if directory.endswith(('/', '\\')): + return directory + + # Append slash + directory += '\\' if os.name == 'nt' else '/' + + return directory + + def get_vid_save_path() -> str: """ Returns output path from config variables, will set default to root of project\\out\\videos\\ @@ -118,12 +130,13 @@ def get_vid_save_path() -> str: default_path = PROJ_ROOT / 'out' / 'videos' if not default_path.exists(): default_path.mkdir(parents=True, exist_ok=True) + + default_path = str(default_path) return default_path - # if not vid_download_path.endswith(('/', '\\')): - # vid_download_path += '\\' if os.name == 'nt' else '/' + vid_download_path = str(Path(vid_download_path)) - return str(Path(vid_download_path).resolve()) + return directory_append_slash(vid_download_path) def get_output_path() -> str: @@ -138,12 +151,13 @@ def get_output_path() -> str: default_path = PROJ_ROOT / 'out' if not default_path.exists(): default_path.mkdir(parents=True, exist_ok=True) - return default_path - - # if not output_path.endswith(('/', '\\')): - # output_path += '\\' if os.name == 'nt' else '/' + + default_path = str(default_path) + + return directory_append_slash(default_path) - return str(Path(output_path).resolve()) + output_path = str(Path(output_path)) + return directory_append_slash(output_path) def send_code_snippet_to_ide(filename: str, code_snippet: str) -> bool: @@ -292,7 +306,6 @@ def add_video_to_user_data(filename: str, video_title: str, video_hash: str, you if user_data is None: return - # video_capture = cv2.VideoCapture(f'{get_vid_save_path()}{filename}') video_path = str(Path(get_vid_save_path(), f'{filename}').resolve()) video_capture = cv2.VideoCapture(video_path) if not video_capture.isOpened(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 16dc7d7..fd70318 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -117,9 +117,11 @@ def test_hash_string(): def test_get_output_path(mocker): + project_root = str(utils.PROJ_ROOT) + default_path = utils.directory_append_slash(os.path.join(project_root, 'out')) test_output_paths = { - "c:\\users\\program files\\app": "c:\\users\\program files\\app\\", - "output_path": os.path.dirname(os.getcwd()) + "\\out\\", + "C:\\Users\\program files\\app": "C:\\Users\\program files\\app\\", + "output_path": default_path, "videos\\my_videos\\": "videos\\my_videos\\", } for paths in test_output_paths: From cde5e43685931bd6f00e8b5fcb93d2a283d6b2de Mon Sep 17 00:00:00 2001 From: ca20110820 Date: Thu, 30 May 2024 12:50:46 +0800 Subject: [PATCH 3/5] refactor(tests_utils): updated test_get_output_path method to check the host system This requires updating due to using Ubuntu in GitHub workflows for testing. --- app/utils.py | 10 ++++++++-- tests/test_utils.py | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app/utils.py b/app/utils.py index efb9e95..f412455 100644 --- a/app/utils.py +++ b/app/utils.py @@ -108,11 +108,17 @@ def read_user_data() -> Union[Any, None]: def directory_append_slash(directory: str) -> str: - # Check if directory already have slash + """ + Append a trailing slash to a directory path if it doesn't already have one. + :param directory: The directory path to which a trailing slash will be appended, if missing. + :return: The directory path with a trailing slash appended, if necessary. + """ + + # Check if directory already have trailing slash if directory.endswith(('/', '\\')): return directory - # Append slash + # Append trailing slash directory += '\\' if os.name == 'nt' else '/' return directory diff --git a/tests/test_utils.py b/tests/test_utils.py index fd70318..82106dc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -119,11 +119,25 @@ def test_hash_string(): def test_get_output_path(mocker): project_root = str(utils.PROJ_ROOT) default_path = utils.directory_append_slash(os.path.join(project_root, 'out')) + test_output_paths = { - "C:\\Users\\program files\\app": "C:\\Users\\program files\\app\\", "output_path": default_path, - "videos\\my_videos\\": "videos\\my_videos\\", } + + assert os.name in ['nt', 'posix'] + + # Windows + if os.name == 'nt': + test_output_paths["c:\\users\\program files\\app"] = "c:\\users\\program files\\app\\" + test_output_paths["videos\\my_videos\\"] = "videos\\my_videos\\" + + # Linux or macOS (Note: GitHub Runner is using ubuntu) + else: + assert os.name == 'posix' + users_dir = 'home' if 'home' in os.path.expanduser("~") else 'Users' + test_output_paths[f"/{users_dir}/program_files/app"] = f"/{users_dir}/program_files/app/" + test_output_paths["videos/my_videos/"] = "videos/my_videos/" + for paths in test_output_paths: mocker.patch("app.utils.config", return_value=paths) assert utils.get_output_path() == test_output_paths[paths] From 4f026528be78fc9c4c374a206b80f6b867c9d45d Mon Sep 17 00:00:00 2001 From: ca20110820 Date: Thu, 30 May 2024 13:25:34 +0800 Subject: [PATCH 4/5] refactor(app): minor refactoring and removed pathlib Restored original implementations of serve_video and upload_video functions in app.py script. --- app/app.py | 6 ++---- app/utils.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/app.py b/app/app.py index c24560a..1b2a22d 100644 --- a/app/app.py +++ b/app/app.py @@ -1,4 +1,3 @@ -from pathlib import Path import os.path import logging import shutil @@ -104,7 +103,7 @@ def serve_video(): Serve local/downloaded video file to view/template :return: Video file """ - video_path = str(Path(utils.get_vid_save_path(), f'{filename}').resolve()) + video_path = f'{utils.get_vid_save_path()}{filename}' return send_file(video_path) @@ -173,8 +172,7 @@ def upload_video(): if file: if not os.path.exists(f"{utils.get_vid_save_path()}"): os.makedirs(f"{utils.get_vid_save_path()}") - save_path = Path(utils.get_vid_save_path(), file.filename).resolve() - file.save(save_path) + file.save(f"{utils.get_vid_save_path()}" + file.filename) global filename filename = file.filename file_hash = utils.hash_video_file(filename) diff --git a/app/utils.py b/app/utils.py index f412455..e8e748f 100644 --- a/app/utils.py +++ b/app/utils.py @@ -138,7 +138,7 @@ def get_vid_save_path() -> str: default_path.mkdir(parents=True, exist_ok=True) default_path = str(default_path) - return default_path + return directory_append_slash(default_path) vid_download_path = str(Path(vid_download_path)) From 649a6574671f100fbc82c3728bb188f8a01f118b Mon Sep 17 00:00:00 2001 From: ca20110820 Date: Tue, 4 Jun 2024 19:46:52 +0800 Subject: [PATCH 5/5] minor type hint fixes --- app/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/utils.py b/app/utils.py index e8e748f..cb3190a 100644 --- a/app/utils.py +++ b/app/utils.py @@ -8,7 +8,7 @@ import time import cv2 from json import JSONDecodeError -from typing import Union, Optional, Any, List +from typing import Union, Optional, Any, List, Dict import openai import pytesseract from pytube import YouTube @@ -85,7 +85,7 @@ def format_timestamp(seconds: int) -> str: return f'{str(minutes).zfill(2)}:{str(remaining_seconds).zfill(2)}' -def read_user_data() -> Union[Any, None]: +def read_user_data() -> Optional[Any]: """ Reads the users data from json file :return: Returns user data as json @@ -383,7 +383,7 @@ def get_setup_progress() -> List[str]: return setup_progress -def parse_video_data() -> List[list]: +def parse_video_data() -> dict: """ Gets all video data from userdata storage and parses all data for in progress videos :return: Array containing two arrays, 1 with all videos 1 with in progress videos