From 20ab7381f3dd9091a47f0c1fde1e044a1fa39009 Mon Sep 17 00:00:00 2001 From: lexara-prime-ai Date: Tue, 4 Jun 2024 09:26:08 +0000 Subject: [PATCH 1/4] Updated error handling on hyper module. --- hyper/hyper/constants.py | 4 +- hyper/hyper/server.py | 154 +++++++++++++++++++++++------------- scripts/bash/python_deps.sh | 22 +++--- 3 files changed, 111 insertions(+), 69 deletions(-) diff --git a/hyper/hyper/constants.py b/hyper/hyper/constants.py index 1343b13..c76a558 100644 --- a/hyper/hyper/constants.py +++ b/hyper/hyper/constants.py @@ -1,3 +1,3 @@ class CONSTANTS: - FILE_PATH = './tableau/data' - FILE_NAME = 'wspr_spots.csv' \ No newline at end of file + FILE_PATH = "./tableau/data" + FILE_NAME = "wspr_spots.csv" diff --git a/hyper/hyper/server.py b/hyper/hyper/server.py index 518535e..ee76466 100644 --- a/hyper/hyper/server.py +++ b/hyper/hyper/server.py @@ -2,77 +2,119 @@ import csv import os -import python_wrapper.python_wrapper import constants +import python_wrapper.python_wrapper + class Server: def __init__(self): self.write_path = os.path.join( - constants.CONSTANTS.FILE_PATH, 'wspr_spot_data.csv') + constants.CONSTANTS.FILE_PATH, "wspr_spot_data.csv" + ) async def write_to_csv(self): """ - Args: self. - return type: () + Args: self. + return type: () """ - output = await python_wrapper.python_wrapper.get_wspr_spots("10", "JSON") - data = output.get_data() - - # Display data that's being fetched for [DEBUG] purposes. - await self.display_data(data) - - # Write data to csv. -> - write_path = self.write_path - print('\nWrite path: \n', write_path) - - # Check if directory exists. - if not os.path.exists(constants.CONSTANTS.FILE_PATH): - os.makedirs(constants.CONSTANTS.FILE_PATH) - - with open(write_path, mode='w', newline='') as file: - writer = csv.writer(file) - writer.writerow([ - 'ID', 'Time', 'Band', 'RX Sign', 'RX Lat', 'RX Lon', 'RX Loc', - 'TX Sign', 'TX Lat', 'TX Lon', 'TX Loc', 'Distance', 'Azimuth', - 'RX Azimuth', 'Frequency', 'Power', 'SNR', 'Drift', 'Version', 'Code' - ]) - - for record in data: - writer.writerow([ - record['id'], record['time'], record['band'], record['rx_sign'], - record['rx_lat'], record['rx_lon'], record['rx_loc'], record['tx_sign'], - record['tx_lat'], record['tx_lon'], record['tx_loc'], record['distance'], - record['azimuth'], record['rx_azimuth'], record['frequency'], - record['power'], record['snr'], record['drift'], record['version'], record['code'] - ]) + + try: + output = await python_wrapper.python_wrapper.get_wspr_spots("10", "JSON") + data = output.get_data() + + # Display data that's being fetched for [DEBUG] purposes. + await self.display_data(data) + + # Write data to csv. -> + write_path = self.write_path + print("\nWrite path: \n", write_path) + + # Check if directory exists. + if not os.path.exists(constants.CONSTANTS.FILE_PATH): + os.makedirs(constants.CONSTANTS.FILE_PATH) + + with open(write_path, mode="w", newline="") as file: + writer = csv.writer(file) + writer.writerow( + [ + "ID", + "Time", + "Band", + "RX Sign", + "RX Lat", + "RX Lon", + "RX Loc", + "TX Sign", + "TX Lat", + "TX Lon", + "TX Loc", + "Distance", + "Azimuth", + "RX Azimuth", + "Frequency", + "Power", + "SNR", + "Drift", + "Version", + "Code", + ] + ) + + for record in data: + writer.writerow( + [ + record["id"], + record["time"], + record["band"], + record["rx_sign"], + record["rx_lat"], + record["rx_lon"], + record["rx_loc"], + record["tx_sign"], + record["tx_lat"], + record["tx_lon"], + record["tx_loc"], + record["distance"], + record["azimuth"], + record["rx_azimuth"], + record["frequency"], + record["power"], + record["snr"], + record["drift"], + record["version"], + record["code"], + ] + ) + except Exception as e: + print("An [ERROR] occurred: ", e) async def display_data(self, data): """ - Args: self, data -> WsprSpot dict. - return type: () + Args: self, data -> WsprSpot dict. + return type: () """ for record in data: - id_field = record['id'] - time_field = record['time'] - band_field = record['band'] - rx_sign_field = record['rx_sign'] - rx_lat_field = record['rx_lat'] - rx_lon_field = record['rx_lon'] - rx_loc_field = record['rx_loc'] - tx_sign_field = record['tx_sign'] - tx_lat_field = record['tx_lat'] - tx_lon_field = record['tx_lon'] - tx_loc_field = record['tx_loc'] - distance_field = record['distance'] - azimuth_field = record['azimuth'] - rx_azimuth_field = record['rx_azimuth'] - frequency_field = record['frequency'] - power_field = record['power'] - snr_field = record['snr'] - drift_field = record['drift'] - version_field = record['version'] - code_field = record['code'] + id_field = record["id"] + time_field = record["time"] + band_field = record["band"] + rx_sign_field = record["rx_sign"] + rx_lat_field = record["rx_lat"] + rx_lon_field = record["rx_lon"] + rx_loc_field = record["rx_loc"] + tx_sign_field = record["tx_sign"] + tx_lat_field = record["tx_lat"] + tx_lon_field = record["tx_lon"] + tx_loc_field = record["tx_loc"] + distance_field = record["distance"] + azimuth_field = record["azimuth"] + rx_azimuth_field = record["rx_azimuth"] + frequency_field = record["frequency"] + power_field = record["power"] + snr_field = record["snr"] + drift_field = record["drift"] + version_field = record["version"] + code_field = record["code"] # Verify content. print("\nFetching [ROW] >\n") diff --git a/scripts/bash/python_deps.sh b/scripts/bash/python_deps.sh index 600ea26..66b1c89 100755 --- a/scripts/bash/python_deps.sh +++ b/scripts/bash/python_deps.sh @@ -6,11 +6,11 @@ sudo apt-get update # Install pip if it is not already installed echo "Checking for pip..." -if ! command -v pip &> /dev/null; then - echo "pip not found. Installing pip..." - sudo apt-get install -y python3-pip +if ! command -v pip &>/dev/null; then + echo "pip not found. Installing pip..." + sudo apt-get install -y python3-pip else - echo "pip is already installed." + echo "pip is already installed." fi # Install mkdocs @@ -20,15 +20,15 @@ pip install tableauhyperapi # Verify installation echo "Verifying mkdocs installation..." -if python3 -c "import mkdocs" &> /dev/null; then - echo "mkdocs successfully installed." +if python3 -c "import mkdocs" &>/dev/null; then + echo "mkdocs successfully installed." else - echo "Failed to install mkdocs." + echo "Failed to install mkdocs." fi echo "Verifying tableauhyperapi installation..." -if python3 -c "import tableauhyperapi" &> /dev/null; then - echo "tableauhyperapi successfully installed." +if python3 -c "import tableauhyperapi" &>/dev/null; then + echo "tableauhyperapi successfully installed." else - echo "Failed to install tableauhyperapi." -fi \ No newline at end of file + echo "Failed to install tableauhyperapi." +fi From 51c01c91d60d1e8f84f18370f5687211e6c98dda Mon Sep 17 00:00:00 2001 From: lexara-prime-ai Date: Tue, 4 Jun 2024 09:43:45 +0000 Subject: [PATCH 2/4] Refactored script. --- hyper/hyper/drive.py | 0 scripts/bash/python_deps.sh | 54 +++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 hyper/hyper/drive.py diff --git a/hyper/hyper/drive.py b/hyper/hyper/drive.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/bash/python_deps.sh b/scripts/bash/python_deps.sh index 66b1c89..8ca403a 100755 --- a/scripts/bash/python_deps.sh +++ b/scripts/bash/python_deps.sh @@ -1,11 +1,6 @@ #!/bin/bash -# Update the package list -echo "Updating package list..." -sudo apt-get update - -# Install pip if it is not already installed -echo "Checking for pip..." +# Check if pip is installed if ! command -v pip &>/dev/null; then echo "pip not found. Installing pip..." sudo apt-get install -y python3-pip @@ -13,22 +8,35 @@ else echo "pip is already installed." fi -# Install mkdocs -echo "Installing dependencies [mkdocs], [tableauhyperapi]..." -pip install mkdocs -pip install tableauhyperapi +# List of modules to install +modules=("mkdocs" "tableauhyperapi" "google-api-python-client" "google-auth-httplib2" "google-auth-oauthlib") -# Verify installation -echo "Verifying mkdocs installation..." -if python3 -c "import mkdocs" &>/dev/null; then - echo "mkdocs successfully installed." -else - echo "Failed to install mkdocs." -fi +# Install modules +echo "Installing dependencies: ${modules[*]}..." +pip install "${modules[@]}" --upgrade -echo "Verifying tableauhyperapi installation..." -if python3 -c "import tableauhyperapi" &>/dev/null; then - echo "tableauhyperapi successfully installed." -else - echo "Failed to install tableauhyperapi." -fi +# Function to verify module installation +verify_installation() { + local module=$1 + local import_name=$2 + echo "Verifying ${module} installation..." + if python3 -c "import ${import_name}" &>/dev/null; then + echo "${module} successfully installed." + else + echo "Failed to install ${module}." + fi +} + +# Dictionary of module to import name mappings +declare -A module_import_map=( + ["mkdocs"]="mkdocs" + ["tableauhyperapi"]="tableauhyperapi" + ["google-api-python-client"]="googleapiclient" + ["google-auth-httplib2"]="google_auth_httplib2" + ["google-auth-oauthlib"]="google_auth_oauthlib" +) + +# Verify installation of each module +for module in "${!module_import_map[@]}"; do + verify_installation "${module}" "${module_import_map[${module}]}" +done From 67430a30bc2da8d086353920e33b68f56cd96080 Mon Sep 17 00:00:00 2001 From: lexara-prime-ai Date: Tue, 4 Jun 2024 09:45:49 +0000 Subject: [PATCH 3/4] Added code comments for script --- scripts/bash/python_deps.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/bash/python_deps.sh b/scripts/bash/python_deps.sh index 8ca403a..708c78e 100755 --- a/scripts/bash/python_deps.sh +++ b/scripts/bash/python_deps.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Check if pip is installed +# Check if pip is installed. if ! command -v pip &>/dev/null; then echo "pip not found. Installing pip..." sudo apt-get install -y python3-pip @@ -8,14 +8,13 @@ else echo "pip is already installed." fi -# List of modules to install +# Modules that will be installed/upgraded. modules=("mkdocs" "tableauhyperapi" "google-api-python-client" "google-auth-httplib2" "google-auth-oauthlib") -# Install modules echo "Installing dependencies: ${modules[*]}..." pip install "${modules[@]}" --upgrade -# Function to verify module installation +# Verify module installation. verify_installation() { local module=$1 local import_name=$2 @@ -27,7 +26,7 @@ verify_installation() { fi } -# Dictionary of module to import name mappings +# The following dictionary contains module to import name mappings. declare -A module_import_map=( ["mkdocs"]="mkdocs" ["tableauhyperapi"]="tableauhyperapi" @@ -36,7 +35,7 @@ declare -A module_import_map=( ["google-auth-oauthlib"]="google_auth_oauthlib" ) -# Verify installation of each module +# Verify installation of each module. for module in "${!module_import_map[@]}"; do verify_installation "${module}" "${module_import_map[${module}]}" done From 9d3fa0e2e918a7d6883369fa8db987f669dcbb4f Mon Sep 17 00:00:00 2001 From: lexara-prime-ai Date: Tue, 4 Jun 2024 12:01:30 +0000 Subject: [PATCH 4/4] Added logic to upload to Google Drive. --- .gitignore | 1 + hyper/hyper/constants.py | 9 ++++++--- hyper/hyper/drive.py | 33 +++++++++++++++++++++++++++++++++ hyper/hyper/server.py | 14 ++++++++------ scripts/bash/python_deps.sh | 7 +++++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 53f10a5..9db29d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .terraform *.tfstate .env +service_account.json # Added by cargo /target diff --git a/hyper/hyper/constants.py b/hyper/hyper/constants.py index c76a558..b63f89f 100644 --- a/hyper/hyper/constants.py +++ b/hyper/hyper/constants.py @@ -1,3 +1,6 @@ -class CONSTANTS: - FILE_PATH = "./tableau/data" - FILE_NAME = "wspr_spots.csv" +FILE_PATH = "./tableau/data" +FILE_NAME = "wspr_spot_data.csv" +FULL_PATH = FILE_PATH + "/" + FILE_NAME +SCOPES = ["https://www.googleapis.com/auth/drive"] +SERVICE_ACCOUNT_FILE = "service_account.json" +PARENT_FOLDER_ID = "1uVnscmoxu91XT1LMnZoOOn3iCTirlEII" diff --git a/hyper/hyper/drive.py b/hyper/hyper/drive.py index e69de29..73ca03f 100644 --- a/hyper/hyper/drive.py +++ b/hyper/hyper/drive.py @@ -0,0 +1,33 @@ +import constants +from google.oauth2 import service_account +from googleapiclient.discovery import build + + +def authenticate(): + try: + print("\n[Authenticating] service user...") + + credentials = service_account.Credentials.from_service_account_file( + constants.SERVICE_ACCOUNT_FILE, scopes=constants.SCOPES + ) + + return credentials + except Exception as e: + print("\n[ERROR] -> Failed to [Authenticate] service user: \n", e) + + +def upload_to_drive(file_path): + try: + credentials = authenticate() + service = build("drive", "v3", credentials=credentials) + + file_metadata = { + "name": "wspr_spot_data", + "parents": [constants.PARENT_FOLDER_ID], + } + + print("[Uploading] file to Google Drive...\n") + + service.files().create(body=file_metadata, media_body=file_path).execute() + except Exception as e: + print("\n[ERROR] -> Failed to upload to Google Drive: \n", e) diff --git a/hyper/hyper/server.py b/hyper/hyper/server.py index ee76466..3025e3b 100644 --- a/hyper/hyper/server.py +++ b/hyper/hyper/server.py @@ -3,15 +3,14 @@ import os import constants +import drive import python_wrapper.python_wrapper class Server: def __init__(self): - self.write_path = os.path.join( - constants.CONSTANTS.FILE_PATH, "wspr_spot_data.csv" - ) + self.write_path = os.path.join(constants.FILE_PATH, "wspr_spot_data.csv") async def write_to_csv(self): """ @@ -20,7 +19,7 @@ async def write_to_csv(self): """ try: - output = await python_wrapper.python_wrapper.get_wspr_spots("10", "JSON") + output = await python_wrapper.python_wrapper.get_wspr_spots("1000", "JSON") data = output.get_data() # Display data that's being fetched for [DEBUG] purposes. @@ -31,8 +30,8 @@ async def write_to_csv(self): print("\nWrite path: \n", write_path) # Check if directory exists. - if not os.path.exists(constants.CONSTANTS.FILE_PATH): - os.makedirs(constants.CONSTANTS.FILE_PATH) + if not os.path.exists(constants.FILE_PATH): + os.makedirs(constants.FILE_PATH) with open(write_path, mode="w", newline="") as file: writer = csv.writer(file) @@ -86,6 +85,9 @@ async def write_to_csv(self): record["code"], ] ) + + # Upload [output] file to Google Drive. + drive.upload_to_drive(constants.FULL_PATH) except Exception as e: print("An [ERROR] occurred: ", e) diff --git a/scripts/bash/python_deps.sh b/scripts/bash/python_deps.sh index 708c78e..9c69854 100755 --- a/scripts/bash/python_deps.sh +++ b/scripts/bash/python_deps.sh @@ -39,3 +39,10 @@ declare -A module_import_map=( for module in "${!module_import_map[@]}"; do verify_installation "${module}" "${module_import_map[${module}]}" done + + +# An array modules contains the names of all the modules to be installed. +# The pip install command installs or upgrades all the modules listed in the array. +# A dictionary module_import_map maps module names to their respective import names. +# The verify_installation function takes a module name and its import name as arguments, checks if the module can be imported, and prints the appropriate message. +# The script iterates over the module_import_map dictionary to verify the installation of each module using the verify_installation function. \ No newline at end of file