From 11a7e414f7c4c9129cd26d487214cb8b037791b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sat, 11 May 2024 20:44:16 -0400 Subject: [PATCH 01/10] running with new package --- pybus.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pybus.py b/pybus.py index 4de59af..14e585e 100644 --- a/pybus.py +++ b/pybus.py @@ -9,12 +9,13 @@ import pytz from bs4 import BeautifulSoup from flask import request -from flask_api import FlaskAPI, status +#from flask_api import FlaskAPI, status +from flask import Flask -app = FlaskAPI(__name__) +app = Flask(__name__) # dynamodb table information -USERS_TABLE = os.environ['USERS_TABLE'] +USERS_TABLE = os.environ.get('USERS_TABLE', 'users-table-prod') dynamodb = boto3.resource('dynamodb', 'us-east-1') table = dynamodb.Table(USERS_TABLE) @@ -149,7 +150,7 @@ def parse_bus(): return_message = "Variable dest=" + direction.lower() + " invalid. Must be dest=" + destination[ 0].lower() \ + " or dest=" + destination[1].lower() - return return_message, status.HTTP_400_BAD_REQUEST + return return_message, 400 ### # Filtering output value to give @@ -209,7 +210,7 @@ def custom_sort(t): complete_return_value.extend(complete_return_value_sjsr[:direction_max]) complete_return_value.extend(complete_return_value_mtrl[:direction_max]) - return complete_return_value, status.HTTP_200_OK + return complete_return_value, 200 if __name__ == "__main__": From 0a958173f362d0456b015ad7f628d633b84bb079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sat, 11 May 2024 22:21:01 -0400 Subject: [PATCH 02/10] extracting function under app.route / --- pybus.py | 382 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 198 insertions(+), 184 deletions(-) diff --git a/pybus.py b/pybus.py index 14e585e..e1a650b 100644 --- a/pybus.py +++ b/pybus.py @@ -7,211 +7,225 @@ from botocore.exceptions import ClientError from pytz import timezone import pytz +from typing import List, Dict, Any from bs4 import BeautifulSoup from flask import request -#from flask_api import FlaskAPI, status from flask import Flask app = Flask(__name__) - +# if needed add logging +# app.logger.(my_variable) # dynamodb table information + USERS_TABLE = os.environ.get('USERS_TABLE', 'users-table-prod') +URL = ('http://www.ville.saint-jean-sur-richelieu.qc.ca/' + + 'transport-en-commun/Documents/horaires/96.html') +DYNAMODB_DATA = [] dynamodb = boto3.resource('dynamodb', 'us-east-1') table = dynamodb.Table(USERS_TABLE) +def remove_html_tags(text): + clean = re.compile('<.*?>') + return re.sub(clean, '', text) + +def list_of_speeds(argument): + """ + Filtering output value to give + """ + argument = remove_html_tags(argument) + switcher = { + 'S': "Super Express ", + 'E': "Express ", + 'E ☀<': "Express ☀ ", + 'L': "Locale ", + 'L ☀': "Locale ☀ ", + 'A': "Autoroute 30 ", + 'A ☀': "Autoroute 30 ☀ ", + } + return switcher.get(argument, "Wrong speed ") + +def populate_list_direction(city_start, list_start, city_end, list_end): + # TODO, aucune idée de ce que sa fait + for i in city_start: + i = str(i) + if i != '\n': + list_start.append(re.sub("<.*?>", "", i)) + for i in city_end: + i = str(i) + if i != '\n': + list_end.append(re.sub("<.*?>", "", i)) + +def custom_sort(t): + """ + Sorting bus rides based on the start hour + TODO, aucune idée de ce que sa fait + """ + dest, speed, hour_start, minutes_start, hour_end, minutes_end = t.split(':') + return hour_start, minutes_start + +def get_data_from_url(URL, remaining_retries=10, timeout=30): + try: + response_web = urllib.request.urlopen(URL, timeout=timeout) + return response_web.read() + except Exception as E: + if remaining_retries > 0: + print(f"{remaining_retries} remaining failed for {URL} exception is : {str(E)}") + return get_data_from_url(URL, remaining_retries -1, timeout) + else: + raise -@app.before_first_request -### -# try to get information from dynamodb -### -def get_or_update_dynamodb_data(): - ### - # TODO - # fix AWS credentials for the git server - ### +def get_data_from_db(): try: response = table.scan() - global dynamodb_data - dynamodb_data = response['Items'] except ClientError as e: print(e.response['Error']['Message']) print("Can't fetch data from dynamodb") - if len(dynamodb_data) <= 4: - ### - # Updating the dynamoDB only if empty - ### - url = ('http://www.ville.saint-jean-sur-richelieu.qc.ca/' + - 'transport-en-commun/Documents/horaires/96.html') - for tries in range(1, 11): - try: - response_web = urllib.request.urlopen(url, timeout=30) - html_doc = response_web.read() - except Exception as E: - print("Try " + str(tries) + "/10 failed for " + url + " exception is : " + str(E)) - continue - ### - # Parsing the collected data - ### - soup = BeautifulSoup(html_doc, 'html.parser') - dir_list = soup.find_all('div', attrs={"id": "div-horaires"}) - dir_to_mtrl_table = dir_list[0].find('table') - dir_from_mtrl_table = dir_list[3].find('table') - - speed_to_mtrl = dir_to_mtrl_table.find_all('div', attrs={"align": "center"}) - start_to_mtrl = dir_to_mtrl_table.find_all('tr')[1] - end_to_mtrl = dir_to_mtrl_table.find_all('tr')[-1] - - speed_to_sjsr = dir_from_mtrl_table.find_all('div', attrs={"align": "center"}) - start_to_sjsr = dir_from_mtrl_table.find_all('tr')[1] - end_to_sjsr = dir_from_mtrl_table.find_all('tr')[-1] - - start_to_mtrl_lst = [] - end_to_mtrl_lst = [] - start_to_sjsr_lst = [] - end_to_sjsr_lst = [] - - def populate_list_direction(city_start, list_start, city_end, list_end): - for i in city_start: - i = str(i) - if i != '\n': - list_start.append(re.sub("<.*?>", "", i)) - for i in city_end: - i = str(i) - if i != '\n': - list_end.append(re.sub("<.*?>", "", i)) - - populate_list_direction(start_to_mtrl, start_to_mtrl_lst, end_to_mtrl, end_to_mtrl_lst) - populate_list_direction(start_to_sjsr, start_to_sjsr_lst, end_to_sjsr, end_to_sjsr_lst) - ### - # Adding Saint-Jean-Sur-Le-Richelieu bus runs information - ### - for start, end, speed in zip(start_to_sjsr_lst, end_to_sjsr_lst, speed_to_sjsr): - try: - speed = str(speed) - destination = 'sjsr' - concat_response = (destination + ';' - + start + ';' - + end + ';' - + speed) - insert_data = table.put_item( - TableName=USERS_TABLE, - Item={ - 'data': str(concat_response) - } - ) - except ClientError as e: - print(e.response['Error']['Message']) - ### - # Adding Montreal bus runs information - ### - for start, end, speed in zip(start_to_mtrl_lst, end_to_mtrl_lst, speed_to_mtrl): - try: - speed = str(speed) - destination = 'mtrl' - concat_response = (destination + ';' - + start + ';' - + end + ';' - + speed) - insert_data = table.put_item( - TableName=USERS_TABLE, - Item={ - 'data': str(concat_response) - } - ) - except ClientError as e: - print(e.response['Error']['Message']) + # TODO, do we need to handle failures ? + return response['Items'] + +def parse_data(data) -> [List, List, List, List]: + soup = BeautifulSoup(data, 'html.parser') + dir_list = soup.find_all('div', attrs={"id": "div-horaires"}) + dir_to_mtrl_table = dir_list[0].find('table') + dir_from_mtrl_table = dir_list[3].find('table') + + speed_to_mtrl = dir_to_mtrl_table.find_all('div', attrs={"align": "center"}) + start_to_mtrl = dir_to_mtrl_table.find_all('tr')[1] + end_to_mtrl = dir_to_mtrl_table.find_all('tr')[-1] + + speed_to_sjsr = dir_from_mtrl_table.find_all('div', attrs={"align": "center"}) + start_to_sjsr = dir_from_mtrl_table.find_all('tr')[1] + end_to_sjsr = dir_from_mtrl_table.find_all('tr')[-1] + + start_to_mtrl_lst = [] + end_to_mtrl_lst = [] + start_to_sjsr_lst = [] + end_to_sjsr_lst = [] + + populate_list_direction(start_to_mtrl, start_to_mtrl_lst, end_to_mtrl, end_to_mtrl_lst) + populate_list_direction(start_to_sjsr, start_to_sjsr_lst, end_to_sjsr, end_to_sjsr_lst) + + return start_to_mtrl_lst, end_to_mtrl_lst, start_to_sjsr_lst, end_to_sjsr_lst + +def update_data_to_db(): + html_doc = get_data_from_url(URL) + start_to_mtrl_lst, end_to_mtrl_lst, start_to_sjsr_lst, end_to_sjsr_lst = parse_data(html_doc) + ### + # Adding Saint-Jean-Sur-Le-Richelieu bus runs information + ### + for start, end, speed in zip(start_to_sjsr_lst, end_to_sjsr_lst, speed_to_sjsr): + try: + speed = str(speed) + destination = 'sjsr' + concat_response = (f"{destination};{start};{end};{speed}") + insert_data = table.put_item( + TableName=USERS_TABLE, + Item={ + 'data': str(concat_response) + } + ) + except ClientError as e: + print(e.response['Error']['Message']) + ### + # Adding Montreal bus runs information + ### + for start, end, speed in zip(start_to_mtrl_lst, end_to_mtrl_lst, speed_to_mtrl): + try: + speed = str(speed) + destination = 'mtrl' + concat_response = (destination + ';' + + start + ';' + + end + ';' + + speed) + insert_data = table.put_item( + TableName=USERS_TABLE, + Item={ + 'data': str(concat_response) + } + ) + except ClientError as e: + print(e.response['Error']['Message']) +@app.before_first_request +### +# try to get information from dynamodb +### +def refresh_data(): + global DYNAMODB_DATA + DYNAMODB_DATA = get_data_from_db() + if len(DYNAMODB_DATA) <= 4: + update_data_to_db() + @app.route("/", methods=['GET']) def parse_bus(): - if request.method == 'GET': - ### - # Defining Montreal timezone to match data source timezone - ### - localtime = datetime.now(pytz.utc) - montreal_tz = timezone('America/Montreal') - date = localtime.astimezone(montreal_tz) - date_time_hours = date.strftime("%H") - date_time_minutes = date.strftime("%M") - ### - # Parsing given parameters - ### - if request.args.get("max"): - direction_max = request.args.get("max", "") - direction_max = int(direction_max) - else: - direction_max = 10 - if request.args.get("dest"): - direction = request.args.get("dest", "") - else: - direction = 'all' - destination = ['sjsr', 'mtrl', 'all'] - if direction.lower() not in destination: - return_message = "Variable dest=" + direction.lower() + " invalid. Must be dest=" + destination[ - 0].lower() \ - + " or dest=" + destination[1].lower() - return return_message, 400 - - ### - # Filtering output value to give - ## - def list_of_speeds(argument): - switcher = { - '
S
': "Super Express ", - '
S ☀
': "Super Express ☀", - '
E
': "Express ", - '
E ☀
': "Express ☀ ", - '
L
': "Locale ", - '
L ☀
': "Locale ☀ ", - '
A
': "Autoroute 30 ", - '
A ☀
': "Autoroute 30 ☀ ", - } - return switcher.get(argument, "Wrong speed ") - - complete_return_value_mtrl = [] - complete_return_value_sjsr = [] - complete_return_value = [] - - def populate_complete(bus_data): - for entry in bus_data: - # Entry example - # sjsr;17:06;17:46;
S
- dest, start, end, speed = entry['data'].split(';') - loop_hours, loop_minutes = start.split(':') - combined_loop_minutes = int(loop_hours)*60 + int(loop_minutes) - combined_date_minutes = int(date_time_hours)*60 + int(date_time_minutes) - if combined_loop_minutes >= combined_date_minutes: - value = ("Autobus destination " + dest + " : " - + list_of_speeds(speed) + " Depart:" - + start + " Arriver:" + end) - if dest == 'sjsr': - complete_return_value_sjsr.append(value) - elif dest == 'mtrl': - complete_return_value_mtrl.append(value) - - populate_complete(dynamodb_data) - ### - # Sorting bus rides based on the start hour - ### - - def custom_sort(t): - dest, speed, hour_start, minutes_start, hour_end, minutes_end = t.split(':') - return hour_start, minutes_start - complete_return_value_sjsr = sorted(complete_return_value_sjsr, key=custom_sort) - complete_return_value_mtrl = sorted(complete_return_value_mtrl, key=custom_sort) - ### - # Adding only the bus rides up to direction_max - ### - if direction == 'sjsr': - complete_return_value = complete_return_value_sjsr[:direction_max] - elif direction == 'mtrl': - complete_return_value = complete_return_value_mtrl[:direction_max] - else: - complete_return_value.extend(complete_return_value_sjsr[:direction_max]) - complete_return_value.extend(complete_return_value_mtrl[:direction_max]) + global DYNAMODB_DATA + ### + # Defining Montreal timezone to match data source timezone + ### + localtime = datetime.now(pytz.utc) + montreal_tz = timezone('America/Montreal') + date = localtime.astimezone(montreal_tz) + date_time_hours = date.strftime("%H") + date_time_minutes = date.strftime("%M") + ### + # Parsing given parameters + ### + if request.args.get("max"): + direction_max = request.args.get("max", "") + direction_max = int(direction_max) + else: + direction_max = 10 + if request.args.get("dest"): + direction = request.args.get("dest", "") + else: + direction = 'all' + destination = ['sjsr', 'mtrl', 'all'] + if direction.lower() not in destination: + return_message = "Variable dest=" + direction.lower() + " invalid. Must be dest=" + destination[ + 0].lower() \ + + " or dest=" + destination[1].lower() + return return_message, 400 + + complete_return_value_mtrl = [] + complete_return_value_sjsr = [] + complete_return_value = [] + + def populate_complete(bus_data): + for entry in bus_data: + # Entry example + # sjsr;17:06;17:46;
S
+ dest, start, end, speed = entry['data'].split(';') + loop_hours, loop_minutes = start.split(':') + combined_loop_minutes = int(loop_hours)*60 + int(loop_minutes) + combined_date_minutes = int(date_time_hours)*60 + int(date_time_minutes) + if combined_loop_minutes >= combined_date_minutes: + value = ("Autobus destination " + dest + " : " + + list_of_speeds(speed) + " Depart:" + + start + " Arriver:" + end) + if dest == 'sjsr': + complete_return_value_sjsr.append(value) + elif dest == 'mtrl': + complete_return_value_mtrl.append(value) + + populate_complete(DYNAMODB_DATA) + complete_return_value_sjsr = sorted(complete_return_value_sjsr, key=custom_sort) + complete_return_value_mtrl = sorted(complete_return_value_mtrl, key=custom_sort) + ### + # Adding only the bus rides up to direction_max + ### + if direction == 'sjsr': + complete_return_value = complete_return_value_sjsr[:direction_max] + elif direction == 'mtrl': + complete_return_value = complete_return_value_mtrl[:direction_max] + else: + complete_return_value.extend(complete_return_value_sjsr[:direction_max]) + complete_return_value.extend(complete_return_value_mtrl[:direction_max]) - return complete_return_value, 200 + return complete_return_value, 200 if __name__ == "__main__": - app.run() + app.run( + debug=True # remove me + ) From 08cce477bb6c96964b1263de8d216134c3fe8fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sat, 11 May 2024 22:48:20 -0400 Subject: [PATCH 03/10] rendering of the response text extracted into simpler functions --- pybus.py | 84 ++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/pybus.py b/pybus.py index e1a650b..39be7f9 100644 --- a/pybus.py +++ b/pybus.py @@ -156,19 +156,42 @@ def refresh_data(): DYNAMODB_DATA = get_data_from_db() if len(DYNAMODB_DATA) <= 4: update_data_to_db() - -@app.route("/", methods=['GET']) -def parse_bus(): - global DYNAMODB_DATA - ### + +def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): # Defining Montreal timezone to match data source timezone - ### localtime = datetime.now(pytz.utc) montreal_tz = timezone('America/Montreal') date = localtime.astimezone(montreal_tz) date_time_hours = date.strftime("%H") date_time_minutes = date.strftime("%M") - ### + + rendered_data = [] + # Entry example + # sjsr;17:06;17:46;
S
+ found_sjsr = 0 + found_mtrl = 0 + for entry in data: + dest, start, end, speed = entry['data'].split(';') + loop_hours, loop_minutes = start.split(':') + combined_loop_minutes = int(loop_hours)*60 + int(loop_minutes) + combined_date_minutes = int(date_time_hours)*60 + int(date_time_minutes) + if combined_loop_minutes >= combined_date_minutes: + value = ("Autobus destination " + dest + " : " + + list_of_speeds(speed) + " Depart:" + + start + " Arriver:" + end) + if dest == 'sjsr' and sjsr is True and found_sjsr < direction_max: + rendered_data.append(value) + found_sjsr = found_sjsr + 1 + elif dest == 'mtrl' and mtrl is True and found_mtrl < direction_max: + rendered_data.append(value) + found_mtrl = found_mtrl +1 + + return sorted(rendered_data, key=custom_sort) + +@app.route("/", methods=['GET']) +def parse_bus(): + global DYNAMODB_DATA + ### # Parsing given parameters ### if request.args.get("max"): @@ -186,44 +209,15 @@ def parse_bus(): 0].lower() \ + " or dest=" + destination[1].lower() return return_message, 400 - - complete_return_value_mtrl = [] - complete_return_value_sjsr = [] - complete_return_value = [] - - def populate_complete(bus_data): - for entry in bus_data: - # Entry example - # sjsr;17:06;17:46;
S
- dest, start, end, speed = entry['data'].split(';') - loop_hours, loop_minutes = start.split(':') - combined_loop_minutes = int(loop_hours)*60 + int(loop_minutes) - combined_date_minutes = int(date_time_hours)*60 + int(date_time_minutes) - if combined_loop_minutes >= combined_date_minutes: - value = ("Autobus destination " + dest + " : " - + list_of_speeds(speed) + " Depart:" - + start + " Arriver:" + end) - if dest == 'sjsr': - complete_return_value_sjsr.append(value) - elif dest == 'mtrl': - complete_return_value_mtrl.append(value) - - populate_complete(DYNAMODB_DATA) - complete_return_value_sjsr = sorted(complete_return_value_sjsr, key=custom_sort) - complete_return_value_mtrl = sorted(complete_return_value_mtrl, key=custom_sort) - ### - # Adding only the bus rides up to direction_max - ### - if direction == 'sjsr': - complete_return_value = complete_return_value_sjsr[:direction_max] - elif direction == 'mtrl': - complete_return_value = complete_return_value_mtrl[:direction_max] - else: - complete_return_value.extend(complete_return_value_sjsr[:direction_max]) - complete_return_value.extend(complete_return_value_mtrl[:direction_max]) - - return complete_return_value, 200 - + + sjsr = True if (direction == 'sjsr') else False + mtrl = True if (direction == 'mtrl') else False + return render_bus_data( + DYNAMODB_DATA, + sjsr, + mtrl, + direction_max + ), 200 if __name__ == "__main__": app.run( From 352bbb0ad26d2a121dd645a697e0e54909d6d0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sat, 11 May 2024 23:43:33 -0400 Subject: [PATCH 04/10] new url --- pybus.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pybus.py b/pybus.py index 39be7f9..7935b6f 100644 --- a/pybus.py +++ b/pybus.py @@ -18,8 +18,7 @@ # dynamodb table information USERS_TABLE = os.environ.get('USERS_TABLE', 'users-table-prod') -URL = ('http://www.ville.saint-jean-sur-richelieu.qc.ca/' + - 'transport-en-commun/Documents/horaires/96.html') +URL = 'https://sjsr.ca/horaires/96.html' DYNAMODB_DATA = [] dynamodb = boto3.resource('dynamodb', 'us-east-1') table = dynamodb.Table(USERS_TABLE) From 98f6bcbeef5b7115da448948e6e088725280689e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 00:03:55 -0400 Subject: [PATCH 05/10] fixing update data parsing --- pybus.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pybus.py b/pybus.py index 7935b6f..8abc248 100644 --- a/pybus.py +++ b/pybus.py @@ -45,14 +45,18 @@ def list_of_speeds(argument): def populate_list_direction(city_start, list_start, city_end, list_end): # TODO, aucune idée de ce que sa fait + app.logger.info(city_start, list_start, city_end, list_end) for i in city_start: i = str(i) if i != '\n': - list_start.append(re.sub("<.*?>", "", i)) + #list_start.append(re.sub("<.*?>", "", i)) + list_start.append(remove_html_tags(i)) for i in city_end: i = str(i) if i != '\n': - list_end.append(re.sub("<.*?>", "", i)) + #list_end.append(re.sub("<.*?>", "", i)) + list_start.append(remove_html_tags(i)) + app.logger.info(city_start, list_start, city_end, list_end) def custom_sort(t): """ @@ -96,15 +100,12 @@ def parse_data(data) -> [List, List, List, List]: start_to_sjsr = dir_from_mtrl_table.find_all('tr')[1] end_to_sjsr = dir_from_mtrl_table.find_all('tr')[-1] - start_to_mtrl_lst = [] - end_to_mtrl_lst = [] - start_to_sjsr_lst = [] - end_to_sjsr_lst = [] + start_to_mtrl = [remove_html_tags(i) for i in start_to_mtrl] + end_to_mtrl = [remove_html_tags(i) for i in end_to_mtrl] + start_to_sjsr = [remove_html_tags(i) for i in start_to_sjsr] + end_to_sjsr = [remove_html_tags(i) for i in end_to_sjsr] - populate_list_direction(start_to_mtrl, start_to_mtrl_lst, end_to_mtrl, end_to_mtrl_lst) - populate_list_direction(start_to_sjsr, start_to_sjsr_lst, end_to_sjsr, end_to_sjsr_lst) - - return start_to_mtrl_lst, end_to_mtrl_lst, start_to_sjsr_lst, end_to_sjsr_lst + return start_to_mtrl, end_to_mtrl, start_to_sjsr, end_to_sjsr def update_data_to_db(): html_doc = get_data_from_url(URL) @@ -152,7 +153,8 @@ def update_data_to_db(): ### def refresh_data(): global DYNAMODB_DATA - DYNAMODB_DATA = get_data_from_db() + # TODO, uncomment me ! + #DYNAMODB_DATA = get_data_from_db() if len(DYNAMODB_DATA) <= 4: update_data_to_db() From 09b2ff0c7393a111248dc21deb365b88b81b301d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 00:10:23 -0400 Subject: [PATCH 06/10] remove uneccesarry function --- pybus.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pybus.py b/pybus.py index 8abc248..733f039 100644 --- a/pybus.py +++ b/pybus.py @@ -100,20 +100,20 @@ def parse_data(data) -> [List, List, List, List]: start_to_sjsr = dir_from_mtrl_table.find_all('tr')[1] end_to_sjsr = dir_from_mtrl_table.find_all('tr')[-1] - start_to_mtrl = [remove_html_tags(i) for i in start_to_mtrl] - end_to_mtrl = [remove_html_tags(i) for i in end_to_mtrl] - start_to_sjsr = [remove_html_tags(i) for i in start_to_sjsr] - end_to_sjsr = [remove_html_tags(i) for i in end_to_sjsr] + start_to_mtrl = [remove_html_tags(str(i)) for i in start_to_mtrl] + end_to_mtrl = [remove_html_tags(str(i)) for i in end_to_mtrl] + start_to_sjsr = [remove_html_tags(str(i)) for i in start_to_sjsr] + end_to_sjsr = [remove_html_tags(str(i)) for i in end_to_sjsr] - return start_to_mtrl, end_to_mtrl, start_to_sjsr, end_to_sjsr + return start_to_mtrl, end_to_mtrl, speed_to_mtrl, start_to_sjsr, end_to_sjsr, speed_to_sjsr def update_data_to_db(): html_doc = get_data_from_url(URL) - start_to_mtrl_lst, end_to_mtrl_lst, start_to_sjsr_lst, end_to_sjsr_lst = parse_data(html_doc) + start_to_mtrl, end_to_mtrl, speed_to_mtrl, start_to_sjsr, end_to_sjsr, speed_to_sjsr = parse_data(html_doc) ### # Adding Saint-Jean-Sur-Le-Richelieu bus runs information ### - for start, end, speed in zip(start_to_sjsr_lst, end_to_sjsr_lst, speed_to_sjsr): + for start, end, speed in zip(start_to_sjsr, end_to_sjsr, speed_to_sjsr): try: speed = str(speed) destination = 'sjsr' @@ -129,7 +129,7 @@ def update_data_to_db(): ### # Adding Montreal bus runs information ### - for start, end, speed in zip(start_to_mtrl_lst, end_to_mtrl_lst, speed_to_mtrl): + for start, end, speed in zip(start_to_mtrl, end_to_mtrl, speed_to_mtrl): try: speed = str(speed) destination = 'mtrl' From fb8ca72d6eed9f4c453cd380a5c2c384d2cc1aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 00:16:12 -0400 Subject: [PATCH 07/10] arg parsing --- pybus.py | 37 +++++-------------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/pybus.py b/pybus.py index 733f039..9770032 100644 --- a/pybus.py +++ b/pybus.py @@ -43,21 +43,6 @@ def list_of_speeds(argument): } return switcher.get(argument, "Wrong speed ") -def populate_list_direction(city_start, list_start, city_end, list_end): - # TODO, aucune idée de ce que sa fait - app.logger.info(city_start, list_start, city_end, list_end) - for i in city_start: - i = str(i) - if i != '\n': - #list_start.append(re.sub("<.*?>", "", i)) - list_start.append(remove_html_tags(i)) - for i in city_end: - i = str(i) - if i != '\n': - #list_end.append(re.sub("<.*?>", "", i)) - list_start.append(remove_html_tags(i)) - app.logger.info(city_start, list_start, city_end, list_end) - def custom_sort(t): """ Sorting bus rides based on the start hour @@ -192,25 +177,13 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): @app.route("/", methods=['GET']) def parse_bus(): global DYNAMODB_DATA - ### # Parsing given parameters - ### - if request.args.get("max"): - direction_max = request.args.get("max", "") - direction_max = int(direction_max) - else: - direction_max = 10 - if request.args.get("dest"): - direction = request.args.get("dest", "") - else: - direction = 'all' + direction_max = int(request.args.get("max", 10)) + direction = request.args.get("dest", "all").lower() destination = ['sjsr', 'mtrl', 'all'] - if direction.lower() not in destination: - return_message = "Variable dest=" + direction.lower() + " invalid. Must be dest=" + destination[ - 0].lower() \ - + " or dest=" + destination[1].lower() - return return_message, 400 - + if direction not in destination: + return f"Variable dest={direction} invalid. Must be dest={destination[0]} or dest={destination}", 400 + sjsr = True if (direction == 'sjsr') else False mtrl = True if (direction == 'mtrl') else False return render_bus_data( From f621c48b0e7f41a02e6f54ce42d78295076b6172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 00:21:30 -0400 Subject: [PATCH 08/10] change value --- pybus.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pybus.py b/pybus.py index 9770032..9129c7c 100644 --- a/pybus.py +++ b/pybus.py @@ -162,9 +162,8 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): combined_loop_minutes = int(loop_hours)*60 + int(loop_minutes) combined_date_minutes = int(date_time_hours)*60 + int(date_time_minutes) if combined_loop_minutes >= combined_date_minutes: - value = ("Autobus destination " + dest + " : " - + list_of_speeds(speed) + " Depart:" - + start + " Arriver:" + end) + speed = list_of_speeds(speed) + value = f"Autobus destination {dest}: {speed} Depart:{start} Arriver:{end}" if dest == 'sjsr' and sjsr is True and found_sjsr < direction_max: rendered_data.append(value) found_sjsr = found_sjsr + 1 From a52ba1e10090f5994970df1546b0e71a8ddd8ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 00:30:14 -0400 Subject: [PATCH 09/10] working --- pybus.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pybus.py b/pybus.py index 9129c7c..09bcaa4 100644 --- a/pybus.py +++ b/pybus.py @@ -142,6 +142,7 @@ def refresh_data(): #DYNAMODB_DATA = get_data_from_db() if len(DYNAMODB_DATA) <= 4: update_data_to_db() + DYNAMODB_DATA = get_data_from_db() def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): # Defining Montreal timezone to match data source timezone @@ -150,7 +151,9 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): date = localtime.astimezone(montreal_tz) date_time_hours = date.strftime("%H") date_time_minutes = date.strftime("%M") - + + app.logger.info('TEST') + app.logger.info(data) rendered_data = [] # Entry example # sjsr;17:06;17:46;
S
@@ -164,13 +167,14 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): if combined_loop_minutes >= combined_date_minutes: speed = list_of_speeds(speed) value = f"Autobus destination {dest}: {speed} Depart:{start} Arriver:{end}" + app.logger.info(f"Autobus destination {dest}: {speed} Depart:{start} Arriver:{end}") if dest == 'sjsr' and sjsr is True and found_sjsr < direction_max: rendered_data.append(value) found_sjsr = found_sjsr + 1 elif dest == 'mtrl' and mtrl is True and found_mtrl < direction_max: rendered_data.append(value) found_mtrl = found_mtrl +1 - + app.logger.info(rendered_data) return sorted(rendered_data, key=custom_sort) @app.route("/", methods=['GET']) @@ -183,12 +187,10 @@ def parse_bus(): if direction not in destination: return f"Variable dest={direction} invalid. Must be dest={destination[0]} or dest={destination}", 400 - sjsr = True if (direction == 'sjsr') else False - mtrl = True if (direction == 'mtrl') else False return render_bus_data( DYNAMODB_DATA, - sjsr, - mtrl, + True if (direction == 'sjsr' or direction == 'all') else False, + True if (direction == 'mtrl' or direction == 'all ') else False, direction_max ), 200 From 14c0cd59937eb099a583f56a612d1b5ca0ae411a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Bellavance?= Date: Sun, 12 May 2024 01:01:40 -0400 Subject: [PATCH 10/10] all working --- pybus.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pybus.py b/pybus.py index 09bcaa4..7b46db2 100644 --- a/pybus.py +++ b/pybus.py @@ -138,13 +138,11 @@ def update_data_to_db(): ### def refresh_data(): global DYNAMODB_DATA - # TODO, uncomment me ! - #DYNAMODB_DATA = get_data_from_db() if len(DYNAMODB_DATA) <= 4: + DYNAMODB_DATA = get_data_from_db() update_data_to_db() - DYNAMODB_DATA = get_data_from_db() -def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): +def render_bus_data(data: List, sjsr: bool, mtrl: bool, direction_max: int): # Defining Montreal timezone to match data source timezone localtime = datetime.now(pytz.utc) montreal_tz = timezone('America/Montreal') @@ -152,8 +150,6 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): date_time_hours = date.strftime("%H") date_time_minutes = date.strftime("%M") - app.logger.info('TEST') - app.logger.info(data) rendered_data = [] # Entry example # sjsr;17:06;17:46;
S
@@ -167,14 +163,12 @@ def render_bus_data(data, sjsr: bool, mtrl: bool, direction_max: int): if combined_loop_minutes >= combined_date_minutes: speed = list_of_speeds(speed) value = f"Autobus destination {dest}: {speed} Depart:{start} Arriver:{end}" - app.logger.info(f"Autobus destination {dest}: {speed} Depart:{start} Arriver:{end}") - if dest == 'sjsr' and sjsr is True and found_sjsr < direction_max: + if dest == 'sjsr' and sjsr is True and found_sjsr < (direction_max / 2): rendered_data.append(value) found_sjsr = found_sjsr + 1 - elif dest == 'mtrl' and mtrl is True and found_mtrl < direction_max: + if dest == 'mtrl' and mtrl is True and found_mtrl < (direction_max / 2): rendered_data.append(value) found_mtrl = found_mtrl +1 - app.logger.info(rendered_data) return sorted(rendered_data, key=custom_sort) @app.route("/", methods=['GET']) @@ -186,11 +180,10 @@ def parse_bus(): destination = ['sjsr', 'mtrl', 'all'] if direction not in destination: return f"Variable dest={direction} invalid. Must be dest={destination[0]} or dest={destination}", 400 - return render_bus_data( DYNAMODB_DATA, True if (direction == 'sjsr' or direction == 'all') else False, - True if (direction == 'mtrl' or direction == 'all ') else False, + True if (direction == 'mtrl' or direction == 'all') else False, direction_max ), 200