From 62ebe6526fd67853536a9ca4d45ac78018bcf2a1 Mon Sep 17 00:00:00 2001 From: lyricnz Date: Fri, 15 Mar 2024 00:25:54 +0000 Subject: [PATCH 1/8] [MegaLinter] Apply linters fixes --- code/nbn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/nbn.py b/code/nbn.py index ddcd403577..14e78332a3 100644 --- a/code/nbn.py +++ b/code/nbn.py @@ -1,7 +1,7 @@ +import difflib import logging import urllib.parse -import difflib import diskcache import requests from requests.adapters import HTTPAdapter @@ -43,7 +43,7 @@ def get_nbn_loc_id(self, key: str, address: str) -> str: suggestions = sorted( suggestions, key=lambda s: difflib.SequenceMatcher(None, address, s["formattedAddress"]).ratio(), - reverse=True + reverse=True, ) if suggestions: loc_id = result["suggestions"][0]["id"] From 2aecca36792c13236a130d53c8c4bae60bdc3d74 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Fri, 15 Mar 2024 11:29:50 +1100 Subject: [PATCH 2/8] Generate state-breakdown of connection types over time. Also CSV exports of same (per state) --- code/adhoc_tools.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/code/adhoc_tools.py b/code/adhoc_tools.py index c319483598..af70595eb9 100644 --- a/code/adhoc_tools.py +++ b/code/adhoc_tools.py @@ -16,6 +16,7 @@ import utils from bs4 import BeautifulSoup from tabulate import tabulate +import csv NBN_UPGRADE_DATES_URL = ( "https://www.nbnco.com.au/corporate-information/media-centre/media-statements/nbnco-announces-suburbs-and" @@ -257,6 +258,36 @@ def generate_all_suburbs_nbn_tallies(): utils.write_json_file("results/all-suburbs-nbn-tallies.json", tallies, indent=1) +def generate_state_breakdown(): + """Generate results/breakdown.STATE.csv containing history of connection-types by state""" + output = {} + all_ctypes = set() + for date, state_info in utils.read_json_file('results/breakdown-suburbs.json').items(): + logging.info("Processing %s", date) + output[date] = {} + for state, suburb_list in state_info.items(): + # logging.info(" State: %s", state) + state_tally = {} + for suburb, connections in suburb_list.items(): + # logging.info(" State: %s", suburb) + for ctype, ccount in connections.items(): + state_tally[ctype] = state_tally.get(ctype, 0) + ccount + all_ctypes.add(ctype) + output[date][state] = state_tally + utils.write_json_file('results/breakdown-state.json', output) + + # write CSV per state + for state in data.STATES: + rows = [ + {'date': date} | {ctype: output[date].get(state, {}).get(ctype, 0) for ctype in all_ctypes} + for date in output + ] + with open(f'results/breakdown.{state}.csv', 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(rows[0].keys()) + writer.writerows(r.values() for r in rows + + if __name__ == "__main__": LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() logging.basicConfig(level=LOGLEVEL, format="%(asctime)s %(levelname)s %(threadName)s %(message)s") From a04fa70a5ef5a429f38183c1c587d4086de620c5 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Fri, 15 Mar 2024 12:05:47 +1100 Subject: [PATCH 3/8] Update adhoc_tools.py --- code/adhoc_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/adhoc_tools.py b/code/adhoc_tools.py index af70595eb9..06eb5db960 100644 --- a/code/adhoc_tools.py +++ b/code/adhoc_tools.py @@ -285,7 +285,7 @@ def generate_state_breakdown(): with open(f'results/breakdown.{state}.csv', 'w', newline='') as f: writer = csv.writer(f) writer.writerow(rows[0].keys()) - writer.writerows(r.values() for r in rows + writer.writerows(r.values() for r in rows) if __name__ == "__main__": From 7cbad2d9709aeba92b0d9940609fa616180558b6 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Fri, 15 Mar 2024 12:11:05 +1100 Subject: [PATCH 4/8] reformat --- code/adhoc_tools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/code/adhoc_tools.py b/code/adhoc_tools.py index 06eb5db960..0dfe1c05a6 100644 --- a/code/adhoc_tools.py +++ b/code/adhoc_tools.py @@ -1,4 +1,5 @@ import argparse +import csv import glob import logging import os @@ -7,16 +8,16 @@ from collections import Counter, OrderedDict from datetime import datetime, timedelta +import requests +from bs4 import BeautifulSoup +from tabulate import tabulate + import data import db import geojson import main -import requests import suburbs import utils -from bs4 import BeautifulSoup -from tabulate import tabulate -import csv NBN_UPGRADE_DATES_URL = ( "https://www.nbnco.com.au/corporate-information/media-centre/media-statements/nbnco-announces-suburbs-and" From 6c51f9ca8d98d0e6125624c2a0a99aa5eb368445 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Fri, 15 Mar 2024 14:20:56 +1100 Subject: [PATCH 5/8] Remove whitespace --- code/adhoc_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/adhoc_tools.py b/code/adhoc_tools.py index 0dfe1c05a6..220ca710b8 100644 --- a/code/adhoc_tools.py +++ b/code/adhoc_tools.py @@ -288,7 +288,7 @@ def generate_state_breakdown(): writer.writerow(rows[0].keys()) writer.writerows(r.values() for r in rows) - + if __name__ == "__main__": LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() logging.basicConfig(level=LOGLEVEL, format="%(asctime)s %(levelname)s %(threadName)s %(message)s") From 0ced3f210f69651114bf5d029b4d6b9981ef6afc Mon Sep 17 00:00:00 2001 From: lyricnz Date: Fri, 15 Mar 2024 03:22:11 +0000 Subject: [PATCH 6/8] [MegaLinter] Apply linters fixes --- code/adhoc_tools.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/code/adhoc_tools.py b/code/adhoc_tools.py index 220ca710b8..84705a0336 100644 --- a/code/adhoc_tools.py +++ b/code/adhoc_tools.py @@ -8,16 +8,15 @@ from collections import Counter, OrderedDict from datetime import datetime, timedelta -import requests -from bs4 import BeautifulSoup -from tabulate import tabulate - import data import db import geojson import main +import requests import suburbs import utils +from bs4 import BeautifulSoup +from tabulate import tabulate NBN_UPGRADE_DATES_URL = ( "https://www.nbnco.com.au/corporate-information/media-centre/media-statements/nbnco-announces-suburbs-and" @@ -263,7 +262,7 @@ def generate_state_breakdown(): """Generate results/breakdown.STATE.csv containing history of connection-types by state""" output = {} all_ctypes = set() - for date, state_info in utils.read_json_file('results/breakdown-suburbs.json').items(): + for date, state_info in utils.read_json_file("results/breakdown-suburbs.json").items(): logging.info("Processing %s", date) output[date] = {} for state, suburb_list in state_info.items(): @@ -275,15 +274,15 @@ def generate_state_breakdown(): state_tally[ctype] = state_tally.get(ctype, 0) + ccount all_ctypes.add(ctype) output[date][state] = state_tally - utils.write_json_file('results/breakdown-state.json', output) + utils.write_json_file("results/breakdown-state.json", output) # write CSV per state for state in data.STATES: rows = [ - {'date': date} | {ctype: output[date].get(state, {}).get(ctype, 0) for ctype in all_ctypes} + {"date": date} | {ctype: output[date].get(state, {}).get(ctype, 0) for ctype in all_ctypes} for date in output ] - with open(f'results/breakdown.{state}.csv', 'w', newline='') as f: + with open(f"results/breakdown.{state}.csv", "w", newline="") as f: writer = csv.writer(f) writer.writerow(rows[0].keys()) writer.writerows(r.values() for r in rows) From c56f7464733e6948f76c0ae89d1a4e69546f2ce4 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Fri, 15 Mar 2024 19:38:32 +1100 Subject: [PATCH 7/8] Automatically create state-by-state breakdown --- code/update_breakdown.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/update_breakdown.py b/code/update_breakdown.py index ea73738f7b..d0af3b2ed8 100755 --- a/code/update_breakdown.py +++ b/code/update_breakdown.py @@ -5,7 +5,7 @@ from datetime import datetime import utils -from adhoc_tools import get_tech_and_upgrade_breakdown +from adhoc_tools import get_tech_and_upgrade_breakdown, generate_state_breakdown from tabulate import tabulate @@ -40,3 +40,4 @@ def print_breakdowns(breakdowns): logging.basicConfig(level=logging.INFO) bd = update_breakdown() print_breakdowns(bd) + generate_state_breakdown() From 15078588ad909cbcf36e61a4c8337c04df8245c8 Mon Sep 17 00:00:00 2001 From: lyricnz Date: Fri, 15 Mar 2024 08:40:00 +0000 Subject: [PATCH 8/8] [MegaLinter] Apply linters fixes --- code/update_breakdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/update_breakdown.py b/code/update_breakdown.py index d0af3b2ed8..3d78faa1b2 100755 --- a/code/update_breakdown.py +++ b/code/update_breakdown.py @@ -5,7 +5,7 @@ from datetime import datetime import utils -from adhoc_tools import get_tech_and_upgrade_breakdown, generate_state_breakdown +from adhoc_tools import generate_state_breakdown, get_tech_and_upgrade_breakdown from tabulate import tabulate