From b98b3411944295e7982ac832a34d55e8f326dcef Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Mon, 18 Dec 2023 17:41:31 +0100 Subject: [PATCH] KML generation of remaining invaders --- README.md | 15 ++++++ generate_to_flash.py | 76 +++++++++++++++++++++++++++- kml.py | 72 ++++++++++++++++++++++++++ scraper.py | 117 +++++++++++++++---------------------------- 4 files changed, 200 insertions(+), 80 deletions(-) create mode 100644 kml.py diff --git a/README.md b/README.md index 0b63611..76aba92 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ python -m venv .venv ## Usage +### Scraping known Invaders and locations + First, scrape all known invaders together with their status and location: ``` @@ -28,6 +30,19 @@ This will generate 3 files: * `data/all-invaders.kml`, a ready to import KML file with all the invaders as bookmarks/placemarks. +### Generating a KML of all remaining invaders + +Provided you have a KML file of your flashed invaders (or a GPX and convert it +to KML), you can run + +``` +./.venv/bin/python generate_to_flash.py KML_FILE_OF_FLASHED_INVADERS +``` + +to generate a `data/invaders-to-flash.kml` KML file containing all the known +invaders locations that you did not flash and that are known to be active at +the moment. + ## License diff --git a/generate_to_flash.py b/generate_to_flash.py index 3b19feb..cd702d0 100644 --- a/generate_to_flash.py +++ b/generate_to_flash.py @@ -1,8 +1,80 @@ #!/uhttpssr/bin/env python +import csv +import re import sys -import simplekml +import fastkml + +from kml import generate_kml + -# TODO if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit('Usage: %s KML_FILE' % sys.argv[0]) + + # Load user-provided KML file + k = fastkml.kml.KML() + with open(sys.argv[1], 'rb') as fh: + k.from_string(fh.read()) + + # Iterate over Document, Folder etc. to find features + # TODO: Only support nested structure with a single Folder + features = list(k.features()) + while True: + try: + features = list(features[0].features()) + except AttributeError: + break + + # Load all the known invaders with locations + with open('data/invaders-with-locations.csv', 'r') as fh: + reader = csv.DictReader(fh) + invaders = list(reader) + invaders_by_name = { + invader['name']: invader + for invader in invaders + } + + # Match flashed invaders (from the KML) with known invaders + matched_invaders = [] + for feature in features: + feature_name = feature.name.upper().replace('-', '_') + name = re.search('[A-Z]+_[0-9]+', feature_name) + if not name: + print('Ignored:', feature_name) + continue + name = name.group(0) + + invader_match = None + while not invader_match: + try: + invader_match = invaders_by_name[name] + except KeyError: + if '_0' not in name: + print('Not found:', name) + break + name = name.replace('_0', '_') + matched_invaders.append(name) + + # Output a filtered KML file with only the remaining invaders (working and + # not flashed) + + # Filter out already flashed invaders + invaders_to_flash = [ + invader + for invader in invaders + if invader['name'] not in matched_invaders + ] + # Filter out unflashable invaders + working_invaders_to_flash = [ + invader + for invader in invaders_to_flash + if invader['status_description'] not in ['Détruit !', 'Non visible'] + ] + generate_kml( + invaders=working_invaders_to_flash, + out_file='data/invaders-to-flash.kml', + name='Invaders left to flash', + ) + print('%s invaders left to flash!' % len(working_invaders_to_flash)) diff --git a/kml.py b/kml.py new file mode 100644 index 0000000..df72e0a --- /dev/null +++ b/kml.py @@ -0,0 +1,72 @@ +import fastkml +from pygeoif.geometry import Point + + +def _compute_color(invader): + """ + Compute KML color based on invader points and status. + """ + if invader['status_description'] == 'OK': + if int(invader['points']) == 100: + return 'purple' + elif int(invader['points']) == 50: + return 'pink' + elif invader['status_description'] in ['Détruit !', 'Non visible']: + return 'red' + elif invader['status_description'] in ['Dégradé', 'Très dégradé']: + return 'brown' + return 'yellow' + + +def generate_kml( + invaders, + out_file, + name, +): + """ + Generate a GPX file from all known invaders with their locations. + """ + kml = fastkml.kml.KML() + + ns = '{http://www.opengis.net/kml/2.2}' + d = fastkml.kml.Document( + ns, + name=name, + styles=[ + fastkml.styles.Style( + ns, + 'placemark-%s' % color, + styles=[ + fastkml.styles.IconStyle( + ns, + icon_href=( + 'https://omaps.app/placemarks/placemark-%s.png' % + color + ) + ) + ] + ) + for color in ['pink', 'purple', 'red', 'brown', 'yellow'] + ] + ) + kml.append(d) + + for invader in invaders: + if not invader['lat'] and not invader['lon']: + # No GPS coordinates + continue + name = invader['name'] + description = ( + 'status=%s ; points=%s ; image="%s" ; desc="%s"' % + (invader['status_description'], invader['points'], + invader['picture_url'], invader['description']) + ) + p = fastkml.kml.Placemark( + ns, name=name, description=description, + styleUrl=('#placemark-%s' % _compute_color(invader)) + ) + p.geometry = Point(invader['lon'], invader['lat']) + d.append(p) + + with open(out_file, 'w') as fh: + fh.write(kml.to_string(prettyprint=True)) diff --git a/scraper.py b/scraper.py index 738566f..1f46abf 100644 --- a/scraper.py +++ b/scraper.py @@ -5,14 +5,15 @@ """ import collections import csv +import os import re from typing import TypedDict, List -import fastkml import requests from bs4 import BeautifulSoup, ResultSet from prettytable import PrettyTable, MARKDOWN -from pygeoif.geometry import Point + +from kml import generate_kml DOMAIN = 'https://www.invader-spotter.art/' URL = DOMAIN + 'listing.php' @@ -39,6 +40,20 @@ class Invader(TypedDict): picture_url: str +def _confirm(dialog): + """ + Ask user to enter Y or N (case-insensitive). + :return: True if the answer is Y. + :rtype: bool + + Source:https://gist.github.com/gurunars/4470c97c916e7b3c4731469c69671d06 + """ + answer = "" + while answer not in ["y", "n"]: + answer = input("%s [Y/N]? " % dialog).lower() + return answer == "y" + + def scrape(url=URL, out_file='data/invaders-dump.csv'): """ Scrape all invaders pages from invader-spotter.art. @@ -155,85 +170,37 @@ def add_locations( writer.writerows(invaders) -def _compute_color(invader): - """ - Compute KML color based on invader points and status. - """ - if invader['status_description'] == 'OK': - if int(invader['points']) == 100: - return 'pink' - elif int(invader['points']) == 50: - return 'purple' - elif invader['status_description'].startswith('Détruit'): - return 'red' - elif invader['status_description'] == 'Dégradé': - return 'brown' - return 'yellow' - - -def generate_kml( - in_file='data/invaders-with-locations.csv', - out_file='data/all-invaders.kml' -): - """ - Generate a GPX file from all known invaders with their locations. - """ - kml = fastkml.kml.KML() +if __name__ == '__main__': + # Scrape data from invader-spotter.art + force_rescrape = 'yes' + if os.path.isfile('data/invaders-dump.csv'): + force_rescrape = _confirm('Force rescrape') - ns = '{http://www.opengis.net/kml/2.2}' - d = fastkml.kml.Document( - ns, - name='All invaders', - styles=[ - fastkml.styles.Style( - ns, - 'placemark-%s' % color, - styles=[ - fastkml.styles.IconStyle( - ns, - icon_href=( - 'https://omaps.app/placemarks/placemark-%s.png' % - color - ) - ) - ] - ) - for color in ['pink', 'purple', 'red', 'brown', 'yellow'] - ] - ) - kml.append(d) + if force_rescrape: + scrape() - missing_coordinates_by_status = collections.defaultdict(list) + # Fuse with locations information + add_locations() - with open(in_file, 'r') as fh: + # Generate a KML of all the available invaders + with open('data/invaders-with-locations.csv', 'r') as fh: reader = csv.DictReader(fh) invaders = list(reader) + generate_kml( + invaders=invaders, + out_file='data/all-invaders.kml', + name='All invaders', + ) + # Debug stats + missing_coordinates_by_status = collections.defaultdict(list) for invader in invaders: - if not invader['lat'] and not invader['lon']: - # No GPS coordinates - status = invader['status_description'] - missing_coordinates_by_status[status].append( - invader['name'] - ) + if invader['lat'] or invader['lon']: continue - name = invader['name'] - description = ( - 'status=%s ; points=%s ; image="%s" ; desc="%s"' % - (invader['status_description'], invader['points'], - invader['picture_url'], invader['description']) + status = invader['status_description'] + missing_coordinates_by_status[status].append( + invader['name'] ) - p = fastkml.kml.Placemark( - ns, name=name, description=description, - styleUrl=('#placemark-%s' % _compute_color(invader)) - ) - p.geometry = Point(invader['lon'], invader['lat']) - d.append(p) - - with open(out_file, 'w') as fh: - fh.write(kml.to_string(prettyprint=True)) - - # Debug table = PrettyTable() table.field_names = ["Status", "# of missing locations"] table.align["Status"] = "l" @@ -241,9 +208,3 @@ def generate_kml( for status, items in missing_coordinates_by_status.items(): table.add_row([status, len(items)]) print(table.get_string(sortby="# of missing locations", reversesort=True)) - - -if __name__ == '__main__': - # scrape() - add_locations() - generate_kml()