From 70ed4feda82876945a4edaad95461ea6649bb4c1 Mon Sep 17 00:00:00 2001 From: Martin Mader Date: Fri, 9 Feb 2024 16:41:42 +0100 Subject: [PATCH 1/6] feat: extend available country code list to most of europe --- README.md | 7 +- charging_stations_pipelines/__init__.py | 72 +++++++++++++++++++ .../deduplication/merger.py | 11 +-- .../pipelines/at/__init__.py | 3 - .../pipelines/at/econtrol.py | 24 ++++--- .../pipelines/nobil/nobil_pipeline.py | 7 +- .../pipelines/ocm/ocm.py | 4 +- .../pipelines/ocm/ocm_extractor.py | 29 ++++---- .../pipelines/osm/__init__.py | 15 ---- .../pipelines/osm/osm.py | 36 +++++----- .../pipelines/osm/osm_receiver.py | 10 +-- .../pipelines/pipeline_factory.py | 4 +- list-countries.py | 5 ++ main.py | 40 ++++++++--- 14 files changed, 171 insertions(+), 96 deletions(-) create mode 100644 charging_stations_pipelines/__init__.py create mode 100644 list-countries.py diff --git a/README.md b/README.md index 44158f3..0a38042 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ All steps are decoupled, so it's easy to integrate your own data source. The primary data sources for electronic vehicle charging stations that are generally available across Europe are Open Street Map (OSM) and Open Charge Map (OCM). -For each country, we then integrate the official government data source (where available). +For some countries, we are able to then integrate the official government data source. Just to name a few examples, the government data source we use for Germany is Bundesnetzagentur (BNA), or the National Chargepoint Registry (NCR) for the UK. @@ -148,6 +148,8 @@ Then run migration eCharm can be run similar to a command line tool. Run `python main.py -h` to see the full list of command line options. +Run `python list-countries.py` to see a list of supported countries +together with information about the availability of a governmental data source, and availability in OSM and OCM. Here are a few example commands for running tasks: @@ -169,7 +171,8 @@ tasks, since eCharm is not (yet) clever with updating data from consecutive impo python main.py import merge --countries de it --delete_data ``` -Currently, we support `at`, `de`,`gb`,`fr`, `it`, `nor` and `swe` as country codes. +Currently, we support the [ISO 3166-1 alpha 2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) +country codes of most European countries. #### Export all original (un-merged) station data for Germany in csv format: diff --git a/charging_stations_pipelines/__init__.py b/charging_stations_pipelines/__init__.py new file mode 100644 index 0000000..ee6e5f7 --- /dev/null +++ b/charging_stations_pipelines/__init__.py @@ -0,0 +1,72 @@ +from pathlib import Path +from typing import Final + + +class CountryInfo: + def __init__(self, name: str, gov: bool, osm: bool, ocm: bool): + self.name = name + self.gov = gov + self.osm = osm + self.ocm = ocm + + +COUNTRIES: Final[dict[str, CountryInfo]] = dict( + [ + ("AD", CountryInfo(name="Andorra", gov=False, osm=True, ocm=True)), + ("AL", CountryInfo(name="Albania", gov=False, osm=True, ocm=True)), + ("AT", CountryInfo(name="Austria", gov=True, osm=True, ocm=True)), + ("BA", CountryInfo(name="Bosnia and Herzegovina", gov=False, osm=True, ocm=True)), + ("BE", CountryInfo(name="Belgium", gov=False, osm=True, ocm=True)), + ("BG", CountryInfo(name="Bulgaria", gov=False, osm=True, ocm=True)), + ("BY", CountryInfo(name="Belarus", gov=False, osm=True, ocm=True)), + ("CH", CountryInfo(name="Switzerland", gov=False, osm=True, ocm=True)), + ("CY", CountryInfo(name="Cyprus", gov=False, osm=True, ocm=True)), + ("CZ", CountryInfo(name="Czech Republic", gov=False, osm=True, ocm=True)), + ("DE", CountryInfo(name="Germany", gov=True, osm=True, ocm=True)), + ("DK", CountryInfo(name="Denmark", gov=False, osm=True, ocm=True)), + ("EE", CountryInfo(name="Estonia", gov=False, osm=True, ocm=True)), + ("ES", CountryInfo(name="Spain", gov=False, osm=True, ocm=True)), + ("FI", CountryInfo(name="Finland", gov=False, osm=True, ocm=True)), + ("FR", CountryInfo(name="France", gov=False, osm=True, ocm=True)), + ("GB", CountryInfo(name="United Kingdom", gov=True, osm=True, ocm=True)), + ("GR", CountryInfo(name="Greece", gov=False, osm=True, ocm=True)), + ("HR", CountryInfo(name="Croatia", gov=False, osm=True, ocm=True)), + ("HU", CountryInfo(name="Hungary", gov=False, osm=True, ocm=True)), + ("IE", CountryInfo(name="Ireland", gov=False, osm=True, ocm=True)), + ("IS", CountryInfo(name="Iceland", gov=False, osm=True, ocm=True)), + ("IT", CountryInfo(name="Italy", gov=False, osm=True, ocm=True)), + ("LI", CountryInfo(name="Liechtenstein", gov=False, osm=True, ocm=True)), + ("LT", CountryInfo(name="Lithuania", gov=False, osm=True, ocm=True)), + ("LU", CountryInfo(name="Luxembourg", gov=False, osm=True, ocm=True)), + ("LV", CountryInfo(name="Latvia", gov=False, osm=True, ocm=True)), + ("MC", CountryInfo(name="Monaco", gov=False, osm=True, ocm=True)), + ("MD", CountryInfo(name="Moldova", gov=False, osm=True, ocm=True)), + ("ME", CountryInfo(name="Montenegro", gov=False, osm=True, ocm=True)), + ("MK", CountryInfo(name="North Macedonia", gov=False, osm=True, ocm=True)), + ("MT", CountryInfo(name="Malta", gov=False, osm=True, ocm=True)), + ("NL", CountryInfo(name="Netherlands", gov=False, osm=True, ocm=True)), + ("NO", CountryInfo(name="Norway", gov=True, osm=True, ocm=True)), + ("PL", CountryInfo(name="Poland", gov=False, osm=True, ocm=True)), + ("PT", CountryInfo(name="Portugal", gov=False, osm=True, ocm=True)), + ("RO", CountryInfo(name="Romania", gov=False, osm=True, ocm=True)), + ("RS", CountryInfo(name="Serbia", gov=False, osm=True, ocm=True)), + ("SE", CountryInfo(name="Sweden", gov=True, osm=True, ocm=True)), + ("SI", CountryInfo(name="Slovenia", gov=False, osm=True, ocm=True)), + ("SK", CountryInfo(name="Slovakia", gov=False, osm=True, ocm=True)), + ("SM", CountryInfo(name="San Marino", gov=False, osm=True, ocm=True)), + ("UA", CountryInfo(name="Ukraine", gov=False, osm=True, ocm=True)), + ("VA", CountryInfo(name="Vatican City", gov=False, osm=True, ocm=False)), + ("XK", CountryInfo(name="Kosovo", gov=False, osm=True, ocm=True)), + ] +) + +COUNTRY_CODES: list[str] = list(COUNTRIES.keys()) +OSM_COUNTRY_CODES: list[str] = list({k: v for k, v in COUNTRIES.items() if v.osm}.keys()) +OCM_COUNTRY_CODES: list[str] = list({k: v for k, v in COUNTRIES.items() if v.ocm}.keys()) +GOV_COUNTRY_CODES: list[str] = list({k: v for k, v in COUNTRIES.items() if v.gov}.keys()) + +PROJ_ROOT: Final[Path] = Path(__file__).parents[1] +"""The root directory of the project.""" + +PROJ_DATA_DIR: Final[Path] = PROJ_ROOT / "data" +"""The path to the data folder.""" diff --git a/charging_stations_pipelines/deduplication/merger.py b/charging_stations_pipelines/deduplication/merger.py index d4949ad..2c2d5e1 100644 --- a/charging_stations_pipelines/deduplication/merger.py +++ b/charging_stations_pipelines/deduplication/merger.py @@ -32,13 +32,14 @@ def __init__(self, country_code: str, config: configparser, db_engine, is_test: "AT": "AT_ECONTROL", "FR": "FRGOV", "GB": "GBGOV", - "IT": "", # No gov source for Italy so far (5.4.2023) - "NOR": "NOBIL", - "SWE": "NOBIL", + "NO": "NOBIL", + "SE": "NOBIL", } if country_code not in country_code_to_gov_source: - raise Exception(f"country code '{country_code}' unknown in merger") - self.gov_source = country_code_to_gov_source[country_code] + logger.info(f"No governmental data source available for country code '{country_code}'") + self.gov_source = "" + else: + self.gov_source = country_code_to_gov_source[country_code] @staticmethod def merge_attributes(station: pd.Series, duplicates_to_merge: pd.DataFrame): diff --git a/charging_stations_pipelines/pipelines/at/__init__.py b/charging_stations_pipelines/pipelines/at/__init__.py index 5f687af..da805c0 100644 --- a/charging_stations_pipelines/pipelines/at/__init__.py +++ b/charging_stations_pipelines/pipelines/at/__init__.py @@ -3,6 +3,3 @@ DATA_SOURCE_KEY: Final[str] = "AT_ECONTROL" """The data source key for the e-control data source.""" - -SCOPE_COUNTRIES: Final[list[str]] = ["AT"] -"""The list of country codes covered by the e-control data source.""" diff --git a/charging_stations_pipelines/pipelines/at/econtrol.py b/charging_stations_pipelines/pipelines/at/econtrol.py index dd5ac58..86b8663 100644 --- a/charging_stations_pipelines/pipelines/at/econtrol.py +++ b/charging_stations_pipelines/pipelines/at/econtrol.py @@ -15,7 +15,7 @@ from tqdm import tqdm from charging_stations_pipelines.pipelines import Pipeline -from charging_stations_pipelines.pipelines.at import DATA_SOURCE_KEY, SCOPE_COUNTRIES +from charging_stations_pipelines.pipelines.at import DATA_SOURCE_KEY from charging_stations_pipelines.pipelines.at.econtrol_crawler import get_data from charging_stations_pipelines.pipelines.at.econtrol_mapper import ( map_address, @@ -78,19 +78,21 @@ def run(self): try: station = map_station(datapoint, self.country_code) - # Count stations with country codes that are not in the scope of the pipeline - if station.country_code not in SCOPE_COUNTRIES: - stats["count_country_mismatch_stations"] += 1 - # Address mapping station.address = map_address(datapoint, self.country_code, None) - # Count stations which have an invalid address - if station.address and station.address.country and station.address.country not in SCOPE_COUNTRIES: - stats["count_country_mismatch_stations"] += 1 - - # Count stations which have a mismatching country code between Station and Address - if station.country_code != station.address.country: + # Count stations that have some kind of country code mismatch + if ( + # Count stations which have an invalid country code in address + (station.address and station.address.country and station.address.country != "AT") + # Count stations which have a mismatching country code between Station and Address + or ( + station.country_code is not None + and station.address is not None + and station.address.country is not None + and station.country_code != station.address.country + ) + ): stats["count_country_mismatch_stations"] += 1 # Charging point diff --git a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py index 5f46520..0dd0d02 100644 --- a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py +++ b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py @@ -137,10 +137,11 @@ def _map_charging_to_domain(nobil_station: NobilStation) -> Charging: def _load_datadump_and_write_to_target(path_to_target, country_code: str): + nobil_api_country_code = "NOR" if country_code == "NO" else "SWE" nobil_api_key = os.getenv("NOBIL_APIKEY") link_to_datadump = ( f"https://nobil.no/api/server/datadump.php?apikey=" - f"{nobil_api_key}&countrycode={country_code}&format=json&file=true" + f"{nobil_api_key}&countrycode={nobil_api_country_code}&format=json&file=true" ) download_file(link_to_datadump, path_to_target) @@ -157,13 +158,13 @@ def __init__( ): super().__init__(config, session, online) - accepted_country_codes = ["NOR", "SWE"] + accepted_country_codes = ["NO", "SE"] reject_if(country_code.upper() not in accepted_country_codes, "Invalid country code ") self.country_code = country_code.upper() def run(self): """Run the pipeline.""" - logger.info("Running NOR/SWE GOV Pipeline...") + logger.info(f"Running {self.country_code} GOV Pipeline...") path_to_target = Path(__file__).parent.parent.parent.parent.joinpath("data/" + self.country_code + "_gov.json") if self.online: logger.info("Retrieving Online Data") diff --git a/charging_stations_pipelines/pipelines/ocm/ocm.py b/charging_stations_pipelines/pipelines/ocm/ocm.py index 5d70dfe..d38d256 100644 --- a/charging_stations_pipelines/pipelines/ocm/ocm.py +++ b/charging_stations_pipelines/pipelines/ocm/ocm.py @@ -40,7 +40,9 @@ def __init__( def _retrieve_data(self): data_dir: str = os.path.join(pathlib.Path(__file__).parent.resolve(), "../../..", "data") pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True) - tmp_file_path = os.path.join(data_dir, self.config["OCM"]["filename"]) + country_dir = os.path.join(data_dir, self.country_code) + pathlib.Path(country_dir).mkdir(parents=True, exist_ok=True) + tmp_file_path = os.path.join(country_dir, self.config["OCM"]["filename"]) if self.online: logger.info("Retrieving Online Data") ocm_extractor(tmp_file_path, self.country_code) diff --git a/charging_stations_pipelines/pipelines/ocm/ocm_extractor.py b/charging_stations_pipelines/pipelines/ocm/ocm_extractor.py index bdf4219..0f49ddb 100644 --- a/charging_stations_pipelines/pipelines/ocm/ocm_extractor.py +++ b/charging_stations_pipelines/pipelines/ocm/ocm_extractor.py @@ -1,7 +1,7 @@ import json import logging import os -import pathlib +from pathlib import Path import re import shutil import subprocess @@ -10,6 +10,8 @@ import pandas as pd from packaging import version +from charging_stations_pipelines import PROJ_DATA_DIR + logger = logging.getLogger(__name__) @@ -56,17 +58,12 @@ def merge_connections(row, connection_types): return pd.merge(frame, connection_types, how="left", left_on="ConnectionTypeID", right_on="ID") -def ocm_extractor(tmp_file_path: str, country_code: str): +def ocm_extractor(extracted_data_file_path: str, country_code: str): """This method extracts Open Charge Map (OCM) data for a given country and saves it to a specified file.""" - # OCM export contains norwegian data under country code "NO" and that's why we need to rename it to "NO" - if country_code == "NOR": - country_code = "NO" - if country_code == "SWE": - country_code = "SE" - project_data_dir: str = pathlib.Path(tmp_file_path).parent.resolve().name - data_root_dir: str = os.path.join(project_data_dir, "ocm-export") - data_dir: str = os.path.join(data_root_dir, f"data/{country_code}") + project_data_dir: Path = PROJ_DATA_DIR + ocm_source_dir: Path = project_data_dir / "ocm-export" + data_dir: Path = ocm_source_dir / "data" / country_code try: git_version_raw: str = subprocess.check_output(["git", "--version"]).decode() @@ -86,7 +83,7 @@ def ocm_extractor(tmp_file_path: str, country_code: str): raise RuntimeError("Git version must be >= 2.25.0!") if (not os.path.isdir(data_dir)) or len(os.listdir(data_dir)) == 0: - shutil.rmtree(data_root_dir, ignore_errors=True) + shutil.rmtree(ocm_source_dir, ignore_errors=True) subprocess.call( [ "git", @@ -101,17 +98,17 @@ def ocm_extractor(tmp_file_path: str, country_code: str): ) subprocess.call( ["git", "sparse-checkout", "init", "--cone"], - cwd=data_root_dir, + cwd=ocm_source_dir, stdout=subprocess.PIPE, ) subprocess.call( ["git", "sparse-checkout", "set", f"data/{country_code}"], - cwd=data_root_dir, + cwd=ocm_source_dir, stdout=subprocess.PIPE, ) - subprocess.call(["git", "checkout"], cwd=data_root_dir, stdout=subprocess.PIPE) + subprocess.call(["git", "checkout"], cwd=ocm_source_dir, stdout=subprocess.PIPE) else: - subprocess.call(["git", "pull"], cwd=data_root_dir, stdout=subprocess.PIPE) + subprocess.call(["git", "pull"], cwd=ocm_source_dir, stdout=subprocess.PIPE) records: List = [] for subdir, dirs, files in os.walk(os.path.join(data_dir)): @@ -162,4 +159,4 @@ def ocm_extractor(tmp_file_path: str, country_code: str): how="left", ) - pd_merged_with_operators.reset_index(drop=True).to_json(tmp_file_path, orient="index") + pd_merged_with_operators.reset_index(drop=True).to_json(extracted_data_file_path, orient="index") diff --git a/charging_stations_pipelines/pipelines/osm/__init__.py b/charging_stations_pipelines/pipelines/osm/__init__.py index fd551f5..25048f5 100644 --- a/charging_stations_pipelines/pipelines/osm/__init__.py +++ b/charging_stations_pipelines/pipelines/osm/__init__.py @@ -1,21 +1,6 @@ """The OpenStreetMap (OSM) data source pipeline.""" from typing import Final -from typing_extensions import KeysView DATA_SOURCE_KEY: Final[str] = "OSM" """The data source key for the OpenStreetMap (OSM) data source.""" - -COUNTRY_CODE_TO_AREA_MAP: Final[dict[str, str]] = { - "DE": "Deutschland", - "AT": "Österreich", - "FR": "France métropolitaine", - "GB": "United Kingdom", - "IT": "Italia", - "NOR": "Norge", - "SWE": "Sverige", -} -"""The mapping of country codes to the corresponding area names for the OSM API.""" - -SCOPE_COUNTRIES: Final[KeysView[str]] = COUNTRY_CODE_TO_AREA_MAP.keys() -"""The list of country codes covered by the AT data source.""" diff --git a/charging_stations_pipelines/pipelines/osm/osm.py b/charging_stations_pipelines/pipelines/osm/osm.py index d34d192..7a4e155 100644 --- a/charging_stations_pipelines/pipelines/osm/osm.py +++ b/charging_stations_pipelines/pipelines/osm/osm.py @@ -3,17 +3,16 @@ import configparser import json import logging -import os -import pathlib +from pathlib import Path from sqlalchemy.orm import Session from tqdm import tqdm +from charging_stations_pipelines import PROJ_DATA_DIR, OSM_COUNTRY_CODES from charging_stations_pipelines.pipelines import Pipeline from charging_stations_pipelines.pipelines.osm import ( DATA_SOURCE_KEY, osm_mapper, - SCOPE_COUNTRIES, ) from charging_stations_pipelines.pipelines.osm.osm_receiver import get_osm_data from charging_stations_pipelines.pipelines.station_table_updater import ( @@ -39,9 +38,11 @@ def __init__( self.country_code = country_code def retrieve_data(self): - data_dir: str = os.path.join(pathlib.Path(__file__).parent.resolve(), "../../..", "data") - pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True) - tmp_file_path = os.path.join(data_dir, self.config[DATA_SOURCE_KEY]["filename"]) + data_dir: Path = PROJ_DATA_DIR + data_dir.mkdir(parents=True, exist_ok=True) + country_dir = data_dir / self.country_code + country_dir.mkdir(parents=True, exist_ok=True) + tmp_file_path = country_dir / self.config[DATA_SOURCE_KEY]["filename"] if self.online: logger.info("Retrieving Online Data") get_osm_data(self.country_code, tmp_file_path) @@ -60,23 +61,20 @@ def run(self): try: station = osm_mapper.map_station_osm(entry, self.country_code) - # Count stations with country codes that are not in the scope of the pipeline - if station.country_code not in SCOPE_COUNTRIES: - stats["count_country_mismatch_stations"] += 1 - # Address mapping station.address = osm_mapper.map_address_osm(entry, None) - # Count stations which have an invalid address - if station.address and station.address.country and station.address.country not in SCOPE_COUNTRIES: - stats["count_country_mismatch_stations"] += 1 - - # Count stations which have a mismatching country code between Station and Address + # Count stations that have some kind of country code mismatch if ( - station.country_code is not None - and station.address is not None - and station.address.country is not None - and station.country_code != station.address.country + # Count stations which have an invalid country code in address + (station.address and station.address.country and station.address.country not in OSM_COUNTRY_CODES) + # Count stations which have a mismatching country code between Station and Address + or ( + station.country_code is not None + and station.address is not None + and station.address.country is not None + and station.country_code != station.address.country + ) ): stats["count_country_mismatch_stations"] += 1 diff --git a/charging_stations_pipelines/pipelines/osm/osm_receiver.py b/charging_stations_pipelines/pipelines/osm/osm_receiver.py index a7866a3..6352220 100644 --- a/charging_stations_pipelines/pipelines/osm/osm_receiver.py +++ b/charging_stations_pipelines/pipelines/osm/osm_receiver.py @@ -6,7 +6,6 @@ from requests import Response from charging_stations_pipelines.pipelines.osm import ( - COUNTRY_CODE_TO_AREA_MAP, DATA_SOURCE_KEY, ) @@ -15,8 +14,7 @@ def get_osm_data(country_code: str, tmp_data_path): """This method retrieves OpenStreetMap (OSM) data for a specific country based on its country code. The OSM data includes information about charging stations in the specified country. - The `country_code` parameter is a string representing the two-letter country code. Valid country codes are "DE" - for Germany, "FR" for France, "GB" for the United Kingdom, "IT" for Italy, "NOR" for Norway, and "SWE" for Sweden. + The `country_code` parameter is a string representing the ISO3166-1 alpha-2 country code. The `tmp_data_path parameter` is a string representing the path to save the downloaded OSM data. The OSM data will be saved in JSON format. @@ -34,16 +32,12 @@ def get_osm_data(country_code: str, tmp_data_path): :type tmp_data_path: str :return: None """ - if country_code not in COUNTRY_CODE_TO_AREA_MAP: - raise Exception(f"country code '{country_code}' unknown for {DATA_SOURCE_KEY}") - - area_name = COUNTRY_CODE_TO_AREA_MAP[country_code] query_params = { "data": f""" [out:json]; - area[name="{area_name}"]; + area["ISO3166-1"="{country_code}"][admin_level=2]; // gather results ( diff --git a/charging_stations_pipelines/pipelines/pipeline_factory.py b/charging_stations_pipelines/pipelines/pipeline_factory.py index 041cda2..0530ba5 100644 --- a/charging_stations_pipelines/pipelines/pipeline_factory.py +++ b/charging_stations_pipelines/pipelines/pipeline_factory.py @@ -29,8 +29,8 @@ def pipeline_factory(db_session: Session, country="DE", online=True) -> Pipeline "DE": BnaPipeline(config, db_session, online), "FR": FraPipeline(config, db_session, online), "GB": GbPipeline(config, db_session, online), - "NOR": NobilPipeline(db_session, "NOR", online), - "SWE": NobilPipeline(db_session, "SWE", online), + "NO": NobilPipeline(db_session, "NO", online), + "SE": NobilPipeline(db_session, "SE", online), } return pipelines[country] if country in pipelines else EmptyPipeline() diff --git a/list-countries.py b/list-countries.py new file mode 100644 index 0000000..05d03e0 --- /dev/null +++ b/list-countries.py @@ -0,0 +1,5 @@ +import json + +from charging_stations_pipelines import COUNTRIES + +print(json.dumps(COUNTRIES, default=vars, indent=4)) diff --git a/main.py b/main.py index 87e1e40..7bf0a77 100644 --- a/main.py +++ b/main.py @@ -7,7 +7,14 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from charging_stations_pipelines import db_utils, settings +from charging_stations_pipelines import ( + db_utils, + settings, + COUNTRY_CODES, + OCM_COUNTRY_CODES, + OSM_COUNTRY_CODES, + GOV_COUNTRY_CODES, +) from charging_stations_pipelines.deduplication.merger import StationMerger from charging_stations_pipelines.pipelines.ocm.ocm import OcmPipeline from charging_stations_pipelines.pipelines.osm.osm import OsmPipeline @@ -26,7 +33,7 @@ def parse_args(args): """This method is used to parse the command line arguments for the eCharm program.""" valid_task_options = ["import", "merge", "export", "testdata"] - valid_country_options = ["DE", "AT", "FR", "GB", "IT", "NOR", "SWE"] + valid_country_options = COUNTRY_CODES valid_export_format_options = ["csv", "GeoJSON"] parser = argparse.ArgumentParser( @@ -56,7 +63,9 @@ def parse_args(args): nargs="+", type=lambda s: s.upper(), metavar="", - help="specifies the countries for which to perform the given tasks. " + help="specifies the countries for which to perform the given tasks, " + "given as ISO 3166-1 alpha-2 code, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2. " + "Run 'python list-countries.py' for more information. " "The country-codes must be one or several of %(choices)s (case-insensitive). " "If not specified, the given tasks are run for all available countries", ) @@ -145,14 +154,23 @@ def run_import(countries: list[str], online: bool, delete_data: bool): logger.info(f"Importing data for country: {country}...") db_session = sessionmaker(bind=get_db_engine())() - gov_pipeline = pipeline_factory(db_session, country, online) - gov_pipeline.run() - - osm = OsmPipeline(country, config, db_session, online) - osm.run() - - ocm = OcmPipeline(country, config, db_session, online) - ocm.run() + if country not in GOV_COUNTRY_CODES: + logger.info(f"no governmental data available for country code '{country}'... skipping GOV pipeline") + else: + gov_pipeline = pipeline_factory(db_session, country, online) + gov_pipeline.run() + + if country not in OSM_COUNTRY_CODES: + logger.info(f"country code '{country}' unknown for OSM... skipping OSM pipeline") + else: + osm = OsmPipeline(country, config, db_session, online) + osm.run() + + if country not in OCM_COUNTRY_CODES: + logger.info(f"country code '{country}' unknown for OCM... skipping OCM pipeline") + else: + ocm = OcmPipeline(country, config, db_session, online) + ocm.run() logger.info("Finished importing data.") From 772c67032dc271f71209b264acdccd4d53e553a2 Mon Sep 17 00:00:00 2001 From: Martin Mader Date: Fri, 16 Feb 2024 10:32:40 +0100 Subject: [PATCH 2/6] refactor: nobil pipeline online property and data path handling --- .../pipelines/nobil/nobil_pipeline.py | 9 +++++++-- .../pipelines/pipeline_factory.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py index 0dd0d02..2d242ca 100644 --- a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py +++ b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session from tqdm import tqdm +from charging_stations_pipelines import PROJ_DATA_DIR from charging_stations_pipelines.models.address import Address from charging_stations_pipelines.models.charging import Charging from charging_stations_pipelines.models.station import Station @@ -151,9 +152,9 @@ class NobilPipeline(Pipeline): def __init__( self, + config: configparser, session: Session, country_code: str, - config: configparser, online: bool = False, ): super().__init__(config, session, online) @@ -165,7 +166,11 @@ def __init__( def run(self): """Run the pipeline.""" logger.info(f"Running {self.country_code} GOV Pipeline...") - path_to_target = Path(__file__).parent.parent.parent.parent.joinpath("data/" + self.country_code + "_gov.json") + data_dir: Path = PROJ_DATA_DIR + data_dir.mkdir(parents=True, exist_ok=True) + country_dir = data_dir / self.country_code + country_dir.mkdir(parents=True, exist_ok=True) + path_to_target = country_dir / "nobil.json" if self.online: logger.info("Retrieving Online Data") _load_datadump_and_write_to_target(path_to_target, self.country_code) diff --git a/charging_stations_pipelines/pipelines/pipeline_factory.py b/charging_stations_pipelines/pipelines/pipeline_factory.py index 0530ba5..1752460 100644 --- a/charging_stations_pipelines/pipelines/pipeline_factory.py +++ b/charging_stations_pipelines/pipelines/pipeline_factory.py @@ -29,8 +29,8 @@ def pipeline_factory(db_session: Session, country="DE", online=True) -> Pipeline "DE": BnaPipeline(config, db_session, online), "FR": FraPipeline(config, db_session, online), "GB": GbPipeline(config, db_session, online), - "NO": NobilPipeline(db_session, "NO", online), - "SE": NobilPipeline(db_session, "SE", online), + "NO": NobilPipeline(config, db_session, "NO", online), + "SE": NobilPipeline(config, db_session, "SE", online), } return pipelines[country] if country in pipelines else EmptyPipeline() From ac17f2172b038d664604e6395b5ebd0ce7f8bf9c Mon Sep 17 00:00:00 2001 From: Martin Mader Date: Fri, 16 Feb 2024 11:47:32 +0100 Subject: [PATCH 3/6] refactor: improve path handling for import tasks --- .../pipelines/at/econtrol.py | 10 ++-------- .../pipelines/de/bna.py | 9 ++------- .../pipelines/fr/france.py | 8 ++------ .../pipelines/gb/gbgov.py | 10 +++------- .../pipelines/nobil/nobil_pipeline.py | 10 ++-------- .../pipelines/ocm/ocm.py | 10 ++-------- .../pipelines/osm/osm.py | 11 +++-------- charging_stations_pipelines/shared.py | 18 +++++++++++++----- 8 files changed, 29 insertions(+), 57 deletions(-) diff --git a/charging_stations_pipelines/pipelines/at/econtrol.py b/charging_stations_pipelines/pipelines/at/econtrol.py index 86b8663..fc47f1e 100644 --- a/charging_stations_pipelines/pipelines/at/econtrol.py +++ b/charging_stations_pipelines/pipelines/at/econtrol.py @@ -7,8 +7,6 @@ import collections import configparser import logging -import os -import pathlib import pandas as pd from sqlalchemy.orm import Session @@ -25,6 +23,7 @@ from charging_stations_pipelines.pipelines.station_table_updater import ( StationTableUpdater, ) +from charging_stations_pipelines.shared import country_import_data_path logger = logging.getLogger(__name__) @@ -40,7 +39,6 @@ class EcontrolAtPipeline(Pipeline): :ivar config: A `configparser` object containing configurations for the pipeline. :ivar session: A `Session` object representing the session used for database operations. :ivar online: A boolean indicating whether the pipeline should retrieve data online. - :ivar data_dir: A string representing the directory where data files will be stored. """ def __init__(self, config: configparser, session: Session, online: bool = False): @@ -49,12 +47,8 @@ def __init__(self, config: configparser, session: Session, online: bool = False) # Is always 'AT' for this pipeline self.country_code = "AT" - relative_dir = os.path.join("../../..", "data") - self.data_dir = os.path.join(pathlib.Path(__file__).parent.resolve(), relative_dir) - def _retrieve_data(self): - pathlib.Path(self.data_dir).mkdir(parents=True, exist_ok=True) - tmp_data_path = os.path.join(self.data_dir, self.config[DATA_SOURCE_KEY]["filename"]) + tmp_data_path = country_import_data_path(self.country_code) / self.config[DATA_SOURCE_KEY]["filename"] if self.online: logger.info("Retrieving Online Data") get_data(tmp_data_path) diff --git a/charging_stations_pipelines/pipelines/de/bna.py b/charging_stations_pipelines/pipelines/de/bna.py index 3a336de..0db49ba 100644 --- a/charging_stations_pipelines/pipelines/de/bna.py +++ b/charging_stations_pipelines/pipelines/de/bna.py @@ -2,8 +2,6 @@ import configparser import logging -import pathlib -from typing import Final import pandas as pd from sqlalchemy.orm import Session @@ -18,7 +16,7 @@ ) from ...pipelines import Pipeline from ...pipelines.station_table_updater import StationTableUpdater -from ...shared import load_excel_file +from ...shared import load_excel_file, country_import_data_path logger = logging.getLogger(__name__) @@ -30,11 +28,8 @@ def __init__(self, config: configparser, session: Session, online: bool = False) # All BNA data is from Germany self.country_code = "DE" - self.data_dir: Final[pathlib.Path] = (pathlib.Path(__file__).parents[3] / "data").resolve() - def retrieve_data(self): - self.data_dir.mkdir(parents=True, exist_ok=True) - tmp_data_path = self.data_dir / self.config[DATA_SOURCE_KEY]["filename"] + tmp_data_path = country_import_data_path(self.country_code) / self.config[DATA_SOURCE_KEY]["filename"] if self.online: logger.info("Retrieving Online Data") diff --git a/charging_stations_pipelines/pipelines/fr/france.py b/charging_stations_pipelines/pipelines/fr/france.py index f2fd8fe..e4a36c4 100644 --- a/charging_stations_pipelines/pipelines/fr/france.py +++ b/charging_stations_pipelines/pipelines/fr/france.py @@ -1,8 +1,6 @@ """Pipeline for retrieving data from the French government website.""" import logging -import os -import pathlib import pandas as pd import requests as requests @@ -18,16 +16,14 @@ from charging_stations_pipelines.pipelines.station_table_updater import ( StationTableUpdater, ) -from charging_stations_pipelines.shared import download_file, reject_if +from charging_stations_pipelines.shared import download_file, reject_if, country_import_data_path logger = logging.getLogger(__name__) class FraPipeline(Pipeline): def _retrieve_data(self): - data_dir = os.path.join(pathlib.Path(__file__).parent.resolve(), "../../..", "data") - pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True) - tmp_data_path = os.path.join(data_dir, self.config["FRGOV"]["filename"]) + tmp_data_path = country_import_data_path("FR") / self.config["FRGOV"]["filename"] if self.online: logger.info("Retrieving Online Data") self.download_france_gov_file(tmp_data_path) diff --git a/charging_stations_pipelines/pipelines/gb/gbgov.py b/charging_stations_pipelines/pipelines/gb/gbgov.py index 3e252fd..824f720 100644 --- a/charging_stations_pipelines/pipelines/gb/gbgov.py +++ b/charging_stations_pipelines/pipelines/gb/gbgov.py @@ -3,8 +3,6 @@ import configparser import json import logging -import os -import pathlib from typing import Optional from sqlalchemy.orm import Session @@ -19,7 +17,7 @@ from charging_stations_pipelines.pipelines.station_table_updater import ( StationTableUpdater, ) -from charging_stations_pipelines.shared import JSON +from charging_stations_pipelines.shared import JSON, country_import_data_path logger = logging.getLogger(__name__) @@ -31,9 +29,7 @@ def __init__(self, config: configparser, session: Session, online: bool = False) self.data: Optional[JSON] = None def _retrieve_data(self): - data_dir: str = os.path.join(pathlib.Path(__file__).parent.resolve(), "../../..", "data") - pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True) - tmp_file_path = os.path.join(data_dir, self.config["GBGOV"]["filename"]) + tmp_file_path = country_import_data_path("GB") / self.config["GBGOV"]["filename"] if self.online: logger.info("Retrieving Online Data") get_gb_data(tmp_file_path) @@ -51,7 +47,7 @@ def run(self): for entry in self.data.get("ChargeDevice", []): mapped_address = map_address_gb(entry, None) mapped_charging = map_charging_gb(entry) - mapped_station = map_station_gb(entry, " GB") + mapped_station = map_station_gb(entry, self.country_code) mapped_station.address = mapped_address mapped_station.charging = mapped_charging station_updater.update_station(station=mapped_station, data_source_key="GBGOV") diff --git a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py index 2d242ca..427cbed 100644 --- a/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py +++ b/charging_stations_pipelines/pipelines/nobil/nobil_pipeline.py @@ -4,19 +4,17 @@ import logging import os from _decimal import Decimal -from pathlib import Path from geoalchemy2.shape import from_shape from shapely.geometry import Point from sqlalchemy.orm import Session from tqdm import tqdm -from charging_stations_pipelines import PROJ_DATA_DIR from charging_stations_pipelines.models.address import Address from charging_stations_pipelines.models.charging import Charging from charging_stations_pipelines.models.station import Station from charging_stations_pipelines.pipelines import Pipeline -from charging_stations_pipelines.shared import download_file, load_json_file, reject_if +from charging_stations_pipelines.shared import download_file, load_json_file, reject_if, country_import_data_path logger = logging.getLogger(__name__) @@ -166,11 +164,7 @@ def __init__( def run(self): """Run the pipeline.""" logger.info(f"Running {self.country_code} GOV Pipeline...") - data_dir: Path = PROJ_DATA_DIR - data_dir.mkdir(parents=True, exist_ok=True) - country_dir = data_dir / self.country_code - country_dir.mkdir(parents=True, exist_ok=True) - path_to_target = country_dir / "nobil.json" + path_to_target = country_import_data_path(self.country_code) / "nobil.json" if self.online: logger.info("Retrieving Online Data") _load_datadump_and_write_to_target(path_to_target, self.country_code) diff --git a/charging_stations_pipelines/pipelines/ocm/ocm.py b/charging_stations_pipelines/pipelines/ocm/ocm.py index d38d256..1c5febd 100644 --- a/charging_stations_pipelines/pipelines/ocm/ocm.py +++ b/charging_stations_pipelines/pipelines/ocm/ocm.py @@ -3,8 +3,6 @@ import configparser import json import logging -import os -import pathlib from sqlalchemy.orm import Session from tqdm import tqdm @@ -19,7 +17,7 @@ from charging_stations_pipelines.pipelines.station_table_updater import ( StationTableUpdater, ) -from charging_stations_pipelines.shared import JSON +from charging_stations_pipelines.shared import JSON, country_import_data_path logger = logging.getLogger(__name__) @@ -38,11 +36,7 @@ def __init__( self.data: JSON = None def _retrieve_data(self): - data_dir: str = os.path.join(pathlib.Path(__file__).parent.resolve(), "../../..", "data") - pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True) - country_dir = os.path.join(data_dir, self.country_code) - pathlib.Path(country_dir).mkdir(parents=True, exist_ok=True) - tmp_file_path = os.path.join(country_dir, self.config["OCM"]["filename"]) + tmp_file_path = country_import_data_path(self.country_code) / self.config["OCM"]["filename"] if self.online: logger.info("Retrieving Online Data") ocm_extractor(tmp_file_path, self.country_code) diff --git a/charging_stations_pipelines/pipelines/osm/osm.py b/charging_stations_pipelines/pipelines/osm/osm.py index 7a4e155..7b4e5df 100644 --- a/charging_stations_pipelines/pipelines/osm/osm.py +++ b/charging_stations_pipelines/pipelines/osm/osm.py @@ -3,12 +3,11 @@ import configparser import json import logging -from pathlib import Path from sqlalchemy.orm import Session from tqdm import tqdm -from charging_stations_pipelines import PROJ_DATA_DIR, OSM_COUNTRY_CODES +from charging_stations_pipelines import OSM_COUNTRY_CODES from charging_stations_pipelines.pipelines import Pipeline from charging_stations_pipelines.pipelines.osm import ( DATA_SOURCE_KEY, @@ -18,7 +17,7 @@ from charging_stations_pipelines.pipelines.station_table_updater import ( StationTableUpdater, ) -from charging_stations_pipelines.shared import JSON +from charging_stations_pipelines.shared import JSON, country_import_data_path logger = logging.getLogger(__name__) @@ -38,11 +37,7 @@ def __init__( self.country_code = country_code def retrieve_data(self): - data_dir: Path = PROJ_DATA_DIR - data_dir.mkdir(parents=True, exist_ok=True) - country_dir = data_dir / self.country_code - country_dir.mkdir(parents=True, exist_ok=True) - tmp_file_path = country_dir / self.config[DATA_SOURCE_KEY]["filename"] + tmp_file_path = country_import_data_path(self.country_code) / self.config[DATA_SOURCE_KEY]["filename"] if self.online: logger.info("Retrieving Online Data") get_osm_data(self.country_code, tmp_file_path) diff --git a/charging_stations_pipelines/shared.py b/charging_stations_pipelines/shared.py index d1eeb19..f1ecfbe 100644 --- a/charging_stations_pipelines/shared.py +++ b/charging_stations_pipelines/shared.py @@ -2,20 +2,19 @@ import configparser import json import logging -import os -import pathlib import re from collections.abc import Iterable from datetime import datetime +from pathlib import Path from typing import Any, Dict, List, Optional, TypeVar, Union import pandas as pd import requests from dateutil import parser -logger = logging.getLogger(__name__) +from charging_stations_pipelines import PROJ_ROOT, PROJ_DATA_DIR -current_dir = os.path.join(pathlib.Path(__file__).parent.parent.resolve()) +logger = logging.getLogger(__name__) _PlainJSON = Union[None, bool, int, float, str, List["_PlainJSON"], Dict[str, "_PlainJSON"]] @@ -28,7 +27,7 @@ def init_config(): """Initializes the configuration from the config.ini file.""" cfg: configparser = configparser.RawConfigParser() - cfg.read(os.path.join(os.path.join(current_dir, "config", "config.ini"))) + cfg.read(PROJ_ROOT / "config" / "config.ini") return cfg @@ -229,3 +228,12 @@ def download_file(url: str, target_file: str) -> None: output = open(target_file, "wb") output.write(resp.content) output.close() + + +def country_import_data_path(country_code: str) -> Path: + """Creates a directory 'PROJ_DATA_DIR/country_code/' if not already existing and returns the path""" + data_dir: Path = PROJ_DATA_DIR + data_dir.mkdir(parents=True, exist_ok=True) + country_dir = data_dir / country_code + country_dir.mkdir(parents=True, exist_ok=True) + return country_dir From 192c14dfa98342a075ce5075faa34d8e7cd30b5f Mon Sep 17 00:00:00 2001 From: Martin Mader Date: Fri, 16 Feb 2024 13:21:28 +0100 Subject: [PATCH 4/6] fix: country code and date mapping for GB GOV pipeline --- .../pipelines/gb/gb_mapper.py | 15 ++++++--------- charging_stations_pipelines/pipelines/gb/gbgov.py | 2 +- charging_stations_pipelines/shared.py | 2 ++ test/test_shared.py | 2 ++ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/charging_stations_pipelines/pipelines/gb/gb_mapper.py b/charging_stations_pipelines/pipelines/gb/gb_mapper.py index 94eaf3f..1edfebd 100644 --- a/charging_stations_pipelines/pipelines/gb/gb_mapper.py +++ b/charging_stations_pipelines/pipelines/gb/gb_mapper.py @@ -7,27 +7,24 @@ from charging_stations_pipelines.models.address import Address from charging_stations_pipelines.models.charging import Charging from charging_stations_pipelines.models.station import Station -from charging_stations_pipelines.shared import check_coordinates +from charging_stations_pipelines.shared import check_coordinates, parse_date logger = logging.getLogger(__name__) # functions for GB gov data: -def map_station_gb(entry, country_code: str): +def map_station_gb(entry): datasource = "GBGOV" lat: float = check_coordinates(entry.get("ChargeDeviceLocation").get("Latitude")) long: float = check_coordinates(entry.get("ChargeDeviceLocation").get("Longitude")) - operator: Optional[str] = entry.get("DeviceController").get("OrganisationName") new_station = Station() - new_station.country_code = country_code + new_station.country_code = "GB" new_station.source_id = entry.get("ChargeDeviceId") - new_station.operator = operator + new_station.operator = entry.get("DeviceController").get("OrganisationName") new_station.data_source = datasource new_station.point = from_shape(Point(float(long), float(lat))) - new_station.date_created = entry.get("DateCreated") - new_station.date_updated = entry.get("DateUpdated") - # TODO: find way to parse date into desired format - # parse_date having issues with "date out of range" at value 0" + new_station.date_created = parse_date(entry.get("DateCreated")) + new_station.date_updated = parse_date(entry.get("DateUpdated")) return new_station diff --git a/charging_stations_pipelines/pipelines/gb/gbgov.py b/charging_stations_pipelines/pipelines/gb/gbgov.py index 824f720..6b4518a 100644 --- a/charging_stations_pipelines/pipelines/gb/gbgov.py +++ b/charging_stations_pipelines/pipelines/gb/gbgov.py @@ -47,7 +47,7 @@ def run(self): for entry in self.data.get("ChargeDevice", []): mapped_address = map_address_gb(entry, None) mapped_charging = map_charging_gb(entry) - mapped_station = map_station_gb(entry, self.country_code) + mapped_station = map_station_gb(entry) mapped_station.address = mapped_address mapped_station.charging = mapped_charging station_updater.update_station(station=mapped_station, data_source_key="GBGOV") diff --git a/charging_stations_pipelines/shared.py b/charging_stations_pipelines/shared.py index f1ecfbe..e2e2640 100644 --- a/charging_stations_pipelines/shared.py +++ b/charging_stations_pipelines/shared.py @@ -69,6 +69,8 @@ def parse_date(date_str: Optional[str]) -> Optional[datetime]: :param date_str: The string representation of a date. :return: A datetime object representing the parsed date, or None if the date could not be parsed. """ + if date_str is None: + return None try: return parser.parse(date_str) except (parser.ParserError, TypeError) as e: diff --git a/test/test_shared.py b/test/test_shared.py index 416f8bf..0006808 100644 --- a/test/test_shared.py +++ b/test/test_shared.py @@ -43,8 +43,10 @@ def test_check_coordinates(): def test_str_parse_date(): assert parse_date("2022-01-01") == datetime(2022, 1, 1) assert parse_date("2023-03-29T17:45:00Z").isoformat() == "2023-03-29T17:45:00+00:00" + assert parse_date("2020-05-04 10:33:41").isoformat() == "2020-05-04T10:33:41" assert parse_date(None) is None assert parse_date("abc") is None + assert parse_date("0000-00-00 00:00:00") is None def test_str_strip_whitespace(): From c6643733794298e80631428fdafb2799fe7155e7 Mon Sep 17 00:00:00 2001 From: Martin Mader Date: Fri, 16 Feb 2024 14:02:02 +0100 Subject: [PATCH 5/6] refactor: improve list countries script --- list-countries.py | 29 ++++++++++++++++++++++++++++- requirements.txt | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/list-countries.py b/list-countries.py index 05d03e0..feb9fca 100644 --- a/list-countries.py +++ b/list-countries.py @@ -1,5 +1,32 @@ +import argparse import json +import sys + +import pandas from charging_stations_pipelines import COUNTRIES -print(json.dumps(COUNTRIES, default=vars, indent=4)) + +def parse_args(args): + parser = argparse.ArgumentParser( + description="List country codes and other country related metadata available in eCharm", + epilog="Example: python list-countries.py --json", + ) + + parser.add_argument( + "-j", + "--json", + action="store_true", + help="output country information in json format.", + ) + + return parser.parse_args(args) + + +command_line_args = parse_args(sys.argv[1:]) +countries_json = json.dumps(COUNTRIES, default=vars, indent=4) +if command_line_args.json: + print(countries_json) +else: + df = pandas.read_json(json.dumps(COUNTRIES, default=vars, indent=4), orient="index") + print(df.to_markdown(tablefmt="fancy_grid")) diff --git a/requirements.txt b/requirements.txt index 524ed6b..461ee50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,6 +54,7 @@ six==1.16.0 smmap==5.0.0 soupsieve==2.3.2.post1 SQLAlchemy==1.3.17 +tabulate==0.9.0 tqdm==4.62.3 uritemplate==4.1.1 urllib3==1.26.11 From 4d789066679f9f4517576e9d7ba7408af73d17ae Mon Sep 17 00:00:00 2001 From: Stefan Pernpaintner Date: Mon, 19 Feb 2024 15:47:05 +0100 Subject: [PATCH 6/6] reactivate import of governmental data for France --- charging_stations_pipelines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charging_stations_pipelines/__init__.py b/charging_stations_pipelines/__init__.py index ee6e5f7..384ac2e 100644 --- a/charging_stations_pipelines/__init__.py +++ b/charging_stations_pipelines/__init__.py @@ -27,7 +27,7 @@ def __init__(self, name: str, gov: bool, osm: bool, ocm: bool): ("EE", CountryInfo(name="Estonia", gov=False, osm=True, ocm=True)), ("ES", CountryInfo(name="Spain", gov=False, osm=True, ocm=True)), ("FI", CountryInfo(name="Finland", gov=False, osm=True, ocm=True)), - ("FR", CountryInfo(name="France", gov=False, osm=True, ocm=True)), + ("FR", CountryInfo(name="France", gov=True, osm=True, ocm=True)), ("GB", CountryInfo(name="United Kingdom", gov=True, osm=True, ocm=True)), ("GR", CountryInfo(name="Greece", gov=False, osm=True, ocm=True)), ("HR", CountryInfo(name="Croatia", gov=False, osm=True, ocm=True)),