Merge pull request #68 from comsysto/fix/FRGOV-pipeline-not-working

Fix FR governmental data pipeline

.github/workflows/check-datasources.yml

@@ -24,13 +24,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip install -r test/requirements.txt
 
-      #      - name: Run integration tests (only)
-      #        run: |
-      #          pip install -r test/requirements.txt
-      #          pytest -m "integration_test"
-
-      - name: "[DE/BNA] Real data validity checks"
+      - name: "[DE/BNA] Data retrieval check"
         run: |
-          pip install -r test/requirements.txt
           pytest tests/integration/test_int_de_bna.py
+
+      - name: "[FR] Data retrieval check"
+        run: |
+          pytest tests/integration/test_int_fr_france.py

.github/workflows/python-app.yml

@@ -45,4 +45,4 @@ jobs:
       - name: Run tests
         run: |
           pip install -r test/requirements.txt
-          pytest
+          pytest -m "not check_datasource"

charging_stations_pipelines/pipelines/fr/france.py

@@ -31,12 +31,7 @@ def _retrieve_data(self):
         if self.online:
             logger.info("Retrieving Online Data")
             self.download_france_gov_file(tmp_data_path)
-        self.data = pd.read_csv(
-            os.path.join(data_dir, "france_stations.csv"),
-            delimiter=",",
-            encoding="utf-8",
-            encoding_errors="replace",
-        )
+        self.data = self.load_csv_file(tmp_data_path)
 
     def run(self):
         logger.info("Running FR GOV Pipeline...")
@@ -55,7 +50,7 @@ def run(self):
     @staticmethod
     def download_france_gov_file(target_file):
         """Download a file from the French government website."""
-        base_url = "https://transport.data.gouv.fr/resources/79624"
+        base_url = "https://transport.data.gouv.fr/resources/81548"
 
         r = requests.get(base_url, headers={"User-Agent": "Mozilla/5.0"})
         soup = BeautifulSoup(r.content, "html.parser")
@@ -73,3 +68,13 @@ def download_france_gov_file(target_file):
             "Could not determine source for french government data",
         )
         download_file(link_to_dataset[0]["href"], target_file)
+
+    @staticmethod
+    def load_csv_file(target_file):
+        return pd.read_csv(
+            target_file,
+            delimiter=",",
+            encoding="utf-8",
+            encoding_errors="replace",
+            low_memory=False,
+        )

charging_stations_pipelines/pipelines/fr/france_mapper.py

@@ -41,15 +41,11 @@ def map_station_fra(row: pd.Series) -> Station:
             float(check_coordinates(row.get("consolidated_latitude"))),
         )
     )
-    station.date_created = row.get("date_mise_en_service").strptime("%Y-%m-%d")
-    station.date_updated = row.get("date_maj").strptime("%Y-%m-%d")
 
     if not pd.isna(row.get("date_mise_en_service")):
         station.date_created = datetime.strptime(row.get("date_mise_en_service"), "%Y-%m-%d")
     if not pd.isna(row.get("date_maj")):
         station.date_updated = datetime.strptime(row.get("date_maj"), "%Y-%m-%d")
-    else:
-        station.date_updated = datetime.now
 
     return station
 

pyproject.toml

@@ -27,5 +27,6 @@ testpaths = [
 
 # Declare custom markers
 markers = [
-    "integration_test: marks tests as integration tests, which are bit slow (deselect with '-m \"integration_test\"')",
+    "integration_test: marks tests as integration tests (deselect with '-m \"not integration_test\"')",
+    "check_datasource: marks tests as datasource check for scheduled github action (deselect with '-m \"not check_datasource\"')",
 ]

tests/integration/test_int_de_bna.py

@@ -58,25 +58,16 @@ def bna_data():
 
 
 @pytest.mark.integration_test
+@pytest.mark.check_datasource
 def test_file_size(bna_data):
     bna_file_name, _ = bna_data
     # Check file size of the downloaded file
-    assert os.path.getsize(bna_file_name) >= 8_602_458  # ~ 9 MB
+    assert os.path.getsize(bna_file_name) >= 1_000  # actual file is ~ 9 MB, just make sure it is not quasi empty here
 
 
 @pytest.mark.integration_test
+@pytest.mark.check_datasource
 def test_dataframe_schema(bna_data):
     _, bna_in_data = bna_data
     # Check schema of the downloaded Excel file
     assert verify_schema_follows(bna_in_data, EXPECTED_DATA_SCHEMA), "Mismatch in schema of the downloaded Excel file!"
-
-
-@pytest.mark.integration_test
-def test_dataframe_shape(bna_data):
-    _, bna_in_data = bna_data
-    # Check shape of the dataframe
-    # Not exact check, because file grows over time
-    # Expected: at least 54,223 rows and 23 columns
-    num_rows, num_cols = bna_in_data.shape
-    assert num_rows >= 54_223, "Mismatch in dataframe shape: too few rows!"
-    assert num_cols >= 23, "Mismatch in dataframe shape: too few columns!"

tests/integration/test_int_fr_france.py

@@ -2,17 +2,45 @@
 
 import os
 import tempfile
-
 import pytest
 
 from charging_stations_pipelines.pipelines.fr.france import FraPipeline
-from test.shared import skip_if_github
+from tests.test_utils import verify_schema_follows
 
+EXPECTED_DATA_SCHEMA = {
+    "id_station_itinerance": "object",
+    "nom_operateur": "object",
+    "consolidated_longitude": "float64",
+    "consolidated_latitude": "float64",
+    "date_mise_en_service": "object",
+    "date_maj": "object",
+    "nbre_pdc": "int64",
+    "adresse_station": "object",
+    "consolidated_commune": "object",
+    "consolidated_code_postal": "float64",
+}
 
-@pytest.mark.integration_test
-@pytest.mark.skipif(skip_if_github(), reason="Skip the test when running on Github")
-def test_download_france_gov_file():
-    """Test the download function."""
+
+@pytest.fixture(scope="module")
+def fr_data():
+    """Setup method for tests. Executes once at the beginning of the test session (and not before each test)."""
+    # Download real FR GOV data to a temporary file
     with tempfile.NamedTemporaryFile() as temp_file:
         FraPipeline.download_france_gov_file(temp_file.name)
-        assert os.path.getsize(temp_file.name) >= 47_498_370  # ~ 50 MB
+        fr_dataframe = FraPipeline.load_csv_file(temp_file.name)
+        yield temp_file.name, fr_dataframe
+
+
+@pytest.mark.integration_test
+@pytest.mark.check_datasource
+def test_download_france_gov_file(fr_data):
+    """Test the download function."""
+    fr_filename, _ = fr_data
+    assert os.path.getsize(fr_filename) >= 1_000  # actual file is ~ 45 MB, just make sure it is not quasi empty here
+
+
+@pytest.mark.integration_test
+@pytest.mark.check_datasource
+def test_dataframe_schema(fr_data):
+    _, fr_dataframe = fr_data
+    assert verify_schema_follows(fr_dataframe, EXPECTED_DATA_SCHEMA), "Mismatch in schema of the downloaded csv file!"