FlugHafen pipeline - first commit

.github/workflows/project_feedback.yml

@@ -0,0 +1,31 @@
+name: Run Tests
+run-name: ${{ github.actor }} is running tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      # project and exercise feedbacks
+      - name: Make test executable
+        run: chmod +x /home/runner/work/made-template/made-template/project/tests.sh
+
+      - name: Run project tests 
+        run: /home/runner/work/made-template/made-template/project/tests.sh

.gitignore

@@ -1,3 +1,8 @@
 .DS_Store
 /data/*
-!/data/.gitkeep
+!/data/.gitkeep
+.mypy_cache/
+temp_dir/
+project/__pycache__/
+*.pyc
+*.sqlite

exercises/exercise1.jv

@@ -0,0 +1,74 @@
+pipeline FlugHafen{
+
+    //1. FlugHafen Pipeline connects the blocks via pipes to extract data from a CSV file
+    // in the web to a SQLite file sink.    
+    FlugHafenHttpExtractor 
+        -> FlugHafenTextFileInterpreter;
+
+    //2. The FlugHafenTextFileInterpreter output is used as input for the FlugHafenCsvFileInterpreter
+    // block which is then used as input for the FlugHafenDataSelector block.
+    FlugHafenTextFileInterpreter
+        -> FlugHafenCsvFileInterpreter
+        // -> FlugHafenDatabaseWriter
+        -> FlugHafenDataSelector
+        -> FlugHafenTableInterpreter
+        -> FlugHafenLoader;
+
+    //3. The FlugHafenHttpExtractor block is of type HttpExtractor and the URL is specified.
+    block FlugHafenHttpExtractor oftype HttpExtractor {
+        // URL of the data source
+        url: "https://opendata.rhein-kreis-neuss.de/api/explore/v2.1/catalog/datasets/rhein-kreis-neuss-flughafen-weltweit/exports/csv?lang=en&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B";
+    }
+
+    //4. The FlugHafenTextFileInterpreter block is of type TextFileInterpreter.
+    block FlugHafenTextFileInterpreter oftype TextFileInterpreter { }
+
+    //5. Since we only need a specific range of the data, we use the CellRangeSelector block.
+    block FlugHafenDataSelector oftype CellRangeSelector {
+        // The name of the sheet
+        select: range A1:I*;
+    }
+
+    //6. The FlugHafenCsvFileInterpreter block is of type CSVInterpreter and the delimiter is specified.
+    block FlugHafenCsvFileInterpreter oftype CSVInterpreter {
+        // Specify the separator as a semicolon for the CSV
+        delimiter: ';';
+    }
+
+    // block FlugHafenDatabaseWriter oftype DatabaseWriter {
+    //     // The name of the database
+    //     database: "flughafen.db";
+    //     // The name of the table
+    //     table: "flughafen";
+    // }
+
+    //7. The FlugHafenTableInterpreter block is of type TableInterpreter and the necessary columns are specified.
+    block FlugHafenTableInterpreter oftype TableInterpreter {
+        // The first row contains the header
+        header: true;
+        // The columns of the table
+        columns: [
+            "Lfd. Nummmer" oftype integer,
+            "Name des Flughafens" oftype text,
+            "Ort" oftype text,
+            "Land" oftype text,
+            "IATA" oftype text,
+            "ICAO" oftype text,
+            "Latitude" oftype decimal,
+            "Longitude" oftype decimal,
+            "Altitude" oftype integer,
+
+        ];
+    }
+
+    //8. Finally the FlugHafenLoader block is of type SQLiteLoader and the table name and file name are specified.
+    block FlugHafenLoader oftype SQLiteLoader {
+        // The name of the table
+        table: "airports";
+        // The name of the file
+        file: "airports.sqlite";
+     }
+
+
+}
+

exercises/exercise2.jv

@@ -0,0 +1,85 @@
+pipeline TreePlanting{
+
+    //1. TreePlanting Pipeline connects the blocks via pipes to extract data from a CSV file
+    // in the web to a SQLite file sink.    
+    TreePlantingHttpExtractor 
+        -> TreePlantingTextFileInterpreter;
+
+    //2. The TreePlantingTextFileInterpreter output is used as input for the TreePlantingCsvFileInterpreter
+    // block which is then used as input for the TreePlantingDataSelector block.
+    TreePlantingTextFileInterpreter
+        -> TreePlantingCsvFileInterpreter
+        // -> TreePlantingDatabaseWriter
+        -> TreePlantingBaumartDeutschDeleter
+        -> TreePlantingTableInterpreter
+        -> TreePlantingLoader;
+
+    //3. The TreePlantingHttpExtractor block is of type HttpExtractor and the URL is specified.
+    block TreePlantingHttpExtractor oftype HttpExtractor {
+        // URL of the data source
+        url: "https://opendata.rhein-kreis-neuss.de/api/v2/catalog/datasets/stadt-neuss-herbstpflanzung-2023/exports/csv";
+    }
+
+    //4. The TreePlantingTextFileInterpreter block is of type TextFileInterpreter.
+    block TreePlantingTextFileInterpreter oftype TextFileInterpreter { }
+
+    //6. The TreePlantingCsvFileInterpreter block is of type CSVInterpreter and the delimiter is specified.
+    block TreePlantingCsvFileInterpreter oftype CSVInterpreter {
+        // Specify the separator as a semicolon for the CSV
+        delimiter: ';';
+    }
+
+    //5. The TreePlantingBaumartDeutschDeleter block is of type ColumnDeleter and the column to be deleted is specified.
+    block TreePlantingBaumartDeutschDeleter oftype ColumnDeleter {
+        // The name of the column
+        delete: [column E];
+    }
+
+    //7. The TreePlantingTableInterpreter block is of type TableInterpreter and the necessary columns are specified.
+    block TreePlantingTableInterpreter oftype TableInterpreter {
+        // The first row contains the header
+        header: true;
+        // The columns of the table
+        columns: [
+            "lfd_nr" oftype integer,
+            "stadtteil" oftype Vogelsang,
+            "standort" oftype text,
+            "baumart_botanisch" oftype text,
+            "id" oftype GeoCoordinate,
+            "baumfamilie" oftype text,
+
+        ];
+    }
+
+    block TreePlantingLoader oftype SQLiteLoader {
+        // The name of the table
+        table: "trees";
+        // The name of the file
+        file: "trees.sqlite";
+     }
+
+
+    valuetype Vogelsang oftype text {
+        // The value of the column
+        constraints: [
+            // only allow column values that start with "Vogelsang"
+            VogelsangStadteil
+        ];
+    }
+
+    valuetype GeoCoordinate oftype text {
+        // The value of the column
+        constraints: [
+            // only allow column values that match the pattern of a geo coordinate
+            Geopoints
+        ];
+    }
+
+    constraint VogelsangStadteil on text: value matches(/^Vogelsang*/);
+    //8. Finally the TreePlantingLoader block is of type SQLiteLoader and the table name and file name are specified.
+
+    constraint Geopoints on text: value matches(/^\d{1,3}\.\d+,\s*\d{1,3}\.\d+$/);
+
+
+}
+

exercises/exercise3.jv

@@ -0,0 +1,92 @@
+pipeline WorldBank {
+
+    // 1. Block to extract an XLSX file from the web
+    block WorldBankHttpExtractor oftype HttpExtractor {
+        // URL of the source file containing the data
+        url: "https://thedocs.worldbank.org/en/doc/7d852628d96b9411d43e5d36d5dff941-0050062022/original/Graphs-Chapter-5-02082022.xlsx";
+    }
+
+    // 2. Block to interpret the downloaded file as an XLSX workbook
+    block WorldBankTextXLSXInterpreter oftype XLSXInterpreter { }
+
+    // 3. Block to select the specific sheet "Figure S5.1.2" from the workbook
+    block DataCellSheetpicker oftype SheetPicker {
+        sheetName: "Figure S5.1.2";
+    }
+
+    // 4. Block to specify the cell range of interest in the selected sheet
+    block WorldBankRangeSelector oftype CellRangeSelector {
+        select: range P2:S45;
+    }
+
+    // 5. Block to rename columns for clarity and standardization
+    block NameHeaderWriter oftype CellWriter {
+        at: range A1:D1;
+        write: ["Country Code", "Economy", "GDP per Capita", "Bond Issuance Share"];
+    }
+
+    // 6. Block to interpret and filter the "Bond Issuance Share" column data
+    block BondIssuanceTableInterpreter oftype TableInterpreter {
+        columns: [
+            "Country Code" oftype CountryCodeAlpha3,
+            "Bond Issuance Share" oftype BetweenZeroAndOne,
+        ];
+    }
+
+    // 7. Block to interpret and filter the "GDP per Capita" column data
+    block GDPTableInterpreter oftype TableInterpreter {
+        columns: [
+            "Country Code" oftype CountryCodeAlpha3,
+            "GDP per Capita" oftype PositiveDecimal,
+        ];
+    }
+
+    // 8. Block to load "Bond Issuance Share" data into a SQLite table
+    block BondIssuanceTableLoader oftype SQLiteLoader {
+        table: "bondIssuance"; // Table name in the SQLite database
+        file: "country-stats.sqlite"; // SQLite database file
+    }
+
+    // 9. Block to load "GDP per Capita" data into a SQLite table
+    block GDPTableLoader oftype SQLiteLoader {
+        table: "gdpPerCapita"; // Table name in the SQLite database
+        file: "country-stats.sqlite"; // SQLite database file
+    }
+
+    // 10. Value type to enforce positive decimal values
+    valuetype PositiveDecimal oftype decimal {
+        constraints: [
+            OnlyPositiveDecimal // Constraint: Values must be > 0
+        ];
+    }
+
+    // 11. Value type to enforce decimal values between 0 and 1
+    valuetype BetweenZeroAndOne oftype decimal {
+        constraints: [
+            BetweenZeroAndOneConstraint // Constraint: 0 <= value <= 1
+        ];
+    }
+
+    // 12. Constraints definitions
+    constraint OnlyPositiveDecimal on decimal: value > 0;
+    constraint BetweenZeroAndOneConstraint on decimal: value >= 0 and value <= 1;
+
+    // Pipeline connections
+    WorldBankHttpExtractor
+        -> WorldBankTextXLSXInterpreter;
+
+    WorldBankTextXLSXInterpreter
+        -> DataCellSheetpicker
+        -> WorldBankRangeSelector
+        -> NameHeaderWriter;
+
+    // Bond Issuance pipeline
+    NameHeaderWriter
+        -> BondIssuanceTableInterpreter
+        -> BondIssuanceTableLoader;
+
+    // GDP per Capita pipeline
+    NameHeaderWriter
+        -> GDPTableInterpreter
+        -> GDPTableLoader;
+}