Merge pull request #10 from aschuma/v0.5.x

Preparing 0.5.x - using 'extractors' to control data retrieval
aschuma · Feb 14, 2023 · 078e452 · 078e452
2 parents 2f3c224 + 4eb1275
commit 078e452
Show file tree

Hide file tree

Showing 10 changed files with 236 additions and 100 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,10 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_size = 2
+
diff --git a/README.md b/README.md
@@ -1,4 +1,3 @@
-
 [![version](https://img.shields.io/npm/v/dsb-untis.svg?style=flat-square)](http://npm.im/dsb-untis)
 [![downloads](https://img.shields.io/npm/dm/dsb-untis.svg?style=flat-square)](http://npm-stat.com/charts.html?package=dsb-untis&from=2021-01-01)
 [![MIT License](https://img.shields.io/npm/l/dsb-untis.svg?style=flat-square)](http://opensource.org/licenses/MIT)
@@ -14,8 +13,8 @@ const username = "165931";
 const password = "secret";
 
 const dsbUntis = new DsbUntis(username, password);
-dsbUntis.fetch().then((data) => { 
-    console.log(JSON.stringify(data))
+dsbUntis.fetch().then((data) => {
+  console.log(JSON.stringify(data))
 });
 ```
 
@@ -65,10 +64,36 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Art",
         "Vertr. von / verlegt"
       ],
-      ["6b"],
-      ["15.1.", "Fr", "E", "3 - 4", "Kmt", "", "---", "E", "Entfall", ""],
-      ["9c"],
-      ["15.1.", "Fr", "D", "6", "Nlr", "", "R109", "D", "Vertretung", ""]
+      [
+        "6b"
+      ],
+      [
+        "15.1.",
+        "Fr",
+        "E",
+        "3 - 4",
+        "Kmt",
+        "",
+        "---",
+        "E",
+        "Entfall",
+        ""
+      ],
+      [
+        "9c"
+      ],
+      [
+        "15.1.",
+        "Fr",
+        "D",
+        "6",
+        "Nlr",
+        "",
+        "R109",
+        "D",
+        "Vertretung",
+        ""
+      ]
     ]
   },
   {
@@ -89,7 +114,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Art",
         "Vertr. von / verlegt"
       ],
-      ["6b"],
+      [
+        "6b"
+      ],
       [
         "18.1.",
         "Mo",
@@ -104,7 +131,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Vertretung",
         ""
       ],
-      ["7a"],
+      [
+        "7a"
+      ],
       [
         "18.1.",
         "Mo",
@@ -119,7 +148,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Vertretung",
         ""
       ],
-      ["9a"],
+      [
+        "9a"
+      ],
       [
         "18.1.",
         "Mo",
@@ -134,7 +165,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Entfall",
         ""
       ],
-      ["9b"],
+      [
+        "9b"
+      ],
       [
         "18.1.",
         "Mo",
@@ -149,7 +182,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Entfall",
         ""
       ],
-      ["9c"],
+      [
+        "9c"
+      ],
       [
         "18.1.",
         "Mo",
@@ -164,7 +199,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Entfall",
         ""
       ],
-      ["10a"],
+      [
+        "10a"
+      ],
       [
         "18.1.",
         "Mo",
@@ -193,7 +230,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Vertretung",
         ""
       ],
-      ["10b"],
+      [
+        "10b"
+      ],
       [
         "18.1.",
         "Mo",
@@ -228,7 +267,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Art",
         "Vertr. von / verlegt"
       ],
-      ["10c"],
+      [
+        "10c"
+      ],
       [
         "18.1.",
         "Mo",
@@ -243,7 +284,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
         "Entfall",
         ""
       ],
-      ["Jg2"],
+      [
+        "Jg2"
+      ],
       [
         "18.1.",
         "Mo",
@@ -262,3 +305,9 @@ docker run --rm  -d -e USERNAME="165931" -e PASSWORD="secret" -e FLATFORMAT="fal
   }
 ]
 ```
+
+## Advanced Feature
+
+The returned data can be customized by passing a list of ```extractors``` to the ```fetch``` method.
+There are some built-in extractors located in the
+```src/timetableHtmlExtractors``` folder, however, you can also create your own custom extractors if needed.
diff --git a/docker/server.js b/docker/server.js
@@ -2,11 +2,17 @@ const DsbUntis = require('../src/index.js')
 
 const username = process.env.USERNAME || 'username';
 const password = process.env.PASSWORD || 'password';
-const flatFormat = "true" === process.env.FLATFORMAT;
 
-console.log(`Starting server (USERNAME=${username}, PASSWORD=${password.slice(0,1)}..., FLATFORMAT=${flatFormat})`);
+// see timetableHtmlExtractors
+//
+//  extractorDivMonTitle
+//  extractorTableInfo
+//  extractorTableMonList | extractorTableMonListFlat
+const extractorList = (process.env.EXTRACTORS || 'extractorTableMonList, extractorDivMonTitle').split(",").map(item => item.trim());
+
+console.log(`Starting server (USERNAME=${username}, PASSWORD=${password.slice(0,1)}..., EXTRACTORS=${extractorList})`)
 
 const dsbUntis = new DsbUntis(username, password);
-    
-dsbUntis.listen(8080,flatFormat);
+
+dsbUntis.listen(8080, {extractors: extractorList});
 
diff --git a/examples/extractClassData.js b/examples/extractClassData.js
@@ -63,13 +63,12 @@ const DsbUntis = require("../src/index.js");
 
 const username = process.env.USERNAME || "username";
 const password = process.env.PASSWORD || "password";
-const flatFormat = true;
 
 const dsbUntis = new DsbUntis(username, password);
 const transform = (timetables) => extractClassRows(timetables, clazz);
 
 dsbUntis
-  .fetch(flatFormat)
+  .fetch({extractors :["extractorTableMonListFlat"]})
   .then(transform)
   .then(collapse)
   .then((data) => {

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dsb-untis",
-  "version": "v0.4.3",
+  "version": "v0.5.0",
   "description": "DSB Untis timetable client/proxy server",
   "main": "src/index.js",
   "scripts": {
@@ -27,7 +27,7 @@
     "axios": "0.26.1",
     "console-stamp": "3.0.4",
     "dsbapi": "4.1.1",
-    "jquery": "3.6.0",
+    "jquery": "3.6.3",
     "jsdom": "19.0.0",
     "node-fetch": "2.6.7"
   }

diff --git a/src/index.js b/src/index.js
@@ -1,28 +1,12 @@
 const DSB = require("dsbapi");
 const http = require("http");
-const parseTimetableHtml = require("./timetableHtmlParser");
+const buildInHtmlExtractors = require("./timetableHtmlExtractors/index")
 const fetchTimetables = require("./timetableHtmlFetcher");
 const resolveTimetableUrls = require("./timetableUrlResolver");
 
 require("./jsonHelper")();
-require("console-stamp")(console, "yyyy-MM-dd HH:mm:ss");
-console.debug = () => {};
-
-function flatten(table) {
-  let currentClass = "";
-  return table
-    .map((cells) => {
-      let mappedCells = cells;
-      if (cells.length == 1) {
-        currentClass = cells[0];
-        mappedCells = null;
-      } else {
-        mappedCells = [currentClass, ...mappedCells];
-      }
-      return mappedCells;
-    })
-    .filter((cells) => cells != null);
-}
+console.debug = () => {
+};
 
 class DsbUntis {
   /**
@@ -32,31 +16,45 @@ class DsbUntis {
    * @param {String|Boolean} [cache=false] In the browser just a boolean and in node a path string. If you don't want to use any cache just use undefined, null or false.
    * @param {Axios} [axios=require('axios')] Pass your custom axios instance if you want.
    */
-  constructor(
-    username,
-    password,
-    cookies = "",
-    cache = false,
-    axios = require("axios")
-  ) {
+  constructor(username, password, cookies = "", cache = false, axios = require("axios")) {
     this.dsb = new DSB(username, password, cookies, cache, axios);
   }
 
   /**
    * Fetch data
-   * @param {Boolean} [flat=false] flatten the result table
+   * @param {Object.<String, (String|Number|Function|Array)>} configuration
    * @returns {Promise.<Object>}
    */
-  async fetch(flat = false) {
+  async fetch(configuration = {extractors: [buildInHtmlExtractors.extractorTableMonListFlat]}) {
+
+    const {extractors = [buildInHtmlExtractors.extractorTableMonListFlat]} = configuration;
+    const buildInHtmlExtractorNames = Object.keys(buildInHtmlExtractors);
+
+    console.debug("Configured extractors:", extractors);
+    console.debug("Available extractors:", buildInHtmlExtractorNames);
+
+    const resolvedExtractors = extractors.reduce((acc, functionOrName) => {
+
+      if (typeof functionOrName === 'string') {
+        if (buildInHtmlExtractorNames.includes(functionOrName)) {
+          acc.push(buildInHtmlExtractors[functionOrName]);
+        }
+      } else if (typeof functionOrName === 'function') {
+        acc.push(functionOrName);
+      }
+      return acc;
+    }, []);
+
+    console.debug("Resolved extractors:", resolvedExtractors);
+
     const dsbNodes = await this.dsb.fetch();
-    if (dsbNodes.Resultcode == 0) {
+    if (dsbNodes.Resultcode === 0) {
       const urlList = resolveTimetableUrls(dsbNodes);
       const timetableHtmlList = await fetchTimetables(urlList);
-      const postprocessor = flat ? flatten : (id) => id;
       const data = timetableHtmlList.map((timetableHtml) => {
+        const extractedData = resolvedExtractors.reduce((acc, extractor) => ({...acc, ...extractor(timetableHtml.html)}), {});
         const answer = {
-          ...timetableHtml,
-          table: postprocessor(parseTimetableHtml(timetableHtml.html)),
+          ...timetableHtml, ...extractedData,
         };
         delete answer.html;
         return answer;
@@ -67,15 +65,19 @@ class DsbUntis {
     }
   }
 
+
   /**
    * Start a HTTP server
    * @param {Number} [port=8080] server port
-   * @param {Boolean} [flat=false] flatten the result table
+   * @param {Object.<String, (String|Number|Function|Array)>} configuration
    */
-  listen(port = 8080, flat = false) {
+  listen(port = 8080, configuration = {extractors: [buildInHtmlExtractors.extractorTableMonListFlat]}) {
+
+    require("console-stamp")(console, "yyyy-MM-dd HH:mm:ss");
+
     const requestListener = (req, res) => {
       res.setHeader("Content-Type", "application/json;charset=utf-8");
-      this.fetch(flat)
+      this.fetch(configuration)
         .then((data) => {
           res.writeHead(200);
           res.end(JSON.stringify(data));