Add freecodecamp scraper

dispatcher/backend/docs/openapi_v1.yaml

@@ -849,6 +849,7 @@ paths:
         - oauth: []
       parameters:
       - $ref: '#/components/parameters/TaskIdParameter'
+      - $ref: '#/components/parameters/WorkerNameParameter'
       responses:
         201:
           description: Task Created
@@ -1620,6 +1621,12 @@ components:
       required: true
       schema:
         type: string
+    WorkerNameParameter:
+      in: query
+      name: worker_name
+      required: true
+      schema:
+        type: string
     StatusParameter:
       in: query
       name: status

dispatcher/backend/src/common/enum.py

@@ -112,6 +112,7 @@ class ScheduleCategory:
     wikivoyage = "wikivoyage"
     wiktionary = "wiktionary"
     ifixit = "ifixit"
+    freecodecamp = "freecodecamp"
 
     @classmethod
     def all(cls):
@@ -135,6 +136,7 @@ def all(cls):
             cls.wikivoyage,
             cls.wiktionary,
             cls.ifixit,
+            cls.freecodecamp,
         ]
 
     @classmethod
@@ -165,6 +167,7 @@ class DockerImageName:
     kolibri = "openzim/kolibri"
     wikihow = "openzim/wikihow"
     ifixit = "openzim/ifixit"
+    freecodecamp = "openzim/freecodecamp"
 
     @classmethod
     def all(cls) -> set:
@@ -181,6 +184,7 @@ def all(cls) -> set:
             cls.kolibri,
             cls.wikihow,
             cls.ifixit,
+            cls.freecodecamp,
         }
 
 
@@ -197,6 +201,7 @@ class Offliner:
     kolibri = "kolibri"
     wikihow = "wikihow"
     ifixit = "ifixit"
+    freecodecamp = "freecodecamp"
 
     @classmethod
     def all(cls):
@@ -213,6 +218,7 @@ def all(cls):
             cls.kolibri,
             cls.wikihow,
             cls.ifixit,
+            cls.freecodecamp,
         ]
 
     @classmethod
@@ -236,6 +242,7 @@ def get_image_name(cls, offliner):
             cls.kolibri: DockerImageName.kolibri,
             cls.wikihow: DockerImageName.wikihow,
             cls.ifixit: DockerImageName.ifixit,
+            cls.freecodecamp: DockerImageName.freecodecamp,
         }.get(offliner, "-")
 
 

dispatcher/backend/src/common/schemas/models.py

@@ -20,6 +20,7 @@
     validate_warehouse_path,
 )
 from common.schemas.offliners import (
+    FreeCodeCampFlagsSchema,
     GutenbergFlagsSchema,
     IFixitFlagsSchema,
     KolibriFlagsSchema,
@@ -87,6 +88,7 @@ def get_offliner_schema(offliner):
             Offliner.kolibri: KolibriFlagsSchema,
             Offliner.wikihow: WikihowFlagsSchema,
             Offliner.ifixit: IFixitFlagsSchema,
+            Offliner.freecodecamp: FreeCodeCampFlagsSchema,
         }.get(offliner, Schema)
 
     @validates_schema

dispatcher/backend/src/common/schemas/offliners/__init__.py

@@ -1,4 +1,5 @@
 from common.schemas import SerializableSchema
+from common.schemas.offliners.freecodecamp import FreeCodeCampFlagsSchema
 from common.schemas.offliners.gutenberg import GutenbergFlagsSchema
 from common.schemas.offliners.ifixit import IFixitFlagsSchema
 from common.schemas.offliners.kolibri import KolibriFlagsSchema
@@ -12,6 +13,7 @@
 from common.schemas.offliners.zimit import ZimitFlagsSchema
 
 __all__ = (
+    "FreeCodeCampFlagsSchema",
     "GutenbergFlagsSchema",
     "IFixitFlagsSchema",
     "KolibriFlagsSchema",

dispatcher/backend/src/common/schemas/offliners/freecodecamp.py

@@ -0,0 +1,109 @@
+from marshmallow import fields
+
+from common.schemas import SerializableSchema
+from common.schemas.fields import (
+    validate_output,
+    validate_zim_description,
+    validate_zim_filename,
+    validate_zim_longdescription,
+)
+
+
+class FreeCodeCampFlagsSchema(SerializableSchema):
+    class Meta:
+        ordered = True
+
+    course = fields.String(
+        metadata={
+            "label": "Course(s)",
+            "description": "Course or course list (separated by commas)",
+        },
+        required=True,
+    )
+
+    language = fields.String(
+        metadata={
+            "label": "Language",
+            "description": "Language of zim file and curriculum. Either (without "
+            "quotes) 'ara' (arabic), 'cmn' (chinese), 'lzh' (chinese-traditional), "
+            "'eng' (english), 'spa' (espanol), 'deu' (german), 'ita' (italian), "
+            "'jpn' (japanese), 'por' (portuguese), 'ukr' (ukranian).",
+        },
+        required=True,
+    )
+
+    name = fields.String(
+        metadata={
+            "label": "Name",
+            "description": "ZIM name",
+        },
+        required=True,
+    )
+
+    title = fields.String(
+        metadata={
+            "label": "Title",
+            "description": "ZIM title",
+        },
+        required=True,
+    )
+
+    description = fields.String(
+        metadata={
+            "label": "Description",
+            "description": "Description for your ZIM",
+        },
+        required=True,
+        validate=validate_zim_description,
+    )
+
+    long_description = fields.String(
+        metadata={
+            "label": "Long description",
+            "description": "Optional long description for your ZIM",
+        },
+        validate=validate_zim_longdescription,
+        data_key="long-description",
+    )
+
+    creator = fields.String(
+        metadata={
+            "label": "Content Creator",
+            "description": "Name of content creator. “freeCodeCamp” otherwise",
+        }
+    )
+
+    publisher = fields.String(
+        metadata={
+            "label": "Publisher",
+            "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise",
+        }
+    )
+
+    debug = fields.Boolean(
+        truthy=[True],
+        falsy=[False],
+        metadata={"label": "Debug", "description": "Enable verbose output"},
+    )
+
+    output_dir = fields.String(
+        metadata={
+            "label": "Output folder",
+            "placeholder": "/output",
+            "description": "Output folder for ZIM file(s). Leave it as `/output`",
+        },
+        load_default="/output",
+        dump_default="/output",
+        data_key="output-dir",
+        validate=validate_output,
+    )
+
+    zim_file = fields.String(
+        metadata={
+            "label": "ZIM filename",
+            "description": "ZIM file name (based on --name if not provided). "
+            "Include {period} to insert date period dynamically",
+        },
+        data_key="zim-file",
+        validate=validate_zim_filename,
+    )

dispatcher/backend/src/tests/integration/routes/schedules/test_freecodecamp.py

@@ -0,0 +1,52 @@
+class TestFreeCodeCamp:
+    def test_create_freecodecamp_schedule(
+        self, client, access_token, garbage_collector
+    ):
+        schedule = {
+            "name": "fcc_javascript_test",
+            "category": "freecodecamp",
+            "enabled": False,
+            "tags": [],
+            "language": {"code": "fr", "name_en": "French", "name_native": "Français"},
+            "config": {
+                "task_name": "freecodecamp",
+                "warehouse_path": "/freecodecamp",
+                "image": {"name": "openzim/freecodecamp", "tag": "1.0.0"},
+                "monitor": False,
+                "platform": None,
+                "flags": {},
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        assert response.status_code == 201
+        response_data = response.get_json()
+        garbage_collector.add_schedule_id(response_data["_id"])
+
+        patch_data = {
+            "enabled": True,
+            "flags": {
+                "output-dir": "/output",
+                "course": (
+                    "regular-expressions,basic-javascript,basic-data-structures,"
+                    "debugging,functional-programming,object-oriented-programming,"
+                    "basic-algorithm-scripting,intermediate-algorithm-scripting,"
+                    "javascript-algorithms-and-data-structures-projects"
+                ),
+                "language": "eng",
+                "name": "fcc_en_javascript",
+                "title": "freeCodeCamp Javascript",
+                "description": "FCC Javascript Courses",
+            },
+        }
+
+        url = f"/schedules/{schedule['name']}"
+        response = client.patch(
+            url, json=patch_data, headers={"Authorization": access_token}
+        )
+        assert response.status_code == 204

dispatcher/backend/src/utils/offliners.py

@@ -12,6 +12,7 @@
 
 od = collections.namedtuple("OfflinerDef", ["cmd", "std_output", "std_stats"])
 OFFLINER_DEFS = {
+    Offliner.freecodecamp: od("fcc2zim", True, False),
     Offliner.gutenberg: od("gutenberg2zim", False, False),
     Offliner.sotoki: od("sotoki", True, True),
     Offliner.wikihow: od("wikihow2zim", True, True),

dispatcher/frontend-ui/src/constants.js

@@ -322,17 +322,17 @@ export default {
   cancelable_statuses: cancelable_statuses,
   running_statuses: running_statuses,
   contact_email: "[email protected]",
-  categories: ["gutenberg", "ifixit", "other", "phet", "psiram", "stack_exchange",
+  categories: ["freecodecamp", "gutenberg", "ifixit", "other", "phet", "psiram", "stack_exchange",
                "ted", "openedx", "vikidia", "wikibooks", "wikihow", "wikinews",
                "wikipedia", "wikiquote", "wikisource", "wikispecies", "wikiversity",
                "wikivoyage", "wiktionary"],  // list of categories for fileering
-  warehouse_paths: ["/gutenberg", "/ifixit", "/other", "/phet", "/psiram", "/stack_exchange",
+  warehouse_paths: ["/freecodecamp", "/gutenberg", "/ifixit", "/other", "/phet", "/psiram", "/stack_exchange",
                     "/ted", "/mooc", "/videos", "/vikidia", "/wikibooks", "/wikihow",
                     "/wikinews", "/wikipedia", "/wikiquote", "/wikisource",
                     "/wikiversity", "/wikivoyage", "/wiktionary", "/zimit",
                     "/.hidden/dev", "/.hidden/private", "/.hidden/endless",
                     "/.hidden/bard", "/.hidden/bsf", "/.hidden/custom_apps"],
-  offliners: ["mwoffliner", "youtube", "phet", "gutenberg", "sotoki", "nautilus", "ted", "openedx", "zimit", "kolibri", "wikihow", "ifixit"],
+  offliners: ["mwoffliner", "youtube", "phet", "gutenberg", "sotoki", "nautilus", "ted", "openedx", "zimit", "kolibri", "wikihow", "ifixit", "freecodecamp"],
   periodicities: ["manually", "monthly", "quarterly", "biannualy", "annually"],
   memory_values: [536870912, // 512MiB
                   1073741824,  // 1GiB