Add support for freecodecamp scraper

openzim · Aug 29, 2023 · 9d81dc9 · 9d81dc9
1 parent 9962928
commit 9d81dc9
Show file tree

Hide file tree

Showing 6 changed files with 121 additions and 3 deletions.
diff --git a/dispatcher/backend/src/common/enum.py b/dispatcher/backend/src/common/enum.py
@@ -112,6 +112,7 @@ class ScheduleCategory:
     wikivoyage = "wikivoyage"
     wiktionary = "wiktionary"
     ifixit = "ifixit"
+    freecodecamp = "freecodecamp"
 
     @classmethod
     def all(cls):
@@ -135,6 +136,7 @@ def all(cls):
             cls.wikivoyage,
             cls.wiktionary,
             cls.ifixit,
+            cls.freecodecamp,
         ]
 
     @classmethod
@@ -165,6 +167,7 @@ class DockerImageName:
     kolibri = "openzim/kolibri"
     wikihow = "openzim/wikihow"
     ifixit = "openzim/ifixit"
+    freecodecamp = "openzim/freecodecamp"
 
     @classmethod
     def all(cls) -> set:
@@ -181,6 +184,7 @@ def all(cls) -> set:
             cls.kolibri,
             cls.wikihow,
             cls.ifixit,
+            cls.freecodecamp,
         }
 
 
@@ -197,6 +201,7 @@ class Offliner:
     kolibri = "kolibri"
     wikihow = "wikihow"
     ifixit = "ifixit"
+    freecodecamp = "freecodecamp"
 
     @classmethod
     def all(cls):
@@ -213,6 +218,7 @@ def all(cls):
             cls.kolibri,
             cls.wikihow,
             cls.ifixit,
+            cls.freecodecamp,
         ]
 
     @classmethod
@@ -236,6 +242,7 @@ def get_image_name(cls, offliner):
             cls.kolibri: DockerImageName.kolibri,
             cls.wikihow: DockerImageName.wikihow,
             cls.ifixit: DockerImageName.ifixit,
+            cls.freecodecamp: DockerImageName.freecodecamp,
         }.get(offliner, "-")
 
 

diff --git a/dispatcher/backend/src/common/schemas/models.py b/dispatcher/backend/src/common/schemas/models.py
@@ -20,6 +20,7 @@
     validate_warehouse_path,
 )
 from common.schemas.offliners import (
+    FreeCodeCampFlagsSchema,
     GutenbergFlagsSchema,
     IFixitFlagsSchema,
     KolibriFlagsSchema,
@@ -87,6 +88,7 @@ def get_offliner_schema(offliner):
             Offliner.kolibri: KolibriFlagsSchema,
             Offliner.wikihow: WikihowFlagsSchema,
             Offliner.ifixit: IFixitFlagsSchema,
+            Offliner.freecodecamp: FreeCodeCampFlagsSchema,
         }.get(offliner, Schema)
 
     @validates_schema

diff --git a/dispatcher/backend/src/common/schemas/offliners/__init__.py b/dispatcher/backend/src/common/schemas/offliners/__init__.py
@@ -1,4 +1,5 @@
 from common.schemas import SerializableSchema
+from common.schemas.offliners.freecodecamp import FreeCodeCampFlagsSchema
 from common.schemas.offliners.gutenberg import GutenbergFlagsSchema
 from common.schemas.offliners.ifixit import IFixitFlagsSchema
 from common.schemas.offliners.kolibri import KolibriFlagsSchema
@@ -12,6 +13,7 @@
 from common.schemas.offliners.zimit import ZimitFlagsSchema
 
 __all__ = (
+    "FreeCodeCampFlagsSchema",
     "GutenbergFlagsSchema",
     "IFixitFlagsSchema",
     "KolibriFlagsSchema",

diff --git a/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py b/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py
@@ -0,0 +1,106 @@
+from marshmallow import fields
+
+from common.schemas import SerializableSchema
+from common.schemas.fields import (
+    validate_output,
+    validate_zim_description,
+    validate_zim_filename,
+    validate_zim_longdescription,
+)
+
+
+class FreeCodeCampFlagsSchema(SerializableSchema):
+    class Meta:
+        ordered = True
+
+    course = fields.String(
+        metadata={
+            "label": "Course(s)",
+            "description": "Course or course list (separated by commas)",
+        },
+        required=True,
+    )
+
+    language = fields.String(
+        metadata={
+            "label": "Language",
+            "description": "Language of zim file and curriculum",
+        },
+        required=True,
+    )
+
+    name = fields.String(
+        metadata={
+            "label": "Name",
+            "description": "ZIM name",
+        },
+        required=True,
+    )
+
+    title = fields.String(
+        metadata={
+            "label": "Title",
+            "description": "ZIM title",
+        },
+        required=True,
+    )
+
+    description = fields.String(
+        metadata={
+            "label": "Description",
+            "description": "Description for your ZIM",
+        },
+        required=True,
+        validate=validate_zim_description,
+    )
+
+    long_description = fields.String(
+        metadata={
+            "label": "Long description",
+            "description": "Optional long description for your ZIM",
+        },
+        validate=validate_zim_longdescription,
+        data_key="long-description",
+    )
+
+    creator = fields.String(
+        metadata={
+            "label": "Content Creator",
+            "description": "Name of content creator. “freeCodeCamp” otherwise",
+        }
+    )
+
+    publisher = fields.String(
+        metadata={
+            "label": "Publisher",
+            "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise",
+        }
+    )
+
+    debug = fields.Boolean(
+        truthy=[True],
+        falsy=[False],
+        metadata={"label": "Debug", "description": "Enable verbose output"},
+    )
+
+    output_dir = fields.String(
+        metadata={
+            "label": "Output folder",
+            "placeholder": "/output",
+            "description": "Output folder for ZIM file(s). Leave it as `/output`",
+        },
+        load_default="/output",
+        dump_default="/output",
+        data_key="output-dir",
+        validate=validate_output,
+    )
+
+    zim_file = fields.String(
+        metadata={
+            "label": "ZIM filename",
+            "description": "ZIM file name (based on --name if not provided). "
+            "Include {period} to insert date period dynamically",
+        },
+        data_key="zim-file",
+        validate=validate_zim_filename,
+    )
diff --git a/dispatcher/backend/src/utils/offliners.py b/dispatcher/backend/src/utils/offliners.py
@@ -12,6 +12,7 @@
 
 od = collections.namedtuple("OfflinerDef", ["cmd", "std_output", "std_stats"])
 OFFLINER_DEFS = {
+    Offliner.freecodecamp: od("fcc2zim", "output-dir", False),
     Offliner.gutenberg: od("gutenberg2zim", False, False),
     Offliner.sotoki: od("sotoki", True, True),
     Offliner.wikihow: od("wikihow2zim", True, True),

diff --git a/dispatcher/frontend-ui/src/constants.js b/dispatcher/frontend-ui/src/constants.js
@@ -322,17 +322,17 @@ export default {
   cancelable_statuses: cancelable_statuses,
   running_statuses: running_statuses,
   contact_email: "[email protected]",
-  categories: ["gutenberg", "ifixit", "other", "phet", "psiram", "stack_exchange",
+  categories: ["freecodecamp", "gutenberg", "ifixit", "other", "phet", "psiram", "stack_exchange",
                "ted", "openedx", "vikidia", "wikibooks", "wikihow", "wikinews",
                "wikipedia", "wikiquote", "wikisource", "wikispecies", "wikiversity",
                "wikivoyage", "wiktionary"],  // list of categories for fileering
-  warehouse_paths: ["/gutenberg", "/ifixit", "/other", "/phet", "/psiram", "/stack_exchange",
+  warehouse_paths: ["/freecodecamp", "/gutenberg", "/ifixit", "/other", "/phet", "/psiram", "/stack_exchange",
                     "/ted", "/mooc", "/videos", "/vikidia", "/wikibooks", "/wikihow",
                     "/wikinews", "/wikipedia", "/wikiquote", "/wikisource",
                     "/wikiversity", "/wikivoyage", "/wiktionary", "/zimit",
                     "/.hidden/dev", "/.hidden/private", "/.hidden/endless",
                     "/.hidden/bard", "/.hidden/bsf", "/.hidden/custom_apps"],
-  offliners: ["mwoffliner", "youtube", "phet", "gutenberg", "sotoki", "nautilus", "ted", "openedx", "zimit", "kolibri", "wikihow", "ifixit"],
+  offliners: ["mwoffliner", "youtube", "phet", "gutenberg", "sotoki", "nautilus", "ted", "openedx", "zimit", "kolibri", "wikihow", "ifixit", "freecodecamp"],
   periodicities: ["manually", "monthly", "quarterly", "biannualy", "annually"],
   memory_values: [536870912, // 512MiB
                   1073741824,  // 1GiB