[sc-29393] MongoDB crawler ignore system collections

metaphor/mongodb/README.md

@@ -15,7 +15,7 @@ infer_schema_sample_size: <int> # Number of documents to sample in a collection
 excluded_databases: # Databases to ignore. By default the databases "admin", "config", "local", "system" are excluded.
   - db1
   - db2
-excluded_collections: # Collections to ignore.
+excluded_collections: # Collections to ignore. By default the system specific collections are ignored, see https://www.mongodb.com/docs/manual/reference/system-collections/#database-specific-collections.
   - coll1
   - coll2
 ```

metaphor/mongodb/config.py

@@ -19,7 +19,11 @@ class MongoDBConfig(BaseConfig):
     excluded_databases: Set[str] = Field(
         default_factory=lambda: set(["admin", "config", "local", "system"])
     )
-    excluded_collections: Set[str] = Field(default_factory=set)
+    excluded_collections: Set[str] = Field(
+        default_factory=lambda: set(
+            ["system.buckets", "system.profile", "system.js", "system.views"]
+        )
+    )
 
     @field_validator("auth_mechanism", mode="before")
     def _validate_auth_mechanism(cls, auth_mechanism: str):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.126"
+version = "0.14.127"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]