[sc-29393] MongoDB crawler ignore system collections

metaphor/mongodb/README.md

@@ -15,7 +15,7 @@ infer_schema_sample_size: <int> # Number of documents to sample in a collection
 excluded_databases: # Databases to ignore. By default the databases "admin", "config", "local", "system" are excluded.
   - db1
   - db2
-excluded_collections: # Collections to ignore.
+excluded_collections: # Extra collections to ignore. Note that the system specific collections (`system.views`, `system.profile`, etc.) are always ignored, see https://www.mongodb.com/docs/manual/reference/system-collections/#database-specific-collections for more details.
   - coll1
   - coll2
 ```

metaphor/mongodb/extractor.py

@@ -50,7 +50,13 @@ class MongoDBExtractor(BaseExtractor):
     def __init__(self, config: MongoDBConfig) -> None:
         super().__init__(config)
         self._sample_size = config.infer_schema_sample_size
+
         self._excluded_collections = config.excluded_collections
+        # Always ignore these system collections
+        self._excluded_collections.update(
+            ["system.buckets", "system.profile", "system.js", "system.views"]
+        )
+
         self._excluded_databases = config.excluded_databases
         self.client = config.get_client()
         self._datasets: Dict[str, Dataset] = {}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.126"
+version = "0.14.127"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]