diff --git a/mongo/changelog.d/19244.fixed b/mongo/changelog.d/19244.fixed new file mode 100644 index 0000000000000..120bd5e2976f1 --- /dev/null +++ b/mongo/changelog.d/19244.fixed @@ -0,0 +1 @@ +Skip unauthorized `local` database collections `system.replset`, `replset.election`, and `replset.minvalid` in collection and index stats gathering to avoid permission errors. diff --git a/mongo/datadog_checks/mongo/collectors/base.py b/mongo/datadog_checks/mongo/collectors/base.py index 1e957a812bada..7b7e5b5d139d8 100644 --- a/mongo/datadog_checks/mongo/collectors/base.py +++ b/mongo/datadog_checks/mongo/collectors/base.py @@ -27,6 +27,9 @@ def __init__(self, check, tags): self.metrics_to_collect = self.check.metrics_to_collect self._collection_interval = None self._collector_key = (self.__class__.__name__,) + self._system_collections_skip_stats = { + "local": frozenset(["system.replset", "replset.election", "replset.minvalid"]) + } def collect(self, api): """The main method exposed by the collector classes, needs to be implemented by every subclass. @@ -37,6 +40,15 @@ def compatible_with(self, deployment): """Whether or not this specific collector is compatible with this specific deployment type.""" raise NotImplementedError() + def should_skip_system_collection(self, coll_name): + """Whether or not the collection should be skipped because collStats or indexStats + is not authorized to run on certain system collections. + """ + db_name = getattr(self, "db_name", None) + if not db_name or db_name not in self._system_collections_skip_stats: + return False + return coll_name in self._system_collections_skip_stats[db_name] + def _normalize(self, metric_name, submit_method, prefix=None): """Replace case-sensitive metric name characters, normalize the metric name, prefix and suffix according to its type. diff --git a/mongo/datadog_checks/mongo/collectors/coll_stats.py b/mongo/datadog_checks/mongo/collectors/coll_stats.py index d0f0c971de9ce..5d4cf1836d3ff 100644 --- a/mongo/datadog_checks/mongo/collectors/coll_stats.py +++ b/mongo/datadog_checks/mongo/collectors/coll_stats.py @@ -45,6 +45,10 @@ def _get_collection_stats(self, api, coll_name): def collect(self, api): coll_names = self._get_collections(api) for coll_name in coll_names: + if self.should_skip_system_collection(coll_name): + self.log.debug("Skipping collStats for system collection %s.%s", self.db_name, coll_name) + continue + # Grab the stats from the collection try: collection_stats = self._get_collection_stats(api, coll_name) diff --git a/mongo/datadog_checks/mongo/collectors/index_stats.py b/mongo/datadog_checks/mongo/collectors/index_stats.py index fefe51d3d5389..397467328f7ad 100644 --- a/mongo/datadog_checks/mongo/collectors/index_stats.py +++ b/mongo/datadog_checks/mongo/collectors/index_stats.py @@ -32,6 +32,10 @@ def _get_collections(self, api): def collect(self, api): coll_names = self._get_collections(api) for coll_name in coll_names: + if self.should_skip_system_collection(coll_name): + self.log.debug("Skipping indexStats for system collection %s.%s", self.db_name, coll_name) + continue + try: for stats in api.index_stats(self.db_name, coll_name): idx_name = stats.get('name', 'unknown') diff --git a/mongo/tests/conftest.py b/mongo/tests/conftest.py index f9a075c73197d..409a6d3832ecc 100644 --- a/mongo/tests/conftest.py +++ b/mongo/tests/conftest.py @@ -140,6 +140,7 @@ def instance_integration_autodiscovery(instance_integration): instance["database_autodiscovery"] = { "enabled": True, } + instance.pop("collections", None) return instance diff --git a/mongo/tests/fixtures/$collStats-oplog.rs b/mongo/tests/fixtures/$collStats-oplog.rs index 0e2b6a429cd63..025d1331c5862 100644 --- a/mongo/tests/fixtures/$collStats-oplog.rs +++ b/mongo/tests/fixtures/$collStats-oplog.rs @@ -5,6 +5,24 @@ "localTime": { "$date": "2024-07-01T20:36:49.358Z" }, + "latencyStats": { + "reads": { + "latency": 13165, + "ops": 10 + }, + "writes": { + "latency": 8542, + "ops": 1 + }, + "commands": { + "latency": 0, + "ops": 0 + }, + "transactions": { + "latency": 0, + "ops": 0 + } + }, "storageStats": { "size": 907806, "count": 4341, @@ -196,6 +214,12 @@ "totalIndexSize": 0, "indexSizes": {}, "scaleFactor": 1 + }, + "queryExecStats": { + "collectionScans": { + "total": 81753, + "nonTailable": 81750 + } } } ] \ No newline at end of file diff --git a/mongo/tests/fixtures/list_collection_names-local b/mongo/tests/fixtures/list_collection_names-local new file mode 100644 index 0000000000000..f2ef8afbb9f89 --- /dev/null +++ b/mongo/tests/fixtures/list_collection_names-local @@ -0,0 +1 @@ +["oplog.rs", "replset.minvalid"] \ No newline at end of file diff --git a/mongo/tests/mocked_api.py b/mongo/tests/mocked_api.py index 06776a152acd7..ffb367b1c496c 100644 --- a/mongo/tests/mocked_api.py +++ b/mongo/tests/mocked_api.py @@ -98,7 +98,10 @@ def command(self, command, *args, **_): return json.load(f, object_hook=json_util.object_hook) def list_collection_names(self, session=None, filter=None, comment=None, **kwargs): - with open(os.path.join(HERE, "fixtures", "list_collection_names"), 'r') as f: + filename = f"list_collection_names-{self._db_name}" + if not os.path.exists(os.path.join(HERE, "fixtures", filename)): + filename = "list_collection_names" + with open(os.path.join(HERE, "fixtures", filename), 'r') as f: return json.load(f) def aggregate(self, pipeline, session=None, **kwargs): diff --git a/mongo/tests/results/metrics-collection-autodiscover.json b/mongo/tests/results/metrics-collection-autodiscover.json index 987fc35208f1c..cbf678fa293ec 100644 --- a/mongo/tests/results/metrics-collection-autodiscover.json +++ b/mongo/tests/results/metrics-collection-autodiscover.json @@ -926,61 +926,31 @@ { "name": "mongodb.collection.size", "type": 0, - "value": 5670.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.size", - "type": 0, - "value": 2600.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.avgobjsize", - "type": 0, - "value": 27.0, + "value": 907806.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.avgobjsize", "type": 0, - "value": 26.0, + "value": 209.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.count", "type": 0, - "value": 210.0, + "value": 4341.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.count", - "type": 0, - "value": 100.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -990,17 +960,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.capped", - "type": 0, - "value": 1.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1010,129 +970,47 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.max", - "type": 0, - "value": 10.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.maxsize", "type": 0, - "value": 10.0, + "value": 16777216.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.maxsize", - "type": 0, - "value": 10.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.storagesize", "type": 0, - "value": 16384.0, + "value": 196608.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.storagesize", - "type": 0, - "value": 16384.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.nindexes", "type": 0, - "value": 1.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.nindexes", - "type": 0, - "value": 1.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.indexsizes", - "type": 0, - "value": 16384.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo", - "index:_id_" - ] - }, - { - "name": "mongodb.collection.indexsizes", - "type": 0, - "value": 16384.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar", - "index:_id_" - ] - }, - { - "name": "mongodb.collection.totalindexsize", - "type": 0, - "value": 16384.0, + "value": 0.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.totalindexsize", "type": 0, - "value": 16384.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.reads.latency", - "type": 0, - "value": 13165.0, + "value": 0.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { @@ -1142,17 +1020,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.reads.opsps", - "type": 1, - "value": 10.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { @@ -1162,7 +1030,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1172,17 +1040,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.writes.latency", - "type": 0, - "value": 8542.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1192,17 +1050,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.writes.opsps", - "type": 1, - "value": 1.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1212,27 +1060,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.commands.latency", - "type": 0, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.commands.opsps", - "type": 1, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { @@ -1242,7 +1070,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1252,17 +1080,7 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.transactions.latency", - "type": 0, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { @@ -1272,97 +1090,47 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.transactions.opsps", - "type": 1, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.collectionscans.nontailable", - "type": 0, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.collectionscans.nontailable", "type": 0, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.collectionscans.total", - "type": 0, - "value": 0.0, + "value": 81750.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.collectionscans.total", "type": 0, - "value": 0.0, + "value": 81753.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.collectionscans.nontailableps", "type": 1, - "value": 0.0, + "value": 81750.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:foo" - ] - }, - { - "name": "mongodb.collection.collectionscans.nontailableps", - "type": 1, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:bar" - ] - }, - { - "name": "mongodb.collection.collectionscans.totalps", - "type": 1, - "value": 0.0, - "tags": [ - "server:mongodb://testUser2:*****@localhost:27017/test", - "db:local", - "collection:foo" + "collection:oplog.rs" ] }, { "name": "mongodb.collection.collectionscans.totalps", "type": 1, - "value": 0.0, + "value": 81753.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "db:local", - "collection:bar" + "collection:oplog.rs" ] }, {