From 75fb9fae989fb4d5fdd3e40348899ed2c4fdb285 Mon Sep 17 00:00:00 2001 From: Abdullah Darwish Date: Wed, 31 Aug 2022 04:08:05 +0200 Subject: [PATCH] MongoDB Adapter Batch_Create and Pagination in find() (#122) * mongodb batch_create * try to fix tests * fixed tests and pushed pipfile * fixed tests * handle pagination in find * resolve comments --- Pipfile.lock | 27 +++++++++++--- syngenta_digital_dta/mongo/adapter.py | 12 +++++- tests/syngenta_digital_dta/mongo/mock_data.py | 37 +++++++++++++++++++ .../syngenta_digital_dta/mongo/test_mongo.py | 35 ++++++++++++++---- 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 1ecb53f..0702e7a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -26,19 +26,19 @@ }, "boto3": { "hashes": [ - "sha256:b72496c7eaa45afbdfa48a7c648c3211342582d91c8c1b7330d09c18242132d1", - "sha256:ec1aa3f4c2b68da1a9c01e175086f5f6b1b8b67780fa569ab8875be5bb3fd5ae" + "sha256:818a40b82e4f66b4bdd4fa38fcc3ed0fb26542f7d8c4d15279d4ba1d4762cd95", + "sha256:84b962f18506ad495dc973aeb5168697ee882497c01430dddeba5e8339db7932" ], "index": "pypi", - "version": "==1.24.61" + "version": "==1.24.62" }, "botocore": { "hashes": [ - "sha256:535c8e97ed28a38fd09dd8f4735195e761bbee54e4c6021f3a709a97b1287dd6", - "sha256:99012965e2409665c7d86706862c5a141e01e1c4d2c81cb9409a44200ee59631" + "sha256:69682c874dc8ed1856bffd203786c9591fb76a1946d3dcc516bda1ee6a6989f3", + "sha256:8563c7d8b80e8041667cf35b397f1c399537f6c887e1c501f0064bfd7ae541ba" ], "markers": "python_version >= '3.7'", - "version": "==1.27.61" + "version": "==1.27.62" }, "certifi": { "hashes": [ @@ -394,6 +394,13 @@ "markers": "python_full_version >= '3.7.2'", "version": "==2.12.5" }, + "atomicwrites": { + "hashes": [ + "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11" + ], + "markers": "sys_platform == 'win32'", + "version": "==1.4.1" + }, "attrs": { "hashes": [ "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", @@ -410,6 +417,14 @@ "index": "pypi", "version": "==1.7.0" }, + "colorama": { + "hashes": [ + "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", + "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" + ], + "markers": "sys_platform == 'win32' and sys_platform == 'win32'", + "version": "==0.4.5" + }, "coverage": { "extras": [ "toml" diff --git a/syngenta_digital_dta/mongo/adapter.py b/syngenta_digital_dta/mongo/adapter.py index 264d1b3..01ad25c 100644 --- a/syngenta_digital_dta/mongo/adapter.py +++ b/syngenta_digital_dta/mongo/adapter.py @@ -46,6 +46,16 @@ def create(self, **kwargs): super().publish('create', data, **kwargs) return data + def batch_create(self, **kwargs): + items = [] + for item in kwargs['data']: + item = schema_mapper.map_to_schema(item, self.model_schema_file, self.model_schema) + item['_id'] = item[self.model_identifier] + items.append(item) + self.connection.insert_many(items) + super().publish('batch_create', items, **kwargs) + return items + def read(self, **kwargs): if kwargs.get('operation') == 'query': return self.find(**kwargs) @@ -61,7 +71,7 @@ def find_one(self, **kwargs): return self.connection.find_one(kwargs['query']) def find(self, **kwargs): - results = self.connection.find(kwargs['query']) + results = self.connection.find(kwargs['query'], **kwargs.get('params', {})) return list(results) def update(self, **kwargs): diff --git a/tests/syngenta_digital_dta/mongo/mock_data.py b/tests/syngenta_digital_dta/mongo/mock_data.py index 307e0d4..7c110d6 100644 --- a/tests/syngenta_digital_dta/mongo/mock_data.py +++ b/tests/syngenta_digital_dta/mongo/mock_data.py @@ -1,5 +1,6 @@ import uuid + def get_standard(): return { 'test_id': str(uuid.uuid4()), @@ -17,3 +18,39 @@ def get_standard(): 'created': '2020-10-05', 'modified': '2020-10-05' } + + +def get_items(): + return [{ + 'test_id': str(uuid.uuid4()), + 'test_query_id': str(uuid.uuid4()), + 'object_key': { + 'string_key': 'nothing' + }, + 'array_number': [1, 2, 3], + 'array_objects': [ + { + 'array_string_key': 'a', + 'array_number_key': 1 + } + ], + 'created': '2020-10-05', + 'modified': '2020-10-05' + }, + { + 'test_id': str(uuid.uuid4()), + 'test_query_id': str(uuid.uuid4()), + 'object_key': { + 'string_key': 'nothing' + }, + 'array_number': [1, 2, 3], + 'array_objects': [ + { + 'array_string_key': 'a', + 'array_number_key': 1 + } + ], + 'created': '2020-10-05', + 'modified': '2020-10-05' + } + ] diff --git a/tests/syngenta_digital_dta/mongo/test_mongo.py b/tests/syngenta_digital_dta/mongo/test_mongo.py index 0d0c5be..bf70501 100644 --- a/tests/syngenta_digital_dta/mongo/test_mongo.py +++ b/tests/syngenta_digital_dta/mongo/test_mongo.py @@ -35,6 +35,15 @@ def test_create_succeed(self): self.assertDictEqual(result, data) self.adapter.delete(query={'test_id': result['test_id']}) + def test_batch_create_succeed(self): + data = mock_data.get_items() + result = self.adapter.batch_create(data=data) + for item in result: + item.pop('_id') + self.assertListEqual(result, data) + for item in result: + self.adapter.delete(query={'test_id': item['test_id']}) + def test_create_fail_non_unique(self): data = mock_data.get_standard() data['test_id'] = 'fail-non-unique' @@ -44,7 +53,7 @@ def test_create_fail_non_unique(self): self.assertTrue(False) except: self.assertTrue(True) - self.adapter.delete(query={'test_id': data['test_id']}) # clean up + self.adapter.delete(query={'test_id': data['test_id']}) # clean up def test_read(self): data = mock_data.get_standard() @@ -52,14 +61,14 @@ def test_read(self): result = self.adapter.read(query={'test_id': data['test_id']}) result.pop('_id') self.assertDictEqual(result, data) - self.adapter.delete(query={'test_id': data['test_id']}) # clean up + self.adapter.delete(query={'test_id': data['test_id']}) # clean up def test_query_allowed(self): data = mock_data.get_standard() self.adapter.create(data=data) result = self.adapter.query(operation='find', query={'test_id': data['test_id']}) self.assertTrue(len(list(result)) == 1) - self.adapter.delete(query={'test_id': data['test_id']}) # clean up + self.adapter.delete(query={'test_id': data['test_id']}) # clean up def test_not_query_allowed(self): data = mock_data.get_standard() @@ -79,16 +88,28 @@ def test_read_many(self): count += 1 results = self.adapter.read(query={'test_query_id': 'some-query'}, operation='query') for result in results: - self.adapter.delete(query={'test_id': result['test_id']}) # clean up + self.adapter.delete(query={'test_id': result['test_id']}) # clean up self.assertTrue(len(results) >= 3) + def test_read_many_pagination(self): + count = 0 + while count < 10: + data = mock_data.get_standard() + data['test_query_id'] = 'some-query' + self.adapter.create(data=data) + count += 1 + results = self.adapter.read(query={'test_query_id': 'some-query'}, operation='query', params={'skip': 5, 'limit': 5}) + for result in results: + self.adapter.delete(query={'test_id': result['test_id']}) # clean up + self.assertEqual(len(results), 5) + def test_update_success(self): data = mock_data.get_standard() self.adapter.create(data=data) - data['array_number'] = [4,5,6] + data['array_number'] = [4, 5, 6] result = self.adapter.update(query={'test_id': data['test_id']}, data=data, update_list_operation='replace') self.assertDictEqual(result, data) - self.adapter.delete(query={'test_id': data['test_id']}) # clean up + self.adapter.delete(query={'test_id': data['test_id']}) # clean up def test_update_fail(self): data = mock_data.get_standard() @@ -103,7 +124,7 @@ def test_upsert(self): result = self.adapter.upsert(query={'test_id': data['test_id']}, data=data) result.pop('_id') self.assertDictEqual(result, data) - self.adapter.delete(query={'test_id': data['test_id']}) # clean up + self.adapter.delete(query={'test_id': data['test_id']}) # clean up def test_delete(self): data = mock_data.get_standard()