diff --git a/.travis.yml b/.travis.yml index 5d3a4a4..187f95d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,15 @@ env: global: - ELASTIC_2_URL=http://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.3/elasticsearch-2.4.3.tar.gz - ELASTIC_5_URL=http://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.tar.gz + - ELASTIC_6_URL=http://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.5.tar.gz # We need to manually set JAVA_HOME to Java 8 too. - JAVA_HOME=/usr/lib/jvm/java-8-oracle jobs: fast_finish: true include: + - python: &latest_py3 3.6 + env: MONGODB=3.2.11 ELASTIC=6.8.5 ELASTIC_URL=$ELASTIC_6_URL TOXENV=elastic6 - python: &latest_py3 3.6 env: MONGODB=3.2.11 ELASTIC=5.1.1 ELASTIC_URL=$ELASTIC_5_URL TOXENV=elastic5 - python: 3.4 diff --git a/README.rst b/README.rst index e42f524..4b057b7 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,12 @@ For use with an Elasticsearch 5.x server, install with:: .. note:: Version 0.3.0 added support for Elasticsearch 5.x. +Elasticsearch 6.x +----------------- + +For use with an Elasticsearch 6.x server, install with:: + + pip install 'elastic2-doc-manager[elastic6]' Amazon Elasticsearch Service ---------------------------- diff --git a/mongo_connector/doc_managers/elastic2_doc_manager.py b/mongo_connector/doc_managers/elastic2_doc_manager.py index 7a829a5..8114c91 100755 --- a/mongo_connector/doc_managers/elastic2_doc_manager.py +++ b/mongo_connector/doc_managers/elastic2_doc_manager.py @@ -212,8 +212,15 @@ def __init__( self.auto_commit_interval = auto_commit_interval self.auto_send_interval = kwargs.get("autoSendInterval", DEFAULT_SEND_INTERVAL) + + # es6 deprecates support for multiple document types + # using default_type for consistency + # Will try and use multiple doc types only if explicity specified + self.create_multi_type = kwargs.get("createMultiType", False) + self.default_type = kwargs.get("defaultType", "_doc") self.meta_index_name = meta_index_name - self.meta_type = meta_type + self.meta_type = meta_type if self.create_multi_type else self.default_type + self.unique_key = unique_key self.chunk_size = chunk_size self.has_attachment_mapping = False @@ -226,7 +233,7 @@ def __init__( def _index_and_mapping(self, namespace): """Helper method for getting the index and type from a namespace.""" index, doc_type = namespace.split(".", 1) - return index.lower(), doc_type + return index.lower(), (self.default_type, doc_type)[self.create_multi_type] def stop(self): """Stop the auto-commit thread.""" @@ -258,6 +265,7 @@ def handle_command(self, doc, namespace, timestamp): if doc.get("create"): db, coll = self.command_helper.map_collection(db, doc["create"]) + coll = (self.default_type, coll)[self.create_multi_type] if db and coll: self.elastic.indices.put_mapping( index=db.lower(), doc_type=coll, body={"_source": {"enabled": True}} @@ -265,6 +273,7 @@ def handle_command(self, doc, namespace, timestamp): if doc.get("drop"): db, coll = self.command_helper.map_collection(db, doc["drop"]) + coll = (self.default_type, coll)[self.create_multi_type] if db and coll: # This will delete the items in coll, but not get rid of the # mapping. @@ -321,6 +330,7 @@ def update(self, document_id, update_spec, namespace, timestamp): def upsert(self, doc, namespace, timestamp, update_spec=None): """Insert a document into Elasticsearch.""" index, doc_type = self._index_and_mapping(namespace) + # No need to duplicate '_id' in source document doc_id = str(doc.pop("_id")) metadata = {"ns": namespace, "_ts": timestamp} diff --git a/setup.py b/setup.py index 7fb3eea..49f6f6f 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ "aws": ["boto3 >= 1.4.0", "requests-aws-sign >= 0.1.2"], "elastic2": ["elasticsearch>=2.0.0,<3.0.0"], "elastic5": ["elasticsearch>=5.0.0,<6.0.0"], + "elastic6": ["elasticsearch>=6.2.0,<7.0.0"], }, packages=["mongo_connector", "mongo_connector.doc_managers"], classifiers=[ diff --git a/tox.ini b/tox.ini index 68ddd89..8e119e5 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ commands_post = extras = elastic2: elastic2 elastic5: elastic5 + elastic6: elastic6 passenv = # the username to use if running the tests with authentication enabled DB_USER