diff --git a/metis-dataset-cleaner/.gitignore b/metis-dataset-cleaner/.gitignore new file mode 100644 index 0000000..651b5c8 --- /dev/null +++ b/metis-dataset-cleaner/.gitignore @@ -0,0 +1,3 @@ +##Add to ignore to not commit by mistake +/src/main/resources/*.properties + diff --git a/metis-dataset-cleaner/README.MD b/metis-dataset-cleaner/README.MD new file mode 100644 index 0000000..81dd62f --- /dev/null +++ b/metis-dataset-cleaner/README.MD @@ -0,0 +1,3 @@ +**Tool to clean dataset by id on demand** +This project contains functionality to remove a dataset by id. +It is required to avoid automatically initializing indices generation, which could block read operations on a live environment. diff --git a/metis-dataset-cleaner/pom.xml b/metis-dataset-cleaner/pom.xml new file mode 100644 index 0000000..2e3808f --- /dev/null +++ b/metis-dataset-cleaner/pom.xml @@ -0,0 +1,123 @@ + + + 4.0.0 + + eu.europeana.metis + metis-dataset-cleaner + 1.0-SNAPSHOT + jar + metis-dataset-cleaner + + + UTF-8 + 3.8.1 + 3.2.3 + 12-SNAPSHOT + 2.16.8 + + 1.7.30 + 2.17.1 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-starter-log4j2 + + + org.apache.logging.log4j + log4j-slf4j-impl + ${version.log4j} + + + eu.europeana.metis + metis-core-service + ${version.metis} + + + eu.europeana.metis + metis-indexing + ${version.metis} + + + eu.europeana.corelib + corelib-storage + ${version.corelib} + + + + + + + org.springframework.boot + spring-boot-dependencies + ${version.spring.boot} + pom + import + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${version.maven.compiler.plugin} + + 21 + + + + org.springframework.boot + spring-boot-maven-plugin + ${version.spring.boot} + + + + repackage + + + + + + + + + + + libs-release + libs-release + https://artifactory.eanadev.org/artifactory/libs-release + + true + + + false + + + + libs-snapshot + libs-snapshot + https://artifactory.eanadev.org/artifactory/libs-snapshot + + false + + + true + + + + diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/ApplicationInitializer.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/ApplicationInitializer.java new file mode 100644 index 0000000..c3e4146 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/ApplicationInitializer.java @@ -0,0 +1,26 @@ +package eu.europeana.metis.cleaner; + +import eu.europeana.metis.cleaner.common.PropertyFileLoader; +import eu.europeana.metis.cleaner.utilities.IndexWrapper; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ApplicationInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationInitializer.class); + private static final Properties indexingProperties = new Properties(); + private final IndexWrapper indexWrapper; + + public ApplicationInitializer() { + PropertyFileLoader.loadPropertyFile("", + "application.properties", + indexingProperties); + LOGGER.info("Indexing properties loaded."); + indexWrapper = IndexWrapper.getInstance(indexingProperties); + } + + public IndexWrapper getIndexWrapper() { + return indexWrapper; + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/MetisDatasetCleaner.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/MetisDatasetCleaner.java new file mode 100644 index 0000000..f0241eb --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/MetisDatasetCleaner.java @@ -0,0 +1,112 @@ +package eu.europeana.metis.cleaner; + +import eu.europeana.indexing.IndexingProperties; +import eu.europeana.metis.cleaner.common.TargetIndexingDatabase; +import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MetisDatasetCleaner implements ApplicationRunner { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetisDatasetCleaner.class); + + public static void main(String[] args) { + SpringApplication.run(MetisDatasetCleaner.class, args); + } + + @Override + public void run(ApplicationArguments args) throws Exception { + LOGGER.info("Starting cleaning database script"); + ApplicationInitializer applicationInitializer = new ApplicationInitializer(); + // Usage help + LOGGER.info("Usage: where # is record or dataset id number."); + LOGGER.info(" where X is path to a rdf xml record ready for preview and publish."); + LOGGER.info( + "mvn spring-boot:run -Dspring-boot.run.arguments=\"--record.id=#\" or \"--dataset.id=#\" or --index.file=\"X\" or --index.files=\"Y,Z\""); + LOGGER.info("java -jar metis-dataset-cleaner-1.0-SNAPSHOT.jar --record.id=# or --dataset.id=#"); + LOGGER.info("java -jar metis-dataset-cleaner-1.0-SNAPSHOT.jar --index.file=\"X\" or --index.files=\"Y,Z\""); + + // Reading command-Line arguments + LOGGER.info("Application started with command-line arguments: {}", Arrays.toString(args.getSourceArgs())); + LOGGER.info("NonOptionArgs: {}", args.getNonOptionArgs()); + LOGGER.info("OptionNames: {}", args.getOptionNames()); + + for (String name : args.getOptionNames()) { + LOGGER.info("arg-{}={}", name, args.getOptionValues(name)); + } + + final boolean containsRecordId = args.containsOption("record.id"); + if (containsRecordId) { + LOGGER.info("::Contains record.id::"); + final String recordId = args.getOptionValues("record.id").getFirst(); + LOGGER.info("cleaning preview record.id: {}", recordId); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PREVIEW).remove(recordId); + LOGGER.info("cleaning publish record.id: {}", recordId); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PUBLISH).remove(recordId); + } + + final boolean containsDatasetId = args.containsOption("dataset.id"); + if (containsDatasetId) { + LOGGER.info("::Contains dataset.id::"); + final String datasetId = args.getOptionValues("dataset.id").getFirst(); + LOGGER.info("cleaning preview dataset.id: {}", datasetId); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PREVIEW) + .removeAll(datasetId, Date.from(Instant.now())); + LOGGER.info("cleaning publish dataset.id: {}", datasetId); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PUBLISH) + .removeAll(datasetId, Date.from(Instant.now())); + } + + final boolean containsRecordFile = args.containsOption("index.file"); + if (containsRecordFile) { + LOGGER.info("::Contains index.file::"); + final String fileName = args.getOptionValues("index.file").getFirst(); + try (FileInputStream fileInputStream = new FileInputStream(fileName)) { + final String fileData = new String(fileInputStream.readAllBytes(), StandardCharsets.UTF_8); + IndexingProperties indexingProperties = new IndexingProperties(Date.from(Instant.now()), true, null, true, true); + LOGGER.info("contents: {}", fileData); + LOGGER.info("indexing preview file: {}", fileName); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PREVIEW) + .index(fileData, indexingProperties); + LOGGER.info("indexing publish file: {}", fileName); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PUBLISH) + .index(fileData, indexingProperties); + } + } + + final boolean containsRecordFiles = args.containsOption("index.files"); + if (containsRecordFiles) { + LOGGER.info("::Contains index.files::"); + final String fileNames = args.getOptionValues("index.files").getFirst(); + List recordList = new ArrayList<>(fileNames.split(",").length); + for (String fName : fileNames.split(",")) { + try (FileInputStream fileInputStream = new FileInputStream(fName)) { + final String fileData = new String(fileInputStream.readAllBytes(), StandardCharsets.UTF_8); + LOGGER.info("contents: {}", fileData); + recordList.add(fileData); + } + } + IndexingProperties indexingProperties = new IndexingProperties(Date.from(Instant.now()), true, null, true, true); + LOGGER.info("indexing preview record list"); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PREVIEW) + .index(recordList, indexingProperties); + LOGGER.info("indexing publish record list"); + applicationInitializer.getIndexWrapper().getIndexer(TargetIndexingDatabase.PUBLISH) + .index(recordList, indexingProperties); + } + + LOGGER.info("Finished cleaning database script"); + System.exit(0); + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/IndexingProperties.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/IndexingProperties.java new file mode 100644 index 0000000..44c704a --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/IndexingProperties.java @@ -0,0 +1,173 @@ +package eu.europeana.metis.cleaner.common; + +public class IndexingProperties { + + private String mongoInstances; + private int mongoPortNumber; + private String mongoDbName; + private String mongoRedirectsDbName; + private String mongoUsername; + private String mongoPassword; + private String mongoAuthDb; + private String mongoUseSSL; + private String mongoReadPreference; + private String mongoApplicationName; + private int mongoPoolSize; + + private String solrInstances; + + private String zookeeperInstances; + private int zookeeperPortNumber; + private String zookeeperChroot; + private String zookeeperDefaultCollection; + + @Override + public String toString() { + return "IndexingProperties{" + + "mongoInstances='" + mongoInstances + '\'' + + ", mongoPortNumber=" + mongoPortNumber + + ", mongoDbName='" + mongoDbName + '\'' + + ", mongoRedirectsDbName='" + mongoRedirectsDbName + '\'' + + ", mongoUsername='" + mongoUsername + '\'' + + ", mongoPassword='" + mongoPassword + '\'' + + ", mongoAuthDb='" + mongoAuthDb + '\'' + + ", mongoUseSSL='" + mongoUseSSL + '\'' + + ", mongoReadPreference='" + mongoReadPreference + '\'' + + ", mongoApplicationName='" + mongoApplicationName + '\'' + + ", mongoPoolSize=" + mongoPoolSize + + ", solrInstances='" + solrInstances + '\'' + + ", zookeeperInstances='" + zookeeperInstances + '\'' + + ", zookeeperPortNumber=" + zookeeperPortNumber + + ", zookeeperChroot='" + zookeeperChroot + '\'' + + ", zookeeperDefaultCollection='" + zookeeperDefaultCollection + '\'' + + '}'; + } + + public String getMongoInstances() { + return mongoInstances; + } + + public void setMongoInstances(String mongoInstances) { + this.mongoInstances = mongoInstances; + } + + public int getMongoPortNumber() { + return mongoPortNumber; + } + + public void setMongoPortNumber(int mongoPortNumber) { + this.mongoPortNumber = mongoPortNumber; + } + + public String getMongoDbName() { + return mongoDbName; + } + + public void setMongoDbName(String mongoDbName) { + this.mongoDbName = mongoDbName; + } + + public String getMongoRedirectsDbName() { + return mongoRedirectsDbName; + } + + public void setMongoRedirectsDbName(String mongoRedirectsDbName) { + this.mongoRedirectsDbName = mongoRedirectsDbName; + } + + public String getMongoUsername() { + return mongoUsername; + } + + public void setMongoUsername(String mongoUsername) { + this.mongoUsername = mongoUsername; + } + + public String getMongoPassword() { + return mongoPassword; + } + + public void setMongoPassword(String mongoPassword) { + this.mongoPassword = mongoPassword; + } + + public String getMongoAuthDb() { + return mongoAuthDb; + } + + public void setMongoAuthDb(String mongoAuthDb) { + this.mongoAuthDb = mongoAuthDb; + } + + public String getMongoUseSSL() { + return mongoUseSSL; + } + + public void setMongoUseSSL(String mongoUseSSL) { + this.mongoUseSSL = mongoUseSSL; + } + + public String getMongoReadPreference() { + return mongoReadPreference; + } + + public void setMongoReadPreference(String mongoReadPreference) { + this.mongoReadPreference = mongoReadPreference; + } + + public String getMongoApplicationName() { + return mongoApplicationName; + } + + public void setMongoApplicationName(String mongoApplicationName) { + this.mongoApplicationName = mongoApplicationName; + } + + public int getMongoPoolSize() { + return mongoPoolSize; + } + + public void setMongoPoolSize(int mongoPoolSize) { + this.mongoPoolSize = mongoPoolSize; + } + + public String getSolrInstances() { + return solrInstances; + } + + public void setSolrInstances(String solrInstances) { + this.solrInstances = solrInstances; + } + + public String getZookeeperInstances() { + return zookeeperInstances; + } + + public void setZookeeperInstances(String zookeeperInstances) { + this.zookeeperInstances = zookeeperInstances; + } + + public int getZookeeperPortNumber() { + return zookeeperPortNumber; + } + + public void setZookeeperPortNumber(int zookeeperPortNumber) { + this.zookeeperPortNumber = zookeeperPortNumber; + } + + public String getZookeeperChroot() { + return zookeeperChroot; + } + + public void setZookeeperChroot(String zookeeperChroot) { + this.zookeeperChroot = zookeeperChroot; + } + + public String getZookeeperDefaultCollection() { + return zookeeperDefaultCollection; + } + + public void setZookeeperDefaultCollection(String zookeeperDefaultCollection) { + this.zookeeperDefaultCollection = zookeeperDefaultCollection; + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/PropertyFileLoader.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/PropertyFileLoader.java new file mode 100644 index 0000000..70f93f6 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/PropertyFileLoader.java @@ -0,0 +1,48 @@ +package eu.europeana.metis.cleaner.common; + +import com.google.common.base.Throwables; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for reading eCloud topology properties. + */ +public class PropertyFileLoader { + + private static final Logger LOGGER = LoggerFactory.getLogger(PropertyFileLoader.class); + + public static void loadPropertyFile(String defaultPropertyFilename, String providedPropertyFilename, Properties properties) { + try { + PropertyFileLoader reader = new PropertyFileLoader(); + if (!"".equals(defaultPropertyFilename)) { + reader.loadDefaultPropertyFile(defaultPropertyFilename, properties); + } + if (!"".equals(providedPropertyFilename)) { + reader.loadProvidedPropertyFile(providedPropertyFilename, properties); + } + } catch (IOException e) { + LOGGER.error(Throwables.getStackTraceAsString(e)); + } + } + + void loadDefaultPropertyFile(String defaultPropertyFilename, Properties properties) throws IOException { + InputStream propertiesInputStream = Thread.currentThread().getContextClassLoader() + .getResourceAsStream(defaultPropertyFilename); + if (propertiesInputStream == null) { + throw new FileNotFoundException(defaultPropertyFilename); + } + properties.load(propertiesInputStream); + + } + + void loadProvidedPropertyFile(String fileName, Properties properties) throws IOException { + try (FileInputStream fileInput = new FileInputStream(fileName)) { + properties.load(fileInput); + } + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/TargetIndexingDatabase.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/TargetIndexingDatabase.java new file mode 100644 index 0000000..ad51fad --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/common/TargetIndexingDatabase.java @@ -0,0 +1,23 @@ +package eu.europeana.metis.cleaner.common; + +import java.util.ArrayList; +import java.util.List; + +public enum TargetIndexingDatabase { + PUBLISH, + PREVIEW; + private static List values = initializeTargetIndexingDatabaseValues(); + + public static List getTargetIndexingDatabaseValues() { + return values; + } + + private static List initializeTargetIndexingDatabaseValues() { + List values = new ArrayList<>(TargetIndexingDatabase.values().length); + for (TargetIndexingDatabase targetIndexingDatabase : TargetIndexingDatabase.values()) { + values.add(targetIndexingDatabase.toString()); + } + return values; + } +} + diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapper.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapper.java new file mode 100644 index 0000000..2e7832d --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapper.java @@ -0,0 +1,99 @@ +package eu.europeana.metis.cleaner.utilities; + + +import eu.europeana.metis.cleaner.common.TargetIndexingDatabase; +import eu.europeana.indexing.Indexer; +import eu.europeana.indexing.IndexerFactory; +import eu.europeana.indexing.IndexingSettings; +import eu.europeana.indexing.exception.IndexingException; +import eu.europeana.metis.cleaner.common.IndexingProperties; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.EnumMap; +import java.util.Map; +import java.util.Properties; +import javax.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Wraps operations on the index + */ +public class IndexWrapper { + + private static final Logger LOGGER = LoggerFactory.getLogger( + IndexWrapper.class); + private static IndexWrapper instance; + protected final IndexingProperties previewIndexingProperties; + protected final IndexingProperties publishIndexingProperties; + protected final Map indexers = new EnumMap<>(TargetIndexingDatabase.class); + + public IndexWrapper(IndexingProperties previewIndexingProperties, + IndexingProperties publishIndexingProperties) { + this.previewIndexingProperties = previewIndexingProperties; + this.publishIndexingProperties = publishIndexingProperties; + try { + prepareIndexers(); + } catch (IndexingException | URISyntaxException e) { + throw new IndexWrapperException("Unable to load indexers", e); + } + } + + public IndexWrapper(Properties properties) { + previewIndexingProperties = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(properties, + IndexingType.PREVIEW); + publishIndexingProperties = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(properties, + IndexingType.PUBLISH); + } + + public static synchronized IndexWrapper getInstance(IndexingProperties previewIndexingProperties, + IndexingProperties publishIndexingProperties) { + if (instance == null) { + instance = new IndexWrapper(previewIndexingProperties, publishIndexingProperties); + } + return instance; + } + + public static synchronized IndexWrapper getInstance(Properties indexingPropertyFile) { + if (instance == null) { + IndexingProperties previewIndexingProperties + = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(indexingPropertyFile, IndexingType.PREVIEW); + IndexingProperties publishIndexingProperties + = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(indexingPropertyFile, IndexingType.PUBLISH); + instance = new IndexWrapper(previewIndexingProperties, publishIndexingProperties); + } + return instance; + } + + public Indexer getIndexer(TargetIndexingDatabase targetIndexingDatabase) { + return indexers.get(targetIndexingDatabase); + } + + @PreDestroy + private void close() { + indexers.values().forEach(indexer -> { + try { + indexer.close(); + } catch (IOException e) { + LOGGER.error("Unable to close indexer", e); + } + }); + } + + public enum IndexingType { + PUBLISH, + PREVIEW + } + + protected void prepareIndexers() throws IndexingException, URISyntaxException { + IndexingSettings indexingSettings; + IndexingSettingsGenerator indexingSettingsGenerator; + + indexingSettingsGenerator = new IndexingSettingsGenerator(previewIndexingProperties, publishIndexingProperties); + + indexingSettings = indexingSettingsGenerator.generateForPreview(); + indexers.put(TargetIndexingDatabase.PREVIEW, new IndexerFactory(indexingSettings).getIndexer()); + indexingSettings = indexingSettingsGenerator.generateForPublish(); + indexers.put(TargetIndexingDatabase.PUBLISH, new IndexerFactory(indexingSettings).getIndexer()); + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapperException.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapperException.java new file mode 100644 index 0000000..8ba6832 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexWrapperException.java @@ -0,0 +1,8 @@ +package eu.europeana.metis.cleaner.utilities; + +public class IndexWrapperException extends RuntimeException { + + public IndexWrapperException(String message, Exception e) { + super(message, e); + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertiesTransformer.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertiesTransformer.java new file mode 100644 index 0000000..d49e4f2 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertiesTransformer.java @@ -0,0 +1,64 @@ +package eu.europeana.metis.cleaner.utilities; + + +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.DELIMITER; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_APPLICATION_NAME; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_AUTH_DB; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_DB_NAME; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_INSTANCES; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_POOL_SIZE; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_PORT_NUMBER; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_READ_PREFERENCE; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_REDIRECTS_DB_NAME; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_SECRET; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_USERNAME; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.MONGO_USE_SSL; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.PREVIEW_PREFIX; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.PUBLISH_PREFIX; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.SOLR_INSTANCES; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.ZOOKEEPER_CHROOT; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.ZOOKEEPER_DEFAULT_COLLECTION; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.ZOOKEEPER_INSTANCES; +import static eu.europeana.metis.cleaner.utilities.IndexingPropertyNames.ZOOKEEPER_PORT_NUMBER; + +import eu.europeana.metis.cleaner.common.IndexingProperties; +import eu.europeana.metis.cleaner.utilities.IndexWrapper.IndexingType; +import java.util.Properties; + +public final class IndexingPropertiesTransformer { + + private IndexingPropertiesTransformer() { + } + + public static IndexingProperties getIndexingPropertiesFromPropertyFile(Properties properties, IndexingType type) { + String prefix = ""; + switch (type) { + case PUBLISH: + prefix = PUBLISH_PREFIX + DELIMITER; + break; + case PREVIEW: + prefix = PREVIEW_PREFIX + DELIMITER; + break; + } + IndexingProperties indexingProperties = new IndexingProperties(); + indexingProperties.setMongoDbName(properties.getProperty(prefix + MONGO_DB_NAME)); + indexingProperties.setMongoUsername(properties.getProperty(prefix + MONGO_USERNAME)); + indexingProperties.setMongoPassword(properties.getProperty(prefix + MONGO_SECRET)); + indexingProperties.setMongoApplicationName(properties.getProperty(prefix + MONGO_APPLICATION_NAME)); + indexingProperties.setMongoInstances(properties.getProperty(prefix + MONGO_INSTANCES)); + indexingProperties.setMongoPoolSize(Integer.parseInt(properties.getProperty(prefix + MONGO_POOL_SIZE))); + indexingProperties.setMongoRedirectsDbName(properties.getProperty(prefix + MONGO_REDIRECTS_DB_NAME)); + indexingProperties.setMongoPortNumber(Integer.parseInt(properties.getProperty(prefix + MONGO_PORT_NUMBER))); + indexingProperties.setMongoAuthDb(properties.getProperty(prefix + MONGO_AUTH_DB)); + indexingProperties.setMongoUseSSL(properties.getProperty(prefix + MONGO_USE_SSL)); + indexingProperties.setMongoReadPreference(properties.getProperty(prefix + MONGO_READ_PREFERENCE)); + + indexingProperties.setSolrInstances(properties.getProperty(prefix + SOLR_INSTANCES)); + + indexingProperties.setZookeeperChroot(properties.getProperty(prefix + ZOOKEEPER_CHROOT)); + indexingProperties.setZookeeperInstances(properties.getProperty(prefix + ZOOKEEPER_INSTANCES)); + indexingProperties.setZookeeperDefaultCollection(properties.getProperty(prefix + ZOOKEEPER_DEFAULT_COLLECTION)); + indexingProperties.setZookeeperPortNumber(Integer.parseInt(properties.getProperty(prefix + ZOOKEEPER_PORT_NUMBER))); + return indexingProperties; + } +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertyNames.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertyNames.java new file mode 100644 index 0000000..4f7f472 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingPropertyNames.java @@ -0,0 +1,30 @@ +package eu.europeana.metis.cleaner.utilities; + +public final class IndexingPropertyNames { + + public static final String PREVIEW_PREFIX = "indexing.preview"; + public static final String PUBLISH_PREFIX = "indexing.publish"; + public static final String MONGO_INSTANCES = "mongoInstances"; + public static final String MONGO_PORT_NUMBER = "mongoPortNumber"; + public static final String MONGO_DB_NAME = "mongoDbName"; + public static final String MONGO_REDIRECTS_DB_NAME = "mongoRedirectsDbName"; + public static final String MONGO_USERNAME = "mongoUsername"; + public static final String MONGO_SECRET = "mongoPassword"; + public static final String MONGO_USE_SSL = "mongoUseSSL"; + public static final String MONGO_READ_PREFERENCE = "mongoReadPreference"; + public static final String MONGO_APPLICATION_NAME = "mongoApplicationName"; + public static final String MONGO_POOL_SIZE = "mongoPoolSize"; + public static final String MONGO_AUTH_DB = "mongoAuthDB"; + // + public static final String SOLR_INSTANCES = "solrInstances"; + // + public static final String ZOOKEEPER_INSTANCES = "zookeeperInstances"; + public static final String ZOOKEEPER_PORT_NUMBER = "zookeeperPortNumber"; + public static final String ZOOKEEPER_CHROOT = "zookeeperChroot"; + public static final String ZOOKEEPER_DEFAULT_COLLECTION = "zookeeperDefaultCollection"; + public static final String DELIMITER = "."; + + private IndexingPropertyNames() { + } + +} diff --git a/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingSettingsGenerator.java b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingSettingsGenerator.java new file mode 100644 index 0000000..84b94e4 --- /dev/null +++ b/metis-dataset-cleaner/src/main/java/eu/europeana/metis/cleaner/utilities/IndexingSettingsGenerator.java @@ -0,0 +1,127 @@ +package eu.europeana.metis.cleaner.utilities; + + +import eu.europeana.indexing.IndexingSettings; +import eu.europeana.indexing.exception.IndexingException; +import eu.europeana.metis.cleaner.common.IndexingProperties; +import eu.europeana.metis.cleaner.utilities.IndexWrapper.IndexingType; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Optional; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class IndexingSettingsGenerator { + + private static final Logger LOGGER = LoggerFactory.getLogger(IndexingSettingsGenerator.class); + + protected final IndexingProperties previewIndexingProperties; + protected final IndexingProperties publishIndexingProperties; + + public IndexingSettingsGenerator(IndexingProperties previewIndexingProperties, IndexingProperties publishIndexingProperties) { + this.previewIndexingProperties = previewIndexingProperties; + this.publishIndexingProperties = publishIndexingProperties; + } + + public IndexingSettingsGenerator(Properties indexingPropertyFile) { + this.previewIndexingProperties + = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(indexingPropertyFile, IndexingType.PREVIEW); + this.publishIndexingProperties + = IndexingPropertiesTransformer.getIndexingPropertiesFromPropertyFile(indexingPropertyFile, IndexingType.PUBLISH); + } + + public IndexingSettings generateForPreview() throws IndexingException, URISyntaxException { + if (isNotDefinedFor(previewIndexingProperties)) { + return null; + } + IndexingSettings indexingSettings = new IndexingSettings(); + prepareSettingFor(previewIndexingProperties, indexingSettings); + return indexingSettings; + } + + public IndexingSettings generateForPublish() throws IndexingException, URISyntaxException { + if (isNotDefinedFor(publishIndexingProperties)) { + return null; + } + IndexingSettings indexingSettings = new IndexingSettings(); + prepareSettingFor(publishIndexingProperties, indexingSettings); + return indexingSettings; + } + + private boolean isNotDefinedFor(IndexingProperties indexingProperties) { + return indexingProperties.getMongoInstances() == null; + } + + private void prepareSettingFor(IndexingProperties indexingProperties, IndexingSettings indexingSettings) + throws IndexingException, URISyntaxException { + prepareMongoSettings(indexingSettings, indexingProperties); + prepareSolrSetting(indexingSettings, indexingProperties); + prepareZookeeperSettings(indexingSettings, indexingProperties); + } + + private void prepareMongoSettings(IndexingSettings indexingSettings, IndexingProperties indexingProperties) + throws IndexingException { + String mongoInstances = indexingProperties.getMongoInstances(); + int mongoPort = indexingProperties.getMongoPortNumber(); + String[] instances = mongoInstances.trim().split(","); + for (String instance : instances) { + indexingSettings.addMongoHost(new InetSocketAddress(instance, mongoPort)); + } + indexingSettings + .setMongoDatabaseName(indexingProperties.getMongoDbName()); + indexingSettings.setRecordRedirectDatabaseName(indexingProperties.getMongoRedirectsDbName()); + + if (mongoCredentialsProvidedFor(indexingProperties)) { + indexingSettings.setMongoCredentials( + indexingProperties.getMongoUsername(), + indexingProperties.getMongoPassword(), + indexingProperties.getMongoAuthDb()); + } else { + LOGGER.info("Mongo credentials not provided"); + } + + Optional optionalMongoPoolSize = Optional.ofNullable(indexingProperties.getMongoPoolSize()); + optionalMongoPoolSize.ifPresentOrElse( + indexingSettings::setMongoMaxConnectionPoolSize, + () -> LOGGER.warn("Mongo max connection pool size not provided")); + + if (indexingProperties.getMongoUseSSL() != null && indexingProperties.getMongoUseSSL().equalsIgnoreCase("true")) { + indexingSettings.setMongoEnableSsl(); + } + indexingSettings + .setMongoReadPreference(indexingProperties.getMongoReadPreference()); + indexingSettings.setMongoApplicationName(indexingProperties.getMongoApplicationName()); + + } + + private boolean mongoCredentialsProvidedFor(IndexingProperties indexingProperties) { + return !"".equals(indexingProperties.getMongoUsername()) && + !"".equals(indexingProperties.getMongoPassword()) && + !"".equals(indexingProperties.getMongoAuthDb()); + } + + private void prepareSolrSetting(IndexingSettings indexingSettings, IndexingProperties indexingProperties) + throws URISyntaxException, IndexingException { + String solrInstances = indexingProperties.getSolrInstances(); + String[] instances = solrInstances.trim().split(","); + for (String instance : instances) { + indexingSettings.addSolrHost(new URI(instance)); + } + } + + private void prepareZookeeperSettings(IndexingSettings indexingSettings, IndexingProperties indexingProperties) + throws IndexingException { + String zookeeperInstances = indexingProperties.getZookeeperInstances(); + int zookeeperPort = indexingProperties.getZookeeperPortNumber(); + String[] instances = zookeeperInstances.trim().split(","); + for (String instance : instances) { + indexingSettings.addZookeeperHost(new InetSocketAddress(instance, zookeeperPort)); + } + indexingSettings + .setZookeeperChroot(indexingProperties.getZookeeperChroot()); + indexingSettings.setZookeeperDefaultCollection(indexingProperties.getZookeeperDefaultCollection()); + } +} diff --git a/metis-dataset-cleaner/src/main/resources/application.properties.example b/metis-dataset-cleaner/src/main/resources/application.properties.example new file mode 100644 index 0000000..7c91b69 --- /dev/null +++ b/metis-dataset-cleaner/src/main/resources/application.properties.example @@ -0,0 +1,59 @@ +#Spring +#logging.config=log4j2.xml +spring.autoconfigure.exclude=\ + org.springframework.boot.autoconfigure.mongo.embedded.EmbeddedMongoAutoConfiguration, \ + org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, \ + org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration + +#Truststore +truststore.path= +truststore.password= + +#Mongo +mongo.hosts= +mongo.ports= +mongo.authenticationDatabase= +mongo.username= +mongo.password= +mongo.enableSsl= +mongo.database= +mongo.applicationName= + +indexing.preview.mongoInstances= +indexing.preview.mongoPortNumber= +indexing.preview.mongoDbName= +indexing.preview.mongoRedirectsDbName= +indexing.preview.mongoUsername= +indexing.preview.mongoPassword= +indexing.preview.mongoAuthDB= +indexing.preview.mongoUseSSL= +indexing.preview.mongoReadPreference= +indexing.preview.mongoPoolSize= + +indexing.preview.solrInstances= + +indexing.preview.zookeeperInstances= +indexing.preview.zookeeperPortNumber= +indexing.preview.zookeeperChroot= +indexing.preview.zookeeperDefaultCollection= + +indexing.preview.mongoApplicationName= + +indexing.publish.mongoInstances= +indexing.publish.mongoPortNumber= +indexing.publish.mongoDbName= +indexing.publish.mongoRedirectsDbName= +indexing.publish.mongoUsername= +indexing.publish.mongoPassword= +indexing.publish.mongoAuthDB= +indexing.publish.mongoUseSSL= +indexing.publish.mongoReadPreference= +indexing.publish.mongoPoolSize= +indexing.publish.mongoApplicationName= + +indexing.publish.solrInstances= + +indexing.publish.zookeeperInstances= +indexing.publish.zookeeperPortNumber= +indexing.publish.zookeeperChroot= +indexing.publish.zookeeperDefaultCollection= diff --git a/metis-dataset-cleaner/src/main/resources/log4j2.xml b/metis-dataset-cleaner/src/main/resources/log4j2.xml new file mode 100644 index 0000000..6d088df --- /dev/null +++ b/metis-dataset-cleaner/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file