Skip to content

Commit

Permalink
Merge pull request #8490 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history 
Add PostgreSQL Audit Log Desination
  • Loading branch information
@mandy-chessell
mandy-chessell authored Nov 14, 2024
2 parents 227dba4 + 16a4096 commit 0d6a30d
Show file tree
Hide file tree
Showing 45 changed files with 1,343 additions and 312 deletions.
4 changes: 2 additions & 2 deletions open-metadata-distribution/omag-server-platform/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ dependencies {
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-console-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-event-topic-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-file-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-jdbc-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-postgres-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-slf4j-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:cohort-registry-store-connectors:cohort-registry-file-store-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:graph-repository-connector')
Expand Down Expand Up @@ -196,7 +196,7 @@ distributions {
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-console-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-event-topic-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-file-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-jdbc-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-postgres-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-slf4j-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:cohort-registry-store-connectors:cohort-registry-file-store-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-archive-connectors:open-metadata-archive-file-connector').jar }
Expand Down
3 changes: 2 additions & 1 deletion ...data-implementation/adapters/open-connectors/connector-configuration-factory/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ dependencies {
implementation project(':open-metadata-implementation:frameworks:open-metadata-framework')
implementation project(':open-metadata-implementation:frameworks:open-connector-framework')
implementation project(':open-metadata-implementation:repository-services:repository-services-apis')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:data-store-connectors:jdbc-resource-connector')
compileOnly 'com.fasterxml.jackson.core:jackson-annotations'
implementation 'org.apache.commons:commons-lang3'

// The following are only included at runtime for the full platform (ie adminChassisOnly is not set as a property)
if (!project.hasProperty("adminChassisOnly")) {
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-file-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-jdbc-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-postgres-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-console-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-slf4j-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:audit-log-connectors:audit-log-event-topic-connector')
Expand Down
107 changes: 71 additions & 36 deletions ...java/org/odpi/openmetadata/adapters/repositoryservices/ConnectorConfigurationFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package org.odpi.openmetadata.adapters.repositoryservices;


import org.odpi.openmetadata.adapters.connectors.resource.jdbc.controls.JDBCConfigurationProperty;
import org.odpi.openmetadata.adapters.repositoryservices.postgres.repositoryconnector.controls.PostgresConfigurationProperty;
import org.odpi.openmetadata.frameworks.auditlog.AuditLogRecordSeverityLevel;
import org.odpi.openmetadata.frameworks.connectors.ConnectorProvider;
Expand All @@ -20,7 +21,6 @@

import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -50,25 +50,25 @@ public class ConnectorConfigurationFactory
private static final String defaultCohortTopicConnectorRootName = defaultTopicRootName + "cohort.";


private static final String FILE_BASED_SERVER_CONFIG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.adminservices.configurationstore.file.FileBasedServerConfigStoreProvider";
private static final String IN_MEMORY_OPEN_METADATA_TOPIC_PROVIDER = "org.odpi.openmetadata.adapters.eventbus.topic.inmemory.InMemoryOpenMetadataTopicProvider";
private static final String KAFKA_OPEN_METADATA_TOPIC_PROVIDER = "org.odpi.openmetadata.adapters.eventbus.topic.kafka.KafkaOpenMetadataTopicProvider";
private static final String FILE_BASED_OPEN_METADATA_ARCHIVE_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.archiveconnector.file.FileBasedOpenMetadataArchiveStoreProvider";
private static final String CONSOLE_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider";
private static final String EVENT_TOPIC_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.eventtopic.EventTopicAuditLogStoreProvider";
private static final String FILE_BASED_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.file.FileBasedAuditLogStoreProvider";
private static final String JDBC_BASED_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.jdbc.JDBCAuditLogDestinationProvider";
private static final String SLF_4_J_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.slf4j.SLF4JAuditLogStoreProvider";
private static final String FILE_BASED_REGISTRY_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.cohortregistrystore.file.FileBasedRegistryStoreProvider";
private static final String GRAPH_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.graphrepository.repositoryconnector.GraphOMRSRepositoryConnectorProvider";
private static final String POSTGRES_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.postgres.repositoryconnector.PostgresOMRSRepositoryConnectorProvider";
private static final String YAML_SECRETS_STORE_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.connectors.secretsstore.yaml.YAMLSecretsStoreProvider";
private static final String JDBC_RESOURCE_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.connectors.resource.jdbc.JDBCResourceConnectorProvider";
private static final String XTDB_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.xtdb.repositoryconnector.XTDBOMRSRepositoryConnectorProvider";
private static final String IN_MEMORY_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider";
private static final String READ_ONLY_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.readonly.repositoryconnector.ReadOnlyOMRSRepositoryConnectorProvider";
private static final String OMRS_REST_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider";
private static final String OMRS_TOPIC_PROVIDER = "org.odpi.openmetadata.repositoryservices.connectors.omrstopic.OMRSTopicProvider";
private static final String FILE_BASED_SERVER_CONFIG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.adminservices.configurationstore.file.FileBasedServerConfigStoreProvider";
private static final String IN_MEMORY_OPEN_METADATA_TOPIC_PROVIDER = "org.odpi.openmetadata.adapters.eventbus.topic.inmemory.InMemoryOpenMetadataTopicProvider";
private static final String KAFKA_OPEN_METADATA_TOPIC_PROVIDER = "org.odpi.openmetadata.adapters.eventbus.topic.kafka.KafkaOpenMetadataTopicProvider";
private static final String FILE_BASED_OPEN_METADATA_ARCHIVE_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.archiveconnector.file.FileBasedOpenMetadataArchiveStoreProvider";
private static final String CONSOLE_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider";
private static final String EVENT_TOPIC_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.eventtopic.EventTopicAuditLogStoreProvider";
private static final String FILE_BASED_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.file.FileBasedAuditLogStoreProvider";
private static final String POSTGRES_BASED_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.postgres.PostgreSQLAuditLogDestinationProvider";
private static final String SLF_4_J_AUDIT_LOG_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.slf4j.SLF4JAuditLogStoreProvider";
private static final String FILE_BASED_REGISTRY_STORE_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.cohortregistrystore.file.FileBasedRegistryStoreProvider";
private static final String GRAPH_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.graphrepository.repositoryconnector.GraphOMRSRepositoryConnectorProvider";
private static final String POSTGRES_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.postgres.repositoryconnector.PostgresOMRSRepositoryConnectorProvider";
private static final String YAML_SECRETS_STORE_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.connectors.secretsstore.yaml.YAMLSecretsStoreProvider";
private static final String JDBC_RESOURCE_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.connectors.resource.jdbc.JDBCResourceConnectorProvider";
private static final String XTDB_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.xtdb.repositoryconnector.XTDBOMRSRepositoryConnectorProvider";
private static final String IN_MEMORY_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider";
private static final String READ_ONLY_OMRS_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.readonly.repositoryconnector.ReadOnlyOMRSRepositoryConnectorProvider";
private static final String OMRS_REST_REPOSITORY_CONNECTOR_PROVIDER = "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider";
private static final String OMRS_TOPIC_PROVIDER = "org.odpi.openmetadata.repositoryservices.connectors.omrstopic.OMRSTopicProvider";

private static final Logger log = LoggerFactory.getLogger(ConnectorConfigurationFactory.class);

Expand Down Expand Up @@ -123,8 +123,8 @@ public Connection getServerConfigConnectionForRetrieveAll()
*/
public Connection getDefaultAuditLogConnection()
{
List<AuditLogRecordSeverityLevel> supportedSeverityDefinitions = Arrays.asList(AuditLogRecordSeverityLevel.values());
List<String> supportedSeverities = new ArrayList<>();
AuditLogRecordSeverityLevel[] supportedSeverityDefinitions = AuditLogRecordSeverityLevel.values();
List<String> supportedSeverities = new ArrayList<>();

for (AuditLogRecordSeverityLevel severityDefinition : supportedSeverityDefinitions)
{
Expand Down Expand Up @@ -229,29 +229,64 @@ public Connection getFileBasedAuditLogConnection(String localServerName,
* Return the connection for the JDBC-based audit log.
* By default, the File-based Audit log is stored in a directory called localServerName.ffdc.
*
* @param connectionString address of database schema
* @param supportedSeverities list of severities that should be logged to this destination (empty list means all)
* @param storageProperties properties used to configure access to the database
* @return OCF Connection used to create the file based audit logger
*/
public Connection getJDBCBasedAuditLogConnection(String connectionString,
List<String> supportedSeverities)
public Connection getPostgreSQLBasedAuditLogConnection(Map<String, Object> storageProperties)
{
final String destinationName = "JDBCDatabase";
Connection secretsStoreConnection = new Connection();
Endpoint endpoint = new Endpoint();

Endpoint endpoint = new Endpoint();
if (storageProperties.containsKey(JDBCConfigurationProperty.SECRETS_STORE.getName()))
{
endpoint.setAddress(storageProperties.get(JDBCConfigurationProperty.SECRETS_STORE.getName()).toString());
}
else
{
endpoint.setAddress(JDBCConfigurationProperty.SECRETS_STORE.getExample());
}

endpoint.setAddress(connectionString);
secretsStoreConnection.setEndpoint(endpoint);
secretsStoreConnection.setConnectorType(getConnectorType(YAML_SECRETS_STORE_CONNECTOR_PROVIDER));
secretsStoreConnection.setConfigurationProperties(storageProperties);

Connection connection = new Connection();
List<EmbeddedConnection> embeddedConnections = new ArrayList<>();
EmbeddedConnection embeddedConnection = new EmbeddedConnection();

connection.setQualifiedName(destinationName + " at " + connectionString);
connection.setDisplayName(destinationName + " at " + connectionString);
connection.setEndpoint(endpoint);
connection.setConnectorType(getConnectorType(JDBC_BASED_AUDIT_LOG_STORE_PROVIDER));
embeddedConnection.setEmbeddedConnection(secretsStoreConnection);
embeddedConnections.add(embeddedConnection);

setSupportedAuditLogSeverities(supportedSeverities, connection);
VirtualConnection jdbcResourceConnection = new VirtualConnection();

return connection;
jdbcResourceConnection.setEmbeddedConnections(embeddedConnections);
jdbcResourceConnection.setConnectorType(getConnectorType(JDBC_RESOURCE_CONNECTOR_PROVIDER));
jdbcResourceConnection.setConfigurationProperties(storageProperties);

endpoint = new Endpoint();

if (storageProperties.containsKey(JDBCConfigurationProperty.DATABASE_URL.getName()))
{
endpoint.setAddress(storageProperties.get(JDBCConfigurationProperty.DATABASE_URL.getName()).toString());
}
else
{
endpoint.setAddress(JDBCConfigurationProperty.DATABASE_URL.getExample());
}
jdbcResourceConnection.setEndpoint(endpoint);

embeddedConnections = new ArrayList<>();
embeddedConnection = new EmbeddedConnection();
embeddedConnection.setEmbeddedConnection(jdbcResourceConnection);
embeddedConnections.add(embeddedConnection);

VirtualConnection jdbcAuditLogConnection = new VirtualConnection();

jdbcAuditLogConnection.setDisplayName("PostgreSQL Database Schema for Audit Log events");
jdbcAuditLogConnection.setConnectorType(getConnectorType(POSTGRES_BASED_AUDIT_LOG_STORE_PROVIDER));
jdbcAuditLogConnection.setConfigurationProperties(storageProperties);
jdbcAuditLogConnection.setEmbeddedConnections(embeddedConnections);

return jdbcAuditLogConnection;
}


Expand Down
11 changes: 6 additions & 5 deletions ...n/java/org/odpi/openmetadata/adapters/connectors/resource/jdbc/JDBCResourceConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

package org.odpi.openmetadata.adapters.connectors.resource.jdbc;

import org.odpi.openmetadata.adapters.connectors.resource.jdbc.controls.JDBCConfigurationProperty;
import org.odpi.openmetadata.adapters.connectors.resource.jdbc.ffdc.JDBCAuditCode;
import org.odpi.openmetadata.adapters.connectors.resource.jdbc.ffdc.JDBCErrorCode;
import org.odpi.openmetadata.adapters.connectors.resource.jdbc.properties.JDBCDataValue;
Expand Down Expand Up @@ -106,9 +107,9 @@ public void start() throws ConnectorCheckedException

if (configurationProperties != null)
{
if (configurationProperties.get(JDBCResourceConnectorProvider.JDBC_DATABASE_NAME) != null)
if (configurationProperties.get(JDBCConfigurationProperty.DATABASE_NAME.getName()) != null)
{
jdbcDatabaseName = configurationProperties.get(JDBCResourceConnectorProvider.JDBC_DATABASE_NAME).toString();
jdbcDatabaseName = configurationProperties.get(JDBCConfigurationProperty.DATABASE_NAME.getName()).toString();
}
}

Expand All @@ -122,9 +123,9 @@ public void start() throws ConnectorCheckedException

if (configurationProperties != null)
{
if (configurationProperties.get(JDBCResourceConnectorProvider.JDBC_CONNECTION_TIMEOUT) != null)
if (configurationProperties.get(JDBCConfigurationProperty.JDBC_CONNECTION_TIMEOUT.getName()) != null)
{
Object connectionTimeoutOption = configurationProperties.get(JDBCResourceConnectorProvider.JDBC_CONNECTION_TIMEOUT);
Object connectionTimeoutOption = configurationProperties.get(JDBCConfigurationProperty.JDBC_CONNECTION_TIMEOUT.getName());

if (connectionTimeoutOption != null)
{
Expand All @@ -137,7 +138,7 @@ public void start() throws ConnectorCheckedException
}
}

Object driverManagerClassName = configurationProperties.get(JDBCResourceConnectorProvider.JDBC_DRIVER_MANAGER_CLASS_NAME);
Object driverManagerClassName = configurationProperties.get(JDBCConfigurationProperty.JDBC_DRIVER_MANAGER_CLASS_NAME.getName());

if (driverManagerClassName != null)
{
Expand Down
Loading

0 comments on commit 0d6a30d

Please sign in to comment.