Skip to content

Commit

Permalink
Merge pull request #8477 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history
Initial implementation of postgres repository connector
  • Loading branch information
mandy-chessell authored Nov 6, 2024
2 parents 7cf47f0 + 2ea04a1 commit bf15e2f
Show file tree
Hide file tree
Showing 160 changed files with 15,305 additions and 431 deletions.
2 changes: 2 additions & 0 deletions Content-Organization.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ The Egeria core repository contains the core Egeria functionality, and is organi
* **[open-metadata-collection-store-connectors](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors)** contains connectors that support mappings to different vendors' metadata repositories.
* **[graph-repository-connector](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors/graph-repository-connector)** - provides a local repository that uses a graph store as its persistence store.
* **[inmemory-repository-connector](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors/inmemory-repository-connector)** - provides a local repository that is entirely in memory. It is useful for testing/developing OMASs and demos.
* **[postgres-repository-connector](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors/postgres-repository-connector)** - provides a local repository that is soring its metadata in a postgres database. Each server/repository has its own database schema.
* **[xtdb-repository-connector](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors/xtdb-repository-connector)** - provides a local repository built on the XTDB repository. This allows different types of storage to be used.
* **[omrs-rest-repository-connector](open-metadata-implementation/adapters/open-connectors/repository-services-connectors/open-metadata-collection-store-connectors/omrs-rest-repository-connector)** - enables IBM Information Governance Catalog to support open metadata.
* **[rest-client-connectors](open-metadata-implementation/adapters/open-connectors/rest-client-connectors)** contains connector implementations for issuing REST calls.
* **[secrets-store-connectors](open-metadata-implementation/adapters/open-connectors/rest-client-connectors)** contains connector implementations for retrieving secrets from external sources.
Expand Down
7 changes: 6 additions & 1 deletion application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,10 @@ platform.security.secrets.collection=userDirectory
platform.placeholder.variables=\
{\
"kafkaEndpoint" : "localhost:9092",\
"egeriaEndpoint" : "https://localhost:9443"\
"egeriaEndpoint" : "https://localhost:9443",\
"postgreSQLDatabaseURL" : "jdbc:postgresql://localhost:5432/egeria",\
"secretsStore" : "loading_bay/secrets/defaults.omsecrets",\
"postgreSQLServerCollectionName" : "PostgreSQL Server:LocalPostgreSQL1"\
}

################################################
Expand Down Expand Up @@ -143,6 +146,8 @@ logging.level.root=OFF
logging.level.org.springframework=ERROR
logging.level.org.springframework.boot.web.embedded.tomcat=INFO
logging.level.org.odpi.openmetadata.platformchassis.springboot=INFO
#tracing SQL calls
#logging.level.org.odpi.openmetadata.adapters.connectors.resource.jdbc=DEBUG
#tracing REST calls
#logging.level.org.odpi.openmetadata.commonservices.ffdc.RESTCallLogger=DEBUG

Expand Down
5 changes: 4 additions & 1 deletion container.application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,10 @@ platform.security.secrets.collection=userDirectory
platform.placeholder.variables=\
{\
"kafkaEndpoint" : "host.docker.internal:9192",\
"egeriaEndpoint" : "https://host.docker.internal:9443"\
"egeriaEndpoint" : "https://host.docker.internal:9443",\
"postgreSQLDatabaseURL" : "jdbc:postgresql://host.docker.internal:5432/egeria",\
"secretsStore" : "loading_bay/secrets/defaults.omsecrets",\
"postgreSQLServerCollectionName" : "PostgreSQL Server:LocalPostgreSQL1"\
}

################################################
Expand Down
2 changes: 1 addition & 1 deletion content-packs/APIsContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/ApacheAtlasContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/ApacheKafkaContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoBusinessSystemsArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoClinicalTrialsTemplatesArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoComboArchive.omarchive

Large diffs are not rendered by default.

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoGovernanceProgramArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoOrganizationArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoSustainabilityArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CocoTypesArchive.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CoreContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/ObservabilityContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/OpenMetadataTypes.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/PostgresContentPack.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/SimpleAPICatalog.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/SimpleCatalog.omarchive

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions content-packs/SimpleCatalogV1.omarchive

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions content-packs/SimpleCatalogV2.omarchive

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions content-packs/SimpleCatalogV3.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/SimpleDataCatalog.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/SimpleEventCatalog.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/SimpleGovernanceCatalog.omarchive

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/UnityCatalogContentPack.omarchive

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,128 +1,156 @@
# Egeria-open-metadata-conformance-suite

@baseURL=https://localhost:9443
@adminUserId=garygeeke

@ctsServer=ctsServer
@ctsUserId=cts
@ctsPassword=secret

@tutServer=postgres-metadata-store
@tutPlatformURLroot=https://localhost:9443

@cohort=ctsCohort

###
# group: Configuration
# @name Query cSuite server's configuration
# Query the current configuration of the conformance suite server.
GET {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/configuration
GET {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/configuration

###
# group: Configuration
# @name Set cSuite server's user Id
# Set up the user Id that this server should use on open metadata requests.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/server-user-id?
id={{user}}
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/server-user-id?
id={{ctsUserId}}

###
# group: Configuration
# @name Set cSuite server's password
# Set up the password that the conformance suite server should use on open metadata requests.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/server-user-password?
password={{password}}
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/server-user-password?
password={{ctsPassword}}

###
# group: Configuration
# @name Set up event bus to connect to cohort topics
# Set up the common properties needed to call your event bus. These properties are passed in the request body.
# This request just sets up the defaults that will be incorporated into any future configuration that includes an event bus topic.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/event-bus
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/event-bus
Content-Type: application/json


{
"producer": {
"bootstrap.servers": "{{kafkaep}}"
},
"consumer":{
"bootstrap.servers": "{{kafkaep}}"
}
"producer":
{
"bootstrap.servers":"{{kafkaEndpoint}}"
},
"consumer":
{
"bootstrap.servers":"{{kafkaEndpoint}}"
}
}

###
# group: Configuration
# @name Enable access to the cohort
# @name Enable access to the cohort (CTS Server)
# Request registration to a cohort.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/cohorts/{{cohort}}
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/cohorts/{{cohort}}

###
# group: Configuration
# @name Disable access to the cohort
# Unregister from a cohort.
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/cohorts/{{cohort}}
# @name Enable access to the cohort (TUT Server)
# Request registration to a cohort.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{tutServer}}/cohorts/{{cohort}}

###
# group: Configuration
# @name Configure the test of an open metadata repository
# Activate the repository workbench tests to validate the behaviour of a specific repository.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/conformance-suite-workbenches/repository-workbench/repositories/{{tutServer}}
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/conformance-suite-workbenches/repository-workbench/repositories
Content-Type: application/json

{
"class": "RepositoryConformanceWorkbenchConfig",
"tutRepositoryServerName": "{{tutServer}}"
}


###
# group: Configuration
# @name Configure the test of an open metadata server platform
# Activate the repository workbench tests to validate the behaviour of a specific server platform.
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/conformance-suite-workbenches/platform-workbench/platforms/{{tutPlatformURLroot}}
POST {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/conformance-suite-workbenches/platform-workbench/platforms
Content-Type: application/json

{
"urlRoot": "{{tutPlatformURLroot}}"
}


###
# group: Configuration
# @name Delete the repository workbench configuration
# Remove all of the configuration for the open metadata conformance suite.
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/conformance-suite-workbenches/repository-workbench
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/conformance-suite-workbenches/repository-workbench

###
# group: Configuration
# @name Delete the platform workbench configuration
# Remove the configuration for the platform workbench of the open metadata conformance suite.
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/conformance-suite-workbenches/platform-workbench
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/conformance-suite-workbenches/platform-workbench

###
# group: Configuration
# @name Delete the conformance suite configuration
# Remove all of the configuration for the open metadata conformance suite.
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{server}}/conformance-suite-workbenches
DELETE {{baseURL}}/open-metadata/admin-services/users/{{adminUserId}}/servers/{{ctsServer}}/conformance-suite-workbenches

###
# group: Starting/Stopping CTS server
# @name Activate conformance suite using config
# Activate open metadata conformance suite services.
POST {{baseURL}}/open-metadata/platform-services/users/{{adminUserId}}/server-platform/servers/{{server}}/instance
POST {{baseURL}}/open-metadata/platform-services/users/{{adminUserId}}/server-platform/servers/{{ctsServer}}/instance

###
# group: Starting/Stopping CTS server
# @name Deactivate conformance suite server
# Deactivate open metadata conformance suite services for this server.
DELETE {{baseURL}}/open-metadata/platform-services/users/{{adminUserId}}/server-platform/servers/{{server}}/instance
DELETE {{baseURL}}/open-metadata/platform-services/users/{{adminUserId}}/server-platform/servers/{{ctsServer}}/instance

###
# group: Getting Results
# @name Retrieve platform workbench results
# Retrieve the results from the platform workbench tests.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/workbenches/platform-workbench
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/workbenches/platform-workbench

###
# group: Getting Results
# @name Retrieve repository workbench results
# Retrieve the results from the repository workbench tests.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/workbenches/repository-workbench
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/workbenches/repository-workbench

###
# group: Getting Results
# @name Retrieve full report of results
# Retrieve all of the results of the conformance suite testing.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report

###
# group: Getting Results
# @name Retrieve summary report of results
# Retrieve all of the results of the conformance suite testing.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/summary
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/summary

###
# group: Getting Results
# @name Retrieve report of failed test cases
# Retrieve the results of all failed test cases.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/test-cases/failed
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/test-cases/failed

###
# group: Getting Results
# @name Retrieve report of a specific test case
# Retrieve the results of all failed test cases.
GET {{baseURL}}/servers/{{server}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/test-cases/repository-metadata-collection
GET {{baseURL}}/servers/{{ctsServer}}/open-metadata/conformance-suite/users/{{adminUserId}}/report/test-cases/repository-entity-reference-copy-lifecycle-ConceptModelElement
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public TestOpenMetadataOrigin(PlatformConformanceWorkPad workPad)
*/
protected void run() throws Exception
{
final String urlTemplate = "/open-metadata/platform-services/users/" + workPad.getLocalServerUserId() + "/server-platform-origin";
final String urlTemplate = "/open-metadata/platform-services/users/" + workPad.getLocalServerUserId() + "/server-platform/origin";

RestTemplate restTemplate = new RestTemplate();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,8 @@ private PrimitivePropertyValue getPrimitivePropertyValue(String propertyN
PrimitivePropertyValue propertyValue = new PrimitivePropertyValue();

propertyValue.setPrimitiveDefCategory(propertyType.getPrimitiveDefCategory());
propertyValue.setTypeGUID(propertyType.getGUID());
propertyValue.setTypeName(propertyType.getName());

switch (propertyType.getPrimitiveDefCategory())
{
Expand Down Expand Up @@ -333,7 +335,8 @@ private PrimitivePropertyValue getPrimitivePropertyValue(String propertyN
PrimitivePropertyValue propertyValue = new PrimitivePropertyValue();

propertyValue.setPrimitiveDefCategory(propertyType.getPrimitiveDefCategory());

propertyValue.setTypeGUID(propertyType.getGUID());
propertyValue.setTypeName(propertyType.getName());

boolean distinct = attrUnique || (instanceCount%2)!=0;

Expand Down Expand Up @@ -602,9 +605,10 @@ protected InstanceProperties getMinPropertiesForInstance(String userId, TypeDef
AttributeCardinality attributeCardinality = typeDefAttribute.getAttributeCardinality();

if (attributeCardinality == AttributeCardinality.AT_LEAST_ONE_ORDERED ||
attributeCardinality == AttributeCardinality.AT_LEAST_ONE_UNORDERED) {

if (category == AttributeTypeDefCategory.PRIMITIVE) {
attributeCardinality == AttributeCardinality.AT_LEAST_ONE_UNORDERED)
{
if (category == AttributeTypeDefCategory.PRIMITIVE)
{
PrimitiveDef primitiveDef = (PrimitiveDef) attributeType;
propertyMap.put(attributeName, this.getPrimitivePropertyValue(attributeName, primitiveDef));
}
Expand Down
2 changes: 2 additions & 0 deletions open-metadata-distribution/omag-server-platform/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ dependencies {
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:graph-repository-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:omrs-rest-repository-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:inmemory-repository-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:postgres-repository-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:xtdb-repository-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:data-store-connectors:file-connectors:basic-file-connector')
implementation project(':open-metadata-implementation:adapters:open-connectors:data-store-connectors:file-connectors:csv-file-connector')
Expand Down Expand Up @@ -201,6 +202,7 @@ distributions {
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-archive-connectors:open-metadata-archive-file-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-archive-connectors:open-metadata-archive-directory-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:inmemory-repository-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:postgres-repository-connector').jar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:graph-repository-connector').fatJar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:xtdb-repository-connector').shadowJar }
from { project(':open-metadata-implementation:adapters:open-connectors:repository-services-connectors:open-metadata-collection-store-connectors:omrs-rest-repository-connector').jar }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ private String getAssetLineageMermaidGraph(AssetLineageGraph assetLineageGraph)
mermaidGraph.append(assetLineageGraph.getProperties().getDisplayName());
mermaidGraph.append(" [");
mermaidGraph.append(assetLineageGraph.getElementHeader().getGUID());
mermaidGraph.append("]\n---\nflowchart TB\n%%{init: {\"flowchart\": {\"htmlLabels\": false}} }%%\n\n");
mermaidGraph.append("]\n---\nflowchart TD\n%%{init: {\"flowchart\": {\"htmlLabels\": false}} }%%\n\n");

List<String> usedNodeNames = new ArrayList<>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.odpi.openmetadata.frameworks.openmetadata.properties.ArchiveProperties;
import org.odpi.openmetadata.frameworks.openmetadata.properties.glossaries.*;
import org.odpi.openmetadata.frameworks.openmetadata.properties.schema.DataFieldValuesProperties;
import org.odpi.openmetadata.frameworkservices.gaf.rest.HistoryRequestBody;

import java.util.Date;
import java.util.List;
Expand Down Expand Up @@ -3856,8 +3857,8 @@ public List<GlossaryTermElement> getGlossaryTermHistory(String u
int validatedPageSize = invalidParameterHandler.validatePaging(startFrom, pageSize, methodName);

HistoryRequestBody requestBody = new HistoryRequestBody();
requestBody.setAssetManagerGUID(assetManagerGUID);
requestBody.setAssetManagerName(assetManagerName);
requestBody.setExternalScopeGUID(assetManagerGUID);
requestBody.setExternalScopeName(assetManagerName);
requestBody.setFromTime(fromTime);
requestBody.setToTime(toTime);
requestBody.setEffectiveTime(effectiveTime);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ dependencies {
implementation project(':open-metadata-implementation:access-services:asset-manager:asset-manager-topic-connectors')
implementation project(':open-metadata-implementation:common-services:repository-handler')
implementation project(':open-metadata-implementation:common-services:metadata-security:metadata-security-server')
implementation project(':open-metadata-implementation:framework-services:gaf-metadata-management:gaf-metadata-api')
implementation project(':open-metadata-implementation:common-services:multi-tenant')
implementation project(':open-metadata-implementation:common-services:ffdc-services')
implementation project(':open-metadata-implementation:common-services:generic-handlers')
Expand Down
Loading

0 comments on commit bf15e2f

Please sign in to comment.