Skip to content

Commit

Permalink
Merge pull request #8285 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history 
Set up instance sync direction in Unity Catalog connectors
  • Loading branch information
@mandy-chessell
mandy-chessell authored Jul 17, 2024
2 parents 3d43a0d + 7abc0ee commit e7eeae7
Show file tree
Hide file tree
Showing 16 changed files with 349 additions and 162 deletions.
2 changes: 1 addition & 1 deletion bom/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ ext {
logbackVersion = '1.5.6'
lettuceVersion = '6.3.2.RELEASE'
// TODO: Version 9 now available
luceneVersion = '8.11.2'
luceneVersion = '8.11.3'
openlineageVersion = '1.16.0'
ossVersion = '4.16.0'
// TODO: Held as data engine breaks
Expand Down
3 changes: 2 additions & 1 deletion ...en-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-catalogs.http
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,11 @@ Content-Type: application/json

###
# @name patch catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test4
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test41
Content-Type: application/json

{
"new_name" : "clinical_trials",
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
Expand Down
2 changes: 1 addition & 1 deletion ...rs/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ Content-Type: application/json

###

@serverGUID=3189db57-1a25-421b-8c18-c428340428b4
@serverGUID=add server guid here

###
# @name getTechnologyTypeElements (UC Server)
Expand Down
75 changes: 60 additions & 15 deletions ...rc/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/ffdc/UCAuditCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ public enum UCAuditCode implements AuditLogMessageSet
"Use the details from the error message to determine the cause of the error and retry the request once it is resolved."),

/**
* UNITY-CATALOG-CONNECTOR-0002 - The {0} survey action service is unable to retrieve details of any databases for Unity Catalog Server {1}
* UNITY-CATALOG-CONNECTOR-0002 - The {0} connector is unable to retrieve details of any catalogs for Unity Catalog Server {1} ({2})
*/
NO_DATABASES( "UNITY-CATALOG-CONNECTOR-0002",
NO_CATALOGS( "UNITY-CATALOG-CONNECTOR-0002",
AuditLogRecordSeverityLevel.INFO,
"The {0} survey action service is unable to retrieve details of any databases for Unity Catalog Server {1} ({2})",
"The survey terminates.",
"This may not be an error if there are no user database on the database server. If there are, check the permissions associated with the database userId."),
"The {0} connector is unable to retrieve details of any catalogs for Unity Catalog Server {1} ({2})",
"The connector terminates.",
"Check that this server is a Unity Catalog (UC) Server. If it is, ensure the permissions are set up so this connector can retrieve information from the server."),

/**
* UNITY-CATALOG-CONNECTOR-0003 - The {0} integration connector has catalogued data set {1} ({2})
Expand Down Expand Up @@ -79,55 +79,100 @@ public enum UCAuditCode implements AuditLogMessageSet
"Use the details from the error message to determine the class of the connector. " +
"Update the connector type associated with Unity Catalog's Connection in the metadata store."),

/**
* UNITY-CATALOG-CONNECTOR-0007 - The {0} OSS Unity Catalog Connector has been supplied with a friendship connector with GUID {1}
*/
FRIENDSHIP_GUID("UNITY-CATALOG-CONNECTOR-0007",
AuditLogRecordSeverityLevel.INFO,
"The {0} OSS Unity Catalog Connector has been supplied with a friendship connector with GUID {1}",
"The friendship connector is an integration connector that is able to synchronize the contents inside a Unity Catalog (UC) connector. Therefore, they will cooperate to synchronize the contents of the Unity Catalog with the open metadata ecosystem.",
"No action is required, this message is just to acknowledge that that the two integration connectors are going to collaborate to catalog the entire contents of Unity Catalog"),

/**
* UNITY-CATALOG-CONNECTOR-0008 - The {0} OSS Unity Inside Catalog Synchronizer Connector only works with catalog targets
*/
IGNORING_ENDPOINT("UNITY-CATALOG-CONNECTOR-0008",
AuditLogRecordSeverityLevel.INFO,
"The {0} OSS Unity Inside Catalog Synchronizer Connector only works with catalog targets",
"The connector is ignoring the Unity Catalog (UC) server instance that are configured directly through its connection.",
"Update the integration connector's configuration to use catalog targets."),

/**
* UNITY-CATALOG-CONNECTOR-0009 - The {0} Connector has added a catalog target relationship {1} from friendship connector {2} to Unity Catalog Server Asset {3} for Unity Catalog (UC) {4}
*/
NEW_CATALOG_TARGET("UNITY-CATALOG-CONNECTOR-0009",
AuditLogRecordSeverityLevel.INFO,
"The {0} Connector has added a catalog target relationship {1} from friendship connector {2} to Unity Catalog Server Asset {3} for Unity Catalog (UC) {4}",
"The connector has requested that its friendship connector starts to catalog a new Unity Catalog (UC) catalog.",
"Verify that the cataloguing starts the next time that the friendship connector refreshes."),

/**
* UNITY-CATALOG-CONNECTOR-0010 - The {0} Connector has detected a change in the identity of element {1}. The original id was from {2} and now it is {3} in Unity Catalog (UC) {4}
*/
IDENTITY_MISMATCH("UNITY-CATALOG-CONNECTOR-0010",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the identity of element {1}. The original id was {2} and now it is {3} in Unity Catalog (UC) {4}",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog.",
"The {0} Connector has detected a change in the identity of element {1}. The original id was from {2} and now it is {3} in Unity Catalog (UC) {4}",
"The connector will continue to synchronize metadata with Unity Catalog (UC). This element will not be updated in with system and will remain out of sync.",
"Verify that the element is being correctly synchronized. It is possible that there are two different elements with the same name. Also investigate why this element changed in Unity Catalog when it is owned by the open metadata ecosystem. It may be that the structure of the element was changed and the UC element was replaced to reflect the new data structure. If this is a planned change then all is ok, if it is unexpected then take steps to repair the data source and prevent it happening again."),

TABLE_UPDATE("UNITY-CATALOG-CONNECTOR-0011",
/**
* UNITY-CATALOG-CONNECTOR-0011 - The {0} Connector has detected a change in the open metadata that controls the definition of catalog {1} but is unable to update the catalog in Unity Catalog (UC) at {2}
*/
CATALOG_UPDATE("UNITY-CATALOG-CONNECTOR-0011",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the open metadata that controls the definition of catalog {1} but is unable to update the catalog in Unity Catalog (UC) at {2}",
"The connector will continue to synchronize metadata with Unity Catalog (UC). However, this catalog's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog uses the PATCH request which is not supported by Java.",
"Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."),

/**
* UNITY-CATALOG-CONNECTOR-0012 - The {0} Connector has detected a change in the open metadata that controls the definition of schema {1} but is unable to update the schema in Unity Catalog (UC) at {2}
*/
SCHEMA_UPDATE("UNITY-CATALOG-CONNECTOR-0012",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the open metadata that controls the definition of schema {1} but is unable to update the schema in Unity Catalog (UC) at {2}",
"The connector will continue to synchronize metadata with Unity Catalog (UC). However, this schema's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog uses the PATCH request which is not supported by Java.",
"Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."),

/**
* UNITY-CATALOG-CONNECTOR-0013 - The {0} Connector has detected a change in the open metadata that controls the definition of table {1} but is unable to update the table in Unity Catalog (UC) at {2}
*/
TABLE_UPDATE("UNITY-CATALOG-CONNECTOR-0013",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the open metadata that controls the definition of table {1} but is unable to update the table in Unity Catalog (UC) at {2}",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this table will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support an update request.",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this table's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support an update request.",
"Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."),

VOLUME_UPDATE("UNITY-CATALOG-CONNECTOR-0012",
/**
* UNITY-CATALOG-CONNECTOR-0014 - The {0} Connector has detected a change in the open metadata element {1} that controls the definition of volume {2} but is unable to update the volume in Unity Catalog (UC) at {3}
*/
VOLUME_UPDATE("UNITY-CATALOG-CONNECTOR-0014",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the open metadata element {1} that controls the definition of volume {2} but is unable to update the volume in Unity Catalog (UC) at {3}",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this volume will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this volume's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.",
"Validate that the change in the open metadata ecosystem is intended. If it is, manually update the volume in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."),

FUNCTION_UPDATE("UNITY-CATALOG-CONNECTOR-0013",
/**
* UNITY-CATALOG-CONNECTOR-0015 - The {0} Connector has detected a change in the open metadata element {1} that controls the definition of function {2} but is unable to update the function in Unity Catalog (UC) at {3}
*/
FUNCTION_UPDATE("UNITY-CATALOG-CONNECTOR-0015",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has detected a change in the open metadata element {1} that controls the definition of function {2} but is unable to update the function in Unity Catalog (UC) at {3}",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this function will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this function's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.",
"Validate that the change in the open metadata ecosystem is intended. If it is, manually update the volume in Unity Catalog using the CLI. There may also need to be some data migration in the underlying called data sources."),

UC_ELEMENT_DELETE("UNITY-CATALOG-CONNECTOR-0014",
/**
* UNITY-CATALOG-CONNECTOR-0016 - The {0} Connector has detected that the open metadata element that controls the definition of element {1} in Unity Catalog (UC) at {2} has been deleted
*/
UC_ELEMENT_DELETE("UNITY-CATALOG-CONNECTOR-0016",
AuditLogRecordSeverityLevel.INFO,
"The {0} Connector has detected that the open metadata element that controls the definition of element {1} in Unity Catalog (UC) at {2} has been deleted",
"The connector will delete the element in Unity Catalog (UC) catalog.",
"Validate that the change in the open metadata ecosystem is intended. If it is, no additional action is required. If the element is still required, investigate what happened to the element in the open metadata ecosystem. It may have moved zones or metadata collections or its security controls changed, making it invisible to the connector. Or it may have been soft-deleted which means it can be restored."),

MISSING_METHOD("UNITY-CATALOG-CONNECTOR-0015",
/**
* UNITY-CATALOG-CONNECTOR-0017 - The {0} Connector has called method {1} that is not implemented and so is not able to synchronize element {2} with the Unity Catalog (UC) at {3}
*/
MISSING_METHOD("UNITY-CATALOG-CONNECTOR-0017",
AuditLogRecordSeverityLevel.ACTION,
"The {0} Connector has called method {1} that is not implemented and so is not able to synchronize element {2} with the Unity Catalog (UC) at {3}",
"The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, the current element will remain unchanged and out of sync with the open metadata ecosystem.",
Expand Down
2 changes: 1 addition & 1 deletion ...nmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ public CatalogInfo updateCatalog(String name,
requestBody.setComment(comment);
requestBody.setProperties(properties);

if (newName == null)
if ((newName == null) || (newName.equals(name)))
{
/*
* new_name must not be null.
Expand Down
63 changes: 56 additions & 7 deletions ...nmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@


import org.odpi.openmetadata.adapters.connectors.unitycatalog.controls.UnityCatalogPlaceholderProperty;
import org.odpi.openmetadata.adapters.connectors.unitycatalog.ffdc.UCAuditCode;
import org.odpi.openmetadata.adapters.connectors.unitycatalog.properties.ElementBase;
import org.odpi.openmetadata.adapters.connectors.unitycatalog.resource.OSSUnityCatalogResourceConnector;
import org.odpi.openmetadata.frameworks.auditlog.AuditLog;
Expand All @@ -14,6 +15,7 @@
import org.odpi.openmetadata.frameworks.connectors.properties.beans.ElementStatus;
import org.odpi.openmetadata.frameworks.governanceaction.controls.PlaceholderProperty;
import org.odpi.openmetadata.frameworks.governanceaction.properties.ExternalIdentifierProperties;
import org.odpi.openmetadata.frameworks.governanceaction.properties.MetadataCorrelationHeader;
import org.odpi.openmetadata.frameworks.governanceaction.search.ElementProperties;
import org.odpi.openmetadata.frameworks.governanceaction.search.PropertyHelper;
import org.odpi.openmetadata.frameworks.integration.context.OpenMetadataAccess;
Expand All @@ -26,9 +28,7 @@
import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataType;
import org.odpi.openmetadata.integrationservices.catalog.connector.CatalogIntegratorContext;

import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.*;

/**
* Common functions for the synchronizing between Egeria and Unity Catalog.
Expand Down Expand Up @@ -210,10 +210,11 @@ protected void deleteElementInEgeria(MemberElement memberElement) throws Invalid
* @param schemaName name of the schema
* @return external identifier properties
*/
protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBase ucElement,
String schemaName,
String elementName,
String id)
protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBase ucElement,
String schemaName,
String elementName,
String id,
PermittedSynchronization instanceSynchronizationDirection)
{
ExternalIdentifierProperties externalIdentifierProperties = new ExternalIdentifierProperties();

Expand All @@ -232,6 +233,8 @@ protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBa

externalIdentifierProperties.setMappingProperties(mappingProperties);

externalIdentifierProperties.setSynchronizationDirection(instanceSynchronizationDirection);

return externalIdentifierProperties;
}

Expand Down Expand Up @@ -280,6 +283,52 @@ protected void addPropertyFacet(String parentGUID,
}



/**
* Check that the name of the third party element has not changed with respect to Egeria.
*
* @param thirdPartyExternalIdentifier id from Unity Catalog
* @param memberElement element from Egeria
* @return boolean
*/
protected boolean noMismatchInExternalIdentifier(String thirdPartyExternalIdentifier,
MemberElement memberElement)
{
final String methodName = "noMismatchInExternalIdentifier";

if ((thirdPartyExternalIdentifier == null) || (memberElement == null) || (memberElement.getExternalIdentifiers() == null))
{
return true;
}

List<String> externalIdentifiers = new ArrayList<>();

for (MetadataCorrelationHeader correlationHeader : memberElement.getExternalIdentifiers())
{
if ((correlationHeader != null) && (correlationHeader.getExternalIdentifier() != null))
{
externalIdentifiers.add(correlationHeader.getExternalIdentifier());

if (thirdPartyExternalIdentifier.equals(correlationHeader.getExternalIdentifier()))
{
return true;
}
}
}

if (externalIdentifiers.isEmpty())
{
return true;
}

auditLog.logMessage(methodName, UCAuditCode.IDENTITY_MISMATCH.getMessageDefinition(connectorName,
externalIdentifiers.toString(),
thirdPartyExternalIdentifier,
ucServerEndpoint));
return false;
}


/**
* Extract the comment for a UC element from the description attribute in an egeria element.
*
Expand Down
Loading

0 comments on commit e7eeae7

Please sign in to comment.