From 2a6d6c15c5a1d948c9a8751559b95db3551c6a25 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Wed, 30 Oct 2024 12:52:21 +0800 Subject: [PATCH] [enhance](hive)hive event code optimization #42637 (#42876) cherry pick from #42637 Co-authored-by: zhangdong <493738387@qq.com> --- .../hive/event/AddPartitionEvent.java | 8 +- .../hive/event/AlterDatabaseEvent.java | 8 +- .../hive/event/AlterPartitionEvent.java | 6 +- .../hive/event/AlterTableEvent.java | 10 +- .../hive/event/CreateDatabaseEvent.java | 4 +- .../hive/event/CreateTableEvent.java | 8 +- .../hive/event/DropDatabaseEvent.java | 4 +- .../hive/event/DropPartitionEvent.java | 8 +- .../datasource/hive/event/DropTableEvent.java | 6 +- .../datasource/hive/event/IgnoredEvent.java | 2 +- .../datasource/hive/event/InsertEvent.java | 6 +- .../datasource/hive/event/MetastoreEvent.java | 167 +++++------------- .../hive/event/MetastoreEventsProcessor.java | 2 +- 13 files changed, 81 insertions(+), 158 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java index ffc7b95ff59aad..e582c3d2662bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java @@ -59,7 +59,7 @@ private AddPartitionEvent(NotificationEvent event, super(event, catalogName); Preconditions.checkArgument(getEventType().equals(MetastoreEventType.ADD_PARTITION)); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { AddPartitionMessage addPartitionMessage = MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -98,18 +98,18 @@ protected static List getEvents(NotificationEvent event, @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]", catalogName, dbName, tblName, + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]", catalogName, dbName, tblName, partitionNames.toString()); // bail out early if there are not partitions to process if (partitionNames.isEmpty()) { - infoLog("Partition list is empty. Ignoring this event."); + logInfo("Partition list is empty. Ignoring this event."); return; } Env.getCurrentEnv().getCatalogMgr() .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, eventTime, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java index ef828025e7f719..334930861b6ef6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java @@ -67,7 +67,7 @@ private AlterDatabaseEvent(NotificationEvent event, dbNameAfter = dbAfter.getName(); } catch (Exception e) { throw new MetastoreNotificationException( - debugString("Unable to parse the alter database message"), e); + getMsgWithEventInfo("Unable to parse the alter database message"), e); } // this is a rename event if either dbName of before and after object changed isRename = !dbBefore.getName().equalsIgnoreCase(dbAfter.getName()); @@ -82,7 +82,7 @@ private void processRename() throws DdlException { throw new DdlException("Only support ExternalCatalog Databases"); } if (catalog.getDbNullable(dbAfter.getName()) != null) { - infoLog("AlterExternalDatabase canceled, because dbAfter has exist, " + logInfo("AlterExternalDatabase canceled, because dbAfter has exist, " + "catalogName:[{}],dbName:[{}]", catalogName, dbAfter.getName()); return; @@ -113,10 +113,10 @@ protected void process() throws MetastoreNotificationException { return; } // only can change properties,we do nothing - infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); + logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName); } catch (Exception e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java index 569d9878d7afaa..d9898f68d982f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java @@ -65,7 +65,7 @@ private AlterPartitionEvent(NotificationEvent event, super(event, catalogName); Preconditions.checkArgument(getEventType().equals(MetastoreEventType.ALTER_PARTITION)); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { AlterPartitionMessage alterPartitionMessage = MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -109,7 +109,7 @@ protected static List getEvents(NotificationEvent event, @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNameBefore:[{}],partitionNameAfter:[{}]", + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNameBefore:[{}],partitionNameAfter:[{}]", catalogName, dbName, tblName, partitionNameBefore, partitionNameAfter); if (isRename) { Env.getCurrentEnv().getCatalogMgr() @@ -125,7 +125,7 @@ protected void process() throws MetastoreNotificationException { } } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java index 2283646b64adfa..43999fac8ce392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java @@ -59,7 +59,7 @@ private AlterTableEvent(NotificationEvent event, String catalogName) { super(event, catalogName); Preconditions.checkArgument(MetastoreEventType.ALTER_TABLE.equals(getEventType())); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { JSONAlterTableMessage alterTableMessage = (JSONAlterTableMessage) MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -70,7 +70,7 @@ private AlterTableEvent(NotificationEvent event, String catalogName) { tblNameAfter = tableAfter.getTableName(); } catch (Exception e) { throw new MetastoreNotificationException( - debugString("Unable to parse the alter table message"), e); + getMsgWithEventInfo("Unable to parse the alter table message"), e); } // this is a rename event if either dbName or tblName of before and after object changed isRename = !tableBefore.getDbName().equalsIgnoreCase(tableAfter.getDbName()) @@ -111,7 +111,7 @@ private void processRename() throws DdlException { boolean hasExist = Env.getCurrentEnv().getCatalogMgr() .externalTableExistInLocal(tableAfter.getDbName(), tableAfter.getTableName(), catalogName); if (hasExist) { - infoLog("AlterExternalTable canceled,because tableAfter has exist, " + logInfo("AlterExternalTable canceled,because tableAfter has exist, " + "catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tableAfter.getTableName()); return; @@ -143,7 +143,7 @@ public String getTblNameAfter() { @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableBefore:[{}],tableAfter:[{}]", catalogName, dbName, + logInfo("catalogName:[{}],dbName:[{}],tableBefore:[{}],tableAfter:[{}]", catalogName, dbName, tableBefore.getTableName(), tableAfter.getTableName()); if (isRename) { processRename(); @@ -161,7 +161,7 @@ protected void process() throws MetastoreNotificationException { eventTime); } catch (Exception e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java index 2dd4c5671bcee7..b11a9eaa072b07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java @@ -54,11 +54,11 @@ protected static List getEvents(NotificationEvent event, @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); + logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName); Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java index e6c3e2e7eae9d0..8a22a479ad7e84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java @@ -48,7 +48,7 @@ private CreateTableEvent(NotificationEvent event, String catalogName) throws Met super(event, catalogName); Preconditions.checkArgument(MetastoreEventType.CREATE_TABLE.equals(getEventType())); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { CreateTableMessage createTableMessage = MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -57,7 +57,7 @@ private CreateTableEvent(NotificationEvent event, String catalogName) throws Met hmsTbl.setTableName(hmsTbl.getTableName().toLowerCase(Locale.ROOT)); } catch (Exception e) { throw new MetastoreNotificationException( - debugString("Unable to deserialize the event message"), e); + getMsgWithEventInfo("Unable to deserialize the event message"), e); } } @@ -78,12 +78,12 @@ protected boolean willChangeTableName() { @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName); + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName); Env.getCurrentEnv().getCatalogMgr() .registerExternalTableFromEvent(dbName, hmsTbl.getTableName(), catalogName, eventTime, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java index 107ce591a42b03..0649106661e6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java @@ -53,12 +53,12 @@ protected static List getEvents(NotificationEvent event, @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); + logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName); Env.getCurrentEnv().getCatalogMgr() .unregisterExternalDatabase(dbName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java index dd443010289126..737ad8f28b9051 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java @@ -57,7 +57,7 @@ private DropPartitionEvent(NotificationEvent event, super(event, catalogName); Preconditions.checkArgument(getEventType().equals(MetastoreEventType.DROP_PARTITION)); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { DropPartitionMessage dropPartitionMessage = MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -97,11 +97,11 @@ protected static List getEvents(NotificationEvent event, @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]", catalogName, dbName, tblName, + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]", catalogName, dbName, tblName, partitionNames.toString()); // bail out early if there are not partitions to process if (partitionNames.isEmpty()) { - infoLog("Partition list is empty. Ignoring this event."); + logInfo("Partition list is empty. Ignoring this event."); return; } Env.getCurrentEnv().getCatalogMgr() @@ -109,7 +109,7 @@ protected void process() throws MetastoreNotificationException { partitionNames, eventTime, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java index 6dcb16dedad369..dd67d6605274f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java @@ -48,7 +48,7 @@ private DropTableEvent(NotificationEvent event, super(event, catalogName); Preconditions.checkArgument(MetastoreEventType.DROP_TABLE.equals(getEventType())); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); try { JSONDropTableMessage dropTableMessage = (JSONDropTableMessage) MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat()) @@ -77,11 +77,11 @@ protected boolean willChangeTableName() { @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tableName); + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tableName); Env.getCurrentEnv().getCatalogMgr().unregisterExternalTable(dbName, tableName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java index e7e6643e647b98..bebfc8c2384f16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java @@ -38,6 +38,6 @@ protected static List getEvents(NotificationEvent event, @Override public void process() { - infoLog("Ignoring unknown event type " + metastoreNotificationEvent.getEventType()); + logInfo("Ignoring unknown event type " + metastoreNotificationEvent.getEventType()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java index 7b76d4913d51f2..4b4abd0264d53d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java @@ -42,7 +42,7 @@ private InsertEvent(NotificationEvent event, String catalogName) { super(event, catalogName); Preconditions.checkArgument(getEventType().equals(MetastoreEventType.INSERT)); Preconditions - .checkNotNull(event.getMessage(), debugString("Event message is null")); + .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event message is null")); } protected static List getEvents(NotificationEvent event, String catalogName) { @@ -62,7 +62,7 @@ protected boolean willChangeTableName() { @Override protected void process() throws MetastoreNotificationException { try { - infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName); + logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName); /** * Only when we use hive client to execute a `INSERT INTO TBL SELECT * ...` or `INSERT INTO TBL ...` sql * to a non-partitioned table then the hms will generate an insert event, and there is not @@ -75,7 +75,7 @@ protected void process() throws MetastoreNotificationException { eventTime); } catch (DdlException e) { throw new MetastoreNotificationException( - debugString("Failed to process event"), e); + getMsgWithEventInfo("Failed to process event"), e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java index 695dd57b215072..b0ec23b0661021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.hive.event; import org.apache.doris.datasource.MetaIdMappingsLog; -import org.apache.doris.datasource.hive.HMSCachedClient; import com.google.common.collect.ImmutableList; import org.apache.commons.lang3.StringUtils; @@ -31,43 +30,29 @@ import java.util.Map; /** - * Abstract base class for all MetastoreEvents. A MetastoreEvent is an object used to - * process a NotificationEvent received from metastore. + * The wrapper parent class of the NotificationEvent class */ public abstract class MetastoreEvent { private static final Logger LOG = LogManager.getLogger(MetastoreEvent.class); - // String.format compatible string to prepend event id and type - private static final String STR_FORMAT_EVENT_ID_TYPE = "EventId: %d EventType: %s "; - // logger format compatible string to prepend to a log formatted message - private static final String LOG_FORMAT_EVENT_ID_TYPE = "EventId: {} EventType: {} "; - - // the notification received from metastore which is processed by this - protected final NotificationEvent event; + protected final String catalogName; - // dbName from the event protected final String dbName; - // tblName from the event protected final String tblName; - // eventId of the event. Used instead of calling getter on event everytime protected final long eventId; - // eventTime of the event. Used instead of calling getter on event everytime protected final long eventTime; - // eventType from the NotificationEvent protected final MetastoreEventType eventType; - // Actual notificationEvent object received from Metastore + protected final NotificationEvent event; protected final NotificationEvent metastoreNotificationEvent; - protected final String catalogName; - // for test protected MetastoreEvent(long eventId, String catalogName, String dbName, - String tblName, MetastoreEventType eventType) { + String tblName, MetastoreEventType eventType) { this.eventId = eventId; this.eventTime = -1L; this.catalogName = catalogName; @@ -103,127 +88,46 @@ protected MetastoreEvent(NotificationEvent event, String catalogName) { this.catalogName = catalogName; } - public long getEventId() { - return eventId; - } - - public MetastoreEventType getEventType() { - return eventType; - } - - public String getDbName() { - return dbName; - } - - public String getTblName() { - return tblName; - } - /** - * Checks if the given event can be batched into this event. Default behavior is - * to return false which can be overridden by a sub-class. + * Can batch processing be performed to improve processing performance * - * @param event The event under consideration to be batched into this event. - * @return false if event cannot be batched into this event; otherwise true. + * @param event + * @return */ protected boolean canBeBatched(MetastoreEvent event) { return false; } - /** - * Adds the given event into the batch of events represented by this event. Default - * implementation is to return null. Sub-classes must override this method to - * implement batching. - * - * @param event The event which needs to be added to the batch. - * @return The batch event which represents all the events batched into this event - * until now including the given event. - */ - protected MetastoreEvent addToBatchEvents(MetastoreEvent event) { - return null; - } - - /** - * Returns the number of events represented by this event. For most events this is 1. - * In case of batch events this could be more than 1. - */ - protected int getNumberOfEvents() { - return 1; - } - - /** - * Certain events like ALTER_TABLE or ALTER_PARTITION implement logic to ignore - * some events because they do not affect query results. - * - * @return true if this event can be skipped. - */ - protected boolean canBeSkipped() { - return false; - } - - /** - * Process the information available in the NotificationEvent. - * Better not to call (direct/indirect) apis of {@link HMSCachedClient} - * during handling hms events (Reference to https://github.com/apache/doris/pull/19120). - * Try to add some fallback strategies if it is highly necessary. - */ protected abstract void process() throws MetastoreNotificationException; - /** - * Helper method to get debug string with helpful event information prepended to the - * message. This can be used to generate helpful exception messages - * - * @param msgFormatString String value to be used in String.format() for the given message - * @param args args to the String.format() for the given msgFormatString - */ - protected String debugString(String msgFormatString, Object... args) { - String formatString = STR_FORMAT_EVENT_ID_TYPE + msgFormatString; - Object[] formatArgs = getLogFormatArgs(args); - return String.format(formatString, formatArgs); - } - - /** - * Helper method to generate the format args after prepending the event id and type - */ - private Object[] getLogFormatArgs(Object[] args) { - Object[] formatArgs = new Object[args.length + 2]; - formatArgs[0] = eventId; - formatArgs[1] = eventType; - int i = 2; - for (Object arg : args) { - formatArgs[i] = arg; - i++; - } - return formatArgs; + protected String getMsgWithEventInfo(String formatSuffix, Object... args) { + String format = "EventId: %d EventType: %s " + formatSuffix; + Object[] argsWithEventInfo = getArgsWithEventInfo(args); + return String.format(format, argsWithEventInfo); } - /** - * Logs at info level the given log formatted string and its args. The log formatted - * string should have {} pair at the appropriate location in the string for each arg - * value provided. This method prepends the event id and event type before logging the - * message. No-op if the log level is not at INFO - */ - protected void infoLog(String logFormattedStr, Object... args) { + protected void logInfo(String formatSuffix, Object... args) { if (!LOG.isInfoEnabled()) { return; } - String formatString = LOG_FORMAT_EVENT_ID_TYPE + logFormattedStr; - Object[] formatArgs = getLogFormatArgs(args); - LOG.info(formatString, formatArgs); + String format = "EventId: {} EventType: {} " + formatSuffix; + Object[] argsWithEventInfo = getArgsWithEventInfo(args); + LOG.info(format, argsWithEventInfo); } /** - * Similar to infoLog excepts logs at debug level + * Add event information to the parameters */ - protected void debugLog(String logFormattedStr, Object... args) { - if (!LOG.isDebugEnabled()) { - return; - } - String formatString = LOG_FORMAT_EVENT_ID_TYPE + logFormattedStr; - Object[] formatArgs = getLogFormatArgs(args); - if (LOG.isDebugEnabled()) { - LOG.debug(formatString, formatArgs); + private Object[] getArgsWithEventInfo(Object[] args) { + Object[] res = new Object[args.length + 2]; + res[0] = eventId; + res[1] = eventType; + int i = 2; + for (Object arg : args) { + res[i] = arg; + i++; } + return res; } protected String getPartitionName(Map part, List partitionColNames) { @@ -253,8 +157,27 @@ protected List transferToMetaIdMappings() { return ImmutableList.of(); } + public String getDbName() { + return dbName; + } + + public String getTblName() { + return tblName; + } + + public long getEventId() { + return eventId; + } + + public MetastoreEventType getEventType() { + return eventType; + } + @Override public String toString() { - return String.format(STR_FORMAT_EVENT_ID_TYPE, eventId, eventType); + return "MetastoreEvent{" + + "eventId=" + eventId + + ", eventType=" + eventType + + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java index cbd0bfb5fa6fb5..73054773402bad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java @@ -165,7 +165,7 @@ private void doExecute(List events, HMSExternalCatalog hmsExtern } catch (HMSClientException hmsClientException) { if (hmsClientException.getCause() != null && hmsClientException.getCause() instanceof NoSuchObjectException) { - LOG.warn(event.debugString("Failed to process event and skip"), hmsClientException); + LOG.warn(event.getMsgWithEventInfo("Failed to process event and skip"), hmsClientException); } else { updateLastSyncedEventId(hmsExternalCatalog, event.getEventId() - 1); throw hmsClientException;