From 0775e1909a8a5f89a6eafbed6ebef56f56c42599 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Fri, 25 Aug 2023 15:18:35 -0700 Subject: [PATCH 01/34] initial imp --- .../streaming/SnowflakeSinkServiceV2.java | 14 +- .../streaming/TopicPartitionChannel.java | 112 +++++---- .../SnowflakeTelemetryChannelStatus.java | 218 ++++++++++++++++++ .../SnowflakeTelemetryServiceV2.java | 4 +- .../SnowflakeTelemetryBasicInfo.java | 4 +- .../SnowflakeTelemetryServiceFactory.java | 2 +- .../telemetry/TelemetryConstants.java | 3 + .../internal/ConnectionServiceIT.java | 2 +- .../streaming/TopicPartitionChannelIT.java | 2 + .../streaming/TopicPartitionChannelTest.java | 126 ++++++++-- .../SnowflakeTelemetryChannelStatusTest.java | 59 +++++ 11 files changed, 462 insertions(+), 84 deletions(-) create mode 100644 src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java rename src/main/java/com/snowflake/kafka/connector/internal/streaming/{ => telemetry}/SnowflakeTelemetryServiceV2.java (92%) create mode 100644 src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java index 4083e2af3..62ef75a0b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java @@ -8,6 +8,7 @@ import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; import com.snowflake.kafka.connector.Utils; import com.snowflake.kafka.connector.dlq.KafkaRecordErrorReporter; @@ -15,6 +16,7 @@ import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.SnowflakeErrors; import com.snowflake.kafka.connector.internal.SnowflakeSinkService; +import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; import com.snowflake.kafka.connector.records.SnowflakeMetadataConfig; @@ -69,6 +71,7 @@ public class SnowflakeSinkServiceV2 implements SnowflakeSinkService { // default is true unless the configuration provided is false; // If this is true, we will enable Mbean for required classes and emit JMX metrics for monitoring private boolean enableCustomJMXMonitoring = SnowflakeSinkConnectorConfig.JMX_OPT_DEFAULT; + private MetricsJmxReporter metricsJmxReporter; /** * Fetching this from {@link org.apache.kafka.connect.sink.SinkTaskContext}'s {@link @@ -128,6 +131,12 @@ public SnowflakeSinkServiceV2( this.partitionsToChannel = new HashMap<>(); this.tableName2SchemaEvolutionPermission = new HashMap<>(); + + // jmx + String connectorName = conn == null || Strings.isNullOrEmpty(this.conn.getConnectorName()) ? + "default_connector" : + this.conn.getConnectorName(); + this.metricsJmxReporter = new MetricsJmxReporter(new MetricRegistry(), connectorName); } @VisibleForTesting @@ -242,7 +251,8 @@ private void createStreamingChannelForTopicPartition( this.conn, this.recordService, this.conn.getTelemetryClient(), - this.enableCustomJMXMonitoring)); + this.enableCustomJMXMonitoring, + this.metricsJmxReporter)); } /** @@ -306,7 +316,7 @@ public long getOffset(TopicPartition topicPartition) { partitionChannelKey(topicPartition.topic(), topicPartition.partition()); if (partitionsToChannel.containsKey(partitionChannelKey)) { long offset = partitionsToChannel.get(partitionChannelKey).getOffsetSafeToCommitToKafka(); - partitionsToChannel.get(partitionChannelKey).setLatestConsumerOffset(offset); + partitionsToChannel.get(partitionChannelKey).setLatestConsumerOffset(offset, false); return offset; } else { LOGGER.warn( diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 4f3d4cded..9ee04f38b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -23,6 +23,7 @@ import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; import com.snowflake.kafka.connector.records.SnowflakeJsonSchema; @@ -53,6 +54,7 @@ import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTaskContext; +import org.checkerframework.checker.units.qual.A; /** * This is a wrapper on top of Streaming Ingest Channel which is responsible for ingesting rows to @@ -91,7 +93,7 @@ public class TopicPartitionChannel { // This offset represents the data persisted in Snowflake. More specifically it is the Snowflake // offset determined from the insertRows API call. It is set after calling the fetchOffsetToken // API for this channel - private final AtomicLong offsetPersistedInSnowflake = + protected final AtomicLong offsetPersistedInSnowflake = new AtomicLong(NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE); // This offset represents the data buffered in KC. More specifically it is the KC offset to ensure @@ -177,6 +179,8 @@ public class TopicPartitionChannel { // Reference to the Snowflake connection service private final SnowflakeConnectionService conn; + private final SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus; + /** * Used to send telemetry to Snowflake. Currently, TelemetryClient created from a Snowflake * Connection Object, i.e. not a session-less Client @@ -207,7 +211,8 @@ public TopicPartitionChannel( null, /* Null Connection */ new RecordService(null /* Null Telemetry Service*/), null, - false); + false, + null); } /** @@ -240,7 +245,8 @@ public TopicPartitionChannel( SnowflakeConnectionService conn, RecordService recordService, SnowflakeTelemetryService telemetryService, - boolean enableCustomJMXMonitoring) { + boolean enableCustomJMXMonitoring, + MetricsJmxReporter metricsJmxReporter) { this.streamingIngestClient = Preconditions.checkNotNull(streamingIngestClient); Preconditions.checkState(!streamingIngestClient.isClosed()); this.topicPartition = Preconditions.checkNotNull(topicPartition); @@ -271,11 +277,15 @@ public TopicPartitionChannel( this.enableSchemaEvolution = this.enableSchematization && hasSchemaEvolutionPermission; + // telemetry and metrics + this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); + // Open channel and reset the offset in kafka this.channel = Preconditions.checkNotNull(openChannelForTable()); final long lastCommittedOffsetToken = fetchOffsetTokenWithRetry(); - this.offsetPersistedInSnowflake.set(lastCommittedOffsetToken); - this.processedOffset.set(lastCommittedOffsetToken); + this.setOffsetPersistedInSnowflake(lastCommittedOffsetToken); + this.setProcessedOffset(lastCommittedOffsetToken); + if (lastCommittedOffsetToken != NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE) { this.sinkTaskContext.offset(this.topicPartition, lastCommittedOffsetToken + 1L); } else { @@ -284,12 +294,6 @@ public TopicPartitionChannel( + " correct offset instead", this.getChannelName()); } - - if (enableCustomJMXMonitoring) { - MetricsJmxReporter metricsJmxReporter = - new MetricsJmxReporter(new MetricRegistry(), conn.getConnectorName()); - this.registerChannelJMXMetrics(channelName, metricsJmxReporter); - } } /** @@ -309,7 +313,7 @@ public void insertRecordToBuffer(SinkRecord kafkaSinkRecord) { // Set the consumer offset to be the first record that Kafka sends us if (latestConsumerOffset == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE) { - latestConsumerOffset = kafkaSinkRecord.kafkaOffset(); + this.setLatestConsumerOffset(kafkaSinkRecord.kafkaOffset(), true); } // Ignore adding to the buffer until we see the expected offset value @@ -325,7 +329,7 @@ public void insertRecordToBuffer(SinkRecord kafkaSinkRecord) { bufferLock.lock(); try { this.streamingBuffer.insert(kafkaSinkRecord); - this.processedOffset.set(kafkaSinkRecord.kafkaOffset()); + this.setProcessedOffset(kafkaSinkRecord.kafkaOffset()); // # of records or size based flushing if (this.streamingBufferThreshold.shouldFlushOnBufferByteSize( streamingBuffer.getBufferSizeBytes()) @@ -937,8 +941,8 @@ private void resetChannelMetadataAfterRecovery( // Need to update the in memory processed offset otherwise if same offset is send again, it // might get rejected. - this.offsetPersistedInSnowflake.set(offsetRecoveredFromSnowflake); - this.processedOffset.set(offsetRecoveredFromSnowflake); + this.setOffsetPersistedInSnowflake(offsetRecoveredFromSnowflake); + this.setProcessedOffset(offsetRecoveredFromSnowflake); // State that there was some exception and only clear that state when we have received offset // starting from offsetRecoveredFromSnowflake @@ -1036,6 +1040,8 @@ private SnowflakeStreamingIngestChannel openChannelForTable() { public void closeChannel() { try { this.channel.close().get(); + this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); + this.snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { final String errMsg = String.format( @@ -1065,6 +1071,24 @@ public String getChannelName() { return this.channel.getFullyQualifiedName(); } + // ------ SETTERS ------ // + private void setOffsetPersistedInSnowflake(long offsetPersistedInSnowflake) { + this.offsetPersistedInSnowflake.set(offsetPersistedInSnowflake); + this.snowflakeTelemetryChannelStatus.setOffsetPersistedInSnowflake(offsetPersistedInSnowflake); + } + + private void setProcessedOffset(long processedOffset) { + this.processedOffset.set(processedOffset); + this.snowflakeTelemetryChannelStatus.setProcessedOffset(processedOffset); + } + + protected void setLatestConsumerOffset(long consumerOffset, boolean force) { + if (force || consumerOffset > this.latestConsumerOffset) { + this.latestConsumerOffset = consumerOffset; + this.snowflakeTelemetryChannelStatus.setLatestConsumerOffset(consumerOffset); + } + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -1080,6 +1104,16 @@ protected long getOffsetPersistedInSnowflake() { return this.offsetPersistedInSnowflake.get(); } + @VisibleForTesting + protected long getProcessedOffset() { + return this.processedOffset.get(); + } + + @VisibleForTesting + protected long getLatestConsumerOffset() { + return this.latestConsumerOffset; + } + @VisibleForTesting protected boolean isPartitionBufferEmpty() { return streamingBuffer.isEmpty(); @@ -1095,10 +1129,9 @@ protected SnowflakeTelemetryService getTelemetryServiceV2() { return this.telemetryServiceV2; } - protected void setLatestConsumerOffset(long consumerOffset) { - if (consumerOffset > this.latestConsumerOffset) { - this.latestConsumerOffset = consumerOffset; - } + @VisibleForTesting + protected SnowflakeTelemetryChannelStatus getSnowflakeTelemetryChannelStatus() { + return this.snowflakeTelemetryChannelStatus; } /** @@ -1317,45 +1350,4 @@ public String toString() { return "[" + this.name() + "]"; } } - - /** - * Registers all the Metrics inside the metricRegistry. - * - * @param channelName channelName - * @param metricsJmxReporter wrapper class for registering all metrics related to above connector - * and channel - */ - public void registerChannelJMXMetrics( - final String channelName, MetricsJmxReporter metricsJmxReporter) { - MetricRegistry currentMetricRegistry = metricsJmxReporter.getMetricRegistry(); - - // Lazily remove all registered metrics from the registry since this can be invoked during - // partition reassignment - LOGGER.debug( - "Registering metrics for channel:{}, existing:{}", - channelName, - metricsJmxReporter.getMetricRegistry().getMetrics().keySet().toString()); - metricsJmxReporter.removeMetricsFromRegistry(channelName); - - try { - // offset - currentMetricRegistry.register( - constructMetricName( - channelName, OFFSET_SUB_DOMAIN, MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), - (Gauge) this.offsetPersistedInSnowflake::get); - - currentMetricRegistry.register( - constructMetricName(channelName, OFFSET_SUB_DOMAIN, MetricsUtil.PROCESSED_OFFSET), - (Gauge) this.processedOffset::get); - - currentMetricRegistry.register( - constructMetricName(channelName, OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), - (Gauge) () -> this.latestConsumerOffset); - - } catch (IllegalArgumentException ex) { - LOGGER.warn("Metrics already present:{}", ex.getMessage()); - } - - metricsJmxReporter.start(); - } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java new file mode 100644 index 000000000..8c6f5dd55 --- /dev/null +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Snowflake Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.snowflake.kafka.connector.internal.streaming.telemetry; + +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.FILE_COUNT_SUB_DOMAIN; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.FILE_COUNT_TABLE_STAGE_INGESTION_FAIL; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.LATENCY_SUB_DOMAIN; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.LATEST_CONSUMER_OFFSET; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.OFFSET_SUB_DOMAIN; +import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.constructMetricName; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_COMMIT_LAG_FILE_COUNT; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_COMMIT_LAG_MS; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_INGESTION_LAG_FILE_COUNT; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_INGESTION_LAG_MS; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_KAFKA_LAG_MS; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_KAFKA_LAG_RECORD_COUNT; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.BYTE_NUMBER; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CLEANER_RESTART_COUNT; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.COMMITTED_OFFSET; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.END_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_ON_INGESTION; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_ON_STAGE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_PURGED; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_TABLE_STAGE_BROKEN_RECORD; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_TABLE_STAGE_INGEST_FAIL; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FLUSHED_OFFSET; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.MEMORY_USAGE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PIPE_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PROCESSED_OFFSET; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PURGED_OFFSET; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.RECORD_NUMBER; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.STAGE_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; +import com.snowflake.kafka.connector.internal.metrics.MetricsUtil.EventType; +import java.util.Arrays; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.LongUnaryOperator; + +import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; +import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; +import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; +import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.checkerframework.checker.units.qual.A; + +/** + * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake + * when the TopicPartitionChannel closes + * + *

Most of the data sent to Snowflake is an aggregated data. + */ +public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo { + public static final long NUM_METRICS = 3; // update when new metrics are added + private static final long INITIAL_OFFSET = -1; + + // channel properties + private final String channelName; + private final boolean enableCustomJMXConfig; + private final MetricsJmxReporter metricsJmxReporter; + + private final AtomicLong startTime; + + // offsets + private AtomicLong offsetPersistedInSnowflake; + private AtomicLong processedOffset; + private long latestConsumerOffset; + + public SnowflakeTelemetryChannelStatus( + final String tableName, + final String channelName, + final boolean enableCustomJMXConfig, + final MetricsJmxReporter metricsJmxReporter) { + super(tableName); + this.channelName = channelName; + this.enableCustomJMXConfig = enableCustomJMXConfig; + this.metricsJmxReporter = metricsJmxReporter; + + this.startTime = new AtomicLong(System.currentTimeMillis()); + + this.offsetPersistedInSnowflake = new AtomicLong(INITIAL_OFFSET); + this.processedOffset = new AtomicLong(INITIAL_OFFSET); + this.latestConsumerOffset = INITIAL_OFFSET; + if (enableCustomJMXConfig) { + if (metricsJmxReporter == null) { + LOGGER.error("Invalid metrics JMX reporter, no metrics will be reported"); + } else { + this.registerChannelJMXMetrics(); + } + } + } + + @Override + public boolean isEmpty() { + // Check that all properties are still at the default value. + return this.offsetPersistedInSnowflake.get() == INITIAL_OFFSET + && this.processedOffset.get() == INITIAL_OFFSET + && this.latestConsumerOffset == INITIAL_OFFSET; + } + + @Override + public void dumpTo(ObjectNode msg) { + msg.put(TelemetryConstants.TABLE_NAME, this.tableName); + msg.put(TelemetryConstants.CHANNEL_NAME, this.channelName); + + msg.put(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, offsetPersistedInSnowflake.get()); + msg.put(TelemetryConstants.PROCESSED_OFFSET, processedOffset.get()); + msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, latestConsumerOffset); + + final long currTime = System.currentTimeMillis(); + msg.put(TelemetryConstants.START_TIME, startTime.getAndSet(currTime)); + msg.put(TelemetryConstants.END_TIME, currTime); + } + + /** + * Registers all the Metrics inside the metricRegistry. + */ + private void registerChannelJMXMetrics() { + LOGGER.debug( + "Registering new metrics for channel:{}, removing existing metrics:{}", + this.channelName, + this.metricsJmxReporter.getMetricRegistry().getMetrics().keySet().toString()); + this.metricsJmxReporter.removeMetricsFromRegistry(this.channelName); + + MetricRegistry currentMetricRegistry = this.metricsJmxReporter.getMetricRegistry(); + + try { + // offsets + currentMetricRegistry.register( + constructMetricName( + this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), + (Gauge) this.offsetPersistedInSnowflake::get); + + currentMetricRegistry.register( + constructMetricName(this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.PROCESSED_OFFSET), + (Gauge) this.processedOffset::get); + + currentMetricRegistry.register( + constructMetricName(this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), + (Gauge) () -> this.latestConsumerOffset); + } catch (IllegalArgumentException ex) { + LOGGER.warn("Metrics already present:{}", ex.getMessage()); + } + + this.metricsJmxReporter.start(); + } + + /** + * Unregisters the JMX metrics + */ + public void unregisterChannelJMXMetrics() { + if (this.metricsJmxReporter != null) { + LOGGER.debug( + "Removing metrics for channel:{}, existing metrics:{}", + this.channelName, + metricsJmxReporter.getMetricRegistry().getMetrics().keySet().toString()); + this.metricsJmxReporter.removeMetricsFromRegistry(this.channelName); + } + } + + public long getOffsetPersistedInSnowflake() { + return this.offsetPersistedInSnowflake.get(); + } + + public void setOffsetPersistedInSnowflake(long offsetPersistedInSnowflake) { + this.offsetPersistedInSnowflake.set(offsetPersistedInSnowflake); + } + + public long getProcessedOffset() { + return this.processedOffset.get(); + } + + public void setProcessedOffset(long processedOffset) { + this.processedOffset.set(processedOffset); + } + + public long getLatestConsumerOffset() { + return this.latestConsumerOffset; + } + + public void setLatestConsumerOffset(long latestConsumerOffset) { + this.latestConsumerOffset = latestConsumerOffset; + } + + public MetricsJmxReporter getMetricsJmxReporter() { + return this.metricsJmxReporter; + } +} diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeTelemetryServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java similarity index 92% rename from src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeTelemetryServiceV2.java rename to src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java index d14941aa3..b379e9579 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeTelemetryServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java @@ -15,8 +15,9 @@ * under the License. */ -package com.snowflake.kafka.connector.internal.streaming; +package com.snowflake.kafka.connector.internal.streaming.telemetry; +import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import java.sql.Connection; @@ -41,7 +42,6 @@ public SnowflakeTelemetryServiceV2(Connection conn) { @Override public void reportKafkaPartitionUsage( SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) { - throw new IllegalStateException("Snowpipe Streaming Doesnt Have Pipe Usage"); } @Override diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java index 352615a34..8c81db637 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java @@ -7,9 +7,9 @@ /** Minimum information needed to sent to Snowflake through Telemetry API */ public abstract class SnowflakeTelemetryBasicInfo { - final String tableName; + public final String tableName; - static final KCLogger LOGGER = new KCLogger(SnowflakeTelemetryBasicInfo.class.getName()); + public static final KCLogger LOGGER = new KCLogger(SnowflakeTelemetryBasicInfo.class.getName()); /** * Base Constructor. Accepts a tableName and StageName. diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceFactory.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceFactory.java index 9f666112a..4e7d3b449 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceFactory.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceFactory.java @@ -1,7 +1,7 @@ package com.snowflake.kafka.connector.internal.telemetry; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import com.snowflake.kafka.connector.internal.streaming.SnowflakeTelemetryServiceV2; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import java.sql.Connection; /** diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 1a1ac72d1..457d361d1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -8,11 +8,14 @@ public final class TelemetryConstants { public static final String TABLE_NAME = "table_name"; public static final String STAGE_NAME = "stage_name"; public static final String PIPE_NAME = "pipe_name"; + public static final String CHANNEL_NAME = "channel_name"; public static final String PROCESSED_OFFSET = "processed-offset"; public static final String FLUSHED_OFFSET = "flushed-offset"; public static final String COMMITTED_OFFSET = "committed-offset"; public static final String PURGED_OFFSET = "purged-offset"; + public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; public static final String RECORD_NUMBER = "record_number"; public static final String BYTE_NUMBER = "byte_number"; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/ConnectionServiceIT.java b/src/test/java/com/snowflake/kafka/connector/internal/ConnectionServiceIT.java index 23b0d720c..66078b016 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/ConnectionServiceIT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/ConnectionServiceIT.java @@ -6,7 +6,7 @@ import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; import com.snowflake.kafka.connector.Utils; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import com.snowflake.kafka.connector.internal.streaming.SnowflakeTelemetryServiceV2; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryServiceV1; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java index dcc48da28..8e51d762e 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java @@ -12,6 +12,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; + +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; import org.apache.kafka.common.TopicPartition; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index ffa905167..a6dec629c 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -7,8 +7,11 @@ import static com.snowflake.kafka.connector.internal.TestUtils.createBigAvroRecords; import static com.snowflake.kafka.connector.internal.TestUtils.createNativeJsonSinkRecords; import static com.snowflake.kafka.connector.internal.streaming.StreamingUtils.MAX_GET_OFFSET_TOKEN_RETRIES; +import static com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus.NUM_METRICS; +import static org.mockito.ArgumentMatchers.eq; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.jmx.JmxReporter; import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; import com.snowflake.kafka.connector.dlq.InMemoryKafkaRecordErrorReporter; import com.snowflake.kafka.connector.dlq.KafkaRecordErrorReporter; @@ -16,6 +19,7 @@ import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.TestUtils; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; import java.nio.charset.StandardCharsets; @@ -97,6 +101,7 @@ public void setupEachTest() { mockStreamingChannel = Mockito.mock(SnowflakeStreamingIngestChannel.class); mockKafkaRecordErrorReporter = Mockito.mock(KafkaRecordErrorReporter.class); mockSinkTaskContext = Mockito.mock(SinkTaskContext.class); + mockSnowflakeConnectionService = Mockito.mock(SnowflakeConnectionService.class); mockTelemetryService = Mockito.mock(SnowflakeTelemetryService.class); Mockito.when(mockStreamingClient.isClosed()).thenReturn(false); Mockito.when(mockStreamingClient.openChannel(ArgumentMatchers.any(OpenChannelRequest.class))) @@ -104,7 +109,7 @@ public void setupEachTest() { Mockito.when(mockStreamingChannel.getFullyQualifiedName()).thenReturn(TEST_CHANNEL_NAME); this.topicPartition = new TopicPartition(TOPIC, PARTITION); this.sfConnectorConfig = TestUtils.getConfig(); - this.streamingBufferThreshold = new StreamingBufferThreshold(10, 10_000, 1); + this.streamingBufferThreshold = new StreamingBufferThreshold(1, 10_000, 1); this.sfConnectorConfig.put( SnowflakeSinkConnectorConfig.ENABLE_SCHEMATIZATION_CONFIG, Boolean.toString(this.enableSchematization)); @@ -229,7 +234,7 @@ public void testCloseChannelException() throws Exception { mockSnowflakeConnectionService, new RecordService(mockTelemetryService), mockTelemetryService, - false); + false, null); topicPartitionChannel.closeChannel(); } @@ -495,7 +500,7 @@ public void testInsertRowsWithSchemaEvolution() throws Exception { conn, new RecordService(), mockTelemetryService, - false); + false, null); final int noOfRecords = 3; List records = @@ -621,7 +626,7 @@ public void testInsertRows_ValidationResponseHasErrors_NoErrorTolerance() throws mockSnowflakeConnectionService, new RecordService(mockTelemetryService), mockTelemetryService, - false); + false, null); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -817,27 +822,116 @@ public void testBigAvroBufferBytesThreshold() throws Exception { } @Test - public void testRegisterJmxMetrics() { + public void testTopicPartitionChannelMetrics() throws Exception { + // variables + int noOfRecords = 5; + + // setup jmxreporter MetricRegistry metricRegistry = Mockito.spy(MetricRegistry.class); MetricsJmxReporter metricsJmxReporter = Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); + // setup insert + Mockito.when( + mockStreamingChannel.insertRows( + ArgumentMatchers.any(Iterable.class), ArgumentMatchers.any(String.class))) + .thenReturn(new InsertValidationResponse()); + Mockito.when( + mockStreamingChannel.insertRow( + ArgumentMatchers.any(Map.class), ArgumentMatchers.any(String.class))) + .thenReturn(new InsertValidationResponse()); + Mockito.when(mockStreamingChannel.close()).thenReturn(Mockito.mock(CompletableFuture.class)); + TopicPartitionChannel topicPartitionChannel = new TopicPartitionChannel( - mockStreamingClient, - topicPartition, + this.mockStreamingClient, + this.topicPartition, TEST_CHANNEL_NAME, TEST_TABLE_NAME, - streamingBufferThreshold, - sfConnectorConfig, - mockKafkaRecordErrorReporter, - mockSinkTaskContext); + this.enableSchematization, + this.streamingBufferThreshold, + this.sfConnectorConfig, + this.mockKafkaRecordErrorReporter, + this.mockSinkTaskContext, + this.mockSnowflakeConnectionService, + new RecordService(), + this.mockTelemetryService, + true, + metricsJmxReporter); + + // insert records + List records = + TestUtils.createJsonStringSinkRecords(0, noOfRecords, TOPIC, PARTITION); + records.forEach(topicPartitionChannel::insertRecordToBuffer); + + Thread.sleep(this.streamingBufferThreshold.getFlushTimeThresholdSeconds() + 1); + topicPartitionChannel.insertBufferedRecordsIfFlushTimeThresholdReached(); + + // verify metrics + SnowflakeTelemetryChannelStatus resultStatus = topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); + + assert resultStatus.getOffsetPersistedInSnowflake() == topicPartitionChannel.getOffsetPersistedInSnowflake(); + assert resultStatus.getOffsetPersistedInSnowflake() == -1; + assert resultStatus.getProcessedOffset() == topicPartitionChannel.getProcessedOffset(); + assert resultStatus.getProcessedOffset() == noOfRecords - 1; + assert resultStatus.getLatestConsumerOffset() == topicPartitionChannel.getLatestConsumerOffset(); + assert resultStatus.getLatestConsumerOffset() == 0; + + assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == SnowflakeTelemetryChannelStatus.NUM_METRICS; + + // verify telemetry was sent when channel closed + topicPartitionChannel.closeChannel(); + Mockito.verify(this.mockTelemetryService, Mockito.times(1)).reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), eq(true)); + assert topicPartitionChannel.getSnowflakeTelemetryChannelStatus().getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == 0; + } - // test - topicPartitionChannel.registerChannelJMXMetrics(TEST_CHANNEL_NAME, metricsJmxReporter); - // verify 3 metrics registered and started - Mockito.verify(metricsJmxReporter, Mockito.times(1)).start(); - Mockito.verify(metricRegistry, Mockito.times(3)).register(Mockito.anyString(), Mockito.any()); + @Test + public void testTopicPartitionChannelInvalidJmxReporter() throws Exception { + // variables + int noOfRecords = 5; + + // setup insert + Mockito.when( + mockStreamingChannel.insertRows( + ArgumentMatchers.any(Iterable.class), ArgumentMatchers.any(String.class))) + .thenReturn(new InsertValidationResponse()); + Mockito.when( + mockStreamingChannel.insertRow( + ArgumentMatchers.any(Map.class), ArgumentMatchers.any(String.class))) + .thenReturn(new InsertValidationResponse()); + Mockito.when(mockStreamingChannel.close()).thenReturn(Mockito.mock(CompletableFuture.class)); + + TopicPartitionChannel topicPartitionChannel = + new TopicPartitionChannel( + this.mockStreamingClient, + this.topicPartition, + TEST_CHANNEL_NAME, + TEST_TABLE_NAME, + this.enableSchematization, + this.streamingBufferThreshold, + this.sfConnectorConfig, + this.mockKafkaRecordErrorReporter, + this.mockSinkTaskContext, + this.mockSnowflakeConnectionService, + new RecordService(), + this.mockTelemetryService, + true, + null); + + // insert records + List records = + TestUtils.createJsonStringSinkRecords(0, noOfRecords, TOPIC, PARTITION); + records.forEach(topicPartitionChannel::insertRecordToBuffer); + + Thread.sleep(this.streamingBufferThreshold.getFlushTimeThresholdSeconds() + 1); + topicPartitionChannel.insertBufferedRecordsIfFlushTimeThresholdReached(); + + // verify no errors are thrown with invalid jmx reporter but enabled jmx monitoring + SnowflakeTelemetryChannelStatus resultStatus = topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); + assert resultStatus.getMetricsJmxReporter() == null; + + topicPartitionChannel.closeChannel(); + assert resultStatus.getMetricsJmxReporter() == null; } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java new file mode 100644 index 000000000..38623fd8e --- /dev/null +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -0,0 +1,59 @@ +package com.snowflake.kafka.connector.internal.streaming.telemetry; + +import com.codahale.metrics.MetricRegistry; +import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import org.junit.Test; +import org.mockito.Mockito; + +import static com.snowflake.kafka.connector.internal.TestUtils.TEST_CONNECTOR_NAME; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class SnowflakeTelemetryChannelStatusTest { + private final String tableName = "tableName"; + private final String channelName = "channelName"; + private final String connectorName = "connectorName"; + + @Test + public void testRegisterAndUnregisterJmxMetrics() { + MetricRegistry metricRegistry = Mockito.spy(MetricRegistry.class); + MetricsJmxReporter metricsJmxReporter = + Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); + + // test register + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, true, metricsJmxReporter); + verify(metricsJmxReporter, times(1)).start(); + verify(metricRegistry, times(3)).register(Mockito.anyString(), Mockito.any()); + verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); + + // test unregister + snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + verify(metricsJmxReporter, times(2)).removeMetricsFromRegistry(channelName); + } + + @Test + public void testDisabledJmx() { + MetricRegistry metricRegistry = Mockito.spy(MetricRegistry.class); + MetricsJmxReporter metricsJmxReporter = + Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); + + // test register + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, false, metricsJmxReporter); + verify(metricsJmxReporter, times(0)).start(); + verify(metricRegistry, times(0)).register(Mockito.anyString(), Mockito.any()); + verify(metricsJmxReporter, times(0)).removeMetricsFromRegistry(channelName); + + // test unregister + snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); + } + + @Test + public void testInvalidJmxReporter() { + // invalid jmx reporter should not error out + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, true, null); + snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + } + + +} From 273c8d70f596ee456d69d821d7e05967d40d0882 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Fri, 25 Aug 2023 15:18:39 -0700 Subject: [PATCH 02/34] autoformatting --- .../streaming/SnowflakeSinkServiceV2.java | 7 +- .../streaming/TopicPartitionChannel.java | 10 +-- .../SnowflakeTelemetryChannelStatus.java | 70 ++++--------------- .../SnowflakeTelemetryServiceV2.java | 3 +- .../streaming/TopicPartitionChannelIT.java | 3 +- .../streaming/TopicPartitionChannelTest.java | 38 ++++++---- .../SnowflakeTelemetryChannelStatusTest.java | 19 ++--- 7 files changed, 56 insertions(+), 94 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java index 62ef75a0b..f3b0f8c80 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java @@ -133,9 +133,10 @@ public SnowflakeSinkServiceV2( this.tableName2SchemaEvolutionPermission = new HashMap<>(); // jmx - String connectorName = conn == null || Strings.isNullOrEmpty(this.conn.getConnectorName()) ? - "default_connector" : - this.conn.getConnectorName(); + String connectorName = + conn == null || Strings.isNullOrEmpty(this.conn.getConnectorName()) + ? "default_connector" + : this.conn.getConnectorName(); this.metricsJmxReporter = new MetricsJmxReporter(new MetricRegistry(), connectorName); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 9ee04f38b..dcd98f90b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -2,15 +2,11 @@ import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG; import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.ERRORS_TOLERANCE_CONFIG; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.OFFSET_SUB_DOMAIN; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.constructMetricName; import static com.snowflake.kafka.connector.internal.streaming.StreamingUtils.DURATION_BETWEEN_GET_OFFSET_TOKEN_RETRY; import static com.snowflake.kafka.connector.internal.streaming.StreamingUtils.MAX_GET_OFFSET_TOKEN_RETRIES; import static java.time.temporal.ChronoUnit.SECONDS; import static org.apache.kafka.common.record.TimestampType.NO_TIMESTAMP_TYPE; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; @@ -22,7 +18,6 @@ import com.snowflake.kafka.connector.internal.PartitionBuffer; import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; -import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; @@ -54,7 +49,6 @@ import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTaskContext; -import org.checkerframework.checker.units.qual.A; /** * This is a wrapper on top of Streaming Ingest Channel which is responsible for ingesting rows to @@ -278,7 +272,9 @@ public TopicPartitionChannel( this.enableSchemaEvolution = this.enableSchematization && hasSchemaEvolutionPermission; // telemetry and metrics - this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); + this.snowflakeTelemetryChannelStatus = + new SnowflakeTelemetryChannelStatus( + tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); // Open channel and reset the offset in kafka this.channel = Preconditions.checkNotNull(openChannelForTable()); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 8c6f5dd55..7eb3b0dba 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -17,66 +17,20 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.FILE_COUNT_SUB_DOMAIN; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.FILE_COUNT_TABLE_STAGE_INGESTION_FAIL; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.LATENCY_SUB_DOMAIN; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.LATEST_CONSUMER_OFFSET; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE; -import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.OFFSET_SUB_DOMAIN; import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.constructMetricName; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_COMMIT_LAG_FILE_COUNT; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_COMMIT_LAG_MS; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_INGESTION_LAG_FILE_COUNT; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_INGESTION_LAG_MS; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_KAFKA_LAG_MS; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.AVERAGE_KAFKA_LAG_RECORD_COUNT; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.BYTE_NUMBER; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CLEANER_RESTART_COUNT; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.COMMITTED_OFFSET; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.END_TIME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_ON_INGESTION; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_ON_STAGE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_PURGED; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_TABLE_STAGE_BROKEN_RECORD; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_TABLE_STAGE_INGEST_FAIL; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FLUSHED_OFFSET; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.MEMORY_USAGE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PIPE_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PROCESSED_OFFSET; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PURGED_OFFSET; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.RECORD_NUMBER; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.STAGE_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.codahale.metrics.Gauge; -import com.codahale.metrics.Meter; -import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; -import com.snowflake.kafka.connector.internal.metrics.MetricsUtil.EventType; -import java.util.Arrays; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.LongUnaryOperator; - import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; -import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; +import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; -import org.checkerframework.checker.units.qual.A; /** - * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake - * when the TopicPartitionChannel closes + * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake when the + * TopicPartitionChannel closes * *

Most of the data sent to Snowflake is an aggregated data. */ @@ -142,9 +96,7 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.END_TIME, currTime); } - /** - * Registers all the Metrics inside the metricRegistry. - */ + /** Registers all the Metrics inside the metricRegistry. */ private void registerChannelJMXMetrics() { LOGGER.debug( "Registering new metrics for channel:{}, removing existing metrics:{}", @@ -158,15 +110,19 @@ private void registerChannelJMXMetrics() { // offsets currentMetricRegistry.register( constructMetricName( - this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), + this.channelName, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), (Gauge) this.offsetPersistedInSnowflake::get); currentMetricRegistry.register( - constructMetricName(this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.PROCESSED_OFFSET), + constructMetricName( + this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.PROCESSED_OFFSET), (Gauge) this.processedOffset::get); currentMetricRegistry.register( - constructMetricName(this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), + constructMetricName( + this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), (Gauge) () -> this.latestConsumerOffset); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); @@ -175,9 +131,7 @@ private void registerChannelJMXMetrics() { this.metricsJmxReporter.start(); } - /** - * Unregisters the JMX metrics - */ + /** Unregisters the JMX metrics */ public void unregisterChannelJMXMetrics() { if (this.metricsJmxReporter != null) { LOGGER.debug( diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java index b379e9579..0a3d40806 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java @@ -41,8 +41,7 @@ public SnowflakeTelemetryServiceV2(Connection conn) { @Override public void reportKafkaPartitionUsage( - SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) { - } + SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) {} @Override public ObjectNode getObjectNode() { diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java index 8e51d762e..289b87f54 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java @@ -7,13 +7,12 @@ import com.snowflake.kafka.connector.internal.SnowflakeSinkService; import com.snowflake.kafka.connector.internal.SnowflakeSinkServiceFactory; import com.snowflake.kafka.connector.internal.TestUtils; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; - -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; import org.apache.kafka.common.TopicPartition; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index a6dec629c..26b9e2cb2 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -7,11 +7,9 @@ import static com.snowflake.kafka.connector.internal.TestUtils.createBigAvroRecords; import static com.snowflake.kafka.connector.internal.TestUtils.createNativeJsonSinkRecords; import static com.snowflake.kafka.connector.internal.streaming.StreamingUtils.MAX_GET_OFFSET_TOKEN_RETRIES; -import static com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus.NUM_METRICS; import static org.mockito.ArgumentMatchers.eq; import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.jmx.JmxReporter; import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; import com.snowflake.kafka.connector.dlq.InMemoryKafkaRecordErrorReporter; import com.snowflake.kafka.connector.dlq.KafkaRecordErrorReporter; @@ -234,7 +232,8 @@ public void testCloseChannelException() throws Exception { mockSnowflakeConnectionService, new RecordService(mockTelemetryService), mockTelemetryService, - false, null); + false, + null); topicPartitionChannel.closeChannel(); } @@ -500,7 +499,8 @@ public void testInsertRowsWithSchemaEvolution() throws Exception { conn, new RecordService(), mockTelemetryService, - false, null); + false, + null); final int noOfRecords = 3; List records = @@ -626,7 +626,8 @@ public void testInsertRows_ValidationResponseHasErrors_NoErrorTolerance() throws mockSnowflakeConnectionService, new RecordService(mockTelemetryService), mockTelemetryService, - false, null); + false, + null); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -868,24 +869,34 @@ public void testTopicPartitionChannelMetrics() throws Exception { topicPartitionChannel.insertBufferedRecordsIfFlushTimeThresholdReached(); // verify metrics - SnowflakeTelemetryChannelStatus resultStatus = topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); + SnowflakeTelemetryChannelStatus resultStatus = + topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); - assert resultStatus.getOffsetPersistedInSnowflake() == topicPartitionChannel.getOffsetPersistedInSnowflake(); + assert resultStatus.getOffsetPersistedInSnowflake() + == topicPartitionChannel.getOffsetPersistedInSnowflake(); assert resultStatus.getOffsetPersistedInSnowflake() == -1; assert resultStatus.getProcessedOffset() == topicPartitionChannel.getProcessedOffset(); assert resultStatus.getProcessedOffset() == noOfRecords - 1; - assert resultStatus.getLatestConsumerOffset() == topicPartitionChannel.getLatestConsumerOffset(); + assert resultStatus.getLatestConsumerOffset() + == topicPartitionChannel.getLatestConsumerOffset(); assert resultStatus.getLatestConsumerOffset() == 0; - assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == SnowflakeTelemetryChannelStatus.NUM_METRICS; + assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() + == SnowflakeTelemetryChannelStatus.NUM_METRICS; // verify telemetry was sent when channel closed topicPartitionChannel.closeChannel(); - Mockito.verify(this.mockTelemetryService, Mockito.times(1)).reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), eq(true)); - assert topicPartitionChannel.getSnowflakeTelemetryChannelStatus().getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == 0; + Mockito.verify(this.mockTelemetryService, Mockito.times(1)) + .reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), eq(true)); + assert topicPartitionChannel + .getSnowflakeTelemetryChannelStatus() + .getMetricsJmxReporter() + .getMetricRegistry() + .getMetrics() + .size() + == 0; } - @Test public void testTopicPartitionChannelInvalidJmxReporter() throws Exception { // variables @@ -928,7 +939,8 @@ public void testTopicPartitionChannelInvalidJmxReporter() throws Exception { topicPartitionChannel.insertBufferedRecordsIfFlushTimeThresholdReached(); // verify no errors are thrown with invalid jmx reporter but enabled jmx monitoring - SnowflakeTelemetryChannelStatus resultStatus = topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); + SnowflakeTelemetryChannelStatus resultStatus = + topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); assert resultStatus.getMetricsJmxReporter() == null; topicPartitionChannel.closeChannel(); diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java index 38623fd8e..453248b4e 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -1,14 +1,14 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; +import static com.snowflake.kafka.connector.internal.TestUtils.TEST_CONNECTOR_NAME; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.codahale.metrics.MetricRegistry; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import org.junit.Test; import org.mockito.Mockito; -import static com.snowflake.kafka.connector.internal.TestUtils.TEST_CONNECTOR_NAME; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - public class SnowflakeTelemetryChannelStatusTest { private final String tableName = "tableName"; private final String channelName = "channelName"; @@ -21,7 +21,8 @@ public void testRegisterAndUnregisterJmxMetrics() { Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); // test register - SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, true, metricsJmxReporter); + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = + new SnowflakeTelemetryChannelStatus(tableName, channelName, true, metricsJmxReporter); verify(metricsJmxReporter, times(1)).start(); verify(metricRegistry, times(3)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); @@ -38,7 +39,8 @@ public void testDisabledJmx() { Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); // test register - SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, false, metricsJmxReporter); + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = + new SnowflakeTelemetryChannelStatus(tableName, channelName, false, metricsJmxReporter); verify(metricsJmxReporter, times(0)).start(); verify(metricRegistry, times(0)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(0)).removeMetricsFromRegistry(channelName); @@ -51,9 +53,8 @@ public void testDisabledJmx() { @Test public void testInvalidJmxReporter() { // invalid jmx reporter should not error out - SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, true, null); + SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = + new SnowflakeTelemetryChannelStatus(tableName, channelName, true, null); snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); } - - } From 340d0d006f8bc39fbd6a76daaded22a442a08de7 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Fri, 25 Aug 2023 17:47:09 -0700 Subject: [PATCH 03/34] actually send telem --- .../streaming/SnowflakeSinkServiceV2.java | 2 +- .../streaming/TopicPartitionChannel.java | 10 +++--- .../SnowflakeTelemetryChannelStatus.java | 4 +-- .../SnowflakeTelemetryServiceV2.java | 9 +++-- .../telemetry/SnowflakeTelemetryService.java | 36 ++++++++++++++----- .../SnowflakeTelemetryServiceV1.java | 16 ++------- .../SnowflakeTelemetryChannelStatusTest.java | 6 ++-- 7 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java index f3b0f8c80..396778540 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2.java @@ -317,7 +317,7 @@ public long getOffset(TopicPartition topicPartition) { partitionChannelKey(topicPartition.topic(), topicPartition.partition()); if (partitionsToChannel.containsKey(partitionChannelKey)) { long offset = partitionsToChannel.get(partitionChannelKey).getOffsetSafeToCommitToKafka(); - partitionsToChannel.get(partitionChannelKey).setLatestConsumerOffset(offset, false); + partitionsToChannel.get(partitionChannelKey).setLatestConsumerOffset(offset); return offset; } else { LOGGER.warn( diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index dcd98f90b..34fe9f595 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -87,7 +87,7 @@ public class TopicPartitionChannel { // This offset represents the data persisted in Snowflake. More specifically it is the Snowflake // offset determined from the insertRows API call. It is set after calling the fetchOffsetToken // API for this channel - protected final AtomicLong offsetPersistedInSnowflake = + private final AtomicLong offsetPersistedInSnowflake = new AtomicLong(NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE); // This offset represents the data buffered in KC. More specifically it is the KC offset to ensure @@ -309,7 +309,7 @@ public void insertRecordToBuffer(SinkRecord kafkaSinkRecord) { // Set the consumer offset to be the first record that Kafka sends us if (latestConsumerOffset == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE) { - this.setLatestConsumerOffset(kafkaSinkRecord.kafkaOffset(), true); + this.setLatestConsumerOffset(kafkaSinkRecord.kafkaOffset()); } // Ignore adding to the buffer until we see the expected offset value @@ -1037,7 +1037,7 @@ public void closeChannel() { try { this.channel.close().get(); this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); - this.snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { final String errMsg = String.format( @@ -1078,8 +1078,8 @@ private void setProcessedOffset(long processedOffset) { this.snowflakeTelemetryChannelStatus.setProcessedOffset(processedOffset); } - protected void setLatestConsumerOffset(long consumerOffset, boolean force) { - if (force || consumerOffset > this.latestConsumerOffset) { + protected void setLatestConsumerOffset(long consumerOffset) { + if (consumerOffset > this.latestConsumerOffset) { this.latestConsumerOffset = consumerOffset; this.snowflakeTelemetryChannelStatus.setLatestConsumerOffset(consumerOffset); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 7eb3b0dba..b5a6d26ef 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -131,8 +131,8 @@ private void registerChannelJMXMetrics() { this.metricsJmxReporter.start(); } - /** Unregisters the JMX metrics */ - public void unregisterChannelJMXMetrics() { + /** Unregisters the JMX metrics if possible */ + public void tryUnregisterChannelJMXMetrics() { if (this.metricsJmxReporter != null) { LOGGER.debug( "Removing metrics for channel:{}, existing metrics:{}", diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java index 0a3d40806..eb283f8f4 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java @@ -17,11 +17,13 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; +import com.google.common.annotations.VisibleForTesting; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import java.sql.Connection; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryClient; /** @@ -39,9 +41,10 @@ public SnowflakeTelemetryServiceV2(Connection conn) { this.telemetry = TelemetryClient.createTelemetry(conn); } - @Override - public void reportKafkaPartitionUsage( - SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) {} + @VisibleForTesting + SnowflakeTelemetryServiceV2(Telemetry telemetry) { + this.telemetry = telemetry; + } @Override public ObjectNode getObjectNode() { diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index febc43836..8903e3777 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -17,6 +17,8 @@ import com.snowflake.kafka.connector.internal.KCLogger; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; import java.util.Map; + +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; @@ -49,6 +51,7 @@ public abstract class SnowflakeTelemetryService { private static final String VERSION = "version"; private static final String KAFKA_VERSION = "kafka_version"; protected static final String IS_PIPE_CLOSING = "is_pipe_closing"; + protected static final String IS_CHANNEL_CLOSING = "is_channel_closing"; // Telemetry instance fetched from JDBC protected Telemetry telemetry; @@ -90,7 +93,7 @@ public void reportKafkaConnectStart( dataObjectNode.put(KAFKA_VERSION, AppInfoParser.getVersion()); addUserConnectorPropertiesToDataNode(userProvidedConfig, dataObjectNode); - send(SnowflakeTelemetryServiceV1.TelemetryType.KAFKA_START, dataObjectNode); + send(SnowflakeTelemetryService.TelemetryType.KAFKA_START, dataObjectNode); } /** @@ -104,7 +107,7 @@ public void reportKafkaConnectStop(final long startTime) { msg.put(START_TIME, startTime); msg.put(END_TIME, System.currentTimeMillis()); - send(SnowflakeTelemetryServiceV1.TelemetryType.KAFKA_STOP, msg); + send(SnowflakeTelemetryService.TelemetryType.KAFKA_STOP, msg); } /** @@ -118,20 +121,34 @@ public void reportKafkaConnectFatalError(final String errorDetail) { msg.put(TIME, System.currentTimeMillis()); msg.put(ERROR_NUMBER, errorDetail); - send(SnowflakeTelemetryServiceV1.TelemetryType.KAFKA_FATAL_ERROR, msg); + send(SnowflakeTelemetryService.TelemetryType.KAFKA_FATAL_ERROR, msg); } /** * report connector's partition usage. * - *

It depends on the underlying implementation of Kafka connector, i.e weather it is Snowpipe - * or Snowpipe Streaming - * * @param partitionStatus SnowflakePipeStatus object * @param isClosing is the underlying pipe/channel closing */ - public abstract void reportKafkaPartitionUsage( - final SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing); + public void reportKafkaPartitionUsage( + final SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) { + if (partitionStatus.isEmpty()) { + return; + } + ObjectNode msg = getObjectNode(); + + partitionStatus.dumpTo(msg); + + if (partitionStatus.getClass().equals(SnowflakeTelemetryPipeStatus.class)) { + msg.put(IS_PIPE_CLOSING, isClosing); + send(TelemetryType.KAFKA_PIPE_USAGE, msg); + } else if (partitionStatus.getClass().equals(SnowflakeTelemetryChannelStatus.class)) { + msg.put(IS_CHANNEL_CLOSING, isClosing); + send(TelemetryType.KAFKA_CHANNEL_USAGE, msg); + } else { + LOGGER.error("Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming, instead got {}", TelemetryType.KAFKA_PIPE_USAGE, TelemetryType.KAFKA_CHANNEL_USAGE, partitionStatus.getClass()); + } + } /** * Get default object Node which will be part of every telemetry being sent to snowflake. Based on @@ -270,7 +287,8 @@ enum TelemetryType { KAFKA_STOP("kafka_stop"), KAFKA_FATAL_ERROR("kafka_fatal_error"), KAFKA_PIPE_USAGE("kafka_pipe_usage"), - KAFKA_PIPE_START("kafka_pipe_start"); + KAFKA_PIPE_START("kafka_pipe_start"), + KAFKA_CHANNEL_USAGE("kafka_channel_usage"); private final String name; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java index 122b36522..7d3ed3d94 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java @@ -3,6 +3,8 @@ import com.google.common.annotations.VisibleForTesting; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; import java.sql.Connection; + +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryClient; @@ -19,20 +21,6 @@ public class SnowflakeTelemetryServiceV1 extends SnowflakeTelemetryService { this.telemetry = telemetry; } - @Override - public void reportKafkaPartitionUsage( - final SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) { - if (partitionStatus.isEmpty()) { - return; - } - ObjectNode msg = getObjectNode(); - - partitionStatus.dumpTo(msg); - msg.put(IS_PIPE_CLOSING, isClosing); - - send(TelemetryType.KAFKA_PIPE_USAGE, msg); - } - @Override public ObjectNode getObjectNode() { ObjectNode objectNode = getDefaultObjectNode(IngestionMethodConfig.SNOWPIPE); diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java index 453248b4e..04b0263ad 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -28,7 +28,7 @@ public void testRegisterAndUnregisterJmxMetrics() { verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); // test unregister - snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); verify(metricsJmxReporter, times(2)).removeMetricsFromRegistry(channelName); } @@ -46,7 +46,7 @@ public void testDisabledJmx() { verify(metricsJmxReporter, times(0)).removeMetricsFromRegistry(channelName); // test unregister - snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); } @@ -55,6 +55,6 @@ public void testInvalidJmxReporter() { // invalid jmx reporter should not error out SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus(tableName, channelName, true, null); - snowflakeTelemetryChannelStatus.unregisterChannelJMXMetrics(); + snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } } From ad0a854f2008b52d77c7968610870e641eb5a739 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Fri, 25 Aug 2023 17:47:15 -0700 Subject: [PATCH 04/34] autoformatting --- .../telemetry/SnowflakeTelemetryServiceV2.java | 1 - .../internal/telemetry/SnowflakeTelemetryService.java | 10 +++++++--- .../telemetry/SnowflakeTelemetryServiceV1.java | 2 -- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java index eb283f8f4..9f41c4b5c 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java @@ -19,7 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import java.sql.Connection; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index 8903e3777..9f939c5e8 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -16,9 +16,8 @@ import com.snowflake.kafka.connector.Utils; import com.snowflake.kafka.connector.internal.KCLogger; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import java.util.Map; - import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; +import java.util.Map; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; @@ -146,7 +145,12 @@ public void reportKafkaPartitionUsage( msg.put(IS_CHANNEL_CLOSING, isClosing); send(TelemetryType.KAFKA_CHANNEL_USAGE, msg); } else { - LOGGER.error("Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming, instead got {}", TelemetryType.KAFKA_PIPE_USAGE, TelemetryType.KAFKA_CHANNEL_USAGE, partitionStatus.getClass()); + LOGGER.error( + "Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming," + + " instead got {}", + TelemetryType.KAFKA_PIPE_USAGE, + TelemetryType.KAFKA_CHANNEL_USAGE, + partitionStatus.getClass()); } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java index 7d3ed3d94..bba06b7e1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1.java @@ -3,8 +3,6 @@ import com.google.common.annotations.VisibleForTesting; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; import java.sql.Connection; - -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryClient; From ec5352ee40997dad097251ab1abb9e4e55e69c83 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 28 Aug 2023 13:58:30 -0700 Subject: [PATCH 05/34] refactoer test --- .../SnowflakeTelemetryServiceV2.java | 2 +- .../SnowflakeTelemetryChannelStatusTest.java | 3 +- .../SnowflakeTelemetryServiceTest.java | 298 ++++++++++++++++++ .../SnowflakeTelemetryServiceV1Test.java | 190 ----------- 4 files changed, 301 insertions(+), 192 deletions(-) rename src/test/java/com/snowflake/kafka/connector/internal/{streaming => }/telemetry/SnowflakeTelemetryChannelStatusTest.java (94%) create mode 100644 src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java delete mode 100644 src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1Test.java diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java index 9f41c4b5c..e866eb4f4 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryServiceV2.java @@ -41,7 +41,7 @@ public SnowflakeTelemetryServiceV2(Connection conn) { } @VisibleForTesting - SnowflakeTelemetryServiceV2(Telemetry telemetry) { + public SnowflakeTelemetryServiceV2(Telemetry telemetry) { this.telemetry = telemetry; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java similarity index 94% rename from src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java rename to src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index 04b0263ad..57980f64a 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -1,4 +1,4 @@ -package com.snowflake.kafka.connector.internal.streaming.telemetry; +package com.snowflake.kafka.connector.internal.telemetry; import static com.snowflake.kafka.connector.internal.TestUtils.TEST_CONNECTOR_NAME; import static org.mockito.Mockito.times; @@ -6,6 +6,7 @@ import com.codahale.metrics.MetricRegistry; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java new file mode 100644 index 000000000..85dbfc3bd --- /dev/null +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -0,0 +1,298 @@ +package com.snowflake.kafka.connector.internal.telemetry; + +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_COUNT_RECORDS; +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_FLUSH_TIME_SEC; +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_SIZE_BYTES; +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.INGESTION_METHOD_OPT; +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.KEY_CONVERTER_CONFIG_FIELD; +import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.VALUE_CONVERTER_CONFIG_FIELD; + +import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; +import com.snowflake.kafka.connector.Utils; +import com.snowflake.kafka.connector.internal.SnowflakeErrors; +import com.snowflake.kafka.connector.internal.TestUtils; +import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; +import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; +import net.snowflake.client.jdbc.telemetry.Telemetry; +import net.snowflake.client.jdbc.telemetry.TelemetryData; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class SnowflakeTelemetryServiceTest { + @Parameterized.Parameters(name = "ingestionMethod: {0}") + public static List input() { + return Arrays.asList(IngestionMethodConfig.SNOWPIPE, IngestionMethodConfig.SNOWPIPE_STREAMING); + } + + private final IngestionMethodConfig ingestionMethodConfig; + private final SnowflakeTelemetryService snowflakeTelemetryService; + private final long startTime; + private final Map config; + + private MockTelemetryClient mockTelemetryClient; + + + public static final String KAFKA_STRING_CONVERTER = + "org.apache.kafka.connect.storage.StringConverter"; + public static final String KAFKA_CONFLUENT_AVRO_CONVERTER = + "io.confluent.connect.avro.AvroConverter"; + + + public SnowflakeTelemetryServiceTest(IngestionMethodConfig ingestionMethodConfig) { + this.startTime = System.currentTimeMillis(); + this.ingestionMethodConfig = ingestionMethodConfig; + this.mockTelemetryClient = new MockTelemetryClient(); + + if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + this.snowflakeTelemetryService = new SnowflakeTelemetryServiceV1(this.mockTelemetryClient); + this.config = TestUtils.getConfig(); + } else { + this.snowflakeTelemetryService = new SnowflakeTelemetryServiceV2(this.mockTelemetryClient); + this.config = TestUtils.getConfForStreaming(); + SnowflakeSinkConnectorConfig.setDefaultValues(this.config); + } + + this.snowflakeTelemetryService.setAppName("TEST_APP"); + this.snowflakeTelemetryService.setTaskID("1"); + } + + @Test + public void testReportKafkaStart() { + addKeyAndValueConvertersToConfigMap(this.config); + + // test report start + this.snowflakeTelemetryService.reportKafkaConnectStart( + System.currentTimeMillis(), this.config); + + // validate data sent + LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); + Assert.assertEquals(1, sentData.size()); + + JsonNode allNode = sentData.get(0).getMessage(); + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_START.toString(), allNode.get("type").asText()); + Assert.assertEquals("kafka_connector", allNode.get("source").asText()); + Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); + + JsonNode dataNode = allNode.get("data"); + Assert.assertNotNull(dataNode); + Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); + Assert.assertTrue( + dataNode + .get(INGESTION_METHOD_OPT) + .asText() + .equalsIgnoreCase(this.ingestionMethodConfig.toString())); + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); + + validateBufferProperties(dataNode); + validateKeyAndValueConverter(dataNode); + } + + @Test + public void testReportKafkaStop() { + // test report start + this.snowflakeTelemetryService.reportKafkaConnectStop( + System.currentTimeMillis()); + + // validate data sent + LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); + Assert.assertEquals(1, sentData.size()); + + JsonNode allNode = sentData.get(0).getMessage(); + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_STOP.toString(), allNode.get("type").asText()); + Assert.assertEquals("kafka_connector", allNode.get("source").asText()); + Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); + + JsonNode dataNode = allNode.get("data"); + Assert.assertNotNull(dataNode); + Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); + Assert.assertTrue( + dataNode + .get(INGESTION_METHOD_OPT) + .asInt()== this.ingestionMethodConfig.ordinal()); + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); + } + + @Test + public void testReportKafkaConnectFatalError() { + Exception exception = SnowflakeErrors.ERROR_0003.getException("test exception"); + + // test report start + this.snowflakeTelemetryService.reportKafkaConnectFatalError(exception.getMessage()); + + // validate data sent + LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); + Assert.assertEquals(1, sentData.size()); + + JsonNode allNode = sentData.get(0).getMessage(); + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_FATAL_ERROR.toString(), allNode.get("type").asText()); + Assert.assertEquals("kafka_connector", allNode.get("source").asText()); + Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); + + JsonNode dataNode = allNode.get("data"); + Assert.assertNotNull(dataNode); + Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); + Assert.assertTrue( + dataNode + .get(INGESTION_METHOD_OPT) + .asInt()== this.ingestionMethodConfig.ordinal()); + Assert.assertTrue( + dataNode.get("time").asLong() <= System.currentTimeMillis() + && dataNode.get("time").asLong() >= this.startTime); + Assert.assertEquals( + dataNode.get("error_number").asText(), exception.getMessage()); + } + + + @Test + public void testReportKafkaPartitionUsage() { + SnowflakeTelemetryBasicInfo partitionUsage; + + if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + SnowflakeTelemetryPipeStatus pipeStatus = new SnowflakeTelemetryPipeStatus("tableName", "stageName", "pipeName", false, null); + pipeStatus.setProcessedOffset(1); + pipeStatus.setFlushedOffset(2); + pipeStatus.setCommittedOffset(3); + + partitionUsage = pipeStatus; + } else { + SnowflakeTelemetryChannelStatus channelStatus = new SnowflakeTelemetryChannelStatus("tableName", "channelName",false, null); + channelStatus.setProcessedOffset(1); + channelStatus.setLatestConsumerOffset(2); + channelStatus.setOffsetPersistedInSnowflake(3); + + partitionUsage = channelStatus; + } + + // test report start + this.snowflakeTelemetryService.reportKafkaPartitionUsage(partitionUsage, false); + + // validate data sent + LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); + Assert.assertEquals(1, sentData.size()); + + JsonNode allNode = sentData.get(0).getMessage(); + Assert.assertEquals("kafka_connector", allNode.get("source").asText()); + Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); + + JsonNode dataNode = allNode.get("data"); + Assert.assertNotNull(dataNode); + Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); + Assert.assertTrue( + dataNode + .get(INGESTION_METHOD_OPT) + .asInt()== this.ingestionMethodConfig.ordinal()); + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); + Assert.assertEquals(1, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); + + if(ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); + Assert.assertEquals(2, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); + Assert.assertEquals(3, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); + } else { + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); + Assert.assertEquals(2, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); + Assert.assertEquals(3, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + } + } + + private void addKeyAndValueConvertersToConfigMap(Map userProvidedConfig) { + userProvidedConfig.put(KEY_CONVERTER_CONFIG_FIELD, KAFKA_STRING_CONVERTER); + userProvidedConfig.put( + SnowflakeSinkConnectorConfig.VALUE_CONVERTER_CONFIG_FIELD, KAFKA_CONFLUENT_AVRO_CONVERTER); + } + + private void validateKeyAndValueConverter(JsonNode dataNode) { + Assert.assertTrue(dataNode.has(KEY_CONVERTER_CONFIG_FIELD)); + Assert.assertTrue( + dataNode.get(KEY_CONVERTER_CONFIG_FIELD).asText().equalsIgnoreCase(KAFKA_STRING_CONVERTER)); + + Assert.assertTrue(dataNode.has(VALUE_CONVERTER_CONFIG_FIELD)); + Assert.assertTrue( + dataNode + .get(VALUE_CONVERTER_CONFIG_FIELD) + .asText() + .equalsIgnoreCase(KAFKA_CONFLUENT_AVRO_CONVERTER)); + } + + private void validateBufferProperties(JsonNode dataNode) { + Assert.assertTrue(dataNode.has(BUFFER_SIZE_BYTES)); + Assert.assertTrue(isNumeric(dataNode.get(BUFFER_SIZE_BYTES).asText())); + + Assert.assertTrue(dataNode.has(BUFFER_COUNT_RECORDS)); + Assert.assertTrue(isNumeric(dataNode.get(BUFFER_COUNT_RECORDS).asText())); + + Assert.assertTrue(dataNode.has(BUFFER_FLUSH_TIME_SEC)); + Assert.assertTrue(isNumeric(dataNode.get(BUFFER_FLUSH_TIME_SEC).asText())); + } + + private static boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + try { + Long.parseLong(strNum); + } catch (NumberFormatException nfe) { + return false; + } + return true; + } + + private static class MockTelemetryClient implements Telemetry { + + private final LinkedList telemetryDataList; + + private final LinkedList sentTelemetryData; + + private ExecutorService executor = Executors.newSingleThreadExecutor(); + + public MockTelemetryClient() { + this.telemetryDataList = new LinkedList<>(); + this.sentTelemetryData = new LinkedList<>(); + } + + @Override + public void addLogToBatch(TelemetryData telemetryData) { + this.telemetryDataList.add(telemetryData); + } + + @Override + public void close() { + this.telemetryDataList.clear(); + this.sentTelemetryData.clear(); + } + + @Override + public Future sendBatchAsync() { + return executor.submit(() -> true); + } + + @Override + public void postProcess(String s, String s1, int i, Throwable throwable) {} + + public LinkedList getSentTelemetryData() { + this.sentTelemetryData.addAll(telemetryDataList); + this.telemetryDataList.clear(); + return sentTelemetryData; + } + } +} diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1Test.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1Test.java deleted file mode 100644 index f1d698eda..000000000 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceV1Test.java +++ /dev/null @@ -1,190 +0,0 @@ -package com.snowflake.kafka.connector.internal.telemetry; - -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_COUNT_RECORDS; -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_FLUSH_TIME_SEC; -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_SIZE_BYTES; -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.INGESTION_METHOD_OPT; -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.KEY_CONVERTER_CONFIG_FIELD; -import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.VALUE_CONVERTER_CONFIG_FIELD; -import static com.snowflake.kafka.connector.internal.TestUtils.getConfig; - -import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; -import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import java.util.LinkedList; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; -import net.snowflake.client.jdbc.telemetry.Telemetry; -import net.snowflake.client.jdbc.telemetry.TelemetryData; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class SnowflakeTelemetryServiceV1Test { - - MockTelemetryClient mockTelemetryClient; - - SnowflakeTelemetryServiceV1 snowflakeTelemetryServiceV1; - - public static final String KAFKA_STRING_CONVERTER = - "org.apache.kafka.connect.storage.StringConverter"; - public static final String KAFKA_CONFLUENT_AVRO_CONVERTER = - "io.confluent.connect.avro.AvroConverter"; - - @Before - public void beforeEachTest() { - mockTelemetryClient = new MockTelemetryClient(); - snowflakeTelemetryServiceV1 = new SnowflakeTelemetryServiceV1(mockTelemetryClient); - snowflakeTelemetryServiceV1.setAppName("TEST_APP"); - - snowflakeTelemetryServiceV1.setTaskID("1"); - } - - @Test - public void testReportKafkaStartSnowpipeAtleastOnce() { - - Map userProvidedConfig = getConfig(); - userProvidedConfig.put(INGESTION_METHOD_OPT, IngestionMethodConfig.SNOWPIPE.toString()); - addKeyAndValueConvertersToConfigMap(userProvidedConfig); - - snowflakeTelemetryServiceV1.reportKafkaConnectStart( - System.currentTimeMillis(), userProvidedConfig); - - Assert.assertEquals(1, mockTelemetryClient.getSentTelemetryData().size()); - - TelemetryData kafkaStartTelemetryData = mockTelemetryClient.getSentTelemetryData().get(0); - - ObjectNode messageSent = kafkaStartTelemetryData.getMessage(); - - JsonNode dataNode = messageSent.get("data"); - - Assert.assertNotNull(dataNode); - Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); - Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asText() - .equalsIgnoreCase(IngestionMethodConfig.SNOWPIPE.toString())); - - validateBufferProperties(dataNode); - - validateKeyAndValueConverter(dataNode); - } - - @Test - public void testReportKafkaStartSnowpipeStreaming() { - - Map userProvidedConfig = getConfig(); - userProvidedConfig.put( - INGESTION_METHOD_OPT, IngestionMethodConfig.SNOWPIPE_STREAMING.toString()); - - addKeyAndValueConvertersToConfigMap(userProvidedConfig); - - snowflakeTelemetryServiceV1.reportKafkaConnectStart( - System.currentTimeMillis(), userProvidedConfig); - - Assert.assertEquals(1, mockTelemetryClient.getSentTelemetryData().size()); - - TelemetryData kafkaStartTelemetryData = mockTelemetryClient.getSentTelemetryData().get(0); - - ObjectNode messageSent = kafkaStartTelemetryData.getMessage(); - - JsonNode dataNode = messageSent.get("data"); - - Assert.assertNotNull(dataNode); - Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); - Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asText() - .equalsIgnoreCase(IngestionMethodConfig.SNOWPIPE_STREAMING.toString())); - - validateBufferProperties(dataNode); - - validateKeyAndValueConverter(dataNode); - } - - private void addKeyAndValueConvertersToConfigMap(Map userProvidedConfig) { - userProvidedConfig.put(KEY_CONVERTER_CONFIG_FIELD, KAFKA_STRING_CONVERTER); - userProvidedConfig.put( - SnowflakeSinkConnectorConfig.VALUE_CONVERTER_CONFIG_FIELD, KAFKA_CONFLUENT_AVRO_CONVERTER); - } - - private void validateKeyAndValueConverter(JsonNode dataNode) { - Assert.assertTrue(dataNode.has(KEY_CONVERTER_CONFIG_FIELD)); - Assert.assertTrue( - dataNode.get(KEY_CONVERTER_CONFIG_FIELD).asText().equalsIgnoreCase(KAFKA_STRING_CONVERTER)); - - Assert.assertTrue(dataNode.has(VALUE_CONVERTER_CONFIG_FIELD)); - Assert.assertTrue( - dataNode - .get(VALUE_CONVERTER_CONFIG_FIELD) - .asText() - .equalsIgnoreCase(KAFKA_CONFLUENT_AVRO_CONVERTER)); - } - - private void validateBufferProperties(JsonNode dataNode) { - Assert.assertTrue(dataNode.has(BUFFER_SIZE_BYTES)); - Assert.assertTrue(isNumeric(dataNode.get(BUFFER_SIZE_BYTES).asText())); - - Assert.assertTrue(dataNode.has(BUFFER_COUNT_RECORDS)); - Assert.assertTrue(isNumeric(dataNode.get(BUFFER_COUNT_RECORDS).asText())); - - Assert.assertTrue(dataNode.has(BUFFER_FLUSH_TIME_SEC)); - Assert.assertTrue(isNumeric(dataNode.get(BUFFER_FLUSH_TIME_SEC).asText())); - } - - private static boolean isNumeric(String strNum) { - if (strNum == null) { - return false; - } - try { - Long.parseLong(strNum); - } catch (NumberFormatException nfe) { - return false; - } - return true; - } - - private static class MockTelemetryClient implements Telemetry { - - private final LinkedList telemetryDataList; - - private final LinkedList sentTelemetryData; - - private ExecutorService executor = Executors.newSingleThreadExecutor(); - - public MockTelemetryClient() { - this.telemetryDataList = new LinkedList<>(); - this.sentTelemetryData = new LinkedList<>(); - } - - @Override - public void addLogToBatch(TelemetryData telemetryData) { - this.telemetryDataList.add(telemetryData); - } - - @Override - public void close() { - this.telemetryDataList.clear(); - this.sentTelemetryData.clear(); - } - - @Override - public Future sendBatchAsync() { - return executor.submit(() -> true); - } - - @Override - public void postProcess(String s, String s1, int i, Throwable throwable) {} - - public LinkedList getSentTelemetryData() { - this.sentTelemetryData.addAll(telemetryDataList); - this.telemetryDataList.clear(); - return sentTelemetryData; - } - } -} From 59af12c8acf5d3daa833ecfda7267f3a2f21bc3a Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 28 Aug 2023 16:23:47 -0700 Subject: [PATCH 06/34] added creation --- .../streaming/TopicPartitionChannel.java | 19 +++- .../SnowflakeTelemetryChannelCreation.java | 55 +++++++++++ .../telemetry/SnowflakeTelemetryService.java | 23 ++++- .../streaming/TopicPartitionChannelTest.java | 3 + .../SnowflakeTelemetryServiceTest.java | 91 ++++++++++++++++--- 5 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 34fe9f595..8078b42f4 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -18,6 +18,7 @@ import com.snowflake.kafka.connector.internal.PartitionBuffer; import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; @@ -275,6 +276,9 @@ public TopicPartitionChannel( this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); + if (this.telemetryServiceV2 != null) { + this.telemetryServiceV2.reportKafkaPartitionStart(new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); + } // Open channel and reset the offset in kafka this.channel = Preconditions.checkNotNull(openChannelForTable()); @@ -754,7 +758,9 @@ private void handleInsertRowsFailures( String.format( "Error inserting Records using Streaming API with msg:%s", insertErrors.get(0).getException().getMessage()); - this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + if (this.telemetryServiceV2 != null) { + this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + } throw new DataException(errMsg, insertErrors.get(0).getException()); } } @@ -1036,14 +1042,21 @@ private SnowflakeStreamingIngestChannel openChannelForTable() { public void closeChannel() { try { this.channel.close().get(); - this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); + + if (this.telemetryServiceV2 != null) { + this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); + } + this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { final String errMsg = String.format( "Failure closing Streaming Channel name:%s msg:%s", this.getChannelName(), e.getMessage()); - this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + + if (this.telemetryServiceV2 != null) { + this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + } LOGGER.error(errMsg, e); } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java new file mode 100644 index 000000000..47ad3d03d --- /dev/null +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -0,0 +1,55 @@ +package com.snowflake.kafka.connector.internal.streaming.telemetry; + +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_REPROCESS_PURGE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_RESTART; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_PIPE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_STAGE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PIPE_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.STAGE_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; + +import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; +import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; + +/** + * This object is send only once when pipe starts No concurrent modification is made on this object, + * thus no lock is required. + */ +public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { + boolean isReuseTable = false; // is the create reusing existing table + long startTime; // start time of the pipe + private final String channelName; + + public SnowflakeTelemetryChannelCreation( + final String tableName, String channelName) { + super(tableName); + this.channelName = channelName; + this.startTime = System.currentTimeMillis(); + } + + @Override + public void dumpTo(ObjectNode msg) { + msg.put(TABLE_NAME, this.tableName); + msg.put(CHANNEL_NAME, this.channelName); + + msg.put(IS_REUSE_TABLE, this.isReuseTable); + msg.put(START_TIME, startTime); + } + + @Override + public boolean isEmpty() { + throw new IllegalStateException( + "Empty function doesnt apply to:" + this.getClass().getSimpleName()); + } + + public void setReuseTable(boolean reuseTable) { + isReuseTable = reuseTable; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } +} diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index 9f939c5e8..af7fd5ef1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -16,6 +16,7 @@ import com.snowflake.kafka.connector.Utils; import com.snowflake.kafka.connector.internal.KCLogger; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import java.util.Map; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; @@ -165,14 +166,25 @@ public void reportKafkaPartitionUsage( /** * report connector partition start * - * @param pipeCreation SnowflakeTelemetryBasicInfor object + * @param partitionCreation SnowflakeTelemetryBasicInfor object */ - public void reportKafkaPartitionStart(final SnowflakeTelemetryBasicInfo pipeCreation) { + public void reportKafkaPartitionStart(final SnowflakeTelemetryBasicInfo partitionCreation) { ObjectNode msg = getObjectNode(); - pipeCreation.dumpTo(msg); + partitionCreation.dumpTo(msg); - send(SnowflakeTelemetryServiceV1.TelemetryType.KAFKA_PIPE_START, msg); + if (partitionCreation.getClass().equals(SnowflakeTelemetryPipeCreation.class)) { + send(TelemetryType.KAFKA_PIPE_START, msg); + } else if (partitionCreation.getClass().equals(SnowflakeTelemetryChannelCreation.class)) { + send(TelemetryType.KAFKA_CHANNEL_START, msg); + } else { + LOGGER.error( + "Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming," + + " instead got {}", + TelemetryType.KAFKA_PIPE_START, + TelemetryType.KAFKA_CHANNEL_START, + partitionCreation.getClass()); + } } /** @@ -292,7 +304,8 @@ enum TelemetryType { KAFKA_FATAL_ERROR("kafka_fatal_error"), KAFKA_PIPE_USAGE("kafka_pipe_usage"), KAFKA_PIPE_START("kafka_pipe_start"), - KAFKA_CHANNEL_USAGE("kafka_channel_usage"); + KAFKA_CHANNEL_USAGE("kafka_channel_usage"), + KAFKA_CHANNEL_START("kafka_channel_start"); private final String name; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 26b9e2cb2..4db3584c3 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -17,6 +17,7 @@ import com.snowflake.kafka.connector.internal.SnowflakeConnectionService; import com.snowflake.kafka.connector.internal.TestUtils; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.records.RecordService; @@ -888,6 +889,8 @@ public void testTopicPartitionChannelMetrics() throws Exception { topicPartitionChannel.closeChannel(); Mockito.verify(this.mockTelemetryService, Mockito.times(1)) .reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), eq(true)); + Mockito.verify(this.mockTelemetryService, Mockito.times(1)) + .reportKafkaPartitionStart(Mockito.any(SnowflakeTelemetryChannelCreation.class)); assert topicPartitionChannel .getSnowflakeTelemetryChannelStatus() .getMetricsJmxReporter() diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 85dbfc3bd..e3a179e1b 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; @@ -42,10 +43,8 @@ public static List input() { private final SnowflakeTelemetryService snowflakeTelemetryService; private final long startTime; private final Map config; - private MockTelemetryClient mockTelemetryClient; - public static final String KAFKA_STRING_CONVERTER = "org.apache.kafka.connect.storage.StringConverter"; public static final String KAFKA_CONFLUENT_AVRO_CONVERTER = @@ -71,7 +70,7 @@ public SnowflakeTelemetryServiceTest(IngestionMethodConfig ingestionMethodConfig } @Test - public void testReportKafkaStart() { + public void testReportKafkaConnectStart() { addKeyAndValueConvertersToConfigMap(this.config); // test report start @@ -104,7 +103,7 @@ public void testReportKafkaStart() { } @Test - public void testReportKafkaStop() { + public void testReportKafkaConnectStop() { // test report start this.snowflakeTelemetryService.reportKafkaConnectStop( System.currentTimeMillis()); @@ -132,10 +131,10 @@ public void testReportKafkaStop() { @Test public void testReportKafkaConnectFatalError() { - Exception exception = SnowflakeErrors.ERROR_0003.getException("test exception"); + final String expectedException = SnowflakeErrors.ERROR_0003.getException("test exception").getMessage(); // test report start - this.snowflakeTelemetryService.reportKafkaConnectFatalError(exception.getMessage()); + this.snowflakeTelemetryService.reportKafkaConnectFatalError(expectedException); // validate data sent LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); @@ -157,23 +156,26 @@ public void testReportKafkaConnectFatalError() { dataNode.get("time").asLong() <= System.currentTimeMillis() && dataNode.get("time").asLong() >= this.startTime); Assert.assertEquals( - dataNode.get("error_number").asText(), exception.getMessage()); + dataNode.get("error_number").asText(), expectedException); } - @Test public void testReportKafkaPartitionUsage() { SnowflakeTelemetryBasicInfo partitionUsage; + final String expectedTableName = "tableName"; + final String expectedStageName = "stageName"; + final String expectedPipeName = "pipeName"; + final String expectedChannelName = "channelName"; if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { - SnowflakeTelemetryPipeStatus pipeStatus = new SnowflakeTelemetryPipeStatus("tableName", "stageName", "pipeName", false, null); + SnowflakeTelemetryPipeStatus pipeStatus = new SnowflakeTelemetryPipeStatus(expectedTableName, expectedStageName, expectedPipeName, false, null); pipeStatus.setProcessedOffset(1); pipeStatus.setFlushedOffset(2); pipeStatus.setCommittedOffset(3); partitionUsage = pipeStatus; } else { - SnowflakeTelemetryChannelStatus channelStatus = new SnowflakeTelemetryChannelStatus("tableName", "channelName",false, null); + SnowflakeTelemetryChannelStatus channelStatus = new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName,false, null); channelStatus.setProcessedOffset(1); channelStatus.setLatestConsumerOffset(2); channelStatus.setOffsetPersistedInSnowflake(3); @@ -203,15 +205,82 @@ public void testReportKafkaPartitionUsage() { dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertEquals(1, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); + Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); if(ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); Assert.assertEquals(2, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); Assert.assertEquals(3, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); + Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); + Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); Assert.assertEquals(2, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); Assert.assertEquals(3, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + Assert.assertEquals(expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); + } + } + + @Test + public void testReportKafkaPartitionStart() { + SnowflakeTelemetryBasicInfo partitionCreation; + final String expectedTableName = "tableName"; + final String expectedStageName = "stageName"; + final String expectedPipeName = "pipeName"; + final String expectedChannelName = "channelName"; + + if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + SnowflakeTelemetryPipeCreation pipeCreation = new SnowflakeTelemetryPipeCreation(expectedTableName, expectedStageName, expectedPipeName); + pipeCreation.setReuseTable(true); + pipeCreation.setReusePipe(true); + pipeCreation.setReuseStage(true); + pipeCreation.setFileCountReprocessPurge(10); + pipeCreation.setFileCountRestart(11); + + partitionCreation = pipeCreation; + } else { + SnowflakeTelemetryChannelCreation channelCreation = new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName); + channelCreation.setReuseTable(true); + + partitionCreation = channelCreation; + } + + // test report start + this.snowflakeTelemetryService.reportKafkaPartitionStart(partitionCreation); + + // validate data sent + LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); + Assert.assertEquals(1, sentData.size()); + + JsonNode allNode = sentData.get(0).getMessage(); + Assert.assertEquals("kafka_connector", allNode.get("source").asText()); + Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); + + JsonNode dataNode = allNode.get("data"); + Assert.assertNotNull(dataNode); + Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); + Assert.assertTrue( + dataNode + .get(INGESTION_METHOD_OPT) + .asInt() + == this.ingestionMethodConfig.ordinal()); + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); + Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_TABLE).asBoolean()); + Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); + + if(ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_START.toString(), allNode.get("type").asText()); + Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_PIPE).asBoolean()); + Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_STAGE).asBoolean()); + Assert.assertEquals(10, dataNode.get(TelemetryConstants.FILE_COUNT_REPROCESS_PURGE).asInt()); + Assert.assertEquals(11, dataNode.get(TelemetryConstants.FILE_COUNT_RESTART).asInt()); + Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); + Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); + } else { + Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); + Assert.assertEquals(expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } } @@ -257,7 +326,7 @@ private static boolean isNumeric(String strNum) { return true; } - private static class MockTelemetryClient implements Telemetry { + public static class MockTelemetryClient implements Telemetry { private final LinkedList telemetryDataList; From 038aa7f74fd9899ab951120f17b169d52a9413c2 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 28 Aug 2023 16:23:50 -0700 Subject: [PATCH 07/34] autoformatting --- .../streaming/TopicPartitionChannel.java | 6 +- .../SnowflakeTelemetryChannelCreation.java | 9 +- .../SnowflakeTelemetryServiceTest.java | 95 ++++++++++--------- 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 8078b42f4..acdfbe5b7 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -277,7 +277,8 @@ public TopicPartitionChannel( new SnowflakeTelemetryChannelStatus( tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); if (this.telemetryServiceV2 != null) { - this.telemetryServiceV2.reportKafkaPartitionStart(new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); + this.telemetryServiceV2.reportKafkaPartitionStart( + new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); } // Open channel and reset the offset in kafka @@ -1044,7 +1045,8 @@ public void closeChannel() { this.channel.close().get(); if (this.telemetryServiceV2 != null) { - this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); + this.telemetryServiceV2.reportKafkaPartitionUsage( + this.snowflakeTelemetryChannelStatus, true); } this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 47ad3d03d..8c9a83c6f 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -1,13 +1,7 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_REPROCESS_PURGE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.FILE_COUNT_RESTART; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_PIPE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_STAGE; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.PIPE_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.STAGE_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; @@ -23,8 +17,7 @@ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicIn long startTime; // start time of the pipe private final String channelName; - public SnowflakeTelemetryChannelCreation( - final String tableName, String channelName) { + public SnowflakeTelemetryChannelCreation(final String tableName, String channelName) { super(tableName); this.channelName = channelName; this.startTime = System.currentTimeMillis(); diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index e3a179e1b..983fc6f70 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -12,7 +12,9 @@ import com.snowflake.kafka.connector.internal.SnowflakeErrors; import com.snowflake.kafka.connector.internal.TestUtils; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; - +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -20,10 +22,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryData; @@ -50,7 +48,6 @@ public static List input() { public static final String KAFKA_CONFLUENT_AVRO_CONVERTER = "io.confluent.connect.avro.AvroConverter"; - public SnowflakeTelemetryServiceTest(IngestionMethodConfig ingestionMethodConfig) { this.startTime = System.currentTimeMillis(); this.ingestionMethodConfig = ingestionMethodConfig; @@ -74,15 +71,16 @@ public void testReportKafkaConnectStart() { addKeyAndValueConvertersToConfigMap(this.config); // test report start - this.snowflakeTelemetryService.reportKafkaConnectStart( - System.currentTimeMillis(), this.config); + this.snowflakeTelemetryService.reportKafkaConnectStart(System.currentTimeMillis(), this.config); // validate data sent LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); Assert.assertEquals(1, sentData.size()); JsonNode allNode = sentData.get(0).getMessage(); - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_START.toString(), allNode.get("type").asText()); + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_START.toString(), + allNode.get("type").asText()); Assert.assertEquals("kafka_connector", allNode.get("source").asText()); Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); @@ -96,7 +94,7 @@ public void testReportKafkaConnectStart() { .equalsIgnoreCase(this.ingestionMethodConfig.toString())); Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); validateBufferProperties(dataNode); validateKeyAndValueConverter(dataNode); @@ -105,15 +103,16 @@ public void testReportKafkaConnectStart() { @Test public void testReportKafkaConnectStop() { // test report start - this.snowflakeTelemetryService.reportKafkaConnectStop( - System.currentTimeMillis()); + this.snowflakeTelemetryService.reportKafkaConnectStop(System.currentTimeMillis()); // validate data sent LinkedList sentData = this.mockTelemetryClient.getSentTelemetryData(); Assert.assertEquals(1, sentData.size()); JsonNode allNode = sentData.get(0).getMessage(); - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_STOP.toString(), allNode.get("type").asText()); + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_STOP.toString(), + allNode.get("type").asText()); Assert.assertEquals("kafka_connector", allNode.get("source").asText()); Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); @@ -121,9 +120,7 @@ public void testReportKafkaConnectStop() { Assert.assertNotNull(dataNode); Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asInt()== this.ingestionMethodConfig.ordinal()); + dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); @@ -131,7 +128,8 @@ public void testReportKafkaConnectStop() { @Test public void testReportKafkaConnectFatalError() { - final String expectedException = SnowflakeErrors.ERROR_0003.getException("test exception").getMessage(); + final String expectedException = + SnowflakeErrors.ERROR_0003.getException("test exception").getMessage(); // test report start this.snowflakeTelemetryService.reportKafkaConnectFatalError(expectedException); @@ -141,7 +139,9 @@ public void testReportKafkaConnectFatalError() { Assert.assertEquals(1, sentData.size()); JsonNode allNode = sentData.get(0).getMessage(); - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_FATAL_ERROR.toString(), allNode.get("type").asText()); + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_FATAL_ERROR.toString(), + allNode.get("type").asText()); Assert.assertEquals("kafka_connector", allNode.get("source").asText()); Assert.assertEquals(Utils.VERSION, allNode.get("version").asText()); @@ -149,14 +149,11 @@ public void testReportKafkaConnectFatalError() { Assert.assertNotNull(dataNode); Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asInt()== this.ingestionMethodConfig.ordinal()); + dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); Assert.assertTrue( dataNode.get("time").asLong() <= System.currentTimeMillis() && dataNode.get("time").asLong() >= this.startTime); - Assert.assertEquals( - dataNode.get("error_number").asText(), expectedException); + Assert.assertEquals(dataNode.get("error_number").asText(), expectedException); } @Test @@ -168,14 +165,17 @@ public void testReportKafkaPartitionUsage() { final String expectedChannelName = "channelName"; if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { - SnowflakeTelemetryPipeStatus pipeStatus = new SnowflakeTelemetryPipeStatus(expectedTableName, expectedStageName, expectedPipeName, false, null); + SnowflakeTelemetryPipeStatus pipeStatus = + new SnowflakeTelemetryPipeStatus( + expectedTableName, expectedStageName, expectedPipeName, false, null); pipeStatus.setProcessedOffset(1); pipeStatus.setFlushedOffset(2); pipeStatus.setCommittedOffset(3); partitionUsage = pipeStatus; } else { - SnowflakeTelemetryChannelStatus channelStatus = new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName,false, null); + SnowflakeTelemetryChannelStatus channelStatus = + new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName, false, null); channelStatus.setProcessedOffset(1); channelStatus.setLatestConsumerOffset(2); channelStatus.setOffsetPersistedInSnowflake(3); @@ -198,26 +198,30 @@ public void testReportKafkaPartitionUsage() { Assert.assertNotNull(dataNode); Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asInt()== this.ingestionMethodConfig.ordinal()); + dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertEquals(1, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); - if(ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); + if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), + allNode.get("type").asText()); Assert.assertEquals(2, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); Assert.assertEquals(3, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), + allNode.get("type").asText()); Assert.assertEquals(2, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); - Assert.assertEquals(3, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); - Assert.assertEquals(expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); + Assert.assertEquals( + 3, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + Assert.assertEquals( + expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } } @@ -230,7 +234,9 @@ public void testReportKafkaPartitionStart() { final String expectedChannelName = "channelName"; if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { - SnowflakeTelemetryPipeCreation pipeCreation = new SnowflakeTelemetryPipeCreation(expectedTableName, expectedStageName, expectedPipeName); + SnowflakeTelemetryPipeCreation pipeCreation = + new SnowflakeTelemetryPipeCreation( + expectedTableName, expectedStageName, expectedPipeName); pipeCreation.setReuseTable(true); pipeCreation.setReusePipe(true); pipeCreation.setReuseStage(true); @@ -239,7 +245,8 @@ public void testReportKafkaPartitionStart() { partitionCreation = pipeCreation; } else { - SnowflakeTelemetryChannelCreation channelCreation = new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName); + SnowflakeTelemetryChannelCreation channelCreation = + new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName); channelCreation.setReuseTable(true); partitionCreation = channelCreation; @@ -260,18 +267,17 @@ public void testReportKafkaPartitionStart() { Assert.assertNotNull(dataNode); Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( - dataNode - .get(INGESTION_METHOD_OPT) - .asInt() - == this.ingestionMethodConfig.ordinal()); + dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_TABLE).asBoolean()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); - if(ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_START.toString(), allNode.get("type").asText()); + if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_START.toString(), + allNode.get("type").asText()); Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_PIPE).asBoolean()); Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_STAGE).asBoolean()); Assert.assertEquals(10, dataNode.get(TelemetryConstants.FILE_COUNT_REPROCESS_PURGE).asInt()); @@ -279,8 +285,11 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { - Assert.assertEquals(SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); - Assert.assertEquals(expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); + Assert.assertEquals( + SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), + allNode.get("type").asText()); + Assert.assertEquals( + expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } } From ad9b223649d0ab4a8bd94d4f30866f570108834d Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 28 Aug 2023 16:37:18 -0700 Subject: [PATCH 08/34] make telem nonnull required --- .../streaming/TopicPartitionChannel.java | 18 +++---- .../SnowflakeSinkTaskStreamingTest.java | 3 +- .../streaming/TopicPartitionChannelIT.java | 4 +- .../streaming/TopicPartitionChannelTest.java | 48 ++++++++++++------- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index acdfbe5b7..f548f4871 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -192,7 +192,8 @@ public TopicPartitionChannel( final BufferThreshold streamingBufferThreshold, final Map sfConnectorConfig, KafkaRecordErrorReporter kafkaRecordErrorReporter, - SinkTaskContext sinkTaskContext) { + SinkTaskContext sinkTaskContext, + SnowflakeTelemetryService telemetryService) { this( streamingIngestClient, topicPartition, @@ -204,8 +205,8 @@ public TopicPartitionChannel( kafkaRecordErrorReporter, sinkTaskContext, null, /* Null Connection */ - new RecordService(null /* Null Telemetry Service*/), - null, + new RecordService(telemetryService), + telemetryService, false, null); } @@ -254,7 +255,7 @@ public TopicPartitionChannel( this.conn = conn; this.recordService = recordService; - this.telemetryServiceV2 = telemetryService; + this.telemetryServiceV2 = Preconditions.checkNotNull(telemetryService); this.previousFlushTimeStampMs = System.currentTimeMillis(); @@ -276,10 +277,9 @@ public TopicPartitionChannel( this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); - if (this.telemetryServiceV2 != null) { this.telemetryServiceV2.reportKafkaPartitionStart( new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); - } + // Open channel and reset the offset in kafka this.channel = Preconditions.checkNotNull(openChannelForTable()); @@ -759,9 +759,7 @@ private void handleInsertRowsFailures( String.format( "Error inserting Records using Streaming API with msg:%s", insertErrors.get(0).getException().getMessage()); - if (this.telemetryServiceV2 != null) { this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); - } throw new DataException(errMsg, insertErrors.get(0).getException()); } } @@ -1044,10 +1042,8 @@ public void closeChannel() { try { this.channel.close().get(); - if (this.telemetryServiceV2 != null) { this.telemetryServiceV2.reportKafkaPartitionUsage( this.snowflakeTelemetryChannelStatus, true); - } this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { @@ -1056,9 +1052,7 @@ public void closeChannel() { "Failure closing Streaming Channel name:%s msg:%s", this.getChannelName(), e.getMessage()); - if (this.telemetryServiceV2 != null) { this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); - } LOGGER.error(errMsg, e); } } diff --git a/src/test/java/com/snowflake/kafka/connector/SnowflakeSinkTaskStreamingTest.java b/src/test/java/com/snowflake/kafka/connector/SnowflakeSinkTaskStreamingTest.java index 3e8678645..3c6516ec2 100644 --- a/src/test/java/com/snowflake/kafka/connector/SnowflakeSinkTaskStreamingTest.java +++ b/src/test/java/com/snowflake/kafka/connector/SnowflakeSinkTaskStreamingTest.java @@ -91,7 +91,8 @@ public void testSinkTaskInvalidRecord_InMemoryDLQ() throws Exception { new StreamingBufferThreshold(10, 10_000, 1), config, errorReporter, - inMemorySinkTaskContext); + inMemorySinkTaskContext, + mockTelemetryService); Map topicPartitionChannelMap = Collections.singletonMap(partitionChannelKey(topicName, partition), topicPartitionChannel); diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java index 289b87f54..d90cc3f4c 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java @@ -21,6 +21,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; public class TopicPartitionChannelIT { @@ -90,7 +91,8 @@ public void testAutoChannelReopenOn_OffsetTokenSFException() throws Exception { new StreamingBufferThreshold(10, 10_000, 1), config, new InMemoryKafkaRecordErrorReporter(), - new InMemorySinkTaskContext(Collections.singleton(topicPartition))); + new InMemorySinkTaskContext(Collections.singleton(topicPartition)), + conn.getTelemetryClient()); // since channel is updated, try to insert data again or may be call getOffsetToken // We will reopen the channel in since the older channel in service is stale because we diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 4db3584c3..622dba160 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -125,7 +125,8 @@ public void testTopicPartitionChannelInit_streamingClientClosed() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); } @Test @@ -141,7 +142,8 @@ public void testFetchOffsetTokenWithRetry_null() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); } @@ -160,7 +162,8 @@ public void testFetchOffsetTokenWithRetry_validLong() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); Assert.assertEquals(100L, topicPartitionChannel.fetchOffsetTokenWithRetry()); } @@ -184,7 +187,8 @@ public void testFirstRecordForChannel() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); JsonConverter converter = new JsonConverter(); HashMap converterConfig = new HashMap(); @@ -253,7 +257,8 @@ public void testFetchOffsetTokenWithRetry_SFException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -286,7 +291,8 @@ public void testFetchOffsetTokenWithRetry_validOffsetTokenAfterThreeSFExceptions streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); int expectedRetries = MAX_GET_OFFSET_TOKEN_RETRIES; Mockito.verify(mockStreamingClient, Mockito.times(2)).openChannel(ArgumentMatchers.any()); @@ -315,7 +321,8 @@ public void testFetchOffsetTokenWithRetry_InvalidNumber() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); try { topicPartitionChannel.fetchOffsetTokenWithRetry(); @@ -346,7 +353,8 @@ public void testFetchOffsetTokenWithRetry_NullPointerException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -373,7 +381,8 @@ public void testFetchOffsetTokenWithRetry_RuntimeException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -406,7 +415,8 @@ public void testInsertRows_SuccessAfterReopenChannel() throws Exception { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); final int noOfRecords = 5; // Since record 0 was not able to ingest, all records in this batch will not be added into the // buffer. @@ -543,7 +553,8 @@ public void testInsertRows_GetOffsetTokenFailureAfterReopenChannel() throws Exce streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -581,7 +592,8 @@ public void testInsertRows_RuntimeException() throws Exception { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -669,7 +681,8 @@ public void testInsertRows_ValidationResponseHasErrors_ErrorTolerance_ALL() thro new StreamingBufferThreshold(1000, 10_000_000, 10000), sfConnectorConfigWithErrors, kafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -713,7 +726,8 @@ public void testInsertRows_ValidationResponseHasErrors_ErrorTolerance_ALL_LogEna streamingBufferThreshold, sfConnectorConfigWithErrors, kafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -752,7 +766,8 @@ public void testBufferBytesThreshold() throws Exception { bufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); // Sending 5 records will trigger a buffer bytes based threshold after 4 records have been // added. Size of each record after serialization to Json is 260 Bytes @@ -800,7 +815,8 @@ public void testBigAvroBufferBytesThreshold() throws Exception { bufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext); + mockSinkTaskContext, + mockTelemetryService); // Sending 3 records will trigger a buffer bytes based threshold after 2 records have been // added. Size of each record after serialization to Json is ~6 KBytes From f576c0f8e25450e8e7e2730303dbc9c460940c17 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 28 Aug 2023 16:37:21 -0700 Subject: [PATCH 09/34] autoformatting --- .../streaming/TopicPartitionChannel.java | 12 ++-- .../streaming/TopicPartitionChannelIT.java | 1 - .../streaming/TopicPartitionChannelTest.java | 60 +++++++++---------- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index f548f4871..1bc4a6807 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -277,9 +277,8 @@ public TopicPartitionChannel( this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, channelName, enableCustomJMXMonitoring, metricsJmxReporter); - this.telemetryServiceV2.reportKafkaPartitionStart( - new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); - + this.telemetryServiceV2.reportKafkaPartitionStart( + new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); // Open channel and reset the offset in kafka this.channel = Preconditions.checkNotNull(openChannelForTable()); @@ -759,7 +758,7 @@ private void handleInsertRowsFailures( String.format( "Error inserting Records using Streaming API with msg:%s", insertErrors.get(0).getException().getMessage()); - this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); throw new DataException(errMsg, insertErrors.get(0).getException()); } } @@ -1042,8 +1041,7 @@ public void closeChannel() { try { this.channel.close().get(); - this.telemetryServiceV2.reportKafkaPartitionUsage( - this.snowflakeTelemetryChannelStatus, true); + this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { @@ -1052,7 +1050,7 @@ public void closeChannel() { "Failure closing Streaming Channel name:%s msg:%s", this.getChannelName(), e.getMessage()); - this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); + this.telemetryServiceV2.reportKafkaConnectFatalError(errMsg); LOGGER.error(errMsg, e); } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java index d90cc3f4c..55867e2b8 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelIT.java @@ -21,7 +21,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mock; public class TopicPartitionChannelIT { diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 622dba160..7e7cbc1ab 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -142,8 +142,8 @@ public void testFetchOffsetTokenWithRetry_null() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); } @@ -162,8 +162,8 @@ public void testFetchOffsetTokenWithRetry_validLong() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); Assert.assertEquals(100L, topicPartitionChannel.fetchOffsetTokenWithRetry()); } @@ -187,8 +187,8 @@ public void testFirstRecordForChannel() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); JsonConverter converter = new JsonConverter(); HashMap converterConfig = new HashMap(); @@ -257,8 +257,8 @@ public void testFetchOffsetTokenWithRetry_SFException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -291,8 +291,8 @@ public void testFetchOffsetTokenWithRetry_validOffsetTokenAfterThreeSFExceptions streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); int expectedRetries = MAX_GET_OFFSET_TOKEN_RETRIES; Mockito.verify(mockStreamingClient, Mockito.times(2)).openChannel(ArgumentMatchers.any()); @@ -321,8 +321,8 @@ public void testFetchOffsetTokenWithRetry_InvalidNumber() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); try { topicPartitionChannel.fetchOffsetTokenWithRetry(); @@ -353,8 +353,8 @@ public void testFetchOffsetTokenWithRetry_NullPointerException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -381,8 +381,8 @@ public void testFetchOffsetTokenWithRetry_RuntimeException() { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); try { Assert.assertEquals(-1L, topicPartitionChannel.fetchOffsetTokenWithRetry()); @@ -415,8 +415,8 @@ public void testInsertRows_SuccessAfterReopenChannel() throws Exception { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); final int noOfRecords = 5; // Since record 0 was not able to ingest, all records in this batch will not be added into the // buffer. @@ -553,8 +553,8 @@ public void testInsertRows_GetOffsetTokenFailureAfterReopenChannel() throws Exce streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -592,8 +592,8 @@ public void testInsertRows_RuntimeException() throws Exception { streamingBufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -681,8 +681,8 @@ public void testInsertRows_ValidationResponseHasErrors_ErrorTolerance_ALL() thro new StreamingBufferThreshold(1000, 10_000_000, 10000), sfConnectorConfigWithErrors, kafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -726,8 +726,8 @@ public void testInsertRows_ValidationResponseHasErrors_ErrorTolerance_ALL_LogEna streamingBufferThreshold, sfConnectorConfigWithErrors, kafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); List records = TestUtils.createJsonStringSinkRecords(0, 1, TOPIC, PARTITION); @@ -766,8 +766,8 @@ public void testBufferBytesThreshold() throws Exception { bufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); // Sending 5 records will trigger a buffer bytes based threshold after 4 records have been // added. Size of each record after serialization to Json is 260 Bytes @@ -815,8 +815,8 @@ public void testBigAvroBufferBytesThreshold() throws Exception { bufferThreshold, sfConnectorConfig, mockKafkaRecordErrorReporter, - mockSinkTaskContext, - mockTelemetryService); + mockSinkTaskContext, + mockTelemetryService); // Sending 3 records will trigger a buffer bytes based threshold after 2 records have been // added. Size of each record after serialization to Json is ~6 KBytes From 92bf65f2c9713e318b509bd3e443bb9c6d9a2721 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 30 Aug 2023 15:31:49 -0700 Subject: [PATCH 10/34] personal nits --- .../SnowflakeTelemetryChannelCreation.java | 23 +++++++++++++++---- .../SnowflakeTelemetryChannelStatus.java | 2 -- .../telemetry/SnowflakeTelemetryService.java | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 8c9a83c6f..3f6403384 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2023 Snowflake Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; @@ -14,7 +31,7 @@ */ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { boolean isReuseTable = false; // is the create reusing existing table - long startTime; // start time of the pipe + long startTime; // start time of the channel private final String channelName; public SnowflakeTelemetryChannelCreation(final String tableName, String channelName) { @@ -41,8 +58,4 @@ public boolean isEmpty() { public void setReuseTable(boolean reuseTable) { isReuseTable = reuseTable; } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index b5a6d26ef..02f84c1ba 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -40,7 +40,6 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo // channel properties private final String channelName; - private final boolean enableCustomJMXConfig; private final MetricsJmxReporter metricsJmxReporter; private final AtomicLong startTime; @@ -57,7 +56,6 @@ public SnowflakeTelemetryChannelStatus( final MetricsJmxReporter metricsJmxReporter) { super(tableName); this.channelName = channelName; - this.enableCustomJMXConfig = enableCustomJMXConfig; this.metricsJmxReporter = metricsJmxReporter; this.startTime = new AtomicLong(System.currentTimeMillis()); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index af7fd5ef1..e8829ed40 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -166,7 +166,7 @@ public void reportKafkaPartitionUsage( /** * report connector partition start * - * @param partitionCreation SnowflakeTelemetryBasicInfor object + * @param partitionCreation SnowflakeTelemetryBasicInfo object */ public void reportKafkaPartitionStart(final SnowflakeTelemetryBasicInfo partitionCreation) { ObjectNode msg = getObjectNode(); @@ -227,7 +227,7 @@ protected ObjectNode getDefaultObjectNode(IngestionMethodConfig ingestionMethodC * @param type type of Data * @param data JsonData to wrap in a json field called data */ - protected void send(SnowflakeTelemetryServiceV1.TelemetryType type, JsonNode data) { + protected void send(SnowflakeTelemetryService.TelemetryType type, JsonNode data) { ObjectNode msg = MAPPER.createObjectNode(); msg.put(SOURCE, KAFKA_CONNECTOR); msg.put(TYPE, type.toString()); From 16833c619672773f878e8e68f4d8fa5d236aeec3 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 30 Aug 2023 15:45:33 -0700 Subject: [PATCH 11/34] refactoring --- .../SnowflakeTelemetryChannelCreation.java | 3 +- .../SnowflakeTelemetryChannelStatus.java | 3 +- .../SnowflakeTelemetryBasicInfo.java | 4 ++- .../SnowflakeTelemetryPipeCreation.java | 2 +- .../SnowflakeTelemetryPipeStatus.java | 2 +- .../telemetry/SnowflakeTelemetryService.java | 31 +++---------------- 6 files changed, 13 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 3f6403384..1409f62f5 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -23,6 +23,7 @@ import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; +import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; /** @@ -35,7 +36,7 @@ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicIn private final String channelName; public SnowflakeTelemetryChannelCreation(final String tableName, String channelName) { - super(tableName); + super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START); this.channelName = channelName; this.startTime = System.currentTimeMillis(); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 02f84c1ba..851ef01c7 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -24,6 +24,7 @@ import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; +import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; @@ -54,7 +55,7 @@ public SnowflakeTelemetryChannelStatus( final String channelName, final boolean enableCustomJMXConfig, final MetricsJmxReporter metricsJmxReporter) { - super(tableName); + super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); this.channelName = channelName; this.metricsJmxReporter = metricsJmxReporter; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java index 8c81db637..8a0a0bdb9 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java @@ -8,6 +8,7 @@ /** Minimum information needed to sent to Snowflake through Telemetry API */ public abstract class SnowflakeTelemetryBasicInfo { public final String tableName; + public final SnowflakeTelemetryService.TelemetryType telemetryType; public static final KCLogger LOGGER = new KCLogger(SnowflakeTelemetryBasicInfo.class.getName()); @@ -16,10 +17,11 @@ public abstract class SnowflakeTelemetryBasicInfo { * * @param tableName Checks for Nullability */ - public SnowflakeTelemetryBasicInfo(final String tableName) { + public SnowflakeTelemetryBasicInfo(final String tableName, SnowflakeTelemetryService.TelemetryType telemetryType) { Preconditions.checkArgument( !Strings.isNullOrEmpty(tableName), "tableName cannot be null or empty"); this.tableName = tableName; + this.telemetryType = telemetryType; } /** diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeCreation.java index 627f03cf0..a0ef9b3af 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeCreation.java @@ -29,7 +29,7 @@ public class SnowflakeTelemetryPipeCreation extends SnowflakeTelemetryBasicInfo public SnowflakeTelemetryPipeCreation( final String tableName, final String stageName, final String pipeName) { - super(tableName); + super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_START); this.stageName = stageName; this.pipeName = pipeName; this.startTime = System.currentTimeMillis(); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeStatus.java index 3efd397b5..27ab511fd 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryPipeStatus.java @@ -132,7 +132,7 @@ public SnowflakeTelemetryPipeStatus( final String pipeName, final boolean enableCustomJMXConfig, final MetricsJmxReporter metricsJmxReporter) { - super(tableName); + super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE); this.stageName = stageName; this.pipeName = pipeName; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index e8829ed40..31910ff4a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -138,21 +138,9 @@ public void reportKafkaPartitionUsage( ObjectNode msg = getObjectNode(); partitionStatus.dumpTo(msg); + msg.put(partitionStatus.telemetryType == TelemetryType.KAFKA_PIPE_USAGE ? IS_PIPE_CLOSING : IS_CHANNEL_CLOSING, isClosing); - if (partitionStatus.getClass().equals(SnowflakeTelemetryPipeStatus.class)) { - msg.put(IS_PIPE_CLOSING, isClosing); - send(TelemetryType.KAFKA_PIPE_USAGE, msg); - } else if (partitionStatus.getClass().equals(SnowflakeTelemetryChannelStatus.class)) { - msg.put(IS_CHANNEL_CLOSING, isClosing); - send(TelemetryType.KAFKA_CHANNEL_USAGE, msg); - } else { - LOGGER.error( - "Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming," - + " instead got {}", - TelemetryType.KAFKA_PIPE_USAGE, - TelemetryType.KAFKA_CHANNEL_USAGE, - partitionStatus.getClass()); - } + send(partitionStatus.telemetryType, msg); } /** @@ -173,18 +161,7 @@ public void reportKafkaPartitionStart(final SnowflakeTelemetryBasicInfo partitio partitionCreation.dumpTo(msg); - if (partitionCreation.getClass().equals(SnowflakeTelemetryPipeCreation.class)) { - send(TelemetryType.KAFKA_PIPE_START, msg); - } else if (partitionCreation.getClass().equals(SnowflakeTelemetryChannelCreation.class)) { - send(TelemetryType.KAFKA_CHANNEL_START, msg); - } else { - LOGGER.error( - "Unknown telemetry info given. Must be of type {} for snowpipe or {} for streaming," - + " instead got {}", - TelemetryType.KAFKA_PIPE_START, - TelemetryType.KAFKA_CHANNEL_START, - partitionCreation.getClass()); - } + send(partitionCreation.telemetryType, msg); } /** @@ -298,7 +275,7 @@ protected void addUserConnectorPropertiesToDataNode( ENABLE_STREAMING_CLIENT_OPTIMIZATION_DEFAULT + "")); } - enum TelemetryType { + public enum TelemetryType { KAFKA_START("kafka_start"), KAFKA_STOP("kafka_stop"), KAFKA_FATAL_ERROR("kafka_fatal_error"), From 59786f0db7041e0a4495e918df1ad5bc55a846ef Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 30 Aug 2023 15:45:37 -0700 Subject: [PATCH 12/34] autoformatting --- .../internal/telemetry/SnowflakeTelemetryBasicInfo.java | 3 ++- .../internal/telemetry/SnowflakeTelemetryService.java | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java index 8a0a0bdb9..33686fa35 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryBasicInfo.java @@ -17,7 +17,8 @@ public abstract class SnowflakeTelemetryBasicInfo { * * @param tableName Checks for Nullability */ - public SnowflakeTelemetryBasicInfo(final String tableName, SnowflakeTelemetryService.TelemetryType telemetryType) { + public SnowflakeTelemetryBasicInfo( + final String tableName, SnowflakeTelemetryService.TelemetryType telemetryType) { Preconditions.checkArgument( !Strings.isNullOrEmpty(tableName), "tableName cannot be null or empty"); this.tableName = tableName; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index 31910ff4a..856044b57 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -16,8 +16,6 @@ import com.snowflake.kafka.connector.Utils; import com.snowflake.kafka.connector.internal.KCLogger; import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; -import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; import java.util.Map; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; @@ -138,7 +136,11 @@ public void reportKafkaPartitionUsage( ObjectNode msg = getObjectNode(); partitionStatus.dumpTo(msg); - msg.put(partitionStatus.telemetryType == TelemetryType.KAFKA_PIPE_USAGE ? IS_PIPE_CLOSING : IS_CHANNEL_CLOSING, isClosing); + msg.put( + partitionStatus.telemetryType == TelemetryType.KAFKA_PIPE_USAGE + ? IS_PIPE_CLOSING + : IS_CHANNEL_CLOSING, + isClosing); send(partitionStatus.telemetryType, msg); } From 1bc3dfb54209bc8f69bb8f84cc30893247598276 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Thu, 7 Sep 2023 11:29:52 -0700 Subject: [PATCH 13/34] push for comparison --- .../streaming/TopicPartitionChannel.java | 10 +++++ .../SnowflakeTelemetryChannelStatus.java | 25 ++++++++++- .../streaming/TopicPartitionChannelTest.java | 42 +------------------ .../SnowflakeTelemetryServiceTest.java | 8 ++-- 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index c7f3e5814..dbd8ed6a1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -1095,6 +1095,16 @@ protected long getOffsetPersistedInSnowflake() { return this.offsetPersistedInSnowflake.get(); } + @VisibleForTesting + protected long getProcessedOffset() { + return this.processedOffset.get(); + } + + @VisibleForTesting + protected long getLatestConsumerOffset() { + return this.latestConsumerOffset.get(); + } + @VisibleForTesting protected boolean isPartitionBufferEmpty() { return streamingBuffer.isEmpty(); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index e813f586a..3840a403a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -21,6 +21,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; +import com.google.common.annotations.VisibleForTesting; import com.snowflake.kafka.connector.internal.metrics.MetricsJmxReporter; import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; @@ -28,6 +29,7 @@ import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.checkerframework.checker.units.qual.A; /** * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake when the @@ -42,6 +44,7 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo // channel properties private final String channelName; private final MetricsJmxReporter metricsJmxReporter; + private final AtomicLong startTime; // offsets private final AtomicLong offsetPersistedInSnowflake; @@ -66,6 +69,9 @@ public SnowflakeTelemetryChannelStatus( AtomicLong offsetPersistedInSnowflake, AtomicLong processedOffset, AtomicLong latestConsumerOffset) { + super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); + + this.startTime = new AtomicLong(System.currentTimeMillis()); this.channelName = channelName; this.metricsJmxReporter = metricsJmxReporter; @@ -87,7 +93,7 @@ public boolean isEmpty() { // Check that all properties are still at the default value. return this.offsetPersistedInSnowflake.get() == INITIAL_OFFSET && this.processedOffset.get() == INITIAL_OFFSET - && this.latestConsumerOffset == INITIAL_OFFSET; + && this.latestConsumerOffset.get() == INITIAL_OFFSET; } @Override @@ -97,7 +103,7 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, processedOffset.get()); - msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, latestConsumerOffset); + msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, latestConsumerOffset.get()); final long currTime = System.currentTimeMillis(); msg.put(TelemetryConstants.START_TIME, startTime.getAndSet(currTime)); @@ -158,4 +164,19 @@ public void tryUnregisterChannelJMXMetrics() { public MetricsJmxReporter getMetricsJmxReporter() { return this.metricsJmxReporter; } + + @VisibleForTesting + public long getOffsetPersistedInSnowflake() { + return this.offsetPersistedInSnowflake.get(); + } + + @VisibleForTesting + public long getProcessedOffset() { + return this.processedOffset.get(); + } + + @VisibleForTesting + public long getLatestConsumerOffset() { + return this.latestConsumerOffset.get(); + } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 090900d5b..f3aa6ff3b 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -3,11 +3,13 @@ import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.ERRORS_DEAD_LETTER_QUEUE_TOPIC_NAME_CONFIG; import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.ERRORS_LOG_ENABLE_CONFIG; import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.ERRORS_TOLERANCE_CONFIG; +import static com.snowflake.kafka.connector.internal.TestUtils.TEST_CONNECTOR_NAME; import static com.snowflake.kafka.connector.internal.TestUtils.createBigAvroRecords; import static com.snowflake.kafka.connector.internal.TestUtils.createNativeJsonSinkRecords; import static com.snowflake.kafka.connector.internal.streaming.StreamingUtils.MAX_GET_OFFSET_TOKEN_RETRIES; import static org.mockito.ArgumentMatchers.eq; +import com.codahale.metrics.MetricRegistry; import com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig; import com.snowflake.kafka.connector.dlq.InMemoryKafkaRecordErrorReporter; import com.snowflake.kafka.connector.dlq.KafkaRecordErrorReporter; @@ -963,44 +965,4 @@ public void testTopicPartitionChannelInvalidJmxReporter() throws Exception { topicPartitionChannel.closeChannel(); assert resultStatus.getMetricsJmxReporter() == null; } - - @Test - public void testRegisterJmxMetrics() { - MetricRegistry metricRegistry = Mockito.spy(MetricRegistry.class); - MetricsJmxReporter metricsJmxReporter = - Mockito.spy(new MetricsJmxReporter(metricRegistry, TEST_CONNECTOR_NAME)); - - TopicPartitionChannel topicPartitionChannel = - new TopicPartitionChannel( - mockStreamingClient, - topicPartition, - TEST_CHANNEL_NAME, - TEST_TABLE_NAME, - this.enableSchematization, - this.streamingBufferThreshold, - this.sfConnectorConfig, - this.mockKafkaRecordErrorReporter, - this.mockSinkTaskContext, - this.mockSnowflakeConnectionService, - new RecordService(), - this.mockTelemetryService, - true, - null); - - // insert records - List records = - TestUtils.createJsonStringSinkRecords(0, noOfRecords, TOPIC, PARTITION); - records.forEach(topicPartitionChannel::insertRecordToBuffer); - - Thread.sleep(this.streamingBufferThreshold.getFlushTimeThresholdSeconds() + 1); - topicPartitionChannel.insertBufferedRecordsIfFlushTimeThresholdReached(); - - // verify no errors are thrown with invalid jmx reporter but enabled jmx monitoring - SnowflakeTelemetryChannelStatus resultStatus = - topicPartitionChannel.getSnowflakeTelemetryChannelStatus(); - assert resultStatus.getMetricsJmxReporter() == null; - - topicPartitionChannel.closeChannel(); - assert resultStatus.getMetricsJmxReporter() == null; - } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 983fc6f70..ad9b94fe7 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -22,9 +22,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; + import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryData; +import org.checkerframework.checker.units.qual.A; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -175,10 +178,7 @@ public void testReportKafkaPartitionUsage() { partitionUsage = pipeStatus; } else { SnowflakeTelemetryChannelStatus channelStatus = - new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName, false, null); - channelStatus.setProcessedOffset(1); - channelStatus.setLatestConsumerOffset(2); - channelStatus.setOffsetPersistedInSnowflake(3); + new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName, false, null, new AtomicLong(1), new AtomicLong(2), new AtomicLong(3)); partitionUsage = channelStatus; } From 9a281a76afe340c5328a25f0b741324334b48fad Mon Sep 17 00:00:00 2001 From: revi cheng Date: Thu, 7 Sep 2023 11:29:59 -0700 Subject: [PATCH 14/34] autoformatting --- .../telemetry/SnowflakeTelemetryChannelStatus.java | 1 - .../internal/streaming/TopicPartitionChannelTest.java | 10 +++++----- .../telemetry/SnowflakeTelemetryServiceTest.java | 11 ++++++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 3840a403a..dad003701 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -29,7 +29,6 @@ import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; -import org.checkerframework.checker.units.qual.A; /** * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake when the diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index f3aa6ff3b..7e7cbc1ab 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -908,11 +908,11 @@ public void testTopicPartitionChannelMetrics() throws Exception { Mockito.verify(this.mockTelemetryService, Mockito.times(1)) .reportKafkaPartitionStart(Mockito.any(SnowflakeTelemetryChannelCreation.class)); assert topicPartitionChannel - .getSnowflakeTelemetryChannelStatus() - .getMetricsJmxReporter() - .getMetricRegistry() - .getMetrics() - .size() + .getSnowflakeTelemetryChannelStatus() + .getMetricsJmxReporter() + .getMetricRegistry() + .getMetrics() + .size() == 0; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index ad9b94fe7..e3eda8ca4 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -23,11 +23,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; - import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; import net.snowflake.client.jdbc.telemetry.Telemetry; import net.snowflake.client.jdbc.telemetry.TelemetryData; -import org.checkerframework.checker.units.qual.A; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -178,7 +176,14 @@ public void testReportKafkaPartitionUsage() { partitionUsage = pipeStatus; } else { SnowflakeTelemetryChannelStatus channelStatus = - new SnowflakeTelemetryChannelStatus(expectedTableName, expectedChannelName, false, null, new AtomicLong(1), new AtomicLong(2), new AtomicLong(3)); + new SnowflakeTelemetryChannelStatus( + expectedTableName, + expectedChannelName, + false, + null, + new AtomicLong(1), + new AtomicLong(2), + new AtomicLong(3)); partitionUsage = channelStatus; } From ba46a21869bcbce52dcbe21ad8927e6704f73dc7 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Thu, 7 Sep 2023 12:56:13 -0700 Subject: [PATCH 15/34] fix tests --- .../streaming/TopicPartitionChannel.java | 2 +- .../SnowflakeTelemetryChannelCreation.java | 6 ++-- .../SnowflakeTelemetryChannelStatus.java | 10 +++--- .../streaming/SnowflakeSinkServiceV2IT.java | 17 ++++++++-- .../SnowflakeTelemetryServiceTest.java | 31 ++++++++++++------- 5 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index dbd8ed6a1..5c8dcbe0b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -1048,8 +1048,8 @@ public void closeChannel() { try { this.channel.close().get(); + // telemetry and metrics this.telemetryServiceV2.reportKafkaPartitionUsage(this.snowflakeTelemetryChannelStatus, true); - this.snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } catch (InterruptedException | ExecutionException e) { final String errMsg = diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 1409f62f5..20f27b9e4 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -27,13 +27,13 @@ import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; /** - * This object is send only once when pipe starts No concurrent modification is made on this object, + * This object is sent only once when a channel starts. No concurrent modification is made on this object, * thus no lock is required. */ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { - boolean isReuseTable = false; // is the create reusing existing table - long startTime; // start time of the channel + private final long startTime; // start time of the channel private final String channelName; + private boolean isReuseTable = false; // is the channel reusing existing table public SnowflakeTelemetryChannelCreation(final String tableName, String channelName) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index dad003701..c425cc86f 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -18,6 +18,7 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.constructMetricName; +import static com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel.NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; @@ -32,13 +33,12 @@ /** * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake when the - * TopicPartitionChannel closes + * TopicPartitionChannel closes. Also creates and registers various metrics with JMX * *

Most of the data sent to Snowflake is an aggregated data. */ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo { public static final long NUM_METRICS = 3; // update when new metrics are added - private static final long INITIAL_OFFSET = -1; // channel properties private final String channelName; @@ -90,9 +90,9 @@ public SnowflakeTelemetryChannelStatus( @Override public boolean isEmpty() { // Check that all properties are still at the default value. - return this.offsetPersistedInSnowflake.get() == INITIAL_OFFSET - && this.processedOffset.get() == INITIAL_OFFSET - && this.latestConsumerOffset.get() == INITIAL_OFFSET; + return this.offsetPersistedInSnowflake.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE + && this.processedOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE + && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; } @Override diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index 9adab3cc4..d4b1716ab 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -13,7 +13,9 @@ import com.snowflake.kafka.connector.internal.SnowflakeSinkServiceFactory; import com.snowflake.kafka.connector.internal.TestUtils; import com.snowflake.kafka.connector.internal.metrics.MetricsUtil; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation; import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus; +import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryServiceV2; import com.snowflake.kafka.connector.records.SnowflakeConverter; import com.snowflake.kafka.connector.records.SnowflakeJsonConverter; import io.confluent.connect.avro.AvroConverter; @@ -42,6 +44,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.Mockito; @RunWith(Parameterized.class) public class SnowflakeSinkServiceV2IT { @@ -352,11 +355,16 @@ public void testStreamingIngestion() throws Exception { public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws Exception { Map config = getConfig(); SnowflakeSinkConnectorConfig.setDefaultValues(config); - conn.createTable(table); + + // set up telemetry service spy + SnowflakeConnectionService connectionService = Mockito.spy(this.conn); + connectionService.createTable(table); + SnowflakeTelemetryServiceV2 telemetryService = Mockito.spy((SnowflakeTelemetryServiceV2) this.conn.getTelemetryClient()); + Mockito.when(connectionService.getTelemetryClient()).thenReturn(telemetryService); // opens a channel for partition 0, table and topic SnowflakeSinkService service = - SnowflakeSinkServiceFactory.builder(conn, IngestionMethodConfig.SNOWPIPE_STREAMING, config) + SnowflakeSinkServiceFactory.builder(connectionService, IngestionMethodConfig.SNOWPIPE_STREAMING, config) .setRecordNumber(5) .setFlushTime(5) .setErrorReporter(new InMemoryKafkaRecordErrorReporter()) @@ -464,12 +472,17 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E .getValue() == recordsInPartition2; + // verify telemetry + Mockito.verify(telemetryService, Mockito.times(2)).reportKafkaPartitionStart(Mockito.any(SnowflakeTelemetryChannelCreation.class)); + service.closeAll(); // verify metrics closed assert !service .getMetricRegistry(SnowflakeSinkServiceV2.partitionChannelKey(topic, partition)) .isPresent(); + + Mockito.verify(telemetryService, Mockito.times(2)).reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), Mockito.eq(true)); } @Test diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index e3eda8ca4..ee7b123bf 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -159,19 +159,26 @@ public void testReportKafkaConnectFatalError() { @Test public void testReportKafkaPartitionUsage() { - SnowflakeTelemetryBasicInfo partitionUsage; + // expected values final String expectedTableName = "tableName"; final String expectedStageName = "stageName"; final String expectedPipeName = "pipeName"; final String expectedChannelName = "channelName"; + final long expectedProcessedOffset = 1; + final long expectedFlushedOffset = 2; + final long expectedCommittedOffset = 3; + final long expectedOffsetPersistedInSnowflake = 4; + final long expectedLatestConsumerOffset = 5; + + SnowflakeTelemetryBasicInfo partitionUsage; if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { SnowflakeTelemetryPipeStatus pipeStatus = new SnowflakeTelemetryPipeStatus( expectedTableName, expectedStageName, expectedPipeName, false, null); - pipeStatus.setProcessedOffset(1); - pipeStatus.setFlushedOffset(2); - pipeStatus.setCommittedOffset(3); + pipeStatus.setFlushedOffset(expectedFlushedOffset); + pipeStatus.setProcessedOffset(expectedProcessedOffset); + pipeStatus.setCommittedOffset(expectedCommittedOffset); partitionUsage = pipeStatus; } else { @@ -181,9 +188,9 @@ public void testReportKafkaPartitionUsage() { expectedChannelName, false, null, - new AtomicLong(1), - new AtomicLong(2), - new AtomicLong(3)); + new AtomicLong(expectedOffsetPersistedInSnowflake), + new AtomicLong(expectedProcessedOffset), + new AtomicLong(expectedLatestConsumerOffset)); partitionUsage = channelStatus; } @@ -207,24 +214,24 @@ public void testReportKafkaPartitionUsage() { Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); - Assert.assertEquals(1, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); + Assert.assertEquals(expectedProcessedOffset, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); - Assert.assertEquals(2, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); - Assert.assertEquals(3, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); + Assert.assertEquals(expectedFlushedOffset, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); + Assert.assertEquals(expectedCommittedOffset, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); - Assert.assertEquals(2, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); + Assert.assertEquals(expectedLatestConsumerOffset, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); Assert.assertEquals( - 3, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } From 25b222089193cc410473bce9442140e73de61701 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Thu, 7 Sep 2023 12:56:17 -0700 Subject: [PATCH 16/34] autoformatting --- .../SnowflakeTelemetryChannelCreation.java | 4 ++-- .../streaming/SnowflakeSinkServiceV2IT.java | 13 +++++++++---- .../telemetry/SnowflakeTelemetryServiceTest.java | 16 +++++++++++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 20f27b9e4..95725c209 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -27,8 +27,8 @@ import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; /** - * This object is sent only once when a channel starts. No concurrent modification is made on this object, - * thus no lock is required. + * This object is sent only once when a channel starts. No concurrent modification is made on this + * object, thus no lock is required. */ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { private final long startTime; // start time of the channel diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index d4b1716ab..fa2a89774 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -359,12 +359,14 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E // set up telemetry service spy SnowflakeConnectionService connectionService = Mockito.spy(this.conn); connectionService.createTable(table); - SnowflakeTelemetryServiceV2 telemetryService = Mockito.spy((SnowflakeTelemetryServiceV2) this.conn.getTelemetryClient()); + SnowflakeTelemetryServiceV2 telemetryService = + Mockito.spy((SnowflakeTelemetryServiceV2) this.conn.getTelemetryClient()); Mockito.when(connectionService.getTelemetryClient()).thenReturn(telemetryService); // opens a channel for partition 0, table and topic SnowflakeSinkService service = - SnowflakeSinkServiceFactory.builder(connectionService, IngestionMethodConfig.SNOWPIPE_STREAMING, config) + SnowflakeSinkServiceFactory.builder( + connectionService, IngestionMethodConfig.SNOWPIPE_STREAMING, config) .setRecordNumber(5) .setFlushTime(5) .setErrorReporter(new InMemoryKafkaRecordErrorReporter()) @@ -473,7 +475,8 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E == recordsInPartition2; // verify telemetry - Mockito.verify(telemetryService, Mockito.times(2)).reportKafkaPartitionStart(Mockito.any(SnowflakeTelemetryChannelCreation.class)); + Mockito.verify(telemetryService, Mockito.times(2)) + .reportKafkaPartitionStart(Mockito.any(SnowflakeTelemetryChannelCreation.class)); service.closeAll(); @@ -482,7 +485,9 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E .getMetricRegistry(SnowflakeSinkServiceV2.partitionChannelKey(topic, partition)) .isPresent(); - Mockito.verify(telemetryService, Mockito.times(2)).reportKafkaPartitionUsage(Mockito.any(SnowflakeTelemetryChannelStatus.class), Mockito.eq(true)); + Mockito.verify(telemetryService, Mockito.times(2)) + .reportKafkaPartitionUsage( + Mockito.any(SnowflakeTelemetryChannelStatus.class), Mockito.eq(true)); } @Test diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index ee7b123bf..1f57c0c0a 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -214,24 +214,30 @@ public void testReportKafkaPartitionUsage() { Assert.assertTrue( dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); - Assert.assertEquals(expectedProcessedOffset, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); + Assert.assertEquals( + expectedProcessedOffset, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); - Assert.assertEquals(expectedFlushedOffset, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); - Assert.assertEquals(expectedCommittedOffset, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); + Assert.assertEquals( + expectedFlushedOffset, dataNode.get(TelemetryConstants.FLUSHED_OFFSET).asLong()); + Assert.assertEquals( + expectedCommittedOffset, dataNode.get(TelemetryConstants.COMMITTED_OFFSET).asLong()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); - Assert.assertEquals(expectedLatestConsumerOffset, dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); Assert.assertEquals( - expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + expectedLatestConsumerOffset, + dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); + Assert.assertEquals( + expectedOffsetPersistedInSnowflake, + dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } From d2c71e5e6d589542a8b3895fdd71b6abfa8d2a6f Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 11 Sep 2023 10:51:50 -0700 Subject: [PATCH 17/34] pr reviews --- .../telemetry/SnowflakeTelemetryChannelStatus.java | 9 ++++----- .../internal/telemetry/SnowflakeTelemetryService.java | 3 --- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index c425cc86f..99fc2a290 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -43,7 +43,7 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo // channel properties private final String channelName; private final MetricsJmxReporter metricsJmxReporter; - private final AtomicLong startTime; + private final long startTime; // offsets private final AtomicLong offsetPersistedInSnowflake; @@ -52,8 +52,7 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo /** * Creates a new object tracking {@link - * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX - * TODO @rcheng: update comment when extends telemetryBasicInfo + * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX and send telemetry data to snowflake * * @param tableName the table the channel is ingesting to * @param channelName the name of the TopicPartitionChannel to track @@ -70,7 +69,7 @@ public SnowflakeTelemetryChannelStatus( AtomicLong latestConsumerOffset) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); - this.startTime = new AtomicLong(System.currentTimeMillis()); + this.startTime = System.currentTimeMillis(); this.channelName = channelName; this.metricsJmxReporter = metricsJmxReporter; @@ -105,7 +104,7 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, latestConsumerOffset.get()); final long currTime = System.currentTimeMillis(); - msg.put(TelemetryConstants.START_TIME, startTime.getAndSet(currTime)); + msg.put(TelemetryConstants.START_TIME, this.startTime); msg.put(TelemetryConstants.END_TIME, currTime); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java index 856044b57..4cb70eef5 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryService.java @@ -130,9 +130,6 @@ public void reportKafkaConnectFatalError(final String errorDetail) { */ public void reportKafkaPartitionUsage( final SnowflakeTelemetryBasicInfo partitionStatus, boolean isClosing) { - if (partitionStatus.isEmpty()) { - return; - } ObjectNode msg = getObjectNode(); partitionStatus.dumpTo(msg); From 2ffa3bdc6cbfb6ae045828339b23182948966d19 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Mon, 11 Sep 2023 10:51:53 -0700 Subject: [PATCH 18/34] autoformatting --- .../streaming/telemetry/SnowflakeTelemetryChannelStatus.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 99fc2a290..9e65ce94b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -52,7 +52,8 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo /** * Creates a new object tracking {@link - * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX and send telemetry data to snowflake + * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX and + * send telemetry data to snowflake * * @param tableName the table the channel is ingesting to * @param channelName the name of the TopicPartitionChannel to track From 3388e2568f9b783afe0df273f36437460a159352 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Tue, 12 Sep 2023 10:28:27 -0700 Subject: [PATCH 19/34] add channel open --- .../streaming/TopicPartitionChannel.java | 6 ++++- .../SnowflakeTelemetryChannelStatus.java | 24 ++++++++++++------- .../telemetry/TelemetryConstants.java | 2 ++ .../SnowflakeTelemetryChannelStatusTest.java | 9 ++++--- .../SnowflakeTelemetryServiceTest.java | 3 ++- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 5c8dcbe0b..a0f2f403b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -104,6 +104,8 @@ public class TopicPartitionChannel { private final AtomicLong latestConsumerOffset = new AtomicLong(NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE); + private final AtomicLong channelTryOpenCount = new AtomicLong(0); + /** * Offsets are reset in kafka when one of following cases arises in which we rely on source of * truth (Which is Snowflake's committed offsetToken) @@ -289,7 +291,8 @@ public TopicPartitionChannel( metricsJmxReporter, this.offsetPersistedInSnowflake, this.processedOffset, - this.latestConsumerOffset); + this.latestConsumerOffset, + this.channelTryOpenCount); this.telemetryServiceV2.reportKafkaPartitionStart( new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); @@ -1037,6 +1040,7 @@ private SnowflakeStreamingIngestChannel openChannelForTable() { .build(); LOGGER.info( "Opening a channel with name:{} for table name:{}", this.channelName, this.tableName); + this.channelTryOpenCount.incrementAndGet(); return streamingIngestClient.openChannel(channelRequest); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 9e65ce94b..5a7b19676 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -28,6 +28,7 @@ import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; + import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; @@ -38,7 +39,7 @@ *

Most of the data sent to Snowflake is an aggregated data. */ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo { - public static final long NUM_METRICS = 3; // update when new metrics are added + public static final long NUM_METRICS = 4; // update when new metrics are added // channel properties private final String channelName; @@ -50,6 +51,9 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo private final AtomicLong processedOffset; private final AtomicLong latestConsumerOffset; + // channel metrics + private final AtomicLong channelTryOpenCount; + /** * Creates a new object tracking {@link * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX and @@ -65,9 +69,10 @@ public SnowflakeTelemetryChannelStatus( final String channelName, final boolean enableCustomJMXConfig, final MetricsJmxReporter metricsJmxReporter, - AtomicLong offsetPersistedInSnowflake, - AtomicLong processedOffset, - AtomicLong latestConsumerOffset) { + final AtomicLong offsetPersistedInSnowflake, + final AtomicLong processedOffset, + final AtomicLong latestConsumerOffset, + final AtomicLong channelTryOpenCount) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); this.startTime = System.currentTimeMillis(); @@ -77,6 +82,7 @@ public SnowflakeTelemetryChannelStatus( this.offsetPersistedInSnowflake = offsetPersistedInSnowflake; this.processedOffset = processedOffset; this.latestConsumerOffset = latestConsumerOffset; + this.channelTryOpenCount = channelTryOpenCount; if (enableCustomJMXConfig) { if (metricsJmxReporter == null) { @@ -92,7 +98,8 @@ public boolean isEmpty() { // Check that all properties are still at the default value. return this.offsetPersistedInSnowflake.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE && this.processedOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE - && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; + && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE + && this.channelTryOpenCount.get() == 0; } @Override @@ -100,9 +107,10 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); msg.put(TelemetryConstants.CHANNEL_NAME, this.channelName); - msg.put(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, offsetPersistedInSnowflake.get()); - msg.put(TelemetryConstants.PROCESSED_OFFSET, processedOffset.get()); - msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, latestConsumerOffset.get()); + msg.put(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); + msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); + msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); + msg.put(TelemetryConstants.CHANNEL_TRY_OPEN_COUNT, this.channelTryOpenCount.get()); final long currTime = System.currentTimeMillis(); msg.put(TelemetryConstants.START_TIME, this.startTime); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 457d361d1..79e7e5dcd 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -49,4 +49,6 @@ public final class TelemetryConstants { public static final String IS_REUSE_PIPE = "is_reuse_pipe"; public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; + + public static final String CHANNEL_TRY_OPEN_COUNT = "channel_try_open_count"; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index 9a20f03ed..0936d5bc9 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -31,7 +31,8 @@ public void testRegisterAndUnregisterJmxMetrics() { metricsJmxReporter, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1)); + new AtomicLong(-1), + new AtomicLong(0)); verify(metricsJmxReporter, times(1)).start(); verify(metricRegistry, times(3)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); @@ -56,7 +57,8 @@ public void testDisabledJmx() { metricsJmxReporter, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1)); + new AtomicLong(-1), + new AtomicLong(0)); verify(metricsJmxReporter, times(0)).start(); verify(metricRegistry, times(0)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(0)).removeMetricsFromRegistry(channelName); @@ -77,7 +79,8 @@ public void testInvalidJmxReporter() { null, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1)); + new AtomicLong(-1), + new AtomicLong(0)); snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 1f57c0c0a..b4056116e 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -190,7 +190,8 @@ public void testReportKafkaPartitionUsage() { null, new AtomicLong(expectedOffsetPersistedInSnowflake), new AtomicLong(expectedProcessedOffset), - new AtomicLong(expectedLatestConsumerOffset)); + new AtomicLong(expectedLatestConsumerOffset), + new AtomicLong(0)); partitionUsage = channelStatus; } From ffa0b7b70d2656483c56a0d30c1d6e57757111e1 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Tue, 12 Sep 2023 10:36:04 -0700 Subject: [PATCH 20/34] add to metrics --- .../kafka/connector/internal/metrics/MetricsUtil.java | 5 +++++ .../telemetry/SnowflakeTelemetryChannelStatus.java | 9 +++++++++ .../internal/streaming/TopicPartitionChannelTest.java | 1 + .../telemetry/SnowflakeTelemetryChannelStatusTest.java | 3 +-- .../telemetry/SnowflakeTelemetryServiceTest.java | 6 +++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index 479ce9ab4..8051fb248 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -91,6 +91,11 @@ public class MetricsUtil { public static final String LATENCY_SUB_DOMAIN = "latencies"; + // Partition related constants + public static final String PARTITION_SUB_DOMAIN = "partition"; + + public static final String CHANNEL_TRY_OPEN_COUNT = "channel-try-open-count"; + public enum EventType { /** * Time difference between the record put into kafka to record fetched into Kafka Connector Can diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 5a7b19676..937312381 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -145,6 +145,12 @@ private void registerChannelJMXMetrics() { constructMetricName( this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); + + // channel + currentMetricRegistry.register( + constructMetricName( + this.channelName, MetricsUtil.PARTITION_SUB_DOMAIN, MetricsUtil.CHANNEL_TRY_OPEN_COUNT), (Gauge) this.channelTryOpenCount::get); + } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); } @@ -186,4 +192,7 @@ public long getProcessedOffset() { public long getLatestConsumerOffset() { return this.latestConsumerOffset.get(); } + + @VisibleForTesting + public long getChannelTryOpenCount() { return this.channelTryOpenCount.get();} } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 7e7cbc1ab..00109e24f 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -897,6 +897,7 @@ public void testTopicPartitionChannelMetrics() throws Exception { assert resultStatus.getLatestConsumerOffset() == topicPartitionChannel.getLatestConsumerOffset(); assert resultStatus.getLatestConsumerOffset() == 0; + assert resultStatus.getChannelTryOpenCount() == 1; assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == SnowflakeTelemetryChannelStatus.NUM_METRICS; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index 0936d5bc9..f7c6009ed 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -14,7 +14,6 @@ public class SnowflakeTelemetryChannelStatusTest { private final String tableName = "tableName"; private final String channelName = "channelName"; - private final String connectorName = "connectorName"; @Test public void testRegisterAndUnregisterJmxMetrics() { @@ -34,7 +33,7 @@ public void testRegisterAndUnregisterJmxMetrics() { new AtomicLong(-1), new AtomicLong(0)); verify(metricsJmxReporter, times(1)).start(); - verify(metricRegistry, times(3)).register(Mockito.anyString(), Mockito.any()); + verify(metricRegistry, times((int) SnowflakeTelemetryChannelStatus.NUM_METRICS)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); // test unregister diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index b4056116e..16cd18007 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -169,6 +169,7 @@ public void testReportKafkaPartitionUsage() { final long expectedCommittedOffset = 3; final long expectedOffsetPersistedInSnowflake = 4; final long expectedLatestConsumerOffset = 5; + final long expectedChannelOpenCount = 0; SnowflakeTelemetryBasicInfo partitionUsage; @@ -191,7 +192,7 @@ public void testReportKafkaPartitionUsage() { new AtomicLong(expectedOffsetPersistedInSnowflake), new AtomicLong(expectedProcessedOffset), new AtomicLong(expectedLatestConsumerOffset), - new AtomicLong(0)); + new AtomicLong(expectedChannelOpenCount)); partitionUsage = channelStatus; } @@ -239,6 +240,9 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals( expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + Assert.assertEquals( + expectedChannelOpenCount, + dataNode.get(TelemetryConstants.CHANNEL_TRY_OPEN_COUNT).asLong()); Assert.assertEquals( expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); } From fd9e6927cd94f89e47d3efd7d5148c22d07f334a Mon Sep 17 00:00:00 2001 From: revi cheng Date: Tue, 12 Sep 2023 13:33:41 -0700 Subject: [PATCH 21/34] autoformatting --- .../telemetry/SnowflakeTelemetryChannelStatus.java | 13 +++++++++---- .../SnowflakeTelemetryChannelStatusTest.java | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 937312381..9da9303db 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -28,7 +28,6 @@ import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; import com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants; - import java.util.concurrent.atomic.AtomicLong; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; @@ -107,7 +106,8 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); msg.put(TelemetryConstants.CHANNEL_NAME, this.channelName); - msg.put(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); + msg.put( + TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); msg.put(TelemetryConstants.CHANNEL_TRY_OPEN_COUNT, this.channelTryOpenCount.get()); @@ -149,7 +149,10 @@ private void registerChannelJMXMetrics() { // channel currentMetricRegistry.register( constructMetricName( - this.channelName, MetricsUtil.PARTITION_SUB_DOMAIN, MetricsUtil.CHANNEL_TRY_OPEN_COUNT), (Gauge) this.channelTryOpenCount::get); + this.channelName, + MetricsUtil.PARTITION_SUB_DOMAIN, + MetricsUtil.CHANNEL_TRY_OPEN_COUNT), + (Gauge) this.channelTryOpenCount::get); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); @@ -194,5 +197,7 @@ public long getLatestConsumerOffset() { } @VisibleForTesting - public long getChannelTryOpenCount() { return this.channelTryOpenCount.get();} + public long getChannelTryOpenCount() { + return this.channelTryOpenCount.get(); + } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index f7c6009ed..d0e750b66 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -33,7 +33,8 @@ public void testRegisterAndUnregisterJmxMetrics() { new AtomicLong(-1), new AtomicLong(0)); verify(metricsJmxReporter, times(1)).start(); - verify(metricRegistry, times((int) SnowflakeTelemetryChannelStatus.NUM_METRICS)).register(Mockito.anyString(), Mockito.any()); + verify(metricRegistry, times((int) SnowflakeTelemetryChannelStatus.NUM_METRICS)) + .register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(1)).removeMetricsFromRegistry(channelName); // test unregister From fbf77193d727cfe77130689f8b28fffb9c425d3f Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:26:53 -0700 Subject: [PATCH 22/34] pr reviews --- .../internal/metrics/MetricsUtil.java | 24 +++-- .../streaming/TopicPartitionChannel.java | 11 +- .../SnowflakeTelemetryChannelCreation.java | 2 +- .../SnowflakeTelemetryChannelStatus.java | 39 +++---- .../telemetry/TelemetryConstants.java | 14 ++- .../streaming/SnowflakeSinkServiceV2IT.java | 101 ++++++++---------- .../streaming/TopicPartitionChannelTest.java | 2 +- .../SnowflakeTelemetryChannelStatusTest.java | 4 + .../SnowflakeTelemetryServiceTest.java | 15 ++- 9 files changed, 114 insertions(+), 98 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index 8051fb248..eb755fc8b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -5,6 +5,20 @@ /** All metrics related constants. Mainly for JMX */ public class MetricsUtil { + public class Streaming { + /** + * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset + * description + */ + public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; + + /** + * The number of times the current topic partition channel tried opening a channel + */ + public static final String CURRENT_TP_CHANNEL_OPEN_COUNT = "current-tp-channel-open-count"; + } + public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; // Offset related constants @@ -70,14 +84,6 @@ public class MetricsUtil { */ public static final String PURGED_OFFSET = "purged-offset"; - /** - * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset - * description - */ - public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; - - public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - // Buffer related constants public static final String BUFFER_SUB_DOMAIN = "buffer"; @@ -94,8 +100,6 @@ public class MetricsUtil { // Partition related constants public static final String PARTITION_SUB_DOMAIN = "partition"; - public static final String CHANNEL_TRY_OPEN_COUNT = "channel-try-open-count"; - public enum EventType { /** * Time difference between the record put into kafka to record fetched into Kafka Connector Can diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index a0f2f403b..31576891b 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -104,7 +104,8 @@ public class TopicPartitionChannel { private final AtomicLong latestConsumerOffset = new AtomicLong(NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE); - private final AtomicLong channelTryOpenCount = new AtomicLong(0); + // how many times this TopicPartitionChannel called openChannel + private final AtomicLong currentTpChannelOpenCount = new AtomicLong(0); /** * Offsets are reset in kafka when one of following cases arises in which we rely on source of @@ -283,16 +284,20 @@ public TopicPartitionChannel( this.processedOffset.set(lastCommittedOffsetToken); // setup telemetry and metrics + String connectorName = conn.getConnectorName() == null || conn.getConnectorName().isEmpty() ? + "default_connector_name" : + conn.getConnectorName(); this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, + connectorName, channelName, enableCustomJMXMonitoring, metricsJmxReporter, this.offsetPersistedInSnowflake, this.processedOffset, this.latestConsumerOffset, - this.channelTryOpenCount); + this.currentTpChannelOpenCount); this.telemetryServiceV2.reportKafkaPartitionStart( new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); @@ -1040,7 +1045,7 @@ private SnowflakeStreamingIngestChannel openChannelForTable() { .build(); LOGGER.info( "Opening a channel with name:{} for table name:{}", this.channelName, this.tableName); - this.channelTryOpenCount.incrementAndGet(); + this.currentTpChannelOpenCount.incrementAndGet(); return streamingIngestClient.openChannel(channelRequest); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 95725c209..22b7dadca 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -17,7 +17,7 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 9da9303db..38c8aa0a1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -41,9 +41,10 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo public static final long NUM_METRICS = 4; // update when new metrics are added // channel properties + private final String connectorName; private final String channelName; private final MetricsJmxReporter metricsJmxReporter; - private final long startTime; + private final long channelCreationTime; // offsets private final AtomicLong offsetPersistedInSnowflake; @@ -51,7 +52,7 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo private final AtomicLong latestConsumerOffset; // channel metrics - private final AtomicLong channelTryOpenCount; + private final AtomicLong currentTpChannelOpenCount; /** * Creates a new object tracking {@link @@ -65,6 +66,7 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo */ public SnowflakeTelemetryChannelStatus( final String tableName, + final String connectorName, final String channelName, final boolean enableCustomJMXConfig, final MetricsJmxReporter metricsJmxReporter, @@ -74,14 +76,15 @@ public SnowflakeTelemetryChannelStatus( final AtomicLong channelTryOpenCount) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); - this.startTime = System.currentTimeMillis(); + this.channelCreationTime = System.currentTimeMillis(); + this.connectorName = connectorName; this.channelName = channelName; this.metricsJmxReporter = metricsJmxReporter; this.offsetPersistedInSnowflake = offsetPersistedInSnowflake; this.processedOffset = processedOffset; this.latestConsumerOffset = latestConsumerOffset; - this.channelTryOpenCount = channelTryOpenCount; + this.currentTpChannelOpenCount = channelTryOpenCount; if (enableCustomJMXConfig) { if (metricsJmxReporter == null) { @@ -98,23 +101,24 @@ public boolean isEmpty() { return this.offsetPersistedInSnowflake.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE && this.processedOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE - && this.channelTryOpenCount.get() == 0; + && this.currentTpChannelOpenCount.get() == 0; } @Override public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); - msg.put(TelemetryConstants.CHANNEL_NAME, this.channelName); + msg.put(TelemetryConstants.CONNECTOR_NAME, this.connectorName); + msg.put(TelemetryConstants.Streaming.CHANNEL_NAME, this.channelName); msg.put( - TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); + TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); - msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); - msg.put(TelemetryConstants.CHANNEL_TRY_OPEN_COUNT, this.channelTryOpenCount.get()); + msg.put(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); + msg.put(TelemetryConstants.CURRENT_TP_CHANNEL_OPEN_COUNT, this.currentTpChannelOpenCount.get()); final long currTime = System.currentTimeMillis(); - msg.put(TelemetryConstants.START_TIME, this.startTime); - msg.put(TelemetryConstants.END_TIME, currTime); + msg.put(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME, this.channelCreationTime); + msg.put(TelemetryConstants.Streaming.CHANNEL_CLOSE_TIME, currTime); } /** Registers all the Metrics inside the metricRegistry. */ @@ -133,7 +137,7 @@ private void registerChannelJMXMetrics() { constructMetricName( this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), + MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE), (Gauge) this.offsetPersistedInSnowflake::get); currentMetricRegistry.register( @@ -143,7 +147,7 @@ private void registerChannelJMXMetrics() { currentMetricRegistry.register( constructMetricName( - this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), + this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); // channel @@ -151,9 +155,8 @@ private void registerChannelJMXMetrics() { constructMetricName( this.channelName, MetricsUtil.PARTITION_SUB_DOMAIN, - MetricsUtil.CHANNEL_TRY_OPEN_COUNT), - (Gauge) this.channelTryOpenCount::get); - + MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT), + (Gauge) this.currentTpChannelOpenCount::get); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); } @@ -197,7 +200,7 @@ public long getLatestConsumerOffset() { } @VisibleForTesting - public long getChannelTryOpenCount() { - return this.channelTryOpenCount.get(); + public long getCurrentTpChannelOpenCount() { + return this.currentTpChannelOpenCount.get(); } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 79e7e5dcd..2930731f2 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -8,14 +8,12 @@ public final class TelemetryConstants { public static final String TABLE_NAME = "table_name"; public static final String STAGE_NAME = "stage_name"; public static final String PIPE_NAME = "pipe_name"; - public static final String CHANNEL_NAME = "channel_name"; + public static final String CONNECTOR_NAME = "connector_name"; public static final String PROCESSED_OFFSET = "processed-offset"; public static final String FLUSHED_OFFSET = "flushed-offset"; public static final String COMMITTED_OFFSET = "committed-offset"; public static final String PURGED_OFFSET = "purged-offset"; - public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; - public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; public static final String RECORD_NUMBER = "record_number"; public static final String BYTE_NUMBER = "byte_number"; @@ -50,5 +48,13 @@ public final class TelemetryConstants { public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; - public static final String CHANNEL_TRY_OPEN_COUNT = "channel_try_open_count"; + public static final String CURRENT_TP_CHANNEL_OPEN_COUNT = "channel_try_open_count"; + + public class Streaming { + public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; + public static final String CHANNEL_NAME = "channel_name"; + public static final String CHANNEL_CREATION_TIME = "channel_creation_time"; + public static final String CHANNEL_CLOSE_TIME = "channel_close_time"; + } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index fa2a89774..dcfb4d38b 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.SchemaBuilder; @@ -417,62 +418,8 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E == SnowflakeTelemetryChannelStatus.NUM_METRICS * 2; // two partitions // partition 1 - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE)) - .getValue() - == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.PROCESSED_OFFSET)) - .getValue() - == recordsInPartition1 - 1; - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.LATEST_CONSUMER_OFFSET)) - .getValue() - == recordsInPartition1; - - // partition 2 - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition2), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE)) - .getValue() - == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition2), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.PROCESSED_OFFSET)) - .getValue() - == recordsInPartition2 - 1; - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - SnowflakeSinkServiceV2.partitionChannelKey(topic, partition2), - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.LATEST_CONSUMER_OFFSET)) - .getValue() - == recordsInPartition2; + this.verifyPartitionMetrics(metricRegistry, partitionChannelKey(topic, partition), NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, recordsInPartition1 - 1, recordsInPartition1, 1, this.conn.getConnectorName()); + this.verifyPartitionMetrics(metricRegistry, partitionChannelKey(topic, partition2), NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, recordsInPartition2 - 1, recordsInPartition2, 1, this.conn.getConnectorName()); // verify telemetry Mockito.verify(telemetryService, Mockito.times(2)) @@ -490,6 +437,48 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E Mockito.any(SnowflakeTelemetryChannelStatus.class), Mockito.eq(true)); } + private void verifyPartitionMetrics(Map metricRegistry, String partitionChannelKey, long offsetPersistedInSnowflake, long processedOffset, long latestConsumerOffset, long currentTpChannelOpenCount, String connectorName) { + // offsets + assert (long) + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE)) + .getValue() + == offsetPersistedInSnowflake; + assert (long) + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.PROCESSED_OFFSET)) + .getValue() + == processedOffset; + assert (long) + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET)) + .getValue() + == latestConsumerOffset; + + // channel + assert (long) + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.PARTITION_SUB_DOMAIN, + MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT)) + .getValue() + == currentTpChannelOpenCount; + } + @Test public void testStreamingIngest_multipleChannelPartitionsWithTopic2Table() throws Exception { final int partitionCount = 3; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 00109e24f..359f50f47 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -897,7 +897,7 @@ public void testTopicPartitionChannelMetrics() throws Exception { assert resultStatus.getLatestConsumerOffset() == topicPartitionChannel.getLatestConsumerOffset(); assert resultStatus.getLatestConsumerOffset() == 0; - assert resultStatus.getChannelTryOpenCount() == 1; + assert resultStatus.getCurrentTpChannelOpenCount() == 1; assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == SnowflakeTelemetryChannelStatus.NUM_METRICS; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index d0e750b66..571d49b15 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -13,6 +13,7 @@ public class SnowflakeTelemetryChannelStatusTest { private final String tableName = "tableName"; + private final String connectorName = "connectorName"; private final String channelName = "channelName"; @Test @@ -25,6 +26,7 @@ public void testRegisterAndUnregisterJmxMetrics() { SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, + connectorName, channelName, true, metricsJmxReporter, @@ -52,6 +54,7 @@ public void testDisabledJmx() { SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, + connectorName, channelName, false, metricsJmxReporter, @@ -74,6 +77,7 @@ public void testInvalidJmxReporter() { SnowflakeTelemetryChannelStatus snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, + connectorName, channelName, true, null, diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 16cd18007..9f77a74a3 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -163,6 +163,7 @@ public void testReportKafkaPartitionUsage() { final String expectedTableName = "tableName"; final String expectedStageName = "stageName"; final String expectedPipeName = "pipeName"; + final String expectedConnectorName = "connectorName"; final String expectedChannelName = "channelName"; final long expectedProcessedOffset = 1; final long expectedFlushedOffset = 2; @@ -186,6 +187,7 @@ public void testReportKafkaPartitionUsage() { SnowflakeTelemetryChannelStatus channelStatus = new SnowflakeTelemetryChannelStatus( expectedTableName, + expectedConnectorName, expectedChannelName, false, null, @@ -236,15 +238,18 @@ public void testReportKafkaPartitionUsage() { allNode.get("type").asText()); Assert.assertEquals( expectedLatestConsumerOffset, - dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); + dataNode.get(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET).asLong()); Assert.assertEquals( expectedOffsetPersistedInSnowflake, - dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + dataNode.get(TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( expectedChannelOpenCount, - dataNode.get(TelemetryConstants.CHANNEL_TRY_OPEN_COUNT).asLong()); + dataNode.get(TelemetryConstants.CURRENT_TP_CHANNEL_OPEN_COUNT).asLong()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); + expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); + Assert.assertEquals( + expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText() + ); } } @@ -312,7 +317,7 @@ public void testReportKafkaPartitionStart() { SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.CHANNEL_NAME).asText()); + expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); } } From 262ecc6adccf314c4a8bb38a14d0702ec1ce642e Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:26:57 -0700 Subject: [PATCH 23/34] autoformatting --- .../internal/metrics/MetricsUtil.java | 5 +- .../streaming/TopicPartitionChannel.java | 7 +- .../SnowflakeTelemetryChannelCreation.java | 2 +- .../SnowflakeTelemetryChannelStatus.java | 7 +- .../streaming/SnowflakeSinkServiceV2IT.java | 84 ++++++++++++------- .../SnowflakeTelemetryServiceTest.java | 3 +- 6 files changed, 65 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index eb755fc8b..6c8efef5d 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -11,11 +11,10 @@ public class Streaming { * description */ public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - /** - * The number of times the current topic partition channel tried opening a channel - */ + /** The number of times the current topic partition channel tried opening a channel */ public static final String CURRENT_TP_CHANNEL_OPEN_COUNT = "current-tp-channel-open-count"; } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 31576891b..1095dfe3e 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -284,9 +284,10 @@ public TopicPartitionChannel( this.processedOffset.set(lastCommittedOffsetToken); // setup telemetry and metrics - String connectorName = conn.getConnectorName() == null || conn.getConnectorName().isEmpty() ? - "default_connector_name" : - conn.getConnectorName(); + String connectorName = + conn.getConnectorName() == null || conn.getConnectorName().isEmpty() + ? "default_connector_name" + : conn.getConnectorName(); this.snowflakeTelemetryChannelStatus = new SnowflakeTelemetryChannelStatus( tableName, diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 22b7dadca..7a39bee1c 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -17,9 +17,9 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 38c8aa0a1..5ff3c0c4d 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -111,7 +111,8 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.Streaming.CHANNEL_NAME, this.channelName); msg.put( - TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); + TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE, + this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); msg.put(TelemetryConstants.CURRENT_TP_CHANNEL_OPEN_COUNT, this.currentTpChannelOpenCount.get()); @@ -147,7 +148,9 @@ private void registerChannelJMXMetrics() { currentMetricRegistry.register( constructMetricName( - this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET), + this.channelName, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); // channel diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index dcfb4d38b..5baa11e0d 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.SchemaBuilder; @@ -418,8 +417,22 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E == SnowflakeTelemetryChannelStatus.NUM_METRICS * 2; // two partitions // partition 1 - this.verifyPartitionMetrics(metricRegistry, partitionChannelKey(topic, partition), NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, recordsInPartition1 - 1, recordsInPartition1, 1, this.conn.getConnectorName()); - this.verifyPartitionMetrics(metricRegistry, partitionChannelKey(topic, partition2), NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, recordsInPartition2 - 1, recordsInPartition2, 1, this.conn.getConnectorName()); + this.verifyPartitionMetrics( + metricRegistry, + partitionChannelKey(topic, partition), + NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, + recordsInPartition1 - 1, + recordsInPartition1, + 1, + this.conn.getConnectorName()); + this.verifyPartitionMetrics( + metricRegistry, + partitionChannelKey(topic, partition2), + NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE, + recordsInPartition2 - 1, + recordsInPartition2, + 1, + this.conn.getConnectorName()); // verify telemetry Mockito.verify(telemetryService, Mockito.times(2)) @@ -437,45 +450,52 @@ public void testStreamingIngest_multipleChannelPartitions_withMetrics() throws E Mockito.any(SnowflakeTelemetryChannelStatus.class), Mockito.eq(true)); } - private void verifyPartitionMetrics(Map metricRegistry, String partitionChannelKey, long offsetPersistedInSnowflake, long processedOffset, long latestConsumerOffset, long currentTpChannelOpenCount, String connectorName) { + private void verifyPartitionMetrics( + Map metricRegistry, + String partitionChannelKey, + long offsetPersistedInSnowflake, + long processedOffset, + long latestConsumerOffset, + long currentTpChannelOpenCount, + String connectorName) { // offsets assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - partitionChannelKey, - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE)) - .getValue() + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE)) + .getValue() == offsetPersistedInSnowflake; assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - partitionChannelKey, - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.PROCESSED_OFFSET)) - .getValue() + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.PROCESSED_OFFSET)) + .getValue() == processedOffset; assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - partitionChannelKey, - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET)) - .getValue() + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.OFFSET_SUB_DOMAIN, + MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET)) + .getValue() == latestConsumerOffset; // channel assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - partitionChannelKey, - MetricsUtil.PARTITION_SUB_DOMAIN, - MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT)) - .getValue() + metricRegistry + .get( + MetricsUtil.constructMetricName( + partitionChannelKey, + MetricsUtil.PARTITION_SUB_DOMAIN, + MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT)) + .getValue() == currentTpChannelOpenCount; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 9f77a74a3..8abf346e2 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -248,8 +248,7 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals( expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); Assert.assertEquals( - expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText() - ); + expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } } From 766b7702c1b1c9510d65dfa92af6f6b1484494cb Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:29:48 -0700 Subject: [PATCH 24/34] remove channel open count --- .../internal/metrics/MetricsUtil.java | 3 --- .../streaming/TopicPartitionChannel.java | 7 +---- .../SnowflakeTelemetryChannelStatus.java | 26 +++---------------- .../telemetry/TelemetryConstants.java | 2 -- .../streaming/SnowflakeSinkServiceV2IT.java | 11 -------- .../streaming/TopicPartitionChannelTest.java | 1 - .../SnowflakeTelemetryChannelStatusTest.java | 9 +++---- .../SnowflakeTelemetryServiceTest.java | 7 +---- 8 files changed, 8 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index 6c8efef5d..db906743e 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -13,9 +13,6 @@ public class Streaming { public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - - /** The number of times the current topic partition channel tried opening a channel */ - public static final String CURRENT_TP_CHANNEL_OPEN_COUNT = "current-tp-channel-open-count"; } public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 1095dfe3e..a9a6db65c 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -104,9 +104,6 @@ public class TopicPartitionChannel { private final AtomicLong latestConsumerOffset = new AtomicLong(NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE); - // how many times this TopicPartitionChannel called openChannel - private final AtomicLong currentTpChannelOpenCount = new AtomicLong(0); - /** * Offsets are reset in kafka when one of following cases arises in which we rely on source of * truth (Which is Snowflake's committed offsetToken) @@ -297,8 +294,7 @@ public TopicPartitionChannel( metricsJmxReporter, this.offsetPersistedInSnowflake, this.processedOffset, - this.latestConsumerOffset, - this.currentTpChannelOpenCount); + this.latestConsumerOffset); this.telemetryServiceV2.reportKafkaPartitionStart( new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); @@ -1046,7 +1042,6 @@ private SnowflakeStreamingIngestChannel openChannelForTable() { .build(); LOGGER.info( "Opening a channel with name:{} for table name:{}", this.channelName, this.tableName); - this.currentTpChannelOpenCount.incrementAndGet(); return streamingIngestClient.openChannel(channelRequest); } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 5ff3c0c4d..28ce41a54 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -38,7 +38,7 @@ *

Most of the data sent to Snowflake is an aggregated data. */ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo { - public static final long NUM_METRICS = 4; // update when new metrics are added + public static final long NUM_METRICS = 3; // update when new metrics are added // channel properties private final String connectorName; @@ -51,9 +51,6 @@ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo private final AtomicLong processedOffset; private final AtomicLong latestConsumerOffset; - // channel metrics - private final AtomicLong currentTpChannelOpenCount; - /** * Creates a new object tracking {@link * com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} metrics with JMX and @@ -72,8 +69,7 @@ public SnowflakeTelemetryChannelStatus( final MetricsJmxReporter metricsJmxReporter, final AtomicLong offsetPersistedInSnowflake, final AtomicLong processedOffset, - final AtomicLong latestConsumerOffset, - final AtomicLong channelTryOpenCount) { + final AtomicLong latestConsumerOffset) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); this.channelCreationTime = System.currentTimeMillis(); @@ -84,7 +80,6 @@ public SnowflakeTelemetryChannelStatus( this.offsetPersistedInSnowflake = offsetPersistedInSnowflake; this.processedOffset = processedOffset; this.latestConsumerOffset = latestConsumerOffset; - this.currentTpChannelOpenCount = channelTryOpenCount; if (enableCustomJMXConfig) { if (metricsJmxReporter == null) { @@ -100,8 +95,7 @@ public boolean isEmpty() { // Check that all properties are still at the default value. return this.offsetPersistedInSnowflake.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE && this.processedOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE - && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE - && this.currentTpChannelOpenCount.get() == 0; + && this.latestConsumerOffset.get() == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE; } @Override @@ -115,7 +109,6 @@ public void dumpTo(ObjectNode msg) { this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); - msg.put(TelemetryConstants.CURRENT_TP_CHANNEL_OPEN_COUNT, this.currentTpChannelOpenCount.get()); final long currTime = System.currentTimeMillis(); msg.put(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME, this.channelCreationTime); @@ -152,14 +145,6 @@ private void registerChannelJMXMetrics() { MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); - - // channel - currentMetricRegistry.register( - constructMetricName( - this.channelName, - MetricsUtil.PARTITION_SUB_DOMAIN, - MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT), - (Gauge) this.currentTpChannelOpenCount::get); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); } @@ -201,9 +186,4 @@ public long getProcessedOffset() { public long getLatestConsumerOffset() { return this.latestConsumerOffset.get(); } - - @VisibleForTesting - public long getCurrentTpChannelOpenCount() { - return this.currentTpChannelOpenCount.get(); - } } diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 2930731f2..6d0711616 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -48,8 +48,6 @@ public final class TelemetryConstants { public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; - public static final String CURRENT_TP_CHANNEL_OPEN_COUNT = "channel_try_open_count"; - public class Streaming { public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index 5baa11e0d..9fa23fee8 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -486,17 +486,6 @@ private void verifyPartitionMetrics( MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET)) .getValue() == latestConsumerOffset; - - // channel - assert (long) - metricRegistry - .get( - MetricsUtil.constructMetricName( - partitionChannelKey, - MetricsUtil.PARTITION_SUB_DOMAIN, - MetricsUtil.Streaming.CURRENT_TP_CHANNEL_OPEN_COUNT)) - .getValue() - == currentTpChannelOpenCount; } @Test diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java index 359f50f47..7e7cbc1ab 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannelTest.java @@ -897,7 +897,6 @@ public void testTopicPartitionChannelMetrics() throws Exception { assert resultStatus.getLatestConsumerOffset() == topicPartitionChannel.getLatestConsumerOffset(); assert resultStatus.getLatestConsumerOffset() == 0; - assert resultStatus.getCurrentTpChannelOpenCount() == 1; assert resultStatus.getMetricsJmxReporter().getMetricRegistry().getMetrics().size() == SnowflakeTelemetryChannelStatus.NUM_METRICS; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index 571d49b15..d8cbdac52 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -32,8 +32,7 @@ public void testRegisterAndUnregisterJmxMetrics() { metricsJmxReporter, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1), - new AtomicLong(0)); + new AtomicLong(-1)); verify(metricsJmxReporter, times(1)).start(); verify(metricRegistry, times((int) SnowflakeTelemetryChannelStatus.NUM_METRICS)) .register(Mockito.anyString(), Mockito.any()); @@ -60,8 +59,7 @@ public void testDisabledJmx() { metricsJmxReporter, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1), - new AtomicLong(0)); + new AtomicLong(-1)); verify(metricsJmxReporter, times(0)).start(); verify(metricRegistry, times(0)).register(Mockito.anyString(), Mockito.any()); verify(metricsJmxReporter, times(0)).removeMetricsFromRegistry(channelName); @@ -83,8 +81,7 @@ public void testInvalidJmxReporter() { null, new AtomicLong(-1), new AtomicLong(-1), - new AtomicLong(-1), - new AtomicLong(0)); + new AtomicLong(-1)); snowflakeTelemetryChannelStatus.tryUnregisterChannelJMXMetrics(); } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 8abf346e2..f4510443b 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -170,7 +170,6 @@ public void testReportKafkaPartitionUsage() { final long expectedCommittedOffset = 3; final long expectedOffsetPersistedInSnowflake = 4; final long expectedLatestConsumerOffset = 5; - final long expectedChannelOpenCount = 0; SnowflakeTelemetryBasicInfo partitionUsage; @@ -193,8 +192,7 @@ public void testReportKafkaPartitionUsage() { null, new AtomicLong(expectedOffsetPersistedInSnowflake), new AtomicLong(expectedProcessedOffset), - new AtomicLong(expectedLatestConsumerOffset), - new AtomicLong(expectedChannelOpenCount)); + new AtomicLong(expectedLatestConsumerOffset)); partitionUsage = channelStatus; } @@ -242,9 +240,6 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals( expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); - Assert.assertEquals( - expectedChannelOpenCount, - dataNode.get(TelemetryConstants.CURRENT_TP_CHANNEL_OPEN_COUNT).asLong()); Assert.assertEquals( expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); Assert.assertEquals( From 0faac75f325a013291e8d86026838dcfe643c2a4 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:38:30 -0700 Subject: [PATCH 25/34] start time -> channel creation time --- .../streaming/TopicPartitionChannel.java | 5 ++++- .../SnowflakeTelemetryChannelCreation.java | 9 +++++---- .../SnowflakeTelemetryChannelStatus.java | 3 ++- .../SnowflakeTelemetryChannelStatusTest.java | 3 +++ .../SnowflakeTelemetryServiceTest.java | 20 ++++++++++++------- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index a9a6db65c..758537734 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -244,6 +244,8 @@ public TopicPartitionChannel( SnowflakeTelemetryService telemetryService, boolean enableCustomJMXMonitoring, MetricsJmxReporter metricsJmxReporter) { + final long startTime = System.currentTimeMillis(); + this.streamingIngestClient = Preconditions.checkNotNull(streamingIngestClient); Preconditions.checkState(!streamingIngestClient.isClosed()); this.topicPartition = Preconditions.checkNotNull(topicPartition); @@ -290,13 +292,14 @@ public TopicPartitionChannel( tableName, connectorName, channelName, + startTime, enableCustomJMXMonitoring, metricsJmxReporter, this.offsetPersistedInSnowflake, this.processedOffset, this.latestConsumerOffset); this.telemetryServiceV2.reportKafkaPartitionStart( - new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName)); + new SnowflakeTelemetryChannelCreation(this.tableName, this.channelName, startTime)); if (lastCommittedOffsetToken != NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE) { this.sinkTaskContext.offset(this.topicPartition, lastCommittedOffsetToken + 1L); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 7a39bee1c..8ffc84e1c 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -19,6 +19,7 @@ import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_CREATION_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; @@ -31,14 +32,14 @@ * object, thus no lock is required. */ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { - private final long startTime; // start time of the channel + private final long channelCreationTime; // start time of the channel private final String channelName; private boolean isReuseTable = false; // is the channel reusing existing table - public SnowflakeTelemetryChannelCreation(final String tableName, String channelName) { + public SnowflakeTelemetryChannelCreation(final String tableName, final String channelName, final long startTime) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START); this.channelName = channelName; - this.startTime = System.currentTimeMillis(); + this.channelCreationTime = startTime; } @Override @@ -47,7 +48,7 @@ public void dumpTo(ObjectNode msg) { msg.put(CHANNEL_NAME, this.channelName); msg.put(IS_REUSE_TABLE, this.isReuseTable); - msg.put(START_TIME, startTime); + msg.put(CHANNEL_CREATION_TIME, channelCreationTime); } @Override diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 28ce41a54..17b45a783 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -65,6 +65,7 @@ public SnowflakeTelemetryChannelStatus( final String tableName, final String connectorName, final String channelName, + final long startTime, final boolean enableCustomJMXConfig, final MetricsJmxReporter metricsJmxReporter, final AtomicLong offsetPersistedInSnowflake, @@ -72,7 +73,7 @@ public SnowflakeTelemetryChannelStatus( final AtomicLong latestConsumerOffset) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE); - this.channelCreationTime = System.currentTimeMillis(); + this.channelCreationTime = startTime; this.connectorName = connectorName; this.channelName = channelName; this.metricsJmxReporter = metricsJmxReporter; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java index d8cbdac52..c8f2466ac 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryChannelStatusTest.java @@ -28,6 +28,7 @@ public void testRegisterAndUnregisterJmxMetrics() { tableName, connectorName, channelName, + 1234, true, metricsJmxReporter, new AtomicLong(-1), @@ -55,6 +56,7 @@ public void testDisabledJmx() { tableName, connectorName, channelName, + 1234, false, metricsJmxReporter, new AtomicLong(-1), @@ -77,6 +79,7 @@ public void testInvalidJmxReporter() { tableName, connectorName, channelName, + 1234, true, null, new AtomicLong(-1), diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index f4510443b..7fa2fbeda 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -165,6 +165,7 @@ public void testReportKafkaPartitionUsage() { final String expectedPipeName = "pipeName"; final String expectedConnectorName = "connectorName"; final String expectedChannelName = "channelName"; + final long expectedChannelCreationTime = 1234; final long expectedProcessedOffset = 1; final long expectedFlushedOffset = 2; final long expectedCommittedOffset = 3; @@ -188,6 +189,7 @@ public void testReportKafkaPartitionUsage() { expectedTableName, expectedConnectorName, expectedChannelName, + expectedChannelCreationTime, false, null, new AtomicLong(expectedOffsetPersistedInSnowflake), @@ -213,14 +215,14 @@ public void testReportKafkaPartitionUsage() { Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); - Assert.assertTrue( - dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertEquals( expectedProcessedOffset, dataNode.get(TelemetryConstants.PROCESSED_OFFSET).asLong()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_USAGE.toString(), allNode.get("type").asText()); @@ -231,6 +233,8 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { + Assert.assertTrue( + dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -254,6 +258,7 @@ public void testReportKafkaPartitionStart() { final String expectedStageName = "stageName"; final String expectedPipeName = "pipeName"; final String expectedChannelName = "channelName"; + final long expectedChannelCreationTime = 1234; if (this.ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { SnowflakeTelemetryPipeCreation pipeCreation = @@ -268,7 +273,7 @@ public void testReportKafkaPartitionStart() { partitionCreation = pipeCreation; } else { SnowflakeTelemetryChannelCreation channelCreation = - new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName); + new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName, expectedChannelCreationTime); channelCreation.setReuseTable(true); partitionCreation = channelCreation; @@ -290,13 +295,13 @@ public void testReportKafkaPartitionStart() { Assert.assertTrue(dataNode.has(INGESTION_METHOD_OPT)); Assert.assertTrue( dataNode.get(INGESTION_METHOD_OPT).asInt() == this.ingestionMethodConfig.ordinal()); - Assert.assertTrue( - dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertTrue(dataNode.get(TelemetryConstants.IS_REUSE_TABLE).asBoolean()); Assert.assertEquals(expectedTableName, dataNode.get(TelemetryConstants.TABLE_NAME).asText()); if (ingestionMethodConfig == IngestionMethodConfig.SNOWPIPE) { + Assert.assertTrue( + dataNode.get(TelemetryConstants.START_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.START_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_PIPE_START.toString(), allNode.get("type").asText()); @@ -307,6 +312,7 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { + Assert.assertTrue(dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); From 9d5943da378418a2864b13a7fce4b28215339783 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:38:34 -0700 Subject: [PATCH 26/34] autoformatting --- .../telemetry/SnowflakeTelemetryChannelCreation.java | 4 ++-- .../telemetry/SnowflakeTelemetryServiceTest.java | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 8ffc84e1c..e1c5c1d65 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -18,7 +18,6 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.START_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_CREATION_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; @@ -36,7 +35,8 @@ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicIn private final String channelName; private boolean isReuseTable = false; // is the channel reusing existing table - public SnowflakeTelemetryChannelCreation(final String tableName, final String channelName, final long startTime) { + public SnowflakeTelemetryChannelCreation( + final String tableName, final String channelName, final long startTime) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START); this.channelName = channelName; this.channelCreationTime = startTime; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 7fa2fbeda..c8b136279 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -234,7 +234,8 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); + dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() + == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -273,7 +274,8 @@ public void testReportKafkaPartitionStart() { partitionCreation = pipeCreation; } else { SnowflakeTelemetryChannelCreation channelCreation = - new SnowflakeTelemetryChannelCreation(expectedTableName, expectedChannelName, expectedChannelCreationTime); + new SnowflakeTelemetryChannelCreation( + expectedTableName, expectedChannelName, expectedChannelCreationTime); channelCreation.setReuseTable(true); partitionCreation = channelCreation; @@ -312,7 +314,9 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { - Assert.assertTrue(dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); + Assert.assertTrue( + dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() + == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); From a47cbcf6f637ad64d55862961af2301c208dc27b Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:41:52 -0700 Subject: [PATCH 27/34] clarify to tpchannel --- .../SnowflakeTelemetryChannelCreation.java | 16 +++++++------- .../SnowflakeTelemetryChannelStatus.java | 7 +++---- .../telemetry/TelemetryConstants.java | 6 +++--- .../SnowflakeTelemetryServiceTest.java | 21 +++++++++++-------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index e1c5c1d65..4d385a620 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -18,8 +18,8 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_CREATION_TIME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.TP_CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; @@ -31,24 +31,24 @@ * object, thus no lock is required. */ public class SnowflakeTelemetryChannelCreation extends SnowflakeTelemetryBasicInfo { - private final long channelCreationTime; // start time of the channel - private final String channelName; + private final long tpChannelCreationTime; // start time of the channel + private final String tpChannelName; private boolean isReuseTable = false; // is the channel reusing existing table public SnowflakeTelemetryChannelCreation( final String tableName, final String channelName, final long startTime) { super(tableName, SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START); - this.channelName = channelName; - this.channelCreationTime = startTime; + this.tpChannelName = channelName; + this.tpChannelCreationTime = startTime; } @Override public void dumpTo(ObjectNode msg) { msg.put(TABLE_NAME, this.tableName); - msg.put(CHANNEL_NAME, this.channelName); + msg.put(TP_CHANNEL_NAME, this.tpChannelName); msg.put(IS_REUSE_TABLE, this.isReuseTable); - msg.put(CHANNEL_CREATION_TIME, channelCreationTime); + msg.put(TP_CHANNEL_CREATION_TIME, tpChannelCreationTime); } @Override diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 17b45a783..15a316104 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -103,7 +103,7 @@ public boolean isEmpty() { public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); msg.put(TelemetryConstants.CONNECTOR_NAME, this.connectorName); - msg.put(TelemetryConstants.Streaming.CHANNEL_NAME, this.channelName); + msg.put(TelemetryConstants.Streaming.TP_CHANNEL_NAME, this.channelName); msg.put( TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE, @@ -111,9 +111,8 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); - final long currTime = System.currentTimeMillis(); - msg.put(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME, this.channelCreationTime); - msg.put(TelemetryConstants.Streaming.CHANNEL_CLOSE_TIME, currTime); + msg.put(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME, this.channelCreationTime); + msg.put(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME, System.currentTimeMillis()); } /** Registers all the Metrics inside the metricRegistry. */ diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 6d0711616..d23dfa147 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -51,8 +51,8 @@ public final class TelemetryConstants { public class Streaming { public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - public static final String CHANNEL_NAME = "channel_name"; - public static final String CHANNEL_CREATION_TIME = "channel_creation_time"; - public static final String CHANNEL_CLOSE_TIME = "channel_close_time"; + public static final String TP_CHANNEL_NAME = "tp_channel_name"; + public static final String TP_CHANNEL_CREATION_TIME = "tp_channel_creation_time"; + public static final String TP_CHANNEL_CLOSE_TIME = "tp_channel_close_time"; } } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index c8b136279..344d0b2a3 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -164,8 +164,8 @@ public void testReportKafkaPartitionUsage() { final String expectedStageName = "stageName"; final String expectedPipeName = "pipeName"; final String expectedConnectorName = "connectorName"; - final String expectedChannelName = "channelName"; - final long expectedChannelCreationTime = 1234; + final String expectedTpChannelName = "channelName"; + final long expectedTpChannelCreationTime = 1234; final long expectedProcessedOffset = 1; final long expectedFlushedOffset = 2; final long expectedCommittedOffset = 3; @@ -188,8 +188,8 @@ public void testReportKafkaPartitionUsage() { new SnowflakeTelemetryChannelStatus( expectedTableName, expectedConnectorName, - expectedChannelName, - expectedChannelCreationTime, + expectedTpChannelName, + expectedTpChannelCreationTime, false, null, new AtomicLong(expectedOffsetPersistedInSnowflake), @@ -234,8 +234,11 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() - == expectedChannelCreationTime); + dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME).asLong() + == expectedTpChannelCreationTime); + Assert.assertTrue( + dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -246,7 +249,7 @@ public void testReportKafkaPartitionUsage() { expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); + expectedTpChannelName, dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } @@ -315,13 +318,13 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.CHANNEL_CREATION_TIME).asLong() + dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.Streaming.CHANNEL_NAME).asText()); + expectedChannelName, dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); } } From 09855b280692048ccb1b3dec68ecf30c693f2e4e Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:41:56 -0700 Subject: [PATCH 28/34] autoformatting --- .../telemetry/SnowflakeTelemetryServiceTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 344d0b2a3..086144d9d 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -237,8 +237,10 @@ public void testReportKafkaPartitionUsage() { dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME).asLong() == expectedTpChannelCreationTime); Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); + dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() + <= System.currentTimeMillis() + && dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() + >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -249,7 +251,8 @@ public void testReportKafkaPartitionUsage() { expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( - expectedTpChannelName, dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); + expectedTpChannelName, + dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } From 56dffbdb55db559d5194a621c2e377b6743d03d0 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:58:47 -0700 Subject: [PATCH 29/34] no streaming subclass --- .../internal/metrics/MetricsUtil.java | 31 +++++++++---------- .../SnowflakeTelemetryChannelCreation.java | 4 +-- .../SnowflakeTelemetryChannelStatus.java | 14 ++++----- .../telemetry/TelemetryConstants.java | 13 ++++---- .../streaming/SnowflakeSinkServiceV2IT.java | 4 +-- .../SnowflakeTelemetryServiceTest.java | 16 +++++----- 6 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index db906743e..beb232f0e 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -5,21 +5,6 @@ /** All metrics related constants. Mainly for JMX */ public class MetricsUtil { - public class Streaming { - /** - * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset - * description - */ - public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; - - public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - } - - public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; - - // Offset related constants - public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; - /** * Number of files we call insertFiles API in snowpipe. Note: There is currently a limitation of * 5k files being sent to a single rest request. So these metric has no one to one relation @@ -93,8 +78,20 @@ public class Streaming { public static final String LATENCY_SUB_DOMAIN = "latencies"; - // Partition related constants - public static final String PARTITION_SUB_DOMAIN = "partition"; + ////////////////////////////////////////////// Streaming Constants ////////////////////////////////////////////// + /** + * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset + * description + */ + public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; + + public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; + + // Offset related constants + public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; + ////////////////////////////////////////////// Streaming Constants ////////////////////////////////////////////// public enum EventType { /** diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 4d385a620..bc21549b1 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -18,8 +18,8 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.Streaming.TP_CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_CREATION_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 15a316104..35b426461 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -103,16 +103,16 @@ public boolean isEmpty() { public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); msg.put(TelemetryConstants.CONNECTOR_NAME, this.connectorName); - msg.put(TelemetryConstants.Streaming.TP_CHANNEL_NAME, this.channelName); + msg.put(TelemetryConstants.TP_CHANNEL_NAME, this.channelName); msg.put( - TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE, + TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); - msg.put(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); + msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); - msg.put(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME, this.channelCreationTime); - msg.put(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME, System.currentTimeMillis()); + msg.put(TelemetryConstants.TP_CHANNEL_CREATION_TIME, this.channelCreationTime); + msg.put(TelemetryConstants.TP_CHANNEL_CLOSE_TIME, System.currentTimeMillis()); } /** Registers all the Metrics inside the metricRegistry. */ @@ -131,7 +131,7 @@ private void registerChannelJMXMetrics() { constructMetricName( this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE), + MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE), (Gauge) this.offsetPersistedInSnowflake::get); currentMetricRegistry.register( @@ -143,7 +143,7 @@ private void registerChannelJMXMetrics() { constructMetricName( this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET), + MetricsUtil.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index d23dfa147..b1977017d 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -48,11 +48,10 @@ public final class TelemetryConstants { public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; - public class Streaming { - public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; - public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - public static final String TP_CHANNEL_NAME = "tp_channel_name"; - public static final String TP_CHANNEL_CREATION_TIME = "tp_channel_creation_time"; - public static final String TP_CHANNEL_CLOSE_TIME = "tp_channel_close_time"; - } + /** Streaming Constants */ + public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; + public static final String TP_CHANNEL_NAME = "tp_channel_name"; + public static final String TP_CHANNEL_CREATION_TIME = "tp_channel_creation_time"; + public static final String TP_CHANNEL_CLOSE_TIME = "tp_channel_close_time"; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java index 9fa23fee8..b2185eb2c 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/streaming/SnowflakeSinkServiceV2IT.java @@ -465,7 +465,7 @@ private void verifyPartitionMetrics( MetricsUtil.constructMetricName( partitionChannelKey, MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE)) + MetricsUtil.OFFSET_PERSISTED_IN_SNOWFLAKE)) .getValue() == offsetPersistedInSnowflake; assert (long) @@ -483,7 +483,7 @@ private void verifyPartitionMetrics( MetricsUtil.constructMetricName( partitionChannelKey, MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.Streaming.LATEST_CONSUMER_OFFSET)) + MetricsUtil.LATEST_CONSUMER_OFFSET)) .getValue() == latestConsumerOffset; } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 086144d9d..db7c09586 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -234,25 +234,25 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME).asLong() + dataNode.get(TelemetryConstants.TP_CHANNEL_CREATION_TIME).asLong() == expectedTpChannelCreationTime); Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() + dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CLOSE_TIME).asLong() + && dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); Assert.assertEquals( expectedLatestConsumerOffset, - dataNode.get(TelemetryConstants.Streaming.LATEST_CONSUMER_OFFSET).asLong()); + dataNode.get(TelemetryConstants.LATEST_CONSUMER_OFFSET).asLong()); Assert.assertEquals( expectedOffsetPersistedInSnowflake, - dataNode.get(TelemetryConstants.Streaming.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); + dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( expectedTpChannelName, - dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); + dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } @@ -321,13 +321,13 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_CREATION_TIME).asLong() + dataNode.get(TelemetryConstants.TP_CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.Streaming.TP_CHANNEL_NAME).asText()); + expectedChannelName, dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); } } From b2c036d9bf65d453784e0af6b2bdb833b430f188 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 13:58:50 -0700 Subject: [PATCH 30/34] autoformatting --- .../kafka/connector/internal/metrics/MetricsUtil.java | 6 ++++-- .../telemetry/SnowflakeTelemetryChannelCreation.java | 2 +- .../telemetry/SnowflakeTelemetryChannelStatus.java | 7 ++----- .../connector/internal/telemetry/TelemetryConstants.java | 1 + .../internal/telemetry/SnowflakeTelemetryServiceTest.java | 6 ++---- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index beb232f0e..1fca77e18 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -78,7 +78,8 @@ public class MetricsUtil { public static final String LATENCY_SUB_DOMAIN = "latencies"; - ////////////////////////////////////////////// Streaming Constants ////////////////////////////////////////////// + ////////////////////////////////////////////// Streaming Constants + // ////////////////////////////////////////////// /** * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset * description @@ -91,7 +92,8 @@ public class MetricsUtil { // Offset related constants public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; - ////////////////////////////////////////////// Streaming Constants ////////////////////////////////////////////// + ////////////////////////////////////////////// Streaming Constants + // ////////////////////////////////////////////// public enum EventType { /** diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index bc21549b1..5967fe351 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -18,9 +18,9 @@ package com.snowflake.kafka.connector.internal.streaming.telemetry; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_CREATION_TIME; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index 35b426461..fc9dcd5ea 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -106,8 +106,7 @@ public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TP_CHANNEL_NAME, this.channelName); msg.put( - TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, - this.offsetPersistedInSnowflake.get()); + TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); @@ -141,9 +140,7 @@ private void registerChannelJMXMetrics() { currentMetricRegistry.register( constructMetricName( - this.channelName, - MetricsUtil.OFFSET_SUB_DOMAIN, - MetricsUtil.LATEST_CONSUMER_OFFSET), + this.channelName, MetricsUtil.OFFSET_SUB_DOMAIN, MetricsUtil.LATEST_CONSUMER_OFFSET), (Gauge) this.latestConsumerOffset::get); } catch (IllegalArgumentException ex) { LOGGER.warn("Metrics already present:{}", ex.getMessage()); diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index b1977017d..98d8da69a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -50,6 +50,7 @@ public final class TelemetryConstants { /** Streaming Constants */ public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; + public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; public static final String TP_CHANNEL_NAME = "tp_channel_name"; public static final String TP_CHANNEL_CREATION_TIME = "tp_channel_creation_time"; diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index db7c09586..1eb1f2200 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -239,8 +239,7 @@ public void testReportKafkaPartitionUsage() { Assert.assertTrue( dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() - >= this.startTime); + && dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -251,8 +250,7 @@ public void testReportKafkaPartitionUsage() { expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( - expectedTpChannelName, - dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); + expectedTpChannelName, dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } From a951535aea3136b029f62cb756c7e3090d4fa467 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 14:05:33 -0700 Subject: [PATCH 31/34] tp -> topicparition and comment --- .../connector/internal/metrics/MetricsUtil.java | 6 ++---- .../telemetry/SnowflakeTelemetryChannelCreation.java | 8 ++++---- .../telemetry/SnowflakeTelemetryChannelStatus.java | 6 +++--- .../internal/telemetry/TelemetryConstants.java | 11 ++++++----- .../telemetry/SnowflakeTelemetryServiceTest.java | 12 ++++++------ 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index 1fca77e18..391bc1a7a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -78,8 +78,7 @@ public class MetricsUtil { public static final String LATENCY_SUB_DOMAIN = "latencies"; - ////////////////////////////////////////////// Streaming Constants - // ////////////////////////////////////////////// + //************ Streaming Constants ************// /** * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset * description @@ -87,13 +86,12 @@ public class MetricsUtil { public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; + //********** ^ Streaming Constants ^ **********// public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; // Offset related constants public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; - ////////////////////////////////////////////// Streaming Constants - // ////////////////////////////////////////////// public enum EventType { /** diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java index 5967fe351..9de9ef32a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelCreation.java @@ -19,8 +19,8 @@ import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.IS_REUSE_TABLE; import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TABLE_NAME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_CREATION_TIME; -import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TP_CHANNEL_NAME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TOPIC_PARTITION_CHANNEL_CREATION_TIME; +import static com.snowflake.kafka.connector.internal.telemetry.TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryBasicInfo; import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService; @@ -45,10 +45,10 @@ public SnowflakeTelemetryChannelCreation( @Override public void dumpTo(ObjectNode msg) { msg.put(TABLE_NAME, this.tableName); - msg.put(TP_CHANNEL_NAME, this.tpChannelName); + msg.put(TOPIC_PARTITION_CHANNEL_NAME, this.tpChannelName); msg.put(IS_REUSE_TABLE, this.isReuseTable); - msg.put(TP_CHANNEL_CREATION_TIME, tpChannelCreationTime); + msg.put(TOPIC_PARTITION_CHANNEL_CREATION_TIME, tpChannelCreationTime); } @Override diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index fc9dcd5ea..a72c90051 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -103,15 +103,15 @@ public boolean isEmpty() { public void dumpTo(ObjectNode msg) { msg.put(TelemetryConstants.TABLE_NAME, this.tableName); msg.put(TelemetryConstants.CONNECTOR_NAME, this.connectorName); - msg.put(TelemetryConstants.TP_CHANNEL_NAME, this.channelName); + msg.put(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME, this.channelName); msg.put( TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE, this.offsetPersistedInSnowflake.get()); msg.put(TelemetryConstants.PROCESSED_OFFSET, this.processedOffset.get()); msg.put(TelemetryConstants.LATEST_CONSUMER_OFFSET, this.latestConsumerOffset.get()); - msg.put(TelemetryConstants.TP_CHANNEL_CREATION_TIME, this.channelCreationTime); - msg.put(TelemetryConstants.TP_CHANNEL_CLOSE_TIME, System.currentTimeMillis()); + msg.put(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CREATION_TIME, this.channelCreationTime); + msg.put(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME, System.currentTimeMillis()); } /** Registers all the Metrics inside the metricRegistry. */ diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index 98d8da69a..ae061234a 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -48,11 +48,12 @@ public final class TelemetryConstants { public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; - /** Streaming Constants */ + //************ Streaming Constants ************// public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; - public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - public static final String TP_CHANNEL_NAME = "tp_channel_name"; - public static final String TP_CHANNEL_CREATION_TIME = "tp_channel_creation_time"; - public static final String TP_CHANNEL_CLOSE_TIME = "tp_channel_close_time"; + + public static final String TOPIC_PARTITION_CHANNEL_NAME = "topic_partition_channel_name"; + public static final String TOPIC_PARTITION_CHANNEL_CREATION_TIME = "topic_partition_channel_creation_time"; + public static final String TOPIC_PARTITION_CHANNEL_CLOSE_TIME = "topic_partition_channel_close_time"; + //********** ^ Streaming Constants ^ **********// } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index 1eb1f2200..afbdec3cb 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -234,12 +234,12 @@ public void testReportKafkaPartitionUsage() { Assert.assertEquals(expectedStageName, dataNode.get(TelemetryConstants.STAGE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.TP_CHANNEL_CREATION_TIME).asLong() + dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CREATION_TIME).asLong() == expectedTpChannelCreationTime); Assert.assertTrue( - dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() + dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.TP_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); + && dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -250,7 +250,7 @@ public void testReportKafkaPartitionUsage() { expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( - expectedTpChannelName, dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); + expectedTpChannelName, dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } @@ -319,13 +319,13 @@ public void testReportKafkaPartitionStart() { Assert.assertEquals(expectedPipeName, dataNode.get(TelemetryConstants.PIPE_NAME).asText()); } else { Assert.assertTrue( - dataNode.get(TelemetryConstants.TP_CHANNEL_CREATION_TIME).asLong() + dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CREATION_TIME).asLong() == expectedChannelCreationTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.TP_CHANNEL_NAME).asText()); + expectedChannelName, dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); } } From 947b0914cfac3e65783dc9e272ae06e9379309c0 Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 14:05:37 -0700 Subject: [PATCH 32/34] autoformatting --- .../kafka/connector/internal/metrics/MetricsUtil.java | 4 ++-- .../internal/telemetry/TelemetryConstants.java | 10 ++++++---- .../telemetry/SnowflakeTelemetryServiceTest.java | 9 ++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index 391bc1a7a..b719fb085 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -78,7 +78,7 @@ public class MetricsUtil { public static final String LATENCY_SUB_DOMAIN = "latencies"; - //************ Streaming Constants ************// + // ************ Streaming Constants ************// /** * See {@link com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannel} for offset * description @@ -86,7 +86,7 @@ public class MetricsUtil { public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; - //********** ^ Streaming Constants ^ **********// + // ********** ^ Streaming Constants ^ **********// public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; diff --git a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java index ae061234a..de02dda73 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/telemetry/TelemetryConstants.java @@ -48,12 +48,14 @@ public final class TelemetryConstants { public static final String FILE_COUNT_RESTART = "file_count_restart"; public static final String FILE_COUNT_REPROCESS_PURGE = "file_count_reprocess_purge"; - //************ Streaming Constants ************// + // ************ Streaming Constants ************// public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset"; public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; public static final String TOPIC_PARTITION_CHANNEL_NAME = "topic_partition_channel_name"; - public static final String TOPIC_PARTITION_CHANNEL_CREATION_TIME = "topic_partition_channel_creation_time"; - public static final String TOPIC_PARTITION_CHANNEL_CLOSE_TIME = "topic_partition_channel_close_time"; - //********** ^ Streaming Constants ^ **********// + public static final String TOPIC_PARTITION_CHANNEL_CREATION_TIME = + "topic_partition_channel_creation_time"; + public static final String TOPIC_PARTITION_CHANNEL_CLOSE_TIME = + "topic_partition_channel_close_time"; + // ********** ^ Streaming Constants ^ **********// } diff --git a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java index afbdec3cb..88cb4edd5 100644 --- a/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java +++ b/src/test/java/com/snowflake/kafka/connector/internal/telemetry/SnowflakeTelemetryServiceTest.java @@ -239,7 +239,8 @@ public void testReportKafkaPartitionUsage() { Assert.assertTrue( dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME).asLong() <= System.currentTimeMillis() - && dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME).asLong() >= this.startTime); + && dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_CLOSE_TIME).asLong() + >= this.startTime); Assert.assertEquals( SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_USAGE.toString(), allNode.get("type").asText()); @@ -250,7 +251,8 @@ public void testReportKafkaPartitionUsage() { expectedOffsetPersistedInSnowflake, dataNode.get(TelemetryConstants.OFFSET_PERSISTED_IN_SNOWFLAKE).asLong()); Assert.assertEquals( - expectedTpChannelName, dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); + expectedTpChannelName, + dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); Assert.assertEquals( expectedConnectorName, dataNode.get(TelemetryConstants.CONNECTOR_NAME).asText()); } @@ -325,7 +327,8 @@ public void testReportKafkaPartitionStart() { SnowflakeTelemetryService.TelemetryType.KAFKA_CHANNEL_START.toString(), allNode.get("type").asText()); Assert.assertEquals( - expectedChannelName, dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); + expectedChannelName, + dataNode.get(TelemetryConstants.TOPIC_PARTITION_CHANNEL_NAME).asText()); } } From d8a82263bc33a98e47e58d64bb038f2add62fbdf Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 14:10:11 -0700 Subject: [PATCH 33/34] personal nits --- .../kafka/connector/internal/metrics/MetricsUtil.java | 10 +++++----- .../telemetry/SnowflakeTelemetryChannelStatus.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java index b719fb085..a84150953 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java @@ -5,6 +5,11 @@ /** All metrics related constants. Mainly for JMX */ public class MetricsUtil { + public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; + + // Offset related constants + public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; + /** * Number of files we call insertFiles API in snowpipe. Note: There is currently a limitation of * 5k files being sent to a single rest request. So these metric has no one to one relation @@ -88,11 +93,6 @@ public class MetricsUtil { public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset"; // ********** ^ Streaming Constants ^ **********// - public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector"; - - // Offset related constants - public static final String FILE_COUNT_SUB_DOMAIN = "file-counts"; - public enum EventType { /** * Time difference between the record put into kafka to record fetched into Kafka Connector Can diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java index a72c90051..d67446672 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/telemetry/SnowflakeTelemetryChannelStatus.java @@ -35,7 +35,7 @@ * Extension of {@link SnowflakeTelemetryBasicInfo} class used to send data to snowflake when the * TopicPartitionChannel closes. Also creates and registers various metrics with JMX * - *

Most of the data sent to Snowflake is an aggregated data. + *

Most of the data sent to Snowflake is aggregated data. */ public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo { public static final long NUM_METRICS = 3; // update when new metrics are added From 6b23cdc16c42193ee74fb174bbb8466cac501dda Mon Sep 17 00:00:00 2001 From: revi cheng Date: Wed, 13 Sep 2023 14:36:48 -0700 Subject: [PATCH 34/34] fix npe --- .../connector/internal/streaming/TopicPartitionChannel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java index 758537734..48e2e0ff9 100644 --- a/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java +++ b/src/main/java/com/snowflake/kafka/connector/internal/streaming/TopicPartitionChannel.java @@ -284,7 +284,7 @@ public TopicPartitionChannel( // setup telemetry and metrics String connectorName = - conn.getConnectorName() == null || conn.getConnectorName().isEmpty() + conn == null || conn.getConnectorName() == null || conn.getConnectorName().isEmpty() ? "default_connector_name" : conn.getConnectorName(); this.snowflakeTelemetryChannelStatus =