Expects configuration from SnowflakeStreamingSinkConnector * *
Creates sink service instance, takes records loaded from those Kafka partitions and ingests to
* Snowflake via Sink service
*/
public class SnowflakeSinkTask extends SinkTask {
private static final long WAIT_TIME = 5 * 1000; // 5 sec
private static final int REPEAT_TIME = 12; // 60 sec
// the dynamic logger is intended to be attached per task instance. the instance id will be set
// during task start, however if it is not set, it falls back to the static logger
private static final KCLogger STATIC_LOGGER =
new KCLogger(SnowflakeSinkTask.class.getName() + "_STATIC");
private KCLogger DYNAMIC_LOGGER;
private volatile SnowflakeSinkService sink = null;
// snowflake JDBC connection provides methods to interact with user's
// snowflake
// account and execute queries
private SnowflakeConnectionService conn = null;
// tracks number of tasks the config wants to create
private String taskConfigId = "-1";
private long taskStartTime;
private final SnowflakeSinkTaskAuthorizationExceptionTracker authorizationExceptionTracker =
new SnowflakeSinkTaskAuthorizationExceptionTracker();
// Stores channel error exception detected in preCommit to fail on next put() call
private volatile SnowflakeKafkaConnectorException channelErrorToFailOn = null;
// Periodic telemetry reporter for channel status
private PeriodicTelemetryReporter telemetryReporter = null;
// Task-level JMX metrics (lifecycle, throughput, duration)
private TaskMetrics taskMetrics = TaskMetrics.noop();
/** default constructor, invoked by kafka connect framework */
public SnowflakeSinkTask() {
DYNAMIC_LOGGER = new KCLogger(this.getClass().getName());
}
@VisibleForTesting
public SnowflakeSinkTask(
SnowflakeSinkService service, SnowflakeConnectionService connectionService) {
DYNAMIC_LOGGER = new KCLogger(this.getClass().getName());
this.sink = service;
this.conn = connectionService;
}
private SnowflakeConnectionService getConnection() {
try {
waitFor(() -> conn != null);
} catch (Exception e) {
throw SnowflakeErrors.ERROR_5013.getException();
}
return conn;
}
/**
* Return an instance of SnowflakeConnection if it was set previously by calling Start(). Else,
* return an empty
*
* @return Optional of SnowflakeConnectionService
*/
public Optional Note that calling this method does not perform synchronous cleanup in Snowpipe based
* implementation
*/
@Override
public void stop() {
this.DYNAMIC_LOGGER.info("stopping task {}", this.taskConfigId);
// Stop telemetry reporter first
if (this.telemetryReporter != null) {
this.telemetryReporter.stop();
}
this.taskMetrics.unregister();
if (this.sink != null) {
this.sink.stop();
}
this.DYNAMIC_LOGGER.info(
"task stopped, total task runtime: {} milliseconds",
getDurationFromStartMs(this.taskStartTime));
}
/**
* init ingestion task in Sink service
*
* @param partitions - The list of all partitions that are now assigned to the task
*/
@Override
public void open(final Collection Closes all running task because the parameter of open function contains all partition info
* but not only the new partition
*
* @param partitions - The list of all partitions that were assigned to the task
*/
@Override
public void close(final Collection Note that exceptions thrown during preCommit() are swallowed by Kafka Connect and will not
* cause task failure.
*/
public class SnowflakeSinkTaskAuthorizationExceptionTracker {
private static final String AUTHORIZATION_EXCEPTION_MESSAGE = "Authorization failed after retry";
private boolean authorizationTaskFailureEnabled;
private boolean authorizationErrorReported;
public SnowflakeSinkTaskAuthorizationExceptionTracker() {
this.authorizationTaskFailureEnabled = true;
this.authorizationErrorReported = false;
}
public void updateStateOnTaskStart(Map Expected configuration: including topic names, partition numbers, snowflake connection info
* and credentials info
*
* Creates snowflake internal stages, snowflake tables provides configuration to SinkTasks
* running on Kafka Connect Workers.
*/
public class SnowflakeStreamingSinkConnector extends SinkConnector {
// create logger without correlationId for now
private static final KCLogger LOGGER =
new KCLogger(SnowflakeStreamingSinkConnector.class.getName());
private Map Creates snowflake internal stages and snowflake tables
*
* @param parsedConfig has the configuration settings
*/
@Override
public void start(final Map Cleans up pipes, after making sure there are no pending files to ingest.
*
* Also ensures that there are no leaked stages, no leaked staged files, and no leaked pipes
*/
@Override
public void stop() {
LOGGER.info("SnowflakeStreamingSinkConnector connector stopping...");
setupComplete = false;
if (telemetryClient != null) {
telemetryClient.reportKafkaConnectStop(connectorStartTime);
}
}
/**
* @return Sink task class
*/
@Override
public Class taskClass() {
return SnowflakeSinkTask.class;
}
/**
* taskConfigs method returns a set of configurations for SinkTasks based on the current
* configuration, producing at most 'maxTasks' configurations
*
* @param maxTasks maximum number of SinkTasks for this instance of
* SnowflakeStreamingSinkConnector
* @return a list containing 'maxTasks' copies of the configuration
*/
@Override
public List A URl connection timeout is added in case Maven repo is not reachable in a proxy'd
* environment. Returning false from this method doesn't have any side effects to start the
* connector.
*
* Version upgrade logic:
*
* [SF_KAFKA_CONNECTOR] this is a log message
*
* [SF_KAFKA_CONNECTOR] this is the second line
*
* All log messages should be wrapped by Snowflake tag. Then user can filter out log messages
* output from Snowflake Kafka connector by these tags.
*
* @param format log message format string
* @param vars variable list
* @return log message wrapped by snowflake tag
*/
public static String formatLogMessage(String format, Object... vars) {
return SF_LOG_TAG + " " + formatString(format, vars);
}
public static String formatString(String format, Object... vars) {
for (int i = 0; i < vars.length; i++) {
format = format.replaceFirst("\\{}", Objects.toString(vars[i]).replaceAll("\\$", "\\\\\\$"));
}
return format;
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/config/AuthenticatorType.java
================================================
package com.snowflake.kafka.connector.config;
import static com.snowflake.kafka.connector.Constants.KafkaConnectorConfigParams.SNOWFLAKE_AUTHENTICATOR;
import java.util.Arrays;
import java.util.Locale;
import java.util.stream.Collectors;
/** Authentication method for Snowflake connections. */
public enum AuthenticatorType {
/** Key-pair (JWT) authentication. This is the default. */
SNOWFLAKE_JWT,
/** External OAuth authentication. */
OAUTH;
/** The config string value, matching the v3 connector convention (lowercase with underscores). */
public String toConfigValue() {
return name().toLowerCase(Locale.ROOT);
}
/**
* Parses a config string into an authenticator type (case-insensitive). Returns {@link
* #SNOWFLAKE_JWT} for null or empty input.
*
* @throws IllegalArgumentException for unrecognized values
*/
public static AuthenticatorType fromConfig(String value) {
if (value == null || value.trim().isEmpty()) {
return SNOWFLAKE_JWT;
}
String normalized = value.trim().toUpperCase(Locale.ROOT);
try {
return valueOf(normalized);
} catch (IllegalArgumentException e) {
String validValues =
Arrays.stream(values())
.map(AuthenticatorType::toConfigValue)
.collect(Collectors.joining(", "));
throw new IllegalArgumentException(
"Invalid value '"
+ value.trim()
+ "' for config '"
+ SNOWFLAKE_AUTHENTICATOR
+ "'. Valid values are: "
+ validValues,
e);
}
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/config/CommaSeparatedKeyValueValidator.java
================================================
package com.snowflake.kafka.connector.config;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigException;
/**
* Class which validates key value pairs in the format It doesn't validate the type of values, only making sure the format is correct.
*/
class CommaSeparatedKeyValueValidator implements ConfigDef.Validator {
public CommaSeparatedKeyValueValidator() {}
public void ensureValid(String name, Object value) {
String s = (String) value;
// Validate the comma-separated key-value pairs string
if (s != null && !s.isEmpty() && !isValidCommaSeparatedKeyValueString(s)) {
throw new ConfigException(name, value, "Format: In the beginning of the function we will check if we have the permission to do schema
* evolution, and we will error out if we don't
*
* @param tableName table name
*/
void createTableWithOnlyMetadataColumn(String tableName);
/**
* Calls describe table statement and returns all columns and corresponding types.
*
* @param tableName - table name
* @return Optional.empty() if table does not exist. List of all table columns and their types
* otherwise.
*/
Optional If message is null, we use Enum's toString() method to wrap inside
* SnowflakeKafkaConnectorException
*
* @param msg Message to send to Telemetry Service. Remember, we Strip the message
* @param telemetryService can be null
* @return Exception wrapped in Snowflake Connector Exception
*/
public SnowflakeKafkaConnectorException getException(
String msg, SnowflakeTelemetryService telemetryService) {
if (telemetryService != null) {
telemetryService.reportKafkaConnectFatalError(
getCode() + msg.substring(0, Math.min(msg.length(), 500)));
}
if (msg == null || msg.isEmpty()) {
return new SnowflakeKafkaConnectorException(toString(), code);
} else {
return new SnowflakeKafkaConnectorException(
Utils.formatLogMessage(
"Exception: {}\nError Code: {}\nDetail: {}\nMessage: {}", name, code, detail, msg),
code);
}
}
public String getCode() {
return code;
}
public String getDetail() {
return this.detail;
}
@Override
public String toString() {
return Utils.formatLogMessage("Exception: {}\nError Code: {}\nDetail: {}", name, code, detail);
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/SnowflakeKafkaConnectorException.java
================================================
package com.snowflake.kafka.connector.internal;
public class SnowflakeKafkaConnectorException extends RuntimeException {
private final String code;
public SnowflakeKafkaConnectorException(String msg, String code) {
super(msg);
this.code = code;
}
public String getCode() {
return code;
}
public boolean checkErrorCode(SnowflakeErrors error) {
return this.code.equals(error.getCode());
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/SnowflakeSinkService.java
================================================
package com.snowflake.kafka.connector.internal;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.snowflake.kafka.connector.internal.streaming.channel.TopicPartitionChannel;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.connect.sink.SinkRecord;
/** Background service of data sink, responsible to create/drop pipe and ingest/purge files */
public interface SnowflakeSinkService {
/**
* Start the Task. This should handle any configuration parsing and one-time setup of the task.
*
* @param topicPartition TopicPartition passed from Kafka
*/
void startPartition(TopicPartition topicPartition);
/**
* Start a collection of TopicPartition. This should handle any configuration parsing and one-time
* setup of the task.
*
* @param partitions collection of topic partitions
*/
void startPartitions(Collection Note that calling this method does not perform synchronous cleanup in Snowpipe based
* implementation
*/
void stop();
/**
* retrieve sink service status
*
* @return true is closed
*/
boolean isClosed();
Map Group 1: If https is present or not. (Not required)
*
* Group 2: Is the entire URL including the port number
*
* Group 3: URL until .com
*
* Group 4: Account name (may include org-account/alias)
*
* Group 5: (Everything after accountname or org-accountname until .com)
*
* Group 7: port number
*/
private static final String SNOWFLAKE_URL_REGEX_PATTERN =
"^(https?://)?((([\\w\\d-]+)(\\.[\\w\\d-]+){2,})(:(\\d+))?)/?$";
public SnowflakeURL(String urlStr) {
Pattern pattern = Pattern.compile(SNOWFLAKE_URL_REGEX_PATTERN);
Matcher matcher = pattern.matcher(urlStr.trim().toLowerCase());
if (!matcher.find()) {
throw SnowflakeErrors.ERROR_0007.getException("input url: " + urlStr);
}
ssl = !"http://".equals(matcher.group(1));
url = matcher.group(3);
account = matcher.group(4);
if (matcher.group(7) != null) {
port = Integer.parseInt(matcher.group(7));
} else if (ssl) {
port = 443;
} else {
port = 80;
}
jdbcUrl = "jdbc:snowflake://" + url + ":" + port;
LOGGER.debug("parsed Snowflake URL: {}", urlStr);
}
String getJdbcUrl() {
return jdbcUrl;
}
public String getAccount() {
return account;
}
public boolean sslEnabled() {
return ssl;
}
public String getScheme() {
if (ssl) {
return "https";
} else {
return "http";
}
}
String getFullUrl() {
return url + ":" + port;
}
public String getUrlWithoutPort() {
return url;
}
int getPort() {
return port;
}
@Override
public String toString() {
return getFullUrl();
}
@Override
public String hostWithPort() {
return getFullUrl();
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/StandardSnowflakeConnectionService.java
================================================
package com.snowflake.kafka.connector.internal;
import static com.snowflake.kafka.connector.Utils.TABLE_COLUMN_METADATA;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.snowflake.kafka.connector.internal.schemaevolution.ColumnInfos;
import com.snowflake.kafka.connector.internal.streaming.v2.migration.Ssv1MigrationResponse;
import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService;
import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryServiceFactory;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import net.snowflake.client.api.driver.SnowflakeDriver;
/**
* Implementation of Snowflake Connection Service interface which includes all handshake between KC
* and SF through JDBC connection.
*/
public class StandardSnowflakeConnectionService implements SnowflakeConnectionService {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String COLUMN_COMMENT =
"created by automatic table creation from Snowflake Kafka Connector High Performance";
private static final String SHOW_ICEBERG_TABLES_QUERY = "show iceberg tables like ? limit 1";
private final KCLogger LOGGER = new KCLogger(StandardSnowflakeConnectionService.class.getName());
private final Connection conn;
private final SnowflakeTelemetryService telemetry;
private final String connectorName;
private final String taskID;
StandardSnowflakeConnectionService(
JdbcProperties jdbcProperties, SnowflakeURL url, String connectorName, String taskID) {
this.connectorName = connectorName;
this.taskID = taskID;
Properties proxyProperties = jdbcProperties.getProxyProperties();
Properties combinedProperties = jdbcProperties.getProperties();
try {
if (!proxyProperties.isEmpty()) {
LOGGER.debug("Proxy properties are set, passing in JDBC while creating the connection");
} else {
LOGGER.info("Establishing a JDBC connection with url:{}", url.getJdbcUrl());
}
this.conn = new SnowflakeDriver().connect(url.getJdbcUrl(), combinedProperties);
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_1001.getException(e);
}
this.telemetry =
SnowflakeTelemetryServiceFactory.builder(conn)
.setAppName(this.connectorName)
.setTaskID(this.taskID)
.build();
LOGGER.info("initialized the snowflake connection");
}
@Override
public void createTableWithOnlyMetadataColumn(final String tableName) {
checkConnection();
InternalUtils.assertNotEmpty("tableName", tableName);
String createTableQuery =
"create table if not exists identifier(?) (record_metadata variant comment '"
+ COLUMN_COMMENT
+ "') enable_schema_evolution = true error_logging = true";
try {
PreparedStatement stmt = conn.prepareStatement(createTableQuery);
stmt.setString(1, quoteIdentifier(tableName));
stmt.execute();
stmt.close();
} catch (SQLException e) {
// Snowflake rejects CREATE TABLE IF NOT EXISTS when the name is already taken by an
// ICEBERG TABLE (cross-type conflict is not suppressed by IF NOT EXISTS). KCv4 only
// supports pre-created Iceberg tables; error_logging is not available for them.
// We match on the error message text because Snowflake does not provide a stable SQL
// error code that distinguishes this cross-type conflict from other CREATE TABLE errors.
if (e.getMessage() != null && e.getMessage().contains("already exists as ICEBERG_TABLE")) {
LOGGER.warn(
"Table '{}' is a pre-created Iceberg table. Skipping auto-creation."
+ " Error table functionality is not available for Iceberg tables.",
tableName);
return;
}
throw SnowflakeErrors.ERROR_2007.getException(e);
}
LOGGER.info(
"Created table {} with RECORD_METADATA column and ERROR_LOGGING enabled", tableName);
}
@Override
public boolean tableExist(final String tableName) {
return describeTable(tableName).isPresent();
}
@Override
public boolean pipeExist(final String pipeName) {
LOGGER.info("Calling DESCRIBE PIPE {}", pipeName);
checkConnection();
InternalUtils.assertNotEmpty("pipeName", pipeName);
String query = "desc pipe identifier(?)";
PreparedStatement stmt = null;
boolean exist;
try {
stmt = conn.prepareStatement(query);
stmt.setString(1, pipeName);
stmt.execute();
exist = true;
} catch (SQLException e) {
LOGGER.debug("pipe {} doesn't exist", pipeName);
exist = false;
} finally {
if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
return exist;
}
@Override
public boolean isTableCompatible(final String tableName) {
checkConnection();
InternalUtils.assertNotEmpty("tableName", tableName);
String query = "desc table identifier(?)";
PreparedStatement stmt = null;
ResultSet result = null;
boolean compatible;
try {
stmt = conn.prepareStatement(query);
stmt.setString(1, quoteIdentifier(tableName));
result = stmt.executeQuery();
boolean hasMeta = false;
boolean allNullable = true;
while (result.next()) {
switch (result.getString(1)) {
case TABLE_COLUMN_METADATA:
if (result.getString(2).equals("VARIANT")) {
hasMeta = true;
}
break;
default:
if (result.getString(4).equals("N")) {
allNullable = false;
}
}
}
compatible = hasMeta && allNullable;
} catch (SQLException e) {
LOGGER.debug("Table {} doesn't exist. Exception {}", tableName, e.getStackTrace());
compatible = false;
} finally {
try {
if (result != null) {
result.close();
}
} catch (Exception e) {
e.printStackTrace();
}
try {
if (stmt != null) {
stmt.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
LOGGER.info("Table {} compatibility is {}", tableName, compatible);
return compatible;
}
@Override
public void databaseExists(String databaseName) {
checkConnection();
String query = "use database identifier(?)";
try {
PreparedStatement stmt = conn.prepareStatement(query);
stmt.setString(1, databaseName);
stmt.execute();
stmt.close();
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_2001.getException(e);
}
LOGGER.info("database {} exists", databaseName);
}
@Override
public void schemaExists(String schemaName) {
checkConnection();
String query = "use schema identifier(?)";
boolean foundSchema = false;
try {
PreparedStatement stmt = conn.prepareStatement(query);
stmt.setString(1, schemaName);
stmt.execute();
stmt.close();
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_2001.getException(e);
}
LOGGER.info("schema {} exists", schemaName);
}
@Override
public SnowflakeTelemetryService getTelemetryClient() {
return this.telemetry;
}
@Override
public void close() {
try {
conn.close();
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_2005.getException(e, this.telemetry);
}
LOGGER.info("snowflake connection closed");
}
@Override
public boolean isClosed() {
try {
return conn.isClosed();
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_2006.getException(e, this.telemetry);
}
}
@Override
public String getConnectorName() {
return this.connectorName;
}
/** make sure connection is not closed */
private void checkConnection() {
try {
if (conn.isClosed()) {
throw SnowflakeErrors.ERROR_1003.getException();
}
} catch (SQLException e) {
throw SnowflakeErrors.ERROR_1003.getException(e, this.telemetry);
}
}
/**
* generate pipe definition
*
* @param tableName table name
* @param stageName stage name
* @return pipe definition string
*/
private String pipeDefinition(String tableName, String stageName) {
return "copy into "
+ tableName
+ "(RECORD_METADATA, RECORD_CONTENT) from (select $1:meta, $1:content from"
+ " @"
+ stageName
+ " t) file_format = (type = 'json')";
}
@Override
public Connection getConnection() {
return this.conn;
}
@Override
public Optional This instance is separate for all pipes and hence registration and unregistration of metrics
* is handled per pipe level.
*/
public class MetricsJmxReporter {
static final KCLogger LOGGER = new KCLogger(MetricsJmxReporter.class.getName());
// The registry which will hold pool of all metrics for this instance
private final MetricRegistry metricRegistry;
/**
* Wrapper on top of listeners and metricRegistry for codehale. This will be useful to start the
* jmx metrics when time is appropriate. (Check {@link MetricsJmxReporter#start()}
*/
private final JmxReporter jmxReporter;
public MetricsJmxReporter(MetricRegistry metricRegistry, final String connectorName) {
this.metricRegistry = metricRegistry;
this.jmxReporter = createJMXReporter(connectorName);
}
public MetricRegistry getMetricRegistry() {
return metricRegistry;
}
/**
* This function will internally register all metrics present inside metric registry and will
* register mbeans to the mbeanserver
*/
public void start() {
jmxReporter.start();
}
/**
* This method is called to fetch an object name for all registered metrics. It can be called
* during registration or unregistration. (Internal implementation of codehale)
*
* @param connectorName name of the connector. (From Config)
* @param jmxDomain JMX Domain
* @param metricName metric name used while registering the metric. (Check {@link
* MetricsUtil#channelMetricName(String, String, String)}
* @return Object Name constructed from above three args
*/
@VisibleForTesting
static ObjectName getObjectName(String connectorName, String jmxDomain, String metricName) {
try {
// each metric name is scope:scopeValue/subDomain/metricName
// e.g. "task:task-0/lifecycle/open-count" or "channel:conn_topic_0/offsets/processed-offset"
Iterator We will convert all duration to SECONDS and prefix our metrics with {@link
* MetricsUtil#JMX_METRIC_PREFIX}
*
* @param connectorName connectorName passed inside configuration
* @return JMXReporter instance.
*/
private JmxReporter createJMXReporter(final String connectorName) {
return JmxReporter.forRegistry(this.metricRegistry)
.inDomain(JMX_METRIC_PREFIX)
.convertDurationsTo(TimeUnit.SECONDS)
.createsObjectNamesWith(
(ignoreMeterType, jmxDomain, metricName) ->
getObjectName(connectorName, jmxDomain, metricName))
.build();
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/metrics/MetricsUtil.java
================================================
package com.snowflake.kafka.connector.internal.metrics;
import java.util.Collection;
/** All metrics related constants. Mainly for JMX */
public class MetricsUtil {
public static final String JMX_METRIC_PREFIX = "snowflake.kafka.connector";
// file count related constants
public static final String OFFSET_SUB_DOMAIN = "offsets";
/**
* Offset number that is most recent inside the buffer (In memory buffer)
*
* This is updated every time an offset is sent as put API of SinkTask {@link
* org.apache.kafka.connect.sink.SinkTask#put(Collection)}
*/
public static final String PROCESSED_OFFSET = "processed-offset";
public static final String OFFSET_PERSISTED_IN_SNOWFLAKE = "persisted-in-snowflake-offset";
public static final String LATEST_CONSUMER_OFFSET = "latest-consumer-offset";
/**
* Returns the metric-registry key prefix for a given channel, e.g. {@code "channel:myConn_t_0"}.
* Use this when removing all metrics for a channel via {@link
* MetricsJmxReporter#removeMetricsFromRegistry}.
*/
public static String channelMetricPrefix(final String channelName) {
return "channel:" + channelName;
}
/**
* Construct a channel-level metric name. The resulting MBean will use {@code channel=} as the
* first key property.
*
* Will be of form channel:channelName/subDomain/metricName. The {@code channel:} prefix
* is parsed by {@link MetricsJmxReporter#getObjectName} to produce the MBean key.
*
* @param channelName channel or partition identifier
* @param subDomain categorize this metric (e.g. "offsets")
* @param metricName actual Metric name for which we will use Gauge, Meter, Histogram
* @return concatenized String
*/
public static String channelMetricName(
final String channelName, final String subDomain, final String metricName) {
return channelMetricPrefix(channelName) + "/" + subDomain + "/" + metricName;
}
/**
* Returns the metric-registry key prefix for a given task, e.g. {@code "task:task-0"}. Use this
* when removing all metrics for a task via {@link MetricsJmxReporter#removeMetricsFromRegistry}.
*/
public static String taskMetricPrefix(final String taskPrefix) {
return "task:" + taskPrefix;
}
/**
* Construct a task-level metric name. The resulting MBean will use {@code task=} as the first key
* property.
*
* Will be of form task:taskPrefix/subDomain/metricName
*
* @param taskPrefix task identifier (e.g. "task-0")
* @param subDomain categorize this metric (e.g. "task", "lifecycle")
* @param metricName actual Metric name
* @return concatenized String with scope prefix
*/
public static String taskMetricName(
final String taskPrefix, final String subDomain, final String metricName) {
return taskMetricPrefix(taskPrefix) + "/" + subDomain + "/" + metricName;
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/metrics/NoopTaskMetrics.java
================================================
package com.snowflake.kafka.connector.internal.metrics;
/** Null-object implementation of {@link TaskMetrics}. Every method is a no-op. */
enum NoopTaskMetrics implements TaskMetrics {
INSTANCE;
@Override
public TimingContext timePut() {
return TimingContext.NOOP;
}
@Override
public TimingContext timePreCommit() {
return TimingContext.NOOP;
}
@Override
public TimingContext timeOpen() {
return TimingContext.NOOP;
}
@Override
public TimingContext timeClose() {
return TimingContext.NOOP;
}
@Override
public TimingContext timeSdkClientCreate() {
return TimingContext.NOOP;
}
@Override
public TimingContext timeChannelOpen() {
return TimingContext.NOOP;
}
@Override
public TimingContext timeOffsetFetch() {
return TimingContext.NOOP;
}
@Override
public void recordStartDuration(long nanos) {}
@Override
public void incOpenCount() {}
@Override
public void incCloseCount() {}
@Override
public void incChannelOpenCount() {}
@Override
public void incPreCommitPartitionsSkipped() {}
@Override
public void incBackpressureRewindCount() {}
@Override
public void markPutRecords(long count) {}
@Override
public void setAssignedPartitions(int count) {}
@Override
public void unregister() {}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/metrics/SnowflakeSinkTaskMetrics.java
================================================
package com.snowflake.kafka.connector.internal.metrics;
import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.taskMetricName;
import static com.snowflake.kafka.connector.internal.metrics.MetricsUtil.taskMetricPrefix;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.snowflake.kafka.connector.internal.KCLogger;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
/**
* Manages task-level JMX metrics for a single {@link
* com.snowflake.kafka.connector.SnowflakeSinkTask} instance. Metrics are registered when the task
* starts and unregistered when it stops.
*
* MBean ObjectNames follow the pattern:
*
* All methods are safe to call unconditionally -- the noop implementation is a no-op.
*/
public interface TaskMetrics {
// ---- timing (try-with-resources) ----
TimingContext timePut();
TimingContext timePreCommit();
TimingContext timeOpen();
TimingContext timeClose();
TimingContext timeSdkClientCreate();
TimingContext timeChannelOpen();
TimingContext timeOffsetFetch();
void recordStartDuration(long nanos);
// ---- counters ----
void incOpenCount();
void incCloseCount();
void incChannelOpenCount();
void incPreCommitPartitionsSkipped();
void incBackpressureRewindCount();
// ---- throughput ----
void markPutRecords(long count);
// ---- gauges ----
void setAssignedPartitions(int count);
// ---- lifecycle ----
void unregister();
// ---- timing context ----
@FunctionalInterface
interface TimingContext extends AutoCloseable {
TimingContext NOOP = () -> {};
@Override
void close();
}
// ---- factory ----
static TaskMetrics noop() {
return NoopTaskMetrics.INSTANCE;
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/schemaevolution/ColumnInfos.java
================================================
/*
* Copyright (c) 2026 Snowflake Computing Inc. All rights reserved.
*/
package com.snowflake.kafka.connector.internal.schemaevolution;
import java.util.Objects;
import java.util.Optional;
/** Represents column type and DDL comment for schema evolution. */
public class ColumnInfos {
private final String columnType;
private final String comments;
public ColumnInfos(String columnType, String comments) {
this.columnType = Objects.requireNonNull(columnType, "columnType cannot be null");
this.comments = comments;
}
public ColumnInfos(String columnType) {
this.columnType = Objects.requireNonNull(columnType, "columnType cannot be null");
this.comments = null;
}
public String getColumnType() {
return columnType;
}
public String getComments() {
return comments;
}
public String getDdlComments() {
return Optional.ofNullable(comments)
.map(comment -> String.format(" comment '%s' ", comment.replace("'", "''")))
.orElse(" comment 'column created by schema evolution from Snowflake Kafka Connector' ");
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ColumnInfos that = (ColumnInfos) o;
return Objects.equals(columnType, that.columnType) && Objects.equals(comments, that.comments);
}
@Override
public int hashCode() {
return Objects.hash(columnType, comments);
}
@Override
public String toString() {
return "ColumnInfos{"
+ "columnType='"
+ columnType
+ '\''
+ ", comments='"
+ comments
+ '\''
+ '}';
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/schemaevolution/ColumnTypeMapper.java
================================================
/*
* Copyright (c) 2026 Snowflake Computing Inc. All rights reserved.
*/
package com.snowflake.kafka.connector.internal.schemaevolution;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.kafka.connect.data.Schema;
/** Abstract base for mapping Kafka Connect types to Snowflake DDL types. */
public abstract class ColumnTypeMapper {
public String mapToColumnType(Schema.Type kafkaType) {
return mapToColumnType(kafkaType, null);
}
public abstract String mapToColumnType(Schema.Type kafkaType, String schemaName);
/**
* Map the JSON node type to Kafka type
*
* @param value JSON node
* @return Kafka type
*/
public abstract Schema.Type mapJsonNodeTypeToKafkaType(JsonNode value);
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/schemaevolution/SchemaEvolutionTargetItems.java
================================================
/*
* Copyright (c) 2026 Snowflake Computing Inc. All rights reserved.
*/
package com.snowflake.kafka.connector.internal.schemaevolution;
import java.util.Collections;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import javax.annotation.Nonnull;
/**
* Contains target items for schema evolution: table name, columns to drop non-nullability, and
* columns to add to the table.
*/
public class SchemaEvolutionTargetItems {
private final String tableName;
@Nonnull private final Set Note: Columns must be added BEFORE dropping NOT NULL constraints, otherwise the constraint
* modification will fail if the column doesn't exist yet.
*
* @param targetItems target items for schema evolution
* @param record the sink record that contains the schema and content
*/
public void evolveSchemaIfNeeded(
SchemaEvolutionTargetItems targetItems, SnowflakeSinkRecord record) {
if (!targetItems.hasDataForSchemaEvolution()) {
return;
}
String tableName = targetItems.getTableName();
// Add new columns FIRST (must exist before we can modify constraints)
if (!targetItems.getColumnsToAdd().isEmpty()) {
LOGGER.debug(
"Adding columns to table: {} columns: {}", tableName, targetItems.getColumnsToAdd());
TableSchema tableSchema =
tableSchemaResolver.resolveTableSchemaFromSnowflakeRecord(
record, new ArrayList<>(targetItems.getColumnsToAdd()));
try {
conn.appendColumnsToTable(tableName, tableSchema.getColumnInfos());
} catch (SnowflakeKafkaConnectorException e) {
LOGGER.warn(
"Failure altering table to add column: {}, this could happen when multiple"
+ " partitions try to alter the table at the same time and the warning could be"
+ " ignored",
tableName,
e);
}
}
// Drop NOT NULL constraints AFTER columns exist
if (!targetItems.getColumnsToDropNonNullability().isEmpty()) {
LOGGER.debug(
"Dropping nonNullability for table: {} columns: {}",
tableName,
targetItems.getColumnsToDropNonNullability());
try {
conn.alterNonNullableColumns(
tableName, new ArrayList<>(targetItems.getColumnsToDropNonNullability()));
} catch (SnowflakeKafkaConnectorException e) {
LOGGER.warn(
"Failure altering table to update nullability: {}, this could happen when multiple"
+ " partitions try to alter the table at the same time and the warning could be"
+ " ignored",
tableName,
e);
}
}
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/schemaevolution/TableSchema.java
================================================
/*
* Copyright (c) 2026 Snowflake Computing Inc. All rights reserved.
*/
package com.snowflake.kafka.connector.internal.schemaevolution;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
/** Wrapper around Map of column name to ColumnInfos. */
public class TableSchema {
private final Map return a map of invalid params
*/
private static Map Only SNOWPIPE_STREAMING is supported (SSv2). Legacy SNOWPIPE and SSv1 have been removed.
*/
public enum IngestionMethodConfig {
/* Snowpipe streaming (SSv2) - the only supported ingestion method */
SNOWPIPE_STREAMING;
@Override
public String toString() {
return name().toLowerCase(Locale.ROOT);
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/streaming/LatestCommitedOffsetTokenExecutor.java
================================================
package com.snowflake.kafka.connector.internal.streaming;
import static java.time.temporal.ChronoUnit.SECONDS;
import com.snowflake.kafka.connector.internal.KCLogger;
import dev.failsafe.Failsafe;
import dev.failsafe.FailsafeExecutor;
import dev.failsafe.Fallback;
import dev.failsafe.RetryPolicy;
import dev.failsafe.function.CheckedSupplier;
import java.time.Duration;
/**
* Class that separates Failsafe specific logic (retries and fallback) from the actual channel logic
*/
public class LatestCommitedOffsetTokenExecutor {
private static final KCLogger LOGGER =
new KCLogger(LatestCommitedOffsetTokenExecutor.class.getName());
private static final Duration DURATION_BETWEEN_GET_OFFSET_TOKEN_RETRY = Duration.ofSeconds(1);
protected static final int MAX_GET_OFFSET_TOKEN_RETRIES = 3;
public static FailsafeExecutor This class provides a clean interface to execute channel opening operations with automatic
* retry on HTTP 429 (rate limiting) errors from Snowflake streaming service.
*/
class OpenChannelRetryPolicy {
private static final KCLogger LOGGER = new KCLogger(OpenChannelRetryPolicy.class.getName());
private static final String RATE_LIMIT_MESSAGE_PART = "HTTP Status: 429";
// Retry policy constants
/** Initial delay before the first retry attempt. */
private static final Duration INITIAL_DELAY = Duration.ofSeconds(2);
/** Maximum delay between retry attempts. */
private static final Duration MAX_DELAY = Duration.ofSeconds(8);
/** Exponential backoff multiplier (retry delays: 2s, 4s, 8s max). */
private static final double BACKOFF_MULTIPLIER = 2.0;
/** Random jitter added to retry delays to prevent thundering herd. */
private static final Duration JITTER_DURATION = Duration.ofMillis(200);
/**
* Executes the provided channel opening action with retry handling.
*
* On SFException containing "429" (HTTP rate limiting), it will retry with exponential backoff
* and jitter with unlimited retry attempts. Other exceptions are not retried.
*
* @param channelOpenAction the action to execute (typically openChannelForTable call)
* @param channelName the channel name for logging purposes
* @return the result of the channel opening operation
*/
static SnowflakeStreamingIngestChannel executeWithRetry(
CheckedSupplier StartTask: Called when partitions are assigned. Responsible for generating the POJOs.
*
* Insert and getOffset are called when {@link
* com.snowflake.kafka.connector.SnowflakeSinkTask#put(Collection)} and {@link
* com.snowflake.kafka.connector.SnowflakeSinkTask#preCommit(Map)} APIs are called.
*
* This implementation of SinkService uses Streaming Snowpipe (Streaming Ingestion)
*
* Hence this initializes the channel, opens, closes. The StreamingIngestChannel resides inside
* {@link TopicPartitionChannel} which is per partition.
*/
public class SnowflakeSinkServiceV2 implements SnowflakeSinkService {
private static final KCLogger LOGGER = new KCLogger(SnowflakeSinkServiceV2.class.getName());
private final SnowflakeConnectionService conn;
private final Optional Safety checks: - If validation disabled: Warn that SSv2 Error Table is required to prevent
* task crashes - If validation enabled: Verify DLQ or tolerance=none for safe error handling
*
* @throws IllegalStateException if configuration is unsafe and would cause data loss
*/
private void logValidationConfiguration() {
String errorsTolerance =
taskConfig.isTolerateErrors()
? ConnectorConfigTools.ErrorTolerance.ALL.toString()
: ConnectorConfigTools.ErrorTolerance.NONE.toString();
String dlqTopic = taskConfig.getDlqTopicName();
boolean dlqConfigured = dlqTopic != null && !dlqTopic.trim().isEmpty();
boolean tolerateAll = "all".equalsIgnoreCase(errorsTolerance);
// Check for legacy KC v3 config and warn if present (schematization enabled via task config)
if (taskConfig.isEnableSchematization()) {
LOGGER.warn(
"Config 'snowflake.enable.schematization' is not supported in KC v4. "
+ "Schema evolution is now handled server-side via table property "
+ "'ENABLE_SCHEMA_EVOLUTION'. For pre-created tables, run: "
+ "ALTER TABLE ... SET ENABLE_SCHEMA_EVOLUTION = TRUE");
} else {
LOGGER.info(
"Schematization is disabled — the connector wraps payloads into"
+ " RECORD_CONTENT/RECORD_METADATA.");
}
if (taskConfig.getValidation() != SnowflakeValidation.CLIENT_SIDE) {
// Check each target table for ERROR_LOGGING.
// Note: makes up to 3 network calls per table (tableExist + isIcebergTable +
// hasErrorLoggingEnabled). Acceptable at startup; only runs once per task constructor.
Set Initializes the Channel and partitionsToChannel map with new instance of {@link
* TopicPartitionChannel}
*
* @param topicPartition TopicPartition passed from Kafka
*/
@Override
public void startPartition(TopicPartition topicPartition) {
startPartitions(Set.of(topicPartition));
}
/**
* Ensures tables and pipes exist in Snowflake, then delegates channel creation to the {@link
* PartitionChannelManager}.
*
* @param partitions collection of topic partition
*/
@Override
public void startPartitions(Collection All the channels are closed. The client is still active. Upon rebalance, (inside {@link
* com.snowflake.kafka.connector.SnowflakeSinkTask#open(Collection)} we will reopen the channel.
*
* We will wipe the cache partitionsToChannel so that in {@link
* com.snowflake.kafka.connector.SnowflakeSinkTask#open(Collection)} we reinstantiate and fetch
* offsetToken
*
* @param partitions a list of topic partition
*/
@Override
public void close(Collection Please note: This exception is translated from SFException when Client SDK determines this is
* an invalid insert Operation. (For instance, clientSequencer is bumped up, but we are still
* calling from older clientSequencer number)
*
* Use this exception when a particular channel (Topic Partition) fails to insert Rows into
* Snowflake Table, in this case we will reopen the channel and try to insert same rows again.
*
* (Note: This exception is not when Streaming Snowpipe API returns error in its response)
*/
public class TopicPartitionChannelInsertionException extends RuntimeException {
public TopicPartitionChannelInsertionException(String msg, Throwable t) {
super(msg, t);
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/streaming/channel/TopicPartitionChannel.java
================================================
package com.snowflake.kafka.connector.internal.streaming.channel;
import com.google.common.annotations.VisibleForTesting;
import com.snowflake.ingest.streaming.ChannelStatus;
import com.snowflake.ingest.streaming.SFException;
import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus;
import java.util.concurrent.CompletableFuture;
import org.apache.kafka.connect.sink.SinkRecord;
public interface TopicPartitionChannel {
long NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE = -1L;
/**
* Inserts the record into buffer
*
* Step 1: Initializes this channel by fetching the offsetToken from Snowflake for the first
* time this channel/partition has received offset after start/restart.
*
* Step 2: Decides whether given offset from Kafka needs to be processed and whether it
* qualifies for being added into buffer.
*
* @param kafkaSinkRecord input record from Kafka
* @param isFirstRowPerPartitionInBatch indicates whether the given record is the first record per
* partition in a batch
* @return true if the record was processed (or legitimately skipped as a duplicate), false if
* recovery was triggered and the caller should stop feeding records to this partition for the
* remainder of the batch
*/
boolean insertRecord(SinkRecord kafkaSinkRecord, boolean isFirstRowPerPartitionInBatch);
/**
* Asynchronously closes a channel associated to this partition. Any {@link SFException} occurred
* is swallowed and a successful {@link CompletableFuture} is returned instead.
*/
CompletableFuture If the committed offset token is null (no data committed yet), returns {@link
* #NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE}. Otherwise returns (committedOffset + 1) so that
* Kafka resumes from the next record after a restart.
*
* When {@code tolerateErrors} is false and new ingestion errors are detected, throws a
* connector exception to fail the task.
*
* @param status the channel status, typically from a batch status call
* @param tolerateErrors whether to tolerate ingestion errors (maps to {@code errors.tolerance})
* @return the offset safe to commit to Kafka, or {@link #NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE}
*/
long processChannelStatus(ChannelStatus status, boolean tolerateErrors);
/** Returns the pipe name associated with this channel's SDK client. */
String getPipeName();
default CompletableFuture Most of the data sent to Snowflake is aggregated data.
*/
public class SnowflakeTelemetryChannelStatus extends SnowflakeTelemetryBasicInfo {
public static final long NUM_METRICS = 4; // update when new metrics are added
static final String CHANNEL_RECOVERY_COUNT = "channel-recovery-count";
// channel properties
private final String connectorName;
private final String channelName;
private final Optional This class provides a clean interface to execute append row operations with automatic channel
* recovery on non-retryable {@link SFException}. For retryable backpressure errors, it throws
* {@link BackpressureException} to signal the batch-level insert loop to abandon the batch and
* rewind offsets.
*/
class AppendRowWithFallbackPolicy {
private static final KCLogger LOGGER = new KCLogger(AppendRowWithFallbackPolicy.class.getName());
/** Delay before fallback attempt (channel reopening). */
private static final Duration FALLBACK_DELAY = Duration.ofMillis(500);
/** Random jitter added to fallback delays to prevent retry storms. */
private static final Duration JITTER_DURATION = Duration.ofMillis(200);
/**
* Executes the given action after a delay with jitter to prevent retry storms.
*
* @param action the action to execute after the delay
* @param channelName the channel name for logging purposes
*/
private static void withDelay(CheckedRunnable action, String channelName) throws Throwable {
try {
long delayMs =
FALLBACK_DELAY.toMillis() + (long) (Math.random() * JITTER_DURATION.toMillis());
LOGGER.info("Delaying channel recovery by {}ms for channel: {}", delayMs, channelName);
Thread.sleep(delayMs);
LOGGER.info("Executing channel recovery for channel: {}", channelName);
action.run();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (SFException e) {
// Re-throw SFException unchanged so Fallback can handle it properly
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Executes the provided append row action with fallback handling.
*
* On retryable {@link SFException} (backpressure errors), throws {@link BackpressureException}
* to signal the batch-level insert loop that the batch should be abandoned and offsets should be
* rewound. The channel remains valid.
*
* On non-retryable {@link SFException}, it will execute the fallback supplier to reopen the
* channel and reset offsets after a simple blocking delay with jitter to prevent retry storms.
*
* @param appendRowAction the action to execute (typically channel.appendRow call)
* @param fallbackSupplier the fallback action to execute on non-retryable failure (channel
* reopening logic)
* @param channelName the channel name for logging purposes
*/
/**
* @return true if the append row action succeeded normally, false if the fallback was executed
* (meaning the record was NOT inserted). When this returns false, callers must NOT advance
* processedOffset — the fallback's recovery logic has already reset offset state.
*/
static boolean executeWithFallback(
CheckedRunnable appendRowAction,
FallbackSupplierWithException fallbackSupplier,
String channelName) {
boolean[] succeeded = {true};
Fallback This is used to encapsulate the channel reopening logic that needs to be executed when the
* primary append row operation fails.
*/
@FunctionalInterface
interface FallbackSupplierWithException {
/**
* Executes the fallback logic.
*
* @param exception the original exception that caused the fallback to be triggered
* @throws Exception if the fallback operation fails
*/
void execute(Throwable exception) throws Exception;
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/streaming/v2/BackpressureException.java
================================================
package com.snowflake.kafka.connector.internal.streaming.v2;
import com.google.common.base.Preconditions;
import com.snowflake.ingest.streaming.SFException;
import java.util.Set;
/**
* Unchecked exception thrown when the Snowflake SDK signals backpressure due to memory saturation
* or receiver overload.
*
* This exception wraps {@link SFException} instances with specific error codes indicating
* transient memory pressure. It signals to the batch-level insert loop that the current batch
* should be abandoned and offsets should be rewound, but the channel remains valid and does not
* need to be reopened.
*
* Retryable error codes:
*
* This exception wraps {@link SFException} instances with specific error codes indicating the
* client itself is no longer usable. Unlike {@link BackpressureException} (where the channel
* remains valid), this signals that the client and all its channels must be replaced.
*
* Client-invalid error codes:
*
* Implementations are expected to use compare-and-swap semantics: if the client has already been
* replaced by another caller, the existing replacement should be returned without creating a second
* one.
*/
@FunctionalInterface
public interface ClientRecreator {
/**
* Replaces the given invalid client with a new one.
*
* @param invalidClient the client instance that is no longer valid (identity-compared in the
* pool)
* @return the new client, or the already-replaced client if another caller got there first
*/
SnowflakeStreamingIngestClient recreate(SnowflakeStreamingIngestClient invalidClient);
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/streaming/v2/PipeNameProvider.java
================================================
package com.snowflake.kafka.connector.internal.streaming.v2;
import static com.snowflake.kafka.connector.Constants.DEFAULT_PIPE_NAME_SUFFIX;
/** Class that generates pipe name for Snowpipe Streaming v2 */
public final class PipeNameProvider {
public static String buildDefaultPipeName(String table) {
return table + DEFAULT_PIPE_NAME_SUFFIX;
}
}
================================================
FILE: src/main/java/com/snowflake/kafka/connector/internal/streaming/v2/SnowpipeStreamingPartitionChannel.java
================================================
package com.snowflake.kafka.connector.internal.streaming.v2;
import static com.snowflake.kafka.connector.Constants.KafkaConnectorConfigParams.SNOWFLAKE_SSV1_OFFSET_MIGRATION;
import static com.snowflake.kafka.connector.Constants.KafkaConnectorConfigParams.SNOWFLAKE_SSV1_OFFSET_MIGRATION_INCLUDE_CONNECTOR_NAME;
import static com.snowflake.kafka.connector.internal.SnowflakeErrors.ERROR_5027;
import static com.snowflake.kafka.connector.internal.SnowflakeErrors.ERROR_5028;
import static com.snowflake.kafka.connector.internal.SnowflakeErrors.ERROR_5030;
import com.google.common.annotations.VisibleForTesting;
import com.snowflake.ingest.streaming.ChannelStatus;
import com.snowflake.ingest.streaming.OpenChannelResult;
import com.snowflake.ingest.streaming.SFException;
import com.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
import com.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
import com.snowflake.kafka.connector.config.SinkTaskConfig;
import com.snowflake.kafka.connector.config.SnowflakeValidation;
import com.snowflake.kafka.connector.internal.DescribeTableRow;
import com.snowflake.kafka.connector.internal.KCLogger;
import com.snowflake.kafka.connector.internal.SnowflakeConnectionService;
import com.snowflake.kafka.connector.internal.SnowflakeKafkaConnectorException;
import com.snowflake.kafka.connector.internal.metrics.TaskMetrics;
import com.snowflake.kafka.connector.internal.schemaevolution.SchemaEvolutionTargetItems;
import com.snowflake.kafka.connector.internal.schemaevolution.SnowflakeSchemaEvolutionService;
import com.snowflake.kafka.connector.internal.schemaevolution.ValidationResultMapper;
import com.snowflake.kafka.connector.internal.streaming.StreamingErrorHandler;
import com.snowflake.kafka.connector.internal.streaming.TopicPartitionChannelInsertionException;
import com.snowflake.kafka.connector.internal.streaming.channel.TopicPartitionChannel;
import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelCreation;
import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetryChannelStatus;
import com.snowflake.kafka.connector.internal.streaming.telemetry.SnowflakeTelemetrySsv1Migration;
import com.snowflake.kafka.connector.internal.streaming.v2.channel.PartitionOffsetTracker;
import com.snowflake.kafka.connector.internal.streaming.v2.migration.Ssv1MigrationMode;
import com.snowflake.kafka.connector.internal.streaming.v2.migration.Ssv1MigrationResponse;
import com.snowflake.kafka.connector.internal.telemetry.SnowflakeTelemetryService;
import com.snowflake.kafka.connector.internal.validation.ColumnSchema;
import com.snowflake.kafka.connector.internal.validation.RowValidator;
import com.snowflake.kafka.connector.internal.validation.ValidationResult;
import com.snowflake.kafka.connector.records.SnowflakeSinkRecord;
import java.time.Duration;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.errors.DataException;
import org.apache.kafka.connect.sink.SinkRecord;
public class SnowpipeStreamingPartitionChannel implements TopicPartitionChannel {
private static final KCLogger LOGGER =
new KCLogger(SnowpipeStreamingPartitionChannel.class.getName());
private volatile CompletableFuture We have deliberately not performed retries on insertRows because it might slow down overall
* ingestion and introduce lags in committing offsets to Kafka.
*
* Note that insertRows API does perform channel validation which might throw SFException if
* channel is invalidated.
*/
/**
* @return true if the record was inserted successfully, false if the fallback fired (record was
* NOT inserted)
*/
private boolean insertRowWithFallback(Map Or, in other words, if streaming channel is invalidated, we will reopen the channel and
* reset the kafka offset to last committed offset in Snowflake.
*
* If a valid offset is found from snowflake, we will reset the topicPartition with
* (offsetReturnedFromSnowflake + 1).
*
* @param reason Reason for the channel recovery. Used for logging.
* @return offset which was last present in Snowflake
*/
private void reopenChannel(final String reason) {
LOGGER.warn("{} Channel {} recovery initiated", reason, this.channelName);
if (this.snowflakeTelemetryChannelStatus != null) {
this.snowflakeTelemetryChannelStatus.incRecoveryCount();
}
this.channel =
this.channel
// Close old channel before reopening a new one. We don't want to wait for the channel
// to flush since it will be reopened right away and the in-progress data will be lost.
.thenAccept(
oldChannel -> {
if (!oldChannel.isClosed()) {
LOGGER.info(
"{} Channel {} is not closed before reopening", reason, this.channelName);
closeChannelWithoutFlushing(oldChannel);
}
})
// If the previous init failed, there is no old channel to close.
.exceptionally(
initFailure -> {
LOGGER.warn(
"{} Channel {} had a failed initialization, skipping close: {}",
reason,
this.channelName,
initFailure.getMessage());
return null;
})
.thenApply(
ignored -> {
OpenChannelResult openChannelResult = openChannelForTable(channelName);
final long offsetRecoveredFromSnowflake =
parseOrMigrateOffsetToken(openChannelResult);
if (offsetRecoveredFromSnowflake == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE) {
LOGGER.info(
"{} Channel {} has no offset token. Will use consumer group offset,"
+ " currently {}",
reason,
this.channelName,
offsetTracker.consumerGroupOffsetRef().get());
}
offsetTracker.resetAfterRecovery(offsetRecoveredFromSnowflake);
LOGGER.info(
"{} Channel {} recovery complete, offsetRecoveredFromSnowflake={}",
reason,
this.channelName,
offsetRecoveredFromSnowflake);
return openChannelResult.getChannel();
});
}
/**
* Parses the SSv2 offset from the open-channel result, and if SSv2 has no committed offset yet,
* attempts SSv1 offset migration based on the configured {@link Ssv1MigrationMode}.
*
* Used by both the initial channel open (constructor) and {@link #reopenChannel} so that
* migration behavior is consistent regardless of whether the first open succeeded or failed.
*/
private long parseOrMigrateOffsetToken(OpenChannelResult openChannelResult) {
final long ssv2Offset =
parseOffsetToken(
openChannelResult.getChannelStatus().getLatestCommittedOffsetToken(), channelName);
LOGGER.info("Channel {} has SSv2 offset token {}", channelName, ssv2Offset);
long effectiveOffset = ssv2Offset;
// Only consult SSv1 when SSv2 has no committed offset yet (first-time migration).
// Once SSv2 has its own offset, it is authoritative.
if (ssv2Offset == NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE
&& taskConfig.getSsv1MigrationMode() != Ssv1MigrationMode.SKIP) {
// migrateSsv1ChannelOffset calls SYSTEM$MIGRATE_SSV1_CHANNEL_OFFSET which:
// - returns ssv1ChannelFound=false if the SSv1 channel doesn't exist
// - returns ssv1ChannelFound=true, migratedOffset=null if found but no committed offset
// - returns ssv1ChannelFound=true, migratedOffset=N on success (also writes to SSv2 in FDB)
// - THROWS for SQL/network errors (must not silently proceed --
// falling through to consumer group offset could cause duplicates)
String ssv1Channel =
ssv1ChannelName.orElseThrow(
() ->
new IllegalStateException(
"ssv1ChannelName must be present when migration mode is "
+ taskConfig.getSsv1MigrationMode()));
Ssv1MigrationResponse response =
conn.migrateSsv1ChannelOffset(tableName, ssv1Channel, channelName, pipeName);
Long migrated = response.getMigratedOffset();
if (migrated != null) {
effectiveOffset = migrated;
LOGGER.info(
"SSv2 channel {} has no offset yet, migrating SSv1 offset for {}: {}",
channelName,
ssv1Channel,
effectiveOffset);
} else if (!response.isSsv1ChannelFound()) {
LOGGER.info("SSv1 channel {} not found for SSv2 channel {}", ssv1Channel, channelName);
} else {
LOGGER.info(
"SSv1 channel {} exists but has no committed offset for SSv2 channel {}",
ssv1Channel,
channelName);
}
telemetryService.reportSsv1Migration(
new SnowflakeTelemetrySsv1Migration(
tableName, channelName, ssv1Channel, taskConfig.getSsv1MigrationMode(), response));
if (!response.isSsv1ChannelFound()
&& taskConfig.getSsv1MigrationMode() == Ssv1MigrationMode.STRICT) {
throw new ConnectException(
"Snowpipe Streaming Classic channel "
+ ssv1Channel
+ " not found but the offset token migration mode is set to 'strict'. This can"
+ " happen if new topics are added after migrating from version 3 of the"
+ " connector or if an incorrect value is provided for "
+ SNOWFLAKE_SSV1_OFFSET_MIGRATION_INCLUDE_CONNECTOR_NAME
+ " or the connector name. Validate your settings or set "
+ SNOWFLAKE_SSV1_OFFSET_MIGRATION
+ " to 'best_effort' or 'skip' to fall through to the Kafka consumer group"
+ " offset.");
}
}
return effectiveOffset;
}
/**
* Parses an offset token string into a long value.
*
* @param offsetToken the offset token string (may be null)
* @param channelNameForLogging used in error messages
* @return the parsed long, or {@link
* TopicPartitionChannel#NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE} if null
* @throws ConnectException if the token is non-null but not parsable as long
*/
@VisibleForTesting
static long parseOffsetToken(String offsetToken, String channelNameForLogging) {
if (offsetToken == null) {
return NO_OFFSET_TOKEN_REGISTERED_IN_SNOWFLAKE;
}
try {
return Long.parseLong(offsetToken);
} catch (NumberFormatException ex) {
LOGGER.error(
"The offsetToken string does not contain a parsable long:{} for channel:{}",
offsetToken,
channelNameForLogging);
throw new ConnectException(ex);
}
}
/**
* Returns the offset Token persisted into snowflake.
*
* OffsetToken from Snowflake returns a String and we will convert it into long.
*
* If it is not long parsable, we will throw {@link ConnectException}
*
* @return -1 if no offset is found in snowflake, else the long value of committedOffset in
* snowflake.
*/
private long fetchLatestCommittedOffsetFromSnowflake() {
return fetchLatestOffsetFromChannel(this.getChannel());
}
private static long fetchLatestOffsetFromChannel(SnowflakeStreamingIngestChannel channel) {
String offsetToken = channel.getLatestCommittedOffsetToken();
LOGGER.info(
"Fetched offsetToken for channelName:{}, offset:{}", channel.getChannelName(), offsetToken);
return parseOffsetToken(offsetToken, channel.getChannelName());
}
private void initializeValidation() {
try {
Optional Open channels happens at:
*
* Constructor of TopicPartitionChannel -> which means we will wipe of all states and it will
* call precomputeOffsetTokenForChannel
*
* Failure handling which will call reopen, replace instance variable with new channel and call
* offsetToken/insertRows.
*
* @return new channel which was fetched after open/reopen
*/
private OpenChannelResult openChannelForTable(final String channelName) {
if (cancelled.get()) {
throw new CancellationException("Channel " + channelName + " was cancelled before opening");
}
final OpenChannelResult result;
try (TaskMetrics.TimingContext ignored = taskMetrics.timeChannelOpen()) {
result = streamingClient.openChannel(channelName, null);
}
taskMetrics.incChannelOpenCount();
final ChannelStatus channelStatus = result.getChannelStatus();
if (channelStatus.getStatusCode().equals("SUCCESS")) {
// Capture the initial error count - errors are cumulative and don't reset on channel reopen.
// We only want to fail on NEW errors that occur after the channel was opened.
this.initialErrorCount = channelStatus.getRowsErrorCount();
LOGGER.info(
"Successfully opened streaming channel: {}, initialErrorCount: {}",
channelName,
this.initialErrorCount);
return result;
} else {
LOGGER.error(
"Failed to open channel: {}, error code: {}", channelName, channelStatus.getStatusCode());
throw ERROR_5028.getException(
String.format(
"Failed to open channel %s. Error code %s",
channelName, channelStatus.getStatusCode()));
}
}
@Override
public CompletableFuture
*
*/
public static boolean checkConnectorVersion() {
return checkConnectorVersion(VERSION, fetchAvailableVersionsFromMaven());
}
/**
* Check connector version with provided current version and available versions.
*
* @param currentVersionString current version string
* @param availableVersions list of available version strings from Maven
*/
static boolean checkConnectorVersion(
String currentVersionString, Listsnowflake.kafka.connector:connector=X,task=task-N,category=task|lifecycle,name=metric
*/
public class SnowflakeSinkTaskMetrics implements TaskMetrics {
private static final KCLogger LOGGER = new KCLogger(SnowflakeSinkTaskMetrics.class.getName());
static final String TASK_SUB_DOMAIN = "task";
static final String LIFECYCLE_SUB_DOMAIN = "lifecycle";
// Method duration timers
static final String PUT_DURATION = "put-duration";
static final String PRECOMMIT_DURATION = "precommit-duration";
static final String OPEN_DURATION = "open-duration";
static final String CLOSE_DURATION = "close-duration";
static final String START_DURATION = "start-duration";
// Channel and SDK timers
static final String CHANNEL_OPEN_DURATION = "channel-open-duration";
static final String SDK_CLIENT_CREATE_DURATION = "sdk-client-create-duration";
static final String PRECOMMIT_OFFSET_FETCH_DURATION = "precommit-offset-fetch-duration";
// Throughput
static final String PUT_RECORDS = "put-records";
// Counters
static final String PRECOMMIT_PARTITIONS_SKIPPED = "precommit-partitions-skipped";
static final String OPEN_COUNT = "open-count";
static final String CLOSE_COUNT = "close-count";
static final String CHANNEL_OPEN_COUNT = "channel-open-count";
static final String BACKPRESSURE_REWIND_COUNT = "backpressure-rewind-count";
// Gauges
static final String ASSIGNED_PARTITIONS = "assigned-partitions";
static final String SDK_CLIENT_COUNT = "sdk-client-count";
private final String taskMetricPrefix;
private final MetricsJmxReporter metricsJmxReporter;
// Method duration timers
private final Timer putDuration;
private final Timer preCommitDuration;
private final Timer openDuration;
private final Timer closeDuration;
private final Timer startDuration;
// Channel/SDK timers (aggregated across all channels in this task)
private final Timer channelOpenDuration;
private final Timer sdkClientCreateDuration;
private final Timer preCommitOffsetFetchDuration;
// Throughput
private final Meter putRecords;
// Counters
private final Counter preCommitPartitionsSkipped;
private final Counter openCount;
private final Counter closeCount;
private final Counter channelOpenCount;
private final Counter backpressureRewindCount;
// Gauges (backed by atomics)
private final AtomicInteger assignedPartitions;
public SnowflakeSinkTaskMetrics(
String connectorName, String taskId, MetricsJmxReporter metricsJmxReporter) {
this(connectorName, taskId, metricsJmxReporter, null);
}
public SnowflakeSinkTaskMetrics(
String connectorName,
String taskId,
MetricsJmxReporter metricsJmxReporter,
Supplier
*
*/
public class BackpressureException extends RuntimeException {
private static final Set
*
*/
public class ClientRecreationException extends RuntimeException {
private static final Set