diff --git a/docs/BigQueryExecute-action.md b/docs/BigQueryExecute-action.md index 961b59504b..b7129c9da2 100644 --- a/docs/BigQueryExecute-action.md +++ b/docs/BigQueryExecute-action.md @@ -30,6 +30,14 @@ write BigQuery data to this project. **SQL**: SQL command to execute. +**BQ Job Labels:** Key value pairs to be added as labels to the BigQuery job. Keys must be unique. (Macro Enabled) + +[job_source, type] are system defined labels used by CDAP for internal purpose and cannot be used as label keys. +Macro format is supported. example `key1:val1,key2:val2` + +Keys and values can contain only lowercase letters, numeric characters, underscores, and dashes. +For more information about labels, see [Docs](https://cloud.google.com/bigquery/docs/labels-intro#requirements). + **Dialect**: Dialect of the SQL command. The value must be 'legacy' or 'standard'. If set to 'standard', the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/. If set to 'legacy', BigQuery's legacy SQL dialect will be used for this query. diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java b/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java index 78455fdb4d..87d778a89e 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java +++ b/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java @@ -125,7 +125,7 @@ public void run(ActionContext context) throws Exception { } // Add labels for the BigQuery Execute job. - builder.setLabels(BigQueryUtil.getJobLabels(BigQueryUtil.BQ_JOB_TYPE_EXECUTE_TAG)); + builder.setLabels(BigQueryUtil.getJobLabels(BigQueryUtil.BQ_JOB_TYPE_EXECUTE_TAG, config.getJobLabelKeyValue())); QueryJobConfiguration queryConfig = builder.build(); @@ -205,6 +205,7 @@ public static final class Config extends AbstractBigQueryActionConfig { private static final String DATASET = "dataset"; private static final String TABLE = "table"; private static final String NAME_LOCATION = "location"; + public static final String NAME_BQ_JOB_LABELS = "jobLabels"; private static final int ERROR_CODE_NOT_FOUND = 404; private static final String STORE_RESULTS = "storeResults"; @@ -272,10 +273,17 @@ public static final class Config extends AbstractBigQueryActionConfig { @Description("Whether to store results in a BigQuery Table.") private Boolean storeResults; + @Name(NAME_BQ_JOB_LABELS) + @Macro + @Nullable + @Description("Key value pairs to be added as labels to the BigQuery job. Keys must be unique. [job_source, type] " + + "are reserved keys and cannot be used as label keys.") + protected String jobLabelKeyValue; + private Config(@Nullable String project, @Nullable String serviceAccountType, @Nullable String serviceFilePath, @Nullable String serviceAccountJson, @Nullable String dataset, @Nullable String table, @Nullable String location, @Nullable String cmekKey, @Nullable String dialect, @Nullable String sql, - @Nullable String mode, @Nullable Boolean storeResults) { + @Nullable String mode, @Nullable Boolean storeResults, @Nullable String jobLabelKeyValue) { this.project = project; this.serviceAccountType = serviceAccountType; this.serviceFilePath = serviceFilePath; @@ -288,6 +296,7 @@ private Config(@Nullable String project, @Nullable String serviceAccountType, @N this.sql = sql; this.mode = mode; this.storeResults = storeResults; + this.jobLabelKeyValue = jobLabelKeyValue; } public boolean isLegacySQL() { @@ -328,6 +337,11 @@ public String getTable() { return table; } + @Nullable + public String getJobLabelKeyValue() { + return jobLabelKeyValue; + } + @Override public void validate(FailureCollector failureCollector) { validate(failureCollector, Collections.emptyMap()); @@ -376,9 +390,17 @@ public void validate(FailureCollector failureCollector, Map argu validateCmekKey(failureCollector, arguments); } + if (!containsMacro(NAME_BQ_JOB_LABELS)) { + validateJobLabelKeyValue(failureCollector); + } + failureCollector.getOrThrowException(); } + void validateJobLabelKeyValue(FailureCollector failureCollector) { + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, failureCollector, NAME_BQ_JOB_LABELS); + } + void validateCmekKey(FailureCollector failureCollector, Map arguments) { CryptoKeyName cmekKeyName = CmekUtils.getCmekKey(cmekKey, arguments, failureCollector); //these fields are needed to check if bucket exists or not and for location validation @@ -449,6 +471,7 @@ public static class Builder { private String sql; private String mode; private Boolean storeResults; + private String jobLabelKeyValue; public Builder setProject(@Nullable String project) { this.project = project; @@ -505,6 +528,11 @@ public Builder setSql(@Nullable String sql) { return this; } + public Builder setJobLabelKeyValue(@Nullable String jobLabelKeyValue) { + this.jobLabelKeyValue = jobLabelKeyValue; + return this; + } + public Config build() { return new Config( project, @@ -518,7 +546,8 @@ public Config build() { dialect, sql, mode, - storeResults + storeResults, + jobLabelKeyValue ); } diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/sink/AbstractBigQuerySinkConfig.java b/src/main/java/io/cdap/plugin/gcp/bigquery/sink/AbstractBigQuerySinkConfig.java index bc894cfeac..b888ea4dee 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigquery/sink/AbstractBigQuerySinkConfig.java +++ b/src/main/java/io/cdap/plugin/gcp/bigquery/sink/AbstractBigQuerySinkConfig.java @@ -187,106 +187,8 @@ void validateCmekKey(FailureCollector failureCollector, Map argu validateCmekKeyLocation(cmekKeyName, null, location, failureCollector); } - /** - * Validates job label key value pairs, as per the following rules: - * Keys and values can contain only lowercase letters, numeric characters, underscores, and dashes. - * Defined in the following link: - * Docs - * @param failureCollector failure collector - */ void validateJobLabelKeyValue(FailureCollector failureCollector) { - Set reservedKeys = BigQueryUtil.BQ_JOB_LABEL_SYSTEM_KEYS; - int maxLabels = 64 - reservedKeys.size(); - int maxKeyLength = 63; - int maxValueLength = 63; - - String validLabelKeyRegex = "^[\\p{L}][a-z0-9-_\\p{L}]+$"; - String validLabelValueRegex = "^[a-z0-9-_\\p{L}]+$"; - String capitalLetterRegex = ".*[A-Z].*"; - String labelKeyValue = getJobLabelKeyValue(); - - if (Strings.isNullOrEmpty(labelKeyValue)) { - return; - } - - String[] keyValuePairs = labelKeyValue.split(","); - Set uniqueKeys = new HashSet<>(); - - for (String keyValuePair : keyValuePairs) { - - // Adding a label without a value is valid behavior - // Read more here: https://cloud.google.com/bigquery/docs/adding-labels#adding_a_label_without_a_value - String[] keyValue = keyValuePair.trim().split(":"); - boolean isKeyPresent = keyValue.length == 1 || keyValue.length == 2; - boolean isValuePresent = keyValue.length == 2; - - - if (!isKeyPresent) { - failureCollector.addFailure(String.format("Invalid job label key value pair '%s'.", keyValuePair), - "Job label key value pair should be in the format 'key:value'.") - .withConfigProperty(NAME_BQ_JOB_LABELS); - continue; - } - - // Check if key is reserved - if (reservedKeys.contains(keyValue[0])) { - failureCollector.addFailure(String.format("Invalid job label key '%s'.", keyValue[0]), - "A system label already exists with same name.").withConfigProperty(NAME_BQ_JOB_LABELS); - continue; - } - - String key = keyValue[0]; - String value = isValuePresent ? keyValue[1] : ""; - boolean isKeyValid = true; - boolean isValueValid = true; - - // Key cannot be empty - if (Strings.isNullOrEmpty(key)) { - failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), - "Job label key cannot be empty.").withConfigProperty(NAME_BQ_JOB_LABELS); - isKeyValid = false; - } - - // Key cannot be longer than 63 characters - if (key.length() > maxKeyLength) { - failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), - "Job label key cannot be longer than 63 characters.").withConfigProperty(NAME_BQ_JOB_LABELS); - isKeyValid = false; - } - - // Value cannot be longer than 63 characters - if (value.length() > maxValueLength) { - failureCollector.addFailure(String.format("Invalid job label value '%s'.", value), - "Job label value cannot be longer than 63 characters.").withConfigProperty(NAME_BQ_JOB_LABELS); - isValueValid = false; - } - - if (isKeyValid && (!key.matches(validLabelKeyRegex) || key.matches(capitalLetterRegex))) { - failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), - "Job label key can only contain lowercase letters, numeric characters, " + - "underscores, and dashes. Check docs for more details.") - .withConfigProperty(NAME_BQ_JOB_LABELS); - isKeyValid = false; - } - - if (isValuePresent && isValueValid && - (!value.matches(validLabelValueRegex) || value.matches(capitalLetterRegex))) { - failureCollector.addFailure(String.format("Invalid job label value '%s'.", value), - "Job label value can only contain lowercase letters, numeric characters, " + - "underscores, and dashes.").withConfigProperty(NAME_BQ_JOB_LABELS); - } - - if (isKeyValid && !uniqueKeys.add(key)) { - failureCollector.addFailure(String.format("Duplicate job label key '%s'.", key), - "Job label key should be unique.").withConfigProperty(NAME_BQ_JOB_LABELS); - } - } - // Check if number of labels is greater than 64 - reserved keys - if (uniqueKeys.size() > maxLabels) { - failureCollector.addFailure("Number of job labels exceeds the limit.", - String.format("Number of job labels cannot be greater than %d.", maxLabels)) - .withConfigProperty(NAME_BQ_JOB_LABELS); - } + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, failureCollector, NAME_BQ_JOB_LABELS); } public String getDatasetProject() { diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtil.java b/src/main/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtil.java index cbde42cf04..fbcdb6398a 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtil.java +++ b/src/main/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtil.java @@ -39,6 +39,7 @@ import io.cdap.cdap.etl.api.validation.InvalidConfigPropertyException; import io.cdap.cdap.etl.api.validation.InvalidStageException; import io.cdap.cdap.etl.api.validation.ValidationFailure; +import io.cdap.plugin.gcp.bigquery.sink.AbstractBigQuerySinkConfig; import io.cdap.plugin.gcp.bigquery.sink.BigQuerySink; import io.cdap.plugin.gcp.bigquery.source.BigQuerySource; import io.cdap.plugin.gcp.bigquery.source.BigQuerySourceConfig; @@ -60,6 +61,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -923,4 +925,106 @@ public static String getStagingBucketName(Map arguments, @Nullab } return bucket; } + + /** + * Validates job label key value pairs, as per the following rules: + * Keys and values can contain only lowercase letters, numeric characters, underscores, and dashes. + * Defined in the following link: + * Docs + * @param failureCollector failure collector + */ + public static void validateJobLabelKeyValue(String labelKeyValue, FailureCollector failureCollector, + String stageConfigProperty) { + Set reservedKeys = BQ_JOB_LABEL_SYSTEM_KEYS; + int maxLabels = 64 - reservedKeys.size(); + int maxKeyLength = 63; + int maxValueLength = 63; + + String validLabelKeyRegex = "^[\\p{L}][a-z0-9-_\\p{L}]+$"; + String validLabelValueRegex = "^[a-z0-9-_\\p{L}]+$"; + String capitalLetterRegex = ".*[A-Z].*"; + + if (com.google.api.client.util.Strings.isNullOrEmpty(labelKeyValue)) { + return; + } + + String[] keyValuePairs = labelKeyValue.split(","); + Set uniqueKeys = new HashSet<>(); + + for (String keyValuePair : keyValuePairs) { + + // Adding a label without a value is valid behavior + // Read more here: https://cloud.google.com/bigquery/docs/adding-labels#adding_a_label_without_a_value + String[] keyValue = keyValuePair.trim().split(":"); + boolean isKeyPresent = keyValue.length == 1 || keyValue.length == 2; + boolean isValuePresent = keyValue.length == 2; + + + if (!isKeyPresent) { + failureCollector.addFailure(String.format("Invalid job label key value pair '%s'.", keyValuePair), + "Job label key value pair should be in the format 'key:value'.") + .withConfigProperty(stageConfigProperty); + continue; + } + + // Check if key is reserved + if (reservedKeys.contains(keyValue[0])) { + failureCollector.addFailure(String.format("Invalid job label key '%s'.", keyValue[0]), + "A system label already exists with same name.").withConfigProperty(stageConfigProperty); + continue; + } + + String key = keyValue[0]; + String value = isValuePresent ? keyValue[1] : ""; + boolean isKeyValid = true; + boolean isValueValid = true; + + // Key cannot be empty + if (com.google.api.client.util.Strings.isNullOrEmpty(key)) { + failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), + "Job label key cannot be empty.").withConfigProperty(stageConfigProperty); + isKeyValid = false; + } + + // Key cannot be longer than 63 characters + if (key.length() > maxKeyLength) { + failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), + "Job label key cannot be longer than 63 characters.").withConfigProperty(stageConfigProperty); + isKeyValid = false; + } + + // Value cannot be longer than 63 characters + if (value.length() > maxValueLength) { + failureCollector.addFailure(String.format("Invalid job label value '%s'.", value), + "Job label value cannot be longer than 63 characters.").withConfigProperty(stageConfigProperty); + isValueValid = false; + } + + if (isKeyValid && (!key.matches(validLabelKeyRegex) || key.matches(capitalLetterRegex))) { + failureCollector.addFailure(String.format("Invalid job label key '%s'.", key), + "Job label key can only contain lowercase letters, numeric characters, " + + "underscores, and dashes. Check docs for more details.") + .withConfigProperty(stageConfigProperty); + isKeyValid = false; + } + + if (isValuePresent && isValueValid && + (!value.matches(validLabelValueRegex) || value.matches(capitalLetterRegex))) { + failureCollector.addFailure(String.format("Invalid job label value '%s'.", value), + "Job label value can only contain lowercase letters, numeric characters, " + + "underscores, and dashes.").withConfigProperty(stageConfigProperty); + } + + if (isKeyValid && !uniqueKeys.add(key)) { + failureCollector.addFailure(String.format("Duplicate job label key '%s'.", key), + "Job label key should be unique.").withConfigProperty(stageConfigProperty); + } + } + // Check if number of labels is greater than 64 - reserved keys + if (uniqueKeys.size() > maxLabels) { + failureCollector.addFailure("Number of job labels exceeds the limit.", + String.format("Number of job labels cannot be greater than %d.", maxLabels)) + .withConfigProperty(stageConfigProperty); + } + } } diff --git a/src/test/java/io/cdap/plugin/gcp/bigquery/sink/BigQuerySinkConfigTest.java b/src/test/java/io/cdap/plugin/gcp/bigquery/sink/BigQuerySinkConfigTest.java index dcfb182779..ebaa553df3 100644 --- a/src/test/java/io/cdap/plugin/gcp/bigquery/sink/BigQuerySinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/gcp/bigquery/sink/BigQuerySinkConfigTest.java @@ -129,192 +129,6 @@ public void testValidateTimePartitioningColumnWithNullAndDate() throws Assert.assertEquals(0, collector.getValidationFailures().size()); } - @Test - public void testJobLabelWithDuplicateKeys() { - config.jobLabelKeyValue = "key1:value1,key2:value2,key1:value3"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Duplicate job label key 'key1'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithDuplicateValues() { - config.jobLabelKeyValue = "key1:value1,key2:value2,key3:value1"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithCapitalLetters() { - config.jobLabelKeyValue = "keY1:value1,key2:value2,key3:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key 'keY1'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelStartingWithCapitalLetters() { - config.jobLabelKeyValue = "Key1:value1,key2:value2,key3:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key 'Key1'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithInvalidCharacters() { - config.jobLabelKeyValue = "key1:value1,key2:value2,key3:value1@"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label value 'value1@'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithEmptyKey() { - config.jobLabelKeyValue = ":value1,key2:value2,key3:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key ''.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithEmptyValue() { - config.jobLabelKeyValue = "key1:,key2:value2,key3:value1"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithWrongFormat() { - config.jobLabelKeyValue = "key1=value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key 'key1=value1'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithNull() { - config.jobLabelKeyValue = null; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithReservedKeys() { - config.jobLabelKeyValue = "job_source:value1,type:value2"; - config.validate(collector); - Assert.assertEquals(2, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key 'job_source'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWith65Keys() { - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= 65; i++) { - String key = "key" + i; - String value = "value" + i; - sb.append(key).append(":").append(value).append(","); - } - // remove the last comma - sb.deleteCharAt(sb.length() - 1); - Assert.assertEquals(65, sb.toString().split(",").length); - config.jobLabelKeyValue = sb.toString(); - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Number of job labels exceeds the limit.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithKeyLength64() { - String key64 = "1234567890123456789012345678901234567890123456789012345678901234"; - config.jobLabelKeyValue = key64 + ":value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key '" + key64 + "'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithValueLength64() { - String value64 = "1234567890123456789012345678901234567890123456789012345678901234"; - config.jobLabelKeyValue = "key1:" + value64; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label value '" + value64 + "'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithKeyStartingWithNumber() { - config.jobLabelKeyValue = "1key:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key '1key'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithKeyStartingWithDash() { - config.jobLabelKeyValue = "-key:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key '-key'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithKeyStartingWithHyphen() { - config.jobLabelKeyValue = "_key:value1"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label key '_key'.", - collector.getValidationFailures().get(0).getMessage()); - } - - @Test - public void testJobLabelWithKeyWithChineseCharacter() { - config.jobLabelKeyValue = "中文:value1"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithKeyWithJapaneseCharacter() { - config.jobLabelKeyValue = "日本語:value1"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithValueStartingWithNumber() { - config.jobLabelKeyValue = "key:1value"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithValueStartingWithDash() { - config.jobLabelKeyValue = "key:-value"; - config.validate(collector); - Assert.assertEquals(0, collector.getValidationFailures().size()); - } - - @Test - public void testJobLabelWithValueStartingWithCaptialLetter() { - config.jobLabelKeyValue = "key:Value"; - config.validate(collector); - Assert.assertEquals(1, collector.getValidationFailures().size()); - Assert.assertEquals("Invalid job label value 'Value'.", - collector.getValidationFailures().get(0).getMessage()); - } - @Test public void testValidateColumnNameWithValidColumnName() { String columnName = "test"; diff --git a/src/test/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtilTest.java b/src/test/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtilTest.java index 4bc7aeb299..41d9c73a46 100644 --- a/src/test/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtilTest.java +++ b/src/test/java/io/cdap/plugin/gcp/bigquery/util/BigQueryUtilTest.java @@ -21,10 +21,12 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.validation.ValidationFailure; +import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.gcp.bigquery.util.BigQueryTypeSize.BigNumeric; import io.cdap.plugin.gcp.bigquery.util.BigQueryTypeSize.Numeric; import io.cdap.plugin.gcp.common.GCPUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -44,6 +46,12 @@ public class BigQueryUtilTest { private static final String BUCKET_PREFIX_ARG = "gcp.bigquery.bucket.prefix"; + MockFailureCollector collector; + + @Before + public void setUp() { + collector = new MockFailureCollector(); + } @Test public void testGetTableSchema() { @@ -277,4 +285,189 @@ public void testCRC32LocationDoesNotCollide() { } } + @Test + public void testJobLabelWithDuplicateKeys() { + String jobLabelKeyValue = "key1:value1,key2:value2,key1:value3"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Duplicate job label key 'key1'.", + collector.getValidationFailures().get(0).getMessage()); + } + @Test + public void testJobLabelWithDuplicateValues() { + String jobLabelKeyValue = "key1:value1,key2:value2,key3:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithCapitalLetters() { + String jobLabelKeyValue = "keY1:value1,key2:value2,key3:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key 'keY1'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelStartingWithCapitalLetters() { + String jobLabelKeyValue = "Key1:value1,key2:value2,key3:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key 'Key1'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithInvalidCharacters() { + String jobLabelKeyValue = "key1:value1,key2:value2,key3:value1@"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label value 'value1@'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithEmptyKey() { + String jobLabelKeyValue = ":value1,key2:value2,key3:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key ''.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithEmptyValue() { + String jobLabelKeyValue = "key1:,key2:value2,key3:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithWrongFormat() { + String jobLabelKeyValue = "key1=value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key 'key1=value1'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithNull() { + String jobLabelKeyValue = null; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithReservedKeys() { + String jobLabelKeyValue = "job_source:value1,type:value2"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(2, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key 'job_source'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWith65Keys() { + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= 65; i++) { + String key = "key" + i; + String value = "value" + i; + sb.append(key).append(":").append(value).append(","); + } + // remove the last comma + sb.deleteCharAt(sb.length() - 1); + Assert.assertEquals(65, sb.toString().split(",").length); + String jobLabelKeyValue = sb.toString(); + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Number of job labels exceeds the limit.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithKeyLength64() { + String key64 = "1234567890123456789012345678901234567890123456789012345678901234"; + String jobLabelKeyValue = key64 + ":value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key '" + key64 + "'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithValueLength64() { + String value64 = "1234567890123456789012345678901234567890123456789012345678901234"; + String jobLabelKeyValue = "key1:" + value64; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label value '" + value64 + "'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithKeyStartingWithNumber() { + String jobLabelKeyValue = "1key:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key '1key'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithKeyStartingWithDash() { + String jobLabelKeyValue = "-key:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key '-key'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithKeyStartingWithHyphen() { + String jobLabelKeyValue = "_key:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label key '_key'.", + collector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testJobLabelWithKeyWithChineseCharacter() { + String jobLabelKeyValue = "中文:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithKeyWithJapaneseCharacter() { + String jobLabelKeyValue = "日本語:value1"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithValueStartingWithNumber() { + String jobLabelKeyValue = "key:1value"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithValueStartingWithDash() { + String jobLabelKeyValue = "key:-value"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testJobLabelWithValueStartingWithCaptialLetter() { + String jobLabelKeyValue = "key:Value"; + BigQueryUtil.validateJobLabelKeyValue(jobLabelKeyValue, collector, "test"); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Invalid job label value 'Value'.", + collector.getValidationFailures().get(0).getMessage()); + } + } diff --git a/widgets/BigQueryExecute-action.json b/widgets/BigQueryExecute-action.json index 9e8942e813..ffba65ca53 100644 --- a/widgets/BigQueryExecute-action.json +++ b/widgets/BigQueryExecute-action.json @@ -28,6 +28,17 @@ { "label": "Advanced", "properties": [ + { + "name": "jobLabels", + "label": "BQ Job Labels", + "widget-type": "keyvalue", + "widget-attributes": { + "delimiter": ",", + "kv-delimiter": ":", + "key-placeholder": "Label key", + "value-placeholder": "Label value" + } + }, { "name": "dialect", "label": "Dialect", diff --git a/widgets/BigQueryMultiTable-batchsink.json b/widgets/BigQueryMultiTable-batchsink.json index d0f3c5f1a1..be6e3f0c41 100644 --- a/widgets/BigQueryMultiTable-batchsink.json +++ b/widgets/BigQueryMultiTable-batchsink.json @@ -131,7 +131,7 @@ "label": "Advanced", "properties": [ { - "name": "jobLabel", + "name": "jobLabels", "label": "BQ Job Labels", "widget-type": "keyvalue", "widget-attributes": {