diff --git a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java index aee2942ca1..d167eff7e8 100644 --- a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java +++ b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java @@ -31,6 +31,7 @@ interface BigQueryWriteOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery output table", + groupName = "Target", optional = true, helpText = "The BigQuery table location to write the output to. Use the format `:.`. The table's schema must match the input objects.") @@ -57,6 +58,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, description = "Database identifier", + groupName = "Source", helpText = "The database unique identifier (UUID).", example = "cf7af129-d33a-498f-ad06-d97a6ee6eb7") @Validation.Required @@ -69,6 +71,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, description = "Cassandra keyspace", + groupName = "Source", regexes = {"^[a-zA-Z0-9][a-zA-Z0-9_]{0,47}$"}, helpText = "The name of the Cassandra keyspace inside of the Astra database.") String getAstraKeyspace(); @@ -79,6 +82,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 4, description = "Cassandra table", + groupName = "Source", regexes = {"^[a-zA-Z][a-zA-Z0-9_]*$"}, helpText = "The name of the table inside of the Cassandra database.", example = "my_table") @@ -91,6 +95,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 5, optional = true, + groupName = "Source", description = "Cassandra CQL Query", helpText = "The query to use to filter rows instead of reading the whole table.") @SuppressWarnings("unused") @@ -102,6 +107,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, optional = true, + groupName = "Source", description = "Astra Database Region", helpText = "If not provided, a default is chosen, which is useful with multi-region databases.") diff --git a/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java b/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java index 0a70cb4514..da160380f9 100644 --- a/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java +++ b/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java @@ -120,6 +120,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { order = 1, optional = false, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, + groupName = "Source", description = "Azure Event Hub endpoint", helpText = "Server IP or DNS for Azure Eventhub Endpoint", example = "mynamespace.servicebus.windows.net:9093") @@ -132,6 +133,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { order = 2, optional = false, regexes = {"[a-zA-Z0-9._-]+"}, + groupName = "Source", description = "Azure Eventhub topic(s) to read the input from", helpText = "Azure Eventhub topic(s) to read the input from", example = "topic") @@ -143,6 +145,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 3, description = "Output Pub/Sub topic", + groupName = "Target", helpText = "The name of the topic to which data should published, in the format of 'projects/your-project-id/topics/your-topic-name'", example = "projects/your-project-id/topics/your-topic-name") diff --git a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java index aaf13282e8..2a9b75340e 100644 --- a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java +++ b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java @@ -146,6 +146,7 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery table to export", + groupName = "Source", helpText = "The BigQuery input table location.", example = "your-project:your-dataset.your-table-name") @Required @@ -156,6 +157,7 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.GcsWriteFile( order = 2, description = "Output Cloud Storage file(s)", + groupName = "Target", helpText = "The Cloud Storage folder to write the Parquet files to.", example = "gs://your-bucket/export/") @Required diff --git a/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java b/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java index c9d6d5e6a0..d200a11343 100644 --- a/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java +++ b/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java @@ -75,6 +75,7 @@ public interface BigtableToHbasePipelineOptions /** Hbase specific configs. Mirrors configurations on hbase-site.xml. */ @TemplateParameter.Text( description = "Zookeeper quorum host", + groupName = "Target", helpText = "Zookeeper quorum host, corresponds to hbase.zookeeper.quorum host") String getHbaseZookeeperQuorumHost(); @@ -83,6 +84,7 @@ public interface BigtableToHbasePipelineOptions @TemplateParameter.Text( optional = true, description = "Zookeeper quorum port", + groupName = "Target", helpText = "Zookeeper quorum port, corresponds to hbase.zookeeper.quorum port") @Default.String("2181") String getHbaseZookeeperQuorumPort(); @@ -91,6 +93,7 @@ public interface BigtableToHbasePipelineOptions @TemplateParameter.Text( description = "Hbase root directory", + groupName = "Target", helpText = "Hbase root directory, corresponds to hbase.rootdir") String getHbaseRootDir(); diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index 949543c75a..d8e02f542c 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -146,6 +146,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Source Bigtable Instance ID", helpText = "The source Bigtable instance ID.") @@ -156,6 +157,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Source Cloud Bigtable table ID", helpText = "The source Bigtable table ID.") @Validation.Required @@ -166,6 +168,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.ProjectId( order = 3, optional = true, + groupName = "Source", description = "Source Cloud Bigtable Project ID", helpText = "The Bigtable project ID. The default is the project for the Dataflow job.") @Default.String("") @@ -192,6 +195,7 @@ interface ReadChangeStreamOptions extends BigtableCommonOptions.ReadOptions { @TemplateParameter.Text( order = 1, optional = true, + groupName = "Source", description = "Cloud Bigtable change streams metadata instance ID", helpText = "The Bigtable change streams metadata instance ID.") @Default.String("") @@ -202,6 +206,7 @@ interface ReadChangeStreamOptions extends BigtableCommonOptions.ReadOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Source", description = "Cloud Bigtable change streams metadata table ID", helpText = "The ID of the Bigtable change streams connector metadata table. If not " diff --git a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java index a18fb44d78..46776037d7 100644 --- a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java +++ b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java @@ -96,6 +96,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 1, optional = true, + groupName = "Source", regexes = {"[,a-zA-Z0-9._-]+"}, description = "Pub/Sub topic(s) to read from", helpText = "Comma-separated list of PubSub topics to where CDC data is being pushed.") @@ -106,6 +107,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 2, regexes = {"[^/]+"}, + groupName = "Source", description = "Input subscriptions to the template", helpText = "The comma-separated list of Pub/Sub input subscriptions to read from, in the format `,, ...`") @@ -116,6 +118,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 3, regexes = {".+"}, + groupName = "Target", description = "Output BigQuery dataset for Changelog tables", helpText = "The BigQuery dataset to store the staging tables in, in the format .") @@ -126,6 +129,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 4, regexes = {".+"}, + groupName = "Target", description = "Output BigQuery dataset for replica tables", helpText = "The location of the BigQuery dataset to store the replica tables in, in the format .") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java index 74e87418ad..0d5a41dac2 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java @@ -102,6 +102,7 @@ public interface WriteToGCSAvroOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -112,6 +113,7 @@ public interface WriteToGCSAvroOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java index 568e3f53bb..5f231bbd4e 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java @@ -92,6 +92,7 @@ public interface WriteToGCSParquetOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -102,6 +103,7 @@ public interface WriteToGCSParquetOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java index cf19a6c27e..56f7ff1eb9 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java @@ -107,6 +107,7 @@ public interface WriteToGCSTextOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -117,6 +118,7 @@ public interface WriteToGCSTextOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java index eb1e7c1165..e9ca67261b 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java @@ -39,6 +39,7 @@ public interface DataplexBigQueryToGcsOptions @TemplateParameter.Text( order = 1, optional = false, + groupName = "Source", regexes = { "^(projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+|projects\\/[^\\n\\r\\/]+\\/datasets\\/[^\\n\\r\\/]+)$" }, @@ -54,6 +55,7 @@ public interface DataplexBigQueryToGcsOptions order = 2, optional = true, regexes = {"^[a-zA-Z0-9_-]+(,[a-zA-Z0-9_-]+)*$"}, + groupName = "Source", description = "Source BigQuery tables to tier.", helpText = "A comma-separated list of BigQuery tables to tier. If none specified, all tables will be tiered. Tables should be specified by their name only (no project/dataset prefix). Case-sensitive!") @@ -67,6 +69,7 @@ public interface DataplexBigQueryToGcsOptions regexes = { "^projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+$" }, + groupName = "Target", description = "Dataplex asset name for the destination Cloud Storage bucket.", helpText = "Dataplex asset name for the Cloud Storage bucket to tier data to. Format: projects//locations//lakes//zones//assets/.") diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java index cc25a6e6a1..86bfffa8cd 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java @@ -41,6 +41,7 @@ public interface DataplexJdbcIngestionOptions regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" }, + groupName = "Source", description = "JDBC connection URL string.", helpText = "Url connection string to connect to the JDBC source. Connection string can be passed in" @@ -127,6 +128,7 @@ public interface DataplexJdbcIngestionOptions void setQuery(String query); @TemplateParameter.Text( + groupName = "Target", order = 8, optional = false, regexes = {"^.+$"}, @@ -170,6 +172,7 @@ public interface DataplexJdbcIngestionOptions + "\\r" + "\\/]+$" }, + groupName = "Target", description = "Dataplex output asset ID", helpText = "Dataplex output asset ID to which the results are stored to. Should be in the format of" diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java index 2e1398bc01..a6f512b40e 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java @@ -126,6 +126,7 @@ public interface FileFormatConversionOptions "^(projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+|projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/entities\\/[^\\n\\r\\/,]+(,projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/entities\\/[^\\n\\r\\/,]+)*)$" }, description = "Dataplex asset name or Dataplex entity names for the files to be converted.", + groupName = "Source", helpText = "Dataplex asset or Dataplex entities that contain the input files. Format:" + " projects//locations//lakes//zones//assets/ getFirestoreReadProjectId(); @@ -160,6 +163,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.ProjectId( order = 1, + groupName = "Target", description = "Project ID", helpText = "The Google Cloud project ID of where to write Datastore entities") @Hidden @@ -178,6 +182,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 2, + groupName = "Target", description = "Datastore entity kind", helpText = "Datastore kind under which entities will be written in the output Google Cloud" @@ -198,6 +203,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 3, + groupName = "Target", optional = true, description = "Datastore namespace", helpText = @@ -219,6 +225,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 4, + groupName = "Target", optional = true, regexes = {"^[1-9]+[0-9]*$"}, description = "Expected number of workers", @@ -238,6 +245,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.ProjectId( order = 5, + groupName = "Target", description = "Project ID", helpText = "The Google Cloud project ID of where to write Firestore entities") ValueProvider getFirestoreWriteProjectId(); @@ -246,6 +254,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, + groupName = "Target", description = "Firestore entity kind", helpText = "Firestore kind under which entities will be written in the output Google Cloud" @@ -256,6 +265,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 7, + groupName = "Target", optional = true, description = "Namespace of the Firestore entity", helpText = @@ -267,6 +277,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Integer( order = 8, + groupName = "Target", optional = true, description = "Expected number of workers", helpText = diff --git a/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java b/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java index e248fd8833..54fe4eddfa 100644 --- a/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java +++ b/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java @@ -29,6 +29,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 1, + groupName = "Target", description = "MongoDB Connection URI", helpText = "The MongoDB connection URI in the format mongodb+srv://:@.") String getMongoDbUri(); @@ -37,6 +38,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 2, + groupName = "Target", description = "MongoDB Database", helpText = "Database in MongoDB to store the collection.", example = "my-db") @@ -46,6 +48,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 3, + groupName = "Target", description = "MongoDB collection", helpText = "The name of the collection in the MongoDB database.", example = "my-collection") @@ -59,6 +62,7 @@ public interface BigQueryReadOptions extends PipelineOptions, DataflowPipelineOp @TemplateParameter.BigQueryTable( order = 1, + groupName = "Source", description = "BigQuery source table", helpText = "The BigQuery table to read from.", example = "bigquery-project:dataset.input_table") diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java index 5335712767..052d68e727 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java @@ -26,7 +26,6 @@ public interface JdbcToBigQueryOptions order = 1, optional = false, regexes = {"^.+$"}, - groupName = "Source", description = "Comma-separated Cloud Storage path(s) of the JDBC driver(s)", helpText = "The comma-separated list of driver JAR files.", example = "gs://your-bucket/driver_jar1.jar,gs://your-bucket/driver_jar2.jar") @@ -38,7 +37,6 @@ public interface JdbcToBigQueryOptions order = 2, optional = false, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC driver class name", helpText = "The JDBC driver class name.", example = "com.mysql.jdbc.Driver") @@ -65,7 +63,6 @@ public interface JdbcToBigQueryOptions order = 4, optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, - groupName = "Source", description = "JDBC connection property string.", helpText = "The properties string to use for the JDBC connection. The format of the string must " @@ -82,7 +79,6 @@ public interface JdbcToBigQueryOptions order = 5, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC connection username.", helpText = "The username to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. Remove whitespace characters from the Base64-encoded string.") @@ -93,7 +89,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Password( order = 6, optional = true, - groupName = "Source", description = "JDBC connection password.", helpText = "The password to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. Remove whitespace characters from the Base64-encoded string.") @@ -105,7 +100,6 @@ public interface JdbcToBigQueryOptions order = 7, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC source SQL query", helpText = "The query to run on the source to extract the data. Note that some JDBC SQL and BigQuery types, although sharing the same name, have some differences. " @@ -130,7 +124,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.GcsWriteFolder( order = 9, optional = false, - groupName = "Target", description = "Temporary directory for BigQuery loading process", helpText = "The temporary directory for the BigQuery loading process.", example = "gs://your-bucket/your-files/temp_dir") @@ -141,7 +134,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.KmsEncryptionKey( order = 10, optional = true, - groupName = "Source", description = "Google Cloud KMS key", helpText = "The Cloud KMS encryption key to use to decrypt the username, password, and connection string. If you " @@ -154,7 +146,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Boolean( order = 11, optional = true, - groupName = "Source", description = "Whether to use column alias to map the rows.", helpText = "If set to `true`, the pipeline uses the column alias (`AS`) instead of the column name to map the rows to BigQuery. Defaults to `false`.") @@ -166,7 +157,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Boolean( order = 12, optional = true, - groupName = "Target", description = "Whether to truncate data before writing", helpText = "If set to `true`, the pipeline truncates before loading data into BigQuery. Defaults to `false`, which causes the pipeline to append data.") @@ -178,7 +168,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Text( order = 13, optional = true, - groupName = "Source", description = "The name of a column of numeric type that will be used for partitioning.", helpText = "If this parameter is provided with the name of the `table` defined as an optional parameter, JdbcIO reads the table in parallel by executing multiple instances of the query on the same table (subquery) using ranges. Currently, only supports `Long` partition columns.") @@ -189,7 +178,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Text( order = 14, optional = true, - groupName = "Source", description = "Name of the table in the external database.", helpText = "The table to read from when using partitions. This parameter also accepts a subquery in parentheses.", @@ -201,7 +189,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Integer( order = 15, optional = true, - groupName = "Source", description = "The number of partitions.", helpText = "The number of partitions. With the lower and upper bound, this value forms partition strides for generated `WHERE` clause expressions that are used to split the partition column evenly. When the input is less than `1`, the number is set to `1`.") @@ -212,7 +199,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Long( order = 16, optional = true, - groupName = "Source", description = "Lower bound of partition column.", helpText = "The lower bound to use in the partition scheme. If not provided, this value is automatically inferred by Apache Beam for the supported types.") @@ -223,7 +209,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Long( order = 17, optional = true, - groupName = "Source", description = "Upper bound of partition column", helpText = "The upper bound to use in the partition scheme. If not provided, this value is automatically inferred by Apache Beam for the supported types.") @@ -234,7 +219,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Integer( order = 18, optional = true, - groupName = "Source", description = "Fetch Size", // TODO: remove the "Not used for partitioned reads" once // https://github.com/apache/beam/pull/28999 is released. diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java index cca5c761b2..5b52b066a3 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java @@ -36,6 +36,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" @@ -105,6 +106,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.PubsubTopic( order = 8, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The Pub/Sub topic to publish to, in the format projects//topics/.", diff --git a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java index 6a5cd2c789..0c83a8292f 100644 --- a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java +++ b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java @@ -138,6 +138,7 @@ public String apply(JmsRecord input) { public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = true, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, description = "JMS Host IP", @@ -150,6 +151,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "JMS Queue/Topic Name to read the input from", @@ -174,6 +176,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 4, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the Pub/Sub topic to publish data to, in the format `projects//topics/`.", diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index f85543cdbb..04a75c6d66 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -194,6 +194,7 @@ public interface KafkaToBQOptions */ @TemplateParameter.Text( order = 2, + groupName = "Source", optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", @@ -221,6 +222,7 @@ public interface KafkaToBQOptions @Deprecated @TemplateParameter.Text( order = 3, + groupName = "Source", optional = true, regexes = {"[,a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from", diff --git a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java index f569efa341..0fe4ba9b53 100644 --- a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java +++ b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java @@ -36,6 +36,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Text( order = 1, + groupName = "Source", optional = false, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", @@ -48,6 +49,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[,a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from",