From 9d3fcbc51e4d23dadc6484063c26d5de848652a4 Mon Sep 17 00:00:00 2001 From: Akarys Shorabek Date: Fri, 10 May 2024 07:16:33 +0500 Subject: [PATCH 1/3] Add groupName attribute to v2 template where it's needed (part 1) --- .../options/AstraDbToBigQueryOptions.java | 7 ++++++ .../v2/templates/AzureEventhubToPubsub.java | 3 +++ .../v2/templates/BigQueryToParquet.java | 3 +++ .../BigtableChangeStreamsToHBase.java | 3 +++ .../CdcToBigQueryChangeApplierPipeline.java | 6 +++++ .../auto/dlq/BigQueryDeadletterOptions.java | 1 + .../options/DataplexBigQueryToGcsOptions.java | 7 ++++++ .../options/DataplexJdbcIngestionOptions.java | 16 +++++++++++++ .../DataplexFileFormatConversion.java | 5 ++++ .../v2/templates/DataStreamToBigQuery.java | 20 ++++++++++++++++ .../v2/templates/DataStreamToMongoDB.java | 8 +++++++ .../v2/templates/DataStreamToPostgres.java | 8 +++++++ .../v2/templates/DataStreamToSpanner.java | 23 +++++++++++++++++++ .../v2/templates/DataStreamToSQL.java | 14 +++++++++++ .../v2/templates/FileFormatConversion.java | 2 ++ .../teleport/v2/templates/GCSToSourceDb.java | 8 +++++++ .../v2/templates/GoogleAdsToBigQuery.java | 9 ++++++++ .../options/GCSToElasticsearchOptions.java | 2 ++ ...BigtableChangeStreamToBigQueryOptions.java | 10 ++++++++ .../BigtableChangeStreamsToGcsOptions.java | 11 +++++++++ .../BigtableChangeStreamsToPubSubOptions.java | 12 ++++++++++ .../templates/DLPTextToBigQueryStreaming.java | 6 +++++ .../v2/templates/FirestoreToBigQuery.java | 3 +++ .../v2/utils/FirestoreConverters.java | 14 +++++++++++ .../options/BigQueryToMongoDbOptions.java | 4 ++++ .../v2/options/JdbcToBigQueryOptions.java | 2 ++ .../v2/options/JdbcToPubsubOptions.java | 9 ++++++++ .../teleport/v2/templates/JmsToPubsub.java | 6 +++++ .../v2/templates/KafkaToBigQuery.java | 6 +++++ .../v2/options/KafkaToGCSOptions.java | 4 ++++ 30 files changed, 232 insertions(+) diff --git a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java index aee2942ca1..d5471c0e43 100644 --- a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java +++ b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java @@ -31,6 +31,7 @@ interface BigQueryWriteOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery output table", + groupName = "Target", optional = true, helpText = "The BigQuery table location to write the output to. Use the format `:.`. The table's schema must match the input objects.") @@ -45,6 +46,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, description = "Astra token", + groupName = "Source", helpText = "The token value or secret resource ID.", example = "AstraCS:abcdefghij") @Validation.Required @@ -57,6 +59,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, description = "Database identifier", + groupName = "Source", helpText = "The database unique identifier (UUID).", example = "cf7af129-d33a-498f-ad06-d97a6ee6eb7") @Validation.Required @@ -69,6 +72,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, description = "Cassandra keyspace", + groupName = "Source", regexes = {"^[a-zA-Z0-9][a-zA-Z0-9_]{0,47}$"}, helpText = "The name of the Cassandra keyspace inside of the Astra database.") String getAstraKeyspace(); @@ -79,6 +83,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 4, description = "Cassandra table", + groupName = "Source", regexes = {"^[a-zA-Z][a-zA-Z0-9_]*$"}, helpText = "The name of the table inside of the Cassandra database.", example = "my_table") @@ -91,6 +96,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 5, optional = true, + groupName = "Source", description = "Cassandra CQL Query", helpText = "The query to use to filter rows instead of reading the whole table.") @SuppressWarnings("unused") @@ -102,6 +108,7 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, optional = true, + groupName = "Source", description = "Astra Database Region", helpText = "If not provided, a default is chosen, which is useful with multi-region databases.") diff --git a/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java b/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java index 0a70cb4514..da160380f9 100644 --- a/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java +++ b/v2/azure-eventhub-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/AzureEventhubToPubsub.java @@ -120,6 +120,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { order = 1, optional = false, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, + groupName = "Source", description = "Azure Event Hub endpoint", helpText = "Server IP or DNS for Azure Eventhub Endpoint", example = "mynamespace.servicebus.windows.net:9093") @@ -132,6 +133,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { order = 2, optional = false, regexes = {"[a-zA-Z0-9._-]+"}, + groupName = "Source", description = "Azure Eventhub topic(s) to read the input from", helpText = "Azure Eventhub topic(s) to read the input from", example = "topic") @@ -143,6 +145,7 @@ public interface AzureEventhubToPubsubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 3, description = "Output Pub/Sub topic", + groupName = "Target", helpText = "The name of the topic to which data should published, in the format of 'projects/your-project-id/topics/your-topic-name'", example = "projects/your-project-id/topics/your-topic-name") diff --git a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java index aaf13282e8..cd387a8eda 100644 --- a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java +++ b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java @@ -146,6 +146,7 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery table to export", + groupName = "Source", helpText = "The BigQuery input table location.", example = "your-project:your-dataset.your-table-name") @Required @@ -156,6 +157,7 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.GcsWriteFile( order = 2, description = "Output Cloud Storage file(s)", + groupName = "Target", helpText = "The Cloud Storage folder to write the Parquet files to.", example = "gs://your-bucket/export/") @Required @@ -166,6 +168,7 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.Integer( order = 3, optional = true, + groupName = "Target", description = "Maximum output shards", helpText = "The number of output file shards. The default value is 1.") @Default.Integer(0) diff --git a/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java b/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java index c9d6d5e6a0..d200a11343 100644 --- a/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java +++ b/v2/bigtable-changestreams-to-hbase/src/main/java/com/google/cloud/teleport/v2/templates/BigtableChangeStreamsToHBase.java @@ -75,6 +75,7 @@ public interface BigtableToHbasePipelineOptions /** Hbase specific configs. Mirrors configurations on hbase-site.xml. */ @TemplateParameter.Text( description = "Zookeeper quorum host", + groupName = "Target", helpText = "Zookeeper quorum host, corresponds to hbase.zookeeper.quorum host") String getHbaseZookeeperQuorumHost(); @@ -83,6 +84,7 @@ public interface BigtableToHbasePipelineOptions @TemplateParameter.Text( optional = true, description = "Zookeeper quorum port", + groupName = "Target", helpText = "Zookeeper quorum port, corresponds to hbase.zookeeper.quorum port") @Default.String("2181") String getHbaseZookeeperQuorumPort(); @@ -91,6 +93,7 @@ public interface BigtableToHbasePipelineOptions @TemplateParameter.Text( description = "Hbase root directory", + groupName = "Target", helpText = "Hbase root directory, corresponds to hbase.rootdir") String getHbaseRootDir(); diff --git a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java index a18fb44d78..e8df362013 100644 --- a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java +++ b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java @@ -96,6 +96,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 1, optional = true, + groupName = "Source", regexes = {"[,a-zA-Z0-9._-]+"}, description = "Pub/Sub topic(s) to read from", helpText = "Comma-separated list of PubSub topics to where CDC data is being pushed.") @@ -106,6 +107,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 2, regexes = {"[^/]+"}, + groupName = "Source", description = "Input subscriptions to the template", helpText = "The comma-separated list of Pub/Sub input subscriptions to read from, in the format `,, ...`") @@ -116,6 +118,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 3, regexes = {".+"}, + groupName = "Target", description = "Output BigQuery dataset for Changelog tables", helpText = "The BigQuery dataset to store the staging tables in, in the format .") @@ -126,6 +129,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Text( order = 4, regexes = {".+"}, + groupName = "Target", description = "Output BigQuery dataset for replica tables", helpText = "The location of the BigQuery dataset to store the replica tables in, in the format .") @@ -136,6 +140,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Integer( order = 5, optional = true, + groupName = "Target", description = "Frequency to issue updates to BigQuery tables (seconds).", helpText = "The interval at which the pipeline updates the BigQuery table replicating the MySQL database.") @@ -146,6 +151,7 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Boolean( order = 6, optional = true, + groupName = "Source", description = "Whether to use a single topic for all MySQL table changes.", helpText = "Set this to true if you have configured your Debezium connector to publish all table" diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java index 9d4b9cdf5b..8c8d90d77b 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java @@ -21,6 +21,7 @@ public interface BigQueryDeadletterOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( + groupName = "Target", order = 4, optional = true, description = "Table for messages failed to reach the output table (i.e., Deadletter table)", diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java index eb1e7c1165..051c193cf7 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java @@ -39,6 +39,7 @@ public interface DataplexBigQueryToGcsOptions @TemplateParameter.Text( order = 1, optional = false, + groupName = "Source", regexes = { "^(projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+|projects\\/[^\\n\\r\\/]+\\/datasets\\/[^\\n\\r\\/]+)$" }, @@ -54,6 +55,7 @@ public interface DataplexBigQueryToGcsOptions order = 2, optional = true, regexes = {"^[a-zA-Z0-9_-]+(,[a-zA-Z0-9_-]+)*$"}, + groupName = "Source", description = "Source BigQuery tables to tier.", helpText = "A comma-separated list of BigQuery tables to tier. If none specified, all tables will be tiered. Tables should be specified by their name only (no project/dataset prefix). Case-sensitive!") @@ -67,6 +69,7 @@ public interface DataplexBigQueryToGcsOptions regexes = { "^projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+$" }, + groupName = "Target", description = "Dataplex asset name for the destination Cloud Storage bucket.", helpText = "Dataplex asset name for the Cloud Storage bucket to tier data to. Format: projects//locations//lakes//zones//assets/.") @@ -90,6 +93,7 @@ public interface DataplexBigQueryToGcsOptions @TemplateParameter.Integer( order = 5, + groupName = "Source", description = "Maximum number of parallel requests.", helpText = "The maximum number of parallel requests that will be sent to BigQuery when loading table/partition metadata.") @@ -103,6 +107,7 @@ public interface DataplexBigQueryToGcsOptions order = 6, enumOptions = {@TemplateEnumOption("AVRO"), @TemplateEnumOption("PARQUET")}, optional = true, + groupName = "Target", description = "Output file format in Cloud Storage.", helpText = "Output file format in Cloud Storage. Format: PARQUET or AVRO.") @Default.Enum("PARQUET") @@ -120,6 +125,7 @@ public interface DataplexBigQueryToGcsOptions @TemplateEnumOption("BZIP2") }, optional = true, + groupName = "Target", description = "Output file compression in Cloud Storage.", helpText = "Output file compression. Format: UNCOMPRESSED, SNAPPY, GZIP, or BZIP2. BZIP2 not supported for PARQUET files.") @@ -146,6 +152,7 @@ public interface DataplexBigQueryToGcsOptions @TemplateEnumOption("SKIP") }, optional = true, + groupName = "Target", description = "Action that occurs if a destination file already exists.", helpText = "Specifies the action that occurs if a destination file already exists. Format: OVERWRITE, FAIL, SKIP. If SKIP, only files that don't exist in the destination directory will be processed. If FAIL and at least one file already exists, no data will be processed and an error will be produced.") diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java index cc25a6e6a1..017840153f 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java @@ -41,6 +41,7 @@ public interface DataplexJdbcIngestionOptions regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" }, + groupName = "Source", description = "JDBC connection URL string.", helpText = "Url connection string to connect to the JDBC source. Connection string can be passed in" @@ -52,6 +53,7 @@ public interface DataplexJdbcIngestionOptions void setConnectionURL(String connectionURL); @TemplateParameter.Text( + groupName = "Source", order = 2, optional = false, regexes = {"^.+$"}, @@ -64,6 +66,7 @@ public interface DataplexJdbcIngestionOptions void setDriverClassName(String driverClassName); @TemplateParameter.Text( + groupName = "Source", order = 3, optional = false, regexes = {"^.+$"}, @@ -76,6 +79,7 @@ public interface DataplexJdbcIngestionOptions void setDriverJars(String driverJar); @TemplateParameter.Text( + groupName = "Source", order = 4, optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, @@ -90,6 +94,7 @@ public interface DataplexJdbcIngestionOptions void setConnectionProperties(String connectionProperties); @TemplateParameter.Text( + groupName = "Source", order = 5, optional = true, regexes = {"^.+$"}, @@ -103,6 +108,7 @@ public interface DataplexJdbcIngestionOptions void setUsername(String username); @TemplateParameter.Password( + groupName = "Source", order = 6, optional = true, description = "JDBC connection password.", @@ -115,6 +121,7 @@ public interface DataplexJdbcIngestionOptions void setPassword(String password); @TemplateParameter.Text( + groupName = "Source", order = 7, optional = false, regexes = {"^.+$"}, @@ -127,6 +134,7 @@ public interface DataplexJdbcIngestionOptions void setQuery(String query); @TemplateParameter.Text( + groupName = "Target", order = 8, optional = false, regexes = {"^.+$"}, @@ -144,6 +152,7 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.KmsEncryptionKey( order = 9, optional = true, + groupName = "Target", description = "Google Cloud KMS key", helpText = "If this parameter is provided, password, user name and connection string should all be" @@ -170,6 +179,7 @@ public interface DataplexJdbcIngestionOptions + "\\r" + "\\/]+$" }, + groupName = "Target", description = "Dataplex output asset ID", helpText = "Dataplex output asset ID to which the results are stored to. Should be in the format of" @@ -187,6 +197,7 @@ public interface DataplexJdbcIngestionOptions @TemplateEnumOption("MONTHLY") }, optional = true, + groupName = "Target", description = "The partition scheme when writing the file.", helpText = "The partition scheme when writing the file. Format: DAILY or MONTHLY or HOURLY.") @Default.Enum("DAILY") @@ -197,6 +208,7 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.Text( order = 12, optional = true, + groupName = "Target", description = "The partition column on which the partition is based.", helpText = "The partition column on which the partition is based. The column type must be of" @@ -218,6 +230,7 @@ public interface DataplexJdbcIngestionOptions @TemplateEnumOption("WRITE_EMPTY") }, optional = true, + groupName = "Target", description = "BigQuery write disposition type", helpText = "Strategy to employ if the target file/table exists. If the table exists - should it" @@ -232,6 +245,7 @@ public interface DataplexJdbcIngestionOptions order = 14, enumOptions = {@TemplateEnumOption("AVRO"), @TemplateEnumOption("PARQUET")}, optional = true, + groupName = "Target", description = "Output file format in Cloud Storage.", helpText = "Output file format in Cloud Storage. Format: PARQUET or AVRO.") @Default.Enum("PARQUET") @@ -243,6 +257,7 @@ public interface DataplexJdbcIngestionOptions order = 15, optional = true, description = "Whether to use column alias to map the rows.", + groupName = "Target", helpText = "If enabled (set to true) the pipeline will consider column alias (\"AS\") instead of the" + " column name to map the rows to BigQuery. Defaults to false.") @@ -254,6 +269,7 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.Integer( order = 16, optional = true, + groupName = "Target", description = "Set the data size going to be fetched and loaded in memory per Jdbc call.", helpText = "It should ONLY be used if the default value throws memory errors. If not set, using Beam's default " diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java index 2e1398bc01..c5a96dc5e5 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java @@ -126,6 +126,7 @@ public interface FileFormatConversionOptions "^(projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/assets\\/[^\\n\\r\\/]+|projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/entities\\/[^\\n\\r\\/,]+(,projects\\/[^\\n\\r\\/]+\\/locations\\/[^\\n\\r\\/]+\\/lakes\\/[^\\n\\r\\/]+\\/zones\\/[^\\n\\r\\/]+\\/entities\\/[^\\n\\r\\/,]+)*)$" }, description = "Dataplex asset name or Dataplex entity names for the files to be converted.", + groupName = "Source", helpText = "Dataplex asset or Dataplex entities that contain the input files. Format:" + " projects//locations//lakes//zones//assets/ getFirestoreReadProjectId(); @@ -143,6 +148,7 @@ public interface FirestoreReadOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, + groupName = "Source", optional = true, regexes = {"^[0-9A-Za-z._-]{0,100}$"}, description = "Namespace", @@ -160,6 +166,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.ProjectId( order = 1, + groupName = "Target", description = "Project ID", helpText = "The Google Cloud project ID of where to write Datastore entities") @Hidden @@ -178,6 +185,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 2, + groupName = "Target", description = "Datastore entity kind", helpText = "Datastore kind under which entities will be written in the output Google Cloud" @@ -198,6 +206,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 3, + groupName = "Target", optional = true, description = "Datastore namespace", helpText = @@ -219,6 +228,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 4, + groupName = "Target", optional = true, regexes = {"^[1-9]+[0-9]*$"}, description = "Expected number of workers", @@ -238,6 +248,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.ProjectId( order = 5, + groupName = "Target", description = "Project ID", helpText = "The Google Cloud project ID of where to write Firestore entities") ValueProvider getFirestoreWriteProjectId(); @@ -246,6 +257,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, + groupName = "Target", description = "Firestore entity kind", helpText = "Firestore kind under which entities will be written in the output Google Cloud" @@ -256,6 +268,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 7, + groupName = "Target", optional = true, description = "Namespace of the Firestore entity", helpText = @@ -267,6 +280,7 @@ public interface DatastoreWriteOptions extends PipelineOptions { @TemplateParameter.Integer( order = 8, + groupName = "Target", optional = true, description = "Expected number of workers", helpText = diff --git a/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java b/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java index e248fd8833..54fe4eddfa 100644 --- a/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java +++ b/v2/googlecloud-to-mongodb/src/main/java/com/google/cloud/teleport/v2/mongodb/options/BigQueryToMongoDbOptions.java @@ -29,6 +29,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 1, + groupName = "Target", description = "MongoDB Connection URI", helpText = "The MongoDB connection URI in the format mongodb+srv://:@.") String getMongoDbUri(); @@ -37,6 +38,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 2, + groupName = "Target", description = "MongoDB Database", helpText = "Database in MongoDB to store the collection.", example = "my-db") @@ -46,6 +48,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 3, + groupName = "Target", description = "MongoDB collection", helpText = "The name of the collection in the MongoDB database.", example = "my-collection") @@ -59,6 +62,7 @@ public interface BigQueryReadOptions extends PipelineOptions, DataflowPipelineOp @TemplateParameter.BigQueryTable( order = 1, + groupName = "Source", description = "BigQuery source table", helpText = "The BigQuery table to read from.", example = "bigquery-project:dataset.input_table") diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java index 5335712767..a32de53ef1 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java @@ -247,6 +247,7 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Enum( order = 19, + groupName = "Target", enumOptions = { @TemplateParameter.TemplateEnumOption("CREATE_IF_NEEDED"), @TemplateParameter.TemplateEnumOption("CREATE_NEVER") @@ -262,6 +263,7 @@ public interface JdbcToBigQueryOptions @TemplateParameter.GcsReadFile( order = 20, + groupName = "Target", optional = true, description = "Cloud Storage path to BigQuery JSON schema", helpText = diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java index cca5c761b2..a535deb27d 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java @@ -25,6 +25,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = false, regexes = {"^.+$"}, description = "JDBC driver class name.", @@ -36,6 +37,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" @@ -50,6 +52,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 3, + groupName = "Source", optional = true, regexes = {"^.+$"}, description = "JDBC connection username.", @@ -61,6 +64,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Password( order = 4, + groupName = "Source", optional = true, description = "JDBC connection password.", helpText = @@ -71,6 +75,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 5, + groupName = "Source", optional = false, regexes = {"^.+$"}, description = "Cloud Storage paths for JDBC drivers", @@ -82,6 +87,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 6, + groupName = "Source", optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, description = "JDBC connection property string.", @@ -94,6 +100,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 7, + groupName = "Source", optional = false, regexes = {"^.+$"}, description = "JDBC source SQL query.", @@ -105,6 +112,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.PubsubTopic( order = 8, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The Pub/Sub topic to publish to, in the format projects//topics/.", @@ -116,6 +124,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.KmsEncryptionKey( order = 9, + groupName = "Source", optional = true, description = "Google Cloud KMS key", helpText = diff --git a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java index 6a5cd2c789..2c347869bb 100644 --- a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java +++ b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java @@ -138,6 +138,7 @@ public String apply(JmsRecord input) { public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = true, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, description = "JMS Host IP", @@ -150,6 +151,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "JMS Queue/Topic Name to read the input from", @@ -162,6 +164,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, + groupName = "Source", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "JMS Destination Type to read the input from", @@ -174,6 +177,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 4, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the Pub/Sub topic to publish data to, in the format `projects//topics/`.", @@ -185,6 +189,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 5, + groupName = "Source", description = "JMS Username", helpText = "The username to use for authentication on the JMS server.", example = "sampleusername") @@ -194,6 +199,7 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Password( order = 6, + groupName = "Source", description = "JMS Password", helpText = "The password associated with the provided username.", example = "samplepassword") diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index f85543cdbb..fa660a3c34 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -194,6 +194,7 @@ public interface KafkaToBQOptions */ @TemplateParameter.Text( order = 2, + groupName = "Source", optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", @@ -221,6 +222,7 @@ public interface KafkaToBQOptions @Deprecated @TemplateParameter.Text( order = 3, + groupName = "Source", optional = true, regexes = {"[,a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from", @@ -239,6 +241,7 @@ public interface KafkaToBQOptions @TemplateParameter.BigQueryTable( order = 4, + groupName = "Target", optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = @@ -252,6 +255,7 @@ public interface KafkaToBQOptions @TemplateParameter.Enum( order = 5, + groupName = "Target", enumOptions = { @TemplateParameter.TemplateEnumOption("AVRO"), @TemplateParameter.TemplateEnumOption("JSON") @@ -266,6 +270,7 @@ public interface KafkaToBQOptions @TemplateParameter.GcsReadFile( order = 6, + groupName = "Target", optional = true, description = "Cloud Storage path to the Avro schema file", helpText = "Cloud Storage path to Avro schema file. For example, gs://MyBucket/file.avsc.") @@ -275,6 +280,7 @@ public interface KafkaToBQOptions @TemplateParameter.Boolean( order = 7, + groupName = "Target", optional = true, parentName = "useStorageWriteApi", parentTriggerValues = {"true"}, diff --git a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java index f569efa341..4d9c98bb39 100644 --- a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java +++ b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java @@ -36,6 +36,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Text( order = 1, + groupName = "Source", optional = false, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", @@ -48,6 +49,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[,a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from", @@ -60,6 +62,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Enum( order = 3, + groupName = "Target", enumOptions = { @TemplateEnumOption("TEXT"), @TemplateEnumOption("AVRO"), @@ -76,6 +79,7 @@ public interface KafkaToGCSOptions @TemplateParameter.Duration( order = 4, + groupName = "Target", optional = true, description = "Window duration", helpText = From b687c011ce574e2aecdc430460527254ff12f8ca Mon Sep 17 00:00:00 2001 From: "vitaly.terentyev" Date: Thu, 16 May 2024 15:12:30 +0400 Subject: [PATCH 2/3] Remove unnecessary group names --- .../options/AstraDbToBigQueryOptions.java | 1 - .../v2/templates/BigQueryToParquet.java | 1 - .../CdcToBigQueryChangeApplierPipeline.java | 2 -- .../v2/auto/dlq/BigQueryDeadletterOptions.java | 1 - .../v2/options/BigQueryCommonOptions.java | 1 + .../options/DataplexBigQueryToGcsOptions.java | 4 ---- .../options/DataplexJdbcIngestionOptions.java | 13 ------------- .../DataplexFileFormatConversion.java | 3 --- .../v2/templates/DataStreamToBigQuery.java | 14 -------------- .../v2/templates/DataStreamToMongoDB.java | 3 --- .../v2/templates/DataStreamToPostgres.java | 4 ---- .../v2/templates/DataStreamToSpanner.java | 17 ----------------- .../teleport/v2/templates/DataStreamToSQL.java | 8 -------- .../v2/templates/FileFormatConversion.java | 2 -- .../teleport/v2/templates/GCSToSourceDb.java | 7 ------- .../v2/templates/GoogleAdsToBigQuery.java | 8 -------- .../options/GCSToElasticsearchOptions.java | 2 -- .../BigtableChangeStreamToBigQueryOptions.java | 8 -------- .../BigtableChangeStreamsToGcsOptions.java | 10 ---------- .../BigtableChangeStreamsToPubSubOptions.java | 10 ---------- .../templates/DLPTextToBigQueryStreaming.java | 3 --- .../v2/templates/FirestoreToBigQuery.java | 2 -- .../teleport/v2/utils/FirestoreConverters.java | 3 --- .../v2/options/JdbcToBigQueryOptions.java | 18 ------------------ .../v2/options/JdbcToPubsubOptions.java | 7 ------- .../teleport/v2/templates/JmsToPubsub.java | 3 --- .../teleport/v2/templates/KafkaToBigQuery.java | 4 ---- .../teleport/v2/options/KafkaToGCSOptions.java | 2 -- 28 files changed, 1 insertion(+), 160 deletions(-) diff --git a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java index d5471c0e43..d167eff7e8 100644 --- a/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java +++ b/v2/astradb-to-bigquery/src/main/java/com/google/cloud/teleport/v2/astradb/options/AstraDbToBigQueryOptions.java @@ -46,7 +46,6 @@ interface AstraDbSourceOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, description = "Astra token", - groupName = "Source", helpText = "The token value or secret resource ID.", example = "AstraCS:abcdefghij") @Validation.Required diff --git a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java index cd387a8eda..2a9b75340e 100644 --- a/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java +++ b/v2/bigquery-to-parquet/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToParquet.java @@ -168,7 +168,6 @@ public interface BigQueryToParquetOptions extends PipelineOptions { @TemplateParameter.Integer( order = 3, optional = true, - groupName = "Target", description = "Maximum output shards", helpText = "The number of output file shards. The default value is 1.") @Default.Integer(0) diff --git a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java index e8df362013..46776037d7 100644 --- a/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java +++ b/v2/cdc-parent/cdc-change-applier/src/main/java/com/google/cloud/dataflow/cdc/applier/CdcToBigQueryChangeApplierPipeline.java @@ -140,7 +140,6 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Integer( order = 5, optional = true, - groupName = "Target", description = "Frequency to issue updates to BigQuery tables (seconds).", helpText = "The interval at which the pipeline updates the BigQuery table replicating the MySQL database.") @@ -151,7 +150,6 @@ public interface CdcApplierOptions extends PipelineOptions, BigQueryStorageApiSt @TemplateParameter.Boolean( order = 6, optional = true, - groupName = "Source", description = "Whether to use a single topic for all MySQL table changes.", helpText = "Set this to true if you have configured your Debezium connector to publish all table" diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java index 8c8d90d77b..9d4b9cdf5b 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/BigQueryDeadletterOptions.java @@ -21,7 +21,6 @@ public interface BigQueryDeadletterOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( - groupName = "Target", order = 4, optional = true, description = "Table for messages failed to reach the output table (i.e., Deadletter table)", diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java index e085996fc7..630e4f36a6 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java @@ -36,6 +36,7 @@ public interface WriteOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery output table", + groupName = "Target", helpText = "The BigQuery output table location to write the output to. For example, `:.`." + "Depending on the `createDisposition` specified, the output table might be created automatically using the user provided Avro schema.") diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java index 051c193cf7..e9ca67261b 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexBigQueryToGcsOptions.java @@ -93,7 +93,6 @@ public interface DataplexBigQueryToGcsOptions @TemplateParameter.Integer( order = 5, - groupName = "Source", description = "Maximum number of parallel requests.", helpText = "The maximum number of parallel requests that will be sent to BigQuery when loading table/partition metadata.") @@ -107,7 +106,6 @@ public interface DataplexBigQueryToGcsOptions order = 6, enumOptions = {@TemplateEnumOption("AVRO"), @TemplateEnumOption("PARQUET")}, optional = true, - groupName = "Target", description = "Output file format in Cloud Storage.", helpText = "Output file format in Cloud Storage. Format: PARQUET or AVRO.") @Default.Enum("PARQUET") @@ -125,7 +123,6 @@ public interface DataplexBigQueryToGcsOptions @TemplateEnumOption("BZIP2") }, optional = true, - groupName = "Target", description = "Output file compression in Cloud Storage.", helpText = "Output file compression. Format: UNCOMPRESSED, SNAPPY, GZIP, or BZIP2. BZIP2 not supported for PARQUET files.") @@ -152,7 +149,6 @@ public interface DataplexBigQueryToGcsOptions @TemplateEnumOption("SKIP") }, optional = true, - groupName = "Target", description = "Action that occurs if a destination file already exists.", helpText = "Specifies the action that occurs if a destination file already exists. Format: OVERWRITE, FAIL, SKIP. If SKIP, only files that don't exist in the destination directory will be processed. If FAIL and at least one file already exists, no data will be processed and an error will be produced.") diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java index 017840153f..86bfffa8cd 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/options/DataplexJdbcIngestionOptions.java @@ -53,7 +53,6 @@ public interface DataplexJdbcIngestionOptions void setConnectionURL(String connectionURL); @TemplateParameter.Text( - groupName = "Source", order = 2, optional = false, regexes = {"^.+$"}, @@ -66,7 +65,6 @@ public interface DataplexJdbcIngestionOptions void setDriverClassName(String driverClassName); @TemplateParameter.Text( - groupName = "Source", order = 3, optional = false, regexes = {"^.+$"}, @@ -79,7 +77,6 @@ public interface DataplexJdbcIngestionOptions void setDriverJars(String driverJar); @TemplateParameter.Text( - groupName = "Source", order = 4, optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, @@ -94,7 +91,6 @@ public interface DataplexJdbcIngestionOptions void setConnectionProperties(String connectionProperties); @TemplateParameter.Text( - groupName = "Source", order = 5, optional = true, regexes = {"^.+$"}, @@ -108,7 +104,6 @@ public interface DataplexJdbcIngestionOptions void setUsername(String username); @TemplateParameter.Password( - groupName = "Source", order = 6, optional = true, description = "JDBC connection password.", @@ -121,7 +116,6 @@ public interface DataplexJdbcIngestionOptions void setPassword(String password); @TemplateParameter.Text( - groupName = "Source", order = 7, optional = false, regexes = {"^.+$"}, @@ -152,7 +146,6 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.KmsEncryptionKey( order = 9, optional = true, - groupName = "Target", description = "Google Cloud KMS key", helpText = "If this parameter is provided, password, user name and connection string should all be" @@ -197,7 +190,6 @@ public interface DataplexJdbcIngestionOptions @TemplateEnumOption("MONTHLY") }, optional = true, - groupName = "Target", description = "The partition scheme when writing the file.", helpText = "The partition scheme when writing the file. Format: DAILY or MONTHLY or HOURLY.") @Default.Enum("DAILY") @@ -208,7 +200,6 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.Text( order = 12, optional = true, - groupName = "Target", description = "The partition column on which the partition is based.", helpText = "The partition column on which the partition is based. The column type must be of" @@ -230,7 +221,6 @@ public interface DataplexJdbcIngestionOptions @TemplateEnumOption("WRITE_EMPTY") }, optional = true, - groupName = "Target", description = "BigQuery write disposition type", helpText = "Strategy to employ if the target file/table exists. If the table exists - should it" @@ -245,7 +235,6 @@ public interface DataplexJdbcIngestionOptions order = 14, enumOptions = {@TemplateEnumOption("AVRO"), @TemplateEnumOption("PARQUET")}, optional = true, - groupName = "Target", description = "Output file format in Cloud Storage.", helpText = "Output file format in Cloud Storage. Format: PARQUET or AVRO.") @Default.Enum("PARQUET") @@ -257,7 +246,6 @@ public interface DataplexJdbcIngestionOptions order = 15, optional = true, description = "Whether to use column alias to map the rows.", - groupName = "Target", helpText = "If enabled (set to true) the pipeline will consider column alias (\"AS\") instead of the" + " column name to map the rows to BigQuery. Defaults to false.") @@ -269,7 +257,6 @@ public interface DataplexJdbcIngestionOptions @TemplateParameter.Integer( order = 16, optional = true, - groupName = "Target", description = "Set the data size going to be fetched and loaded in memory per Jdbc call.", helpText = "It should ONLY be used if the default value throws memory errors. If not set, using Beam's default " diff --git a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java index c5a96dc5e5..a6f512b40e 100644 --- a/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java +++ b/v2/dataplex/src/main/java/com/google/cloud/teleport/v2/templates/DataplexFileFormatConversion.java @@ -141,7 +141,6 @@ public interface FileFormatConversionOptions @TemplateParameter.Enum( order = 2, enumOptions = {@TemplateEnumOption("AVRO"), @TemplateEnumOption("PARQUET")}, - groupName = "Target", description = "Output file format in Cloud Storage.", helpText = "Output file format in Cloud Storage. Format: PARQUET or AVRO.") @Required @@ -158,7 +157,6 @@ public interface FileFormatConversionOptions @TemplateEnumOption("BZIP2") }, optional = true, - groupName = "Target", description = "Output file compression in Cloud Storage.", helpText = "Output file compression. Format: UNCOMPRESSED, SNAPPY, GZIP, or BZIP2. BZIP2 not" @@ -193,7 +191,6 @@ public interface FileFormatConversionOptions @TemplateEnumOption("SKIP") }, optional = true, - groupName = "Target", description = "Action that occurs if a destination file already exists.", helpText = "Specifies the action that occurs if a destination file already exists. Format:" diff --git a/v2/datastream-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToBigQuery.java b/v2/datastream-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToBigQuery.java index 43de2e7734..b55fb26204 100644 --- a/v2/datastream-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToBigQuery.java +++ b/v2/datastream-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToBigQuery.java @@ -157,7 +157,6 @@ public interface Options @TemplateParameter.Enum( order = 2, enumOptions = {@TemplateEnumOption("avro"), @TemplateEnumOption("json")}, - groupName = "Source", description = "Datastream output file format (avro/json).", helpText = "The format of the output files produced by Datastream. Value can be 'avro' or 'json'. Defaults to: avro.") @@ -167,7 +166,6 @@ public interface Options void setInputFileFormat(String value); @TemplateParameter.PubsubSubscription( - groupName = "Source", order = 3, description = "The Pub/Sub subscription on the Cloud Storage bucket.", helpText = @@ -177,7 +175,6 @@ public interface Options void setGcsPubSubSubscription(String value); @TemplateParameter.Text( - groupName = "Source", order = 4, optional = true, description = "Name or template for the stream to poll for schema information.", @@ -188,7 +185,6 @@ public interface Options void setStreamName(String value); @TemplateParameter.DateTime( - groupName = "Source", order = 5, optional = true, description = @@ -205,7 +201,6 @@ public interface Options order = 6, optional = true, description = "File read concurrency", - groupName = "Source", helpText = "The number of concurrent DataStream files to read. Default is 10.") @Default.Integer(10) Integer getFileReadConcurrency(); @@ -271,7 +266,6 @@ public interface Options @TemplateParameter.Text( order = 12, - groupName = "Target", optional = true, description = "Fields to be ignored", helpText = @@ -286,7 +280,6 @@ public interface Options void setIgnoreFields(String value); @TemplateParameter.Integer( - groupName = "Target", order = 13, optional = true, description = "The number of minutes between merges for a given table", @@ -297,7 +290,6 @@ public interface Options void setMergeFrequencyMinutes(Integer value); @TemplateParameter.Text( - groupName = "Target", order = 14, description = "Dead letter queue directory.", helpText = @@ -308,7 +300,6 @@ public interface Options void setDeadLetterQueueDirectory(String value); @TemplateParameter.Integer( - groupName = "Target", order = 15, optional = true, description = "The number of minutes between DLQ Retries.", @@ -319,7 +310,6 @@ public interface Options void setDlqRetryMinutes(Integer value); @TemplateParameter.Text( - groupName = "Source", order = 16, optional = true, description = "Datastream API Root URL (only required for testing)", @@ -330,7 +320,6 @@ public interface Options void setDataStreamRootUrl(String value); @TemplateParameter.Boolean( - groupName = "Target", order = 17, optional = true, description = "A switch to disable MERGE queries for the job.", @@ -341,7 +330,6 @@ public interface Options void setApplyMerge(Boolean value); @TemplateParameter.Integer( - groupName = "Target", order = 18, optional = true, parentName = "applyMerge", @@ -355,7 +343,6 @@ public interface Options void setMergeConcurrency(Integer value); @TemplateParameter.Integer( - groupName = "Target", order = 19, optional = true, description = "Partition retention days.", @@ -367,7 +354,6 @@ public interface Options void setPartitionRetentionDays(Integer value); @TemplateParameter.Boolean( - groupName = "Target", order = 20, optional = true, parentName = "useStorageWriteApi", diff --git a/v2/datastream-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToMongoDB.java b/v2/datastream-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToMongoDB.java index 423bf9d59c..f4f05fef60 100644 --- a/v2/datastream-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToMongoDB.java +++ b/v2/datastream-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToMongoDB.java @@ -98,7 +98,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 2, - groupName = "Source", enumOptions = {@TemplateEnumOption("avro"), @TemplateEnumOption("json")}, optional = false, description = "The GCS input format avro/json", @@ -128,7 +127,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.DateTime( order = 5, - groupName = "Source", optional = true, description = "The starting DateTime used to fetch from Cloud Storage " @@ -143,7 +141,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Integer( order = 6, - groupName = "Source", optional = true, description = "File read concurrency", helpText = "The number of concurrent DataStream files to read.") diff --git a/v2/datastream-to-postgres/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToPostgres.java b/v2/datastream-to-postgres/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToPostgres.java index cd5530e8a6..c7fce49a37 100644 --- a/v2/datastream-to-postgres/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToPostgres.java +++ b/v2/datastream-to-postgres/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToPostgres.java @@ -112,7 +112,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.DateTime( order = 2, - groupName = "Source", optional = true, description = "The starting DateTime used to fetch from Cloud Storage " @@ -127,7 +126,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 3, - groupName = "Source", optional = true, description = "Datastream API Root URL (only required for testing)", helpText = "Datastream API Root URL") @@ -159,7 +157,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 6, - groupName = "Target", description = "Database User to connect with.", helpText = "Database User to connect with.") @Default.String("postgres") @@ -169,7 +166,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Password( order = 11, - groupName = "Target", description = "Database Password for given user.", helpText = "Database Password for given user.") @Default.String("postgres") diff --git a/v2/datastream-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSpanner.java b/v2/datastream-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSpanner.java index 540137ab8f..8a8300da7d 100644 --- a/v2/datastream-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSpanner.java +++ b/v2/datastream-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSpanner.java @@ -144,7 +144,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 2, - groupName = "Source", enumOptions = {@TemplateEnumOption("avro"), @TemplateEnumOption("json")}, optional = true, description = "Datastream output file format (avro/json).", @@ -157,7 +156,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.GcsReadFile( order = 3, - groupName = "Target", optional = true, description = "Session File Path in Cloud Storage", helpText = @@ -209,7 +207,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.PubsubSubscription( order = 8, - groupName = "Source", optional = true, description = "The Pub/Sub subscription being used in a Cloud Storage notification policy.", helpText = @@ -232,7 +229,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 10, - groupName = "Target", optional = true, description = "Cloud Spanner shadow table prefix.", helpText = "The prefix used to name shadow tables. Default: `shadow_`.") @@ -243,7 +239,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Boolean( order = 11, - groupName = "Target", optional = true, description = "If true, create shadow tables in Cloud Spanner.", helpText = @@ -255,7 +250,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.DateTime( order = 12, - groupName = "Source", optional = true, description = "The starting DateTime used to fetch from Cloud Storage " @@ -270,7 +264,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Integer( order = 13, - groupName = "Source", optional = true, description = "File read concurrency", helpText = "The number of concurrent DataStream files to read.") @@ -281,7 +274,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 14, - groupName = "Target", optional = true, description = "Dead letter queue directory.", helpText = @@ -294,7 +286,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Integer( order = 15, - groupName = "Target", optional = true, description = "Dead letter queue retry minutes", helpText = "The number of minutes between dead letter queue retries. Defaults to 10.") @@ -305,7 +296,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Integer( order = 16, - groupName = "Target", optional = true, description = "Dead letter queue maximum retry count", helpText = @@ -318,7 +308,6 @@ public interface Options extends PipelineOptions, StreamingOptions { // DataStream API Root Url (only used for testing) @TemplateParameter.Text( order = 17, - groupName = "Source", optional = true, description = "Datastream API Root URL (only required for testing)", helpText = "Datastream API Root URL.") @@ -329,7 +318,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 18, - groupName = "Source", optional = true, description = "Datastream source type (only required for testing)", helpText = @@ -356,7 +344,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 20, - groupName = "Target", optional = true, description = "Run mode - currently supported are : regular or retryDLQ", enumOptions = {@TemplateEnumOption("regular"), @TemplateEnumOption("retryDLQ")}, @@ -368,7 +355,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.GcsReadFile( order = 21, - groupName = "Target", optional = true, helpText = "Transformation context file path in cloud storage used to populate data used in" @@ -381,7 +367,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Integer( order = 22, - groupName = "Target", optional = true, description = "Directory watch duration in minutes. Default: 10 minutes", helpText = @@ -397,7 +382,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 23, - groupName = "Target", enumOptions = { @TemplateEnumOption("LOW"), @TemplateEnumOption("MEDIUM"), @@ -415,7 +399,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.PubsubSubscription( order = 24, - groupName = "Target", optional = true, description = "The Pub/Sub subscription being used in a Cloud Storage notification policy for DLQ" diff --git a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java index d21d103278..9199a31ff9 100644 --- a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java +++ b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java @@ -109,7 +109,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.PubsubSubscription( order = 2, - groupName = "Source", optional = true, description = "The Pub/Sub subscription being used in a Cloud Storage notification policy.", helpText = @@ -121,7 +120,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 3, - groupName = "Source", enumOptions = {@TemplateEnumOption("avro"), @TemplateEnumOption("json")}, optional = true, description = "Datastream output file format (avro/json).", @@ -145,7 +143,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.DateTime( order = 5, - groupName = "Source", optional = true, description = "The starting DateTime used to fetch from Cloud Storage " @@ -161,7 +158,6 @@ public interface Options extends PipelineOptions, StreamingOptions { // DataStream API Root Url (only used for testing) @TemplateParameter.Text( order = 6, - groupName = "Source", optional = true, description = "Datastream API Root URL (only required for testing)", helpText = "Datastream API Root URL") @@ -173,7 +169,6 @@ public interface Options extends PipelineOptions, StreamingOptions { // SQL Connection Parameters @TemplateParameter.Enum( order = 7, - groupName = "Target", optional = true, enumOptions = {@TemplateEnumOption("postgres"), @TemplateEnumOption("mysql")}, description = "SQL Database Type (postgres or mysql).", @@ -205,7 +200,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 10, - groupName = "Target", description = "Database User to connect with.", helpText = "The SQL user with all required permissions to write to all tables in replication.") @@ -215,7 +209,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Password( order = 11, - groupName = "Target", description = "Database Password for given user.", helpText = "The password for the SQL user.") String getDatabasePassword(); @@ -235,7 +228,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 13, - groupName = "Target", optional = true, description = "A map of key/values used to dictate schema name changes", helpText = diff --git a/v2/file-format-conversion/src/main/java/com/google/cloud/teleport/v2/templates/FileFormatConversion.java b/v2/file-format-conversion/src/main/java/com/google/cloud/teleport/v2/templates/FileFormatConversion.java index f9b5ce53a4..02c17dd33a 100644 --- a/v2/file-format-conversion/src/main/java/com/google/cloud/teleport/v2/templates/FileFormatConversion.java +++ b/v2/file-format-conversion/src/main/java/com/google/cloud/teleport/v2/templates/FileFormatConversion.java @@ -90,7 +90,6 @@ public interface FileFormatConversionOptions extends PipelineOptions, CsvPipelineOptions, AvroOptions, ParquetOptions { @TemplateParameter.Enum( order = 1, - groupName = "Source", enumOptions = { @TemplateEnumOption("avro"), @TemplateEnumOption("csv"), @@ -105,7 +104,6 @@ public interface FileFormatConversionOptions @TemplateParameter.Enum( order = 2, - groupName = "Target", enumOptions = {@TemplateEnumOption("avro"), @TemplateEnumOption("parquet")}, description = "File format of the output files.", helpText = "File format of the output files. Needs to be either avro or parquet.") diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/GCSToSourceDb.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/GCSToSourceDb.java index c0d6d7bf3f..7385c5cc45 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/GCSToSourceDb.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/GCSToSourceDb.java @@ -78,7 +78,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.GcsReadFile( order = 1, - groupName = "Source", optional = false, description = "Source shard details file path in Cloud Storage", helpText = @@ -90,7 +89,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.GcsReadFile( order = 2, - groupName = "Source", optional = false, description = "Session File Path in Cloud Storage", helpText = @@ -102,7 +100,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Enum( order = 3, - groupName = "Target", optional = true, description = "Destination source type", enumOptions = {@TemplateEnumOption("mysql")}, @@ -114,7 +111,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 4, - groupName = "Target", optional = true, description = "SourceDB timezone offset", helpText = @@ -181,7 +177,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.ProjectId( order = 9, - groupName = "Source", optional = false, description = "Cloud Spanner Project Id.", helpText = "This is the name of the Cloud Spanner project.") @@ -191,7 +186,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 10, - groupName = "Source", optional = false, description = "Cloud Spanner Instance to store the shard progress when reading from gcs", helpText = "This is the instance to store the shard progress of the files processed.") @@ -201,7 +195,6 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 11, - groupName = "Source", optional = false, description = "Cloud Spanner Database to store the shard progress when reading from gcs", helpText = "This is the database to store the shard progress of the files processed..") diff --git a/v2/google-ads-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/GoogleAdsToBigQuery.java b/v2/google-ads-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/GoogleAdsToBigQuery.java index 8907adc252..c5181047bb 100644 --- a/v2/google-ads-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/GoogleAdsToBigQuery.java +++ b/v2/google-ads-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/GoogleAdsToBigQuery.java @@ -77,7 +77,6 @@ public final class GoogleAdsToBigQuery { public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptions { @TemplateParameter.Long( order = 1, - groupName = "Source", optional = true, description = "Google Ads manager account ID", helpText = "A Google Ads manager account ID to use to access the account IDs.", @@ -88,7 +87,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Text( order = 2, - groupName = "Source", regexes = {"^[0-9]+(,[0-9]+)*$"}, description = "Google Ads account IDs", helpText = "A list of Google Ads account IDs to use to execute the query.", @@ -100,7 +98,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Text( order = 3, - groupName = "Source", description = "Google Ads Query Language query", helpText = "The query to use to get the data. See Google Ads Query Language. For example: `SELECT campaign.id, campaign.name FROM campaign`.", @@ -112,7 +109,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Double( order = 4, - groupName = "Source", description = "Required Google Ads request rate per worker", helpText = "The rate of query requests per second (QPS) to submit to Google Ads. " @@ -125,7 +121,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.GcsReadFile( order = 5, - groupName = "Target", optional = true, description = "BigQuery Table Schema Path", helpText = @@ -139,7 +134,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Text( order = 6, - groupName = "Source", description = "OAuth 2.0 Client ID identifying the application", helpText = "The OAuth 2.0 client ID that identifies the application. See Create a client ID and client secret (https://developers.google.com/google-ads/api/docs/oauth/cloud-project#create_a_client_id_and_client_secret).") @@ -159,7 +153,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Password( order = 8, - groupName = "Source", description = "OAuth 2.0 Refresh Token for the user connecting to the Google Ads API", helpText = "The OAuth 2.0 refresh token to use to connect to the Google Ads API. See 2-Step Verification (https://developers.google.com/google-ads/api/docs/oauth/2sv).") @@ -169,7 +162,6 @@ public interface GoogleAdsToBigQueryOptions extends WriteOptions, GoogleAdsOptio @TemplateParameter.Password( order = 9, - groupName = "Source", description = "Google Ads developer token for the user connecting to the Google Ads API", helpText = "The Google Ads developer token to use to connect to the Google Ads API. See Obtain a developer token (https://developers.google.com/google-ads/api/docs/get-started/dev-token).") diff --git a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java index 9d269673dd..b5d1e728db 100644 --- a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java +++ b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java @@ -32,7 +32,6 @@ public interface GCSToElasticsearchOptions @TemplateParameter.BigQueryTable( order = 1, - groupName = "Target", description = "BigQuery Deadletter table to send failed inserts.", helpText = "The BigQuery dead-letter table to send failed inserts to.", example = "your-project:your-dataset.your-table-name") @@ -43,7 +42,6 @@ public interface GCSToElasticsearchOptions @TemplateParameter.Text( order = 2, - groupName = "Source", optional = true, regexes = {"[a-zA-Z0-9._-]+"}, description = "Input file format", diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamToBigQueryOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamToBigQueryOptions.java index baf214ebb5..81717246b1 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamToBigQueryOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamToBigQueryOptions.java @@ -40,7 +40,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Boolean( order = 2, - groupName = "Target", optional = true, description = "Write rowkeys as BigQuery BYTES", helpText = @@ -53,7 +52,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Boolean( order = 3, - groupName = "Target", optional = true, description = "Write values as BigQuery BYTES", helpText = @@ -66,7 +64,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Boolean( order = 4, - groupName = "Target", optional = true, description = "Write Bigtable timestamp as BigQuery INT", helpText = @@ -92,7 +89,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Text( order = 6, - groupName = "Target", optional = true, description = "BigQuery changelog table name", helpText = @@ -105,7 +101,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Text( order = 7, - groupName = "Target", optional = true, description = "Changelog table will be partitioned at specified granularity", helpText = @@ -119,7 +114,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Long( order = 8, - groupName = "Target", optional = true, description = "Sets partition expiration time in milliseconds", helpText = @@ -132,7 +126,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.Text( order = 9, - groupName = "Target", optional = true, description = "Optional changelog table columns to be disabled", helpText = @@ -146,7 +139,6 @@ public interface BigtableChangeStreamToBigQueryOptions @TemplateParameter.GcsWriteFolder( order = 10, - groupName = "Source", optional = true, description = "Dead letter queue directory", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToGcsOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToGcsOptions.java index 504d527560..77ed3761e0 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToGcsOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToGcsOptions.java @@ -29,7 +29,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Enum( order = 1, - groupName = "Target", enumOptions = {@TemplateEnumOption("TEXT"), @TemplateEnumOption("AVRO")}, optional = true, description = "Output file format", @@ -42,7 +41,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Duration( order = 2, - groupName = "Target", optional = true, description = "Window duration", helpText = @@ -56,7 +54,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Text( order = 3, - groupName = "Source", optional = true, description = "Bigtable Metadata Table Id", helpText = "Table ID used for creating the metadata table.") @@ -66,7 +63,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Enum( order = 4, - groupName = "Target", enumOptions = {@TemplateEnumOption("CHANGELOG_ENTRY"), @TemplateEnumOption("BIGTABLE_ROW")}, optional = true, description = "Output schema format", @@ -93,7 +89,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Text( order = 6, - groupName = "Target", optional = true, description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file. Defaults to \"changelog-\"", @@ -105,7 +100,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Integer( order = 7, - groupName = "Target", optional = true, description = "Number of output file shards", helpText = @@ -120,7 +114,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Integer( order = 7, - groupName = "Target", optional = true, description = "Maximum number of mutations in a batch", helpText = @@ -134,7 +127,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Boolean( order = 8, - groupName = "Target", optional = true, parentName = "outputFileFormat", parentTriggerValues = {"TEXT"}, @@ -149,7 +141,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Boolean( order = 9, - groupName = "Target", optional = true, parentName = "outputFileFormat", parentTriggerValues = {"TEXT"}, @@ -164,7 +155,6 @@ public interface BigtableChangeStreamsToGcsOptions @TemplateParameter.Boolean( order = 10, - groupName = "Target", optional = true, parentName = "outputFileFormat", parentTriggerValues = {"TEXT"}, diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToPubSubOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToPubSubOptions.java index ffd1206642..d4e7b519d1 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToPubSubOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/BigtableChangeStreamsToPubSubOptions.java @@ -43,7 +43,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Enum( order = 2, - groupName = "Target", enumOptions = { @TemplateEnumOption("BINARY"), @TemplateEnumOption("JSON"), @@ -60,7 +59,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Enum( order = 3, - groupName = "Target", enumOptions = { @TemplateEnumOption("AVRO"), @TemplateEnumOption("PROTOCOL_BUFFERS"), @@ -79,7 +77,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Boolean( order = 4, - groupName = "Source", optional = true, description = "Strip values for SetCell mutation", helpText = @@ -91,7 +88,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.GcsWriteFolder( order = 5, - groupName = "Source", optional = true, description = "Dead letter queue directory to store any unpublished change record.", helpText = @@ -104,7 +100,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Integer( order = 6, - groupName = "Source", optional = true, description = "Dead letter queue retry minutes", helpText = "The number of minutes between dead-letter queue retries. Defaults to `10`.") @@ -115,7 +110,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Integer( order = 7, - groupName = "Source", optional = true, description = "Dead letter maximum retries", helpText = "The dead letter maximum retries. Defaults to `5`.") @@ -137,7 +131,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Boolean( order = 8, - groupName = "Source", optional = true, parentName = "messageFormat", parentTriggerValues = {"JSON"}, @@ -152,7 +145,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Boolean( order = 9, - groupName = "Source", optional = true, parentName = "messageFormat", parentTriggerValues = {"JSON"}, @@ -167,7 +159,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Boolean( order = 10, - groupName = "Source", optional = true, parentName = "messageFormat", parentTriggerValues = {"JSON"}, @@ -182,7 +173,6 @@ public interface BigtableChangeStreamsToPubSubOptions @TemplateParameter.Boolean( order = 11, - groupName = "Source", optional = true, description = "Whether or not to disable retries for the DLQ", helpText = "Whether or not to disable retries for the DLQ") diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/DLPTextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/DLPTextToBigQueryStreaming.java index f9f7a2552a..87172f7f4b 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/DLPTextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/DLPTextToBigQueryStreaming.java @@ -302,7 +302,6 @@ public interface TokenizePipelineOptions @TemplateParameter.Text( order = 2, - groupName = "Source", regexes = { "^projects\\/[^\\n" + "\\r" @@ -325,7 +324,6 @@ public interface TokenizePipelineOptions @TemplateParameter.Text( order = 3, - groupName = "Source", optional = true, regexes = { "^projects\\/[^\\n" @@ -346,7 +344,6 @@ public interface TokenizePipelineOptions @TemplateParameter.Integer( order = 4, - groupName = "Source", optional = true, description = "Batch size", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/FirestoreToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/FirestoreToBigQuery.java index 5ebc0949de..be5595e4ac 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/FirestoreToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/FirestoreToBigQuery.java @@ -106,7 +106,6 @@ public interface FirestoreToBigQueryOptions @TemplateParameter.GcsWriteFolder( order = 2, - groupName = "Target", description = "Temporary directory for BigQuery loading process", helpText = "Temporary directory for BigQuery loading process", example = "gs://your-bucket/your-files/temp_dir") @@ -117,7 +116,6 @@ public interface FirestoreToBigQueryOptions @TemplateParameter.GcsReadFile( order = 3, - groupName = "Target", optional = true, description = "Cloud Storage path to BigQuery JSON schema", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/utils/FirestoreConverters.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/utils/FirestoreConverters.java index 2b13111041..63d256b70c 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/utils/FirestoreConverters.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/utils/FirestoreConverters.java @@ -72,7 +72,6 @@ public interface FirestoreReadOptions extends PipelineOptions { */ @TemplateParameter.Text( order = 1, - groupName = "Source", regexes = {"^.+$"}, description = "GQL Query", helpText = "Specifies which Datastore entities to read. Ex: ‘SELECT * FROM MyKind’") @@ -129,7 +128,6 @@ public interface FirestoreReadOptions extends PipelineOptions { @TemplateParameter.Text( order = 4, - groupName = "Source", regexes = {"^.+$"}, description = "GQL Query", helpText = "Specifies which Firestore entities to read. Ex: ‘SELECT * FROM MyKind’") @@ -148,7 +146,6 @@ public interface FirestoreReadOptions extends PipelineOptions { @TemplateParameter.Text( order = 6, - groupName = "Source", optional = true, regexes = {"^[0-9A-Za-z._-]{0,100}$"}, description = "Namespace", diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java index a32de53ef1..052d68e727 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToBigQueryOptions.java @@ -26,7 +26,6 @@ public interface JdbcToBigQueryOptions order = 1, optional = false, regexes = {"^.+$"}, - groupName = "Source", description = "Comma-separated Cloud Storage path(s) of the JDBC driver(s)", helpText = "The comma-separated list of driver JAR files.", example = "gs://your-bucket/driver_jar1.jar,gs://your-bucket/driver_jar2.jar") @@ -38,7 +37,6 @@ public interface JdbcToBigQueryOptions order = 2, optional = false, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC driver class name", helpText = "The JDBC driver class name.", example = "com.mysql.jdbc.Driver") @@ -65,7 +63,6 @@ public interface JdbcToBigQueryOptions order = 4, optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, - groupName = "Source", description = "JDBC connection property string.", helpText = "The properties string to use for the JDBC connection. The format of the string must " @@ -82,7 +79,6 @@ public interface JdbcToBigQueryOptions order = 5, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC connection username.", helpText = "The username to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. Remove whitespace characters from the Base64-encoded string.") @@ -93,7 +89,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Password( order = 6, optional = true, - groupName = "Source", description = "JDBC connection password.", helpText = "The password to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. Remove whitespace characters from the Base64-encoded string.") @@ -105,7 +100,6 @@ public interface JdbcToBigQueryOptions order = 7, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC source SQL query", helpText = "The query to run on the source to extract the data. Note that some JDBC SQL and BigQuery types, although sharing the same name, have some differences. " @@ -130,7 +124,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.GcsWriteFolder( order = 9, optional = false, - groupName = "Target", description = "Temporary directory for BigQuery loading process", helpText = "The temporary directory for the BigQuery loading process.", example = "gs://your-bucket/your-files/temp_dir") @@ -141,7 +134,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.KmsEncryptionKey( order = 10, optional = true, - groupName = "Source", description = "Google Cloud KMS key", helpText = "The Cloud KMS encryption key to use to decrypt the username, password, and connection string. If you " @@ -154,7 +146,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Boolean( order = 11, optional = true, - groupName = "Source", description = "Whether to use column alias to map the rows.", helpText = "If set to `true`, the pipeline uses the column alias (`AS`) instead of the column name to map the rows to BigQuery. Defaults to `false`.") @@ -166,7 +157,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Boolean( order = 12, optional = true, - groupName = "Target", description = "Whether to truncate data before writing", helpText = "If set to `true`, the pipeline truncates before loading data into BigQuery. Defaults to `false`, which causes the pipeline to append data.") @@ -178,7 +168,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Text( order = 13, optional = true, - groupName = "Source", description = "The name of a column of numeric type that will be used for partitioning.", helpText = "If this parameter is provided with the name of the `table` defined as an optional parameter, JdbcIO reads the table in parallel by executing multiple instances of the query on the same table (subquery) using ranges. Currently, only supports `Long` partition columns.") @@ -189,7 +178,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Text( order = 14, optional = true, - groupName = "Source", description = "Name of the table in the external database.", helpText = "The table to read from when using partitions. This parameter also accepts a subquery in parentheses.", @@ -201,7 +189,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Integer( order = 15, optional = true, - groupName = "Source", description = "The number of partitions.", helpText = "The number of partitions. With the lower and upper bound, this value forms partition strides for generated `WHERE` clause expressions that are used to split the partition column evenly. When the input is less than `1`, the number is set to `1`.") @@ -212,7 +199,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Long( order = 16, optional = true, - groupName = "Source", description = "Lower bound of partition column.", helpText = "The lower bound to use in the partition scheme. If not provided, this value is automatically inferred by Apache Beam for the supported types.") @@ -223,7 +209,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Long( order = 17, optional = true, - groupName = "Source", description = "Upper bound of partition column", helpText = "The upper bound to use in the partition scheme. If not provided, this value is automatically inferred by Apache Beam for the supported types.") @@ -234,7 +219,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Integer( order = 18, optional = true, - groupName = "Source", description = "Fetch Size", // TODO: remove the "Not used for partitioned reads" once // https://github.com/apache/beam/pull/28999 is released. @@ -247,7 +231,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.Enum( order = 19, - groupName = "Target", enumOptions = { @TemplateParameter.TemplateEnumOption("CREATE_IF_NEEDED"), @TemplateParameter.TemplateEnumOption("CREATE_NEVER") @@ -263,7 +246,6 @@ public interface JdbcToBigQueryOptions @TemplateParameter.GcsReadFile( order = 20, - groupName = "Target", optional = true, description = "Cloud Storage path to BigQuery JSON schema", helpText = diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java index a535deb27d..5b52b066a3 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java @@ -25,7 +25,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 1, - groupName = "Source", optional = false, regexes = {"^.+$"}, description = "JDBC driver class name.", @@ -52,7 +51,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 3, - groupName = "Source", optional = true, regexes = {"^.+$"}, description = "JDBC connection username.", @@ -64,7 +62,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Password( order = 4, - groupName = "Source", optional = true, description = "JDBC connection password.", helpText = @@ -75,7 +72,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 5, - groupName = "Source", optional = false, regexes = {"^.+$"}, description = "Cloud Storage paths for JDBC drivers", @@ -87,7 +83,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 6, - groupName = "Source", optional = true, regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, description = "JDBC connection property string.", @@ -100,7 +95,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 7, - groupName = "Source", optional = false, regexes = {"^.+$"}, description = "JDBC source SQL query.", @@ -124,7 +118,6 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { @TemplateParameter.KmsEncryptionKey( order = 9, - groupName = "Source", optional = true, description = "Google Cloud KMS key", helpText = diff --git a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java index 2c347869bb..0c83a8292f 100644 --- a/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java +++ b/v2/jms-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/JmsToPubsub.java @@ -164,7 +164,6 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, - groupName = "Source", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "JMS Destination Type to read the input from", @@ -189,7 +188,6 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 5, - groupName = "Source", description = "JMS Username", helpText = "The username to use for authentication on the JMS server.", example = "sampleusername") @@ -199,7 +197,6 @@ public interface JmsToPubsubOptions extends PipelineOptions { @TemplateParameter.Password( order = 6, - groupName = "Source", description = "JMS Password", helpText = "The password associated with the provided username.", example = "samplepassword") diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index fa660a3c34..04a75c6d66 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -241,7 +241,6 @@ public interface KafkaToBQOptions @TemplateParameter.BigQueryTable( order = 4, - groupName = "Target", optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = @@ -255,7 +254,6 @@ public interface KafkaToBQOptions @TemplateParameter.Enum( order = 5, - groupName = "Target", enumOptions = { @TemplateParameter.TemplateEnumOption("AVRO"), @TemplateParameter.TemplateEnumOption("JSON") @@ -270,7 +268,6 @@ public interface KafkaToBQOptions @TemplateParameter.GcsReadFile( order = 6, - groupName = "Target", optional = true, description = "Cloud Storage path to the Avro schema file", helpText = "Cloud Storage path to Avro schema file. For example, gs://MyBucket/file.avsc.") @@ -280,7 +277,6 @@ public interface KafkaToBQOptions @TemplateParameter.Boolean( order = 7, - groupName = "Target", optional = true, parentName = "useStorageWriteApi", parentTriggerValues = {"true"}, diff --git a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java index 4d9c98bb39..0fe4ba9b53 100644 --- a/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java +++ b/v2/kafka-to-gcs/src/main/java/com/google/cloud/teleport/v2/options/KafkaToGCSOptions.java @@ -62,7 +62,6 @@ public interface KafkaToGCSOptions @TemplateParameter.Enum( order = 3, - groupName = "Target", enumOptions = { @TemplateEnumOption("TEXT"), @TemplateEnumOption("AVRO"), @@ -79,7 +78,6 @@ public interface KafkaToGCSOptions @TemplateParameter.Duration( order = 4, - groupName = "Target", optional = true, description = "Window duration", helpText = From 7077ee674ba92bb21c302eb8a8340e8eb894aaf2 Mon Sep 17 00:00:00 2001 From: "vitaly.terentyev" Date: Thu, 16 May 2024 17:41:34 +0400 Subject: [PATCH 3/3] Add group name for common options --- .../teleport/v2/bigtable/options/BigtableCommonOptions.java | 5 +++++ .../google/cloud/teleport/v2/transforms/WriteToGCSAvro.java | 2 ++ .../cloud/teleport/v2/transforms/WriteToGCSParquet.java | 2 ++ .../google/cloud/teleport/v2/transforms/WriteToGCSText.java | 2 ++ 4 files changed, 11 insertions(+) diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index 949543c75a..d8e02f542c 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -146,6 +146,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Source Bigtable Instance ID", helpText = "The source Bigtable instance ID.") @@ -156,6 +157,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Source Cloud Bigtable table ID", helpText = "The source Bigtable table ID.") @Validation.Required @@ -166,6 +168,7 @@ interface ReadOptions extends BigtableCommonOptions { @TemplateParameter.ProjectId( order = 3, optional = true, + groupName = "Source", description = "Source Cloud Bigtable Project ID", helpText = "The Bigtable project ID. The default is the project for the Dataflow job.") @Default.String("") @@ -192,6 +195,7 @@ interface ReadChangeStreamOptions extends BigtableCommonOptions.ReadOptions { @TemplateParameter.Text( order = 1, optional = true, + groupName = "Source", description = "Cloud Bigtable change streams metadata instance ID", helpText = "The Bigtable change streams metadata instance ID.") @Default.String("") @@ -202,6 +206,7 @@ interface ReadChangeStreamOptions extends BigtableCommonOptions.ReadOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Source", description = "Cloud Bigtable change streams metadata table ID", helpText = "The ID of the Bigtable change streams connector metadata table. If not " diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java index 74e87418ad..0d5a41dac2 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSAvro.java @@ -102,6 +102,7 @@ public interface WriteToGCSAvroOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -112,6 +113,7 @@ public interface WriteToGCSAvroOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java index 568e3f53bb..5f231bbd4e 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSParquet.java @@ -92,6 +92,7 @@ public interface WriteToGCSParquetOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -102,6 +103,7 @@ public interface WriteToGCSParquetOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java index cf19a6c27e..56f7ff1eb9 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/WriteToGCSText.java @@ -107,6 +107,7 @@ public interface WriteToGCSTextOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash.", example = "gs://your-bucket/your-path") @@ -117,6 +118,7 @@ public interface WriteToGCSTextOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-")