From c59c51b4aba976994834247f1886eb96144ee2c9 Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 14 Mar 2024 11:51:40 +0300 Subject: [PATCH] Add mandatory SLULA license acceptance flag Since introducing a new license we need to explicitly check if the user has accepted the terms. Therefore a new flag is added. By default it is set to false but can be overrided by either: - setting `license.accept = true` in the config file - setting `env ACCEPT_LIMITED_USE_LICENSE=true` - appending `-Dlicense.accept=true` --- .../aws/databricks.config.minimal.hocon | 3 ++ .../aws/databricks.config.reference.hocon | 5 +++ .../loader/aws/redshift.config.minimal.hocon | 3 ++ .../aws/redshift.config.reference.hocon | 5 +++ .../loader/aws/snowflake.config.minimal.hocon | 3 ++ .../aws/snowflake.config.reference.hocon | 5 +++ .../azure/databricks.config.minimal.hocon | 3 ++ .../azure/databricks.config.reference.hocon | 4 +++ .../azure/snowflake.config.minimal.hocon | 3 ++ .../azure/snowflake.config.reference.hocon | 4 +++ .../gcp/databricks.config.minimal.hocon | 3 ++ .../gcp/databricks.config.reference.hocon | 5 +++ .../loader/gcp/snowflake.config.minimal.hocon | 3 ++ .../gcp/snowflake.config.reference.hocon | 5 +++ .../transformer.batch.config.minimal.hocon | 3 ++ .../transformer.batch.config.reference.hocon | 5 +++ .../transformer.kinesis.config.minimal.hocon | 3 ++ ...transformer.kinesis.config.reference.hocon | 5 +++ .../transformer.kafka.config.minimal.hocon | 3 ++ .../transformer.kafka.config.reference.hocon | 5 +++ .../transformer.pubsub.config.minimal.hocon | 3 ++ .../transformer.pubsub.config.reference.hocon | 5 +++ .../src/main/resources/application.conf | 4 +++ .../transformer/stream/common/Config.scala | 5 +-- .../transformer/stream/common/Run.scala | 7 +++- .../rdbloader/common/config/License.scala | 32 +++++++++++++++++++ .../loader/databricks/ConfigSpec.scala | 18 +++++++---- .../loader/databricks/DatabricksSpec.scala | 3 +- .../src/main/resources/application.conf | 4 +++ .../snowplow/rdbloader/Runner.scala | 2 ++ .../snowplow/rdbloader/config/Config.scala | 5 +-- .../snowplow/rdbloader/ConfigSpec.scala | 6 ++-- .../snowplow/rdbloader/SpecHelpers.scala | 3 +- .../snowplow/loader/redshift/ConfigSpec.scala | 6 ++-- .../loader/snowflake/ConfigSpec.scala | 18 +++++++---- .../src/main/resources/application.conf | 5 +++ .../rdbloader/transformer/batch/Config.scala | 5 +-- .../rdbloader/transformer/batch/Main.scala | 6 +++- .../transformer/batch/ConfigSpec.scala | 9 ++++-- .../transformer/batch/ShredJobSpec.scala | 5 ++- .../stream/kafka/AzuriteBasedDevApp.scala | 1 + .../transformer/stream/kafka/ConfigSpec.scala | 9 ++++-- .../stream/kinesis/ConfigSpec.scala | 9 ++++-- .../stream/pubsub/ConfigSpec.scala | 9 ++++-- 44 files changed, 218 insertions(+), 39 deletions(-) create mode 100644 modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala diff --git a/config/loader/aws/databricks.config.minimal.hocon b/config/loader/aws/databricks.config.minimal.hocon index ea78e7c0c..794e8ba7f 100644 --- a/config/loader/aws/databricks.config.minimal.hocon +++ b/config/loader/aws/databricks.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "messageQueue": "test-queue", "storage" : { "host": "abc.cloud.databricks.com" diff --git a/config/loader/aws/databricks.config.reference.hocon b/config/loader/aws/databricks.config.reference.hocon index f2b4abf54..177fa66e0 100644 --- a/config/loader/aws/databricks.config.reference.hocon +++ b/config/loader/aws/databricks.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Data Lake (S3) region # This field is optional if it can be resolved with AWS region provider chain. # It checks places like env variables, system properties, AWS profile file. diff --git a/config/loader/aws/redshift.config.minimal.hocon b/config/loader/aws/redshift.config.minimal.hocon index 105f56f74..edf369947 100644 --- a/config/loader/aws/redshift.config.minimal.hocon +++ b/config/loader/aws/redshift.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "messageQueue": "test-queue" "storage" : { "type": "redshift" diff --git a/config/loader/aws/redshift.config.reference.hocon b/config/loader/aws/redshift.config.reference.hocon index f2872396a..d04b3a2bf 100644 --- a/config/loader/aws/redshift.config.reference.hocon +++ b/config/loader/aws/redshift.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Specifies the cloud provider that application will be deployed into "cloud": "aws" # Data Lake (S3) region diff --git a/config/loader/aws/snowflake.config.minimal.hocon b/config/loader/aws/snowflake.config.minimal.hocon index 4284f9629..62fdd3bd6 100644 --- a/config/loader/aws/snowflake.config.minimal.hocon +++ b/config/loader/aws/snowflake.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "messageQueue": "test-queue", "storage" : { "type": "snowflake", diff --git a/config/loader/aws/snowflake.config.reference.hocon b/config/loader/aws/snowflake.config.reference.hocon index c78c9f93c..a36d5a8ad 100644 --- a/config/loader/aws/snowflake.config.reference.hocon +++ b/config/loader/aws/snowflake.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Data Lake (S3) region # This field is optional if it can be resolved with AWS region provider chain. # It checks places like env variables, system properties, AWS profile file. diff --git a/config/loader/azure/databricks.config.minimal.hocon b/config/loader/azure/databricks.config.minimal.hocon index a1893a850..9a1968a1f 100644 --- a/config/loader/azure/databricks.config.minimal.hocon +++ b/config/loader/azure/databricks.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name" "messageQueue": { "type": "kafka" diff --git a/config/loader/azure/databricks.config.reference.hocon b/config/loader/azure/databricks.config.reference.hocon index 8ac5e1f1a..8b45a39a6 100644 --- a/config/loader/azure/databricks.config.reference.hocon +++ b/config/loader/azure/databricks.config.reference.hocon @@ -1,4 +1,8 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } # Azure Blob Storage endpoint, should contain container with transformer's output "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name" diff --git a/config/loader/azure/snowflake.config.minimal.hocon b/config/loader/azure/snowflake.config.minimal.hocon index 2adb6b0c3..aa4b6b020 100644 --- a/config/loader/azure/snowflake.config.minimal.hocon +++ b/config/loader/azure/snowflake.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name" "messageQueue": { "type": "kafka" diff --git a/config/loader/azure/snowflake.config.reference.hocon b/config/loader/azure/snowflake.config.reference.hocon index 7780dcfe8..3b2b2b269 100644 --- a/config/loader/azure/snowflake.config.reference.hocon +++ b/config/loader/azure/snowflake.config.reference.hocon @@ -1,4 +1,8 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } # Azure Blob Storage endpoint, should contain container with transformer's output "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name" diff --git a/config/loader/gcp/databricks.config.minimal.hocon b/config/loader/gcp/databricks.config.minimal.hocon index becb70d27..73dba0d9c 100644 --- a/config/loader/gcp/databricks.config.minimal.hocon +++ b/config/loader/gcp/databricks.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "messageQueue": { "type": "pubsub" "subscription": "projects/project-id/subscriptions/subscription-id" diff --git a/config/loader/gcp/databricks.config.reference.hocon b/config/loader/gcp/databricks.config.reference.hocon index 5169934fa..c6fbd4f83 100644 --- a/config/loader/gcp/databricks.config.reference.hocon +++ b/config/loader/gcp/databricks.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Pubsub subscription used by Transformer and Loader to communicate "messageQueue": { "type": "pubsub" diff --git a/config/loader/gcp/snowflake.config.minimal.hocon b/config/loader/gcp/snowflake.config.minimal.hocon index 551f94feb..49c8deea9 100644 --- a/config/loader/gcp/snowflake.config.minimal.hocon +++ b/config/loader/gcp/snowflake.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "messageQueue": { "type": "pubsub" "subscription": "projects/project-id/subscriptions/subscription-id" diff --git a/config/loader/gcp/snowflake.config.reference.hocon b/config/loader/gcp/snowflake.config.reference.hocon index 208b71255..9cb67609c 100644 --- a/config/loader/gcp/snowflake.config.reference.hocon +++ b/config/loader/gcp/snowflake.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Pubsub subscription used by Transformer and Loader to communicate "messageQueue": { "type": "pubsub" diff --git a/config/transformer/aws/transformer.batch.config.minimal.hocon b/config/transformer/aws/transformer.batch.config.minimal.hocon index dda79b114..9a35fed79 100644 --- a/config/transformer/aws/transformer.batch.config.minimal.hocon +++ b/config/transformer/aws/transformer.batch.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "input": "s3://bucket/input/" "output": { "path": "s3://bucket/transformed/" diff --git a/config/transformer/aws/transformer.batch.config.reference.hocon b/config/transformer/aws/transformer.batch.config.reference.hocon index f15a31f96..63126ea09 100644 --- a/config/transformer/aws/transformer.batch.config.reference.hocon +++ b/config/transformer/aws/transformer.batch.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + # Path to enriched archive (must be populated separately with run=YYYY-MM-DD-hh-mm-ss-UUID directories) for S3 input "input": "s3://bucket/input/", diff --git a/config/transformer/aws/transformer.kinesis.config.minimal.hocon b/config/transformer/aws/transformer.kinesis.config.minimal.hocon index c7d0f2557..f0654083c 100644 --- a/config/transformer/aws/transformer.kinesis.config.minimal.hocon +++ b/config/transformer/aws/transformer.kinesis.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "input": { "streamName": "enriched-events" } diff --git a/config/transformer/aws/transformer.kinesis.config.reference.hocon b/config/transformer/aws/transformer.kinesis.config.reference.hocon index ba1043a91..0d9cb9871 100644 --- a/config/transformer/aws/transformer.kinesis.config.reference.hocon +++ b/config/transformer/aws/transformer.kinesis.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + "input": { # kinesis and file are the only options for transformer-kinesis # Optional, default value kinesis diff --git a/config/transformer/azure/transformer.kafka.config.minimal.hocon b/config/transformer/azure/transformer.kafka.config.minimal.hocon index f1c944be4..d1e3c350c 100644 --- a/config/transformer/azure/transformer.kafka.config.minimal.hocon +++ b/config/transformer/azure/transformer.kafka.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "input": { "topicName": "enriched" "bootstrapServers": "localhost:9092" diff --git a/config/transformer/azure/transformer.kafka.config.reference.hocon b/config/transformer/azure/transformer.kafka.config.reference.hocon index 96a92f4e6..23b514c04 100644 --- a/config/transformer/azure/transformer.kafka.config.reference.hocon +++ b/config/transformer/azure/transformer.kafka.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + "input": { "type": "kafka" diff --git a/config/transformer/gcp/transformer.pubsub.config.minimal.hocon b/config/transformer/gcp/transformer.pubsub.config.minimal.hocon index be976a3c4..0b60c0384 100644 --- a/config/transformer/gcp/transformer.pubsub.config.minimal.hocon +++ b/config/transformer/gcp/transformer.pubsub.config.minimal.hocon @@ -1,4 +1,7 @@ { + "license": { + "accept": true + } "input": { "subscription": "projects/project-id/subscriptions/subscription-id" } diff --git a/config/transformer/gcp/transformer.pubsub.config.reference.hocon b/config/transformer/gcp/transformer.pubsub.config.reference.hocon index 8efc09711..6ae79d1c1 100644 --- a/config/transformer/gcp/transformer.pubsub.config.reference.hocon +++ b/config/transformer/gcp/transformer.pubsub.config.reference.hocon @@ -1,4 +1,9 @@ { + # Full license text available in SNOWPLOW-LICENSE.md + "license": { + "accept": true + } + "input": { # Name of the Pubsub subscription with the enriched events "subscription": "projects/project-id/subscriptions/subscription-id" diff --git a/modules/common-transformer-stream/src/main/resources/application.conf b/modules/common-transformer-stream/src/main/resources/application.conf index 15a70dae7..06ea5d68e 100644 --- a/modules/common-transformer-stream/src/main/resources/application.conf +++ b/modules/common-transformer-stream/src/main/resources/application.conf @@ -1,4 +1,8 @@ "snowplow": { + "license": { + "accept": false + "accept": ${?ACCEPT_LIMITED_USE_LICENSE} + } "output": { "compression": "GZIP", diff --git a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala index 997089bf8..76e3961c5 100644 --- a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala +++ b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala @@ -24,7 +24,7 @@ import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Tran import com.snowplowanalytics.snowplow.rdbloader.common.config.implicits._ import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Compression import com.snowplowanalytics.snowplow.rdbloader.common.config.{Kinesis => AWSKinesis} -import com.snowplowanalytics.snowplow.rdbloader.common.config.Region +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, Region} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config.Output.Bad final case class Config( @@ -36,7 +36,8 @@ final case class Config( monitoring: Config.Monitoring, telemetry: Telemetry.Config, featureFlags: TransformerConfig.FeatureFlags, - validations: TransformerConfig.Validations + validations: TransformerConfig.Validations, + license: License ) object Config { diff --git a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala index 3f16523f4..7e1089555 100644 --- a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala +++ b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala @@ -13,6 +13,7 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common import org.typelevel.log4cats.slf4j.Slf4jLogger import cats.Parallel import cats.implicits._ +import cats.data.EitherT import cats.effect._ import scala.concurrent.ExecutionContext @@ -21,6 +22,7 @@ import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.parque import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config.{Monitoring, StreamInput} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.sources.Checkpointer import com.snowplowanalytics.snowplow.rdbloader.common.cloud.{BlobStorage, Queue} +import com.snowplowanalytics.snowplow.rdbloader.common.config.License import com.snowplowanalytics.snowplow.scalatracker.Tracking object Run { @@ -43,7 +45,10 @@ object Run { parquetOps: ParquetOps = ParquetOps.noop ): F[ExitCode] = for { - parsed <- CliConfig.loadConfigFrom[F](buildName, buildDescription)(args: Seq[String]).value + parsed <- CliConfig + .loadConfigFrom[F](buildName, buildDescription)(args: Seq[String]) + .flatTap(c => EitherT.fromEither[F](License.checkLicense(c.config.license))) + .value res <- parsed match { case Right(cliConfig) => Resources diff --git a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala new file mode 100644 index 000000000..0cd680f85 --- /dev/null +++ b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2012-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ +package com.snowplowanalytics.snowplow.rdbloader.common.config + +import cats.implicits._ + +import io.circe.Decoder + +case class License(accept: Boolean) + +object License { + implicit val licenseDecoder: Decoder[License] = { + val truthy = Set("true", "yes", "on", "1") + Decoder + .forProduct1("accept")((s: String) => License(truthy(s.toLowerCase()))) + .or(Decoder.forProduct1("accept")((b: Boolean) => License(b))) + } + + def checkLicense(license: License): Either[String, Unit] = + if (license.accept) + ().asRight + else + "Please accept the terms of the Snowplow Limited Use License Agreement to proceed. See https://docs.snowplow.io/docs/pipeline-components-and-applications/loaders-storage-targets/snowplow-rdb-loader/loading-transformed-data/rdb-loader-configuration-reference/#license for more information on the license and how to configure this.".asLeft +} diff --git a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala index 86085579f..61d828f45 100644 --- a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala +++ b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala @@ -44,7 +44,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -72,7 +73,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -100,7 +102,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -127,7 +130,8 @@ class ConfigSpec extends Specification { readyCheck, initRetries, exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } @@ -153,7 +157,8 @@ class ConfigSpec extends Specification { readyCheck, initRetries, exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } @@ -179,7 +184,8 @@ class ConfigSpec extends Specification { readyCheck, initRetries, exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } diff --git a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala index 8f0e4018e..b0ef7ddfa 100644 --- a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala +++ b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala @@ -247,7 +247,8 @@ object DatabricksSpec { Config.Retries(Config.Strategy.Constant, None, 1.minute, None), Config.Retries(Config.Strategy.Constant, None, 1.minute, None), Config.FeatureFlags(addLoadTstampColumn = true, disableRecovery = Nil), - exampleTelemetry + exampleTelemetry, + exampleLicense ) val target: Target[Unit] = Databricks diff --git a/modules/loader/src/main/resources/application.conf b/modules/loader/src/main/resources/application.conf index 90b3119b7..539849392 100644 --- a/modules/loader/src/main/resources/application.conf +++ b/modules/loader/src/main/resources/application.conf @@ -1,4 +1,8 @@ "snowplow": { + "license": { + "accept": false + "accept": ${?ACCEPT_LIMITED_USE_LICENSE} + } "cloud": "dummy" "messageQueue": { "parallelPullCount": 1, diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala index 088289d74..1136942b8 100644 --- a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala +++ b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala @@ -18,6 +18,7 @@ import doobie.ConnectionIO import org.typelevel.log4cats.slf4j.Slf4jLogger import com.snowplowanalytics.snowplow.rdbloader.dsl._ import com.snowplowanalytics.snowplow.rdbloader.dsl.Environment +import com.snowplowanalytics.snowplow.rdbloader.common.config.License import com.snowplowanalytics.snowplow.rdbloader.config.CliConfig import com.snowplowanalytics.snowplow.scalatracker.Tracking @@ -40,6 +41,7 @@ object Runner { val result = for { parsed <- CliConfig.parse[F](argv) statements <- EitherT.fromEither[F](buildStatements(parsed.config)) + _ <- EitherT.fromEither[F](License.checkLicense(parsed.config.license)) application = Environment .initialize[F, I]( diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala index f5704a95e..efa175f1b 100644 --- a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala +++ b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala @@ -25,7 +25,7 @@ import cron4s.circe._ import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath -import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Region} +import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, License, Region} import com.snowplowanalytics.snowplow.rdbloader.config.Config._ /** @@ -45,7 +45,8 @@ case class Config[+D <: StorageTarget]( readyCheck: Retries, initRetries: Retries, featureFlags: FeatureFlags, - telemetry: Telemetry.Config + telemetry: Telemetry.Config, + license: License ) object Config { diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala index 6facba152..c305fc590 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala @@ -16,7 +16,7 @@ import cats.data.EitherT import cats.effect.IO import org.http4s.implicits._ import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry -import com.snowplowanalytics.snowplow.rdbloader.common.config.Region +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, Region} import com.snowplowanalytics.snowplow.rdbloader.common.RegionSpec import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage import com.snowplowanalytics.snowplow.rdbloader.config.{Config, StorageTarget} @@ -237,6 +237,7 @@ object ConfigSpec { Some("rdb-loader-ce"), Some("1.0.0") ) + val exampleLicense = License(true) val defaultTelemetry = Telemetry.Config( false, @@ -263,7 +264,8 @@ object ConfigSpec { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) def getConfigFromResource[A](resourcePath: String, parse: HoconOrPath => EitherT[IO, String, A]): Either[String, A] = diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala index 11196104d..e8cfbce32 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala @@ -33,7 +33,8 @@ object SpecHelpers { ConfigSpec.exampleReadyCheck, ConfigSpec.exampleInitRetries, ConfigSpec.exampleFeatureFlags, - ConfigSpec.exampleTelemetry + ConfigSpec.exampleTelemetry, + ConfigSpec.exampleLicense ) def asB64(resourcePath: String): String = diff --git a/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala b/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala index c33985886..29b991a90 100644 --- a/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala +++ b/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala @@ -37,7 +37,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -56,7 +57,8 @@ class ConfigSpec extends Specification { exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds), exampleInitRetries.copy(attempts = None, cumulativeBound = Some(10.minutes)), exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } diff --git a/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala b/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala index 6d43ba22f..2696b405d 100644 --- a/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala +++ b/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala @@ -46,7 +46,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -88,7 +89,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -135,7 +137,8 @@ class ConfigSpec extends Specification { exampleReadyCheck, exampleInitRetries, exampleFeatureFlags, - exampleTelemetry + exampleTelemetry, + exampleLicense ) result must beRight(expected) } @@ -157,7 +160,8 @@ class ConfigSpec extends Specification { exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds), exampleInitRetries.copy(attempts = None, cumulativeBound = Some(10.minutes)), exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } @@ -188,7 +192,8 @@ class ConfigSpec extends Specification { exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds), exampleInitRetries.copy(attempts = None, cumulativeBound = Some(10.minutes)), exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } @@ -224,7 +229,8 @@ class ConfigSpec extends Specification { exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds), exampleInitRetries.copy(attempts = None, cumulativeBound = Some(10.minutes)), exampleFeatureFlags, - defaultTelemetry + defaultTelemetry, + exampleLicense ) result must beRight(expected) } diff --git a/modules/transformer-batch/src/main/resources/application.conf b/modules/transformer-batch/src/main/resources/application.conf index 6017ce5ef..946baea9c 100644 --- a/modules/transformer-batch/src/main/resources/application.conf +++ b/modules/transformer-batch/src/main/resources/application.conf @@ -1,4 +1,9 @@ "snowplow": { + "license": { + "accept": false + "accept": ${?ACCEPT_LIMITED_USE_LICENSE} + } + "output": { "compression": "GZIP", "maxRecordsPerFile": 10000 diff --git a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala index a544d8ad7..85b9476cd 100644 --- a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala +++ b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala @@ -22,7 +22,7 @@ import com.snowplowanalytics.snowplow.rdbloader.common._ import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Compression -import com.snowplowanalytics.snowplow.rdbloader.common.config.Region +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, Region} import com.snowplowanalytics.snowplow.rdbloader.common.config.implicits._ final case class Config( @@ -35,7 +35,8 @@ final case class Config( runInterval: Config.RunInterval, featureFlags: TransformerConfig.FeatureFlags, skipSchemas: List[SchemaCriterion], - validations: TransformerConfig.Validations + validations: TransformerConfig.Validations, + license: License ) object Config { diff --git a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Main.scala b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Main.scala index 299603fed..3d7878cc2 100644 --- a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Main.scala +++ b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Main.scala @@ -16,6 +16,7 @@ import org.apache.spark.serializer.KryoSerializer import org.apache.spark.sql.SparkSession import cats.syntax.either._ +import com.snowplowanalytics.snowplow.rdbloader.common.config.License import com.snowplowanalytics.snowplow.rdbloader.transformer.batch.generated.BuildInfo import com.snowplowanalytics.snowplow.rdbloader.transformer.batch.spark.Serialization @@ -30,7 +31,10 @@ object Main { .registerKryoClasses(Serialization.classesToRegister) def main(args: Array[String]): Unit = - CliConfig.loadConfigFrom(BuildInfo.name, BuildInfo.description)(args) match { + (for { + c <- CliConfig.loadConfigFrom(BuildInfo.name, BuildInfo.description)(args) + _ <- License.checkLicense(c.config.license) + } yield c) match { case Right(cli) => val spark = SparkSession .builder() diff --git a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala index 102038a55..ecdf09366 100644 --- a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala +++ b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala @@ -19,7 +19,7 @@ import scala.concurrent.duration._ import com.snowplowanalytics.iglu.core.SchemaCriterion import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Validations -import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Region, TransformerConfig} +import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, License, Region, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.{LoaderMessage, RegionSpec} import org.specs2.mutable.Specification @@ -39,7 +39,8 @@ class ConfigSpec extends Specification { exampleRunInterval, exampleDefaultFeatureFlags, exampleSkipSchemas, - exampleValidations + exampleValidations, + exampleLicense ) result must beRight(expected) } @@ -56,7 +57,8 @@ class ConfigSpec extends Specification { emptyRunInterval, exampleDefaultFeatureFlags, Nil, - emptyValidations + emptyValidations, + exampleLicense ) result must beRight(expected) } @@ -160,6 +162,7 @@ object TransformerConfigSpec { ) val exampleDefaultFeatureFlags = TransformerConfig.FeatureFlags(false, None, false, false) val exampleValidations = Validations(Some(Instant.parse("2021-11-18T11:00:00.00Z"))) + val exampleLicense = License(true) val emptyValidations = Validations(None) val exampleSkipSchemas = List( SchemaCriterion("com.acme", "skipped1", "jsonschema", Some(1), Some(0), Some(0)), diff --git a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ShredJobSpec.scala b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ShredJobSpec.scala index f7cdc6876..2916b1ed7 100644 --- a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ShredJobSpec.scala +++ b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ShredJobSpec.scala @@ -15,6 +15,7 @@ import com.snowplowanalytics.iglu.client.{Client, Resolver} import com.snowplowanalytics.iglu.client.validator.CirceValidator import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ import com.snowplowanalytics.snowplow.rdbloader.common.catsClockIdInstance +import com.snowplowanalytics.snowplow.rdbloader.common.config.License import com.snowplowanalytics.snowplow.rdbloader.common.transformation.parquet.{AtomicFieldsProvider, NonAtomicFieldsProvider} import com.snowplowanalytics.snowplow.rdbloader.common.transformation.parquet.fields.AllFields import com.snowplowanalytics.snowplow.rdbloader.transformer.batch.spark.singleton.IgluSingleton @@ -298,6 +299,7 @@ object ShredJobSpec { } val naturalDeduplication = shredder.deduplication.natural.asJson val configPlain = s"""|{ + |"license": { "accept": true } |"input": "${shredder.input}", |"output" = { | "path": "${shredder.output.path}", @@ -453,7 +455,8 @@ object ShredJobSpec { Config.RunInterval(None, None, None), TransformerConfig.FeatureFlags(false, None, false, false), skipSchemas, - TransformerConfig.Validations(None) + TransformerConfig.Validations(None), + License(true) ) } } diff --git a/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/AzuriteBasedDevApp.scala b/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/AzuriteBasedDevApp.scala index a9a0f68f9..42de9013d 100644 --- a/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/AzuriteBasedDevApp.scala +++ b/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/AzuriteBasedDevApp.scala @@ -38,6 +38,7 @@ object AzuriteBasedDevApp extends IOApp { val appConfig = """ |{ + | "license": { "accept": true } | "input": { | "topicName": "enriched" | "bootstrapServers": "localhost:9092" diff --git a/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/ConfigSpec.scala b/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/ConfigSpec.scala index 847c47718..85b444b62 100644 --- a/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/ConfigSpec.scala +++ b/modules/transformer-kafka/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/ConfigSpec.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.kafka import cats.effect.IO import cats.effect.unsafe.implicits.global import com.snowplowanalytics.snowplow.badrows.Processor -import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Validations import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.generated.BuildInfo @@ -41,7 +41,8 @@ class ConfigSpec extends Specification { exampleMonitoringStream, exampleTelemetry, exampleDefaultFeatureFlags, - exampleValidations + exampleValidations, + exampleLicense ) result must beRight(expected) } @@ -57,7 +58,8 @@ class ConfigSpec extends Specification { exampleDefaultMonitoringStream, defaultTelemetry, exampleDefaultFeatureFlags, - emptyValidations + emptyValidations, + exampleLicense ) result must beRight(expected) } @@ -138,6 +140,7 @@ object ConfigSpec { ) val exampleDefaultFeatureFlags = TransformerConfig.FeatureFlags(false, None, true, false) val exampleValidations = Validations(Some(Instant.parse("2021-11-18T11:00:00.00Z"))) + val exampleLicense = License(true) val emptyValidations = Validations(None) val TestProcessor = Processor(BuildInfo.name, BuildInfo.version) } diff --git a/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala b/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala index 9223bdaee..56f4ee56c 100644 --- a/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala +++ b/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala @@ -24,7 +24,7 @@ import com.snowplowanalytics.iglu.core.SchemaCriterion import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.common.config.{Kinesis => AWSKinesis} import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Validations -import com.snowplowanalytics.snowplow.rdbloader.common.config.{Region, TransformerConfig} +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, Region, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.{LoaderMessage, RegionSpec} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.ConfigUtils._ @@ -48,7 +48,8 @@ class ConfigSpec extends Specification { exampleMonitoringStream, exampleTelemetry, exampleDefaultFeatureFlags.copy(enableMaxRecordsPerFile = true), - exampleValidations + exampleValidations, + exampleLicense ) result must beRight(expected) } @@ -64,7 +65,8 @@ class ConfigSpec extends Specification { exampleDefaultMonitoringStream, defaultTelemetry, exampleDefaultFeatureFlags.copy(enableMaxRecordsPerFile = true), - emptyValidations + emptyValidations, + exampleLicense ) result must beRight(expected) } @@ -166,6 +168,7 @@ object ConfigSpec { ) val exampleDefaultFeatureFlags = TransformerConfig.FeatureFlags(false, None, false, false) val exampleValidations = Validations(Some(Instant.parse("2021-11-18T11:00:00.00Z"))) + val exampleLicense = License(true) val emptyValidations = Validations(None) val TestProcessor = Processor(BuildInfo.name, BuildInfo.version) } diff --git a/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala b/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala index d8ab67c0d..61275bd2a 100644 --- a/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala +++ b/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala @@ -19,7 +19,7 @@ import cats.effect.IO import cats.effect.unsafe.implicits.global import com.snowplowanalytics.snowplow.badrows.Processor import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry -import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig +import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Validations import com.snowplowanalytics.snowplow.rdbloader.generated.BuildInfo import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config @@ -43,7 +43,8 @@ class ConfigSpec extends Specification { exampleMonitoringStream, exampleTelemetry, exampleDefaultFeatureFlags, - exampleValidations + exampleValidations, + exampleLicense ) result must beRight(expected) } @@ -59,7 +60,8 @@ class ConfigSpec extends Specification { exampleDefaultMonitoringStream, defaultTelemetry, exampleDefaultFeatureFlags, - emptyValidations + emptyValidations, + exampleLicense ) result must beRight(expected) } @@ -142,6 +144,7 @@ object ConfigSpec { ) val exampleDefaultFeatureFlags = TransformerConfig.FeatureFlags(false, None, true, false) val exampleValidations = Validations(Some(Instant.parse("2021-11-18T11:00:00.00Z"))) + val exampleLicense = License(true) val emptyValidations = Validations(None) val TestProcessor = Processor(BuildInfo.name, BuildInfo.version) }