Add mandatory SLULA license acceptance flag

Since introducing a new license we need to explicitly check if the user has accepted the terms. Therefore a new flag is added. By default it is set to false but can be overrided by either: - setting `license.accept = true` in the config file - setting `env ACCEPT_LIMITED_USE_LICENSE=true` - appending `-Dlicense.accept=true`
snowplow · Mar 14, 2024 · c59c51b · c59c51b
1 parent b0bafc6
commit c59c51b
Show file tree

Hide file tree

Showing 44 changed files with 218 additions and 39 deletions.
diff --git a/config/loader/aws/databricks.config.minimal.hocon b/config/loader/aws/databricks.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "messageQueue": "test-queue",
   "storage" : {
     "host": "abc.cloud.databricks.com"

diff --git a/config/loader/aws/databricks.config.reference.hocon b/config/loader/aws/databricks.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Data Lake (S3) region
   # This field is optional if it can be resolved with AWS region provider chain.
   # It checks places like env variables, system properties, AWS profile file.

diff --git a/config/loader/aws/redshift.config.minimal.hocon b/config/loader/aws/redshift.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "messageQueue": "test-queue"
   "storage" : {
     "type": "redshift"

diff --git a/config/loader/aws/redshift.config.reference.hocon b/config/loader/aws/redshift.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Specifies the cloud provider that application will be deployed into
   "cloud": "aws"
   # Data Lake (S3) region

diff --git a/config/loader/aws/snowflake.config.minimal.hocon b/config/loader/aws/snowflake.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "messageQueue": "test-queue",
   "storage" : {
     "type": "snowflake",

diff --git a/config/loader/aws/snowflake.config.reference.hocon b/config/loader/aws/snowflake.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Data Lake (S3) region
   # This field is optional if it can be resolved with AWS region provider chain.
   # It checks places like env variables, system properties, AWS profile file.

diff --git a/config/loader/azure/databricks.config.minimal.hocon b/config/loader/azure/databricks.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name"
   "messageQueue": {
     "type": "kafka"

diff --git a/config/loader/azure/databricks.config.reference.hocon b/config/loader/azure/databricks.config.reference.hocon
@@ -1,4 +1,8 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
 
   # Azure Blob Storage endpoint, should contain container with transformer's output
   "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name"

diff --git a/config/loader/azure/snowflake.config.minimal.hocon b/config/loader/azure/snowflake.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name"
   "messageQueue": {
     "type": "kafka"

diff --git a/config/loader/azure/snowflake.config.reference.hocon b/config/loader/azure/snowflake.config.reference.hocon
@@ -1,4 +1,8 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
 
   # Azure Blob Storage endpoint, should contain container with transformer's output
   "blobStorageEndpoint": "https://accountName.blob.core.windows.net/container-name"

diff --git a/config/loader/gcp/databricks.config.minimal.hocon b/config/loader/gcp/databricks.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "messageQueue": {
     "type": "pubsub"
     "subscription": "projects/project-id/subscriptions/subscription-id"

diff --git a/config/loader/gcp/databricks.config.reference.hocon b/config/loader/gcp/databricks.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Pubsub subscription used by Transformer and Loader to communicate
   "messageQueue": {
     "type": "pubsub"

diff --git a/config/loader/gcp/snowflake.config.minimal.hocon b/config/loader/gcp/snowflake.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "messageQueue": {
     "type": "pubsub"
     "subscription": "projects/project-id/subscriptions/subscription-id"

diff --git a/config/loader/gcp/snowflake.config.reference.hocon b/config/loader/gcp/snowflake.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Pubsub subscription used by Transformer and Loader to communicate
   "messageQueue": {
     "type": "pubsub"

diff --git a/config/transformer/aws/transformer.batch.config.minimal.hocon b/config/transformer/aws/transformer.batch.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "input": "s3://bucket/input/"
   "output": {
     "path": "s3://bucket/transformed/"

diff --git a/config/transformer/aws/transformer.batch.config.reference.hocon b/config/transformer/aws/transformer.batch.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   # Path to enriched archive (must be populated separately with run=YYYY-MM-DD-hh-mm-ss-UUID directories) for S3 input
   "input": "s3://bucket/input/",
 

diff --git a/config/transformer/aws/transformer.kinesis.config.minimal.hocon b/config/transformer/aws/transformer.kinesis.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "input": {
     "streamName": "enriched-events"
   }

diff --git a/config/transformer/aws/transformer.kinesis.config.reference.hocon b/config/transformer/aws/transformer.kinesis.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   "input": {
     # kinesis and file are the only options for transformer-kinesis
     # Optional, default value kinesis

diff --git a/config/transformer/azure/transformer.kafka.config.minimal.hocon b/config/transformer/azure/transformer.kafka.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "input": {
     "topicName": "enriched"
     "bootstrapServers": "localhost:9092"

diff --git a/config/transformer/azure/transformer.kafka.config.reference.hocon b/config/transformer/azure/transformer.kafka.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   "input": {
     "type": "kafka"
 

diff --git a/config/transformer/gcp/transformer.pubsub.config.minimal.hocon b/config/transformer/gcp/transformer.pubsub.config.minimal.hocon
@@ -1,4 +1,7 @@
 {
+  "license": {
+    "accept": true
+  }
   "input": {
     "subscription": "projects/project-id/subscriptions/subscription-id"
   }

diff --git a/config/transformer/gcp/transformer.pubsub.config.reference.hocon b/config/transformer/gcp/transformer.pubsub.config.reference.hocon
@@ -1,4 +1,9 @@
 {
+  # Full license text available in SNOWPLOW-LICENSE.md
+  "license": {
+    "accept": true
+  }
+
   "input": {
     # Name of the Pubsub subscription with the enriched events
     "subscription": "projects/project-id/subscriptions/subscription-id"

diff --git a/modules/common-transformer-stream/src/main/resources/application.conf b/modules/common-transformer-stream/src/main/resources/application.conf
@@ -1,4 +1,8 @@
 "snowplow": {
+  "license": {
+    "accept": false
+    "accept": ${?ACCEPT_LIMITED_USE_LICENSE}
+  }
 
   "output": {
     "compression": "GZIP",

diff --git a/...ain/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala b/...ain/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala
@@ -24,7 +24,7 @@ import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Tran
 import com.snowplowanalytics.snowplow.rdbloader.common.config.implicits._
 import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Compression
 import com.snowplowanalytics.snowplow.rdbloader.common.config.{Kinesis => AWSKinesis}
-import com.snowplowanalytics.snowplow.rdbloader.common.config.Region
+import com.snowplowanalytics.snowplow.rdbloader.common.config.{License, Region}
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config.Output.Bad
 
 final case class Config(
@@ -36,7 +36,8 @@ final case class Config(
   monitoring: Config.Monitoring,
   telemetry: Telemetry.Config,
   featureFlags: TransformerConfig.FeatureFlags,
-  validations: TransformerConfig.Validations
+  validations: TransformerConfig.Validations,
+  license: License
 )
 
 object Config {

diff --git a/...c/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala b/...c/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Run.scala
@@ -13,6 +13,7 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common
 import org.typelevel.log4cats.slf4j.Slf4jLogger
 import cats.Parallel
 import cats.implicits._
+import cats.data.EitherT
 import cats.effect._
 
 import scala.concurrent.ExecutionContext
@@ -21,6 +22,7 @@ import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.parque
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config.{Monitoring, StreamInput}
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.sources.Checkpointer
 import com.snowplowanalytics.snowplow.rdbloader.common.cloud.{BlobStorage, Queue}
+import com.snowplowanalytics.snowplow.rdbloader.common.config.License
 import com.snowplowanalytics.snowplow.scalatracker.Tracking
 
 object Run {
@@ -43,7 +45,10 @@ object Run {
     parquetOps: ParquetOps = ParquetOps.noop
   ): F[ExitCode] =
     for {
-      parsed <- CliConfig.loadConfigFrom[F](buildName, buildDescription)(args: Seq[String]).value
+      parsed <- CliConfig
+                  .loadConfigFrom[F](buildName, buildDescription)(args: Seq[String])
+                  .flatTap(c => EitherT.fromEither[F](License.checkLicense(c.config.license)))
+                  .value
       res <- parsed match {
                case Right(cliConfig) =>
                  Resources

diff --git a/...ommon/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala b/...ommon/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/License.scala
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2012-present Snowplow Analytics Ltd.
+ * All rights reserved.
+ *
+ * This software is made available by Snowplow Analytics, Ltd.,
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
+ * located at https://docs.snowplow.io/limited-use-license-1.0
+ * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
+ * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
+ */
+package com.snowplowanalytics.snowplow.rdbloader.common.config
+
+import cats.implicits._
+
+import io.circe.Decoder
+
+case class License(accept: Boolean)
+
+object License {
+  implicit val licenseDecoder: Decoder[License] = {
+    val truthy = Set("true", "yes", "on", "1")
+    Decoder
+      .forProduct1("accept")((s: String) => License(truthy(s.toLowerCase())))
+      .or(Decoder.forProduct1("accept")((b: Boolean) => License(b)))
+  }
+
+  def checkLicense(license: License): Either[String, Unit] =
+    if (license.accept)
+      ().asRight
+    else
+      "Please accept the terms of the Snowplow Limited Use License Agreement to proceed. See https://docs.snowplow.io/docs/pipeline-components-and-applications/loaders-storage-targets/snowplow-rdb-loader/loading-transformed-data/rdb-loader-configuration-reference/#license for more information on the license and how to configure this.".asLeft
+}
diff --git a/...s-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala b/...s-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala
@@ -44,7 +44,8 @@ class ConfigSpec extends Specification {
         exampleReadyCheck,
         exampleInitRetries,
         exampleFeatureFlags,
-        exampleTelemetry
+        exampleTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }
@@ -72,7 +73,8 @@ class ConfigSpec extends Specification {
         exampleReadyCheck,
         exampleInitRetries,
         exampleFeatureFlags,
-        exampleTelemetry
+        exampleTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }
@@ -100,7 +102,8 @@ class ConfigSpec extends Specification {
         exampleReadyCheck,
         exampleInitRetries,
         exampleFeatureFlags,
-        exampleTelemetry
+        exampleTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }
@@ -127,7 +130,8 @@ class ConfigSpec extends Specification {
         readyCheck,
         initRetries,
         exampleFeatureFlags,
-        defaultTelemetry
+        defaultTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }
@@ -153,7 +157,8 @@ class ConfigSpec extends Specification {
         readyCheck,
         initRetries,
         exampleFeatureFlags,
-        defaultTelemetry
+        defaultTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }
@@ -179,7 +184,8 @@ class ConfigSpec extends Specification {
         readyCheck,
         initRetries,
         exampleFeatureFlags,
-        defaultTelemetry
+        defaultTelemetry,
+        exampleLicense
       )
       result must beRight(expected)
     }

diff --git a/...ader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala b/...ader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala
@@ -247,7 +247,8 @@ object DatabricksSpec {
     Config.Retries(Config.Strategy.Constant, None, 1.minute, None),
     Config.Retries(Config.Strategy.Constant, None, 1.minute, None),
     Config.FeatureFlags(addLoadTstampColumn = true, disableRecovery = Nil),
-    exampleTelemetry
+    exampleTelemetry,
+    exampleLicense
   )
 
   val target: Target[Unit] = Databricks

diff --git a/modules/loader/src/main/resources/application.conf b/modules/loader/src/main/resources/application.conf
@@ -1,4 +1,8 @@
 "snowplow": {
+    "license": {
+      "accept": false
+      "accept": ${?ACCEPT_LIMITED_USE_LICENSE}
+    }
     "cloud": "dummy"
     "messageQueue": {
       "parallelPullCount": 1,

diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Runner.scala
@@ -18,6 +18,7 @@ import doobie.ConnectionIO
 import org.typelevel.log4cats.slf4j.Slf4jLogger
 import com.snowplowanalytics.snowplow.rdbloader.dsl._
 import com.snowplowanalytics.snowplow.rdbloader.dsl.Environment
+import com.snowplowanalytics.snowplow.rdbloader.common.config.License
 import com.snowplowanalytics.snowplow.rdbloader.config.CliConfig
 import com.snowplowanalytics.snowplow.scalatracker.Tracking
 
@@ -40,6 +41,7 @@ object Runner {
     val result = for {
       parsed <- CliConfig.parse[F](argv)
       statements <- EitherT.fromEither[F](buildStatements(parsed.config))
+      _ <- EitherT.fromEither[F](License.checkLicense(parsed.config.license))
       application =
         Environment
           .initialize[F, I](

diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala
@@ -25,7 +25,7 @@ import cron4s.circe._
 import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry
 import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage
 import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath
-import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Region}
+import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, License, Region}
 import com.snowplowanalytics.snowplow.rdbloader.config.Config._
 
 /**
@@ -45,7 +45,8 @@ case class Config[+D <: StorageTarget](
   readyCheck: Retries,
   initRetries: Retries,
   featureFlags: FeatureFlags,
-  telemetry: Telemetry.Config
+  telemetry: Telemetry.Config,
+  license: License
 )
 
 object Config {