From 72beea1ae726f7b5af78dcf3595b78eba214b87e Mon Sep 17 00:00:00 2001 From: Chris Li Date: Sat, 31 Jul 2021 12:40:30 -0700 Subject: [PATCH] Config CI to publish artifacts to JFrog (public) --- .github/PULL_REQUEST_TEMPLATE.md | 7 +- .github/actions/release/action.yml | 18 ++ .github/workflows/build-and-test.yml | 35 ++++ .github/workflows/build_and_test.yaml | 151 ---------------- .github/workflows/docker_build_publish.yaml | 61 ------- .github/workflows/release.yml | 27 +++ .travis.yml | 65 ------- README.md | 16 +- build.gradle | 164 +++++++----------- {dil => cdi-core}/build.gradle | 20 ++- .../configuration/MultistageProperties.java | 20 +-- .../cdi}/configuration/StaticConstants.java | 2 +- .../linkedin/cdi}/connection/Connection.java | 6 +- .../cdi}/connection/HdfsConnection.java | 14 +- .../cdi}/connection/HttpConnection.java | 22 +-- .../cdi}/connection/JdbcConnection.java | 22 +-- .../cdi}/connection/MultistageConnection.java | 12 +- .../cdi}/connection/S3Connection.java | 20 +-- .../converter/AvroNormalizerConverter.java | 8 +- .../converter/JsonNormalizerConverter.java | 8 +- .../RetriableAuthenticationException.java | 2 +- .../cdi}/extractor/AvroExtractor.java | 18 +- .../linkedin/cdi}/extractor/CsvExtractor.java | 26 +-- .../cdi}/extractor/FileDumpExtractor.java | 20 +-- .../cdi}/extractor/JsonExtractor.java | 20 +-- .../cdi}/extractor/MultistageExtractor.java | 42 ++--- .../cdi}/factory/ApacheHttpClientFactory.java | 2 +- .../factory/DefaultJdbcClientFactory.java | 4 +- .../cdi}/factory/DefaultS3ClientFactory.java | 2 +- .../cdi}/factory/HttpClientFactory.java | 2 +- .../cdi}/factory/JdbcClientFactory.java | 2 +- .../cdi}/factory/S3ClientFactory.java | 2 +- .../cdi}/factory/SchemaReaderFactory.java | 6 +- .../cdi}/factory/reader/JsonFileReader.java | 2 +- .../cdi}/factory/reader/SchemaReader.java | 2 +- .../cdi}/filter/AvroSchemaBasedFilter.java | 8 +- .../cdi}/filter/CsvSchemaBasedFilter.java | 6 +- .../cdi}/filter/JsonSchemaBasedFilter.java | 8 +- .../filter/MultistageSchemaBasedFilter.java | 4 +- .../cdi}/filter/SchemaBasedFilter.java | 2 +- .../linkedin/cdi}/keys/AvroExtractorKeys.java | 4 +- .../linkedin/cdi}/keys/CsvExtractorKeys.java | 4 +- .../com/linkedin/cdi}/keys/ExtractorKeys.java | 6 +- .../cdi}/keys/FileDumpExtractorKeys.java | 2 +- .../java/com/linkedin/cdi}/keys/HdfsKeys.java | 4 +- .../java/com/linkedin/cdi}/keys/HttpKeys.java | 6 +- .../java/com/linkedin/cdi}/keys/JdbcKeys.java | 4 +- .../java/com/linkedin/cdi}/keys/JobKeys.java | 20 +-- .../linkedin/cdi}/keys/JsonExtractorKeys.java | 4 +- .../java/com/linkedin/cdi}/keys/S3Keys.java | 2 +- .../preprocessor/GpgDecryptProcessor.java | 6 +- .../preprocessor/GpgEncryptProcessor.java | 4 +- .../cdi}/preprocessor/GunzipProcessor.java | 2 +- .../preprocessor/InputStreamProcessor.java | 2 +- .../preprocessor/OutputStreamProcessor.java | 2 +- .../cdi}/preprocessor/StreamProcessor.java | 2 +- .../com/linkedin/cdi}/source/HdfsSource.java | 8 +- .../com/linkedin/cdi}/source/HttpSource.java | 12 +- .../com/linkedin/cdi}/source/JdbcSource.java | 12 +- .../cdi}/source/MultistageSource.java | 16 +- .../com/linkedin/cdi}/source/S3SourceV2.java | 12 +- .../linkedin/cdi}/util/AvroSchemaUtils.java | 2 +- .../java/com/linkedin/cdi}/util/CsvUtils.java | 2 +- .../java/com/linkedin/cdi}/util/Database.java | 2 +- .../com/linkedin/cdi}/util/DateTimeUtils.java | 2 +- .../linkedin/cdi}/util/EncryptionUtils.java | 2 +- .../com/linkedin/cdi}/util/EndecoUtils.java | 2 +- .../com/linkedin/cdi}/util/HdfsReader.java | 4 +- .../linkedin/cdi}/util/HttpRequestMethod.java | 2 +- .../linkedin/cdi}/util/InputStreamUtils.java | 2 +- .../com/linkedin/cdi}/util/JdbcUtils.java | 2 +- .../linkedin/cdi}/util/JsonElementTypes.java | 2 +- .../cdi}/util/JsonIntermediateSchema.java | 4 +- .../com/linkedin/cdi}/util/JsonParameter.java | 4 +- .../com/linkedin/cdi}/util/JsonUtils.java | 2 +- .../linkedin/cdi}/util/ParameterTypes.java | 2 +- .../com/linkedin/cdi}/util/SchemaBuilder.java | 4 +- .../com/linkedin/cdi}/util/SchemaUtils.java | 2 +- .../com/linkedin/cdi}/util/VariableUtils.java | 2 +- .../cdi}/util/WatermarkDefinition.java | 4 +- .../cdi}/util/WorkUnitPartitionTypes.java | 6 +- .../linkedin/cdi}/util/WorkUnitStatus.java | 2 +- .../MultistagePropertiesTest.java | 2 +- .../connection/HdfsReadConnectionTest.java | 16 +- .../cdi}/connection/HttpConnectionTest.java | 24 +-- .../connection/JdbcReadConnectionTest.java | 26 +-- .../MulstistageReadConnectionTest.java | 12 +- .../S3ReadConnectionOnlineTest.java | 2 +- .../cdi}/connection/S3ReadConnectionTest.java | 12 +- .../AvroNormalizerConverterTest.java | 4 +- .../JsonNormalizerConverterTest.java | 2 +- .../cdi}/extractor/AvroExtractorTest.java | 24 +-- .../cdi}/extractor/CsvExtractorTest.java | 24 +-- .../cdi}/extractor/FileDumpExtractorTest.java | 10 +- .../extractor/FileDumpExtractorTest2.java | 20 +-- .../cdi}/extractor/JsonExtractorTest.java | 26 +-- .../extractor/MultistageExtractorTest.java | 34 ++-- .../factory/ApacheHttpClientFactoryTest.java | 2 +- .../filter/AvroSchemaBasedFilterTest.java | 10 +- .../filter/JsonSchemaBasedFilterTest.java | 6 +- .../MultistageSchemaBasedFilterTest.java | 4 +- .../helpers/GobblinMultiStageTestHelpers.java | 2 +- .../cdi}/helpers/MockedResponseStrings.java | 2 +- .../cdi}/jdbcclient/DatabaseTest.java | 4 +- .../cdi}/keys/FileDumpExtractorKeysTest.java | 2 +- .../com/linkedin/cdi}/keys/JobKeysTest.java | 12 +- .../preprocessor/GpgDecryptProcessorTest.java | 2 +- .../preprocessor/GpgEncryptProcessorTest.java | 2 +- .../preprocessor/GpgPreprocessorTest.java | 2 +- .../preprocessor/GunzipPreprocessorTest.java | 2 +- .../preprocessor/GunzipProcessorTest.java | 2 +- .../linkedin/cdi}/source/HdfsSourceTest.java | 4 +- .../linkedin/cdi}/source/HttpSourceTest.java | 18 +- .../linkedin/cdi}/source/JdbcSourceTest.java | 2 +- .../cdi}/source/MultistageSource2Test.java | 6 +- .../cdi}/source/MultistageSourceTest.java | 14 +- .../linkedin/cdi}/source/S3SourceV2Test.java | 4 +- .../cdi}/util/AvroSchemaUtilsTest.java | 2 +- .../linkedin/cdi}/util/DateTimeUtilsTest.java | 2 +- .../cdi}/util/EncryptionUtilsTest.java | 4 +- .../cdi}/util/HttpRequestMethodTest.java | 2 +- .../cdi}/util/InputStreamUtilsTest.java | 2 +- .../com/linkedin/cdi}/util/JdbcUtilsTest.java | 2 +- .../cdi}/util/JsonElementTypesTest.java | 4 +- .../cdi}/util/JsonIntermediateSchemaTest.java | 2 +- .../linkedin/cdi}/util/JsonParameterTest.java | 4 +- .../com/linkedin/cdi}/util/JsonUtilsTest.java | 2 +- .../linkedin/cdi}/util/SchemaBuilderTest.java | 2 +- .../linkedin/cdi}/util/SchemaUtilsTest.java | 2 +- .../linkedin/cdi}/util/VariableUtilsTest.java | 2 +- .../cdi}/util/WatermarkDefinitionTest.java | 4 +- .../cdi}/util/WorkUnitPartitionTypesTest.java | 2 +- .../cdi}/util/WorkUnitStatusTest.java | 2 +- .../src/test/resources/avro/empty_file.avro | 0 .../test/resources/csv/comma-separated.csv | 0 .../test/resources/csv/common-crawl-files.csv | 0 .../src/test/resources/csv/ctl_d_text.dat | 0 .../src/test/resources/csv/flat.csv | 0 .../resources/csv/flat_uppercase_header.csv | 0 .../resources/csv/flat_without_header.csv | 0 .../src/test/resources/csv/ids.csv | 0 .../src/test/resources/csv/ids_flat.csv | 0 .../test/resources/csv/ids_need_cleansing.csv | 0 .../src/test/resources/csv/tab-separated.csv | 0 .../src/test/resources/gpg/test.csv | 0 .../src/test/resources/gpg/test.csv.gpg | Bin .../src/test/resources/gpg/test.secret | Bin .../src/test/resources/gzip/cc-index.paths.gz | Bin .../resources/json/last-page-with-data.json | 0 .../resources/json/parameter-encryption.json | 0 .../json/sample-data-for-source.json | 4 +- .../resources/json/sample-json-schema.json | 0 .../src/test/resources/key/egress_test.key | 0 .../src/test/resources/key/master.key | 0 .../src/test/resources/key/public.key | 0 .../src/test/resources/key/sftp_test.key | 0 .../sample-data-include-long-file-name.txt | 0 .../src/test/resources/pull/s3-csv.pull | 8 +- .../src/test/resources/pull/s3-filedump.pull | 8 +- .../resources/util/parameter-jsonarray.json | 0 .../resources/util/sample-json-schema.json | 0 .../test/resources/util/sample-schema.json | 0 .../src/test/resources/util/sample.json | 0 defaultEnvironment.gradle | 21 --- flavored-build.gradle | 13 -- gradle.properties | 19 +- {config => gradle}/checkstyle/checkstyle.xml | 2 +- .../checkstyle/suppressions.xml | 0 gradle/ci-release.gradle | 38 ++++ gradle/java-publishing.gradle | 124 +++++++++++++ gradle/release.gradle | 39 +++++ gradle/scripts/bintrayPublishing.gradle | 156 ----------------- gradle/scripts/buildscript.gradle | 17 -- gradle/scripts/ci-support.gradle | 56 ------ gradle/scripts/computeVersions.gradle | 56 ------ gradle/scripts/defaultBuildProperties.gradle | 66 ------- gradle/scripts/dependencyDefinitions.gradle | 44 ----- gradle/scripts/environment.gradle | 14 -- gradle/scripts/globalDependencies.gradle | 65 ------- gradle/scripts/idesSetup.gradle | 25 --- gradle/scripts/javaPlugin.gradle | 13 -- gradle/scripts/javaVersionCheck.gradle | 17 -- gradle/scripts/javadoc.gradle | 97 ----------- gradle/scripts/mavenPublishing.gradle | 107 ------------ gradle/scripts/nexusPublishing.gradle | 128 -------------- gradle/scripts/release.gradle | 164 ------------------ gradle/scripts/repositories.gradle | 2 +- gradle/scripts/restli.gradle | 16 -- gradle/scripts/sourcesJar.gradle | 15 -- gradle/scripts/testSetup.gradle | 2 +- gradle/scripts/utilities.gradle | 34 ---- maven-nexus/maven-install.sh | 46 ----- maven-nexus/maven-nexus.sh | 102 ----------- maven-nexus/upload-to-nexus.sh | 58 ------- maven-sonatype/github-pr-change-log.py | 30 ---- maven-sonatype/maven-install.sh | 45 ----- maven-sonatype/maven-sonatype.sh | 104 ----------- maven-sonatype/upload-to-sonatype.sh | 58 ------- query_github_issues.py | 79 --------- settings.gradle | 24 +-- travis/bintrayDeploy.sh | 26 --- travis/filter-to-failing-test-results.py | 34 ---- travis/junit-errors-to-stdout.sh | 31 ---- travis/junit-xml-format-errors.xsl | 71 -------- travis/test-build.sh | 14 -- travis/test-coverage.sh | 18 -- travis/test-default.sh | 19 -- travis/test-group1.sh | 22 --- travis/test-groups.inc | 1 - travis/test.sh | 33 ---- version.properties | 1 + 211 files changed, 826 insertions(+), 2808 deletions(-) create mode 100644 .github/actions/release/action.yml create mode 100644 .github/workflows/build-and-test.yml delete mode 100644 .github/workflows/build_and_test.yaml delete mode 100644 .github/workflows/docker_build_publish.yaml create mode 100644 .github/workflows/release.yml delete mode 100644 .travis.yml rename {dil => cdi-core}/build.gradle (87%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/configuration/MultistageProperties.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/configuration/StaticConstants.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/Connection.java (90%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/HdfsConnection.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/HttpConnection.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/JdbcConnection.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/MultistageConnection.java (92%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/connection/S3Connection.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/converter/AvroNormalizerConverter.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/converter/JsonNormalizerConverter.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/exception/RetriableAuthenticationException.java (91%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/extractor/AvroExtractor.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/extractor/CsvExtractor.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/extractor/FileDumpExtractor.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/extractor/JsonExtractor.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/extractor/MultistageExtractor.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/ApacheHttpClientFactory.java (93%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/DefaultJdbcClientFactory.java (91%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/DefaultS3ClientFactory.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/HttpClientFactory.java (92%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/JdbcClientFactory.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/S3ClientFactory.java (93%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/SchemaReaderFactory.java (85%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/reader/JsonFileReader.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/factory/reader/SchemaReader.java (91%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/filter/AvroSchemaBasedFilter.java (89%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/filter/CsvSchemaBasedFilter.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/filter/JsonSchemaBasedFilter.java (93%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/filter/MultistageSchemaBasedFilter.java (86%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/filter/SchemaBasedFilter.java (89%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/AvroExtractorKeys.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/CsvExtractorKeys.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/ExtractorKeys.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/FileDumpExtractorKeys.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/HdfsKeys.java (91%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/HttpKeys.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/JdbcKeys.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/JobKeys.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/JsonExtractorKeys.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/keys/S3Keys.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/GpgDecryptProcessor.java (90%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/GpgEncryptProcessor.java (93%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/GunzipProcessor.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/InputStreamProcessor.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/OutputStreamProcessor.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/preprocessor/StreamProcessor.java (91%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/source/HdfsSource.java (89%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/source/HttpSource.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/source/JdbcSource.java (89%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/source/MultistageSource.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/source/S3SourceV2.java (94%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/AvroSchemaUtils.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/CsvUtils.java (96%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/Database.java (97%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/DateTimeUtils.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/EncryptionUtils.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/EndecoUtils.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/HdfsReader.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/HttpRequestMethod.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/InputStreamUtils.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/JdbcUtils.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/JsonElementTypes.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/JsonIntermediateSchema.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/JsonParameter.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/JsonUtils.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/ParameterTypes.java (95%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/SchemaBuilder.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/SchemaUtils.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/VariableUtils.java (99%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/WatermarkDefinition.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/WorkUnitPartitionTypes.java (98%) rename {dil/src/main/java/com/linkedin/dil => cdi-core/src/main/java/com/linkedin/cdi}/util/WorkUnitStatus.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/configuration/MultistagePropertiesTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/HdfsReadConnectionTest.java (89%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/HttpConnectionTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/JdbcReadConnectionTest.java (93%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/MulstistageReadConnectionTest.java (90%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/S3ReadConnectionOnlineTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/connection/S3ReadConnectionTest.java (82%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/converter/AvroNormalizerConverterTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/converter/JsonNormalizerConverterTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/AvroExtractorTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/CsvExtractorTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/FileDumpExtractorTest.java (91%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/FileDumpExtractorTest2.java (94%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/JsonExtractorTest.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/extractor/MultistageExtractorTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/factory/ApacheHttpClientFactoryTest.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/filter/AvroSchemaBasedFilterTest.java (93%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/filter/JsonSchemaBasedFilterTest.java (92%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/filter/MultistageSchemaBasedFilterTest.java (86%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/helpers/GobblinMultiStageTestHelpers.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/helpers/MockedResponseStrings.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/jdbcclient/DatabaseTest.java (95%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/keys/FileDumpExtractorKeysTest.java (93%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/keys/JobKeysTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/preprocessor/GpgDecryptProcessorTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/preprocessor/GpgEncryptProcessorTest.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/preprocessor/GpgPreprocessorTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/preprocessor/GunzipPreprocessorTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/preprocessor/GunzipProcessorTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/HdfsSourceTest.java (90%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/HttpSourceTest.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/JdbcSourceTest.java (93%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/MultistageSource2Test.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/MultistageSourceTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/source/S3SourceV2Test.java (94%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/AvroSchemaUtilsTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/DateTimeUtilsTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/EncryptionUtilsTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/HttpRequestMethodTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/InputStreamUtilsTest.java (96%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/JdbcUtilsTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/JsonElementTypesTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/JsonIntermediateSchemaTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/JsonParameterTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/JsonUtilsTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/SchemaBuilderTest.java (98%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/SchemaUtilsTest.java (97%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/VariableUtilsTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/WatermarkDefinitionTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/WorkUnitPartitionTypesTest.java (99%) rename {dil/src/test/java/com/linkedin/dil => cdi-core/src/test/java/com/linkedin/cdi}/util/WorkUnitStatusTest.java (98%) rename {dil => cdi-core}/src/test/resources/avro/empty_file.avro (100%) rename {dil => cdi-core}/src/test/resources/csv/comma-separated.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/common-crawl-files.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/ctl_d_text.dat (100%) rename {dil => cdi-core}/src/test/resources/csv/flat.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/flat_uppercase_header.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/flat_without_header.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/ids.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/ids_flat.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/ids_need_cleansing.csv (100%) rename {dil => cdi-core}/src/test/resources/csv/tab-separated.csv (100%) rename {dil => cdi-core}/src/test/resources/gpg/test.csv (100%) rename {dil => cdi-core}/src/test/resources/gpg/test.csv.gpg (100%) rename {dil => cdi-core}/src/test/resources/gpg/test.secret (100%) rename {dil => cdi-core}/src/test/resources/gzip/cc-index.paths.gz (100%) rename {dil => cdi-core}/src/test/resources/json/last-page-with-data.json (100%) rename {dil => cdi-core}/src/test/resources/json/parameter-encryption.json (100%) rename {dil => cdi-core}/src/test/resources/json/sample-data-for-source.json (95%) rename {dil => cdi-core}/src/test/resources/json/sample-json-schema.json (100%) rename {dil => cdi-core}/src/test/resources/key/egress_test.key (100%) rename {dil => cdi-core}/src/test/resources/key/master.key (100%) rename {dil => cdi-core}/src/test/resources/key/public.key (100%) rename {dil => cdi-core}/src/test/resources/key/sftp_test.key (100%) rename {dil => cdi-core}/src/test/resources/other/sample-data-include-long-file-name.txt (100%) rename {dil => cdi-core}/src/test/resources/pull/s3-csv.pull (75%) rename {dil => cdi-core}/src/test/resources/pull/s3-filedump.pull (74%) rename {dil => cdi-core}/src/test/resources/util/parameter-jsonarray.json (100%) rename {dil => cdi-core}/src/test/resources/util/sample-json-schema.json (100%) rename {dil => cdi-core}/src/test/resources/util/sample-schema.json (100%) rename {dil => cdi-core}/src/test/resources/util/sample.json (100%) delete mode 100644 defaultEnvironment.gradle delete mode 100644 flavored-build.gradle rename {config => gradle}/checkstyle/checkstyle.xml (90%) rename {config => gradle}/checkstyle/suppressions.xml (100%) create mode 100644 gradle/ci-release.gradle create mode 100644 gradle/java-publishing.gradle create mode 100644 gradle/release.gradle delete mode 100644 gradle/scripts/bintrayPublishing.gradle delete mode 100644 gradle/scripts/buildscript.gradle delete mode 100644 gradle/scripts/ci-support.gradle delete mode 100644 gradle/scripts/computeVersions.gradle delete mode 100644 gradle/scripts/defaultBuildProperties.gradle delete mode 100644 gradle/scripts/environment.gradle delete mode 100644 gradle/scripts/globalDependencies.gradle delete mode 100644 gradle/scripts/idesSetup.gradle delete mode 100644 gradle/scripts/javaPlugin.gradle delete mode 100644 gradle/scripts/javaVersionCheck.gradle delete mode 100644 gradle/scripts/javadoc.gradle delete mode 100644 gradle/scripts/mavenPublishing.gradle delete mode 100644 gradle/scripts/nexusPublishing.gradle delete mode 100644 gradle/scripts/release.gradle delete mode 100644 gradle/scripts/restli.gradle delete mode 100644 gradle/scripts/sourcesJar.gradle delete mode 100644 gradle/scripts/utilities.gradle delete mode 100755 maven-nexus/maven-install.sh delete mode 100755 maven-nexus/maven-nexus.sh delete mode 100755 maven-nexus/upload-to-nexus.sh delete mode 100755 maven-sonatype/github-pr-change-log.py delete mode 100755 maven-sonatype/maven-install.sh delete mode 100755 maven-sonatype/maven-sonatype.sh delete mode 100755 maven-sonatype/upload-to-sonatype.sh delete mode 100755 query_github_issues.py delete mode 100755 travis/bintrayDeploy.sh delete mode 100644 travis/filter-to-failing-test-results.py delete mode 100755 travis/junit-errors-to-stdout.sh delete mode 100755 travis/junit-xml-format-errors.xsl delete mode 100755 travis/test-build.sh delete mode 100755 travis/test-coverage.sh delete mode 100755 travis/test-default.sh delete mode 100755 travis/test-group1.sh delete mode 100644 travis/test-groups.inc delete mode 100755 travis/test.sh create mode 100644 version.properties diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index afff394..7060d18 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,9 @@ -Dear Gobblin maintainers, +Dear DIL maintainers, Please accept this PR. I understand that it will not be reviewed until I have checked off all the steps below! - ### JIRA -- [ ] My PR addresses the following [Gobblin JIRA](https://issues.apache.org/jira/browse/GOBBLIN/) issues and references them in the PR title. For example, "[GOBBLIN-XXX] My Gobblin PR" - - https://issues.apache.org/jira/browse/GOBBLIN-XXX - +- [ ] My PR addresses the following [JIRA](https://jira01.corp.linkedin.com:8443/issues) issues and references them in the PR title. ### Description - [ ] Here are some details about my PR, including screenshots (if applicable): diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml new file mode 100644 index 0000000..4d93651 --- /dev/null +++ b/.github/actions/release/action.yml @@ -0,0 +1,18 @@ +name: 'Publish Releases' +description: 'Runs gradle tasks to publish versions to Artifactory and GitHub.' +outputs: + version: + description: "The version tag this created." + value: ${{ steps.get-tag.outputs.tag }} +runs: + using: "composite" + steps: + - id: get-tag + run: echo "::set-output name=tag::$(./gradlew -q getVersion)" + shell: bash + - run: echo Got version ${{ steps.get-tag.outputs.tag }} + shell: bash + - run: ./gradlew build publishToMavenLocal artifactoryPublish -Partifactory.dryRun + shell: bash + - run: ./gradlew ciPerformRelease + shell: bash \ No newline at end of file diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..b040689 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,35 @@ +name: Build and Run Tests +on: + push: + branches: + - master + paths-ignore: + - 'docs/**' + - '**.md' + pull_request: + branches: + - master + paths-ignore: + - 'docs/**' + - '**.md' + release: + types: [published, edited] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Gradle build (and test) + run: ./gradlew build + - name: Slack failure notification + if: failure() && github.event_name == 'push' + uses: kpritam/slack-job-status-action@v1 + with: + job-status: ${{ job.status }} + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel: github-activities diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml deleted file mode 100644 index d805187..0000000 --- a/.github/workflows/build_and_test.yaml +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -name: Build and Run Tests - -on: - push: - # Publish only on `master` - branches: - - master - pull_request: - branches: - - master - release: - types: [published, edited] - -jobs: - build: - name: Build repository - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - # Stores external dependencies, can be further improved with Gradle 6.1 - - name: Cache Gradle Dependencies - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # Only rebuild cache if build.gradle is changed - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: ${{ runner.os }}-gradle - - name: Build repository - run: | - ./gradlew --no-daemon clean build -x test -x javadoc -x findbugsMain -x findbugsTest -x checkstyleMain -x checkstyleJmh -x checkstyleTest -Dorg.gradle.parallel=true - - test_coverage: - runs-on: ubuntu-latest - name: Generate test coverage - needs: build - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Cache Gradle Dependencies - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # Only rebuild cache if build.gradle is changed - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: ${{ runner.os }}-gradle - - name: Generate code coverage - run: | - ./gradlew -PskipTestGroup=disabledOnCI -Dorg.gradle.parallel=false -DjacocoBuild=1 $GOBBLIN_GRADLE_OPTS jacocoTestCoverage - - static_checks: - name: Run static checks - runs-on: ubuntu-latest - needs: build - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - # Stores external dependencies, can be further improved with Gradle 6.1 - - name: Cache Gradle Dependencies - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # Only rebuild cache if build.gradle is changed - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: ${{ runner.os }}-gradle - - name: Run CheckStyle and FindBugs - run: | - ./gradlew --no-daemon -x javadoc findbugsMain checkstyleMain checkstyleTest checkstyleJmh - - run_tests: - timeout-minutes: 60 - env: - GOBBLIN_GRADLE_OPTS: "--no-daemon -Dgobblin.metastore.testing.embeddedMysqlEnabled=false -PusePreinstalledMysql=true" - strategy: - matrix: - test-group: ["Core Tests", "Service Tests", "Module Tests", "Other Tests"] - fail-fast: false - runs-on: ubuntu-latest - needs: build - services: - mysql: - image: mysql:5.7.32 - env: - MYSQL_USER: testUser - MYSQL_PASSWORD: testPassword - MYSQL_DATABASE: test - MYSQL_ROOT_PASSWORD: password - ports: - - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Verify mysql connection - run: | - sudo apt-get install -y mysql-client - mysql --host 127.0.0.1 --port 3306 -uroot -ppassword -e "SHOW DATABASES" - - name: Cache Gradle Dependencies - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # Only rebuild cache if build.gradle is changed - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: ${{ runner.os }}-gradle - - name: Run test group ${{ matrix.test-group }} - # Write retry logic as integration tests can fail due to timing out or network problems - run: | - ./gradlew getGroupedTests -PgroupName="${{matrix.test-group}}" > temp.txt - TASKS=$(sed -n 's/CI Task: //p' temp.txt) - echo $TASKS - - n=0 - until [ "$n" -ge 3 ] - do - ./gradlew -PskipTestGroup=disabledOnCI $GOBBLIN_GRADLE_OPTS $TASKS -Dorg.gradle.parallel=false && break - n=$((n+1)) - if [[ $n -lt 3 ]]; then - echo "Tests failed, retry attempt number $n" - else - exit 1 - fi - sleep 10 - done diff --git a/.github/workflows/docker_build_publish.yaml b/.github/workflows/docker_build_publish.yaml deleted file mode 100644 index 4c2ba93..0000000 --- a/.github/workflows/docker_build_publish.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -name: Build and Publish Docker image -on: - push: - # Publish only on `master` - branches: - - master - pull_request: - branches: - - master - paths: - - 'gobblin-docker/**' - - '.github/workflows/docker_build_publish.yaml' - release: - types: [published, edited] - -env: - IMAGE_NAME: apache/gobblin - -jobs: - build_and_push_to_registry: - name: Build docker images and publish to DockerHub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Build Docker Tag - id: build_tag - run: | - SHA=`echo ${{ github.sha }} | head -c 7` - if [[ ${{ github.event_name }} == 'release' ]]; then - TAG="${{ env.IMAGE_NAME }}:sha-$SHA, ${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}, ${{ env.IMAGE_NAME }}:latest" - else - TAG="${{ env.IMAGE_NAME }}:sha-$SHA" - fi - echo "tag=$TAG" - echo "::set-output name=tag::$TAG" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: github.event_name != 'pull_request' - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Login to GitHub Container Registry - # if: github.event_name != 'pull_request' - # uses: docker/login-action@v1 - # with: - # registry: ghcr.io - # username: ${{ github.repository_owner }} - # password: ${{ secrets.CR_PAT }} - - name: Build Images and Publish - uses: docker/build-push-action@v2 - with: - tags: ${{ steps.build_tag.outputs.tag }} - push: ${{ github.event_name == 'release' }} - file: ./gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5e3e366 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Release and Publish +on: + push: + branches: + - 'master' + paths-ignore: + - 'docs/**' + - '**.md' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Needed to get all tags + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Release + uses: ./.github/actions/release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_KEY: ${{ secrets.ARTIFACTORY_KEY }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 427e5fb..0000000 --- a/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ -language: java - -dist: bionic -sudo: required - -jdk: - - openjdk8 - -addons: - apt: - packages: - - libaio-dev - - libdbus-glib-1-dev - - xsltproc - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - -before_install: - -services: - - xvfb - - mysql - -stages: - - test - - name: deploy - if: branch = master - -before_script: - - mysql -uroot -e "create user testUser identified by 'testPassword';" - - mysql -uroot -e "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('password')" - -script: - - travis_retry ./travis/test.sh - - travis_retry ./gradlew jacocoTestReport -after_success: - - bash <(cat .codecov_bash) -after_failure: ./travis/junit-errors-to-stdout.sh - -jobs: - include: - - stage: deploy - env: RUN_TEST_GROUP=none - install: skip - script: - - travis_retry ./travis/bintrayDeploy.sh - -env: - jobs: - - RUN_TEST_GROUP=build - - RUN_TEST_GROUP=default - - RUN_TEST_GROUP=group1 - - RUN_TEST_GROUP=coverage - global: - - secure: U72nmzXq7kXcIabiwvrTF+WkNFQxov2ACd8oPxWSHsyzRsfVJN42vT7gS3dLbH5G5claXG2p+rur4ueVffzYSwJ8B9OP6gTB8sNJnGr9zyxni4OJchyKqOYuj+UBpEQC/7qXKMCKnuJndsf1OvndDh/V1SH0DSSUuA6mDtgO/eM= - - secure: WiK7tyFV68xdkIfUlWreUHgEGGjaCBd73O4SbjE9AsbqqF7D+Iu8iRo1OhKQj+6eajUH9Eoev9rVN74FQgUfeNzrOkYsgDysXmyZ7+UxFokijFcATJmIBompA3dySGU2qXeKbJMNuUjXgrRIludaV6h2ahL6Fji42cgK4I3s2qs= - -jdk: - - openjdk8 diff --git a/README.md b/README.md index bc25839..60f988e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # Data Integration Library +[![Build Status](https://github.com/linkedin/data-integration-library/actions/workflows/build-and-test.yml/badge.svg?branch=master)](https://github.com/linkedin/data-integration-library/actions?query=workflow%3A%22Build+and+Run+Tests%22+branch%3Amaster+event%3Apush) +[![Release Status](https://github.com/linkedin/data-integration-library/actions/workflows/release.yml/badge.svg?branch=master)](https://github.com/linkedin/data-integration-library/actions?query=workflow%3A%22Release+and+Publish%22+branch%3Amaster+event%3Apush) LinkedIn Data Integration Library (DIL) is a collection of generic data integration components that can be mix-and-matched to form powerful ready-to-use connectors, which can then be used by data integration frameworks like [Apache Gobblin](https://gobblin.apache.org) or event processing frameworks like [Apache Kafka](https://kafka.apache.org/) to ingress or egress data between cloud services or APIs. @@ -22,10 +24,10 @@ If building the distribution with tests turned on: # Instructions to build the distribution 1. Extract the archive file to your local directory. -2. Skip tests and build the distribution: -Run `./gradlew build -x findbugsMain -x test -x rat -x checkstyleMain` -The distribution will be created in build/distribution directory. -(or) -3. Run tests and build the distribution (requires Maven): -Run `./gradlew build` -The distribution will be created in build/distribution directory. \ No newline at end of file +2. Set JAVA_HOME +3. Build +* Skip tests and build the distribution +> `./gradlew build -x findbugsMain -x test -x rat -x checkstyleMain` + +* Tests and build the distribution (requires Maven): +> `./gradlew build` diff --git a/build.gradle b/build.gradle index d6f6071..218aa80 100644 --- a/build.gradle +++ b/build.gradle @@ -2,129 +2,91 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. - -apply from: 'gradle/scripts/environment.gradle' - buildscript { - apply from: 'gradle/scripts/repositories.gradle' - apply from: 'gradle/scripts/defaultBuildProperties.gradle' - apply from: 'gradle/scripts/computeVersions.gradle' - apply from: 'gradle/scripts/ci-support.gradle' - - apply from: file('gradle/scripts/buildscript.gradle'), to: buildscript + ext.avroVersion = '1.8.1' + ext.awsVersion = '2.10.15' + ext.findBugsVersion = '3.0.0' + ext.gobblinVersion = '0.15.0-dev-9467' + ext.hadoopVersion = '2.3.0' + ext.hiveVersion = '1.0.1' + ext.javaVersion = JavaVersion.VERSION_1_8 + ext.junitJupiterVersion = '5.6.1' + ext.log4jVersion = '1.2.17' + ext.pegasusVersion = '28.3.7' + ext.slf4jVersion = '1.7.21' + apply from: 'gradle/scripts/repositories.gradle' buildscript.repositories.addAll(project.repositories) dependencies { classpath 'org.apache.ant:ant:1.9.4' - classpath 'gradle.plugin.org.inferred:gradle-processors:1.1.2' classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:1.0.2' - classpath 'io.spring.gradle:dependency-management-plugin:0.6.0.RELEASE' classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8' classpath "gradle.plugin.nl.javadude.gradle.plugins:license-gradle-plugin:0.14.0" - classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.+' + classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:latest.release' classpath 'com.github.jengelman.gradle.plugins:shadow:5.2.0' + classpath "org.shipkit:shipkit-auto-version:latest.release" + classpath "org.shipkit:shipkit-changelog:latest.release" } - - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } - } -apply plugin: "com.github.hierynomus.license" - -downloadLicenses { - includeProjectDependencies = true - - ext.apacheTwo = license('Apache License, Version 2.0', 'http://opensource.org/licenses/Apache-2.0') - ext.bsd = license('BSD License', 'http://www.opensource.org/licenses/bsd-license.php') - ext.bsd3 = license('BSD 3-clause', 'https://opensource.org/licenses/BSD-3-Clause') - ext.cddlGplv2 = license('CDDL + GPLv2 with classpath exception', 'https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html') - ext.spl = license('SPL', 'http://opensource.org/licenses/SPL-1.0'); - ext.epl = license('Eclipse Public License - Version 1.0', 'https://opensource.org/licenses/EPL-1.0') +apply from: 'gradle/scripts/dependencyDefinitions.gradle' +apply from: 'gradle/scripts/configureSubprojects.gradle' +apply from: 'gradle/scripts/jacoco-coveralls-support.gradle' +apply from: 'gradle/scripts/testSetup.gradle' +apply from: "gradle/release.gradle" - licenses = [ - (group('com.linkedin.gobblin')) : apacheTwo, - // https://zookeeper.apache.org/ - (group('org.apache.zookeeper')) : apacheTwo, - // http://testng.org/doc/ - (group('org.testng:testng')) : apacheTwo, - // https://db.apache.org/jdo/index.html - (group('javax.jdo')) : apacheTwo, - // Based on other javax licenses - (group('javax.servlet.jsp')) : cddlGplv2, - (group('javax.servlet')) : cddlGplv2, - (group('javax.transaction')) : cddlGplv2, - // http://commons.apache.org/proper/commons-beanutils/ - (group('commons-beanutils')) : apacheTwo, - // http://jakarta.apache.org/regexp/ - (group('regexp')) : apacheTwo, - // http://forge.ow2.org/projects/asm/ - (group('asm')) : bsd, - // https://github.com/codehaus/jettison/blob/master/pom.xml - (group('org.codehaus.jettison')) : apacheTwo, - // https://github.com/stephenc/jcip-annotations/blob/master/LICENSE.txt - (group('net.jcip')) : apacheTwo, - // https://github.com/google/re2j/blob/master/LICENSE - (group('com.google.re2j')) : bsd3, - // https://github.com/findbugsproject/findbugs/issues/128 - (group('com.google.code.findbugs')) : bsd3, - // http://www.beanshell.org/license.html - (group('org.beanshell')) : spl, - // http://www.mchange.com/projects/c3p0/ - (group('c3p0')) : epl, - 'antlr-runtime-3.4.jar' : apacheTwo - ] - aliases = [ - (apacheTwo) : ['The Apache Software License, Version 2.0', 'Apache 2', 'Apache License Version 2.0', - 'Apache License, Version 2.0', 'Apache License 2.0', 'Apache Version 2.0, January 2004', - license('Apache License', 'http://www.apache.org/licenses/LICENSE-2.0')], - (bsd) : ['BSD', license('New BSD License', 'http://www.opensource.org/licenses/bsd-license.php')] - ] - dependencyConfiguration = 'compile' - excludeDependencies = [ - ] +allprojects { + apply plugin: 'idea' + apply plugin: 'eclipse' + apply plugin: 'checkstyle' } -apply from: 'gradle/scripts/repositories.gradle' - -apply plugin: 'org.inferred.processors' -apply plugin: 'io.spring.dependency-management' - -apply from: 'gradle/scripts/configureSubprojects.gradle' -apply from: 'gradle/scripts/idesSetup.gradle' -apply from: 'gradle/scripts/jacoco-coveralls-support.gradle' +subprojects { + dependencies { + // Gradle 5 compatibility + compileOnly externalDependency.lombok + testCompileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok + testAnnotationProcessor externalDependency.lombok + } + project.buildDir = new File(project.rootProject.buildDir, project.name) +} -apply from: 'gradle/scripts/dependencyDefinitions.gradle' -apply from: 'gradle/scripts/restli.gradle' +subprojects { sp -> + apply plugin: 'maven' -apply from: 'gradle/scripts/testSetup.gradle' -apply from: 'gradle/scripts/globalDependencies.gradle' -apply from: 'gradle/scripts/javaPlugin.gradle' + plugins.withType(JavaPlugin) { + dependencies { + testCompile externalDependency.testng + } -apply from: 'gradle/scripts/utilities.gradle' + tasks.withType(Test) { + useTestNG() + } -apply from: 'gradle/scripts/javadoc.gradle' -apply from: 'gradle/scripts/sourcesJar.gradle' + checkstyle { + configDirectory = file("${project.rootDir}/gradle/checkstyle") + sourceSets = [ getProject().sourceSets.main, getProject().sourceSets.test ] + toolVersion = "8.35" + maxWarnings = 0 + ignoreFailures = false + } + } -apply from: 'gradle/scripts/mavenPublishing.gradle' -apply from: 'gradle/scripts/nexusPublishing.gradle' -apply from: 'gradle/scripts/bintrayPublishing.gradle' -apply from: 'gradle/scripts/javaVersionCheck.gradle' -apply from: 'gradle/scripts/release.gradle' + afterEvaluate { + if (project.plugins.hasPlugin('java')) { + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + } + } -/* - * Hack for upgrading pegasus to version 11.0.0. For some reason, the gradle-plugins in - * that version fails to bring in "tools.jar" into the classpath. The rest.li team is actively - * seeking for a clean fix. This part will be removed later when the fix is ready - */ -allprojects { - tasks.matching { it.name == 'generateRestModel' }.all { - doFirst { - it.codegenClasspath += files("${System.getProperty('java.home')}/../lib/tools.jar") + afterEvaluate { + if (project.plugins.hasPlugin('pegasus')) { + dependencies { + dataTemplateCompile spec.product.pegasus.data + restClientCompile spec.product.pegasus.restliClient + } } } } diff --git a/dil/build.gradle b/cdi-core/build.gradle similarity index 87% rename from dil/build.gradle rename to cdi-core/build.gradle index 49e9c34..f5643a6 100644 --- a/dil/build.gradle +++ b/cdi-core/build.gradle @@ -8,11 +8,13 @@ jacocoTestReport { afterEvaluate { classDirectories = files(classDirectories.files.collect { fileTree(dir: it, - exclude: ['com/linkedin/dil/factory']) + exclude: ['com/linkedin/cdi/factory']) }) } } +apply from: "$rootDir/gradle/java-publishing.gradle" + dependencies { compile externalDependency.'gson' compile externalDependency.'gobblin-core' @@ -33,6 +35,7 @@ dependencies { compile externalDependency.'commonsLang3' compile externalDependency.'testng' compile externalDependency.'okhttp' + compile externalDependency.'jhyde' runtime externalDependency.'gobblin-azkaban' runtime externalDependency.'gobblin-kafka-08' @@ -40,7 +43,7 @@ dependencies { runtime externalDependency.'gobblin-metrics' runtime externalDependency.'gobblin-data-management' runtime externalDependency.'gobblin-sql' - runtime externalDependency.gson + runtime externalDependency.'gson' runtime externalDependency.'jodaTime' testCompile externalDependency.'jmockit' @@ -73,12 +76,13 @@ configurations { all*.exclude group: 'org.apache.hive' all*.exclude group: 'ch.qos.logback' all*.exclude group: 'com.ibm.icu', module: 'icu4j' + all*.exclude group: 'org.pentaho' } shadowJar { zip64 true dependencies { - include dependency('com.linkedin.dil:.*') + include dependency('com.linkedin.cdi:.*') include dependency('com.squareup.retrofit:.*') include dependency('com.squareup.okhttp:.*') include dependency('com.squareup.okhttp3:.*') @@ -93,11 +97,11 @@ shadowJar { exclude 'org/slf4j/**' exclude 'javax/ws/rs/**' - relocate('com.squareup', 'gobblin.shaded.com.squareup') - relocate('retrofit', 'gobblin.shaded.retrofit') - relocate('okhttp3', 'gobblin.shaded.okhttp3') - relocate('okio', 'gobblin.shaded.okio') - relocate('org.testng', 'gobblin.shaded.org.testng') + relocate('com.squareup', 'cdi.shaded.com.squareup') + relocate('retrofit', 'cdi.shaded.retrofit') + relocate('okhttp3', 'cdi.shaded.okhttp3') + relocate('okio', 'cdi.shaded.okio') + relocate('org.testng', 'cdi.shaded.org.testng') mergeServiceFiles() } diff --git a/dil/src/main/java/com/linkedin/dil/configuration/MultistageProperties.java b/cdi-core/src/main/java/com/linkedin/cdi/configuration/MultistageProperties.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/configuration/MultistageProperties.java rename to cdi-core/src/main/java/com/linkedin/cdi/configuration/MultistageProperties.java index 266fb15..410cb93 100644 --- a/dil/src/main/java/com/linkedin/dil/configuration/MultistageProperties.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/configuration/MultistageProperties.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.configuration; +package com.linkedin.cdi.configuration; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.factory.DefaultS3ClientFactory; +import com.linkedin.cdi.factory.DefaultS3ClientFactory; /** @@ -285,7 +285,7 @@ public T getDefaultValue() { }, /** * extract.preprocessors define one or more preprocessor classes, see - * {@link com.linkedin.dil.preprocessor.GunzipProcessor} + * {@link com.linkedin.cdi.preprocessor.GunzipProcessor} */ MSTAGE_EXTRACT_PREPROCESSORS("ms.extract.preprocessors", String.class), /** @@ -333,12 +333,12 @@ public Long getMillis(State state) { }, /** * http.client.factory define an indirect way to specify the type of HttpClient to use. - * default = {@link com.linkedin.dil.factory.ApacheHttpClientFactory} + * default = {@link com.linkedin.cdi.factory.ApacheHttpClientFactory} */ MSTAGE_HTTP_CLIENT_FACTORY("ms.http.client.factory", String.class) { @Override public T getDefaultValue() { - return (T) "com.linkedin.dil.factory.ApacheHttpClientFactory"; + return (T) "com.linkedin.cdi.factory.ApacheHttpClientFactory"; } }, /** @@ -389,12 +389,12 @@ public T getDefaultValue() { MSTAGE_HTTP_STATUS_REASONS("ms.http.status.reasons", JsonObject.class), /** * jdbc.client.factory define an indirect way to specify the type of JDBC Client to use. - * default = {@link com.linkedin.dil.factory.DefaultJdbcClientFactory} + * default = {@link com.linkedin.cdi.factory.DefaultJdbcClientFactory} */ MSTAGE_JDBC_CLIENT_FACTORY("ms.jdbc.client.factory", String.class) { @Override public T getDefaultValue() { - return (T) "com.linkedin.dil.factory.DefaultJdbcClientFactory"; + return (T) "com.linkedin.cdi.factory.DefaultJdbcClientFactory"; } }, @@ -514,7 +514,7 @@ public T getDefaultValue() { MSTAGE_S3_CLIENT_FACTORY("ms.s3.client.factory", String.class) { @Override public T getDefaultValue() { - return (T) "com.linkedin.dil.factory.DefaultS3ClientFactory"; + return (T) "com.linkedin.cdi.factory.DefaultS3ClientFactory"; } }, /** @@ -638,7 +638,7 @@ public T getDefaultValue() { MSTAGE_SOURCE_SCHEMA_READER_FACTORY("ms.source.schema.reader.factory", String.class) { @Override public T getDefaultValue() { - return (T) "com.linkedin.dil.factory.reader.MetastoreReader"; + return (T) "com.linkedin.cdi.factory.reader.MetastoreReader"; } }, /** @@ -676,7 +676,7 @@ public T getDefaultValue() { MSTAGE_TARGET_SCHEMA_READER_FACTORY("ms.target.schema.reader.factory", String.class) { @Override public T getDefaultValue() { - return (T) "com.linkedin.dil.factory.reader.MetastoreReader"; + return (T) "com.linkedin.cdi.factory.reader.MetastoreReader"; } }, /** diff --git a/dil/src/main/java/com/linkedin/dil/configuration/StaticConstants.java b/cdi-core/src/main/java/com/linkedin/cdi/configuration/StaticConstants.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/configuration/StaticConstants.java rename to cdi-core/src/main/java/com/linkedin/cdi/configuration/StaticConstants.java index 57bc268..75d5aa7 100644 --- a/dil/src/main/java/com/linkedin/dil/configuration/StaticConstants.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/configuration/StaticConstants.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.configuration; +package com.linkedin.cdi.configuration; import com.google.gson.Gson; diff --git a/dil/src/main/java/com/linkedin/dil/connection/Connection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/Connection.java similarity index 90% rename from dil/src/main/java/com/linkedin/dil/connection/Connection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/Connection.java index c56ecf5..626054f 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/Connection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/Connection.java @@ -2,10 +2,10 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.util.WorkUnitStatus; /** * The connection interface defines core methods that an Extractor would call diff --git a/dil/src/main/java/com/linkedin/dil/connection/HdfsConnection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/HdfsConnection.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/connection/HdfsConnection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/HdfsConnection.java index 589035a..e7eb49b 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/HdfsConnection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/HdfsConnection.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -15,12 +15,12 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.HdfsKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.InputStreamUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.HdfsKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.InputStreamUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.extractor.filebased.FileBasedHelperException; import org.apache.gobblin.source.extractor.filebased.TimestampAwareFileBasedHelper; import org.apache.gobblin.source.extractor.hadoop.HadoopFsHelper; diff --git a/dil/src/main/java/com/linkedin/dil/connection/HttpConnection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/HttpConnection.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/connection/HttpConnection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/HttpConnection.java index 7156649..44b3e06 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/HttpConnection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/HttpConnection.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -21,15 +21,15 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.factory.HttpClientFactory; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.HttpKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.HttpRequestMethod; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.factory.HttpClientFactory; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.HttpKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.HttpRequestMethod; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.http.Header; import org.apache.http.HeaderElement; import org.apache.http.HttpResponse; @@ -37,7 +37,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/connection/JdbcConnection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/JdbcConnection.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/connection/JdbcConnection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/JdbcConnection.java index 9d24103..f24bc5a 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/JdbcConnection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/JdbcConnection.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -24,16 +24,16 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.factory.JdbcClientFactory; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JdbcKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.JdbcUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.SchemaBuilder; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.factory.JdbcClientFactory; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JdbcKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.JdbcUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.SchemaBuilder; +import com.linkedin.cdi.util.WorkUnitStatus; /** * JdbcConnection creates transmission channel with JDBC data provider or JDBC data receiver, diff --git a/dil/src/main/java/com/linkedin/dil/connection/MultistageConnection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/MultistageConnection.java similarity index 92% rename from dil/src/main/java/com/linkedin/dil/connection/MultistageConnection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/MultistageConnection.java index 3275b9a..da9cc92 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/MultistageConnection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/MultistageConnection.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.gson.JsonObject; import lombok.Getter; @@ -10,11 +10,11 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.VariableUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.VariableUtils; +import com.linkedin.cdi.util.WorkUnitStatus; /** * MultistageConnection is a basic implementation of Connection interface. diff --git a/dil/src/main/java/com/linkedin/dil/connection/S3Connection.java b/cdi-core/src/main/java/com/linkedin/cdi/connection/S3Connection.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/connection/S3Connection.java rename to cdi-core/src/main/java/com/linkedin/cdi/connection/S3Connection.java index 0db233d..0030ea9 100644 --- a/dil/src/main/java/com/linkedin/dil/connection/S3Connection.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/connection/S3Connection.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.common.collect.Lists; import java.net.URI; @@ -14,15 +14,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.factory.S3ClientFactory; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.keys.S3Keys; -import com.linkedin.dil.util.EncryptionUtils; -import com.linkedin.dil.util.InputStreamUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.factory.S3ClientFactory; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.keys.S3Keys; +import com.linkedin.cdi.util.EncryptionUtils; +import com.linkedin.cdi.util.InputStreamUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; diff --git a/dil/src/main/java/com/linkedin/dil/converter/AvroNormalizerConverter.java b/cdi-core/src/main/java/com/linkedin/cdi/converter/AvroNormalizerConverter.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/converter/AvroNormalizerConverter.java rename to cdi-core/src/main/java/com/linkedin/cdi/converter/AvroNormalizerConverter.java index c3dc585..ccfa957 100644 --- a/dil/src/main/java/com/linkedin/dil/converter/AvroNormalizerConverter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/converter/AvroNormalizerConverter.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.converter; +package com.linkedin.cdi.converter; import com.google.common.base.Optional; import com.google.gson.JsonArray; @@ -19,12 +19,12 @@ import org.apache.gobblin.converter.SchemaConversionException; import org.apache.gobblin.converter.SingleRecordIterable; import org.apache.gobblin.converter.avro.UnsupportedDateTypeException; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.util.AvroSchemaUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.util.AvroSchemaUtils; import org.apache.gobblin.util.AvroUtils; import org.apache.gobblin.util.EmptyIterable; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/converter/JsonNormalizerConverter.java b/cdi-core/src/main/java/com/linkedin/cdi/converter/JsonNormalizerConverter.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/converter/JsonNormalizerConverter.java rename to cdi-core/src/main/java/com/linkedin/cdi/converter/JsonNormalizerConverter.java index ee97722..c92f549 100644 --- a/dil/src/main/java/com/linkedin/dil/converter/JsonNormalizerConverter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/converter/JsonNormalizerConverter.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.converter; +package com.linkedin.cdi.converter; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -13,11 +13,11 @@ import org.apache.gobblin.configuration.WorkUnitState; import org.apache.gobblin.converter.Converter; import org.apache.gobblin.converter.SingleRecordIterable; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.util.JsonUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.util.JsonUtils; import org.apache.gobblin.util.EmptyIterable; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/exception/RetriableAuthenticationException.java b/cdi-core/src/main/java/com/linkedin/cdi/exception/RetriableAuthenticationException.java similarity index 91% rename from dil/src/main/java/com/linkedin/dil/exception/RetriableAuthenticationException.java rename to cdi-core/src/main/java/com/linkedin/cdi/exception/RetriableAuthenticationException.java index fb086ba..6ae9122 100644 --- a/dil/src/main/java/com/linkedin/dil/exception/RetriableAuthenticationException.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/exception/RetriableAuthenticationException.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.exception; +package com.linkedin.cdi.exception; /** * An {@link Exception} thrown when it can be retried diff --git a/dil/src/main/java/com/linkedin/dil/extractor/AvroExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/AvroExtractor.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/extractor/AvroExtractor.java rename to cdi-core/src/main/java/com/linkedin/cdi/extractor/AvroExtractor.java index 6dbe5d3..76cf7fa 100644 --- a/dil/src/main/java/com/linkedin/dil/extractor/AvroExtractor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/extractor/AvroExtractor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.annotations.VisibleForTesting; import com.google.gson.JsonArray; @@ -22,14 +22,14 @@ import org.apache.commons.lang3.StringUtils; import org.apache.gobblin.configuration.WorkUnitState; import org.apache.gobblin.converter.avro.UnsupportedDateTypeException; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.filter.AvroSchemaBasedFilter; -import com.linkedin.dil.keys.AvroExtractorKeys; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.AvroSchemaUtils; -import com.linkedin.dil.util.JsonIntermediateSchema; -import com.linkedin.dil.util.SchemaUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.filter.AvroSchemaBasedFilter; +import com.linkedin.cdi.keys.AvroExtractorKeys; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.AvroSchemaUtils; +import com.linkedin.cdi.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.SchemaUtils; import org.apache.gobblin.util.AvroUtils; import org.testng.Assert; diff --git a/dil/src/main/java/com/linkedin/dil/extractor/CsvExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/CsvExtractor.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/extractor/CsvExtractor.java rename to cdi-core/src/main/java/com/linkedin/cdi/extractor/CsvExtractor.java index d78b4c1..df5664d 100644 --- a/dil/src/main/java/com/linkedin/dil/extractor/CsvExtractor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/extractor/CsvExtractor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Doubles; @@ -33,18 +33,18 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.filter.CsvSchemaBasedFilter; -import com.linkedin.dil.keys.CsvExtractorKeys; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.preprocessor.InputStreamProcessor; -import com.linkedin.dil.preprocessor.StreamProcessor; -import com.linkedin.dil.util.CsvUtils; -import com.linkedin.dil.util.JsonIntermediateSchema; -import com.linkedin.dil.util.SchemaBuilder; -import com.linkedin.dil.util.SchemaUtils; -import com.linkedin.dil.util.VariableUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.filter.CsvSchemaBasedFilter; +import com.linkedin.cdi.keys.CsvExtractorKeys; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.preprocessor.InputStreamProcessor; +import com.linkedin.cdi.preprocessor.StreamProcessor; +import com.linkedin.cdi.util.CsvUtils; +import com.linkedin.cdi.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.SchemaBuilder; +import com.linkedin.cdi.util.SchemaUtils; +import com.linkedin.cdi.util.VariableUtils; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Period; diff --git a/dil/src/main/java/com/linkedin/dil/extractor/FileDumpExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/FileDumpExtractor.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/extractor/FileDumpExtractor.java rename to cdi-core/src/main/java/com/linkedin/cdi/extractor/FileDumpExtractor.java index d030b7d..81941e6 100644 --- a/dil/src/main/java/com/linkedin/dil/extractor/FileDumpExtractor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/extractor/FileDumpExtractor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -18,15 +18,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.FileDumpExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.preprocessor.InputStreamProcessor; -import com.linkedin.dil.preprocessor.OutputStreamProcessor; -import com.linkedin.dil.preprocessor.StreamProcessor; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.VariableUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.FileDumpExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.preprocessor.InputStreamProcessor; +import com.linkedin.cdi.preprocessor.OutputStreamProcessor; +import com.linkedin.cdi.preprocessor.StreamProcessor; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.VariableUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/dil/src/main/java/com/linkedin/dil/extractor/JsonExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/JsonExtractor.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/extractor/JsonExtractor.java rename to cdi-core/src/main/java/com/linkedin/cdi/extractor/JsonExtractor.java index 1e0c7b8..94fd0cf 100644 --- a/dil/src/main/java/com/linkedin/dil/extractor/JsonExtractor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/extractor/JsonExtractor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; @@ -27,15 +27,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.filter.JsonSchemaBasedFilter; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.keys.JsonExtractorKeys; -import com.linkedin.dil.util.EncryptionUtils; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.SchemaBuilder; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.filter.JsonSchemaBasedFilter; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.keys.JsonExtractorKeys; +import com.linkedin.cdi.util.EncryptionUtils; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.SchemaBuilder; import org.testng.Assert; diff --git a/dil/src/main/java/com/linkedin/dil/extractor/MultistageExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/MultistageExtractor.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/extractor/MultistageExtractor.java rename to cdi-core/src/main/java/com/linkedin/cdi/extractor/MultistageExtractor.java index bb9defc..ed51cc4 100644 --- a/dil/src/main/java/com/linkedin/dil/extractor/MultistageExtractor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/extractor/MultistageExtractor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.collect.ImmutableList; import com.google.gson.JsonArray; @@ -29,25 +29,25 @@ import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.filter.JsonSchemaBasedFilter; -import com.linkedin.dil.filter.MultistageSchemaBasedFilter; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.preprocessor.StreamProcessor; -import com.linkedin.dil.util.DateTimeUtils; -import com.linkedin.dil.util.EncryptionUtils; -import com.linkedin.dil.util.HdfsReader; -import com.linkedin.dil.util.InputStreamUtils; -import com.linkedin.dil.util.JsonIntermediateSchema; -import com.linkedin.dil.util.JsonParameter; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.SchemaBuilder; -import com.linkedin.dil.util.VariableUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.filter.JsonSchemaBasedFilter; +import com.linkedin.cdi.filter.MultistageSchemaBasedFilter; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.preprocessor.StreamProcessor; +import com.linkedin.cdi.util.DateTimeUtils; +import com.linkedin.cdi.util.EncryptionUtils; +import com.linkedin.cdi.util.HdfsReader; +import com.linkedin.cdi.util.InputStreamUtils; +import com.linkedin.cdi.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.JsonParameter; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.SchemaBuilder; +import com.linkedin.cdi.util.VariableUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.extractor.Extractor; import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.joda.time.DateTime; @@ -57,7 +57,7 @@ import org.joda.time.format.DateTimeFormatter; import org.testng.Assert; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/factory/ApacheHttpClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/ApacheHttpClientFactory.java similarity index 93% rename from dil/src/main/java/com/linkedin/dil/factory/ApacheHttpClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/ApacheHttpClientFactory.java index 3b08e2d..06c67e4 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/ApacheHttpClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/ApacheHttpClientFactory.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import org.apache.gobblin.configuration.State; import org.apache.http.client.HttpClient; diff --git a/dil/src/main/java/com/linkedin/dil/factory/DefaultJdbcClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultJdbcClientFactory.java similarity index 91% rename from dil/src/main/java/com/linkedin/dil/factory/DefaultJdbcClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultJdbcClientFactory.java index 8af4a8b..61321fd 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/DefaultJdbcClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultJdbcClientFactory.java @@ -2,12 +2,12 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import java.sql.Connection; import java.sql.DriverManager; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.util.EncryptionUtils; +import com.linkedin.cdi.util.EncryptionUtils; /** diff --git a/dil/src/main/java/com/linkedin/dil/factory/DefaultS3ClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultS3ClientFactory.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/factory/DefaultS3ClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultS3ClientFactory.java index f19d7a2..93db465 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/DefaultS3ClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/DefaultS3ClientFactory.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import org.apache.gobblin.configuration.State; import software.amazon.awssdk.http.SdkHttpClient; diff --git a/dil/src/main/java/com/linkedin/dil/factory/HttpClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/HttpClientFactory.java similarity index 92% rename from dil/src/main/java/com/linkedin/dil/factory/HttpClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/HttpClientFactory.java index ae1f698..775736c 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/HttpClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/HttpClientFactory.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import org.apache.gobblin.configuration.State; import org.apache.http.client.HttpClient; diff --git a/dil/src/main/java/com/linkedin/dil/factory/JdbcClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/JdbcClientFactory.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/factory/JdbcClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/JdbcClientFactory.java index 384e385..b308815 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/JdbcClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/JdbcClientFactory.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import java.sql.Connection; import org.apache.gobblin.configuration.State; diff --git a/dil/src/main/java/com/linkedin/dil/factory/S3ClientFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/S3ClientFactory.java similarity index 93% rename from dil/src/main/java/com/linkedin/dil/factory/S3ClientFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/S3ClientFactory.java index 412653f..3e5de55 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/S3ClientFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/S3ClientFactory.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import org.apache.gobblin.configuration.State; import software.amazon.awssdk.http.SdkHttpClient; diff --git a/dil/src/main/java/com/linkedin/dil/factory/SchemaReaderFactory.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/SchemaReaderFactory.java similarity index 85% rename from dil/src/main/java/com/linkedin/dil/factory/SchemaReaderFactory.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/SchemaReaderFactory.java index ad949a7..c1facdb 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/SchemaReaderFactory.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/SchemaReaderFactory.java @@ -2,12 +2,12 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import com.google.common.annotations.VisibleForTesting; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.factory.reader.SchemaReader; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.factory.reader.SchemaReader; /** diff --git a/dil/src/main/java/com/linkedin/dil/factory/reader/JsonFileReader.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/reader/JsonFileReader.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/factory/reader/JsonFileReader.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/reader/JsonFileReader.java index 7aede16..05ad750 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/reader/JsonFileReader.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/reader/JsonFileReader.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory.reader; +package com.linkedin.cdi.factory.reader; import com.google.gson.JsonElement; import org.apache.gobblin.configuration.State; diff --git a/dil/src/main/java/com/linkedin/dil/factory/reader/SchemaReader.java b/cdi-core/src/main/java/com/linkedin/cdi/factory/reader/SchemaReader.java similarity index 91% rename from dil/src/main/java/com/linkedin/dil/factory/reader/SchemaReader.java rename to cdi-core/src/main/java/com/linkedin/cdi/factory/reader/SchemaReader.java index 41c9a62..6b51f17 100644 --- a/dil/src/main/java/com/linkedin/dil/factory/reader/SchemaReader.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/factory/reader/SchemaReader.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory.reader; +package com.linkedin.cdi.factory.reader; import com.google.gson.JsonElement; import org.apache.gobblin.configuration.State; diff --git a/dil/src/main/java/com/linkedin/dil/filter/AvroSchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/AvroSchemaBasedFilter.java similarity index 89% rename from dil/src/main/java/com/linkedin/dil/filter/AvroSchemaBasedFilter.java rename to cdi-core/src/main/java/com/linkedin/cdi/filter/AvroSchemaBasedFilter.java index d6bf695..a332d0d 100644 --- a/dil/src/main/java/com/linkedin/dil/filter/AvroSchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/AvroSchemaBasedFilter.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; import com.google.common.base.Optional; import lombok.SneakyThrows; @@ -11,9 +11,9 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.keys.AvroExtractorKeys; -import com.linkedin.dil.util.AvroSchemaUtils; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.keys.AvroExtractorKeys; +import com.linkedin.cdi.util.AvroSchemaUtils; +import com.linkedin.cdi.util.JsonIntermediateSchema; import org.apache.gobblin.util.AvroUtils; diff --git a/dil/src/main/java/com/linkedin/dil/filter/CsvSchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/CsvSchemaBasedFilter.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/filter/CsvSchemaBasedFilter.java rename to cdi-core/src/main/java/com/linkedin/cdi/filter/CsvSchemaBasedFilter.java index d2cf2c5..a52d6f2 100644 --- a/dil/src/main/java/com/linkedin/dil/filter/CsvSchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/CsvSchemaBasedFilter.java @@ -2,13 +2,13 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; import java.util.Arrays; import java.util.Set; import lombok.extern.slf4j.Slf4j; -import com.linkedin.dil.keys.CsvExtractorKeys; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.keys.CsvExtractorKeys; +import com.linkedin.cdi.util.JsonIntermediateSchema; /** diff --git a/dil/src/main/java/com/linkedin/dil/filter/JsonSchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java similarity index 93% rename from dil/src/main/java/com/linkedin/dil/filter/JsonSchemaBasedFilter.java rename to cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java index 4facfb0..85a9031 100644 --- a/dil/src/main/java/com/linkedin/dil/filter/JsonSchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java @@ -2,16 +2,16 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import java.util.Map; -import com.linkedin.dil.util.JsonElementTypes; -import com.linkedin.dil.util.JsonIntermediateSchema; -import com.linkedin.dil.util.JsonUtils; +import com.linkedin.cdi.util.JsonElementTypes; +import com.linkedin.cdi.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.JsonUtils; /** diff --git a/dil/src/main/java/com/linkedin/dil/filter/MultistageSchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilter.java similarity index 86% rename from dil/src/main/java/com/linkedin/dil/filter/MultistageSchemaBasedFilter.java rename to cdi-core/src/main/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilter.java index fdc15ec..8ba90a9 100644 --- a/dil/src/main/java/com/linkedin/dil/filter/MultistageSchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilter.java @@ -2,9 +2,9 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.JsonIntermediateSchema; /** diff --git a/dil/src/main/java/com/linkedin/dil/filter/SchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/SchemaBasedFilter.java similarity index 89% rename from dil/src/main/java/com/linkedin/dil/filter/SchemaBasedFilter.java rename to cdi-core/src/main/java/com/linkedin/cdi/filter/SchemaBasedFilter.java index 8a3e673..bc37a5f 100644 --- a/dil/src/main/java/com/linkedin/dil/filter/SchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/SchemaBasedFilter.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; /** diff --git a/dil/src/main/java/com/linkedin/dil/keys/AvroExtractorKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/AvroExtractorKeys.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/keys/AvroExtractorKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/AvroExtractorKeys.java index 5a0c6ef..0958832 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/AvroExtractorKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/AvroExtractorKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import java.util.List; @@ -14,7 +14,7 @@ import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.apache.gobblin.source.workunit.WorkUnit; diff --git a/dil/src/main/java/com/linkedin/dil/keys/CsvExtractorKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/CsvExtractorKeys.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/keys/CsvExtractorKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/CsvExtractorKeys.java index 62f8339..ba77212 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/CsvExtractorKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/CsvExtractorKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import java.util.ArrayDeque; @@ -18,7 +18,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.apache.gobblin.source.workunit.WorkUnit; diff --git a/dil/src/main/java/com/linkedin/dil/keys/ExtractorKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/ExtractorKeys.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/keys/ExtractorKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/ExtractorKeys.java index b2dcfe3..5ee5a35 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/ExtractorKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/ExtractorKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import com.google.gson.JsonArray; @@ -14,8 +14,8 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.preprocessor.StreamProcessor; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.preprocessor.StreamProcessor; import org.apache.gobblin.source.workunit.WorkUnit; import org.joda.time.DateTime; diff --git a/dil/src/main/java/com/linkedin/dil/keys/FileDumpExtractorKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/FileDumpExtractorKeys.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/keys/FileDumpExtractorKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/FileDumpExtractorKeys.java index d7accf0..c5b0c17 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/FileDumpExtractorKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/FileDumpExtractorKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import lombok.AccessLevel; import lombok.Getter; diff --git a/dil/src/main/java/com/linkedin/dil/keys/HdfsKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/HdfsKeys.java similarity index 91% rename from dil/src/main/java/com/linkedin/dil/keys/HdfsKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/HdfsKeys.java index 23873a8..16b9955 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/HdfsKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/HdfsKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import java.util.List; @@ -11,7 +11,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; /** diff --git a/dil/src/main/java/com/linkedin/dil/keys/HttpKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/HttpKeys.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/keys/HttpKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/HttpKeys.java index 12ca627..3180542 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/HttpKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/HttpKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import com.google.gson.JsonObject; @@ -14,8 +14,8 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.util.HttpRequestMethod; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.util.HttpRequestMethod; /** diff --git a/dil/src/main/java/com/linkedin/dil/keys/JdbcKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/JdbcKeys.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/keys/JdbcKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/JdbcKeys.java index e0d1d84..3fd8495 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/JdbcKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/JdbcKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import com.google.gson.JsonObject; @@ -12,7 +12,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; /** diff --git a/dil/src/main/java/com/linkedin/dil/keys/JobKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/JobKeys.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/keys/JobKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/JobKeys.java index 73263af..a4a6027 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/JobKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/JobKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -22,17 +22,17 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.factory.SchemaReaderFactory; -import com.linkedin.dil.factory.reader.SchemaReader; -import com.linkedin.dil.util.DateTimeUtils; -import com.linkedin.dil.util.HdfsReader; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.WorkUnitPartitionTypes; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.factory.SchemaReaderFactory; +import com.linkedin.cdi.factory.reader.SchemaReader; +import com.linkedin.cdi.util.DateTimeUtils; +import com.linkedin.cdi.util.HdfsReader; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.WorkUnitPartitionTypes; import org.joda.time.DateTime; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/keys/JsonExtractorKeys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/JsonExtractorKeys.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/keys/JsonExtractorKeys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/JsonExtractorKeys.java index 9f24359..1b3a7ba 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/JsonExtractorKeys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/JsonExtractorKeys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.common.collect.Lists; import com.google.gson.JsonElement; @@ -14,7 +14,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.apache.gobblin.source.workunit.WorkUnit; diff --git a/dil/src/main/java/com/linkedin/dil/keys/S3Keys.java b/cdi-core/src/main/java/com/linkedin/cdi/keys/S3Keys.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/keys/S3Keys.java rename to cdi-core/src/main/java/com/linkedin/cdi/keys/S3Keys.java index 7608c1d..0c89614 100644 --- a/dil/src/main/java/com/linkedin/dil/keys/S3Keys.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/keys/S3Keys.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import lombok.AccessLevel; import lombok.Getter; diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/GpgDecryptProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessor.java similarity index 90% rename from dil/src/main/java/com/linkedin/dil/preprocessor/GpgDecryptProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessor.java index 81301eb..25a9537 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/GpgDecryptProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; @@ -13,8 +13,8 @@ import org.apache.commons.io.FilenameUtils; import org.apache.gobblin.annotation.Alias; import org.apache.gobblin.codec.StreamCodec; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.util.EncryptionUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.util.EncryptionUtils; /** diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/GpgEncryptProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessor.java similarity index 93% rename from dil/src/main/java/com/linkedin/dil/preprocessor/GpgEncryptProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessor.java index 6df7173..b74bf9e 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/GpgEncryptProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; @@ -11,7 +11,7 @@ import lombok.Setter; import org.apache.commons.io.FilenameUtils; import org.apache.gobblin.codec.StreamCodec; -import com.linkedin.dil.util.EncryptionUtils; +import com.linkedin.cdi.util.EncryptionUtils; /** diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/GunzipProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GunzipProcessor.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/preprocessor/GunzipProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GunzipProcessor.java index 57f3cfe..5dbb36a 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/GunzipProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/GunzipProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/InputStreamProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/InputStreamProcessor.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/preprocessor/InputStreamProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/InputStreamProcessor.java index a783186..fd8b601 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/InputStreamProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/InputStreamProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/OutputStreamProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/OutputStreamProcessor.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/preprocessor/OutputStreamProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/OutputStreamProcessor.java index c932925..4258a6e 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/OutputStreamProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/OutputStreamProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; diff --git a/dil/src/main/java/com/linkedin/dil/preprocessor/StreamProcessor.java b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/StreamProcessor.java similarity index 91% rename from dil/src/main/java/com/linkedin/dil/preprocessor/StreamProcessor.java rename to cdi-core/src/main/java/com/linkedin/cdi/preprocessor/StreamProcessor.java index 32b38fc..07fa468 100644 --- a/dil/src/main/java/com/linkedin/dil/preprocessor/StreamProcessor.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/preprocessor/StreamProcessor.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import java.io.Closeable; import java.io.IOException; diff --git a/dil/src/main/java/com/linkedin/dil/source/HdfsSource.java b/cdi-core/src/main/java/com/linkedin/cdi/source/HdfsSource.java similarity index 89% rename from dil/src/main/java/com/linkedin/dil/source/HdfsSource.java rename to cdi-core/src/main/java/com/linkedin/cdi/source/HdfsSource.java index 8319c33..08f278c 100644 --- a/dil/src/main/java/com/linkedin/dil/source/HdfsSource.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/source/HdfsSource.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import lombok.Getter; import lombok.Setter; @@ -11,9 +11,9 @@ import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.connection.HdfsConnection; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.HdfsKeys; +import com.linkedin.cdi.connection.HdfsConnection; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.HdfsKeys; import org.apache.gobblin.source.extractor.Extractor; diff --git a/dil/src/main/java/com/linkedin/dil/source/HttpSource.java b/cdi-core/src/main/java/com/linkedin/cdi/source/HttpSource.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/source/HttpSource.java rename to cdi-core/src/main/java/com/linkedin/cdi/source/HttpSource.java index 61f2428..1f8fb24 100644 --- a/dil/src/main/java/com/linkedin/dil/source/HttpSource.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/source/HttpSource.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -22,11 +22,11 @@ import org.apache.commons.codec.binary.Base64; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.HttpConnection; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.HttpKeys; -import com.linkedin.dil.util.EncryptionUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.HttpConnection; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.HttpKeys; +import com.linkedin.cdi.util.EncryptionUtils; import org.apache.gobblin.source.extractor.Extractor; diff --git a/dil/src/main/java/com/linkedin/dil/source/JdbcSource.java b/cdi-core/src/main/java/com/linkedin/cdi/source/JdbcSource.java similarity index 89% rename from dil/src/main/java/com/linkedin/dil/source/JdbcSource.java rename to cdi-core/src/main/java/com/linkedin/cdi/source/JdbcSource.java index 7974256..0a4082f 100644 --- a/dil/src/main/java/com/linkedin/dil/source/JdbcSource.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/source/JdbcSource.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import java.sql.Connection; import java.util.concurrent.ConcurrentHashMap; @@ -13,11 +13,11 @@ import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.JdbcConnection; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.JdbcKeys; -import com.linkedin.dil.util.CsvUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.JdbcConnection; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.JdbcKeys; +import com.linkedin.cdi.util.CsvUtils; import org.apache.gobblin.source.extractor.Extractor; diff --git a/dil/src/main/java/com/linkedin/dil/source/MultistageSource.java b/cdi-core/src/main/java/com/linkedin/cdi/source/MultistageSource.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/source/MultistageSource.java rename to cdi-core/src/main/java/com/linkedin/cdi/source/MultistageSource.java index be367e0..540c9af 100644 --- a/dil/src/main/java/com/linkedin/dil/source/MultistageSource.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/source/MultistageSource.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -31,12 +31,12 @@ import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.EndecoUtils; -import com.linkedin.dil.util.HdfsReader; -import com.linkedin.dil.util.WatermarkDefinition; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.EndecoUtils; +import com.linkedin.cdi.util.HdfsReader; +import com.linkedin.cdi.util.WatermarkDefinition; import org.apache.gobblin.source.extractor.Extractor; import org.apache.gobblin.source.extractor.WatermarkInterval; import org.apache.gobblin.source.extractor.extract.AbstractSource; @@ -46,7 +46,7 @@ import org.joda.time.DateTime; import org.testng.Assert; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/source/S3SourceV2.java b/cdi-core/src/main/java/com/linkedin/cdi/source/S3SourceV2.java similarity index 94% rename from dil/src/main/java/com/linkedin/dil/source/S3SourceV2.java rename to cdi-core/src/main/java/com/linkedin/cdi/source/S3SourceV2.java index 65eee49..2a3405b 100644 --- a/dil/src/main/java/com/linkedin/dil/source/S3SourceV2.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/source/S3SourceV2.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -18,11 +18,11 @@ import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.S3Connection; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.S3Keys; -import com.linkedin.dil.util.EndecoUtils; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.S3Connection; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.S3Keys; +import com.linkedin.cdi.util.EndecoUtils; import org.apache.gobblin.source.extractor.Extractor; import software.amazon.awssdk.regions.Region; diff --git a/dil/src/main/java/com/linkedin/dil/util/AvroSchemaUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/AvroSchemaUtils.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/AvroSchemaUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/AvroSchemaUtils.java index a236d94..35a16c9 100644 --- a/dil/src/main/java/com/linkedin/dil/util/AvroSchemaUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/AvroSchemaUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/main/java/com/linkedin/dil/util/CsvUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/CsvUtils.java similarity index 96% rename from dil/src/main/java/com/linkedin/dil/util/CsvUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/CsvUtils.java index cc1299e..c788b38 100644 --- a/dil/src/main/java/com/linkedin/dil/util/CsvUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/CsvUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; diff --git a/dil/src/main/java/com/linkedin/dil/util/Database.java b/cdi-core/src/main/java/com/linkedin/cdi/util/Database.java similarity index 97% rename from dil/src/main/java/com/linkedin/dil/util/Database.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/Database.java index 3bdfe11..7100eba 100644 --- a/dil/src/main/java/com/linkedin/dil/util/Database.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/Database.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.base.Preconditions; import java.net.URI; diff --git a/dil/src/main/java/com/linkedin/dil/util/DateTimeUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/DateTimeUtils.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/DateTimeUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/DateTimeUtils.java index f93e188..e102ec0 100644 --- a/dil/src/main/java/com/linkedin/dil/util/DateTimeUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/DateTimeUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.ImmutableMap; import java.util.Map; diff --git a/dil/src/main/java/com/linkedin/dil/util/EncryptionUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/EncryptionUtils.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/EncryptionUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/EncryptionUtils.java index be9e010..bb96849 100644 --- a/dil/src/main/java/com/linkedin/dil/util/EncryptionUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/EncryptionUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.JsonObject; import org.apache.gobblin.codec.StreamCodec; diff --git a/dil/src/main/java/com/linkedin/dil/util/EndecoUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/EndecoUtils.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/EndecoUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/EndecoUtils.java index 39fcab0..0105285 100644 --- a/dil/src/main/java/com/linkedin/dil/util/EndecoUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/EndecoUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.net.URLDecoder; import java.net.URLEncoder; diff --git a/dil/src/main/java/com/linkedin/dil/util/HdfsReader.java b/cdi-core/src/main/java/com/linkedin/cdi/util/HdfsReader.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/HdfsReader.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/HdfsReader.java index 2a3dc54..a0d54f0 100644 --- a/dil/src/main/java/com/linkedin/dil/util/HdfsReader.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/HdfsReader.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; @@ -29,7 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/util/HttpRequestMethod.java b/cdi-core/src/main/java/com/linkedin/cdi/util/HttpRequestMethod.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/HttpRequestMethod.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/HttpRequestMethod.java index 1f80876..75ac7b1 100644 --- a/dil/src/main/java/com/linkedin/dil/util/HttpRequestMethod.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/HttpRequestMethod.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/dil/src/main/java/com/linkedin/dil/util/InputStreamUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/InputStreamUtils.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/InputStreamUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/InputStreamUtils.java index 264f313..a6abb5c 100644 --- a/dil/src/main/java/com/linkedin/dil/util/InputStreamUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/InputStreamUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/dil/src/main/java/com/linkedin/dil/util/JdbcUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JdbcUtils.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/JdbcUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/JdbcUtils.java index e2128de..14c4b2b 100644 --- a/dil/src/main/java/com/linkedin/dil/util/JdbcUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JdbcUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.ImmutableMap; import java.sql.Blob; diff --git a/dil/src/main/java/com/linkedin/dil/util/JsonElementTypes.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/JsonElementTypes.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java index 319b22b..d94030d 100644 --- a/dil/src/main/java/com/linkedin/dil/util/JsonElementTypes.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; diff --git a/dil/src/main/java/com/linkedin/dil/util/JsonIntermediateSchema.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/JsonIntermediateSchema.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java index f937927..453cf64 100644 --- a/dil/src/main/java/com/linkedin/dil/util/JsonIntermediateSchema.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -16,7 +16,7 @@ import lombok.NonNull; import lombok.Setter; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/util/JsonParameter.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonParameter.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/JsonParameter.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/JsonParameter.java index 4cb3dd4..aba18a6 100644 --- a/dil/src/main/java/com/linkedin/dil/util/JsonParameter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonParameter.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.base.Strings; import com.google.gson.Gson; @@ -13,7 +13,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; diff --git a/dil/src/main/java/com/linkedin/dil/util/JsonUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonUtils.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/JsonUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/JsonUtils.java index de12e57..be7f9cb 100644 --- a/dil/src/main/java/com/linkedin/dil/util/JsonUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.Lists; import com.google.gson.Gson; diff --git a/dil/src/main/java/com/linkedin/dil/util/ParameterTypes.java b/cdi-core/src/main/java/com/linkedin/cdi/util/ParameterTypes.java similarity index 95% rename from dil/src/main/java/com/linkedin/dil/util/ParameterTypes.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/ParameterTypes.java index c85f5b7..da2a4ae 100644 --- a/dil/src/main/java/com/linkedin/dil/util/ParameterTypes.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/ParameterTypes.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; /** * diff --git a/dil/src/main/java/com/linkedin/dil/util/SchemaBuilder.java b/cdi-core/src/main/java/com/linkedin/cdi/util/SchemaBuilder.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/SchemaBuilder.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/SchemaBuilder.java index 9d33038..887226e 100644 --- a/dil/src/main/java/com/linkedin/dil/util/SchemaBuilder.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/SchemaBuilder.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -20,7 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/util/SchemaUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/SchemaUtils.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/SchemaUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/SchemaUtils.java index 08b5ef4..f707f6c 100644 --- a/dil/src/main/java/com/linkedin/dil/util/SchemaUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/SchemaUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.util.List; import lombok.extern.slf4j.Slf4j; diff --git a/dil/src/main/java/com/linkedin/dil/util/VariableUtils.java b/cdi-core/src/main/java/com/linkedin/cdi/util/VariableUtils.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/VariableUtils.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/VariableUtils.java index d079818..6a409f0 100644 --- a/dil/src/main/java/com/linkedin/dil/util/VariableUtils.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/VariableUtils.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.Lists; import com.google.gson.Gson; diff --git a/dil/src/main/java/com/linkedin/dil/util/WatermarkDefinition.java b/cdi-core/src/main/java/com/linkedin/cdi/util/WatermarkDefinition.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/WatermarkDefinition.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/WatermarkDefinition.java index b57cad2..7ee1bf4 100644 --- a/dil/src/main/java/com/linkedin/dil/util/WatermarkDefinition.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/WatermarkDefinition.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -21,7 +21,7 @@ import org.joda.time.Period; import org.testng.Assert; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/util/WorkUnitPartitionTypes.java b/cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitPartitionTypes.java similarity index 98% rename from dil/src/main/java/com/linkedin/dil/util/WorkUnitPartitionTypes.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitPartitionTypes.java index 5ef8641..e1b49e9 100644 --- a/dil/src/main/java/com/linkedin/dil/util/WorkUnitPartitionTypes.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitPartitionTypes.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.Lists; import com.google.gson.JsonObject; @@ -13,10 +13,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.joda.time.DateTime; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; /** diff --git a/dil/src/main/java/com/linkedin/dil/util/WorkUnitStatus.java b/cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitStatus.java similarity index 99% rename from dil/src/main/java/com/linkedin/dil/util/WorkUnitStatus.java rename to cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitStatus.java index fb21dfd..3db3be8 100644 --- a/dil/src/main/java/com/linkedin/dil/util/WorkUnitStatus.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/WorkUnitStatus.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/test/java/com/linkedin/dil/configuration/MultistagePropertiesTest.java b/cdi-core/src/test/java/com/linkedin/cdi/configuration/MultistagePropertiesTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/configuration/MultistagePropertiesTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/configuration/MultistagePropertiesTest.java index 27268c7..fd951b4 100644 --- a/dil/src/test/java/com/linkedin/dil/configuration/MultistagePropertiesTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/configuration/MultistagePropertiesTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.configuration; +package com.linkedin.cdi.configuration; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/test/java/com/linkedin/dil/connection/HdfsReadConnectionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/HdfsReadConnectionTest.java similarity index 89% rename from dil/src/test/java/com/linkedin/dil/connection/HdfsReadConnectionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/HdfsReadConnectionTest.java index 5c707cd..447aebb 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/HdfsReadConnectionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/HdfsReadConnectionTest.java @@ -2,16 +2,16 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import java.util.ArrayList; import java.util.List; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.source.HdfsSource; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.source.HdfsSource; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.extractor.filebased.FileBasedHelperException; import org.apache.gobblin.source.extractor.filebased.TimestampAwareFileBasedHelper; import org.apache.gobblin.source.extractor.hadoop.HadoopFsHelper; @@ -35,11 +35,11 @@ public void testGetFileList() throws Exception { HdfsSource hdfsSource = new HdfsSource(); SourceState sourceState = new SourceState(); - sourceState.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + sourceState.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); sourceState.setProp("ms.source.uri", "/data/test?RE=.*"); WorkUnitState state = new WorkUnitState(hdfsSource.getWorkunits(sourceState).get(0), new State()); - state.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + state.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); state.setProp("ms.source.uri", "/data/test?RE=.*"); hdfsSource.getExtractor(state); @@ -71,11 +71,11 @@ public void testGetFileInputStream() throws Exception { HdfsSource hdfsSource = new HdfsSource(); SourceState sourceState = new SourceState(); - sourceState.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + sourceState.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); sourceState.setProp("ms.source.uri", "/jobs/exttest/data/external/snapshots/test"); WorkUnitState state = new WorkUnitState(hdfsSource.getWorkunits(sourceState).get(0), new State()); - state.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + state.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); state.setProp("ms.source.uri", "/jobs/exttest/data/external/snapshots/test"); hdfsSource.getExtractor(state); diff --git a/dil/src/test/java/com/linkedin/dil/connection/HttpConnectionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/HttpConnectionTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/connection/HttpConnectionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/HttpConnectionTest.java index 6614d8c..b678441 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/HttpConnectionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/HttpConnectionTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -14,16 +14,16 @@ import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.extractor.JsonExtractor; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.HttpKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.util.HttpRequestMethod; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.extractor.JsonExtractor; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.HttpKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.util.HttpRequestMethod; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.gobblin.source.workunit.WorkUnit; import org.apache.http.Header; @@ -46,7 +46,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/connection/JdbcReadConnectionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/JdbcReadConnectionTest.java similarity index 93% rename from dil/src/test/java/com/linkedin/dil/connection/JdbcReadConnectionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/JdbcReadConnectionTest.java index 191605c..d5d3fa8 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/JdbcReadConnectionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/JdbcReadConnectionTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; @@ -18,15 +18,15 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.extractor.MultistageExtractor; -import com.linkedin.dil.util.Database; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JdbcKeys; -import com.linkedin.dil.util.JsonParameter; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.VariableUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.extractor.MultistageExtractor; +import com.linkedin.cdi.util.Database; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JdbcKeys; +import com.linkedin.cdi.util.JsonParameter; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.VariableUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -35,7 +35,7 @@ import org.testng.Assert; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Matchers.*; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; @@ -118,7 +118,7 @@ public void testGetNext() throws UnsupportedEncodingException, SQLException, Ret when(statement.execute(any())).thenReturn(true); doNothing().when(statement).setFetchSize(anyInt()); - String unSupportedExtractor = "com.linkedin.dil.extractor.SomeExtractor"; + String unSupportedExtractor = "com.linkedin.cdi.extractor.SomeExtractor"; when(state.getProp(MSTAGE_EXTRACTOR_CLASS.getConfig(), StringUtils.EMPTY)).thenReturn(unSupportedExtractor); when(JsonParameter.getParametersAsJson(any(), any(), any())).thenReturn(new JsonObject()); Assert.assertNull(conn.executeNext(workUnitStatus)); @@ -127,7 +127,7 @@ public void testGetNext() throws UnsupportedEncodingException, SQLException, Ret conn.setJdbcConnection(jdbcConnection); Assert.assertNull(conn.executeNext(workUnitStatus)); - String supportedExtractor = "com.linkedin.dil.extractor.CsvExtractor"; + String supportedExtractor = "com.linkedin.cdi.extractor.CsvExtractor"; when(state.getProp(MSTAGE_EXTRACTOR_CLASS.getConfig(), StringUtils.EMPTY)).thenReturn(supportedExtractor); when(jdbcSourceKeys.hasSourceSchema()).thenReturn(true); Assert.assertEquals(conn.executeNext(workUnitStatus), workUnitStatus); diff --git a/dil/src/test/java/com/linkedin/dil/connection/MulstistageReadConnectionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/MulstistageReadConnectionTest.java similarity index 90% rename from dil/src/test/java/com/linkedin/dil/connection/MulstistageReadConnectionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/MulstistageReadConnectionTest.java index 5b3e969..dfa9b85 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/MulstistageReadConnectionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/MulstistageReadConnectionTest.java @@ -2,17 +2,17 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import gobblin.configuration.SourceState; import java.io.UnsupportedEncodingException; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.VariableUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.VariableUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; diff --git a/dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionOnlineTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionOnlineTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionOnlineTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionOnlineTest.java index 5264f51..4dd054a 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionOnlineTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionOnlineTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import org.apache.gobblin.runtime.embedded.EmbeddedGobblin; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionTest.java similarity index 82% rename from dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionTest.java index 9344525..faa73ad 100644 --- a/dil/src/test/java/com/linkedin/dil/connection/S3ReadConnectionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/connection/S3ReadConnectionTest.java @@ -2,16 +2,16 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.connection; +package com.linkedin.cdi.connection; import gobblin.runtime.JobState; import java.util.List; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.source.S3SourceV2; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.source.S3SourceV2; import org.apache.gobblin.source.workunit.WorkUnit; import org.testng.Assert; import org.testng.annotations.Test; @@ -23,7 +23,7 @@ public class S3ReadConnectionTest { public void testGetS3HttpClient() { List wus = new MultistageSource().getWorkunits(new SourceState()); WorkUnitState wuState = new WorkUnitState(wus.get(0), new JobState()); - wuState.setProp(MultistageProperties.MSTAGE_HTTP_CLIENT_FACTORY.getConfig(), "com.linkedin.dil.factory.ApacheHttpClientFactory"); + wuState.setProp(MultistageProperties.MSTAGE_HTTP_CLIENT_FACTORY.getConfig(), "com.linkedin.cdi.factory.ApacheHttpClientFactory"); S3SourceV2 source = new S3SourceV2(); SourceState sourceState = new SourceState(); diff --git a/dil/src/test/java/com/linkedin/dil/converter/AvroNormalizerConverterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/converter/AvroNormalizerConverterTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/converter/AvroNormalizerConverterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/converter/AvroNormalizerConverterTest.java index da57651..359b60e 100644 --- a/dil/src/test/java/com/linkedin/dil/converter/AvroNormalizerConverterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/converter/AvroNormalizerConverterTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.converter; +package com.linkedin.cdi.converter; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -14,7 +14,7 @@ import org.apache.gobblin.converter.DataConversionException; import org.apache.gobblin.converter.SchemaConversionException; import org.apache.gobblin.converter.avro.UnsupportedDateTypeException; -import com.linkedin.dil.util.AvroSchemaUtils; +import com.linkedin.cdi.util.AvroSchemaUtils; import org.apache.gobblin.source.workunit.Extract; import org.testng.Assert; import org.testng.annotations.BeforeMethod; diff --git a/dil/src/test/java/com/linkedin/dil/converter/JsonNormalizerConverterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/converter/JsonNormalizerConverterTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/converter/JsonNormalizerConverterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/converter/JsonNormalizerConverterTest.java index 25b09d9..ecc2186 100644 --- a/dil/src/test/java/com/linkedin/dil/converter/JsonNormalizerConverterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/converter/JsonNormalizerConverterTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.converter; +package com.linkedin.cdi.converter; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/test/java/com/linkedin/dil/extractor/AvroExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/AvroExtractorTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/extractor/AvroExtractorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/AvroExtractorTest.java index 0180764..361a9f3 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/AvroExtractorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/AvroExtractorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; @@ -20,15 +20,15 @@ import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.AvroExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.AvroExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.workunit.Extract; import org.apache.gobblin.source.workunit.WorkUnit; import org.apache.gobblin.util.AvroUtils; @@ -44,8 +44,8 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; -import static com.linkedin.dil.configuration.StaticConstants.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.StaticConstants.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/extractor/CsvExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/CsvExtractorTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/extractor/CsvExtractorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/CsvExtractorTest.java index a53a232..e74ee35 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/CsvExtractorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/CsvExtractorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -28,15 +28,15 @@ import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.filter.JsonSchemaBasedFilter; -import com.linkedin.dil.keys.CsvExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.filter.JsonSchemaBasedFilter; +import com.linkedin.cdi.keys.CsvExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.extractor.utils.InputStreamCSVReader; import org.apache.gobblin.source.workunit.WorkUnit; import org.joda.time.DateTime; @@ -52,7 +52,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Mockito.*; @@ -218,7 +218,7 @@ void testExtractGzippedCSV() throws RetriableAuthenticationException { WorkUnitStatus status = WorkUnitStatus.builder().buffer(inputStream).build(); when(sourceState.getProp("ms.output.schema", "" )).thenReturn(""); - when(state.getProp("ms.extract.preprocessors", "")).thenReturn("com.linkedin.dil.preprocessor.GunzipProcessor"); + when(state.getProp("ms.extract.preprocessors", "")).thenReturn("com.linkedin.cdi.preprocessor.GunzipProcessor"); realHttpSource.getWorkunits(sourceState); CsvExtractor extractor = new CsvExtractor(state, realHttpSource.getHttpSourceKeys()); diff --git a/dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest.java similarity index 91% rename from dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest.java index 36f4d5c..eea860e 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import java.io.File; import java.io.FileFilter; @@ -10,10 +10,10 @@ import java.util.List; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.runtime.JobState; import org.apache.gobblin.source.workunit.WorkUnit; import org.mockito.Mockito; diff --git a/dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest2.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest2.java similarity index 94% rename from dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest2.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest2.java index 80cbe0d..e541952 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/FileDumpExtractorTest2.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/FileDumpExtractorTest2.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -19,14 +19,14 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.FileDumpExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.util.VariableUtils; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.FileDumpExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.util.VariableUtils; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.source.workunit.WorkUnit; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -38,7 +38,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/extractor/JsonExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/JsonExtractorTest.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/extractor/JsonExtractorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/JsonExtractorTest.java index 05dae6d..1f3c980 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/JsonExtractorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/JsonExtractorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -21,17 +21,17 @@ import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.connection.MultistageConnection; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.keys.JsonExtractorKeys; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.SchemaBuilder; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.connection.MultistageConnection; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.keys.JsonExtractorKeys; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.SchemaBuilder; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.runtime.JobState; import org.apache.gobblin.source.workunit.WorkUnit; import org.joda.time.DateTime; @@ -41,7 +41,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/extractor/MultistageExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/MultistageExtractorTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/extractor/MultistageExtractorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/extractor/MultistageExtractorTest.java index 5d4e46a..7b89b19 100644 --- a/dil/src/test/java/com/linkedin/dil/extractor/MultistageExtractorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/extractor/MultistageExtractorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.extractor; +package com.linkedin.cdi.extractor; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; @@ -24,17 +24,17 @@ import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.filter.JsonSchemaBasedFilter; -import com.linkedin.dil.keys.ExtractorKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.preprocessor.GpgDecryptProcessor; -import com.linkedin.dil.preprocessor.GunzipProcessor; -import com.linkedin.dil.preprocessor.InputStreamProcessor; -import com.linkedin.dil.source.HttpSource; -import com.linkedin.dil.source.MultistageSource; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.WorkUnitStatus; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.filter.JsonSchemaBasedFilter; +import com.linkedin.cdi.keys.ExtractorKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.preprocessor.GpgDecryptProcessor; +import com.linkedin.cdi.preprocessor.GunzipProcessor; +import com.linkedin.cdi.preprocessor.InputStreamProcessor; +import com.linkedin.cdi.source.HttpSource; +import com.linkedin.cdi.source.MultistageSource; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.WorkUnitStatus; import org.apache.gobblin.runtime.JobState; import org.apache.gobblin.source.workunit.WorkUnit; import org.joda.time.DateTime; @@ -146,10 +146,10 @@ public void testGetOnePreprocessor() { when(state.getProp("ms.activation.property", new JsonObject().toString())).thenReturn( "{\"a\":\"x\"}"); when(state.getProp("ms.extract.preprocessor.parameters", new JsonObject().toString())).thenReturn( - "{\"com.linkedin.dil.preprocessor.GpgProcessor\":" + + "{\"com.linkedin.cdi.preprocessor.GpgProcessor\":" + "{\"keystore_path\" :\"some path\", \"keystore_password\" : \"some password\"}}"); when(state.getProp("ms.extract.preprocessors", new String())).thenReturn( - "com.linkedin.dil.preprocessor.GpgProcessor"); + "com.linkedin.cdi.preprocessor.GpgProcessor"); SourceState sourceState = mock(SourceState.class); when(sourceState.contains("source.conn.use.proxy.url")).thenReturn(true); @@ -173,11 +173,11 @@ public void testGetTwoPreprocessors() { when(state.getProp("ms.activation.property", new JsonObject().toString())).thenReturn( "{\"a\":\"x\"}"); when(state.getProp("ms.extract.preprocessor.parameters", new JsonObject().toString())).thenReturn( - "{\"com.linkedin.dil.preprocessor.GpgProcessor\":" + + "{\"com.linkedin.cdi.preprocessor.GpgProcessor\":" + "{\"keystore_path\" :\"some path\", \"keystore_password\" : \"some password\"}}"); when(state.getProp("ms.extract.preprocessors", new String())).thenReturn( - "com.linkedin.dil.preprocessor.GpgProcessor,"+ - "com.linkedin.dil.preprocessor.GunzipProcessor"); + "com.linkedin.cdi.preprocessor.GpgProcessor,"+ + "com.linkedin.cdi.preprocessor.GunzipProcessor"); SourceState sourceState = mock(SourceState.class); when(sourceState.contains("source.conn.use.proxy.url")).thenReturn(true); diff --git a/dil/src/test/java/com/linkedin/dil/factory/ApacheHttpClientFactoryTest.java b/cdi-core/src/test/java/com/linkedin/cdi/factory/ApacheHttpClientFactoryTest.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/factory/ApacheHttpClientFactoryTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/factory/ApacheHttpClientFactoryTest.java index 3225aa2..7d6ce34 100644 --- a/dil/src/test/java/com/linkedin/dil/factory/ApacheHttpClientFactoryTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/factory/ApacheHttpClientFactoryTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.factory; +package com.linkedin.cdi.factory; import org.apache.gobblin.configuration.State; import org.apache.http.impl.client.CloseableHttpClient; diff --git a/dil/src/test/java/com/linkedin/dil/filter/AvroSchemaBasedFilterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/filter/AvroSchemaBasedFilterTest.java similarity index 93% rename from dil/src/test/java/com/linkedin/dil/filter/AvroSchemaBasedFilterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/filter/AvroSchemaBasedFilterTest.java index c39b215..18bd747 100644 --- a/dil/src/test/java/com/linkedin/dil/filter/AvroSchemaBasedFilterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/filter/AvroSchemaBasedFilterTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -12,10 +12,10 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.exception.RetriableAuthenticationException; -import com.linkedin.dil.keys.AvroExtractorKeys; -import com.linkedin.dil.util.AvroSchemaUtils; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.exception.RetriableAuthenticationException; +import com.linkedin.cdi.keys.AvroExtractorKeys; +import com.linkedin.cdi.util.AvroSchemaUtils; +import com.linkedin.cdi.util.JsonIntermediateSchema; import org.apache.gobblin.source.workunit.Extract; import org.testng.Assert; import org.testng.annotations.BeforeMethod; diff --git a/dil/src/test/java/com/linkedin/dil/filter/JsonSchemaBasedFilterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/filter/JsonSchemaBasedFilterTest.java similarity index 92% rename from dil/src/test/java/com/linkedin/dil/filter/JsonSchemaBasedFilterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/filter/JsonSchemaBasedFilterTest.java index 96601a2..faf0a6f 100644 --- a/dil/src/test/java/com/linkedin/dil/filter/JsonSchemaBasedFilterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/filter/JsonSchemaBasedFilterTest.java @@ -2,13 +2,13 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; import com.google.gson.Gson; import com.google.gson.JsonElement; import java.lang.reflect.Method; -import com.linkedin.dil.util.JsonElementTypes; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.JsonElementTypes; +import com.linkedin.cdi.util.JsonIntermediateSchema; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.BeforeMethod; diff --git a/dil/src/test/java/com/linkedin/dil/filter/MultistageSchemaBasedFilterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilterTest.java similarity index 86% rename from dil/src/test/java/com/linkedin/dil/filter/MultistageSchemaBasedFilterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilterTest.java index fb42bac..c0f29fe 100644 --- a/dil/src/test/java/com/linkedin/dil/filter/MultistageSchemaBasedFilterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/filter/MultistageSchemaBasedFilterTest.java @@ -2,9 +2,9 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.filter; +package com.linkedin.cdi.filter; -import com.linkedin.dil.util.JsonIntermediateSchema; +import com.linkedin.cdi.util.JsonIntermediateSchema; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/dil/src/test/java/com/linkedin/dil/helpers/GobblinMultiStageTestHelpers.java b/cdi-core/src/test/java/com/linkedin/cdi/helpers/GobblinMultiStageTestHelpers.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/helpers/GobblinMultiStageTestHelpers.java rename to cdi-core/src/test/java/com/linkedin/cdi/helpers/GobblinMultiStageTestHelpers.java index 0ab1a8f..9264462 100644 --- a/dil/src/test/java/com/linkedin/dil/helpers/GobblinMultiStageTestHelpers.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/helpers/GobblinMultiStageTestHelpers.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.helpers; +package com.linkedin.cdi.helpers; import java.text.DateFormat; import java.text.SimpleDateFormat; diff --git a/dil/src/test/java/com/linkedin/dil/helpers/MockedResponseStrings.java b/cdi-core/src/test/java/com/linkedin/cdi/helpers/MockedResponseStrings.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/helpers/MockedResponseStrings.java rename to cdi-core/src/test/java/com/linkedin/cdi/helpers/MockedResponseStrings.java index a710c4d..e1fa476 100644 --- a/dil/src/test/java/com/linkedin/dil/helpers/MockedResponseStrings.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/helpers/MockedResponseStrings.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.helpers; +package com.linkedin.cdi.helpers; public class MockedResponseStrings { public MockedResponseStrings() { diff --git a/dil/src/test/java/com/linkedin/dil/jdbcclient/DatabaseTest.java b/cdi-core/src/test/java/com/linkedin/cdi/jdbcclient/DatabaseTest.java similarity index 95% rename from dil/src/test/java/com/linkedin/dil/jdbcclient/DatabaseTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/jdbcclient/DatabaseTest.java index 77d0b2f..64c5ec9 100644 --- a/dil/src/test/java/com/linkedin/dil/jdbcclient/DatabaseTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/jdbcclient/DatabaseTest.java @@ -2,12 +2,12 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.jdbcclient; +package com.linkedin.cdi.jdbcclient; import mockit.Mock; import mockit.MockUp; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.util.Database; +import com.linkedin.cdi.util.Database; import org.apache.gobblin.password.PasswordManager; import org.mockito.Matchers; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/keys/FileDumpExtractorKeysTest.java b/cdi-core/src/test/java/com/linkedin/cdi/keys/FileDumpExtractorKeysTest.java similarity index 93% rename from dil/src/test/java/com/linkedin/dil/keys/FileDumpExtractorKeysTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/keys/FileDumpExtractorKeysTest.java index 973ba37..d153b1e 100644 --- a/dil/src/test/java/com/linkedin/dil/keys/FileDumpExtractorKeysTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/keys/FileDumpExtractorKeysTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/dil/src/test/java/com/linkedin/dil/keys/JobKeysTest.java b/cdi-core/src/test/java/com/linkedin/cdi/keys/JobKeysTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/keys/JobKeysTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/keys/JobKeysTest.java index f408658..f2119b9 100644 --- a/dil/src/test/java/com/linkedin/dil/keys/JobKeysTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/keys/JobKeysTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.keys; +package com.linkedin.cdi.keys; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -13,11 +13,11 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.util.JsonUtils; -import com.linkedin.dil.util.ParameterTypes; -import com.linkedin.dil.util.SchemaBuilder; -import com.linkedin.dil.util.WorkUnitPartitionTypes; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.util.JsonUtils; +import com.linkedin.cdi.util.ParameterTypes; +import com.linkedin.cdi.util.SchemaBuilder; +import com.linkedin.cdi.util.WorkUnitPartitionTypes; import org.mockito.Mockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.testng.PowerMockTestCase; diff --git a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgDecryptProcessorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessorTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/preprocessor/GpgDecryptProcessorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessorTest.java index 30be085..f9f6fae 100644 --- a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgDecryptProcessorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgDecryptProcessorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; diff --git a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgEncryptProcessorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessorTest.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/preprocessor/GpgEncryptProcessorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessorTest.java index 6405412..b45ed47 100644 --- a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgEncryptProcessorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgEncryptProcessorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import java.io.IOException; diff --git a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgPreprocessorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgPreprocessorTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/preprocessor/GpgPreprocessorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgPreprocessorTest.java index 5a6b966..c4014ca 100644 --- a/dil/src/test/java/com/linkedin/dil/preprocessor/GpgPreprocessorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GpgPreprocessorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import com.opencsv.CSVParserBuilder; diff --git a/dil/src/test/java/com/linkedin/dil/preprocessor/GunzipPreprocessorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipPreprocessorTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/preprocessor/GunzipPreprocessorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipPreprocessorTest.java index a52d0d1..de539b1 100644 --- a/dil/src/test/java/com/linkedin/dil/preprocessor/GunzipPreprocessorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipPreprocessorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; diff --git a/dil/src/test/java/com/linkedin/dil/preprocessor/GunzipProcessorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipProcessorTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/preprocessor/GunzipProcessorTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipProcessorTest.java index 60d9da9..b19f589 100644 --- a/dil/src/test/java/com/linkedin/dil/preprocessor/GunzipProcessorTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/preprocessor/GunzipProcessorTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.preprocessor; +package com.linkedin.cdi.preprocessor; import com.google.gson.JsonObject; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/source/HdfsSourceTest.java b/cdi-core/src/test/java/com/linkedin/cdi/source/HdfsSourceTest.java similarity index 90% rename from dil/src/test/java/com/linkedin/dil/source/HdfsSourceTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/HdfsSourceTest.java index 08446b1..1476a4a 100644 --- a/dil/src/test/java/com/linkedin/dil/source/HdfsSourceTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/HdfsSourceTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import gobblin.configuration.SourceState; import gobblin.configuration.State; @@ -24,7 +24,7 @@ public void testGetExtractor() { HdfsSource hdfsSource = new HdfsSource(); SourceState sourceState = new SourceState(); WorkUnitState state = new WorkUnitState(hdfsSource.getWorkunits(sourceState).get(0), new State()); - state.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + state.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); hdfsSource.getExtractor(state); Assert.assertNotNull(hdfsSource.getExtractor(state)); } diff --git a/dil/src/test/java/com/linkedin/dil/source/HttpSourceTest.java b/cdi-core/src/test/java/com/linkedin/cdi/source/HttpSourceTest.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/source/HttpSourceTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/HttpSourceTest.java index a8c1c67..fe4d106 100644 --- a/dil/src/test/java/com/linkedin/dil/source/HttpSourceTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/HttpSourceTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -18,12 +18,12 @@ import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.extractor.JsonExtractor; -import com.linkedin.dil.helpers.GobblinMultiStageTestHelpers; -import com.linkedin.dil.keys.HttpKeys; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.EncryptionUtils; -import com.linkedin.dil.util.ParameterTypes; +import com.linkedin.cdi.extractor.JsonExtractor; +import com.linkedin.cdi.helpers.GobblinMultiStageTestHelpers; +import com.linkedin.cdi.keys.HttpKeys; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.EncryptionUtils; +import com.linkedin.cdi.util.ParameterTypes; import org.apache.gobblin.runtime.embedded.EmbeddedGobblin; import org.apache.gobblin.source.workunit.WorkUnit; import org.mockito.Mockito; @@ -35,8 +35,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; -import static com.linkedin.dil.source.HttpSource.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; +import static com.linkedin.cdi.source.HttpSource.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/source/JdbcSourceTest.java b/cdi-core/src/test/java/com/linkedin/cdi/source/JdbcSourceTest.java similarity index 93% rename from dil/src/test/java/com/linkedin/dil/source/JdbcSourceTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/JdbcSourceTest.java index 8e7e1d3..0570685 100644 --- a/dil/src/test/java/com/linkedin/dil/source/JdbcSourceTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/JdbcSourceTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import gobblin.configuration.SourceState; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/source/MultistageSource2Test.java b/cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSource2Test.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/source/MultistageSource2Test.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSource2Test.java index bd66102..e1c44f7 100644 --- a/dil/src/test/java/com/linkedin/dil/source/MultistageSource2Test.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSource2Test.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -11,7 +11,7 @@ import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.gobblin.configuration.SourceState; -import com.linkedin.dil.util.VariableUtils; +import com.linkedin.cdi.util.VariableUtils; import org.apache.gobblin.source.workunit.WorkUnit; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.testng.PowerMockTestCase; @@ -19,7 +19,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import static com.linkedin.dil.configuration.MultistageProperties.*; +import static com.linkedin.cdi.configuration.MultistageProperties.*; import static org.mockito.Mockito.*; diff --git a/dil/src/test/java/com/linkedin/dil/source/MultistageSourceTest.java b/cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSourceTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/source/MultistageSourceTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSourceTest.java index 5e2a623..30f7d03 100644 --- a/dil/src/test/java/com/linkedin/dil/source/MultistageSourceTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/MultistageSourceTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; @@ -19,12 +19,12 @@ import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.State; import org.apache.gobblin.configuration.WorkUnitState; -import com.linkedin.dil.configuration.MultistageProperties; -import com.linkedin.dil.factory.reader.SchemaReader; -import com.linkedin.dil.keys.JobKeys; -import com.linkedin.dil.util.EndecoUtils; -import com.linkedin.dil.util.WatermarkDefinition; -import com.linkedin.dil.util.WorkUnitPartitionTypes; +import com.linkedin.cdi.configuration.MultistageProperties; +import com.linkedin.cdi.factory.reader.SchemaReader; +import com.linkedin.cdi.keys.JobKeys; +import com.linkedin.cdi.util.EndecoUtils; +import com.linkedin.cdi.util.WatermarkDefinition; +import com.linkedin.cdi.util.WorkUnitPartitionTypes; import org.apache.gobblin.source.extractor.WatermarkInterval; import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.gobblin.source.workunit.Extract; diff --git a/dil/src/test/java/com/linkedin/dil/source/S3SourceV2Test.java b/cdi-core/src/test/java/com/linkedin/cdi/source/S3SourceV2Test.java similarity index 94% rename from dil/src/test/java/com/linkedin/dil/source/S3SourceV2Test.java rename to cdi-core/src/test/java/com/linkedin/cdi/source/S3SourceV2Test.java index 3215dd1..07f72ff 100644 --- a/dil/src/test/java/com/linkedin/dil/source/S3SourceV2Test.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/source/S3SourceV2Test.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.source; +package com.linkedin.cdi.source; import com.google.gson.JsonObject; import java.io.UnsupportedEncodingException; @@ -23,7 +23,7 @@ public void testInitialization() throws UnsupportedEncodingException { S3SourceV2 source = new S3SourceV2(); WorkUnitState state = new WorkUnitState(); state.setProp("ms.source.uri", "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-43/cc-index.paths.gz"); - state.setProp("ms.extractor.class", "com.linkedin.dil.extractor.CsvExtractor"); + state.setProp("ms.extractor.class", "com.linkedin.cdi.extractor.CsvExtractor"); JsonObject params = new JsonObject(); params.addProperty("region", "us-east-1"); params.addProperty("connection_timeout", 30); diff --git a/dil/src/test/java/com/linkedin/dil/util/AvroSchemaUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/AvroSchemaUtilsTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/util/AvroSchemaUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/AvroSchemaUtilsTest.java index a3ff158..13261ea 100644 --- a/dil/src/test/java/com/linkedin/dil/util/AvroSchemaUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/AvroSchemaUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/test/java/com/linkedin/dil/util/DateTimeUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/DateTimeUtilsTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/DateTimeUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/DateTimeUtilsTest.java index 38021b3..fd5da0a 100644 --- a/dil/src/test/java/com/linkedin/dil/util/DateTimeUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/DateTimeUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import org.joda.time.DateTimeZone; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/util/EncryptionUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/EncryptionUtilsTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/util/EncryptionUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/EncryptionUtilsTest.java index ef9b479..bfcb81a 100644 --- a/dil/src/test/java/com/linkedin/dil/util/EncryptionUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/EncryptionUtilsTest.java @@ -2,12 +2,12 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.JsonObject; import gobblin.configuration.SourceState; import org.apache.gobblin.codec.StreamCodec; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.apache.gobblin.password.PasswordManager; import org.mockito.Mock; import org.powermock.api.mockito.PowerMockito; diff --git a/dil/src/test/java/com/linkedin/dil/util/HttpRequestMethodTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/HttpRequestMethodTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/HttpRequestMethodTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/HttpRequestMethodTest.java index bbe33e3..8f15490 100644 --- a/dil/src/test/java/com/linkedin/dil/util/HttpRequestMethodTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/HttpRequestMethodTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; diff --git a/dil/src/test/java/com/linkedin/dil/util/InputStreamUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/InputStreamUtilsTest.java similarity index 96% rename from dil/src/test/java/com/linkedin/dil/util/InputStreamUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/InputStreamUtilsTest.java index 6db1d2c..866e5f8 100644 --- a/dil/src/test/java/com/linkedin/dil/util/InputStreamUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/InputStreamUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.io.IOException; import java.io.InputStream; diff --git a/dil/src/test/java/com/linkedin/dil/util/JdbcUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/JdbcUtilsTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/JdbcUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/JdbcUtilsTest.java index 35bfe46..8b5eef9 100644 --- a/dil/src/test/java/com/linkedin/dil/util/JdbcUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/JdbcUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.sun.rowset.JdbcRowSetImpl; import java.sql.SQLException; diff --git a/dil/src/test/java/com/linkedin/dil/util/JsonElementTypesTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonElementTypesTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/JsonElementTypesTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/JsonElementTypesTest.java index d302a84..f2afcf6 100644 --- a/dil/src/test/java/com/linkedin/dil/util/JsonElementTypesTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonElementTypesTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -10,7 +10,7 @@ import org.testng.Assert; import org.testng.annotations.Test; -import static com.linkedin.dil.util.JsonElementTypes.*; +import static com.linkedin.cdi.util.JsonElementTypes.*; /** diff --git a/dil/src/test/java/com/linkedin/dil/util/JsonIntermediateSchemaTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonIntermediateSchemaTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/JsonIntermediateSchemaTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/JsonIntermediateSchemaTest.java index 9db7466..8a4699d 100644 --- a/dil/src/test/java/com/linkedin/dil/util/JsonIntermediateSchemaTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonIntermediateSchemaTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/dil/src/test/java/com/linkedin/dil/util/JsonParameterTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonParameterTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/JsonParameterTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/JsonParameterTest.java index 30c8b74..2755b99 100644 --- a/dil/src/test/java/com/linkedin/dil/util/JsonParameterTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonParameterTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -11,7 +11,7 @@ import java.io.InputStreamReader; import java.lang.reflect.Method; import org.apache.gobblin.configuration.State; -import com.linkedin.dil.configuration.MultistageProperties; +import com.linkedin.cdi.configuration.MultistageProperties; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/dil/src/test/java/com/linkedin/dil/util/JsonUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonUtilsTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/util/JsonUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/JsonUtilsTest.java index f325d7f..cb688ba 100644 --- a/dil/src/test/java/com/linkedin/dil/util/JsonUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/JsonUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.JsonObject; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/util/SchemaBuilderTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/SchemaBuilderTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/util/SchemaBuilderTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/SchemaBuilderTest.java index 8fb5197..9f6f72f 100644 --- a/dil/src/test/java/com/linkedin/dil/util/SchemaBuilderTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/SchemaBuilderTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.util.HashMap; import org.testng.Assert; diff --git a/dil/src/test/java/com/linkedin/dil/util/SchemaUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/SchemaUtilsTest.java similarity index 97% rename from dil/src/test/java/com/linkedin/dil/util/SchemaUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/SchemaUtilsTest.java index 9d78293..8b23a8c 100644 --- a/dil/src/test/java/com/linkedin/dil/util/SchemaUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/SchemaUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import java.util.Arrays; import java.util.List; diff --git a/dil/src/test/java/com/linkedin/dil/util/VariableUtilsTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/VariableUtilsTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/VariableUtilsTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/VariableUtilsTest.java index 30d4f7b..b08780c 100644 --- a/dil/src/test/java/com/linkedin/dil/util/VariableUtilsTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/VariableUtilsTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonObject; diff --git a/dil/src/test/java/com/linkedin/dil/util/WatermarkDefinitionTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/WatermarkDefinitionTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/WatermarkDefinitionTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/WatermarkDefinitionTest.java index ef3abd9..3b4da30 100644 --- a/dil/src/test/java/com/linkedin/dil/util/WatermarkDefinitionTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/WatermarkDefinitionTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -11,7 +11,7 @@ import org.testng.Assert; import org.testng.annotations.Test; -import static com.linkedin.dil.util.DateTimeUtils.*; +import static com.linkedin.cdi.util.DateTimeUtils.*; @Test diff --git a/dil/src/test/java/com/linkedin/dil/util/WorkUnitPartitionTypesTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitPartitionTypesTest.java similarity index 99% rename from dil/src/test/java/com/linkedin/dil/util/WorkUnitPartitionTypesTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitPartitionTypesTest.java index cc73807..adbb1ff 100644 --- a/dil/src/test/java/com/linkedin/dil/util/WorkUnitPartitionTypesTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitPartitionTypesTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTime; diff --git a/dil/src/test/java/com/linkedin/dil/util/WorkUnitStatusTest.java b/cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitStatusTest.java similarity index 98% rename from dil/src/test/java/com/linkedin/dil/util/WorkUnitStatusTest.java rename to cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitStatusTest.java index 9fd548b..b118d18 100644 --- a/dil/src/test/java/com/linkedin/dil/util/WorkUnitStatusTest.java +++ b/cdi-core/src/test/java/com/linkedin/cdi/util/WorkUnitStatusTest.java @@ -2,7 +2,7 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -package com.linkedin.dil.util; +package com.linkedin.cdi.util; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; diff --git a/dil/src/test/resources/avro/empty_file.avro b/cdi-core/src/test/resources/avro/empty_file.avro similarity index 100% rename from dil/src/test/resources/avro/empty_file.avro rename to cdi-core/src/test/resources/avro/empty_file.avro diff --git a/dil/src/test/resources/csv/comma-separated.csv b/cdi-core/src/test/resources/csv/comma-separated.csv similarity index 100% rename from dil/src/test/resources/csv/comma-separated.csv rename to cdi-core/src/test/resources/csv/comma-separated.csv diff --git a/dil/src/test/resources/csv/common-crawl-files.csv b/cdi-core/src/test/resources/csv/common-crawl-files.csv similarity index 100% rename from dil/src/test/resources/csv/common-crawl-files.csv rename to cdi-core/src/test/resources/csv/common-crawl-files.csv diff --git a/dil/src/test/resources/csv/ctl_d_text.dat b/cdi-core/src/test/resources/csv/ctl_d_text.dat similarity index 100% rename from dil/src/test/resources/csv/ctl_d_text.dat rename to cdi-core/src/test/resources/csv/ctl_d_text.dat diff --git a/dil/src/test/resources/csv/flat.csv b/cdi-core/src/test/resources/csv/flat.csv similarity index 100% rename from dil/src/test/resources/csv/flat.csv rename to cdi-core/src/test/resources/csv/flat.csv diff --git a/dil/src/test/resources/csv/flat_uppercase_header.csv b/cdi-core/src/test/resources/csv/flat_uppercase_header.csv similarity index 100% rename from dil/src/test/resources/csv/flat_uppercase_header.csv rename to cdi-core/src/test/resources/csv/flat_uppercase_header.csv diff --git a/dil/src/test/resources/csv/flat_without_header.csv b/cdi-core/src/test/resources/csv/flat_without_header.csv similarity index 100% rename from dil/src/test/resources/csv/flat_without_header.csv rename to cdi-core/src/test/resources/csv/flat_without_header.csv diff --git a/dil/src/test/resources/csv/ids.csv b/cdi-core/src/test/resources/csv/ids.csv similarity index 100% rename from dil/src/test/resources/csv/ids.csv rename to cdi-core/src/test/resources/csv/ids.csv diff --git a/dil/src/test/resources/csv/ids_flat.csv b/cdi-core/src/test/resources/csv/ids_flat.csv similarity index 100% rename from dil/src/test/resources/csv/ids_flat.csv rename to cdi-core/src/test/resources/csv/ids_flat.csv diff --git a/dil/src/test/resources/csv/ids_need_cleansing.csv b/cdi-core/src/test/resources/csv/ids_need_cleansing.csv similarity index 100% rename from dil/src/test/resources/csv/ids_need_cleansing.csv rename to cdi-core/src/test/resources/csv/ids_need_cleansing.csv diff --git a/dil/src/test/resources/csv/tab-separated.csv b/cdi-core/src/test/resources/csv/tab-separated.csv similarity index 100% rename from dil/src/test/resources/csv/tab-separated.csv rename to cdi-core/src/test/resources/csv/tab-separated.csv diff --git a/dil/src/test/resources/gpg/test.csv b/cdi-core/src/test/resources/gpg/test.csv similarity index 100% rename from dil/src/test/resources/gpg/test.csv rename to cdi-core/src/test/resources/gpg/test.csv diff --git a/dil/src/test/resources/gpg/test.csv.gpg b/cdi-core/src/test/resources/gpg/test.csv.gpg similarity index 100% rename from dil/src/test/resources/gpg/test.csv.gpg rename to cdi-core/src/test/resources/gpg/test.csv.gpg diff --git a/dil/src/test/resources/gpg/test.secret b/cdi-core/src/test/resources/gpg/test.secret similarity index 100% rename from dil/src/test/resources/gpg/test.secret rename to cdi-core/src/test/resources/gpg/test.secret diff --git a/dil/src/test/resources/gzip/cc-index.paths.gz b/cdi-core/src/test/resources/gzip/cc-index.paths.gz similarity index 100% rename from dil/src/test/resources/gzip/cc-index.paths.gz rename to cdi-core/src/test/resources/gzip/cc-index.paths.gz diff --git a/dil/src/test/resources/json/last-page-with-data.json b/cdi-core/src/test/resources/json/last-page-with-data.json similarity index 100% rename from dil/src/test/resources/json/last-page-with-data.json rename to cdi-core/src/test/resources/json/last-page-with-data.json diff --git a/dil/src/test/resources/json/parameter-encryption.json b/cdi-core/src/test/resources/json/parameter-encryption.json similarity index 100% rename from dil/src/test/resources/json/parameter-encryption.json rename to cdi-core/src/test/resources/json/parameter-encryption.json diff --git a/dil/src/test/resources/json/sample-data-for-source.json b/cdi-core/src/test/resources/json/sample-data-for-source.json similarity index 95% rename from dil/src/test/resources/json/sample-data-for-source.json rename to cdi-core/src/test/resources/json/sample-data-for-source.json index 0926cb1..aac61cc 100644 --- a/dil/src/test/resources/json/sample-data-for-source.json +++ b/cdi-core/src/test/resources/json/sample-data-for-source.json @@ -80,7 +80,7 @@ } } ], - "ms.http.client.factory": "com.linkedin.dil.factory.ApacheHttpClientFactory", + "ms.http.client.factory": "com.linkedin.cdi.factory.ApacheHttpClientFactory", "ms.http.request.headers": { "Content-Type": "application/json", "key": "adwpsdfsftcc9cj749fnb8xxsdfsn" @@ -127,5 +127,5 @@ "reason2 for error" ] }, - "ms.extractor.class": "com.linkedin.dil.extractor.JsonExtractor" + "ms.extractor.class": "com.linkedin.cdi.extractor.JsonExtractor" } \ No newline at end of file diff --git a/dil/src/test/resources/json/sample-json-schema.json b/cdi-core/src/test/resources/json/sample-json-schema.json similarity index 100% rename from dil/src/test/resources/json/sample-json-schema.json rename to cdi-core/src/test/resources/json/sample-json-schema.json diff --git a/dil/src/test/resources/key/egress_test.key b/cdi-core/src/test/resources/key/egress_test.key similarity index 100% rename from dil/src/test/resources/key/egress_test.key rename to cdi-core/src/test/resources/key/egress_test.key diff --git a/dil/src/test/resources/key/master.key b/cdi-core/src/test/resources/key/master.key similarity index 100% rename from dil/src/test/resources/key/master.key rename to cdi-core/src/test/resources/key/master.key diff --git a/dil/src/test/resources/key/public.key b/cdi-core/src/test/resources/key/public.key similarity index 100% rename from dil/src/test/resources/key/public.key rename to cdi-core/src/test/resources/key/public.key diff --git a/dil/src/test/resources/key/sftp_test.key b/cdi-core/src/test/resources/key/sftp_test.key similarity index 100% rename from dil/src/test/resources/key/sftp_test.key rename to cdi-core/src/test/resources/key/sftp_test.key diff --git a/dil/src/test/resources/other/sample-data-include-long-file-name.txt b/cdi-core/src/test/resources/other/sample-data-include-long-file-name.txt similarity index 100% rename from dil/src/test/resources/other/sample-data-include-long-file-name.txt rename to cdi-core/src/test/resources/other/sample-data-include-long-file-name.txt diff --git a/dil/src/test/resources/pull/s3-csv.pull b/cdi-core/src/test/resources/pull/s3-csv.pull similarity index 75% rename from dil/src/test/resources/pull/s3-csv.pull rename to cdi-core/src/test/resources/pull/s3-csv.pull index 55f9abf..75865a6 100644 --- a/dil/src/test/resources/pull/s3-csv.pull +++ b/cdi-core/src/test/resources/pull/s3-csv.pull @@ -1,8 +1,8 @@ -source.class=com.linkedin.dil.source.S3SourceV2 -ms.extract.preprocessors=com.linkedin.dil.preprocessor.GunzipProcessor -ms.extractor.class=com.linkedin.dil.extractor.CsvExtractor +source.class=com.linkedin.cdi.source.S3SourceV2 +ms.extract.preprocessors=com.linkedin.cdi.preprocessor.GunzipProcessor +ms.extractor.class=com.linkedin.cdi.extractor.CsvExtractor converter.classes=org.apache.gobblin.converter.csv.CsvToJsonConverterV2,org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter -ms.http.client.factory=com.linkedin.dil.factory.ApacheHttpClientFactory +ms.http.client.factory=com.linkedin.cdi.factory.ApacheHttpClientFactory extract.namespace=com.linkedin.test extract.table.name=test diff --git a/dil/src/test/resources/pull/s3-filedump.pull b/cdi-core/src/test/resources/pull/s3-filedump.pull similarity index 74% rename from dil/src/test/resources/pull/s3-filedump.pull rename to cdi-core/src/test/resources/pull/s3-filedump.pull index ba59680..f8d9cce 100644 --- a/dil/src/test/resources/pull/s3-filedump.pull +++ b/cdi-core/src/test/resources/pull/s3-filedump.pull @@ -1,8 +1,8 @@ -source.class=com.linkedin.dil.source.S3SourceV2 -#ms.extract.preprocessors=com.linkedin.dil.preprocessor.GunzipProcessor -ms.extractor.class=com.linkedin.dil.extractor.FileDumpExtractor +source.class=com.linkedin.cdi.source.S3SourceV2 +#ms.extract.preprocessors=com.linkedin.cdi.preprocessor.GunzipProcessor +ms.extractor.class=com.linkedin.cdi.extractor.FileDumpExtractor converter.classes=org.apache.gobblin.converter.csv.CsvToJsonConverterV2,org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter -ms.http.client.factory=com.linkedin.dil.factory.ApacheHttpClientFactory +ms.http.client.factory=com.linkedin.cdi.factory.ApacheHttpClientFactory extract.namespace=com.linkedin.test extract.table.name=test diff --git a/dil/src/test/resources/util/parameter-jsonarray.json b/cdi-core/src/test/resources/util/parameter-jsonarray.json similarity index 100% rename from dil/src/test/resources/util/parameter-jsonarray.json rename to cdi-core/src/test/resources/util/parameter-jsonarray.json diff --git a/dil/src/test/resources/util/sample-json-schema.json b/cdi-core/src/test/resources/util/sample-json-schema.json similarity index 100% rename from dil/src/test/resources/util/sample-json-schema.json rename to cdi-core/src/test/resources/util/sample-json-schema.json diff --git a/dil/src/test/resources/util/sample-schema.json b/cdi-core/src/test/resources/util/sample-schema.json similarity index 100% rename from dil/src/test/resources/util/sample-schema.json rename to cdi-core/src/test/resources/util/sample-schema.json diff --git a/dil/src/test/resources/util/sample.json b/cdi-core/src/test/resources/util/sample.json similarity index 100% rename from dil/src/test/resources/util/sample.json rename to cdi-core/src/test/resources/util/sample.json diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle deleted file mode 100644 index a84d256..0000000 --- a/defaultEnvironment.gradle +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -repositories { - mavenCentral() -} - -subprojects { - repositories { - mavenCentral() - maven { - url "https://repository.cloudera.com/artifactory/cloudera-repos/" - } - maven { - url "http://conjars.org/repo" - } - } - - project.buildDir = new File(project.rootProject.buildDir, project.name) -} diff --git a/flavored-build.gradle b/flavored-build.gradle deleted file mode 100644 index cef335d..0000000 --- a/flavored-build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -// Check for build customizations -if (project.hasProperty('dilFlavor')) { - def dilFlavorFileName = project.projectDir.toString() + '/flavored-' + project.dilFlavor + '.gradle' - if (file(dilFlavorFileName).exists()) { - println "Using flavor:" + project.dilFlavor + " for project " + project.name - apply from: dilFlavorFileName - } -} - diff --git a/gradle.properties b/gradle.properties index 2941c8c..c97c40e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,23 +2,8 @@ # Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -# Long-running Gradle process speeds up local builds -# To stop the daemon run 'ligradle --stop' -org.gradle.daemon=true - -# Configures only relevant projects to speed up the configuration of large projects -# Useful when specific project/task is invoked +org.gradle.daemon=false org.gradle.configureondemand=true - -# Gradle will run tasks from subprojects in parallel -# Higher CPU usage, faster builds org.gradle.parallel=true - -# Allows generation of idea/eclipse metadata for a specific subproject and its upstream project dependencies -ide.recursive=true - -version=0.0.1 -group=com.linkedin.dil -release=false - +org.gradle.caching=false org.gradle.jvmargs=-Xms512m -Xmx4096m \ No newline at end of file diff --git a/config/checkstyle/checkstyle.xml b/gradle/checkstyle/checkstyle.xml similarity index 90% rename from config/checkstyle/checkstyle.xml rename to gradle/checkstyle/checkstyle.xml index a89898b..f5a69bc 100644 --- a/config/checkstyle/checkstyle.xml +++ b/gradle/checkstyle/checkstyle.xml @@ -10,7 +10,7 @@ - + \ No newline at end of file diff --git a/config/checkstyle/suppressions.xml b/gradle/checkstyle/suppressions.xml similarity index 100% rename from config/checkstyle/suppressions.xml rename to gradle/checkstyle/suppressions.xml diff --git a/gradle/ci-release.gradle b/gradle/ci-release.gradle new file mode 100644 index 0000000..7cab47a --- /dev/null +++ b/gradle/ci-release.gradle @@ -0,0 +1,38 @@ +// CI done via ./github/workflows/gh-release.yml. + +// Task to fail fast and clearly if required properties are not set. +task verifyArtifactoryProperties { + doFirst { + if (!project.hasProperty('artifactory.dryRun')) { + if (System.getenv('ARTIFACTORY_USER') == null) { + throw new Exception("Environment variable ARTIFACTORY_USER not set."); + } + if (System.getenv('ARTIFACTORY_KEY') == null) { + throw new Exception("Environment variable ARTIFACTORY_KEY not set."); + } + } + } +} + +task artifactoryPublishAll { + description = "Runs 'artifactoryPublish' tasks from all projects" + mustRunAfter "githubRelease" // github release is easier to rollback so we run it first +} + +allprojects { + tasks.matching { it.name == "artifactoryPublish" }.all { + it.dependsOn verifyArtifactoryProperties + artifactoryPublishAll.dependsOn it + } +} + +task ciPerformRelease { + description = "Performs the release, intended to be ran on CI" + dependsOn "githubRelease", "artifactoryPublishAll" +} + +task getVersion() { + doFirst { + println version + } +} diff --git a/gradle/java-publishing.gradle b/gradle/java-publishing.gradle new file mode 100644 index 0000000..a00688b --- /dev/null +++ b/gradle/java-publishing.gradle @@ -0,0 +1,124 @@ +assert plugins.hasPlugin(JavaPlugin) + +if (project.hasProperty('disableShipkit')) { + return +} + +apply plugin: 'maven-publish' +apply plugin: 'com.jfrog.artifactory' + +//////////////////////////////////////////////// +// Artifact setup +//////////////////////////////////////////////// + +tasks.withType(Jar) { + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" +} + +task sourcesJar(type: Jar, dependsOn: classes) { + classifier = 'sources' + from sourceSets.main.allSource +} + +javadoc { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('Xwerror', '-quiet') +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier = 'javadoc' + from javadoc.destinationDir +} + +artifacts { + archives sourcesJar + archives javadocJar + + if (plugins.hasPlugin('pegasus')) { + archives mainDataTemplateJar + } +} + +def pomConfig = { + licenses { + license { + name = 'BSD 2-CLAUSE LICENSE' + url = 'https://github.com/linkedin/data-integration-library/blob/master/LICENSE' + distribution = 'repo' + } + } + scm { + url = 'https://github.com/linkedin/data-integration-library.git' + } + issueManagement { + url = 'https://github.com/linkedin/data-integration-library/issues' + system = 'GitHub issues' + } +} + +publishing { + publications { + if (!project.ext.has('publications') || project.ext.publications.contains('jar')) { + jar(MavenPublication) { + from components.java + + artifact javadocJar + artifact sourcesJar + + pom pomConfig + } + } + + if (plugins.hasPlugin('pegasus') && + (!project.ext.has('publications') || project.ext.publications.contains('dataTemplate'))) { + dataTemplate(MavenPublication) { + artifact mainDataTemplateJar + artifactId "$project.name-data-template" + + pom pomConfig + } + } + } +} + +//////////////////////////////////////////////// +// Artifactory publishing +//////////////////////////////////////////////// + +artifactory { + contextUrl = 'https://linkedin.jfrog.io/artifactory' + publish { + repository { + repoKey = 'dil' + username = System.getenv('ARTIFACTORY_USER') + password = System.getenv('ARTIFACTORY_KEY') + maven = true + } + + defaults { + if (project.ext.has('publications')) { + publications(*project.ext.publications) + } else { + publications('jar') + + if (plugins.hasPlugin('pegasus')) { + publications('jar', 'dataTemplate') + } + } + } + } + + resolve { + repository { + repoKey = 'release' + username = System.getenv('ARTIFACTORY_USER') + password = System.getenv('ARTIFACTORY_KEY') + maven = true + } + } +} + +artifactoryPublish { + skip = project.hasProperty('artifactory.dryRun') +} diff --git a/gradle/release.gradle b/gradle/release.gradle new file mode 100644 index 0000000..a879654 --- /dev/null +++ b/gradle/release.gradle @@ -0,0 +1,39 @@ +apply plugin: 'org.shipkit.shipkit-auto-version' +apply plugin: 'org.shipkit.shipkit-changelog' +apply plugin: 'org.shipkit.shipkit-github-release' + + +if (!project.hasProperty('disableShipkit')) { + task checkGitHubToken { + doFirst { + if (System.getenv("GITHUB_TOKEN") == null) { + throw new Exception("Environment variable GITHUB_TOKEN not set."); + } + println "Using repository " + System.getenv("GITHUB_REPOSITORY") + } + } + + tasks.named("generateChangelog") { + dependsOn checkGitHubToken + previousRevision = project.ext.'shipkit-auto-version.previous-tag' + githubToken = System.getenv("GITHUB_TOKEN") + repository = System.getenv("GITHUB_REPOSITORY") + } + + tasks.named("githubRelease") { + dependsOn tasks.named("generateChangelog") + dependsOn checkGitHubToken + repository = System.getenv("GITHUB_REPOSITORY") + changelog = tasks.named("generateChangelog").get().outputFile + githubToken = System.getenv("GITHUB_TOKEN") + newTagRevision = System.getenv("GITHUB_SHA") + } +} + +allprojects { p -> + group = "com.linkedin.cdi" +} + +if (!project.hasProperty('disableShipkit')) { + apply from: file("gradle/ci-release.gradle") +} \ No newline at end of file diff --git a/gradle/scripts/bintrayPublishing.gradle b/gradle/scripts/bintrayPublishing.gradle deleted file mode 100644 index 45e2547..0000000 --- a/gradle/scripts/bintrayPublishing.gradle +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -import org.gradle.api.internal.java.JavaLibrary - -ext.pomAttributes = { - packaging = 'jar' - // optionally artifactId can be defined here - name = 'Apache Gobblin (incubating)' - description = 'A distributed data integration framework for streaming and batch data ecosystems.' - url = 'https://gobblin.apache.org/' - - scm { - url = 'https://git-wip-us.apache.org/repos/asf?p=incubator-gobblin.git;a=tree' - connection = 'scm:http://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' - developerConnection = 'scm:https://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' - } - - licenses { - license { - name = 'BSD 2-CLAUSE LICENSE' - url = 'http://www.opensource.org/licenses/bsd-license.php' - } - } -} - -// Default publication (mavenJava) to publish artifacts as maven modules -// Uses maven-publish plugin and generates the default jar, sources, javadoc -// and pom file (https://docs.gradle.org/current/userguide/publishing_maven.html) -subprojects{ - plugins.withType(JavaPlugin) { - plugins.apply('maven-publish') - publishing { - publications { - mavenJava(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - - pom pomAttributes - - pom.withXml { - //Ensures that correct dependencies are in the pom when subproject declares a project dependency - // with specific target configuration (restClient, dataTemplate, etc.) - //Needed because pom model is lossy and does not carry the 'configuration' information - def dependenciesNode = it.asNode().dependencies[0] - def removed = [] as Set; def added = [] //helps auditing - configurations.runtime.allDependencies.each { d -> - def confToPom = ['restClient': 'rest-client', 'dataTemplate': 'data-template', 'tests': 'test'] - if (d instanceof ProjectDependency && confToPom.containsKey(d.targetConfiguration)) { - boolean dependsOnMainModule = configurations.runtime.allDependencies.any { - it.name == d.name && it.targetConfiguration == 'default' - } - if (!dependsOnMainModule) { - //subproject declares a dependency on target configuration (i.e. path: 'gobblin-rest-api', configuration: 'restClient') - // but does not declare a dependency on the 'default' artifact (i.e. 'gobblin-rest-api') - //we need to remove the 'default' artifact from the pom file - def mainModuleNode = dependenciesNode.find { it.artifactId.text() == d.name } - dependenciesNode.remove(mainModuleNode) - removed.add(d.name) - } - - //adding explicit dependency on the artifact that corresponds to given target configuration - // (i.e. 'gobblin-rest-api-rest-client') - def dependencyNode = dependenciesNode.appendNode('dependency') - dependencyNode.appendNode('groupId', d.group) - def newArtifactId = d.name + "-" + confToPom[d.targetConfiguration] - dependencyNode.appendNode('artifactId', newArtifactId) - dependencyNode.appendNode('version', d.version) - added.add(newArtifactId) - - //adding global exclude rules to the new dependency entries - if (configurations.runtime.excludeRules.size() > 0) { - def exclusionsNode = dependencyNode.appendNode('exclusions') - configurations.runtime.excludeRules.each { rule -> - def exclusionNode = exclusionsNode.appendNode('exclusion') - exclusionNode.appendNode('groupId', rule.group ?: "*") - exclusionNode.appendNode('artifactId', rule.module) - } - } - } - } - if (added || removed) { - logger.lifecycle("Updated pom dependencies in {}, added: {}, removed: {}", project.path, added, removed) - } - } - } - } - } - } - - // Using gradle bintray plugin to publish artifacts to Jfrog bintray - plugins.apply('com.jfrog.bintray') - bintray { - user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - publications = ["mavenJava"] - publish = true - override = project.hasProperty("bintray.override") //[Default: false] - dryRun = project.hasProperty("bintray.dryRun") //[Default: false] - pkg { - repo = 'maven' - name = 'gobblin-github' - userOrg = 'linkedin' - licenses = ['Apache-2.0'] - vcsUrl = 'https://github.com/apache/incubator-gobblin.git' - version { - name = project.version - desc = 'Apache Gobblin' - released = new Date() - // disable gpg signing to speed up publishing - gpg { - sign = false - } - // disable upload to maven central - mavenCentralSync { - sync = false - } - } - } - } - - tasks.bintrayUpload { - doFirst { - println "Running bintrayUpload for $project.name, publications: $project.bintray.publications" - } - dependsOn publishToMavenLocal - } - - ext.addPublicationToBintray = { pubName -> - project.bintray.publications += pubName - } - - ext.addRuntimeDependenciesToPom = { - def dependenciesNode = asNode().appendNode('dependencies') - configurations.runtime.allDependencies.each { - if (it.group != null && it.name != null) { - def dependencyNode = dependenciesNode.appendNode('dependency') - dependencyNode.appendNode('groupId', it.group) - dependencyNode.appendNode('artifactId', it.name) - dependencyNode.appendNode('version', it.version) - - if (it.excludeRules.size() > 0) { - def exclusionsNode = dependencyNode.appendNode('exclusions') - it.excludeRules.each { rule -> - def exclusionNode = exclusionsNode.appendNode('exclusion') - exclusionNode.appendNode('groupId', rule.group) - exclusionNode.appendNode('artifactId', rule.module) - } - } - } - } - } - -} \ No newline at end of file diff --git a/gradle/scripts/buildscript.gradle b/gradle/scripts/buildscript.gradle deleted file mode 100644 index 24db83d..0000000 --- a/gradle/scripts/buildscript.gradle +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -repositories { - repositories { - maven { - // For gradle-nexus-plugin - url 'https://jcenter.bintray.com/' - } - } -} - -dependencies { - classpath 'org.gradle.api.plugins:gradle-nexus-plugin:0.7.1' - classpath 'com.fizzpod:gradle-sweeney-plugin:1.0+' -} \ No newline at end of file diff --git a/gradle/scripts/ci-support.gradle b/gradle/scripts/ci-support.gradle deleted file mode 100644 index 71062ed..0000000 --- a/gradle/scripts/ci-support.gradle +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - - -def corePaths = ["gobblin-binary-management", "gobblin-compaction", "gobblin-core", "gobblin-data-management", - "gobblin-hive-registration", "gobblin-runtime", "gobblin-yarn", "gobblin-metrics-libs", "gobblin-runtime-hadoop"] -def servicePaths = ["gobblin-api", "gobblin-rest-service", "gobblin-restli", "gobblin-service"] -def modulePaths = ["gobblin-modules"] - -task getGroupedTests { - doLast { - def taskNames = subprojects.findAll { - subproject -> subproject.tasks.hasProperty('test') - } - def includedGroups - switch(groupName) { - case "Core Tests": - includedGroups = taskNames.findAll {task -> - corePaths.any { - task.path.contains(it) - } - } - break; - case "Service Tests": - includedGroups = taskNames.findAll {task -> - servicePaths.any { - task.path.contains(it) - } - } - break; - case "Module Tests": - includedGroups = taskNames.findAll {task -> - modulePaths.any { - task.path.contains(it) - } - } - break; - case "Other Tests": - corePaths.addAll(servicePaths) - corePaths.addAll(modulePaths) - includedGroups = taskNames.findAll { task -> - !corePaths.any { - task.path.contains(it) - } - } - break; - default: - includedGroups = taskNames - break; - } - - def groupedTaskNames = includedGroups.collect { task -> task.tasks.findByName('test').getPath() } - println "CI Task: " + groupedTaskNames.join(" ") - } -} diff --git a/gradle/scripts/computeVersions.gradle b/gradle/scripts/computeVersions.gradle deleted file mode 100644 index b850d18..0000000 --- a/gradle/scripts/computeVersions.gradle +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -if (!project.hasProperty('version') || project.version == 'unspecified') { - try { - exec { - commandLine 'git', 'fetch', '-t', 'https://github.com/apache/incubator-gobblin.git', 'master' - } - def versionOut = new ByteArrayOutputStream() - exec { - commandLine 'git', 'describe', '--tags', '--always' - standardOutput versionOut - } - def tagStr = versionOut.toString().trim() - println 'Using latest tag for version: ' + tagStr - if (tagStr.startsWith("gobblin_")) { - project.version = tagStr.substring(8) - } - else { - project.version = tagStr - } - } - catch (Exception e) { - logger.warn("Unable to determine version. Is this a git copy? Using 'unknown'."); - project.version = 'unknown' - } -} - -if (!project.hasProperty('pegasusVersion')) { - project.ext.pegasusVersion = '6.0.12' -} - -if (project.hasProperty('jdkVersion')) { - if (project.jdkVersion.equals(JavaVersion.VERSION_1_8.toString())) { - ext.javaVersion = JavaVersion.VERSION_1_8 - } else { - throw new RuntimeException("Unsupported JDK version: " + project.jdkVersion) - } -} - -// Hadoop validation -if (! hadoopVersion.startsWith('2.')) { - throw new GradleScriptException("Only Hadoop 2.x is supported: " + hadoopVersion); -} - -def gradleVersions = project.gradle.gradleVersion.tokenize(".") -ext.gradleVersionMajor = Integer.parseInt(gradleVersions[0]) -ext.gradleVersionMinor = Integer.parseInt(gradleVersions[1]) -println "Detected Gradle version major=" + gradleVersionMajor + " minor=" + gradleVersionMinor - -ext.dropwizardMetricsVersion = '3.2.3' -ext.findBugsVersion = '3.0.0' -ext.googleVersion = '1.22.0' -ext.slf4jVersion = '1.7.21' -ext.log4jVersion = '1.2.17' diff --git a/gradle/scripts/defaultBuildProperties.gradle b/gradle/scripts/defaultBuildProperties.gradle deleted file mode 100644 index 3e81fd3..0000000 --- a/gradle/scripts/defaultBuildProperties.gradle +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -import org.apache.gobblin.gradle.BuildProperties -import org.apache.gobblin.gradle.BuildProperty - -def BuildProperties BUILD_PROPERTIES = new BuildProperties(project) - .register(new BuildProperty("sonatypeArtifactRepository", "https://oss.sonatype.org/service/local/staging/deploy/maven2/", "Maven repository to publish artifacts")) - .register(new BuildProperty("sonatypeArtifactSnapshotRepository", "https://oss.sonatype.org/content/repositories/snapshots/", "Maven repository to publish artifacts")) - .register(new BuildProperty("nexusArtifactRepository", "https://repository.apache.org/service/local/staging/deploy/maven2", "Maven repository to publish artifacts")) - .register(new BuildProperty("nexusArtifactSnapshotRepository", "https://repository.apache.org/content/repositories/snapshots", "Maven repository to publish artifacts")) - .register(new BuildProperty("gobblinVersion", "0.15.0-dev-9467", "Gobblin dependencies version")) - .register(new BuildProperty("avroVersion", "1.8.1", "Avro dependencies version")) - .register(new BuildProperty("awsVersion", "2.10.15", "AWS dependencies version")) - .register(new BuildProperty("bytemanVersion", "4.0.5", "Byteman dependencies version")) - .register(new BuildProperty("confluentVersion", "2.0.1", "confluent dependencies version")) - .register(new BuildProperty("doNotSignArtifacts", false, "Do not sight Maven artifacts")) - .register(new BuildProperty("gobblinFlavor", "standard", "Build flavor (see http://gobblin.readthedocs.io/en/latest/developer-guide/GobblinModules/)")) - .register(new BuildProperty("hadoopVersion", "2.3.0", "Hadoop dependencies version")) - .register(new BuildProperty("hiveVersion", "1.0.1", "Hive dependencies version")) - .register(new BuildProperty("icebergVersion", "0.10.0", "Iceberg dependencies version")) - .register(new BuildProperty("jdkVersion", JavaVersion.VERSION_1_8.toString(), - "Java languange compatibility; supported versions: " + JavaVersion.VERSION_1_8)) - .register(new BuildProperty("kafka08Version", "0.8.2.2", "Kafka 0.8 dependencies version")) - .register(new BuildProperty("kafka09Version", "0.9.0.1", "Kafka 0.9 dependencies version")) - .register(new BuildProperty("kafka1Version", "1.1.1", "Kafka 1.1 dependencies version")) - .register(new BuildProperty("pegasusVersion", "29.6.4", "Pegasus dependencies version")) - .register(new BuildProperty("publishToMaven", false, "Enable publishing of artifacts to a central Maven repository")) - .register(new BuildProperty("publishToNexus", false, "Enable publishing of artifacts to Nexus")) - .register(new BuildProperty("salesforceVersion", "42.0.0", "Salesforce dependencies version")) - -task buildProperties(description: 'Lists main properties that can be used to customize the build') { - doLast { - BUILD_PROPERTIES.printHelp(); - } -} - -// Gobblin build customization -BUILD_PROPERTIES.ensureDefined('gobblinFlavor') - -// Compiler compatibility -BUILD_PROPERTIES.ensureDefined('jdkVersion') - -BUILD_PROPERTIES.ensureDefined('sonatypeArtifactRepository') -BUILD_PROPERTIES.ensureDefined('sonatypeArtifactSnapshotRepository') -BUILD_PROPERTIES.ensureDefined('nexusArtifactRepository') -BUILD_PROPERTIES.ensureDefined('nexusArtifactSnapshotRepository') -BUILD_PROPERTIES.ensureDefined('doNotSignArtifacts') - -// Library dependencies versions -BUILD_PROPERTIES.ensureDefined('gobblinVersion') -BUILD_PROPERTIES.ensureDefined('avroVersion') -BUILD_PROPERTIES.ensureDefined('awsVersion') -BUILD_PROPERTIES.ensureDefined('bytemanVersion') -BUILD_PROPERTIES.ensureDefined('confluentVersion') -BUILD_PROPERTIES.ensureDefined('hadoopVersion') -BUILD_PROPERTIES.ensureDefined('hiveVersion') -BUILD_PROPERTIES.ensureDefined('icebergVersion') -BUILD_PROPERTIES.ensureDefined('kafka08Version') -BUILD_PROPERTIES.ensureDefined('kafka09Version') -BUILD_PROPERTIES.ensureDefined('kafka1Version') -BUILD_PROPERTIES.ensureDefined('pegasusVersion') -BUILD_PROPERTIES.ensureDefined('salesforceVersion') - -ext.buildProperties = BUILD_PROPERTIES diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle index 2fb5af0..3ff75d8 100644 --- a/gradle/scripts/dependencyDefinitions.gradle +++ b/gradle/scripts/dependencyDefinitions.gradle @@ -2,12 +2,6 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -dependencyManagement { - imports { - mavenBom 'com.amazonaws:aws-java-sdk-bom:1.11.8' - } -} - ext.externalDependency = [ "antlrRuntime": "org.antlr:antlr-runtime:3.5.2", "avro": "org.apache.avro:avro:" + avroVersion, @@ -60,24 +54,10 @@ ext.externalDependency = [ "httpmime": "org.apache.httpcomponents:httpmime:4.5.2", "httpcore": "org.apache.httpcomponents:httpcore:4.4.11", "httpasyncclient": "org.apache.httpcomponents:httpasyncclient:4.1.3", - "icebergHive": "org.apache.iceberg:iceberg-hive-runtime:" + icebergVersion, "jgit": "org.eclipse.jgit:org.eclipse.jgit:5.1.1.201809181055-r", "jmh": "org.openjdk.jmh:jmh-core:1.17.3", "jmhAnnotations": "org.openjdk.jmh:jmh-generator-annprocess:1.17.3", "jollyday": "de.jollyday:jollyday:0.4.9", - "kafka08": "org.apache.kafka:kafka_2.11:" + kafka08Version, - "kafka08Test": "org.apache.kafka:kafka_2.11:" + kafka08Version + ":test", - "kafka08Client": "org.apache.kafka:kafka-clients:" + kafka08Version, - "kafka09": "org.apache.kafka:kafka_2.11:" + kafka09Version, - "kafka09Test": "org.apache.kafka:kafka_2.11:" + kafka09Version + ":test", - "kafka09Client": "org.apache.kafka:kafka-clients:" + kafka09Version, - "kafka1": "org.apache.kafka:kafka_2.11:" + kafka1Version, - "kafka1Test": "org.apache.kafka:kafka_2.11:" + kafka1Version + ":test", - "kafka1Client": "org.apache.kafka:kafka-clients:" + kafka1Version, - "kafka1ClientTest": "org.apache.kafka:kafka-clients:" + kafka1Version + ":test", - "confluentSchemaRegistryClient": "io.confluent:kafka-schema-registry-client:" + confluentVersion, - "confluentAvroSerializer": "io.confluent:kafka-avro-serializer:" + confluentVersion, - "confluentJsonSerializer": "io.confluent:kafka-json-serializer:" + confluentVersion, "zkClient": "com.101tec:zkclient:0.7", "quartz": "org.quartz-scheduler:quartz:2.2.3", "testng": "org.testng:testng:6.9.10", @@ -88,9 +68,6 @@ ext.externalDependency = [ "jasypt": "org.jasypt:jasypt:1.9.2", "jodaTime": "joda-time:joda-time:2.9.3", "jgrapht": "org.jgrapht:jgrapht-core:0.9.2", - "metricsCore": "io.dropwizard.metrics:metrics-core:" + dropwizardMetricsVersion, - "metricsJvm": "io.dropwizard.metrics:metrics-jvm:" + dropwizardMetricsVersion, - "metricsGraphite": "io.dropwizard.metrics:metrics-graphite:" + dropwizardMetricsVersion, "jsch": "com.jcraft:jsch:0.1.54", "jdo2": "javax.jdo:jdo2-api:2.1", "azkaban": "com.linkedin.azkaban:azkaban:2.5.0", @@ -102,8 +79,6 @@ ext.externalDependency = [ "derby": "org.apache.derby:derby:10.12.1.1", "mockito": "org.mockito:mockito-core:1.10.19", "powermock": "org.powermock:powermock-mockito-release-full:1.6.2", - "salesforceWsc": "com.force.api:force-wsc:" + salesforceVersion, - "salesforcePartner": "com.force.api:force-partner-api:" + salesforceVersion, "scala": "org.scala-lang:scala-library:2.11.8", "influxdbJava": "org.influxdb:influxdb-java:2.1", "kryo": "com.esotericsoftware.kryo:kryo:2.22", @@ -112,8 +87,6 @@ ext.externalDependency = [ "mockRunnerJdbc":"com.mockrunner:mockrunner-jdbc:1.0.8", "xerces":"xerces:xercesImpl:2.11.0", "typesafeConfig": "com.typesafe:config:1.2.1", - "byteman": "org.jboss.byteman:byteman:" + bytemanVersion, - "bytemanBmunit": "org.jboss.byteman:byteman-bmunit:" + bytemanVersion, "bcpgJdk15on": "org.bouncycastle:bcpg-jdk15on:1.52", "bcprovJdk15on": "org.bouncycastle:bcprov-jdk15on:1.52", "calciteCore": "org.apache.calcite:calcite-core:1.16.0", @@ -155,16 +128,9 @@ ext.externalDependency = [ "googleAnalytics": "com.google.apis:google-api-services-analytics:v3-rev134-1.22.0", "googleDrive": "com.google.apis:google-api-services-drive:v3-rev42-1.22.0", "googleWebmasters": "com.google.apis:google-api-services-webmasters:v3-rev17-1.22.0", - "googleHttpClient": "com.google.http-client:google-http-client:" + googleVersion, - "googleHttpClientGson": "com.google.http-client:google-http-client-gson:" + googleVersion, - "googleOauthClient": "com.google.oauth-client:google-oauth-client:" + googleVersion, - "googleApiClient": "com.google.api-client:google-api-client:" + googleVersion, "opencsv": "com.opencsv:opencsv:3.8", "grok": "io.thekraken:grok:0.1.5", "hadoopAdl" : "org.apache.hadoop:hadoop-azure-datalake:3.0.0-alpha2", - /** - * Avoiding conflicts with Hive 1.x versions existed in the classpath - */ "orcMapreduce":"org.apache.orc:orc-mapreduce:1.6.5:nohive", "orcCore": "org.apache.orc:orc-core:1.6.5:nohive", "orcTools":"org.apache.orc:orc-tools:1.6.5", @@ -220,13 +186,3 @@ ext.externalDependency = [ "okhttp": "com.squareup.okhttp3:okhttp:4.8.0", "commonsValidator": "commons-validator:commons-validator:1.6" ] - -if (!isDefaultEnvironment) -{ - ext.externalDependency.each { overrideDepKey, overrideDepValue -> - if (externalDependency[overrideDepKey] != null) - { - externalDependency[overrideDepKey] = overrideDepValue - } - } -} diff --git a/gradle/scripts/environment.gradle b/gradle/scripts/environment.gradle deleted file mode 100644 index e9f0d84..0000000 --- a/gradle/scripts/environment.gradle +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -ext.build_script_dir = "${projectDir.path}/build_script" -ext.isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment') - -File getEnvironmentScript() { - final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment) - assert env.isFile() : "The environment script [$env] does not exists or is not a file." - return env -} - -apply from: environmentScript diff --git a/gradle/scripts/globalDependencies.gradle b/gradle/scripts/globalDependencies.gradle deleted file mode 100644 index 456d989..0000000 --- a/gradle/scripts/globalDependencies.gradle +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -import javax.tools.ToolProvider - - -subprojects { - plugins.withType(JavaPlugin) { - configurations { - customTestCompile - compile - dependencies { - if (!project.name.contains('gobblin-elasticsearch-deps')) { - compile(externalDependency.hadoopCommon) { - exclude module: 'servlet-api' - } - compile externalDependency.hadoopClientCore - compile externalDependency.hadoopAnnotations - if (project.name.equals('gobblin-runtime') || project.name.equals('gobblin-test')) { - compile externalDependency.hadoopClientCommon - } - compile(externalDependency.guava) { - force = true - } - } - compile(externalDependency.commonsCodec) { - force = true - } - //Since testCompile inherit from compile, so we cannot use testCompile to import dependency for log4j - customTestCompile externalDependency.log4j - customTestCompile externalDependency.slf4jToLog4j - - // Required to add JDK's tool jar, which is required to run byteman tests. - testCompile (files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs())) - } - if (!project.name.contains('gobblin-aws') && !project.name.contains('gobblin-distribution')) { - configurations.compile.dependencies*.each { - //exclude this jar because we introduce log4j-over-slf4j, these two jars cannot present at the same time - it.exclude group: 'org.slf4j', module: 'slf4j-log4j12' - //exclude log4j related jars to provide a clean log environment - it.exclude group: 'log4j', module: 'log4j' - it.exclude group: 'log4j', module: 'apache-log4j-extras' - } - } - all*.exclude group: 'org.apache.calcite', module: 'calcite-avatica' // replaced by org.apache.calcite.avatica:avatica-core - //exclude this jar in test class path because we are using log4j implementation to test - testCompile.exclude group: 'org.slf4j', module: 'log4j-over-slf4j' - } - test{ - //Add log4j into runtime path - classpath += configurations.customTestCompile - //Add log4j into compile path - sourceSets.test.compileClasspath += configurations.customTestCompile - } - } - - dependencies { - // Gradle 5 compatibility - compileOnly externalDependency.lombok - testCompileOnly externalDependency.lombok - annotationProcessor externalDependency.lombok - testAnnotationProcessor externalDependency.lombok - } -} diff --git a/gradle/scripts/idesSetup.gradle b/gradle/scripts/idesSetup.gradle deleted file mode 100644 index 250df0f..0000000 --- a/gradle/scripts/idesSetup.gradle +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -apply plugin: 'idea' - -subprojects { - plugins.withType(JavaPlugin) { - plugins.apply('idea') - plugins.apply('eclipse') - - // Configure the IDEA plugin to (1) add the codegen as source dirs and (2) work around - // an apparent bug in the plugin which doesn't set the outputDir/testOutputDir as documented - idea.module { - // Gradle docs claim the two settings below are the default, but - // the actual defaults appear to be "out/production/$MODULE_NAME" - // and "out/test/$MODULE_NAME". Changing it so IDEA and gradle share - // the class output directory. - - outputDir = file(sourceSets.main.output.classesDirs.getSingleFile()) - testOutputDir = file(sourceSets.test.output.classesDirs.getSingleFile()) - } - - } -} diff --git a/gradle/scripts/javaPlugin.gradle b/gradle/scripts/javaPlugin.gradle deleted file mode 100644 index 01238e7..0000000 --- a/gradle/scripts/javaPlugin.gradle +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -subprojects { - plugins.withType(JavaPlugin) { - sourceCompatibility = javaVersion - configurations { - // guava-jdk5 conflicts with guava libraries - all*.exclude module: 'guava-jdk5' - } - } -} diff --git a/gradle/scripts/javaVersionCheck.gradle b/gradle/scripts/javaVersionCheck.gradle deleted file mode 100644 index 98e4fca..0000000 --- a/gradle/scripts/javaVersionCheck.gradle +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -gradle.taskGraph.whenReady { - gradle.taskGraph.allTasks.each { task -> - def taskProject = task.project - if (taskProject.hasProperty('requiresJavaVersion') && !taskProject.requiresJavaVersion.equals(javaVersion)) { - logger.warn("WARNING: Project {} requires Java version {} which conflicts with build version {}. COMPILATION DISABLED. Please use -PjdkVersion={} .", - taskProject.name, - taskProject.requiresJavaVersion, - javaVersion, - taskProject.requiresJavaVersion) - task.onlyIf { false } - } - } -} \ No newline at end of file diff --git a/gradle/scripts/javadoc.gradle b/gradle/scripts/javadoc.gradle deleted file mode 100644 index 2303c22..0000000 --- a/gradle/scripts/javadoc.gradle +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -task javadocTarball(type: Tar) { - baseName = "gobblin-javadoc-all" - destinationDir = new File(project.buildDir, baseName) - compression = Compression.GZIP - extension = 'tgz' - description = "Generates a tar-ball with all javadocs to ${destinationDir}/${archiveName}" - doLast { - def indexFile = new File(destinationDir, "index.md") - def version = rootProject.ext.javadocVersion - indexFile << """---- - layout: page - title: Gobblin Javadoc packages ${version} - permalink: /javadoc/${version}/ - ---- - - """ - rootProject.ext.javadocPackages.each { - indexFile << "* [${it}](${it})\n" - } - } -} - -// Javadoc initialization for subprojects -ext.javadocVersion = null != project.version ? project.version.toString() : "latest" -if (ext.javadocVersion.indexOf('-') > 0) { - // Remove any "-" addons from the version - ext.javadocVersion = javadocVersion.substring(0, javadocVersion.indexOf('-')) -} - -ext.javadocPackages = new HashSet() -subprojects.each{Project pr -> - if (file(pr.projectDir.absolutePath + "/src/main/java").exists()) { - rootProject.ext.javadocPackages += pr.name - } -} - -subprojects { - plugins.withType(JavaPlugin) { - - // Sometimes generating javadocs can lead to OOM. This may needs to be increased. - // Also force javadocs to pick up system proxy settings if available - javadoc { - options.encoding = 'UTF-8'; - options.jFlags('-Xmx256m', '-Djava.net.useSystemProxies=true'); - } - - rootProject.tasks.javadocTarball.dependsOn project.tasks.javadoc - if ( rootProject.ext.javadocPackages.contains(project.name)) { - rootProject.tasks.javadocTarball.into(project.name){from(fileTree(dir: "${project.buildDir}/docs/javadoc/"))} - } - } -} - -subprojects { - plugins.withType(JavaPlugin) { - if (isDefaultEnvironment) { - task javadocJar(type: Jar) { - from javadoc - classifier = 'javadoc' - } - artifacts { archives javadocJar } - } - - // Add standard javadoc repositories so we can reference classes in them using @link - tasks.javadoc.options.links "http://typesafehub.github.io/config/latest/api/", - "https://docs.oracle.com/javase/7/docs/api/", - "http://google.github.io/guava/releases/15.0/api/docs/", - "http://hadoop.apache.org/docs/r${rootProject.ext.hadoopVersion}/api/", - "https://hive.apache.org/javadocs/r${rootProject.ext.hiveVersion}/api/", - "http://avro.apache.org/docs/${avroVersion}/api/java/", - "https://dropwizard.github.io/metrics/${dropwizardMetricsVersion}/apidocs/" - } -} - -gradle.projectsEvaluated { - subprojects { - plugins.withType(JavaPlugin) { - getAllDependentProjects(project).each { - tasks.javadoc.options.linksOffline "http://linkedin.github.io/gobblin/javadoc/${javadocVersion}/${it.name}/", - "${rootProject.buildDir}/${it.name}/docs/javadoc/" - } - } - } -} - -//Turn off javadoc lint for Java 8+ -if (JavaVersion.current().isJava8Compatible()) { - allprojects { - tasks.withType(Javadoc) { - options.addStringOption('Xdoclint:none', '-quiet') - } - } -} diff --git a/gradle/scripts/mavenPublishing.gradle b/gradle/scripts/mavenPublishing.gradle deleted file mode 100644 index d35a00b..0000000 --- a/gradle/scripts/mavenPublishing.gradle +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.locks.ReentrantLock - -// Set group for Maven -allprojects { - if (!project.hasProperty('group') || project.group.length() == 0) { - project.ext.group = 'org.apache.gobblin' - project.group = project.ext.group - } -} - -// Maven publishing -ext.buildProperties.ensureDefined('publishToMaven') -if (ext.publishToMaven) { - plugins.apply('maven') - // Workaround for a bug in gradle's "maven" plugin. See https://discuss.gradle.org/t/error-in-parallel-build/7215/3 - project.setProperty("org.gradle.parallel", "false") -} - -ext.signArtifacts = !project.doNotSignArtifacts - -// Maven POM generation is not thread safe, so serialize all the Upload tasks we can use `--parallel`. -// https://issues.gradle.org/browse/GRADLE-2492 -// When we start building with 2.3 and later we should remove this and just add a common output dir for all tasks and let Gradle serialize them -def lock = new ReentrantLock() -def available = lock.newCondition() -def busy = new AtomicBoolean() -def serializedTasks = [] -allprojects { - tasks.matching { it.name == "generatePom" || it instanceof Upload }.all { - serializedTasks << it - doFirst { - lock.lock() - while (busy.get()) { - available.await() - } - busy.set(true) - } - } -} - -gradle.taskGraph.afterTask { - if (it in serializedTasks && lock.heldByCurrentThread) { - busy.set(false) - available.signal() - lock.unlock() - } -} - -subprojects { - plugins.withType(JavaPlugin) { - plugins.apply('maven') - - install { - repositories { - mavenInstaller { - mavenLocal() - pom.project { - packaging 'jar' - name 'Apache Gobblin (incubating)' - description 'A distributed data integration framework for streaming and batch data ecosystems.' - url 'https://gobblin.apache.org/' - } - } - } - } - - // Publishing of maven artifacts for subprojects - if (rootProject.ext.publishToMaven) { - if (rootProject.ext.signArtifacts) { - plugins.apply('signing') - } - - uploadArchives { - repositories { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> - if (rootProject.ext.signArtifacts) { - signing.signPom(deployment) - } - } - - repository(url: rootProject.sonatypeArtifactRepository) { - authentication(userName: rootProject.ext.ossrhUsername, password: rootProject.ext.ossrhPassword) - } - - snapshotRepository(url: rootProject.sonatypeArtifactSnapshotRepository) { - authentication(userName: rootProject.ext.ossrhUsername, password: rootProject.ext.ossrhPassword) - } - - pom.project pomAttributes - } - } - } - - if (rootProject.ext.signArtifacts) { - signing { - sign configurations.archives - } - } - } - } -} diff --git a/gradle/scripts/nexusPublishing.gradle b/gradle/scripts/nexusPublishing.gradle deleted file mode 100644 index e811882..0000000 --- a/gradle/scripts/nexusPublishing.gradle +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.locks.ReentrantLock - -// Set group for Maven -allprojects { - if (!project.hasProperty('group') || project.group.length() == 0) { - project.ext.group = 'org.apache.gobblin' - project.group = project.ext.group - } -} - -// Nexus publishing -ext.buildProperties.ensureDefined('publishToNexus') -if (ext.publishToNexus) { - plugins.apply('maven') - // Workaround for a bug in gradle's "maven" plugin. See https://discuss.gradle.org/t/error-in-parallel-build/7215/3 - project.setProperty("org.gradle.parallel", "false") -} - -ext.signArtifacts = !project.doNotSignArtifacts - -// Maven POM generation is not thread safe, so serialize all the Upload tasks we can use `--parallel`. -// https://issues.gradle.org/browse/GRADLE-2492 -// When we start building with 2.3 and later we should remove this and just add a common output dir for all tasks and let Gradle serialize them -def lock = new ReentrantLock() -def available = lock.newCondition() -def busy = new AtomicBoolean() -def serializedTasks = [] -allprojects { - tasks.matching { it.name == "generatePom" || it instanceof Upload }.all { - serializedTasks << it - doFirst { - lock.lock() - while (busy.get()) { - available.await() - } - busy.set(true) - } - } -} - -gradle.taskGraph.afterTask { - if (it in serializedTasks && lock.heldByCurrentThread) { - busy.set(false) - available.signal() - lock.unlock() - } -} - -ext.pomAttributes = { - packaging 'jar' - // optionally artifactId can be defined here - name 'Apache Gobblin (incubating)' - description 'A distributed data integration framework for streaming and batch data ecosystems.' - url 'https://gobblin.apache.org/' - - scm { - url 'https://git-wip-us.apache.org/repos/asf?p=incubator-gobblin.git;a=tree' - connection 'scm:http://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' - developerConnection 'scm:https://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' - } - - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } -} - -subprojects { - plugins.withType(JavaPlugin) { - plugins.apply('maven') - - install { - repositories { - mavenInstaller { - mavenLocal() - pom.project { - packaging 'jar' - name 'Apache Gobblin (incubating)' - description 'A distributed data integration framework for streaming and batch data ecosystems.' - url 'https://gobblin.apache.org/' - } - } - } - } - - // Publishing of maven artifacts for subprojects - if (rootProject.ext.publishToNexus) { - if (rootProject.ext.signArtifacts) { - plugins.apply('signing') - } - - uploadArchives { - repositories { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> - if (rootProject.ext.signArtifacts) { - signing.signPom(deployment) - } - } - - repository(url: rootProject.nexusArtifactRepository) { - authentication(userName: rootProject.ext.nexusUsername, password: rootProject.ext.nexusPassword) - } - - snapshotRepository(url: rootProject.nexusArtifactSnapshotRepository) { - authentication(userName: rootProject.ext.nexusUsername, password: rootProject.ext.nexusPassword) - } - - pom.project pomAttributes - } - } - } - - if (rootProject.ext.signArtifacts) { - signing { - sign configurations.archives - } - } - } - } -} diff --git a/gradle/scripts/release.gradle b/gradle/scripts/release.gradle deleted file mode 100644 index 3badd61..0000000 --- a/gradle/scripts/release.gradle +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -project(':') { - apply plugin: 'base' - apply plugin: 'signing' -} - -def isRelease = ext.release.toBoolean() - -def releaseVersion = project.version -println String.format("Release Version: %s", releaseVersion) - -// Modify the gradle.properties to indicate whether this is a release. This results in the -// source releases generating artifacts without -SNAPSHOT appended to the version when they are -// built. -task prepare_release_config() { - project.copy { - from "$rootDir/gradle.properties" - into "$rootDir" - rename { filename -> - filename + ".release" - } - filter { line -> - if (isRelease && line.startsWith("release=")) { - "release=true" - } else { - line - } - } - } -} - -task sourceRelease(type: Tar, dependsOn: prepare_release_config) { - description = "Build a source release, specifically excluding the build directories and gradle wrapper files" - compression = Compression.GZIP - - baseName "gobblin-connectors-sources-${releaseVersion}" - - from(project.rootDir) { - // don't include gradle.properties because we use a modified version - exclude "gradle.properties" - exclude '**/build' - exclude '.gradle' - // including gradlew but excluding related jar because of: https://issues.apache.org/jira/browse/LEGAL-288 - // instructions for downloading the gradle-wrapper.jar have been added to the README - exclude 'gradle/wrapper/gradle-wrapper.jar' - exclude '.github' - exclude 'maven-sonatype' - exclude 'travis' - exclude '.classpath*' - exclude '.project*' - exclude '.settings' - exclude '**/.idea' - exclude '**/*.iml' - exclude '**/*.iws' - exclude '**/*.ipr' - exclude '**/.classpath' - exclude '**/.project' - exclude '**/.settings' - exclude '**/*.swp' - exclude '**/*.swo' - exclude '**/*.log' - exclude '**/build/' - exclude '**/.gradle' - exclude 'test-output' - exclude '**/test-output' - exclude 'dist' - exclude 'target' - exclude 'tmp' - exclude 'out' - exclude '**/out' - exclude 'output' - exclude 'gobblin-test/basicTest' - exclude 'gobblin-test/jobOutput' - exclude 'gobblin-test/state-store' - exclude 'gobblin-tesTaskt/metrics' - exclude 'gobblin-test/byteman' - exclude 'gobblin-test/locks' - exclude 'gobblin-test/mr-jobs' - exclude '**/mainGeneratedDataTemplate' - exclude '**/mainGeneratedRest' - exclude 'gobblin-dist' - exclude 'metastore_db' - exclude 'GobblinKey_*.pem' - exclude 'node_modules' - exclude 'package-lock.json' - exclude '**/gen-java' - exclude '**/generated-gobblin-cluster.conf' - exclude 'gobblin-modules/gobblin-couchbase/mock-couchbase' - exclude 'gobblin-modules/gobblin-elasticsearch/test-elasticsearch' - } - - // rename gradle.properties.release to gradle.properties - rename { filename -> - if (filename.equals("gradle.properties.release")) { - "gradle.properties" - } - else { - filename - } - } - - into(baseName) - - // Set destination directory. - destinationDir = file("${project.buildDir}/distribution/source") - - archiveName = "${baseName}.tgz" - doLast { - // Disable md5, since Apache release policy has changed - // ant.checksum file:"$destinationDir/$archiveName", algorithm: "MD5", fileext: ".md5" - ant.checksum file:"$destinationDir/$archiveName", algorithm: "SHA-512", fileext: ".sha512" - } -} - -// Signing requires a user ~/.gradle/gradle.properties file with signing information. -// See: http://www.gradle.org/docs/current/userguide/signing_plugin.html -signing { - // Sign the source archive if the proper configuration is in place to do so. Otherwise - // skip the signing process (it isn't required). This archive can be manually signed if - // needed. - required false - sign sourceRelease -} - -task release(dependsOn: signSourceRelease) - -// Publishing to Apache's Maven repository (Nexus). To install the archives in the -// local repository, run the 'install' task. -//subprojects { -// apply plugin: 'nexus' -// -// nexus { -// attachSources = false -// attachTests = false -// attachJavadoc = false -// sign = true -// repositoryUrl = 'https://repository.apache.org/service/local/staging/deploy/maven2' -// snapshotRepositoryUrl = 'https://repository.apache.org/content/repositories/snapshots' -// } -// -// modifyPom { -// project { -// name 'Apache Gobblin (incubating)' -// description 'A distributed data integration framework for streaming and batch data ecosystems.' -// url 'https://gobblin.apache.org/' -// -// scm { -// url 'https://git-wip-us.apache.org/repos/asf?p=incubator-gobblin.git;a=tree' -// connection 'scm:http://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' -// developerConnection 'scm:https://git-wip-us.apache.org/repos/asf/incubator-gobblin.git' -// } -// -// licenses { -// license { -// name 'The Apache Software License, Version 2.0' -// url 'http://www.apache.org/licenses/LICENSE-2.0.txt' -// } -// } -// } -// } -//} \ No newline at end of file diff --git a/gradle/scripts/repositories.gradle b/gradle/scripts/repositories.gradle index 3a2efce..f21d27e 100644 --- a/gradle/scripts/repositories.gradle +++ b/gradle/scripts/repositories.gradle @@ -11,7 +11,7 @@ repositories { url "http://packages.confluent.io/maven/" } maven { - url "https://linkedin.bintray.com/maven" + url "https://linkedin.jfrog.io/artifactory/open-source/" } jcenter() } diff --git a/gradle/scripts/restli.gradle b/gradle/scripts/restli.gradle deleted file mode 100644 index 7eb9a72..0000000 --- a/gradle/scripts/restli.gradle +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -subprojects { - - afterEvaluate { - // add the standard pegasus dependencies wherever the plugin is used - if (project.plugins.hasPlugin('pegasus')) { - dependencies { - dataTemplateCompile externalDependency.pegasus.data - restClientCompile externalDependency.pegasus.restliClient,externalDependency.pegasus.restliCommon,externalDependency.pegasus.restliTools - } - } - } -} diff --git a/gradle/scripts/sourcesJar.gradle b/gradle/scripts/sourcesJar.gradle deleted file mode 100644 index 7afb676..0000000 --- a/gradle/scripts/sourcesJar.gradle +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -subprojects { - plugins.withType(JavaPlugin) { - if (isDefaultEnvironment) { - task sourcesJar(type: Jar, dependsOn: classes) { - from sourceSets.main.allSource - classifier = 'sources' - } - artifacts { archives sourcesJar } - } - } -} diff --git a/gradle/scripts/testSetup.gradle b/gradle/scripts/testSetup.gradle index fc0ab31..15c24f8 100644 --- a/gradle/scripts/testSetup.gradle +++ b/gradle/scripts/testSetup.gradle @@ -40,7 +40,7 @@ subprojects { } checkstyle { toolVersion = '7.6.1' - configFile = rootProject.file('config/checkstyle/checkstyle.xml') + configFile = rootProject.file('gradle/checkstyle/checkstyle.xml') } } diff --git a/gradle/scripts/utilities.gradle b/gradle/scripts/utilities.gradle deleted file mode 100644 index 44e9863..0000000 --- a/gradle/scripts/utilities.gradle +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2021 LinkedIn Corporation. All rights reserved. -// Licensed under the BSD-2 Clause license. -// See LICENSE in the project root for license information. - -def getAllDependentProjectsImpl(project) { - def projectDependencies = project.configurations.runtime.getAllDependencies().withType(ProjectDependency) - def dependentProjects = projectDependencies*.dependencyProject - if (dependentProjects.size() > 0) { - dependentProjects.each { dependentProjects += getAllDependentProjects(it) } - } - return dependentProjects.unique() -} - -ext.getAllDependentProjects = {getAllDependentProjectsImpl(it)} - -task dotProjectDependencies(description: 'List of gobblin project dependencies in dot format') { - doLast { - println "// ========= Start of project dependency graph ======= " - println "digraph project_dependencies {" - subprojects.each { Project project -> - def project_node_name = project.name.replaceAll("-", "_") - if (project.configurations.findByName("compile") != null) { - project.configurations.compile.dependencies.each { Dependency dep -> - if (dep instanceof ProjectDependency) { - def dep_node_name = dep.dependencyProject.name.replaceAll("-", "_") - println "\t${project_node_name} -> ${dep_node_name};" - } - } - } - } - println "}" - println "// ========= End of project dependency graph ======= " - } -} diff --git a/maven-nexus/maven-install.sh b/maven-nexus/maven-install.sh deleted file mode 100755 index 2d1219f..0000000 --- a/maven-nexus/maven-install.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - - -#group is overiden to support forked repositories - -function print_usage(){ - echo "maven-install.sh --version VERSION [--group GROUP]" -} - -for i in "$@" -do - case "$1" in - --version) - VERSION="$2" - shift - ;; - --group) - GROUP="$2" - shift - ;; - --help) - print_usage - exit 0 - ;; - *) - ;; - esac - shift -done - -if [ -z "$VERSION" ]; then - print_usage - exit -fi -echo VERSION=$VERSION - -if [ -z "$GROUP" ]; then - GROUP="org.apache.gobblin" -fi - - -./gradlew install -Dorg.gradle.parallel=false -Pversion=$VERSION -Pgroup=$GROUP diff --git a/maven-nexus/maven-nexus.sh b/maven-nexus/maven-nexus.sh deleted file mode 100755 index 677d0b3..0000000 --- a/maven-nexus/maven-nexus.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -script_dir=$(dirname $0) -script_name=$(basename $0) -GRADLE="$script_dir/../gradlew" - -function print_usage() { - echo -e "USAGE: $0 [-remote|-local] [-noclean] [gradle_args]" - echo - echo -e "Publishes signed maven artifacts locally ($HOME/.m2/repository) or remotely (Nexus)." - echo -e "\t-local Publish to local repository" - echo -e "\t-noclean Don't run gradlew clean (useful if re-running)" - echo -e "\t-remote Publish to Nexus repository" - echo -e "\t-packages a comma-separated list of gradle paths to publish (e.g. :gobblin-api,:gobblin-core)" - echo - echo -e "NOTES:" - echo -e "\t1. You need the Gobblin PGP key to sign the artifacts. If you don't have it," - echo -e "\t talk to another committer to get it. You also need to add the following to" - echo -e "\t your $HOME/.gradle/gradle.properties file:" - echo - echo -e "signing.keyId=" - echo -e "signing.password=" - echo -e "signing.secretKeyRingFile=$HOME/.gnupg/secring.gpg" - echo - echo -e "\t2. To upload remotely, you'll need a Nexus account. Visit " - echo -e "\t https://repository.apache.org/ to set it up. After" - echo -e "\t that add to your $HOME/.gradle/gradle.properties file:" - echo - echo -e "nexusUsername=" - echo -e "nexusPassword=" - echo - echo -e "\t3. Uploading remotely will upload only to the Nexus staging directory. " - echo -e "\t4. Don't forget to create a gobblin_ tag before publishing remotely!" - echo -e "\t5. Sometimes build with fail with an error" - echo -e "\t '... Failed to interpolate field: private java.lang.String ...'" - echo -e "\t Just re-run with -noclean" -} - -if [ "$#" -eq 0 ] ; then - print_usage - exit -fi - -install_target= -gradle_args= -noclean= -declare -a packages - -# Parse command line -while [ "$#" -gt 0 ] ; do - A="$1" - case "$A" in - -local) - install_target=install - ;; - -noclean) - noclean="1" - ;; - -remote) - install_target=uploadArchives - ;; - -h|--help|-help) - print_usage - exit - ;; - -packages) - shift - packages=( ${1//,/ } ) - ;; - *) - gradle_args="$gradle_args $A" - ;; - esac - shift -done - -if [ -z "${install_target}" ] ; then - echo "${script_name}: missing install target" - exit 1 -fi - -declare -a package_targets -for P in "${packages[@]}" ; do - ptarget="$P:${install_target}" - if [ "${ptarget:0:1}" != ":" ] ; then - ptarget=":$ptarget" - fi - package_targets+=( "$ptarget" ) -done - -if [ "${#packages[@]}" -gt 0 ] ; then - install_target="${package_targets[@]}" -fi - -if [ -z "$noclean" ] ; then - $GRADLE clean -fi -$GRADLE -PpublishToNexus -Porg.gradle.parallel=false -Porg.gradle.daemon=false -xtest $gradle_args $install_target 2>&1 | tee /tmp/${script_name}.out diff --git a/maven-nexus/upload-to-nexus.sh b/maven-nexus/upload-to-nexus.sh deleted file mode 100755 index 7994b81..0000000 --- a/maven-nexus/upload-to-nexus.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -PROG=$(basename $0) - -function usage() { - echo -e "USAGE: $PROG" -} - -# main() - -if [ "$#" -eq 0 ] ; then - usage - exit -fi - -while [ "$#" -gt 0 ] ; do - A="$1" - case "$A" in - -h|--help) - usage - exit - ;; - *) - echo "$PROG: unknown option: $A" - exit 1 - ;; - esac - shift -done - -echo "CLEANING" -./gradlew clean - -upload_all=0 - -for P in :gobblin-admin :gobblin-api :gobblin-azkaban :gobblin-compaction :gobblin-config-management:gobblin-config-core :gobblin-config-management:gobblin-config-client :gobblin-core :gobblin-data-management :gobblin-distribution :gobblin-example :gobblin-metastore :gobblin-metrics :gobblin-oozie :gobblin-rest-service:gobblin-rest-api :gobblin-rest-service:gobblin-rest-client :gobblin-rest-service:gobblin-rest-server :gobblin-runtime :gobblin-salesforce :gobblin-scheduler :gobblin-test :gobblin-iceberg :gobblin-test-harness :gobblin-utility :gobblin-yarn ; do - echo "----------------------------" - if [ "$upload_all" -eq 0 ] ; then - read -p "UPLOAD $P [y(es)|n(o)|a(ll)]" ans - ans=$(echo "$ans" | tr '[:upper:]' '[:lower:]') - if [ "$ans" == "a" -o "$ans" == "all" ] ; then - upload_all=1 - ans="y" - fi - else - ans="y" - fi - if [ "$ans" == "y" -o "$ans" == "yes" ] ; then - ./maven-nexus/maven-nexus.sh -remote -noclean -packages $P - else - echo "Skipping $P" - fi - done - diff --git a/maven-sonatype/github-pr-change-log.py b/maven-sonatype/github-pr-change-log.py deleted file mode 100755 index 739d72f..0000000 --- a/maven-sonatype/github-pr-change-log.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# pylint: disable=line-too-long - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# Usage: Python Script that takes in a range of Gobblin Pull Request Numbers and outputs metadata about each Pull Request -# -# An example output would look like: -# -# * [] [PR 902] Make it possible to specify empty job data publisher -# * [] [PR 903] The underlying Avro CodecFactory only matches lowercase codecs, so we… -# * [] [PR 904] Fixed precondition check for overwriting in datapublisher -# -# The output of this script is meant for the CHANGELOG file that is updated before each Gobblin release. -# There is a pair of [] brackets at the beginning of the build which is meant to containg the project name the PR is related to -# -# The script should be run as follows "./pull-requests-change-log.py [github-username] [github-password] [starting-pr-number] [ending-pr-number] -# For example, to produce the above output the command run was "./pull-requests-change-log.py sahilTakiar [my-password] 900 905" - -import sys -import requests - -for prNumber in range(int(sys.argv[3]), int(sys.argv[4])): - pr = requests.get("https://api.github.com/repos/linkedin/gobblin/pulls/" + str(prNumber), auth=(sys.argv[1], sys.argv[2])).json() - if "state" in pr.keys() and pr["state"] == "closed" and pr["merged"]: - print "* [] [PR " + str(pr["number"]) + "] " + pr["title"] diff --git a/maven-sonatype/maven-install.sh b/maven-sonatype/maven-install.sh deleted file mode 100755 index e07d925..0000000 --- a/maven-sonatype/maven-install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -#group is overiden to support forked repositories - -function print_usage(){ - echo "maven-install.sh --version VERSION [--group GROUP]" -} - -for i in "$@" -do - case "$1" in - --version) - VERSION="$2" - shift - ;; - --group) - GROUP="$2" - shift - ;; - --help) - print_usage - exit 0 - ;; - *) - ;; - esac - shift -done - -if [ -z "$VERSION" ]; then - print_usage - exit -fi -echo VERSION=$VERSION - -if [ -z "$GROUP" ]; then - GROUP="gobblin" -fi - - -./gradlew install -Dorg.gradle.parallel=false -Pversion=$VERSION -Pgroup=$GROUP diff --git a/maven-sonatype/maven-sonatype.sh b/maven-sonatype/maven-sonatype.sh deleted file mode 100755 index 233884e..0000000 --- a/maven-sonatype/maven-sonatype.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -script_dir=$(dirname $0) -script_name=$(basename $0) -GRADLE="$script_dir/../gradlew" - -function print_usage() { - echo -e "USAGE: $0 [-remote|-local] [-noclean] [gradle_args]" - echo - echo -e "Publishes signed maven artifacts locally ($HOME/.m2/repository) or remotely (Sonatype)." - echo -e "\t-local Publish to local repository" - echo -e "\t-noclean Don't run gradlew clean (useful if re-running)" - echo -e "\t-remote Publish to Sonatype repository" - echo -e "\t-packages a comma-separated list of gradle paths to publish (e.g. :gobblin-api,:gobblin-core)" - echo - echo -e "NOTES:" - echo -e "\t1. You need the Gobblin PGP key to sign the artifacts. If you don't have it," - echo -e "\t talk to another committer to get it. You also need to add the following to" - echo -e "\t your $HOME/.gradle/gradle.properties file:" - echo - echo -e "signing.keyId=" - echo -e "signing.password=" - echo -e "signing.secretKeyRingFile=$HOME/.gnupg/secring.gpg" - echo - echo -e "\t2. To upload remotely, you'll need a Sonatype account. Visit " - echo -e "\t https://issues.sonatype.org/secure/Signup!default.jspa to set it up. After" - echo -e "\t that add to your $HOME/.gradle/gradle.properties file:" - echo - echo -e "ossrhUsername=" - echo -e "ossrhPassword=" - echo - echo -e "\t3. Uploading remotely will upload only to the Sonatype staging directory. Follow " - echo -e " the steps at http://central.sonatype.org/pages/releasing-the-deployment.html to" - echo -e " synchronize with Maven Central." - echo -e "\t4. Don't forget to create a gobblin_ tag before publishing remotely!" - echo -e "\t5. Sometimes build with fail with an error" - echo -e "\t '... Failed to interpolate field: private java.lang.String ...'" - echo -e "\t Just re-run with -noclean" -} - -if [ "$#" -eq 0 ] ; then - print_usage - exit -fi - -install_target= -gradle_args= -noclean= -declare -a packages - -# Parse command line -while [ "$#" -gt 0 ] ; do - A="$1" - case "$A" in - -local) - install_target=install - ;; - -noclean) - noclean="1" - ;; - -remote) - install_target=uploadArchives - ;; - -h|--help|-help) - print_usage - exit - ;; - -packages) - shift - packages=( ${1//,/ } ) - ;; - *) - gradle_args="$gradle_args $A" - ;; - esac - shift -done - -if [ -z "${install_target}" ] ; then - echo "${script_name}: missing install target" - exit 1 -fi - -declare -a package_targets -for P in "${packages[@]}" ; do - ptarget="$P:${install_target}" - if [ "${ptarget:0:1}" != ":" ] ; then - ptarget=":$ptarget" - fi - package_targets+=( "$ptarget" ) -done - -if [ "${#packages[@]}" -gt 0 ] ; then - install_target="${package_targets[@]}" -fi - -if [ -z "$noclean" ] ; then - $GRADLE clean -fi -$GRADLE -PpublishToMaven -Porg.gradle.parallel=false -Porg.gradle.daemon=false -xtest $gradle_args $install_target 2>&1 | tee /tmp/${script_name}.out diff --git a/maven-sonatype/upload-to-sonatype.sh b/maven-sonatype/upload-to-sonatype.sh deleted file mode 100755 index 7f684c5..0000000 --- a/maven-sonatype/upload-to-sonatype.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -PROG=$(basename $0) - -function usage() { - echo -e "USAGE: $PROG" -} - -# main() - -if [ "$#" -eq 0 ] ; then - usage - exit -fi - -while [ "$#" -gt 0 ] ; do - A="$1" - case "$A" in - -h|--help) - usage - exit - ;; - *) - echo "$PROG: unknown option: $A" - exit 1 - ;; - esac - shift -done - -echo "CLEANING" -./gradlew clean - -upload_all=0 - -for P in :gobblin-admin :gobblin-api :gobblin-azkaban :gobblin-compaction :gobblin-config-management:gobblin-config-core :gobblin-config-management:gobblin-config-client :gobblin-core :gobblin-data-management :gobblin-distribution :gobblin-example :gobblin-metastore :gobblin-metrics :gobblin-oozie :gobblin-rest-service:gobblin-rest-api :gobblin-rest-service:gobblin-rest-client :gobblin-rest-service:gobblin-rest-server :gobblin-iceberg :gobblin-runtime :gobblin-salesforce :gobblin-scheduler :gobblin-test :gobblin-test-harness :gobblin-utility :gobblin-yarn ; do - echo "----------------------------" - if [ "$upload_all" -eq 0 ] ; then - read -p "UPLOAD $P [y(es)|n(o)|a(ll)]" ans - ans=$(echo "$ans" | tr '[:upper:]' '[:lower:]') - if [ "$ans" == "a" -o "$ans" == "all" ] ; then - upload_all=1 - ans="y" - fi - else - ans="y" - fi - if [ "$ans" == "y" -o "$ans" == "yes" ] ; then - ./maven-sonatype/maven-sonatype.sh -remote -noclean -packages $P - else - echo "Skipping $P" - fi - done - diff --git a/query_github_issues.py b/query_github_issues.py deleted file mode 100755 index 4d10b7f..0000000 --- a/query_github_issues.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/env /usr/bin/python2.6 -# pylint: disable=missing-docstring -# pylint: disable=unused-import -from datetime import date, timedelta -import json -import httplib -from string import Template -import sys -import urllib - -def search_issues(query_terms, sortBy=None, orderDir=None): - # pylint: disable=line-too-long - # pylint: disable=invalid-name - conn = httplib.HTTPSConnection("api.github.com") - params_map = {"q": "+".join(query_terms)} - if sortBy is not None: - params_map["sort"] = sortBy - if orderDir is not None: - params_map["order"] = orderDir - # Note we don't do urlencode because the output is not compatible with the q syntax, e.g. : and / should not be escaped - # params = urllib.urlencode(params_map) - params = "&".join(str(i[0]) + "=" + str(i[1]) for i in params_map.items()) - #print params - - headers = {"User-Agent": "Python App"} - - conn.request("GET", "/search/issues?" + params, headers=headers) - response = conn.getresponse() - if response.status != httplib.OK: - sys.stderr.write("Query error: %s %s: %s" % (response.status, response.reason, response.read())) - sys.exit(1) - result = json.loads(response.read()) - conn.close() - return result - - -def get_created_issues_since(day): - # pylint: disable=line-too-long - return search_issues(query_terms=["repo:linkedin/gobblin", "is:open", "is:issue", "created:>=" + day], - sortBy="created", - orderDir="desc" - ) - -def get_created_issues_last_days(n=10): - # pylint: disable=invalid-name - since_day = (date.today() - timedelta(days=n)).strftime("%Y-%m-%d") - return get_created_issues_since(since_day) - -def simple_issue_list(issues): - # pylint: disable=invalid-name - HEADER_TEMPLATE = Template("$total_count issues found") - ISSUE_TEMPLATE = Template("""------------------ -ISSUE $number : HTML: $html_url JSON: $url -\tCREATED ON: $created_at -\tCREATED BY: $user_login ( $user_name ) $user_html_url -\tASSIGNED TO: $assignee_login ($assignee_name) $assignee_html_url -\tCOMMENTS: $comments -\tUPDATED ON: $updated_at -\tCLOSED ON: $closed_at - -$body -""") - print HEADER_TEMPLATE.substitute(issues) - for issue in issues["items"]: - user_data = issue["user"] - for user_attr in user_data: - issue["user_" + user_attr] = user_data[user_attr] - assignee_data = issue["assignee"] - for assignee_attr in assignee_data: - issue["assignee_" + assignee_attr] = assignee_data[assignee_attr] - print ISSUE_TEMPLATE.safe_substitute(issue) - -def main(argv): - # pylint: disable=unused-argument - issues = get_created_issues_last_days(7) - print simple_issue_list(issues) - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/settings.gradle b/settings.gradle index 8b02f26..b026e9e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,26 +2,4 @@ // Licensed under the BSD-2 Clause license. // See LICENSE in the project root for license information. -def modules = ['dil'] - -// Disable jacoco for now as Kafka 0.8 is the default version and jacoco does not like the same classes -// being declared in different modules -def jacocoBlacklist = new HashSet([ -]) - -modules.each { module -> - include "${module}" - file(module).eachDir { submodule -> - if (!submodule.name.startsWith('.') && !submodule.name.equals('src') && !submodule.name.equals('bin') - && !submodule.name.equals('test-output') && !submodule.name.equals('jobconf')) { - def submoduleId = "${module}:${submodule.name}" - if (System.getProperty('jacocoBuild') == null || !jacocoBlacklist.contains(submoduleId.toString())) { - include submoduleId - } - else { - println "Ignoring blacklisted module ${submoduleId}" - } - } - } -} - +include 'cdi-core' diff --git a/travis/bintrayDeploy.sh b/travis/bintrayDeploy.sh deleted file mode 100755 index 5b47c01..0000000 --- a/travis/bintrayDeploy.sh +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Test script used by Travis to test the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -echo "Starting $0 at " $(date) -PROJECT_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}') - -echo "Project Version: $PROJECT_VERSION" -BUILD_VERSION=$PROJECT_VERSION-dev-${TRAVIS_BUILD_NUMBER} -echo "Build Version: $BUILD_VERSION" - -echo "Pull request: [$TRAVIS_PULL_REQUEST], Travis branch: [$TRAVIS_BRANCH]" -# release only from master when no pull request build -if [ "$TRAVIS_PULL_REQUEST" = "false" ] -then - echo "Uploading artifacts to bintray for version $BUILD_VERSION" - ./gradlew bintrayUpload -Pversion=$BUILD_VERSION -Pbintray.override -fi \ No newline at end of file diff --git a/travis/filter-to-failing-test-results.py b/travis/filter-to-failing-test-results.py deleted file mode 100644 index 8018600..0000000 --- a/travis/filter-to-failing-test-results.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Python script used to filter the list of junit XML test results -# to those that contain errors or failures. -# - -#!/usr/bin/python - -import sys -import fileinput -import xml.etree.ElementTree - -for line in fileinput.input(): - suite = xml.etree.ElementTree.parse(line.rstrip()).getroot() - errors = suite.get("errors") - failures = suite.get("failures") - if (errors is not None and int(errors) > 0) or (failures is not None and int(failures) > 0): - sys.stdout.write(line) - -sys.exit(0) \ No newline at end of file diff --git a/travis/junit-errors-to-stdout.sh b/travis/junit-errors-to-stdout.sh deleted file mode 100755 index bb2cdaf..0000000 --- a/travis/junit-errors-to-stdout.sh +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Script used by Travis builds upon a failure to format and -# print the failing test results to the console. -# - -#!/bin/bash -IFS=' -' -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -ROOTDIR="$1" -if [ -z "$ROOTDIR" ]; then - ROOTDIR="." -fi -echo 'Formatting results...' -FILES=$(find "$ROOTDIR" -path '*/build/*/test-results/*.xml' | python "$DIR/filter-to-failing-test-results.py") -if [ -n "$FILES" ]; then - for file in $FILES; do - echo "Formatting $file" - if [ -f "$file" ]; then - echo '=====================================================' - xsltproc "$DIR/junit-xml-format-errors.xsl" "$file" - fi - done - echo '=====================================================' -else - echo 'No */build/*/test-results/*.xml files found with failing tests.' -fi diff --git a/travis/junit-xml-format-errors.xsl b/travis/junit-xml-format-errors.xsl deleted file mode 100755 index c4b0895..0000000 --- a/travis/junit-xml-format-errors.xsl +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - Testsuite: - -Tests run: - - , Failures: - - , Errors: - - , Time elapsed: - - sec - ---------- ----------- --------- - - - - - - - -Testcase: - - took - - - FAILURE - - ERROR - - SUCCESS - - - - - - - - - - - - - - - ------- Standard output ------ - - - - - - ------- Error output ------ - - - - - \ No newline at end of file diff --git a/travis/test-build.sh b/travis/test-build.sh deleted file mode 100755 index f2aea4c..0000000 --- a/travis/test-build.sh +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Build script used by Travis to clean and assemble the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -echo "Starting $0 at " $(date) -time ./gradlew clean build -x test -x javadoc -Dorg.gradle.parallel=true $GOBBLIN_GRADLE_OPTS diff --git a/travis/test-coverage.sh b/travis/test-coverage.sh deleted file mode 100755 index 8e87989..0000000 --- a/travis/test-coverage.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Test script used by Travis to test the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -script_dir=$(dirname $0) - -source ${script_dir}/test-groups.inc - -echo "Starting $0 at " $(date) -time ./gradlew -PskipTestGroup=disabledOnCI -Dorg.gradle.parallel=false -DjacocoBuild=1 $GOBBLIN_GRADLE_OPTS jacocoTestCoverage diff --git a/travis/test-default.sh b/travis/test-default.sh deleted file mode 100755 index 3d91ff0..0000000 --- a/travis/test-default.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Test script used by Travis to test the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -script_dir=$(dirname $0) - -source ${script_dir}/test-groups.inc - -echo "Starting $0 at " $(date) -echo "GOBBLIN_GRADLE_OPTS=$GOBBLIN_GRADLE_OPTS" -time ./gradlew -PskipTestGroup=disabledOnCI,$TEST_GROUP1 -Dorg.gradle.parallel=false $GOBBLIN_GRADLE_OPTS test diff --git a/travis/test-group1.sh b/travis/test-group1.sh deleted file mode 100755 index 8591e6a..0000000 --- a/travis/test-group1.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Test script used by Travis to test the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -script_dir=$(dirname $0) - -source ${script_dir}/test-groups.inc - -echo "Starting $0 at " $(date) -echo "Precompiling tests:" -rm -rf $HOME/.gradle/caches/ -./gradlew compileTest -Porg.gradle.parallel=false $GOBBLIN_GRADLE_OPTS -echo "Running tests for $TEST_GROUP1" -time ./gradlew -PskipTestGroup=disabledOnCI -PrunTestGroups=$TEST_GROUP1 -Dorg.gradle.parallel=false $GOBBLIN_GRADLE_OPTS test \ No newline at end of file diff --git a/travis/test-groups.inc b/travis/test-groups.inc deleted file mode 100644 index 8eef534..0000000 --- a/travis/test-groups.inc +++ /dev/null @@ -1 +0,0 @@ -TEST_GROUP1=gobbin.yarn,gobblin.runtime,gobblin.cluster,gobblin.compaction,gobblin.util,gobblin.writer diff --git a/travis/test.sh b/travis/test.sh deleted file mode 100755 index fd557d0..0000000 --- a/travis/test.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 LinkedIn Corporation. All rights reserved. -# Licensed under the BSD-2 Clause license. -# See LICENSE in the project root for license information. - -# -# Test script used by Travis to test the -# hadoop1 or hadoop2 versions of gobblin. -# - -#!/bin/bash -set -e - -#free - -RUN_TEST_GROUP=${RUN_TEST_GROUP:-default} - -script_dir=$(dirname $0) -echo "Old GRADLE_OPTS=$GRADLE_OPTS" - -export java_version=$(java -version 2>&1 | grep 'openjdk version' | sed -e 's/openjdk version "\(1\..\).*/\1/') - -echo "Using Java version:${java_version}" - -export GOBBLIN_GRADLE_OPTS="-Dorg.gradle.daemon=false -Dgobblin.metastore.testing.embeddedMysqlEnabled=false -PusePreinstalledMysql=true -PjdkVersion=${java_version}" - -TEST_SCRIPT=${script_dir}/test-${RUN_TEST_GROUP}.sh -if [ -x $TEST_SCRIPT ] ; then - echo "Running test group $RUN_TEST_GROUP" - $TEST_SCRIPT "$@" -else - echo "Test file $TEST_SCRIPT does not exist or is not executable!" - exit 1 -fi diff --git a/version.properties b/version.properties new file mode 100644 index 0000000..88480b9 --- /dev/null +++ b/version.properties @@ -0,0 +1 @@ +version=0.1.* \ No newline at end of file