initial repository

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+Dear Gobblin maintainers,
+
+Please accept this PR. I understand that it will not be reviewed until I have checked off all the steps below!
+
+
+### JIRA
+- [ ] My PR addresses the following [Gobblin JIRA](https://issues.apache.org/jira/browse/GOBBLIN/) issues and references them in the PR title. For example, "[GOBBLIN-XXX] My Gobblin PR"
+    - https://issues.apache.org/jira/browse/GOBBLIN-XXX
+
+
+### Description
+- [ ] Here are some details about my PR, including screenshots (if applicable):
+
+
+### Tests
+- [ ] My PR adds the following unit tests __OR__ does not need testing for this extremely good reason:
+
+
+### Commits
+- [ ] My commits all reference JIRA issues in their subject lines, and I have squashed multiple commits if they address the same issue. In addition, my commits follow the guidelines from "[How to write a good git commit message](http://chris.beams.io/posts/git-commit/)":
+    1. Subject is separated from body by a blank line
+    2. Subject is limited to 50 characters
+    3. Subject does not end with a period
+    4. Subject uses the imperative mood ("add", not "adding")
+    5. Body wraps at 72 characters
+    6. Body explains "what" and "why", not "how"
+

.github/workflows/build_and_test.yaml

@@ -0,0 +1,151 @@
+# Copyright 2021 LinkedIn Corporation. All rights reserved.
+# Licensed under the BSD-2 Clause license.
+# See LICENSE in the project root for license information.
+
+name: Build and Run Tests
+
+on:
+  push:
+    # Publish only on `master`
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  release:
+    types: [published, edited]
+
+jobs:
+  build:
+    name: Build repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      # Stores external dependencies, can be further improved with Gradle 6.1
+      - name: Cache Gradle Dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          # Only rebuild cache if build.gradle is changed
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Build repository
+        run: |
+          ./gradlew --no-daemon clean build -x test -x javadoc -x findbugsMain -x findbugsTest -x checkstyleMain -x checkstyleJmh -x checkstyleTest -Dorg.gradle.parallel=true
+
+  test_coverage:
+    runs-on: ubuntu-latest
+    name: Generate test coverage
+    needs: build
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Cache Gradle Dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          # Only rebuild cache if build.gradle is changed
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Generate code coverage
+        run: |
+          ./gradlew -PskipTestGroup=disabledOnCI -Dorg.gradle.parallel=false -DjacocoBuild=1 $GOBBLIN_GRADLE_OPTS jacocoTestCoverage
+
+  static_checks:
+    name: Run static checks
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      # Stores external dependencies, can be further improved with Gradle 6.1
+      - name: Cache Gradle Dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          # Only rebuild cache if build.gradle is changed
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Run CheckStyle and FindBugs
+        run: |
+          ./gradlew --no-daemon -x javadoc findbugsMain checkstyleMain checkstyleTest checkstyleJmh
+
+  run_tests:
+    timeout-minutes: 60
+    env:
+      GOBBLIN_GRADLE_OPTS: "--no-daemon -Dgobblin.metastore.testing.embeddedMysqlEnabled=false -PusePreinstalledMysql=true"
+    strategy:
+      matrix:
+        test-group: ["Core Tests", "Service Tests", "Module Tests", "Other Tests"]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    needs: build
+    services:
+      mysql:
+        image: mysql:5.7.32
+        env:
+          MYSQL_USER: testUser
+          MYSQL_PASSWORD: testPassword
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: password
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Verify mysql connection
+        run: |
+            sudo apt-get install -y mysql-client
+            mysql --host 127.0.0.1 --port 3306 -uroot -ppassword -e "SHOW DATABASES"
+      - name: Cache Gradle Dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          # Only rebuild cache if build.gradle is changed
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Run test group ${{ matrix.test-group }}
+        # Write retry logic as integration tests can fail due to timing out or network problems
+        run: |
+          ./gradlew getGroupedTests -PgroupName="${{matrix.test-group}}" > temp.txt
+          TASKS=$(sed -n 's/CI Task: //p' temp.txt)
+          echo $TASKS
+
+          n=0
+          until [ "$n" -ge 3 ]
+          do
+            ./gradlew -PskipTestGroup=disabledOnCI $GOBBLIN_GRADLE_OPTS $TASKS -Dorg.gradle.parallel=false && break
+            n=$((n+1))
+            if [[ $n -lt 3 ]]; then
+              echo "Tests failed, retry attempt number $n"
+            else
+              exit 1
+            fi
+            sleep 10
+          done

.github/workflows/docker_build_publish.yaml

@@ -0,0 +1,61 @@
+# Copyright 2021 LinkedIn Corporation. All rights reserved.
+# Licensed under the BSD-2 Clause license.
+# See LICENSE in the project root for license information.
+
+name: Build and Publish Docker image
+on:
+  push:
+    # Publish only on `master`
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'gobblin-docker/**'
+      - '.github/workflows/docker_build_publish.yaml'
+  release:
+    types: [published, edited]
+
+env:
+  IMAGE_NAME: apache/gobblin
+
+jobs:
+  build_and_push_to_registry:
+    name: Build docker images and publish to DockerHub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Build Docker Tag
+        id: build_tag
+        run: |
+          SHA=`echo ${{ github.sha }} | head -c 7`
+          if [[ ${{ github.event_name }} == 'release' ]]; then
+            TAG="${{ env.IMAGE_NAME }}:sha-$SHA, ${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}, ${{ env.IMAGE_NAME }}:latest"
+          else
+            TAG="${{ env.IMAGE_NAME }}:sha-$SHA"
+          fi
+          echo "tag=$TAG"
+          echo "::set-output name=tag::$TAG"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      #      - name: Login to GitHub Container Registry
+      #        if: github.event_name != 'pull_request'
+      #        uses: docker/login-action@v1
+      #        with:
+      #          registry: ghcr.io
+      #          username: ${{ github.repository_owner }}
+      #          password: ${{ secrets.CR_PAT }}
+      - name: Build Images and Publish
+        uses: docker/build-push-action@v2
+        with:
+          tags: ${{ steps.build_tag.outputs.tag }}
+          push: ${{ github.event_name == 'release' }}
+          file: ./gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile

.gitignore

@@ -0,0 +1,59 @@
+.classpath*
+.project*
+.settings
+.DS_Store
+*.tar.gz
+
+# Intellij related
+**/.idea
+**/*.iml
+**/*.iws
+**/*.ipr
+.shelf/
+.ideaDataSources/
+
+# VS Code related
+.vscode
+
+**/.classpath
+**/.project
+**/.settings
+
+**/*.swp
+**/*.swo
+**/*.log
+
+**/build/
+.gradle
+**/.gradle
+gradle.properties.release
+test-output
+**/test-output
+dist
+target
+tmp
+out
+**/out
+output
+/eclipse_build
+.project
+.classpath
+out/
+*/bin/
+**/mainGeneratedDataTemplate
+**/mainGeneratedRest
+**/main/snapshot
+**.tar.gz
+*~
+metastore_db/
+
+# generated nodeJs files
+node_modules/
+package-lock.json
+
+*.out
+
+# generated java files
+**/gen-java/
+
+temp/

.travis.yml

@@ -0,0 +1,65 @@
+language: java
+
+dist: bionic
+sudo: required
+
+jdk:
+  - openjdk8
+
+addons:
+  apt:
+    packages:
+      - libaio-dev
+      - libdbus-glib-1-dev
+      - xsltproc
+
+before_cache:
+  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
+
+cache:
+  directories:
+    - $HOME/.gradle/caches/
+    - $HOME/.gradle/wrapper/
+
+before_install:
+
+services:
+  - xvfb
+  - mysql
+
+stages:
+  - test
+  - name: deploy
+    if: branch = master
+
+before_script:
+  - mysql -uroot -e "create user testUser identified by 'testPassword';"
+  - mysql -uroot -e "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('password')"
+
+script:
+  - travis_retry ./travis/test.sh
+  - travis_retry ./gradlew jacocoTestReport
+after_success:
+  - bash <(cat .codecov_bash)
+after_failure: ./travis/junit-errors-to-stdout.sh
+
+jobs:
+  include:
+    - stage: deploy
+      env: RUN_TEST_GROUP=none
+      install: skip
+      script:
+        - travis_retry ./travis/bintrayDeploy.sh
+
+env:
+  jobs:
+    - RUN_TEST_GROUP=build
+    - RUN_TEST_GROUP=default
+    - RUN_TEST_GROUP=group1
+    - RUN_TEST_GROUP=coverage
+  global:
+    - secure: U72nmzXq7kXcIabiwvrTF+WkNFQxov2ACd8oPxWSHsyzRsfVJN42vT7gS3dLbH5G5claXG2p+rur4ueVffzYSwJ8B9OP6gTB8sNJnGr9zyxni4OJchyKqOYuj+UBpEQC/7qXKMCKnuJndsf1OvndDh/V1SH0DSSUuA6mDtgO/eM=
+    - secure: WiK7tyFV68xdkIfUlWreUHgEGGjaCBd73O4SbjE9AsbqqF7D+Iu8iRo1OhKQj+6eajUH9Eoev9rVN74FQgUfeNzrOkYsgDysXmyZ7+UxFokijFcATJmIBompA3dySGU2qXeKbJMNuUjXgrRIludaV6h2ahL6Fji42cgK4I3s2qs=
+
+jdk:
+  - openjdk8

CHANGELOG.md

@@ -0,0 +1,11 @@
+DIL 0.0.1
+--------------
+
+###Created Date: 4/10/2021
+
+## HIGHLIGHTS
+* Initial publication of DIL to open source
+
+## NEW FEATURES
+
+##IMPROVEMENTS