Skip to content

Commit

Permalink
Merge code, add docs, and redesigned properties with more validation …
Browse files Browse the repository at this point in the history
…capabilities
  • Loading branch information
Chris Li committed Oct 29, 2021
1 parent 72beea1 commit 5e61863
Show file tree
Hide file tree
Showing 248 changed files with 11,402 additions and 5,392 deletions.
48 changes: 42 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,53 @@ LinkedIn Data Integration Library (DIL) is a collection of generic data integrat
- Ingest data from one Rest API and egress to another (Rest API) on cloud

# Requirements
* Java >= 1.8
* JDK 1.8

If building the distribution with tests turned on:
* Maven version 3.5.3

# Instructions to build the distribution
1. Extract the archive file to your local directory.
2. Set JAVA_HOME
2. Set JAVA_HOME to use JDK 1.8 (JDK 11+ not supported)
3. Build
* Skip tests and build the distribution
> `./gradlew build -x findbugsMain -x test -x rat -x checkstyleMain`
* Tests and build the distribution (requires Maven):
> `./gradlew build`
# Instructions to contribute
To contribute, please use submit Pull Request (PR) for committers to merge.
1. Create your own fork on GitHub off the main repository
2. Clone your fork to your local computer
- `git clone https://github.com/<<your-github-login>>/data-integration-library.git`
3. Add upstream and verify
- `git remote add upstream https://github.com/linkedin/data-integration-library.git`
- `git remote -v`
4. Change, test, commit, and push to your fork
- `git status`
- `git add .`
- `git commit -m "comments"`
- `git push origin master`
5. Create Pull Request on GitHub with the following details
- Title
- Detailed description
- Document the tests done
- Links to the updated documents
6. Publish to local Maven repository
- `./gradlew publishToMavenLocal`
7. Refresh your fork
- if upstream has no conflict with your fork, you can go to your forked
repository, and use "Fetch upstream" function to sync up your fork.
- if upstream has conflicts with your fork, GitHub will ask you to create
a pull request to merge.
- if the conflicts are too significant, it is better to just copy
everything from upstream (the main repository) to your fork; that can
be done with the following procedure:
- Follow step 2 and step 3 above
- `git fetch upstream`
- `git reset --hard upstream/master`
- `git push origin +master`
- check your fork should be in sync with the main repository

# Detailed Documents

- [Job Properties](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/summary.md)
- [Job Properties by Category](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/categories.md)
- [Deprecated Job Properties](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/deprecated.md)
5 changes: 0 additions & 5 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ allprojects {

subprojects {
dependencies {
// Gradle 5 compatibility
compileOnly externalDependency.lombok
testCompileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok
testAnnotationProcessor externalDependency.lombok
}
project.buildDir = new File(project.rootProject.buildDir, project.name)
}
Expand Down

This file was deleted.

This file was deleted.

2 changes: 0 additions & 2 deletions cdi-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,8 @@ dependencies {
compile externalDependency.'awsUtils'
compile externalDependency.'commonsValidator'
compile externalDependency.'guava'
compile externalDependency.'lombok'
compile externalDependency.'commonsLang3'
compile externalDependency.'testng'
compile externalDependency.'okhttp'
compile externalDependency.'jhyde'

runtime externalDependency.'gobblin-azkaban'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2021 LinkedIn Corporation. All rights reserved.
// Licensed under the BSD-2 Clause license.
// See LICENSE in the project root for license information.

package com.linkedin.cdi.configuration;

import org.apache.commons.lang3.StringUtils;
import org.apache.gobblin.configuration.State;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
* A Boolean type of property has no default defaultValue, and each property
* has to supply a default value, true or false
*/
public class BooleanProperties extends MultistageProperties<Boolean> {
private static final Logger LOG = LoggerFactory.getLogger(BooleanProperties.class);

/**
* Constructor with explicit default value
* @param config property name
* @param defaultValue default value
*/
BooleanProperties(String config, Boolean defaultValue) {
super(config, Boolean.class, defaultValue);
}

/**
* Validates the value when it is blank
* - No configuration is considered blank
* - A blank string is considered blank
*
* @param state state
* @return true if blank
*/
@Override
public boolean isBlank(State state) {
return !state.contains(getConfig())
|| StringUtils.isBlank(state.getProp(getConfig()));
}

/**
* Validates the value when it is non-blank and accepts blank value
* - A blank configuration is considered valid
* - Any properly formed Boolean is considered valid
* @param state state
* @return true if blank or non-blank and valid
*/
@Override
public boolean isValid(State state) {
if (!isBlank(state)) try {
String value = state.getProp(getConfig());
if (!value.toLowerCase().matches("true|false")) {
LOG.error(errorMessage(state));
return false;
}
// Properly formed Boolean string is valid
Boolean.parseBoolean(state.getProp(getConfig()));
} catch (Exception e) {
LOG.error(errorMessage(state), e.getMessage());
return false;
}
return true;
}

/**
* Validates the value when it is non-blank and rejects blank value
* - only properly formed Boolean string is considered valid
*
* @param state source state
* @return true when the configuration is non-blank and valid
*/
public boolean isValidNonblank(State state) {
return !isBlank(state) && isValid(state);
}

/**
* Retrieves property value from state object if valid and not blank
* otherwise, return default value
*
* @param state state
* @return property value if non-blank and valid, otherwise the default value
*/
protected Boolean getValidNonblankWithDefault(State state) {
if (isValidNonblank(state)) {
return Boolean.parseBoolean(state.getProp(getConfig()));
}
return getDefaultValue();
}
}
Loading

0 comments on commit 5e61863

Please sign in to comment.