From 2f15e386b7f4b2f9a70a1cbece9b23e80a818ac7 Mon Sep 17 00:00:00 2001 From: Rishabh Bohra <19497586+rishabh96b@users.noreply.github.com> Date: Tue, 28 Jan 2020 21:53:11 +0530 Subject: [PATCH] [DCOS-63346] Update to use SDK v0.57.3 (#460) * bump sdk v0.57.3 * bump mockito v2.27.0 * bump gradle v5.1.1 * Bump scheduler jre to v11 --- frameworks/kafka/build.gradle | 4 ++-- frameworks/kafka/universe/resource.json | 2 +- frameworks/kafka/versions.sh | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- test.sh | 9 ++++++++- testing/sdk_security.py | 16 ++++++++++++---- tools/publish_http.py | 2 +- tools/universe/package_builder.py | 2 ++ tools/universe/s3_uploader.py | 3 ++- 9 files changed, 30 insertions(+), 12 deletions(-) diff --git a/frameworks/kafka/build.gradle b/frameworks/kafka/build.gradle index 7d13ab0b..06c7705f 100644 --- a/frameworks/kafka/build.gradle +++ b/frameworks/kafka/build.gradle @@ -18,8 +18,8 @@ repositories { ext { junitVer = "4.12" systemRulesVer = "1.16.0" - mockitoVer = "2.10.0" - dcosSDKVer = "0.57.0" + mockitoVer = "2.27.0" + dcosSDKVer = "0.57.3" } dependencies { diff --git a/frameworks/kafka/universe/resource.json b/frameworks/kafka/universe/resource.json index a7bee0a6..6bdd9a6a 100644 --- a/frameworks/kafka/universe/resource.json +++ b/frameworks/kafka/universe/resource.json @@ -1,7 +1,7 @@ { "assets": { "uris": { - "jre-tar-gz": "{{jre-url}}", + "jre-tar-gz": "{{scheduler-jre-url}}", "libmesos-bundle-tar-gz": "{{libmesos-bundle-url}}", "bootstrap-zip": "https://downloads.mesosphere.com/dcos-commons/artifacts/{{dcos-sdk-version}}/bootstrap.zip", "kafka-tgz": "https://downloads.mesosphere.com/kafka/assets/kafka_{{kafka-version}}.tgz", diff --git a/frameworks/kafka/versions.sh b/frameworks/kafka/versions.sh index 5933dd9d..4d3e574f 100755 --- a/frameworks/kafka/versions.sh +++ b/frameworks/kafka/versions.sh @@ -13,4 +13,4 @@ # instructions at: # https://wiki.mesosphere.com/display/ENGINEERING/Uploading+an+asset+to+production export TEMPLATE_KAFKA_VERSION="2.12-2.4.0" -export TEMPLATE_DCOS_SDK_VERSION="0.57.0" \ No newline at end of file +export TEMPLATE_DCOS_SDK_VERSION="0.57.3" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 115e6ac0..558870da 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/test.sh b/test.sh index 6383ed00..8c315511 100755 --- a/test.sh +++ b/test.sh @@ -92,6 +92,7 @@ headless="false" dind="false" interactive="false" package_registry="false" +disable_diag_collect="false" docker_options="${DOCKER_OPTIONS:=}" docker_command="${DOCKER_COMMAND:=bash ${WORK_DIR}/tools/ci/test_runner.sh ${WORK_DIR}}" docker_image="${DOCKER_IMAGE:-mesosphere/dcos-commons:latest}" @@ -157,6 +158,8 @@ function usage() echo " The AWS profile to use. Only required when using an AWS credentials file with multiple profiles." echo " --dind ${dind}" echo " launch docker daemon inside the container" + echo " --disable-diag-collect ${disable_diag_collect}" + echo " Disable collection of diagnostic logs after a failed test." echo echo "---" echo @@ -254,6 +257,9 @@ while [[ ${#} -gt 0 ]]; do --dind) dind="true" ;; + --disable-diag-collect) + disable_diag_collect="true" + ;; --dcos-files-path) if [[ ! -d "${2}" ]]; then echo "Directory not found: ${arg} ${2}"; exit 1; fi # Resolve abs path: @@ -515,6 +521,7 @@ cat >> "${env_file}" <<-EOF S3_BUCKET=${S3_BUCKET} SECURITY=${security} STUB_UNIVERSE_URL=${STUB_UNIVERSE_URL} + DISABLE_DIAG=${disable_diag_collect} EOF while read -r line; do @@ -540,7 +547,7 @@ fi ######################### Prepare and run command ############################## ################################################################################ -docker pull "${docker_image}" +[ ! -z $(docker images -q "${docker_image}") ] || docker pull "${docker_image}" set +x diff --git a/testing/sdk_security.py b/testing/sdk_security.py index 45dcdf64..841315c5 100644 --- a/testing/sdk_security.py +++ b/testing/sdk_security.py @@ -195,11 +195,15 @@ def create_service_account(service_account_name: str, service_account_secret: st delete_service_account(service_account_name, service_account_secret) log.info("Create keypair") - sdk_cmd.run_cli("security org service-accounts keypair private-key.pem public-key.pem") + sdk_cmd.run_cli( + "security org service-accounts keypair {account}-private-key.pem {account}-public-key.pem".format( + account=service_account_name + ) + ) log.info("Create service account") sdk_cmd.run_cli( - "security org service-accounts create -p public-key.pem " + "security org service-accounts create -p {account}-public-key.pem " '-d "Service account for integration tests" "{account}"'.format( account=service_account_name ) @@ -207,7 +211,7 @@ def create_service_account(service_account_name: str, service_account_secret: st log.info("Create secret") sdk_cmd.run_cli( - 'security secrets create-sa-secret --strict private-key.pem "{account}" "{secret}"'.format( + 'security secrets create-sa-secret --strict {account}-private-key.pem "{account}" "{secret}"'.format( account=service_account_name, secret=service_account_secret ) ) @@ -227,7 +231,10 @@ def delete_service_account(service_account_name: str, service_account_secret: st sdk_cmd.run_cli("security org service-accounts delete {name}".format(name=service_account_name)) # Files generated by service-accounts keypair command should get removed - for keypair_file in ["private-key.pem", "public-key.pem"]: + for keypair_file in [ + "{account}-private-key.pem".format(account=service_account_name), + "{account}-public-key.pem".format(account=service_account_name), + ]: try: os.unlink(keypair_file) except OSError: @@ -282,6 +289,7 @@ def _get_integration_test_foldered_role(service_name: str) -> List[str]: role_basename = service_name.strip("/").replace("/", "__") return [ "test__integration__{}-role".format(role_basename), + "test", "quota", "quota__{}-role".format(role_basename), ] diff --git a/tools/publish_http.py b/tools/publish_http.py index 3d2182b9..136cff22 100755 --- a/tools/publish_http.py +++ b/tools/publish_http.py @@ -193,7 +193,7 @@ def add_repo_to_cli(self, repo_url): subprocess.check_call("dcos package repo remove {}".format(repo["name"]).split()) logger.info("Adding repository: {} {}".format(repo_name, repo_url)) subprocess.check_call( - "dcos package repo add --index=0 {} '{}'".format(repo_name, repo_url).split(" ") + "dcos package repo add --index=0 {} {}".format(repo_name, repo_url).split(" ") ) return True diff --git a/tools/universe/package_builder.py b/tools/universe/package_builder.py index d61c4780..6bdcc30f 100755 --- a/tools/universe/package_builder.py +++ b/tools/universe/package_builder.py @@ -275,6 +275,8 @@ def _generate_packages_dict(self, package_files): package_files[_package_json_filename], object_pairs_hook=collections.OrderedDict ) package_json["releaseVersion"] = 0 + # `lastUpdated` is supported for all packages v3 and above + package_json["lastUpdated"] = round(time.time()) config_json = package_files.get(_config_json_filename) if config_json is not None: diff --git a/tools/universe/s3_uploader.py b/tools/universe/s3_uploader.py index aa98f700..86a22723 100644 --- a/tools/universe/s3_uploader.py +++ b/tools/universe/s3_uploader.py @@ -20,6 +20,7 @@ def __init__(self, s3_directory, dry_run=False): self._aws_region = os.environ.get("AWS_UPLOAD_REGION", "") self._reauth_attempted = False self._dry_run = dry_run + self._acl = os.environ.get("AWS_UPLOAD_ACL", "public-read") def get_s3_directory(self): return self._s3_directory @@ -29,7 +30,7 @@ def upload(self, filepath, content_type=None): cmdlist = ["aws s3"] if self._aws_region: cmdlist.append("--region={}".format(self._aws_region)) - cmdlist.append("cp --acl public-read") + cmdlist.append("cp --acl {}".format(self._acl)) if self._dry_run: cmdlist.append("--dryrun") if content_type is not None: