diff --git a/ci/envs/ci-vsphere-baseline.yml b/ci/envs/ci-vsphere-baseline.yml index 3e61819..854bba2 100644 --- a/ci/envs/ci-vsphere-baseline.yml +++ b/ci/envs/ci-vsphere-baseline.yml @@ -6,7 +6,7 @@ kit: - metrics-emitter ci: job: deploy - spec_tests: false + serial_groups: [deployment] genesis: env: ci-vsphere-baseline diff --git a/ci/pipeline/base.yml b/ci/pipeline/base.yml index d82d33e..2c58233 100644 --- a/ci/pipeline/base.yml +++ b/ci/pipeline/base.yml @@ -82,7 +82,6 @@ meta: VAULT_TOKEN: (( grab meta.vault.token )) INFRASTRUCTURE: (( grab meta.iaas )) - groups: - name: (( grab meta.pipeline )) jobs: diff --git a/ci/pipeline/custom-jobs/.keep b/ci/pipeline/custom-jobs/.keep new file mode 100644 index 0000000..e69de29 diff --git a/ci/pipeline/custom-resources/.keep b/ci/pipeline/custom-resources/.keep new file mode 100644 index 0000000..e69de29 diff --git a/ci/scripts/build-upstream-jobs b/ci/scripts/build-upstream-jobs index 9590c4c..68db417 100755 --- a/ci/scripts/build-upstream-jobs +++ b/ci/scripts/build-upstream-jobs @@ -7,10 +7,11 @@ _lookup() { echo "${1}" | base64 --decode | jq -re "${2}" } -# Read upstream.yml +# Get upstream details from settings.yml update_group=() -upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstream.bosh_releases//[] | .[] | @base64')" -upstream_manifest="$(spruce merge "$base_dir/pipeline/base.yml" "$base_dir/settings.yml" | spruce json | jq -r '.meta.upstream.manifest//{} | @base64')" +upstream_details="$(spruce merge "$base_dir/pipeline/base.yml" "$base_dir/settings.yml" | spruce json | jq -r '.meta.upstream//{}')" +upstream_releases="$(jq -r '.bosh_releases//[] | .[] | @base64' <<< "$upstream_details")" +upstream_manifest="$(jq -r '.manifest//{} | @base64' <<< "$upstream_details")" # Support upstream manifests package="$(_lookup "$upstream_manifest" '.package')" || true @@ -60,11 +61,17 @@ EOF cp "$base_dir/pipeline/optional-resources/upstream-manifest.yml" "$base_dir/pipeline/upstream/upstream-manifest-resource.yml" fi -# For each release in upstream.yml, -for release in $upstream_details ; do - name="$(_lookup "$release" .name)" +# For each upstream release, create a job to update the release +for release in $upstream_releases ; do + name="$(_lookup "$release" '.name')" + watch="$(_lookup "$release" '.watch')" || true + [[ "$watch" == "false" ]] && continue type="$(_lookup "$release" '.type//"bosh-io-release"')" - path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')" + path="$(_lookup "$release" '.path//""')" + if [[ -z "$path" ]] ; then + default_path="$(jq -r '.default_releases_path//"manifests/releases"' <<<"$upstream_details")" + path="$default_path/$name.yml" + fi repo="$(_lookup "$release" '.repository')" if [[ $type == 'bosh-io-release' ]] ; then source=$'\n'" repository: $repo"; diff --git a/ci/scripts/compare-release-specs b/ci/scripts/compare-release-specs index b897651..c6a94fa 100755 --- a/ci/scripts/compare-release-specs +++ b/ci/scripts/compare-release-specs @@ -105,18 +105,24 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then echo echo "Fetching spec diffs..." - if [ -f "${ci_dir}/ci/upstreamrepo.yml" ]; then - upstreamrepo=$(spruce json "${ci_dir}/ci/upstreamrepo.yml") - else - upstreamrepo='{"repos": []}' - fi + upstream="$( + spruce merge --skip-eval "${ci_dir}/ci/pipeline/base.yml" "${ci_dir}/ci/settings.yml" | + spruce json | + jq -r '.meta.upstream.bosh_releases[] | + if .spec_url then {name, "repo":.spec_url} + elif .owner then {name, "repo":"https://github.com/\(.owner)/\(.repository)"} + else {name, "repo":"https://github.com/\(.repository)"} + end' | + jq -s + )" + # TODO: do this in two phases -- first phase pull out all the non-compiled # versions, then run through with the compiled versions, picking up the # non-compiled version's git repo. Also indicate if they are compiled # or not, and if so, what os is the target. (because that may change) repos="$( echo "$curr_releases" \ - | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []}; + | jq --argjson gitrepos "$upstream" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []}; ($url | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) @@ -160,7 +166,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then branch="master" fi - git -C "$rel_dir" pull --depth 1 origin ${branch} > /dev/null 2>&1 + git -C "$rel_dir" pull --depth 1 origin "${branch}" > /dev/null 2>&1 mkdir -p "$workdir/compare-specs" diff --git a/ci/scripts/get-latest-upstream-manifest b/ci/scripts/get-latest-upstream-manifest index 50b59ba..e8416b4 100755 --- a/ci/scripts/get-latest-upstream-manifest +++ b/ci/scripts/get-latest-upstream-manifest @@ -86,9 +86,8 @@ git -C git diff --color=always "$UPSTREAM_PATH" | cat header "Recreate spec-test results to validate upstream" pushd git/spec > /dev/null - # TODO: remove spec/{credhub,vault} if needed? because this will regenerate vault. - rm -rf results/ - ACK_GINKGO_RC=true ginkgo -p . + rm -rf results/ credhub/ + ACK_GINKGO_RC=true ginkgo -p -failFast . popd header "Spec file changes:" diff --git a/ci/scripts/release-notes b/ci/scripts/release-notes index 64fc678..cdb8768 100755 --- a/ci/scripts/release-notes +++ b/ci/scripts/release-notes @@ -55,6 +55,14 @@ sub get_new_commits { return @commits; } +sub get_upstream_settings { + my ($path) = @_; + $path //= $FindBin::Bin."/../"; + my $json = qx(spruce merge --skip-eval $path/pipeline/base.yml $path/settings.yml | spruce json); + my $settings = decode_json($json); + return $settings->{meta}{upstream} || {}; +} + sub get_latest_commit_ref { my ($path) = @_; chomp(my $ref = qx(git -C $path rev-parse HEAD)); @@ -131,6 +139,7 @@ sub parse_commits { } sub get_releases { + # FIXME: This should also check manifest/releases/*.yml or potentially other locations my ($path) = @_; my $filter = "$path/spec/results/*.yml"; my @entries = qx(for x in $filter ; do spruce json \$x | jq -r '.releases[] | "\\(.name) \\(.version) \\(.url)"' 2>/dev/null ; done | sort | uniq); @@ -144,7 +153,7 @@ sub get_releases { my @months = qw(ignored January February March April May June July August September October November December); sub github { - my ($org, $repo, $versions, $type, $orig_url) = @_; + my ($org, $repo, $versions, $orig_url) = @_; my ($url, $day, $mon, $year, $out, $tag); unless ($org && $repo) { @@ -208,9 +217,15 @@ sub calculate_software_updates { printf STDERR " - retrieving previous releases found by spec tests...\n"; my $last_release_releases = get_releases($last_path); - printf STDERR " - checking ci/upstream.yml for more details...\n"; - my $upstream = LoadFile($FindBin::Bin."/../upstream.yml"); - my $sections = $upstream->{sections} || []; + printf STDERR " - checking ci/settings.yml for more details...\n"; + my $upstream = get_upstream_settings(); + my $sections = $upstream->{categories} || [ + { + 'default' => bless( do{\(my $o = 1)}, 'JSON::PP::Boolean' ), # TODO: maybe support a regular expression match (ie .*-cpi) + 'label' => 'Releases', + 'name' => 'releases' + } + ]; my $default_section = (map {$_->{name}} grep {$_->{default}} @$sections)[0] || '-'; delete $upstream->{sections}; @@ -234,7 +249,8 @@ sub calculate_software_updates { next unless scalar(@new_versions); for my $version (@new_versions) { - my $section = (grep {defined($upstream->{$_}{$name})} keys(%$upstream))[0] || $default_section; + my $release_definition = (grep {$_->{name} eq $name} @{$upstream->{bosh_releases}})[0]; + my $section = $release_definition->{category} || $default_section; for my $url (@{$release_candidate_releases->{$name}{$version} || []}) { my ($type,$target); if ($url =~ /$name-$version-([^-]*(?:-[^0-9][^-]*)*)-(\d*(?:\.\d*)*)-/) { @@ -243,16 +259,17 @@ sub calculate_software_updates { } else { $type = 'source'; } - my @prefixes = defined($upstream->{$section}{$name}{tag_prefix}) ? ($upstream->{$section}{$name}{tag_prefix}) : ('v',''); + my @prefixes = defined($release_definition->{tag_prefix}) ? ($release_definition->{tag_prefix}) : ('v',''); my @suffixes = ($version =~ /^\d+$/) ? ('','.0','.0.0') : (''); $found{$section} ||= {}; + my ($org,$repo) = ($release_definition->{owner},$release_definition->{repository}); + ($org,$repo) = ($repo =~ m|([^\\]*)/([^\\]*)|) if $repo && $repo =~ /\// && ! defined($org); unless (defined($found{$section}{"$name/$version"})) { my ($date,$release_url) = github( - $upstream->{$section}{$name}{org}, - $upstream->{$section}{$name}{repo}, + $org, + $repo, [map {my $p = $_; map {$p.$version.$_} @suffixes} @prefixes], - $upstream->{$section}{$name}{type}, $url ); $found{$section}{"$name/$version"} = { @@ -261,7 +278,7 @@ sub calculate_software_updates { url => $release_url, date => $date, form => [], - label => $upstream->{$section}{$name}{label}, + label => $release_definition->{label}, changed => (! scalar(@old_versions)) ? "NEW" : scalar(grep {$version eq $_} @old_versions) ? '' : sprintf('[X](## "was %s")', join(', ',@old_versions)) diff --git a/ci/settings.yml b/ci/settings.yml index 57bea56..5c8ad72 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -8,15 +8,26 @@ meta: exposed: false initial_version: 0.5.1 - upstream: {} + upstream: + bosh_releases: + - name: cf-rabbitmq-multitenant-broker + type: github-release + repository: pivotal-cf/cf-rabbitmq-multitenant-broker-release + access_token: ((github.access_token)) + + - name: cf-rabbitmq-smoke-tests + type: github-release + repository: pivotal-cf/cf-rabbitmq-smoke-tests-release + access_token: ((github.access_token)) + vault: url: ((vault.url)) token: ((vault.token)) aws: - access_key: ((cloudfoundry-community-aws.access_key_id)) - secret_key: ((cloudfoundry-community-aws.secret_access_key)) + access_key: ((cloudfoundry-community-aws.access_key_id)) + secret_key: ((cloudfoundry-community-aws.secret_access_key)) github: owner: genesis-community @@ -25,6 +36,8 @@ meta: main-branch: main private_key: ((github.private_key)) access_token: ((github.access_token)) + name: ((github.name)) + email: ((github.email)) shout: url: ((shout.url)) diff --git a/ci/upstream.yml b/ci/upstream.yml deleted file mode 100644 index ad67827..0000000 --- a/ci/upstream.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# This file contains patterns for generating links to upstream release notes, -# for constituent releases. - -sections: -- name: releases - label: Releases - default: true