2024 standard refactor update

ci/envs/ci-vsphere-baseline.yml

@@ -6,7 +6,7 @@ kit:
   - metrics-emitter
   ci:
     job: deploy
-    spec_tests: false
+    serial_groups: [deployment]
 
 genesis:
   env:      ci-vsphere-baseline

ci/pipeline/base.yml

@@ -82,7 +82,6 @@ meta:
     VAULT_TOKEN:          (( grab meta.vault.token ))
     INFRASTRUCTURE:       (( grab meta.iaas ))
 
-
 groups:
 - name: (( grab meta.pipeline ))
   jobs:

ci/scripts/build-upstream-jobs

@@ -7,10 +7,11 @@ _lookup() {
  echo "${1}" | base64 --decode | jq -re "${2}"
 }
 
-# Read upstream.yml
+# Get upstream details from settings.yml
 update_group=()
-upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstream.bosh_releases//[] | .[] | @base64')"
-upstream_manifest="$(spruce merge "$base_dir/pipeline/base.yml" "$base_dir/settings.yml" | spruce json | jq -r '.meta.upstream.manifest//{} | @base64')"
+upstream_details="$(spruce merge "$base_dir/pipeline/base.yml" "$base_dir/settings.yml" | spruce json | jq -r '.meta.upstream//{}')"
+upstream_releases="$(jq -r '.bosh_releases//[] | .[] | @base64' <<< "$upstream_details")"
+upstream_manifest="$(jq -r '.manifest//{} | @base64' <<< "$upstream_details")"
 
 # Support upstream manifests
 package="$(_lookup "$upstream_manifest" '.package')" || true
@@ -60,11 +61,17 @@ EOF
 	cp "$base_dir/pipeline/optional-resources/upstream-manifest.yml" "$base_dir/pipeline/upstream/upstream-manifest-resource.yml"
 fi
 
-# For each release in upstream.yml,
-for release in $upstream_details ; do
-  name="$(_lookup "$release" .name)"
+# For each upstream release, create a job to update the release
+for release in $upstream_releases ; do
+  name="$(_lookup "$release" '.name')"
+  watch="$(_lookup "$release" '.watch')" || true
+  [[ "$watch" == "false" ]] && continue
   type="$(_lookup "$release" '.type//"bosh-io-release"')"
-  path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')"
+  path="$(_lookup "$release" '.path//""')"
+  if [[ -z "$path" ]] ; then
+    default_path="$(jq -r '.default_releases_path//"manifests/releases"' <<<"$upstream_details")"
+    path="$default_path/$name.yml"
+  fi
   repo="$(_lookup "$release" '.repository')"
   if [[ $type == 'bosh-io-release' ]] ; then
     source=$'\n'"      repository: $repo";

ci/scripts/compare-release-specs

@@ -105,18 +105,24 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
   echo
 
   echo "Fetching spec diffs..."
-  if [ -f "${ci_dir}/ci/upstreamrepo.yml" ]; then
-    upstreamrepo=$(spruce json "${ci_dir}/ci/upstreamrepo.yml")
-  else
-    upstreamrepo='{"repos": []}'
-  fi
+  upstream="$(
+    spruce merge --skip-eval "${ci_dir}/ci/pipeline/base.yml" "${ci_dir}/ci/settings.yml" |
+    spruce json |
+    jq -r '.meta.upstream.bosh_releases[] |
+           if .spec_url then {name, "repo":.spec_url}
+           elif .owner then {name, "repo":"https://github.com/\(.owner)/\(.repository)"}
+           else {name, "repo":"https://github.com/\(.repository)"}
+           end' |
+    jq -s
+  )"
+
   # TODO: do this in two phases -- first phase pull out all the non-compiled
   #       versions, then run through with the compiled versions, picking up the
   #       non-compiled version's git repo.  Also indicate if they are compiled
   #       or not, and if so, what os is the target. (because that may change)
   repos="$(
     echo "$curr_releases" \
-    | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
+    | jq --argjson gitrepos "$upstream" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
         ($url
         | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then
             ($gitrepos.repos | map(select(.name == $name))[0].repo)
@@ -160,7 +166,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
       branch="master"
     fi
 
-    git -C "$rel_dir" pull --depth 1 origin ${branch} > /dev/null 2>&1
+    git -C "$rel_dir" pull --depth 1 origin "${branch}" > /dev/null 2>&1
 
 
     mkdir -p "$workdir/compare-specs"

ci/scripts/get-latest-upstream-manifest

@@ -86,9 +86,8 @@ git -C git diff --color=always "$UPSTREAM_PATH" | cat
 
 header "Recreate spec-test results to validate upstream"
 pushd git/spec > /dev/null
-  # TODO: remove spec/{credhub,vault} if needed? because this will regenerate vault.
-  rm -rf results/
-  ACK_GINKGO_RC=true ginkgo -p .
+  rm -rf results/ credhub/
+  ACK_GINKGO_RC=true ginkgo -p -failFast .
 popd
 
 header "Spec file changes:"

ci/scripts/release-notes

@@ -55,6 +55,14 @@ sub get_new_commits {
 	return @commits;
 }
 
+sub get_upstream_settings {
+	my ($path) = @_;
+	$path //= $FindBin::Bin."/../";
+	my $json = qx(spruce merge --skip-eval $path/pipeline/base.yml $path/settings.yml | spruce json);
+	my $settings = decode_json($json);
+	return $settings->{meta}{upstream} || {};
+}
+
 sub get_latest_commit_ref {
 	my ($path) = @_;
 	chomp(my $ref = qx(git -C $path rev-parse HEAD));
@@ -131,6 +139,7 @@ sub parse_commits {
 }
 
 sub get_releases {
+	# FIXME: This should also check manifest/releases/*.yml or potentially other locations
 	my ($path) = @_;
 	my $filter = "$path/spec/results/*.yml";
 	my @entries = qx(for x in $filter ; do spruce json \$x | jq -r '.releases[] | "\\(.name) \\(.version) \\(.url)"' 2>/dev/null ; done | sort | uniq);
@@ -144,7 +153,7 @@ sub get_releases {
 
 my @months = qw(ignored January February March April May June July August September October November December);
 sub github {
-	my ($org, $repo, $versions, $type, $orig_url) = @_;
+	my ($org, $repo, $versions, $orig_url) = @_;
 	my ($url, $day, $mon, $year, $out, $tag);
 
 	unless ($org && $repo) {
@@ -208,9 +217,15 @@ sub calculate_software_updates {
 	printf STDERR "  - retrieving previous releases found by spec tests...\n";
 	my $last_release_releases = get_releases($last_path);
 
-	printf STDERR "  - checking ci/upstream.yml for more details...\n";
-	my $upstream = LoadFile($FindBin::Bin."/../upstream.yml");
-	my $sections = $upstream->{sections} || [];
+	printf STDERR "  - checking ci/settings.yml for more details...\n";
+	my $upstream = get_upstream_settings();
+	my $sections = $upstream->{categories} || [
+		{
+			'default' => bless( do{\(my $o = 1)}, 'JSON::PP::Boolean' ), # TODO: maybe support a regular expression match (ie .*-cpi)
+			'label' => 'Releases',
+			'name' => 'releases'
+		}
+	];
 	my $default_section = (map {$_->{name}} grep {$_->{default}} @$sections)[0] || '-';
 	delete $upstream->{sections};
 
@@ -234,7 +249,8 @@ sub calculate_software_updates {
 		next unless scalar(@new_versions);
 		for my $version (@new_versions) {
 
-			my $section = (grep {defined($upstream->{$_}{$name})} keys(%$upstream))[0] || $default_section;
+			my $release_definition = (grep {$_->{name} eq $name} @{$upstream->{bosh_releases}})[0];
+			my $section = $release_definition->{category} || $default_section;
 			for my $url (@{$release_candidate_releases->{$name}{$version} || []}) {
 				my ($type,$target);
 				if ($url =~ /$name-$version-([^-]*(?:-[^0-9][^-]*)*)-(\d*(?:\.\d*)*)-/) {
@@ -243,16 +259,17 @@ sub calculate_software_updates {
 				} else {
 					$type = 'source';
 				}
-				my @prefixes = defined($upstream->{$section}{$name}{tag_prefix}) ? ($upstream->{$section}{$name}{tag_prefix}) : ('v','');
+				my @prefixes = defined($release_definition->{tag_prefix}) ? ($release_definition->{tag_prefix}) : ('v','');
 				my @suffixes = ($version =~ /^\d+$/) ? ('','.0','.0.0') : ('');
 
 				$found{$section} ||= {};
+				my ($org,$repo) = ($release_definition->{owner},$release_definition->{repository});
+				($org,$repo) = ($repo =~ m|([^\\]*)/([^\\]*)|) if $repo && $repo =~ /\// && ! defined($org);
 				unless (defined($found{$section}{"$name/$version"})) {
 					my ($date,$release_url) = github(
-						$upstream->{$section}{$name}{org},
-						$upstream->{$section}{$name}{repo},
+						$org,
+						$repo,
 						[map {my $p = $_; map {$p.$version.$_} @suffixes} @prefixes],
-						$upstream->{$section}{$name}{type},
 						$url
 					);
 					$found{$section}{"$name/$version"} = {
@@ -261,7 +278,7 @@ sub calculate_software_updates {
 						url => $release_url,
 						date => $date,
 						form => [],
-						label => $upstream->{$section}{$name}{label},
+						label => $release_definition->{label},
 						changed => (! scalar(@old_versions)) ? "NEW" :
 						           scalar(grep {$version eq $_} @old_versions) ? '' :
 						           sprintf('[X](## "was %s")', join(', ',@old_versions))

ci/settings.yml

@@ -8,15 +8,26 @@ meta:
   exposed:         false
   initial_version: 0.5.1
 
-  upstream: {}
+  upstream: 
+    bosh_releases:
+    - name:         cf-rabbitmq-multitenant-broker
+      type:         github-release
+      repository:   pivotal-cf/cf-rabbitmq-multitenant-broker-release
+      access_token: ((github.access_token))
+
+    - name:         cf-rabbitmq-smoke-tests
+      type:         github-release
+      repository:   pivotal-cf/cf-rabbitmq-smoke-tests-release
+      access_token: ((github.access_token))
+
 
   vault:
     url:       ((vault.url))
     token:     ((vault.token))
 
   aws:
-    access_key:  ((cloudfoundry-community-aws.access_key_id))
-    secret_key:  ((cloudfoundry-community-aws.secret_access_key))
+    access_key: ((cloudfoundry-community-aws.access_key_id))
+    secret_key: ((cloudfoundry-community-aws.secret_access_key))
 
   github:
     owner:        genesis-community
@@ -25,6 +36,8 @@ meta:
     main-branch:  main
     private_key:  ((github.private_key))
     access_token: ((github.access_token))
+    name:         ((github.name))
+    email:        ((github.email))
 
   shout:
     url:      ((shout.url))