From c086ed363165c85f10a4e70a7b21712194e79ded Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Thu, 20 Jul 2023 23:51:56 +0000 Subject: [PATCH] Squashed 'release-tools/' changes from 78c0fb7..4133d1d 4133d1d Merge pull request #226 from msau42/cloudbuild 8d519d2 Pin buildkit to v0.10.6 to workaround v0.11 bug with docker manifest 6e04a03 Merge pull request #224 from msau42/cloudbuild 26fdfff Update cloudbuild image 6613c39 Merge pull request #223 from sunnylovestiramisu/update 0e7ae99 Update k8s image repo url 77e47cc Merge pull request #222 from xinydev/fix-dep-version 155854b Fix dep version mismatch 8f83905 Merge pull request #221 from sunnylovestiramisu/go-update 1d3f94d Update go version to 1.20 to match k/k v1.27 e322ce5 Merge pull request #220 from andyzhangx/fix-golint-error b74a512 test: fix golint error aa61bfd Merge pull request #218 from xing-yang/update_csi_driver 7563d19 Update CSI_PROW_DRIVER_VERSION to v1.11.0 a2171be Merge pull request #216 from msau42/process cb98782 Merge pull request #217 from msau42/owners a11216e add new reviewers and remove inactive reviewers dd98675 Add step for checking builds b66c082 Merge pull request #214 from pohly/junit-fixes b9b6763 filter-junit.go: fix loss of testcases when parsing Ginkgo v2 JUnit d427783 filter-junit.go: preserve system error log 38e1146 prow.sh: publish individual JUnit files as separate artifacts git-subtree-dir: release-tools git-subtree-split: 4133d1df083eaa65bdeddd0530d54278529c7a60 --- KUBERNETES_CSI_OWNERS_ALIASES | 4 +++- SIDECAR_RELEASE_PROCESS.md | 4 +++- build.make | 2 +- cloudbuild.yaml | 2 +- filter-junit.go | 8 ++++---- prow.sh | 23 ++++++++++++++--------- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 344fbe4d..04a07bfb 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -22,17 +22,19 @@ aliases: - ggriffiths - gnufied - humblec + - mauriciopoppe - j-griffith - - Jiawei0227 - jingxu97 - jsafrane - pohly - RaunakShah + - sunnylovestiramisu - xing-yang # This documents who previously contributed to Kubernetes-CSI # as approver. emeritus_approvers: +- Jiawei0227 - lpabon - sbezverk - vladimirvivien diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 8977fbe6..d1cc80b8 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -92,6 +92,8 @@ naming convention `-on-`. 1. Check that all [canary CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. +1. Check that the post-\-push-images builds are succeeding. + [Example](https://k8s-testgrid.appspot.com/sig-storage-image-build#post-external-snapshotter-push-images) 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. 1. Create a new release following a previous release as a template. Be sure to select the correct @@ -102,7 +104,7 @@ naming convention `-on-`. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar diff --git a/build.make b/build.make index ff428890..bceab34d 100644 --- a/build.make +++ b/build.make @@ -148,7 +148,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ export DOCKER_CLI_EXPERIMENTAL=enabled; \ - docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest --driver-opt image=moby/buildkit:v0.10.6; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 4bcffad3..6b76a9ed 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,7 +26,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20210917-12df099d55' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230424-910a2a439d' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/filter-junit.go b/filter-junit.go index 5454092b..aab1b8b6 100644 --- a/filter-junit.go +++ b/filter-junit.go @@ -24,7 +24,6 @@ package main import ( "encoding/xml" "flag" - "io/ioutil" "os" "regexp" ) @@ -56,6 +55,7 @@ type TestCase struct { Name string `xml:"name,attr"` Time string `xml:"time,attr"` SystemOut string `xml:"system-out,omitempty"` + SystemErr string `xml:"system-err,omitempty"` Failure string `xml:"failure,omitempty"` Skipped SkipReason `xml:"skipped,omitempty"` } @@ -95,7 +95,7 @@ func main() { } } else { var err error - data, err = ioutil.ReadFile(input) + data, err = os.ReadFile(input) if err != nil { panic(err) } @@ -109,7 +109,7 @@ func main() { if err := xml.Unmarshal(data, &junitv2); err != nil { panic(err) } - junit = junitv2.TestSuite + junit.TestCases = append(junit.TestCases, junitv2.TestSuite.TestCases...) } } @@ -142,7 +142,7 @@ func main() { panic(err) } } else { - if err := ioutil.WriteFile(*output, data, 0644); err != nil { + if err := os.WriteFile(*output, data, 0644); err != nil { panic(err) } } diff --git a/prow.sh b/prow.sh index 55c3231d..6d68fd9f 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.19" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.20" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below @@ -196,7 +196,7 @@ kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fc # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.8.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.11.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" @@ -245,7 +245,7 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # The version of dep to use for 'make test-vendor'. Ignored if the project doesn't # use dep. Only binary releases of dep are supported (https://github.com/golang/dep/releases). -configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor checking" +configvar CSI_PROW_DEP_VERSION v0.5.4 "golang dep version to be used for vendor checking" # Each job can run one or more of the following tests, identified by # a single word: @@ -469,7 +469,7 @@ install_dep () { if dep version 2>/dev/null | grep -q "version:.*${CSI_PROW_DEP_VERSION}$"; then return fi - run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/v0.5.4/dep-linux-amd64" && + run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/${CSI_PROW_DEP_VERSION}/dep-linux-amd64" && chmod u+x "${CSI_PROW_WORK}/bin/dep" } @@ -1008,7 +1008,10 @@ run_e2e () ( # the full Kubernetes E2E testsuite while only running a few tests. move_junit () { if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then - run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + mkdir -p "${ARTIFACTS}/junit/${name}" && + mkdir -p "${ARTIFACTS}/junit/steps" && + run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && + mv "${ARTIFACTS}"/junit_[0-9]*.xml "${ARTIFACTS}/junit/${name}/" fi } trap move_junit EXIT @@ -1085,13 +1088,14 @@ kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PA EOF chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh + mkdir -p "${ARTIFACTS}/junit/steps" # This cannot run in parallel, because -csi.junitfile output # from different Ginkgo nodes would go to the same file. Also the # staging and target directories are the same. run_with_loggers "${CSI_PROW_WORK}/csi-sanity" \ -ginkgo.v \ - -csi.junitfile "${ARTIFACTS}/junit_sanity.xml" \ + -csi.junitfile "${ARTIFACTS}/junit/steps/junit_sanity.xml" \ -csi.endpoint "dns:///$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' csi-prow-control-plane):$(kubectl get "services/${CSI_PROW_SANITY_SERVICE}" -o "jsonpath={..nodePort}")" \ -csi.stagingdir "/tmp/staging" \ -csi.mountdir "/tmp/mount" \ @@ -1121,7 +1125,8 @@ make_test_to_junit () { # Plain make-test.xml was not delivered as text/xml by the web # server and ignored by spyglass. It seems that the name has to # match junit*.xml. - out="${ARTIFACTS}/junit_make_test.xml" + out="${ARTIFACTS}/junit/steps/junit_make_test.xml" + mkdir -p "$(dirname "$out")" testname= echo "" >>"$out" @@ -1385,8 +1390,8 @@ main () { fi # Merge all junit files into one. This gets rid of duplicated "skipped" tests. - if ls "${ARTIFACTS}"/junit_*.xml 2>/dev/null >&2; then - run_filter_junit -o "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}"/junit_*.xml && rm "${ARTIFACTS}"/junit_*.xml && mv "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}" + if ls "${ARTIFACTS}"/junit/steps/junit_*.xml 2>/dev/null >&2; then + run_filter_junit -o "${ARTIFACTS}/junit_final.xml" "${ARTIFACTS}"/junit/steps/junit_*.xml fi return "$ret"