diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a6a209cd13d..f6aa4d6374c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -94,7 +94,7 @@ jobs: fail-fast: false matrix: redis-version: [7] - go-version: [1.22.x] + go-version: [1.23.x] env: REDIS_IMAGE: redis:${{ matrix.redis-version }} diff --git a/.github/workflows/plugin-compiler-build.yml b/.github/workflows/plugin-compiler-build.yml index aa4939fd603..0a51ef68fd1 100644 --- a/.github/workflows/plugin-compiler-build.yml +++ b/.github/workflows/plugin-compiler-build.yml @@ -11,7 +11,7 @@ on: - "v*" env: - GOLANG_CROSS: 1.22-bullseye + GOLANG_CROSS: 1.23-bullseye concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -77,7 +77,7 @@ jobs: labels: ${{ steps.set-metadata.outputs.labels }} tags: ${{ steps.set-metadata.outputs.tags }} build-args: | - BASE-IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }} + BASE_IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }} GITHUB_SHA=${{ github.sha }} GITHUB_TAG=${{ github.ref_name }} @@ -108,7 +108,7 @@ jobs: labels: ${{ steps.set-metadata-ee.outputs.labels }} tags: ${{ steps.set-metadata-ee.outputs.tags }} build-args: | - BASE-IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }} + BASE_IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }} GITHUB_SHA=${{ github.sha }} GITHUB_TAG=${{ github.ref_name }} BUILD_TAG=ee diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e611642b870..3032690a57f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,9 +41,9 @@ jobs: fail-fast: false matrix: golang_cross: - - 1.22-bullseye + - 1.23-bullseye include: - - golang_cross: 1.22-bullseye + - golang_cross: 1.23-bullseye goreleaser: 'ci/goreleaser/goreleaser.yml' cgo: 1 rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023' @@ -127,12 +127,12 @@ jobs: mask-aws-account-id: false - uses: aws-actions/amazon-ecr-login@v2 id: ecr - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} with: mask-password: 'true' - name: Docker metadata for CI id: ci_metadata_ - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/metadata-action@v5 with: images: ${{ steps.ecr.outputs.registry }}/tyk @@ -146,7 +146,7 @@ jobs: type=semver,pattern={{major}}.{{minor}},prefix=v type=semver,pattern={{version}},prefix=v - name: push image to CI - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/build-push-action@v6 with: context: "dist" @@ -163,7 +163,7 @@ jobs: EDITION= - name: Docker metadata for CI ee id: ci_metadata_ee - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/metadata-action@v5 with: images: ${{ steps.ecr.outputs.registry }}/tyk-ee @@ -177,7 +177,7 @@ jobs: type=semver,pattern={{major}}.{{minor}},prefix=v type=semver,pattern={{version}},prefix=v - name: push image to CI ee - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/build-push-action@v6 with: context: "dist" @@ -207,7 +207,7 @@ jobs: type=semver,pattern={{version}} labels: "org.opencontainers.image.title=tyk-gateway (distroless) \norg.opencontainers.image.description=Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n" - name: push image to prod - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/build-push-action@v6 with: context: "dist" @@ -236,7 +236,7 @@ jobs: type=semver,pattern={{version}} labels: "org.opencontainers.image.title=tyk-gateway Enterprise Edition (distroless) \norg.opencontainers.image.description=Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n" - name: push image to prod ee - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} uses: docker/build-push-action@v6 with: context: "dist" @@ -253,7 +253,7 @@ jobs: EDITION=-ee - name: save deb uses: actions/upload-artifact@v4 - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} with: name: deb retention-days: 1 @@ -263,7 +263,7 @@ jobs: !dist/*fips*.deb - name: save rpm uses: actions/upload-artifact@v4 - if: ${{ matrix.golang_cross == '1.22-bullseye' }} + if: ${{ matrix.golang_cross == '1.23-bullseye' }} with: name: rpm retention-days: 1 diff --git a/Dockerfile b/Dockerfile index c094b71ccd2..0b9d631f9f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,68 +1,28 @@ -FROM debian:bookworm AS assets - -# This Dockerfile facilitates bleeding edge development docker image builds -# directly from source. To build a development image, run `make docker`. -# If you need to tweak the environment for testing, you can override the -# `GO_VERSION` and `PYTHON_VERSION` as docker build arguments. - -ARG GO_VERSION=1.22.6 -ARG PYTHON_VERSION=3.11.6 - -WORKDIR /assets - -RUN apt update && apt install wget -y && \ - wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ - wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz - -FROM debian:bookworm - -ARG GO_VERSION=1.22.6 -ARG PYTHON_VERSION=3.11.6 - -COPY --from=assets /assets/ /tmp/ -WORKDIR /tmp - -# Install Go - -ENV PATH=$PATH:/usr/local/go/bin - -RUN tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \ - go version +ARG GO_VERSION=1.23 +FROM golang:${GO_VERSION}-bullseye # Build essentials RUN apt update && apt install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl wget libbz2-dev -y -# Install $PYTHON_VERSION - ## This just installs whatever is is bullseye, makes docker build (fast/small)-(er) RUN apt install python3 -y -## This runs python code slower, but the process finishes quicker -# RUN tar -xf Python-${PYTHON_VERSION}.tar.xz && ls -la && \ -# cd Python-${PYTHON_VERSION}/ && \ -# ./configure --enable-shared && make build_all && \ -# make altinstall && \ -# ldconfig $PWD - -## This runs python code faster, but is expensive to build and runs regression tests -# RUN tar -xf Python-${PYTHON_VERSION}.tar.xz && ls -la && \ -# cd Python-${PYTHON_VERSION}/ && \ -# ./configure --enable-shared --enable-optimizations && make -j 2 && \ -# make altinstall && \ -# ldconfig $PWD - -# Clean up build assets -RUN find /tmp -type f -delete - # Build gateway RUN mkdir /opt/tyk-gateway WORKDIR /opt/tyk-gateway + ADD go.mod go.sum /opt/tyk-gateway/ -RUN go mod download + +RUN --mount=type=cache,mode=0755,target=/go/pkg/mod \ + --mount=type=cache,mode=0755,target=/root/.cache/go-build \ + go mod download + ADD . /opt/tyk-gateway -RUN make build +RUN --mount=type=cache,mode=0755,target=/go/pkg/mod \ + --mount=type=cache,mode=0755,target=/root/.cache/go-build \ + make build COPY tyk.conf.example tyk.conf diff --git a/Taskfile.yml b/Taskfile.yml index 33c8bd1a8a2..8ce0c1e23ca 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -25,7 +25,7 @@ tasks: docker: desc: "build Tyk gateway internal/tyk-gateway" cmds: - - docker build --platform "linux/amd64" --rm -t internal/tyk-gateway . + - docker build --build-arg GO_VERSION="$(go mod edit -json | jq .Go -r)" --platform "linux/amd64" --rm -t internal/tyk-gateway . sources: - go.mod - go.sum diff --git a/ci/images/plugin-compiler/Dockerfile b/ci/images/plugin-compiler/Dockerfile index f045ce5a784..5f0219afa9a 100644 --- a/ci/images/plugin-compiler/Dockerfile +++ b/ci/images/plugin-compiler/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=tykio/golang-cross:1.22-bullseye +ARG BASE_IMAGE=tykio/golang-cross:1.23-bullseye FROM ${BASE_IMAGE} LABEL description="Image for plugin development" diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index e018caa87b0..5fbd66f2b2a 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -26,6 +26,7 @@ import ( "github.com/TykTechnologies/tyk-pump/analytics" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/internal/debug2" "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" @@ -643,30 +644,55 @@ func TestListenPathTykPrefix(t *testing.T) { } func TestReloadGoroutineLeakWithTest(t *testing.T) { - test.Flaky(t) + newRecord := func() *debug2.Record { + result := debug2.NewRecord() + result.SetIgnores([]string{ + "runtime/pprof.writeRuntimeProfile", + "/root/go/pkg/mod/github.com/!tyk!technologies/leakybucket@v0.0.0-20170301023702-71692c943e3c/memorycache/cache.go:69", + "/root/go/pkg/mod/github.com/pmylund/go-cache@v2.1.0+incompatible/cache.go:1079", + "/root/tyk/tyk/gateway/distributed_rate_limiter.go:31", + "/root/tyk/tyk/gateway/redis_signals.go:68", + }) + + return result + } - before := runtime.NumGoroutine() + before := newRecord() ts := StartTest(nil) ts.Close() - time.Sleep(time.Second) - - after := runtime.NumGoroutine() + time.Sleep(100 * time.Millisecond) + runtime.GC() - if before < after { - t.Errorf("Goroutine leak, was: %d, after reload: %d", before, after) - } + final := newRecord().Since(before) + assert.Equal(t, 0, final.Count(), "final count not zero: %s", final) } func TestReloadGoroutineLeakWithCircuitBreaker(t *testing.T) { ts := StartTest(nil) t.Cleanup(ts.Close) + newRecord := func() *debug2.Record { + result := debug2.NewRecord() + result.SetIgnores([]string{ + "runtime/pprof.writeRuntimeProfile", + "/root/tyk/tyk/gateway/reverse_proxy.go:223", + "/root/tyk/tyk/gateway/api_definition.go:1025", + "/root/tyk/tyk/gateway/distributed_rate_limiter.go:31", + "/root/go/pkg/mod/github.com/pmylund/go-cache@v2.1.0+incompatible/cache.go:1079", + "/root/go/pkg/mod/github.com/!tyk!technologies/circuitbreaker@v2.2.2+incompatible/circuitbreaker.go:202", + }) + + return result + } + globalConf := ts.Gw.GetConfig() globalConf.EnableJSVM = false ts.Gw.SetConfig(globalConf) + stage1 := newRecord() + specs := ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/" UpdateAPIVersion(spec, "v1", func(version *apidef.VersionInfo) { @@ -684,17 +710,13 @@ func TestReloadGoroutineLeakWithCircuitBreaker(t *testing.T) { }) }) - before := runtime.NumGoroutine() - ts.Gw.LoadAPI(specs...) // just doing globalGateway.DoReload() doesn't load anything as BuildAndLoadAPI cleans up folder with API specs time.Sleep(100 * time.Millisecond) + runtime.GC() - after := runtime.NumGoroutine() - - if before < after { - t.Errorf("Goroutine leak, was: %d, after reload: %d", before, after) - } + final := newRecord().Since(stage1) + assert.Equal(t, 0, final.Count(), "final count not zero: %s", final) } func listenProxyProto(ls net.Listener) error { diff --git a/go.mod b/go.mod index 3c10e2b4e2b..c140a431531 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/TykTechnologies/tyk -go 1.22.6 +go 1.23.4 require ( github.com/Jeffail/tunny v0.1.4 diff --git a/internal/debug2/goroutine.go b/internal/debug2/goroutine.go new file mode 100644 index 00000000000..115975597c1 --- /dev/null +++ b/internal/debug2/goroutine.go @@ -0,0 +1,124 @@ +package debug2 + +import ( + "bytes" + "fmt" + "regexp" + "runtime/pprof" + "strings" +) + +// Record captures goroutine states +type Record struct { + buffer *bytes.Buffer + ignores []string +} + +// NewRecord creates a new Record and populates it with the current goroutine dump. +func NewRecord() *Record { + result := &Record{ + buffer: bytes.NewBuffer([]byte{}), + } + + pprof.Lookup("goroutine").WriteTo(result.buffer, 1) + + result.SetIgnores([]string{ + "runtime/pprof.writeRuntimeProfile", + }) + return result +} + +func (r *Record) SetIgnores(ignores []string) { + r.ignores = ignores +} + +var headerMatchRe = regexp.MustCompile(`^[0-9]+ @ 0x.*`) + +// parseGoroutines parses goroutines from the buffer into a map where each key is a +// goroutine header and the value is its stack trace as a slice of strings. +func (r *Record) parseGoroutines() map[string][]string { + goroutines := make(map[string][]string) + var currentHeader string + var currentStack []string + toDelete := []string{} + lines := strings.Split(r.buffer.String(), "\n") + + for _, line := range lines { + var skip bool + for _, ign := range r.ignores { + if strings.Contains(line, ign) { + skip = true + break + } + } + + if skip { + toDelete = append(toDelete, currentHeader) + } + + if headerMatchRe.MatchString(line) { + // Save the previous goroutine and reset + if currentHeader != "" { + goroutines[currentHeader] = currentStack + } + currentHeader = line + currentStack = []string{line} + } else if currentHeader != "" { + // Add stack trace lines to the current goroutine + currentStack = append(currentStack, line) + } + } + + // Save the last goroutine + if currentHeader != "" { + goroutines[currentHeader] = currentStack + } + + for _, key := range toDelete { + delete(goroutines, key) + } + + return goroutines +} + +// Since compares the current Record with another Record and returns a new Record +// containing only the goroutines found in the current Record but not in the last. +func (r *Record) Since(last *Record) *Record { + currentGoroutines := r.parseGoroutines() + lastGoroutines := last.parseGoroutines() + + diffBuffer := bytes.NewBuffer([]byte{}) + for header, stack := range currentGoroutines { + if _, exists := lastGoroutines[header]; !exists { + diffBuffer.WriteString(header + "\n") + for _, line := range stack { + diffBuffer.WriteString(line + "\n") + } + } + } + + return &Record{ + buffer: diffBuffer, + } +} + +// Count returns the number of unique goroutines in the Record. +func (r *Record) Count() int { + return len(r.parseGoroutines()) +} + +// String implements the fmt.Stringer interface, providing a formatted view +// of the goroutines in the Record. +func (r *Record) String() string { + goroutines := r.parseGoroutines() + var builder strings.Builder + builder.WriteString(fmt.Sprintf("Number of goroutines: %d\n", len(goroutines))) + for header, stack := range goroutines { + builder.WriteString("--- Goroutine ---\n") + builder.WriteString(header + "\n") + for _, line := range stack { + builder.WriteString(line + "\n") + } + } + return builder.String() +} diff --git a/internal/debug2/goroutine_test.go b/internal/debug2/goroutine_test.go new file mode 100644 index 00000000000..0d55d582247 --- /dev/null +++ b/internal/debug2/goroutine_test.go @@ -0,0 +1,103 @@ +package debug2_test + +import ( + "context" + "fmt" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/TykTechnologies/tyk/internal/debug2" +) + +func TestNewRecordWithGoroutines(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + time.Sleep(100 * time.Millisecond) + + // Capture the initial state of goroutines + initialRecord := debug2.NewRecord() + + // Create and start a new goroutine + go func() { + time.Sleep(100 * time.Millisecond) + }() + go func() { + time.Sleep(100 * time.Millisecond) + }() + + // Capture the state after starting the goroutine + intermediateRecord := debug2.NewRecord() + // t.Log("The intermediate goroutines:\n", intermediateRecord.String()) + + newGoroutines := intermediateRecord.Since(initialRecord) + assert.Equal(t, 2, newGoroutines.Count(), "Expected new goroutines, but found none") + + for { + // Wait for the goroutine to finish + time.Sleep(100 * time.Millisecond) + runtime.GC() + time.Sleep(10 * time.Millisecond) + + // Capture the state after the goroutine has finished + finalRecord := debug2.NewRecord() + remainingGoroutines := finalRecord.Since(initialRecord) + + // Expecting goroutines clear + if remainingGoroutines.Count() == 0 { + break + } + + if ctx.Err() != nil { + break + } + + fmt.Print(remainingGoroutines.String()) + } + + assert.NoError(t, ctx.Err(), "cancelled goroutine leak check after timeout") +} + +func BenchmarkNewRecordWithGoroutines(b *testing.B) { + // Capture the initial state of goroutines + initialRecord := debug2.NewRecord() + + // Create and start a new goroutine + + var wg sync.WaitGroup + wg.Add(b.N) + + var i int + for i = 0; i < b.N; i++ { + go func() { + defer wg.Done() + + time.Sleep(100 * time.Millisecond) + }() + } + + // Capture the state after starting the goroutine + intermediateRecord := debug2.NewRecord() + b.Logf("Started %d goroutines with sleep", b.N) + b.Log("Intermediate Record count: ", intermediateRecord.Count()) + + wg.Wait() + + runtime.GC() + + // Capture the state after the goroutine has finished + finalRecord := debug2.NewRecord() + b.Log("Finished with finalRecord count: ", finalRecord.Count()) + + // Check that the intermediate record contains the new goroutine + newGoroutines := intermediateRecord.Since(initialRecord) + assert.Greater(b, newGoroutines.Count(), 0, "Expected new goroutines, but found none") + + // Check that the final record no longer contains the new goroutine + remainingGoroutines := finalRecord.Since(initialRecord) + assert.Equal(b, 0, remainingGoroutines.Count(), "Expected no new goroutines, but found: "+remainingGoroutines.String()) +}