poc: use Buf and protocompile for protobuf schemas

MANIFEST.in

@@ -9,5 +9,6 @@ include setup.py
 include setup.cfg
 include LICENSE
 include MANIFEST.in
+include *.so
 
 recursive-exclude examples *~ *.pyc \.*

go/protopace/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env

go/protopace/Makefile

@@ -0,0 +1,101 @@
+# Change these variables as necessary.
+MAIN_PACKAGE_PATH := .
+BINARY_NAME := protopace
+BUILD_DIR := ../../karapace/protobuf/protopace/bin
+
+# ==================================================================================== #
+# HELPERS
+# ==================================================================================== #
+
+## help: print this help message
+.PHONY: help
+help:
+	@echo 'Usage:'
+	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /'
+
+.PHONY: confirm
+confirm:
+	@echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ]
+
+.PHONY: no-dirty
+no-dirty:
+	git diff --exit-code
+
+
+# ==================================================================================== #
+# QUALITY CONTROL
+# ==================================================================================== #
+
+## tidy: format code and tidy modfile
+.PHONY: tidy
+tidy:
+	go fmt ./...
+	go mod tidy -v
+
+## audit: run quality control checks
+.PHONY: audit
+audit:
+	go mod verify
+	go vet ./...
+	go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
+	go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+	go test -race -buildvcs -vet=off ./...
+
+
+# ==================================================================================== #
+# DEVELOPMENT
+# ==================================================================================== #
+
+## test: run all tests
+.PHONY: test
+test:
+	go test -v -race -buildvcs ./...
+
+## test/cover: run all tests and display coverage
+.PHONY: test/cover
+test/cover:
+	go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./...
+	go tool cover -html=/tmp/coverage.out
+
+## build: build the application
+.PHONY: build
+build:
+	# Include additional build steps, like TypeScript, SCSS or Tailwind compilation here...
+	go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH}
+
+## run: run the  application
+.PHONY: run
+run: build
+	/tmp/bin/${BINARY_NAME}
+
+## run/live: run the application with reloading on file changes
+.PHONY: run/live
+run/live:
+	go run github.com/cosmtrek/[email protected] \
+		--build.cmd "make build" --build.bin "/tmp/bin/${BINARY_NAME}" --build.delay "100" \
+		--build.exclude_dir "" \
+		--build.include_ext "go, tpl, tmpl, html, css, scss, js, ts, sql, jpeg, jpg, gif, png, bmp, svg, webp, ico" \
+		--misc.clean_on_exit "true"
+
+
+# ==================================================================================== #
+# OPERATIONS
+# ==================================================================================== #
+
+## release: cross-compile to build dir on mac
+.PHONY: release
+release:
+	CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+	CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+
+	docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/amd64 golang:1.22 env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-amd64.so
+	cp ${BINARY_NAME}-linux-amd64.so ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.so
+	cp ${BINARY_NAME}-linux-amd64.h ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.h
+	rm ${BINARY_NAME}-linux-amd64.so
+	rm ${BINARY_NAME}-linux-amd64.h
+
+	docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/arm64 golang:1.22 env GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-arm64.so
+	cp ${BINARY_NAME}-linux-arm64.so ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.so
+	cp ${BINARY_NAME}-linux-arm64.h ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.h
+	rm ${BINARY_NAME}-linux-arm64.so
+	rm ${BINARY_NAME}-linux-arm64.h

go/protopace/compatibility.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+
+	"github.com/Aiven-Open/karapace/go/protopace/schema"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking"
+	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
+	"github.com/bufbuild/buf/private/pkg/tracing"
+	"go.uber.org/zap"
+)
+
+func Check(schema schema.Schema, previousSchema schema.Schema) error {
+	handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer)
+	ctx := context.Background()
+	image, err := schema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	previousImage, err := previousSchema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	checkConfig, _ := bufconfig.NewEnabledCheckConfig(
+		bufconfig.FileVersionV2,
+		nil,
+		[]string{
+			"FIELD_NO_DELETE",
+			"FILE_SAME_PACKAGE",
+			"FIELD_SAME_NAME",
+			"FIELD_SAME_JSON_NAME",
+			"FILE_NO_DELETE",
+			"ENUM_NO_DELETE",
+		},
+		nil,
+		nil,
+	)
+	config := bufconfig.NewBreakingConfig(checkConfig, false)
+	return handler.Check(ctx, config, previousImage, image)
+}

go/protopace/compatibility_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	s "github.com/Aiven-Open/karapace/go/protopace/schema"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCompatibility(t *testing.T) {
+	assert := assert.New(t)
+
+	data, _ := os.ReadFile("./fixtures/dependency.proto")
+	dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil)
+	assert.NoError(err)
+	assert.NotNil(dependencySchema)
+
+	data, _ = os.ReadFile("./fixtures/test.proto")
+	testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(testSchema)
+
+	data, _ = os.ReadFile("./fixtures/test_previous.proto")
+	previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(previousSchema)
+
+	err = Check(*testSchema, *testSchema)
+	assert.NoError(err)
+
+	err = Check(*testSchema, *previousSchema)
+	assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".")
+}

go/protopace/fixtures/dependency.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package my.awesome.customer.v1;
+
+message NestedValue {
+    string value = 1;
+}
+
+enum Status {
+    UNKNOWN = 0;
+    ACTIVE = 1;
+    INACTIVE = 2;
+}

go/protopace/fixtures/test.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  int32 foo = 5;
+}

go/protopace/fixtures/test_previous.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  string foo = 5;
+}