Simple macro benchmark

Implementation of a simple macro benchmark executing `ec validate image`
with the fixed state and no external dependencies.

For this the OCI distribution registry is run in a container with the
data from the `data/registry/data` and the git repository mirrored in
the `data/git/rhtap-ec-policy.git`.

The benchmark outputs in the golang standard benchmark format that can
be utilized with tools in the golang benchmarking ecosystem.

Reference: https://issues.redhat.com/browse/EC-968

Makefile

@@ -144,6 +144,13 @@ feature_%: ## Run acceptance tests for a single feature file, e.g. make feature_
 scenario_%: build ## Run acceptance tests for a single scenario, e.g. make scenario_inline_policy
 	@cd acceptance && go test -test.run 'TestFeatures/$*'
 
+benchmark_%:
+	cd benchmark/$*
+	go run . -benchnum 10
+
+.PHONY: benchmark
+benchmark: benchmark_simple ## Run benchmarks
+
 .PHONY: ci
 ci: test lint-fix acceptance ## Run the usual required CI tasks
 

benchmark/README.md

@@ -0,0 +1,12 @@
+# Benchmarks of ec CLI
+
+Benchmarks within this directory use the [golang
+benchmarking](golang.org/x/benchmarks/) package and output in the [standard
+benchmark
+format](https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md).
+
+Each benchmark is built as a standalone executable with no external dependency
+other than any data that is contained within it. Benchmarks are run from within
+the directory they're defined in, simply by running `go run .`, additional
+arguments can be passed in, for example `-benchnum 10` to run the benchmark 10
+times.

benchmark/internal/registry/registry.go

@@ -0,0 +1,50 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package registry
+
+import (
+	"context"
+
+	"github.com/testcontainers/testcontainers-go/modules/registry"
+)
+
+type Closer interface {
+	Close()
+}
+
+type registryCloser struct {
+	container *registry.RegistryContainer
+}
+
+func (r *registryCloser) Close() {
+	if r == nil || r.container == nil {
+		return
+	}
+
+	_ = r.container.Terminate(context.Background())
+}
+
+func Launch(data string) (string, Closer, error) {
+	ctx := context.Background()
+	r, err := registry.Run(ctx, "registry:2.8.3", registry.WithData(data))
+	c := &registryCloser{r}
+	if err != nil {
+		return "", c, err
+	}
+
+	return r.RegistryName, c, nil
+}

benchmark/internal/suite/suite.go

@@ -0,0 +1,32 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suite
+
+import (
+	"io"
+
+	"github.com/enterprise-contract/ec-cli/cmd"
+	"github.com/enterprise-contract/ec-cli/cmd/root"
+)
+
+func Execute(args []string) error {
+	c := root.NewRootCmd()
+	cmd.AddCommandsTo(c)
+	c.SetArgs(args)
+	c.SetOutput(io.Discard)
+	return c.Execute()
+}

benchmark/internal/untar/untar.go

@@ -0,0 +1,84 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package untar
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"errors"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+)
+
+func UnTar(a string) (string, error) {
+	dir, err := os.MkdirTemp("", "ec-benchmark")
+	if err != nil {
+		return "", err
+	}
+
+	archive, err := os.Open(a)
+	if err != nil {
+		return "", err
+	}
+	defer archive.Close()
+
+	gz, err := gzip.NewReader(archive)
+	if err != nil {
+		return "", err
+	}
+	defer gz.Close()
+
+	t := tar.NewReader(gz)
+	for {
+		hdr, err := t.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		dst := path.Join(dir, hdr.Name)
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			err = os.MkdirAll(dst, fs.FileMode(hdr.Mode))
+		case tar.TypeLink:
+			err = os.Symlink(path.Join(dir, hdr.Linkname), dst)
+		case tar.TypeReg:
+			var f io.WriteCloser
+			f, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, fs.FileMode(hdr.Mode))
+			if err != nil {
+				break
+			}
+
+			_, err = io.Copy(f, t)
+			if err != nil {
+				break
+			}
+			err = f.Close()
+		}
+
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return dir, nil
+}

benchmark/offliner/README.md

@@ -0,0 +1,8 @@
+# Offliner
+
+A tool to offline, i.e. place all data for an container image in a remote
+registry to a local data directory. This local data directory can be mounted to
+docker.io/registry container registry so running against it eliminates remote
+network access.
+
+Use: `offliner <pinned image reference> <data directory>`