feat: support image policy

Signed-off-by: HIHIA <[email protected]>

cmd/sealer/cmd/cluster/run.go

@@ -392,7 +392,7 @@ func installApplication(appImageName string, envs []string, app *v2.Application,
 		return loadToRegistry(infraDriver, distributor)
 	}
 
-	installer := clusterruntime.NewAppInstaller(infraDriver, distributor, extension)
+	installer := clusterruntime.NewAppInstaller(infraDriver, distributor, extension, imageEngine)
 	err = installer.Install(infraDriver.GetHostIPListByRole(common.MASTER)[0], v2App.GetImageLaunchCmds())
 	if err != nil {
 		return err

common/common.go

@@ -127,6 +127,72 @@ const (
 	WINDOWS            = "windows"
 )
 
+const (
+	ImagePolicyLabelKey         = "app.alpha.sealer.io/image-policy-plugin"
+	ImagePolicyPluginKyverno    = "kyverno"
+	ImagePolicyTemplateYamlName = "image-policy-template.yaml"
+	ImagePolicyTemplate         = `
+apiVersion : kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: [[.name]]-redirect-registry
+  namespace: kube-system
+spec:
+  background: false
+  rules:
+  - name: prepend-registry-containers
+    match:
+      resources:
+        kinds:
+        - Pod
+    preconditions:
+      all:
+      - key: "{{request.operation}}"
+        operator: In
+        value:
+        - CREATE
+        - UPDATE
+    mutate:
+      foreach:
+      - list: "request.object.spec.containers"
+        preconditions:
+          all:
+          - key: "{{element.image}}"
+            operator: AnyIn
+            value: [[.imageList]]
+        patchStrategicMerge:
+          spec:
+            containers:
+            - name: "{{ element.name }}"
+              image: "[[.registry]]/{{ images.containers.{{element.name}}.path}}:{{images.containers.{{element.name}}.tag}}"
+  - name: prepend-registry-initcontainers
+    match:
+      resources:
+        kinds:
+        - Pod
+    preconditions:
+      all:
+      - key: "{{request.operation}}"
+        operator: In
+        value:
+        - CREATE
+        - UPDATE
+    mutate:
+      foreach:
+      - list: "request.object.spec.initContainers"
+        preconditions:
+          all:
+          - key: "{{element.image}}"
+            operator: AnyIn
+            value: [[.imageList]]
+        patchStrategicMerge:
+          spec:
+            initContainers:
+            - name: "{{ element.name }}"
+              image: "[[.registry]]/{{ images.initContainers.{{element.name}}.path}}:{{images.initContainers.{{element.name}}.tag}}"
+`
+)
+
 func GetSealerWorkDir() string {
 	return filepath.Join(GetHomeDir(), ".sealer")
 }

pkg/cluster-runtime/apps.go

@@ -26,7 +26,10 @@ import (
 	"github.com/sealerio/sealer/common"
 	containerruntime "github.com/sealerio/sealer/pkg/container-runtime"
 	v12 "github.com/sealerio/sealer/pkg/define/image/v1"
+	"github.com/sealerio/sealer/pkg/define/options"
 	"github.com/sealerio/sealer/pkg/imagedistributor"
+	"github.com/sealerio/sealer/pkg/imageengine"
+	"github.com/sealerio/sealer/pkg/imagepolicy/kyverno"
 	"github.com/sealerio/sealer/pkg/infradriver"
 	"github.com/sealerio/sealer/pkg/registry"
 	"github.com/sirupsen/logrus"
@@ -36,13 +39,15 @@ type AppInstaller struct {
 	infraDriver infradriver.InfraDriver
 	distributor imagedistributor.Distributor
 	extension   v12.ImageExtension
+	imageEngine imageengine.Interface
 }
 
-func NewAppInstaller(infraDriver infradriver.InfraDriver, distributor imagedistributor.Distributor, extension v12.ImageExtension) AppInstaller {
+func NewAppInstaller(infraDriver infradriver.InfraDriver, distributor imagedistributor.Distributor, extension v12.ImageExtension, imageEngine imageengine.Interface) AppInstaller {
 	return AppInstaller{
 		infraDriver: infraDriver,
 		distributor: distributor,
 		extension:   extension,
+		imageEngine: imageEngine,
 	}
 }
 
@@ -90,6 +95,20 @@ func (i AppInstaller) Launch(master0 net.IP, launchCmds []string) error {
 		ex         = shell.NewLex('\\')
 	)
 
+	var imagePolicyEngine *kyverno.Engine
+	clusterImageName := i.infraDriver.GetClusterImageName()
+	extension, err := i.imageEngine.GetSealerImageExtension(&options.GetImageAnnoOptions{ImageNameOrID: clusterImageName})
+	if err != nil {
+		return err
+	}
+	val, ok := extension.Labels[common.ImagePolicyLabelKey]
+	if ok && val == common.ImagePolicyPluginKyverno {
+		imagePolicyEngine, err = kyverno.NewKyvernoImagePolicyEngine()
+		if err != nil {
+			return err
+		}
+	}
+
 	for _, value := range launchCmds {
 		if value == "" {
 			continue
@@ -102,6 +121,16 @@ func (i AppInstaller) Launch(master0 net.IP, launchCmds []string) error {
 		if err = i.infraDriver.CmdAsync(master0, fmt.Sprintf(common.CdAndExecCmd, rootfsPath, cmdline)); err != nil {
 			return err
 		}
+
+		if imagePolicyEngine != nil {
+			ok, err := imagePolicyEngine.IsImagePolicyApp(cmdline)
+			if err != nil {
+				return err
+			}
+			if ok {
+				imagePolicyEngine.CreateImagePolicyRule(i.infraDriver, i.imageEngine, cmdline)
+			}
+		}
 	}
 
 	return i.save(common.GetDefaultApplicationFile())

pkg/cluster-runtime/installer.go

@@ -202,7 +202,7 @@ func (i *Installer) Install() error {
 		return err
 	}
 
-	appInstaller := NewAppInstaller(i.infraDriver, i.Distributor, extension)
+	appInstaller := NewAppInstaller(i.infraDriver, i.Distributor, extension, i.ImageEngine)
 
 	v2App, err := application.NewV2Application(v2.ConstructApplication(i.Application, cmds, appNames), extension)
 	if err != nil {

pkg/imageengine/buildah/sealer_image_extension.go

@@ -51,6 +51,16 @@ func GetImageExtensionFromAnnotations(annotations map[string]string) (image_v1.I
 	return extension, nil
 }
 
+func (engine *Engine) GetSealerContainerImageList(opts *options.GetImageAnnoOptions) ([]*image_v1.ContainerImage, error) {
+	annotation, err := engine.GetImageAnnotation(opts)
+	result, err := GetContainerImagesFromAnnotations(annotation)
+	if err != nil {
+		return []*image_v1.ContainerImage{}, errors.Wrapf(err, "failed to get %s in image %s", image_v1.SealerImageContainerImageList, opts.ImageNameOrID)
+	}
+
+	return result, nil
+}
+
 func GetContainerImagesFromAnnotations(annotations map[string]string) ([]*image_v1.ContainerImage, error) {
 	var containerImageList []*image_v1.ContainerImage
 	annotationStr := annotations[image_v1.SealerImageContainerImageList]

pkg/imageengine/interface.go

@@ -61,6 +61,8 @@ type Interface interface {
 
 	GetSealerImageExtension(opts *options.GetImageAnnoOptions) (v1.ImageExtension, error)
 
+	GetSealerContainerImageList(opts *options.GetImageAnnoOptions) ([]*v1.ContainerImage, error)
+
 	LookupManifest(name string) (*libimage.ManifestList, error)
 
 	CreateManifest(name string, opts *options.ManifestCreateOpts) error

pkg/imagepolicy/common.go

@@ -0,0 +1,21 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imagepolicy
+
+import "github.com/sealerio/sealer/pkg/imagepolicy/kyverno"
+
+func NewImagePolicyEngine() (Interface, error) {
+	return kyverno.NewKyvernoImagePolicyEngine()
+}

pkg/imagepolicy/interface.go

@@ -0,0 +1,26 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imagepolicy
+
+import (
+	"github.com/sealerio/sealer/pkg/imageengine"
+	"github.com/sealerio/sealer/pkg/infradriver"
+)
+
+type Interface interface {
+	IsImagePolicyApp(appName string) (bool, error)
+
+	CreateImagePolicyRule(infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, appName string) error
+}

pkg/imagepolicy/kyverno/engine.go

@@ -0,0 +1,113 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kyverno
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	runtimeClient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	"github.com/sealerio/sealer/common"
+	imagecommon "github.com/sealerio/sealer/pkg/define/options"
+	"github.com/sealerio/sealer/pkg/imageengine"
+	"github.com/sealerio/sealer/pkg/infradriver"
+	"github.com/sealerio/sealer/pkg/rootfs"
+	"github.com/sealerio/sealer/pkg/runtime"
+	k "github.com/sealerio/sealer/pkg/runtime/kubernetes"
+	apimachineryRuntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+type Engine struct {
+	driver runtime.Driver
+}
+
+func NewKyvernoImagePolicyEngine() (*Engine, error) {
+	driver, err := k.NewKubeDriver(k.AdminKubeConfPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Engine{
+		driver: driver,
+	}, nil
+}
+
+// TODO: make it configurable
+func (engine *Engine) IsImagePolicyApp(appName string) (bool, error) {
+	return appName == "kyverno", nil
+}
+
+func (engine *Engine) CreateImagePolicyRule(infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, appName string) error {
+	imagePolicyTemplateStr := common.ImagePolicyTemplate
+	templatePath := filepath.Clean(filepath.Join(infraDriver.GetClusterRootfsPath(), rootfs.GlobalManager.App().Root(), appName, common.ImagePolicyTemplateYamlName))
+	if _, err := os.Stat(templatePath); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	} else {
+		yamlFile, err := ioutil.ReadFile(templatePath)
+		if err != nil {
+			return err
+		}
+		imagePolicyTemplateStr = string(yamlFile)
+	}
+
+	clusterImageName := infraDriver.GetClusterImageName()
+	imageList, err := imageEngine.GetSealerContainerImageList(&imagecommon.GetImageAnnoOptions{ImageNameOrID: clusterImageName})
+	clusterPolicy := &kyvernov1.ClusterPolicy{}
+
+	var imageListArray []string
+	for _, containerImage := range imageList {
+		imageListArray = append(imageListArray, "\""+containerImage.Image+"\"")
+	}
+
+	ctx := map[string]string{
+		"name":      clusterImageName,
+		"imageList": "[" + strings.Join(imageListArray, ",") + "]",
+		"registry":  infraDriver.GetClusterRegistry().LocalRegistry.Domain,
+	}
+	imagePolicyTemplate, err := SubsituteTemplate(imagePolicyTemplateStr, ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := kyvernov1.AddToScheme(scheme.Scheme); err != nil {
+		return err
+	}
+	decoder := serializer.NewCodecFactory(scheme.Scheme).UniversalDecoder()
+	if err := apimachineryRuntime.DecodeInto(decoder, []byte(imagePolicyTemplate), clusterPolicy); err != nil {
+		return err
+	}
+
+	if err := engine.driver.Create(context.Background(), clusterPolicy, &runtimeClient.CreateOptions{}); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("unable to create image policy: %v", err)
+		}
+
+		if err := engine.driver.Update(context.Background(), clusterPolicy, &runtimeClient.UpdateOptions{}); err != nil {
+			return fmt.Errorf("unable to update image policy: %v", err)
+		}
+	}
+	return nil
+}

pkg/imagepolicy/kyverno/util.go

@@ -0,0 +1,35 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kyverno
+
+import (
+	"bytes"
+	"text/template"
+)
+
+// SubsituteTemplate fills out the templates based on the context
+func SubsituteTemplate(tmpl string, context interface{}) (string, error) {
+	// Use [[]] as identifier to avoid conflicts
+	t, tmplPrsErr := template.New("test").Delims("[[", "]]").Option("missingkey=zero").Parse(tmpl)
+	if tmplPrsErr != nil {
+		return "", tmplPrsErr
+	}
+	writer := bytes.NewBuffer([]byte{})
+	if err := t.Execute(writer, context); nil != err {
+		return "", err
+	}
+
+	return writer.String(), nil
+}