feat: Add plugin support for resource (dragonflyoss#291)

* feature: add plugin support for resource

Signed-off-by: Jim Ma <[email protected]>

* fix: skip link for mockgen import

Signed-off-by: Jim Ma <[email protected]>

* feature: add generate in Makefile

Signed-off-by: Jim Ma <[email protected]>

* fix: remove unused package

Signed-off-by: Jim Ma <[email protected]>

* fix: remove unused test

Signed-off-by: Jim Ma <[email protected]>

* feature: remove unused import package

Signed-off-by: Jim Ma <[email protected]>

Makefile

@@ -16,7 +16,7 @@ PROJECT_NAME := "d7y.io/dragonfly/v2"
 DFGET_NAME := "dfget"
 VERSION := "2.0.0"
 PKG := "$(PROJECT_NAME)"
-PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(manager\)')
+PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/manager/\)')
 GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7)
 GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD)
 DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(GIT_COMMIT)"
@@ -190,6 +190,11 @@ test-coverage:
 	@cat cover.out >> coverage.txt
 .PHONY: test-coverage
 
+# Run go generate
+generate:
+	@go generate ${PKG_LIST}
+.PHONY: generate
+
 # Generate changelog
 changelog:
 	@git-chglog -o CHANGELOG.md

internal/dfpath/dfpath.go

@@ -27,6 +27,7 @@ var (
 	DaemonSockPath = filepath.Join(WorkHome, "daemon.sock")
 	DaemonLockPath = filepath.Join(WorkHome, "daemon.lock")
 	DfgetLockPath  = filepath.Join(WorkHome, "dfget.lock")
+	PluginsDir     = filepath.Join(WorkHome, "plugins")
 )
 
 func init() {

internal/dfplugin/dfplugin.go

@@ -0,0 +1,84 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dfplugin
+
+import (
+	"errors"
+	"fmt"
+	"path"
+	"plugin"
+
+	"d7y.io/dragonfly/v2/internal/dfpath"
+)
+
+const (
+	// PluginFormat indicates the lookup name of a plugin in plugin directory.
+	PluginFormat = "d7y-%s-plugin-%s.so"
+
+	// PluginInitFuncName indicates the function `DragonflyPluginInit` must be implemented in plugin
+	PluginInitFuncName = "DragonflyPluginInit"
+
+	// PluginMetaKeyType indicates the type of a plugin, currently support: resource
+	PluginMetaKeyType = "type"
+	// PluginMetaKeyName indicates the name of a plugin
+	PluginMetaKeyName = "name"
+)
+
+type PluginType string
+
+const (
+	PluginTypeResource = PluginType("resource")
+)
+
+type PluginInitFunc func(option map[string]string) (plugin interface{}, meta map[string]string, err error)
+
+func Load(typ PluginType, name string, option map[string]string) (interface{}, map[string]string, error) {
+	soName := fmt.Sprintf(PluginFormat, string(typ), name)
+	p, err := plugin.Open(path.Join(dfpath.PluginsDir, soName))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	symbol, err := p.Lookup(PluginInitFuncName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// FIXME when use symbol.(PluginInitFunc), ok is always false
+	f, ok := symbol.(func(option map[string]string) (plugin interface{}, meta map[string]string, err error))
+	if !ok {
+		return nil, nil, errors.New("invalid plugin init function signature")
+	}
+
+	i, meta, err := f(option)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if meta == nil {
+		return nil, nil, errors.New("empty plugin metadata")
+	}
+
+	if meta[PluginMetaKeyType] != string(typ) {
+		return nil, nil, errors.New("plugin type not match")
+	}
+
+	if meta[PluginMetaKeyName] != name {
+		return nil, nil, errors.New("plugin name not match")
+	}
+	return i, meta, nil
+}

pkg/source/plugin.go

@@ -0,0 +1,42 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package source
+
+import (
+	"errors"
+
+	"d7y.io/dragonfly/v2/internal/dfplugin"
+)
+
+const (
+	pluginMetadataSchema = "schema"
+)
+
+func LoadPlugin(schema string) (ResourceClient, error) {
+	// TODO init option
+	client, meta, err := dfplugin.Load(dfplugin.PluginTypeResource, schema, map[string]string{})
+	if err != nil {
+		return nil, err
+	}
+	if meta[pluginMetadataSchema] != schema {
+		return nil, errors.New("support schema not match")
+	}
+	if rc, ok := client.(ResourceClient); ok {
+		return rc, err
+	}
+	return nil, errors.New("invalid client, not a ResourceClient")
+}

pkg/source/plugin_test.go

@@ -0,0 +1,75 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package source
+
+import (
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+
+	testifyassert "github.com/stretchr/testify/assert"
+)
+
+func Test_loadPlugin(t *testing.T) {
+	assert := testifyassert.New(t)
+	defer func() {
+		os.Remove("./testdata/d7y-resource-plugin-dfs.so")
+		os.Remove("./testdata/test")
+	}()
+
+	var (
+		cmd    *exec.Cmd
+		output []byte
+		wd     string
+		err    error
+	)
+
+	// TODO can not load golang plugin in testing, because the different building flags
+	// golang runtime will check the runtime hash of all imported packages
+
+	// build plugin
+	cmd = exec.Command("go", "build", "-buildmode=plugin", "-o=./testdata/d7y-resource-plugin-dfs.so", "testdata/plugin/dfs.go")
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+
+	// build test binary
+	cmd = exec.Command("go", "build", "-o=./testdata/test", "testdata/main.go")
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+
+	wd, err = os.Getwd()
+	assert.Nil(err)
+	wd = path.Join(wd, "testdata")
+
+	// execute test binary
+	cmd = exec.Command("./testdata/test", "-plugin-dir", wd)
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+}

pkg/source/source_client.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	logger "d7y.io/dragonfly/v2/internal/dflog"
@@ -62,6 +63,7 @@ type ClientManager interface {
 }
 
 type ClientManagerImpl struct {
+	sync.RWMutex
 	clients map[string]ResourceClient
 }
 
@@ -197,9 +199,28 @@ func (clientMgr *ClientManagerImpl) getSourceClient(rawURL string) (ResourceClie
 	if err != nil {
 		return nil, err
 	}
+	clientMgr.RLock()
 	client, ok := clientMgr.clients[strings.ToLower(parsedURL.Scheme)]
+	clientMgr.RUnlock()
 	if !ok || client == nil {
 		return nil, fmt.Errorf("can not find client for supporting url %s, clients:%v", rawURL, clientMgr.clients)
 	}
 	return client, nil
 }
+
+func (clientMgr *ClientManagerImpl) loadSourcePlugin(schema string) (ResourceClient, error) {
+	clientMgr.Lock()
+	defer clientMgr.Unlock()
+	// double check
+	client, ok := clientMgr.clients[schema]
+	if ok {
+		return client, nil
+	}
+
+	client, err := LoadPlugin(schema)
+	if err != nil {
+		return nil, err
+	}
+	clientMgr.clients[schema] = client
+	return client, nil
+}

pkg/source/testdata/main.go

@@ -0,0 +1,73 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"d7y.io/dragonfly/v2/internal/dfpath"
+	"d7y.io/dragonfly/v2/pkg/source"
+)
+
+func init() {
+	flag.StringVar(&dfpath.PluginsDir, "plugin-dir", ".", "")
+}
+
+func main() {
+	flag.Parse()
+
+	client, err := source.LoadPlugin("dfs")
+	if err != nil {
+		fmt.Printf("load plugin error: %s\n", err)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+
+	l, err := client.GetContentLength(ctx, "", nil)
+	if err != nil {
+		fmt.Printf("get content length error: %s\n", err)
+		os.Exit(1)
+	}
+
+	rc, err := client.Download(ctx, "", nil)
+	if err != nil {
+		fmt.Printf("download error: %s\n", err)
+		os.Exit(1)
+	}
+
+	data, err := ioutil.ReadAll(rc)
+	if err != nil {
+		fmt.Printf("read error: %s\n", err)
+		os.Exit(1)
+	}
+
+	if l != int64(len(data)) {
+		fmt.Printf("content length mismatch\n")
+		os.Exit(1)
+	}
+
+	err = rc.Close()
+	if err != nil {
+		fmt.Printf("close error: %s\n", err)
+		os.Exit(1)
+	}
+}