From 4a3e82c8b8e6aef96fa3f43e73da93a2558676fb Mon Sep 17 00:00:00 2001 From: Jim Ma Date: Thu, 1 Jul 2021 17:08:26 +0800 Subject: [PATCH] feat: Add plugin support for resource (#291) * feature: add plugin support for resource Signed-off-by: Jim Ma * fix: skip link for mockgen import Signed-off-by: Jim Ma * feature: add generate in Makefile Signed-off-by: Jim Ma * fix: remove unused package Signed-off-by: Jim Ma * fix: remove unused test Signed-off-by: Jim Ma * feature: remove unused import package Signed-off-by: Jim Ma --- Makefile | 7 ++- internal/dfpath/dfpath.go | 1 + internal/dfplugin/dfplugin.go | 84 +++++++++++++++++++++++++++++++ pkg/source/plugin.go | 42 ++++++++++++++++ pkg/source/plugin_test.go | 75 +++++++++++++++++++++++++++ pkg/source/source_client.go | 21 ++++++++ pkg/source/testdata/main.go | 73 +++++++++++++++++++++++++++ pkg/source/testdata/plugin/dfs.go | 61 ++++++++++++++++++++++ 8 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 internal/dfplugin/dfplugin.go create mode 100644 pkg/source/plugin.go create mode 100644 pkg/source/plugin_test.go create mode 100644 pkg/source/testdata/main.go create mode 100644 pkg/source/testdata/plugin/dfs.go diff --git a/Makefile b/Makefile index 1b7a6c9f6..a40647d93 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROJECT_NAME := "d7y.io/dragonfly/v2" DFGET_NAME := "dfget" VERSION := "2.0.0" PKG := "$(PROJECT_NAME)" -PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(manager\)') +PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/manager/\)') GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7) GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD) DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(GIT_COMMIT)" @@ -190,6 +190,11 @@ test-coverage: @cat cover.out >> coverage.txt .PHONY: test-coverage +# Run go generate +generate: + @go generate ${PKG_LIST} +.PHONY: generate + # Generate changelog changelog: @git-chglog -o CHANGELOG.md diff --git a/internal/dfpath/dfpath.go b/internal/dfpath/dfpath.go index 24ec651e0..b9158afab 100644 --- a/internal/dfpath/dfpath.go +++ b/internal/dfpath/dfpath.go @@ -27,6 +27,7 @@ var ( DaemonSockPath = filepath.Join(WorkHome, "daemon.sock") DaemonLockPath = filepath.Join(WorkHome, "daemon.lock") DfgetLockPath = filepath.Join(WorkHome, "dfget.lock") + PluginsDir = filepath.Join(WorkHome, "plugins") ) func init() { diff --git a/internal/dfplugin/dfplugin.go b/internal/dfplugin/dfplugin.go new file mode 100644 index 000000000..5f11212c1 --- /dev/null +++ b/internal/dfplugin/dfplugin.go @@ -0,0 +1,84 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfplugin + +import ( + "errors" + "fmt" + "path" + "plugin" + + "d7y.io/dragonfly/v2/internal/dfpath" +) + +const ( + // PluginFormat indicates the lookup name of a plugin in plugin directory. + PluginFormat = "d7y-%s-plugin-%s.so" + + // PluginInitFuncName indicates the function `DragonflyPluginInit` must be implemented in plugin + PluginInitFuncName = "DragonflyPluginInit" + + // PluginMetaKeyType indicates the type of a plugin, currently support: resource + PluginMetaKeyType = "type" + // PluginMetaKeyName indicates the name of a plugin + PluginMetaKeyName = "name" +) + +type PluginType string + +const ( + PluginTypeResource = PluginType("resource") +) + +type PluginInitFunc func(option map[string]string) (plugin interface{}, meta map[string]string, err error) + +func Load(typ PluginType, name string, option map[string]string) (interface{}, map[string]string, error) { + soName := fmt.Sprintf(PluginFormat, string(typ), name) + p, err := plugin.Open(path.Join(dfpath.PluginsDir, soName)) + if err != nil { + return nil, nil, err + } + + symbol, err := p.Lookup(PluginInitFuncName) + if err != nil { + return nil, nil, err + } + + // FIXME when use symbol.(PluginInitFunc), ok is always false + f, ok := symbol.(func(option map[string]string) (plugin interface{}, meta map[string]string, err error)) + if !ok { + return nil, nil, errors.New("invalid plugin init function signature") + } + + i, meta, err := f(option) + if err != nil { + return nil, nil, err + } + + if meta == nil { + return nil, nil, errors.New("empty plugin metadata") + } + + if meta[PluginMetaKeyType] != string(typ) { + return nil, nil, errors.New("plugin type not match") + } + + if meta[PluginMetaKeyName] != name { + return nil, nil, errors.New("plugin name not match") + } + return i, meta, nil +} diff --git a/pkg/source/plugin.go b/pkg/source/plugin.go new file mode 100644 index 000000000..292cf421a --- /dev/null +++ b/pkg/source/plugin.go @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package source + +import ( + "errors" + + "d7y.io/dragonfly/v2/internal/dfplugin" +) + +const ( + pluginMetadataSchema = "schema" +) + +func LoadPlugin(schema string) (ResourceClient, error) { + // TODO init option + client, meta, err := dfplugin.Load(dfplugin.PluginTypeResource, schema, map[string]string{}) + if err != nil { + return nil, err + } + if meta[pluginMetadataSchema] != schema { + return nil, errors.New("support schema not match") + } + if rc, ok := client.(ResourceClient); ok { + return rc, err + } + return nil, errors.New("invalid client, not a ResourceClient") +} diff --git a/pkg/source/plugin_test.go b/pkg/source/plugin_test.go new file mode 100644 index 000000000..d262a1f67 --- /dev/null +++ b/pkg/source/plugin_test.go @@ -0,0 +1,75 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package source + +import ( + "os" + "os/exec" + "path" + "testing" + + testifyassert "github.com/stretchr/testify/assert" +) + +func Test_loadPlugin(t *testing.T) { + assert := testifyassert.New(t) + defer func() { + os.Remove("./testdata/d7y-resource-plugin-dfs.so") + os.Remove("./testdata/test") + }() + + var ( + cmd *exec.Cmd + output []byte + wd string + err error + ) + + // TODO can not load golang plugin in testing, because the different building flags + // golang runtime will check the runtime hash of all imported packages + + // build plugin + cmd = exec.Command("go", "build", "-buildmode=plugin", "-o=./testdata/d7y-resource-plugin-dfs.so", "testdata/plugin/dfs.go") + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } + + // build test binary + cmd = exec.Command("go", "build", "-o=./testdata/test", "testdata/main.go") + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } + + wd, err = os.Getwd() + assert.Nil(err) + wd = path.Join(wd, "testdata") + + // execute test binary + cmd = exec.Command("./testdata/test", "-plugin-dir", wd) + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } +} diff --git a/pkg/source/source_client.go b/pkg/source/source_client.go index 7e5ebd2e3..f7202929d 100644 --- a/pkg/source/source_client.go +++ b/pkg/source/source_client.go @@ -23,6 +23,7 @@ import ( "io" "net/url" "strings" + "sync" "time" logger "d7y.io/dragonfly/v2/internal/dflog" @@ -62,6 +63,7 @@ type ClientManager interface { } type ClientManagerImpl struct { + sync.RWMutex clients map[string]ResourceClient } @@ -197,9 +199,28 @@ func (clientMgr *ClientManagerImpl) getSourceClient(rawURL string) (ResourceClie if err != nil { return nil, err } + clientMgr.RLock() client, ok := clientMgr.clients[strings.ToLower(parsedURL.Scheme)] + clientMgr.RUnlock() if !ok || client == nil { return nil, fmt.Errorf("can not find client for supporting url %s, clients:%v", rawURL, clientMgr.clients) } return client, nil } + +func (clientMgr *ClientManagerImpl) loadSourcePlugin(schema string) (ResourceClient, error) { + clientMgr.Lock() + defer clientMgr.Unlock() + // double check + client, ok := clientMgr.clients[schema] + if ok { + return client, nil + } + + client, err := LoadPlugin(schema) + if err != nil { + return nil, err + } + clientMgr.clients[schema] = client + return client, nil +} diff --git a/pkg/source/testdata/main.go b/pkg/source/testdata/main.go new file mode 100644 index 000000000..5d5ea2403 --- /dev/null +++ b/pkg/source/testdata/main.go @@ -0,0 +1,73 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "os" + + "d7y.io/dragonfly/v2/internal/dfpath" + "d7y.io/dragonfly/v2/pkg/source" +) + +func init() { + flag.StringVar(&dfpath.PluginsDir, "plugin-dir", ".", "") +} + +func main() { + flag.Parse() + + client, err := source.LoadPlugin("dfs") + if err != nil { + fmt.Printf("load plugin error: %s\n", err) + os.Exit(1) + } + + ctx := context.Background() + + l, err := client.GetContentLength(ctx, "", nil) + if err != nil { + fmt.Printf("get content length error: %s\n", err) + os.Exit(1) + } + + rc, err := client.Download(ctx, "", nil) + if err != nil { + fmt.Printf("download error: %s\n", err) + os.Exit(1) + } + + data, err := ioutil.ReadAll(rc) + if err != nil { + fmt.Printf("read error: %s\n", err) + os.Exit(1) + } + + if l != int64(len(data)) { + fmt.Printf("content length mismatch\n") + os.Exit(1) + } + + err = rc.Close() + if err != nil { + fmt.Printf("close error: %s\n", err) + os.Exit(1) + } +} diff --git a/pkg/source/testdata/plugin/dfs.go b/pkg/source/testdata/plugin/dfs.go new file mode 100644 index 000000000..80dd05b0e --- /dev/null +++ b/pkg/source/testdata/plugin/dfs.go @@ -0,0 +1,61 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "context" + "io" + "io/ioutil" + + "d7y.io/dragonfly/v2/pkg/source" +) + +var data = "hello world" + +var _ source.ResourceClient = (*client)(nil) + +type client struct { +} + +func (c *client) GetContentLength(ctx context.Context, url string, header source.RequestHeader) (int64, error) { + return int64(len(data)), nil +} + +func (c *client) IsSupportRange(ctx context.Context, url string, header source.RequestHeader) (bool, error) { + return false, nil +} + +func (c *client) IsExpired(ctx context.Context, url string, header source.RequestHeader, expireInfo map[string]string) (bool, error) { + panic("implement me") +} + +func (c *client) Download(ctx context.Context, url string, header source.RequestHeader) (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewBufferString(data)), nil +} + +func (c *client) DownloadWithResponseHeader(ctx context.Context, url string, header source.RequestHeader) (io.ReadCloser, source.ResponseHeader, error) { + return ioutil.NopCloser(bytes.NewBufferString(data)), map[string]string{}, nil +} + +func (c *client) GetLastModifiedMillis(ctx context.Context, url string, header source.RequestHeader) (int64, error) { + panic("implement me") +} + +func DragonflyPluginInit(option map[string]string) (interface{}, map[string]string, error) { + return &client{}, map[string]string{"type": "resource", "name": "dfs", "schema": "dfs"}, nil +}