feature: dfdaemon supports proxing https registries

Signed-off-by: lowzj <[email protected]>

cmd/dfdaemon/app/options/options.go

@@ -78,6 +78,10 @@ type Options struct {
 	// TrustHosts includes the trusted hosts which dfdaemon forward their
 	// requests directly when dfdaemon is used to http_proxy mode.
 	TrustHosts []string
+
+	// ConfigPath is the path of dfdaemon's configuration file.
+	// default value is: /etc/dragonfly/dfdaemon.yml
+	ConfigPath string
 }
 
 // NewOption returns the default options.
@@ -88,7 +92,6 @@ func NewOption() *Options {
 		if absPath, err := filepath.Abs(path); err == nil {
 			defaultPath = filepath.Dir(absPath) + "/dfget"
 		}
-
 	}
 
 	o := &Options{
@@ -120,4 +123,7 @@ func (o *Options) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&o.DownRule, "rule", "", "download the url by P2P if url matches the specified pattern,format:reg1,reg2,reg3")
 	fs.BoolVar(&o.Notbs, "notbs", true, "not try back source to download if throw exception")
 	fs.StringSliceVar(&o.TrustHosts, "trust-hosts", o.TrustHosts, "list of trusted hosts which dfdaemon forward their requests directly, comma separated.")
+
+	fs.StringVar(&o.ConfigPath, "config", "/etc/dragonfly/dfdaemon.yml",
+		"the path of dfdaemon's configuration file")
 }

dfdaemon/config/config.go

@@ -0,0 +1,149 @@
+/*
+ * Copyright The Dragonfly Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+)
+
+// -----------------------------------------------------------------------------
+// Properties
+
+// NewProperties create a new properties with default values.
+func NewProperties() *Properties {
+	return &Properties{}
+}
+
+// Properties holds all configurable properties of dfdaemon.
+// The default path is '/etc/dragonfly/dfdaemon.yml'
+type Properties struct {
+	Registries []*Registry `yaml:"registries"`
+}
+
+// Load loads properties from config file.
+func (p *Properties) Load(path string) error {
+	if err := p.loadFromYaml(path); err != nil {
+		return err
+	}
+	var tmp []*Registry
+	for _, v := range p.Registries {
+		if v != nil {
+			if err := v.init(); err != nil {
+				return err
+			}
+			tmp = append(tmp, v)
+		}
+	}
+	p.Registries = tmp
+	return nil
+}
+
+func (p *Properties) loadFromYaml(path string) error {
+	yamlFile, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("read yaml config from %s error: %v", path, err)
+	}
+	err = yaml.Unmarshal(yamlFile, p)
+	if err != nil {
+		return fmt.Errorf("unmarshal yaml error:%v", err)
+	}
+	return nil
+}
+
+// -----------------------------------------------------------------------------
+// Registry
+
+// NewRegistry create and init registry proxy with the giving values.
+func NewRegistry(schema, host, cert, regx string) (*Registry, error) {
+	reg := &Registry{
+		Schema: schema,
+		Host:   host,
+		Cert:   cert,
+		Regx:   regx,
+	}
+	if err := reg.init(); err != nil {
+		return nil, err
+	}
+	return reg, nil
+}
+
+// Registry is the proxied registry base information.
+type Registry struct {
+	// Schema can be 'http', 'https' or empty. It will use dfdaemon's schema if
+	// it's empty.
+	Schema string `yaml:"schema"`
+
+	// Host is the host of proxied registry, including ip and port.
+	Host string `yaml:"host"`
+
+	// Cert is the path of server-side certification. It should be provided when
+	// the 'Schema' is 'https' and the dfdaemon is worked on proxy pattern and
+	// the proxied registry is self-certificated.
+	// The server-side certification could be get by using this command:
+	// openssl x509 -in <(openssl s_client -showcerts -servername xxx -connect xxx:443 -prexit 2>/dev/null)
+	Cert string `yaml:"cert"`
+
+	// Regx is a regular expression, dfdaemon use this registry to process the
+	// requests whose host is matched.
+	Regx string `yaml:"regx"`
+
+	compiler  *regexp.Regexp
+	tlsConfig *tls.Config
+}
+
+// Match reports whether the Registry matches the string s.
+func (r *Registry) Match(s string) bool {
+	return r.compiler != nil && r.compiler.MatchString(s)
+}
+
+// TLSConfig returns a initialized tls.Config instance.
+func (r *Registry) TLSConfig() *tls.Config {
+	return r.tlsConfig
+}
+
+func (r *Registry) init() error {
+	c, err := regexp.Compile(r.Regx)
+	if err != nil {
+		return err
+	}
+	r.compiler = c
+	return r.initTLSConfig()
+}
+
+func (r *Registry) initTLSConfig() error {
+	if r.Cert == "" {
+		r.tlsConfig = &tls.Config{InsecureSkipVerify: true}
+		return nil
+	}
+
+	cert, err := ioutil.ReadFile(r.Cert)
+	if err != nil {
+		return err
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(cert) {
+		return fmt.Errorf("invalid certs:%s", r.Cert)
+	}
+	r.tlsConfig = &tls.Config{RootCAs: roots}
+	return nil
+}

dfdaemon/config/config_test.go

@@ -0,0 +1,130 @@
+/*
+ * Copyright The Dragonfly Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/go-check/check"
+)
+
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+func init() {
+	check.Suite(&ConfigTestSuite{})
+}
+
+type ConfigTestSuite struct {
+	workHome string
+}
+
+func (s *ConfigTestSuite) SetUpSuite(c *check.C) {
+	s.workHome, _ = ioutil.TempDir("/tmp", "dfget-ConfigTestSuite-")
+}
+
+func (s *ConfigTestSuite) TearDownSuite(c *check.C) {
+	if s.workHome != "" {
+		os.RemoveAll(s.workHome)
+	}
+}
+
+func (s *ConfigTestSuite) TestProperties_Load(c *check.C) {
+	var f = func(schema ...string) *Properties {
+		var tmp []*Registry
+		for _, s := range schema {
+			r := &Registry{Schema: s}
+			r.init()
+			tmp = append(tmp, r)
+		}
+		return &Properties{Registries: tmp}
+	}
+	var cases = []struct {
+		create   bool
+		content  string
+		errMsg   string
+		expected *Properties
+	}{
+		{create: false, content: "", errMsg: "read yaml", expected: nil},
+		{create: true, content: "-", errMsg: "unmarshal yaml", expected: nil},
+		{create: true, content: "registries:\n - regx: '^['",
+			errMsg: "missing closing", expected: nil},
+		{create: true, content: "registries:\n  -", errMsg: "", expected: f()},
+		{
+			create:   true,
+			content:  "registries:\n  - schema: http",
+			errMsg:   "",
+			expected: f("http"),
+		},
+	}
+	for idx, v := range cases {
+		filename := filepath.Join(s.workHome, fmt.Sprintf("test-%d", idx))
+		if v.create {
+			ioutil.WriteFile(filename, []byte(v.content), os.ModePerm)
+		}
+		p := &Properties{}
+		err := p.Load(filename)
+		if v.expected != nil {
+			c.Assert(err, check.IsNil)
+			c.Assert(len(p.Registries), check.Equals, len(v.expected.Registries))
+			for i := 0; i < len(v.expected.Registries); i++ {
+				c.Assert(p.Registries[i], check.DeepEquals, v.expected.Registries[i])
+			}
+		} else {
+			c.Assert(err, check.NotNil)
+			c.Assert(strings.Contains(err.Error(), v.errMsg), check.Equals, true,
+				check.Commentf("error:%v expected:%s", err, v.errMsg))
+		}
+	}
+}
+
+func (s *ConfigTestSuite) TestRegistry_Match(c *check.C) {
+	var cases = []struct {
+		regx      string
+		str       string
+		errNotNil bool
+		matched   bool
+	}{
+		{regx: "[a.com", str: "a.com", errNotNil: true, matched: false},
+		{regx: "a.com", str: "a.com", matched: true},
+		{regx: "a.com", str: "ba.com", matched: true},
+		{regx: "a.com", str: "a.comm", matched: true},
+		{regx: "^a.com", str: "ba.com", matched: false},
+		{regx: "^a.com$", str: "ba.com", matched: false},
+		{regx: "^a.com$", str: "a.comm", matched: false},
+		{regx: "^a.com$", str: "a.com", matched: true},
+		{regx: "", str: "a.com", matched: true},
+	}
+
+	for _, v := range cases {
+		reg := &Registry{Regx: v.regx}
+		err := reg.init()
+		if v.errNotNil {
+			c.Assert(err, check.NotNil)
+		} else {
+			c.Assert(err, check.IsNil)
+		}
+		c.Assert(reg.Match(v.str), check.Equals, v.matched,
+			check.Commentf("%v", v))
+	}
+}

dfdaemon/global/global.go

@@ -20,7 +20,9 @@ import (
 	"regexp"
 	"sync"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
 )
 
 // CommandParam is a struct that stores all the command line parameters
@@ -76,6 +78,9 @@ var (
 	// DFPattern is the url patterns. Dfdaemon starts downloading by P2P if the downloading url matches DFPattern.
 	DFPattern = make(map[string]*regexp.Regexp)
 
+	// Properties holds all properties get from dfdaemon's config file.
+	Properties *config.Properties
+
 	rwMutex sync.RWMutex
 )
 
@@ -89,7 +94,7 @@ func UpdateDFPattern(reg string) {
 	if compiledReg, err := regexp.Compile(reg); err == nil {
 		DFPattern[reg] = compiledReg
 	} else {
-		log.Warnf("pattern:%s is invalid", reg)
+		logrus.Warnf("pattern:%s is invalid", reg)
 	}
 }
 
@@ -114,7 +119,7 @@ func MatchDfPattern(location string) bool {
 			useGetter = true
 			break
 		}
-		log.Debugf("location:%s not match reg:%s", location, key)
+		logrus.Debugf("location:%s not match reg:%s", location, key)
 	}
 	return useGetter
 }