From f8095bcb9c8d6fd09f3c21a3a0b2c2cdf6ccb067 Mon Sep 17 00:00:00 2001 From: Ox Cart Date: Tue, 25 Jan 2022 17:24:39 -0600 Subject: [PATCH] Added matomo configuration to global template --- config/features_test.go | 45 +++- config_v2/ads.go | 79 ++++++ config_v2/ads_test.go | 41 +++ config_v2/bootstrap.go | 105 ++++++++ config_v2/bootstrap_test.go | 58 +++++ config_v2/client_config.go | 102 ++++++++ config_v2/common_test.go | 163 ++++++++++++ config_v2/config.go | 324 ++++++++++++++++++++++++ config_v2/config_test.go | 257 +++++++++++++++++++ config_v2/configchecker/main.go | 148 +++++++++++ config_v2/embedded_global_test.go | 22 ++ config_v2/features.go | 408 ++++++++++++++++++++++++++++++ config_v2/features_test.go | 201 +++++++++++++++ config_v2/fetcher.go | 162 ++++++++++++ config_v2/fetcher_test.go | 54 ++++ config_v2/global.go | 152 +++++++++++ config_v2/initializer.go | 236 +++++++++++++++++ config_v2/initializer_test.go | 127 ++++++++++ embeddedconfig/global.yaml.tmpl | 6 + 19 files changed, 2676 insertions(+), 14 deletions(-) create mode 100644 config_v2/ads.go create mode 100644 config_v2/ads_test.go create mode 100644 config_v2/bootstrap.go create mode 100644 config_v2/bootstrap_test.go create mode 100644 config_v2/client_config.go create mode 100644 config_v2/common_test.go create mode 100644 config_v2/config.go create mode 100644 config_v2/config_test.go create mode 100644 config_v2/configchecker/main.go create mode 100644 config_v2/embedded_global_test.go create mode 100644 config_v2/features.go create mode 100644 config_v2/features_test.go create mode 100644 config_v2/fetcher.go create mode 100644 config_v2/fetcher_test.go create mode 100644 config_v2/global.go create mode 100644 config_v2/initializer.go create mode 100644 config_v2/initializer_test.go diff --git a/config/features_test.go b/config/features_test.go index e1320ceb06..f88e1693e9 100644 --- a/config/features_test.go +++ b/config/features_test.go @@ -161,10 +161,24 @@ featureoptions: require.Nil(t, mat.Config["k1"]) } +func TestMatomoConfigured(t *testing.T) { + opts := getAnalyticsOptions(t) + matomo := opts.GetProvider(MATOMO) + require.NotNil(t, matomo) + require.NotZero(t, matomo.SampleRate) + require.Equal(t, 1, matomo.Config["idsite"]) +} + +func getAnalyticsOptions(t *testing.T) (opts AnalyticsOptions) { + var g Global + require.NoError(t, yaml.Unmarshal(getGlobalTemplate(t), &g)) + require.NoError(t, g.UnmarshalFeatureOptions(FeatureAnalytics, &opts)) + return +} + func TestReplicaByCountry(t *testing.T) { - require := require.New(t) assert := assert.New(t) - fos := getReplicaOptionsRoot(require) + fos := getReplicaOptionsRoot(t) assert.Contains(fos.ByCountry, "RU") assert.NotContains(fos.ByCountry, "AU") assert.NotEmpty(fos.ByCountry) @@ -177,20 +191,9 @@ func TestReplicaByCountry(t *testing.T) { assert.Equal(fos.ByCountry["IR"].Trackers, globalTrackers) } -func getReplicaOptionsRoot(require *require.Assertions) (fos ReplicaOptionsRoot) { - var w bytes.Buffer - // We could write into a pipe, but that requires concurrency and we're old-school in tests. - require.NoError(template.Must(template.New("").Parse(embeddedconfig.GlobalTemplate)).Execute(&w, nil)) - var g Global - require.NoError(yaml.Unmarshal(w.Bytes(), &g)) - require.NoError(g.UnmarshalFeatureOptions(FeatureReplica, &fos)) - return -} - func TestReplicaProxying(t *testing.T) { - require := require.New(t) assert := assert.New(t) - fos := getReplicaOptionsRoot(require) + fos := getReplicaOptionsRoot(t) numInfohashes := len(fos.ProxyAnnounceTargets) // The default is to announce as a proxy. assert.True(numInfohashes > 0) @@ -199,3 +202,17 @@ func TestReplicaProxying(t *testing.T) { // Iran looks for peers from the default countries. assert.Len(fos.ByCountry["IR"].ProxyPeerInfoHashes, numInfohashes) } + +func getReplicaOptionsRoot(t *testing.T) (fos ReplicaOptionsRoot) { + var g Global + require.NoError(t, yaml.Unmarshal(getGlobalTemplate(t), &g)) + require.NoError(t, g.UnmarshalFeatureOptions(FeatureReplica, &fos)) + return +} + +func getGlobalTemplate(t *testing.T) []byte { + var w bytes.Buffer + // We could write into a pipe, but that requires concurrency and we're old-school in tests. + require.NoError(t, template.Must(template.New("").Parse(embeddedconfig.GlobalTemplate)).Execute(&w, nil)) + return w.Bytes() +} diff --git a/config_v2/ads.go b/config_v2/ads.go new file mode 100644 index 0000000000..74a53f2f07 --- /dev/null +++ b/config_v2/ads.go @@ -0,0 +1,79 @@ +package config + +import ( + "math/rand" + "strings" +) + +const ( + none = "none" + free = "free" + pro = "pro" +) + +// AdSettings are settings to use when showing ads to Android clients +type AdSettings struct { + NativeBannerZoneID string `yaml:"nativebannerzoneid,omitempty"` + StandardBannerZoneID string `yaml:"standardbannerzoneid,omitempty"` + InterstitialZoneID string `yaml:"interstitialzoneid,omitempty"` + DaysToSuppress int `yaml:"daystosuppress,omitempty"` + Percentage float64 + Countries map[string]string +} + +type AdProvider struct { + AdSettings +} + +func (s *AdSettings) GetAdProvider(isPro bool, countryCode string, daysSinceInstalled int) *AdProvider { + if !s.adsEnabled(isPro, countryCode, daysSinceInstalled) { + return nil + } + + return &AdProvider{*s} +} + +func (s *AdSettings) adsEnabled(isPro bool, countryCode string, daysSinceInstalled int) bool { + if s == nil { + return false + } + + if daysSinceInstalled < s.DaysToSuppress { + return false + } + + level := s.Countries[strings.ToLower(countryCode)] + switch level { + case free: + return !isPro + case pro: + return true + default: + return false + } +} + +func (p *AdProvider) GetNativeBannerZoneID() string { + if p == nil { + return "" + } + return p.NativeBannerZoneID +} + +func (p *AdProvider) GetStandardBannerZoneID() string { + if p == nil { + return "" + } + return p.StandardBannerZoneID +} + +func (p *AdProvider) GetInterstitialZoneID() string { + if p == nil { + return "" + } + return p.InterstitialZoneID +} + +func (p *AdProvider) ShouldShowAd() bool { + return rand.Float64() <= p.Percentage/100 +} diff --git a/config_v2/ads_test.go b/config_v2/ads_test.go new file mode 100644 index 0000000000..93aad757aa --- /dev/null +++ b/config_v2/ads_test.go @@ -0,0 +1,41 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAdSettings(t *testing.T) { + s := &AdSettings{ + NativeBannerZoneID: "a", + StandardBannerZoneID: "b", + InterstitialZoneID: "c", + DaysToSuppress: 1, + Percentage: 100, + Countries: map[string]string{ + "ir": "pro", + "ae": "free", + "us": "none", + "uk": "wrong", + }, + } + + assert.True(t, s.adsEnabled(true, "IR", 1)) + assert.True(t, s.adsEnabled(false, "IR", 1)) + assert.False(t, s.adsEnabled(true, "AE", 1)) + assert.True(t, s.adsEnabled(false, "AE", 1)) + assert.False(t, s.adsEnabled(false, "AE", 0)) + assert.False(t, s.adsEnabled(false, "US", 1)) + assert.False(t, s.adsEnabled(false, "UK", 1)) + assert.False(t, s.adsEnabled(false, "Ru", 1)) + + p := s.GetAdProvider(false, "IR", 1) + if assert.NotNil(t, p) { + if assert.True(t, p.ShouldShowAd()) { + assert.Equal(t, s.NativeBannerZoneID, p.GetNativeBannerZoneID()) + assert.Equal(t, s.StandardBannerZoneID, p.GetStandardBannerZoneID()) + assert.Equal(t, s.InterstitialZoneID, p.GetInterstitialZoneID()) + } + } +} diff --git a/config_v2/bootstrap.go b/config_v2/bootstrap.go new file mode 100644 index 0000000000..10974a07f2 --- /dev/null +++ b/config_v2/bootstrap.go @@ -0,0 +1,105 @@ +package config + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/getlantern/yaml" + + "github.com/getlantern/flashlight/common" +) + +var ( + name = ".packaged-lantern.yaml" + lanternYamlName = "lantern.yaml" +) + +// BootstrapSettings provides access to configuration embedded directly in Lantern installation +// packages. On OSX, that means data embedded in the Lantern.app app bundle in +// Lantern.app/Contents/Resources/.lantern.yaml, while on Windows that means data embedded +// in AppData/Roaming/Lantern/.lantern.yaml. This allows customization embedded in the +// installer outside of the auto-updated binary that should only be used under special +// circumstances. +type BootstrapSettings struct { + StartupUrl string +} + +// ReadBootstrapSettings reads packaged settings from pre-determined paths +// on the various OSes. +func ReadBootstrapSettings(configDir string) (*BootstrapSettings, error) { + _, yamlPath, err := bootstrapPath(name) + if err != nil { + return &BootstrapSettings{}, err + } + + ps, er := readSettingsFromFile(yamlPath) + if er != nil { + // This is the local copy of our embedded ration file. This is necessary + // to ensure we remember the embedded ration across auto-updated + // binaries. We write to the local file system instead of to the package + // itself (app bundle on OSX, install directory on Windows) because + // we're not always sure we can write to that directory. + return readSettingsFromFile(filepath.Join(configDir, name)) + } + return ps, nil +} + +// ReadSettingsFromFile reads BootstrapSettings from the yaml file at the specified +// path. +func readSettingsFromFile(yamlPath string) (*BootstrapSettings, error) { + log.Debugf("Opening file at: %v", yamlPath) + data, err := ioutil.ReadFile(yamlPath) + if err != nil { + // This will happen whenever there's no packaged settings, which is often + log.Debugf("Error reading file %v", err) + return &BootstrapSettings{}, err + } + + trimmed := strings.TrimSpace(string(data)) + + log.Debugf("Read bytes: %v", trimmed) + + if trimmed == "" { + log.Debugf("Ignoring empty string") + return &BootstrapSettings{}, errors.New("Empty string") + } + var s BootstrapSettings + err = yaml.Unmarshal([]byte(trimmed), &s) + + if err != nil { + log.Errorf("Could not read yaml: %v", err) + return &BootstrapSettings{}, err + } + return &s, nil +} + +func bootstrapPath(fileName string) (string, string, error) { + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + log.Errorf("Could not get current directory %v", err) + return "", "", err + } + var yamldir string + if common.Platform == "windows" { + yamldir = dir + } else if common.Platform == "darwin" { + // Code signing doesn't like this file in the current directory + // for whatever reason, so we grab it from the Resources/en.lproj + // directory in the app bundle. See: + // https://developer.apple.com/library/mac/technotes/tn2206/_index.html#//apple_ref/doc/uid/DTS40007919-CH1-TNTAG402 + yamldir = dir + "/../Resources/en.lproj" + if _, err := ioutil.ReadDir(yamldir); err != nil { + // This likely means the user originally installed with an older version that didn't include en.lproj + // in the app bundle, so just look in the old location in Resources. + yamldir = dir + "/../Resources" + } + } else if common.Platform == "linux" { + yamldir = dir + "/../" + } + fullPath := filepath.Join(yamldir, fileName) + log.Debugf("Opening bootstrap file from: %v", fullPath) + return yamldir, fullPath, nil +} diff --git a/config_v2/bootstrap_test.go b/config_v2/bootstrap_test.go new file mode 100644 index 0000000000..9a53074bc7 --- /dev/null +++ b/config_v2/bootstrap_test.go @@ -0,0 +1,58 @@ +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/getlantern/yaml" + "github.com/stretchr/testify/assert" + + "github.com/getlantern/flashlight/common" +) + +func TestBootstrapSettings(t *testing.T) { + file, err := ioutil.TempFile("", ".packaged-lantern.yaml") + defer func() { + err := os.Remove(file.Name()) + if err != nil { + log.Errorf("Could not remove file? %v", err) + } + }() + assert.True(t, err == nil, "Should not be an error") + file.Close() + + log.Debugf("File at: %v", file.Name()) + settings := BootstrapSettings{StartupUrl: "test"} + log.Debugf("Settings: %v", settings) + + data, er := yaml.Marshal(&settings) + assert.True(t, er == nil, "Should not be an error") + + e := ioutil.WriteFile(file.Name(), data, 0644) + assert.True(t, e == nil, "Should not be an error") + + ps, errr := readSettingsFromFile(file.Name()) + assert.Equal(t, "test", ps.StartupUrl, "Unexpected startup URL") + assert.NoError(t, errr, "Unable to read settings") + + _, path, err := bootstrapPath(name) + assert.True(t, err == nil, "Should not be an error") + + var dir string + + if common.Platform == "darwin" { + dir, err = filepath.Abs(filepath.Dir(os.Args[0]) + "/../Resources") + } else if common.Platform == "linux" { + dir, err = filepath.Abs(filepath.Dir(os.Args[0]) + "/../") + } + assert.True(t, err == nil, "Should not be an error") + + log.Debugf("Running in %v", dir) + if common.Platform == "darwin" { + assert.Equal(t, dir+"/"+name, path, "Unexpected settings dir") + } else if common.Platform == "linux" { + assert.Equal(t, dir+"/"+name, path, "Unexpected settings dir") + } +} diff --git a/config_v2/client_config.go b/config_v2/client_config.go new file mode 100644 index 0000000000..e6e19756df --- /dev/null +++ b/config_v2/client_config.go @@ -0,0 +1,102 @@ +package config + +import ( + "errors" + + "github.com/getlantern/fronted" +) + +const ( + // for historical reasons, if a provider is unspecified in a masquerade, it + // is treated as a cloudfront masquerade (which was once the only provider) + DefaultFrontedProviderID = "cloudfront" +) + +// ClientConfig captures configuration information for a Client +type ClientConfig struct { + DumpHeaders bool // whether or not to dump headers of requests and responses + Fronted *FrontedConfig + + // Legacy masquerade configuration + // included to test presence for older clients + MasqueradeSets map[string][]*fronted.Masquerade +} + +// Configuration structure for direct domain fronting +type FrontedConfig struct { + Providers map[string]*ProviderConfig +} + +// Configuration structure for a particular fronting provider (cloudfront, akamai, etc) +type ProviderConfig struct { + HostAliases map[string]string + TestURL string + Masquerades []*fronted.Masquerade + Validator *ValidatorConfig + PassthroughPatterns []string +} + +// returns a fronted.ResponseValidator specified by the +// provider config or nil if none was specified +func (p *ProviderConfig) GetResponseValidator(providerID string) fronted.ResponseValidator { + // hard-coded custom validators can be determined here if needed... + + if p.Validator == nil { + return nil + } + + if len(p.Validator.RejectStatus) > 0 { + return fronted.NewStatusCodeValidator(p.Validator.RejectStatus) + } + // ... + + // unknown or empty + return nil +} + +// Configuration struture that specifies a fronted.ResponseValidator +type ValidatorConfig struct { + RejectStatus []int +} + +func newFrontedConfig() *FrontedConfig { + return &FrontedConfig{ + Providers: make(map[string]*ProviderConfig), + } +} + +// NewClientConfig creates a new client config with default values. +func NewClientConfig() *ClientConfig { + return &ClientConfig{ + Fronted: newFrontedConfig(), + } +} + +// Builds a list of fronted.Providers to use based on the configuration +func (c *ClientConfig) FrontedProviders() map[string]*fronted.Provider { + + providers := make(map[string]*fronted.Provider) + for pid, p := range c.Fronted.Providers { + providers[pid] = fronted.NewProvider( + p.HostAliases, + p.TestURL, + p.Masquerades, + p.GetResponseValidator(pid), + p.PassthroughPatterns, + ) + } + return providers +} + +// Check that this ClientConfig is valid +func (c *ClientConfig) Validate() error { + sz := 0 + for _, p := range c.Fronted.Providers { + sz += len(p.Masquerades) + } + if sz == 0 { + return errors.New("No masquerades.") + } + + return nil +} diff --git a/config_v2/common_test.go b/config_v2/common_test.go new file mode 100644 index 0000000000..abd9e0a33f --- /dev/null +++ b/config_v2/common_test.go @@ -0,0 +1,163 @@ +package config + +import ( + "compress/gzip" + "io/ioutil" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + + "github.com/getlantern/rot13" + "github.com/getlantern/yaml" + + "github.com/getlantern/flashlight/chained" +) + +// withTempDir creates a temporary directory, executes the given function and +// then cleans up the temporary directory. inTempDir allows modifying filenames +// to put them in the temporary directory. +func withTempDir(t *testing.T, fn func(inTempDir func(file string) string)) { + tmpDir, err := ioutil.TempDir("", "test") + abortOnError(t, err) + defer os.RemoveAll(tmpDir) + fn(func(file string) string { + return filepath.Join(tmpDir, file) + }) +} + +// writeObfuscatedConfig serializes the given config onto disk as YAML with +// ROT13 encoding. +func writeObfuscatedConfig(t *testing.T, config interface{}, obfuscatedFilename string) { + log.Debugf("Writing obfuscated config to %v", obfuscatedFilename) + + bytes, err := yaml.Marshal(config) + abortOnError(t, err) + + // create new obfuscated file + outfile, err := os.OpenFile(obfuscatedFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + abortOnError(t, err) + defer outfile.Close() + + // write ROT13-encoded config to obfuscated file + out := rot13.NewWriter(outfile) + _, err = out.Write(bytes) + abortOnError(t, err) +} + +func startConfigServer(t *testing.T, config interface{}) (u string, reqCount func() int64) { + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Unable to listen: %s", err) + } + + requests := int64(0) + hs := &http.Server{ + Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + atomic.AddInt64(&requests, 1) + bytes, _ := yaml.Marshal(config) + w := gzip.NewWriter(resp) + defer w.Close() + w.Write(bytes) + }), + } + go func() { + if err = hs.Serve(l); err != nil { + t.Errorf("Unable to serve: %v", err) + } + }() + + port := l.Addr().(*net.TCPAddr).Port + url := "http://localhost:" + strconv.Itoa(port) + return url, func() int64 { + return atomic.LoadInt64(&requests) + } +} + +func newGlobalConfig(t *testing.T) *Global { + global := &Global{} + err := yaml.Unmarshal([]byte(globalYamlTemplate), global) + abortOnError(t, err) + return global +} + +func newProxiesConfig(t *testing.T) map[string]*chained.ChainedServerInfo { + proxies := make(map[string]*chained.ChainedServerInfo) + err := yaml.Unmarshal([]byte(proxiesYamlTemplate), proxies) + abortOnError(t, err) + return proxies +} + +// abortOnError aborts the current test if there's an error +func abortOnError(t *testing.T, err error) { + if err != nil { + abortOnError(t, err) + } +} + +// Certain tests fetch global config from a remote server and store it at +// `global.yaml`. Other tests rely on `global.yaml` matching the +// `fetched-global.yaml` fixture. For tests that fetch config remotely, we must +// delete the config file once the test has completed to avoid causing other +// these other tests to fail in the event that the remote config differs from +// the fixture. +func deleteGlobalConfig() { + os.Remove("global.yaml") +} + +const globalYamlTemplate = ` +version: 0 +cloudconfigca: "" +autoupdateca: "" +updateserverurl: "" +bordareportinterval: 0s +bordasamplepercentage: 0 +pingsamplepercentage: 0 +reportissueemail: "" +client: + dumpheaders: false + fronted: + providers: + cloudfront: + hostaliases: + xyz.getiantem.org: xyz.cloudfront.net + masquerades: &id001 + - domain: cloudfront.net + ipaddress: 54.182.1.120 + testurl: http://zyx.cloudfront.net/ping + validator: + rejectstatus: + - 403 + masqueradesets: + cloudflare: [] + cloudfront: *id001 +adsettings: null +proxiedsites: + delta: + additions: [] + deletions: [] + cloud: + - 0000a-fast-proxy.de +trustedcas: +- commonname: VeriSign Class 3 Public Primary Certification Authority - G5 + cert: "-----BEGIN CERTIFICATE-----\nMIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB\nyjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL\nExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp\nU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW\nZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0\naG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL\nMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW\nZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln\nbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp\nU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y\naXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1\nnmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex\nt0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz\nSdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG\nBO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+\nrCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/\nNIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E\nBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH\nBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy\naXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv\nMzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE\np6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y\n5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK\nWE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ\n4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N\nhnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq\n-----END + CERTIFICATE-----\n" +globalconfigpollinterval: 3s +proxyconfigpollinterval: 1s +` + +const proxiesYamlTemplate = ` +fallback-104.236.192.114: + addr: 104.236.192.114:443 + cert: "-----BEGIN CERTIFICATE-----\nMIIDoDCCAoigAwIBAgIEJSGW+TANBgkqhkiG9w0BAQsFADB4MQswCQYDVQQGEwJVUzETMBEGA1UE\nCBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHQWJ1c2VyczEhMB8GA1UEChMYUnVtcGVsc3RpbHRza2lu\nIEJsZW5kaW5nMR8wHQYDVQQDExZOYWlyIENvaW5hZ2VzIFJvb3RsZXNzMB4XDTE1MTIxMDE5MjI0\nNloXDTE2MTIwOTE5MjI0NloweDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAO\nBgNVBAcTB0FidXNlcnMxITAfBgNVBAoTGFJ1bXBlbHN0aWx0c2tpbiBCbGVuZGluZzEfMB0GA1UE\nAxMWTmFpciBDb2luYWdlcyBSb290bGVzczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nAKjJSnSh8ErJZhxppJLoee80dvMB0RB9xjjXkvhCB/k/PHSsGHzHQp2ywuD50RoMlFq5tBL10Nnx\nBQ8a56lqXdwAfJD74dN2ppA/wyzQ1KFRRp9kZb4l2jry6GArexHS1fnLEh6XgkEf4DLp4FdsBgWk\nB7EMJRh8HRYZLNZXZz+EUx+a9cIRFAoHDYg/CfzhqGk4qC2Wkoty/7LP72dIo4nC5ynzLNX/3HIQ\nTh+6Qt9KxJDC6OyWvputZc2bYcxeEVzx2/3FNsJtuuPiw8kIG1Ji6t+jlFJaE/82LweYPONmbiAd\nHIlXhFamy46dHAWgqG/iNPeQCkKyNbpS1/UAoKUCAwEAAaMyMDAwDwYDVR0RBAgwBocEaOzAcjAd\nBgNVHQ4EFgQUt93T+OqvcAvacIs3c0Qmj7KGp3AwDQYJKoZIhvcNAQELBQADggEBAJq5sYbq9wOl\nEcc87B56GJlLr6ZktGR7vQEvNsMq2YJwv1U4ZuSCuKx3IcuB4i+bvMWcaZRomNhiDbI07GLxYI3L\nSUbjJ4O/MJmUTb/KnmloRYPFPie6nq3sdAePCYwFUPLrz4RhOmII/nxWUqIoMvEFOHN+zRgr2s7n\npAFeLQ5PnWbPovfKCMi+imHlMSSBAWXQnLhfKUmkKfW1libcyV+MOjyhalQNFxHwkgNugLKlh7FN\nnvQl5FfTwrn5y+m3K5CIzFvnz3j7KdKtK3vfbA/Makbi4wc2/Gn2aMcFibFPBJyfSQ1QQkdRVtE4\n5lED/8D2Ekj3qRCtsGnuszJ6Fdk=\n-----END + CERTIFICATE-----\n" + authtoken: OlTIG6BtHaSDC2Iu2FmM7Xv3SYoP1HjjqpRXFGU7Q7719uRwJpQfTKPBfbN020SZ + trusted: true + pluggabletransport: "" + pluggabletransportsettings: {} + kcpsettings: {} + enhttpurl: "" +` diff --git a/config_v2/config.go b/config_v2/config.go new file mode 100644 index 0000000000..73c48798a7 --- /dev/null +++ b/config_v2/config.go @@ -0,0 +1,324 @@ +package config + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "reflect" + "time" + + "github.com/getlantern/rot13" + "github.com/getlantern/yaml" + + "github.com/getlantern/flashlight/common" + "github.com/getlantern/flashlight/ops" +) + +// Source specifies where the config is from when dispatching. +type Source string + +const ( + Saved Source = "saved" + Embedded Source = "embedded" + Fetched Source = "fetched" +) + +// Config is an interface for getting proxy data saved locally, embedded +// in the binary, or fetched over the network. +type Config interface { + + // Saved returns a yaml config from disk. + saved() (interface{}, error) + + // Embedded retrieves a yaml config embedded in the binary. + embedded([]byte) (interface{}, error) + + // Poll polls for new configs from a remote server and saves them to disk for + // future runs. + poll(stopCh chan bool, dispatch func(interface{}), fetcher Fetcher, sleep func() time.Duration) +} + +type config struct { + filePath string + obfuscate bool + saveChan chan interface{} + unmarshaler func([]byte) (interface{}, error) +} + +// options specifies the options to use for piping config data back to the +// dispatch processor function. +type options struct { + + // saveDir is the directory where we should save new configs and also look + // for existing saved configs. + saveDir string + + // onSaveError is called when an error occurs saving the config. May be nil. + onSaveError func(error) + + // obfuscate specifies whether or not to obfuscate the config on disk. + obfuscate bool + + // name specifies the name of the config file both on disk and in the + // embedded config that uses tarfs (the same in the interest of using + // configuration by convention). + name string + + // URL to use for fetching this config. + originURL string + + // userConfig contains data for communicating the user details to upstream + // servers in HTTP headers, such as the pro token and other options. + userConfig common.UserConfig + + // marshaler marshals application specific config to bytes, defaults to + // yaml.Marshal + marshaler func(interface{}) ([]byte, error) + // unmarshaler unmarshals application specific data structure. + unmarshaler func([]byte) (interface{}, error) + + // dispatch is essentially a callback function for processing retrieved + // yaml configs. + dispatch func(cfg interface{}, src Source) + + // embeddedData is the data for embedded configs, using tarfs. + embeddedData []byte + + // sleep the time to sleep between config fetches. + sleep func() time.Duration + + // sticky specifies whether or not to only use the local config and not + // update it with remote data. + sticky bool + + // rt provides the RoundTripper the fetcher should use, which allows us to + // dictate whether the fetcher will use dual fetching (from fronted and + // chained URLs) or not. + rt http.RoundTripper +} + +// pipeConfig creates a new config pipeline for reading a specified type of +// config onto a channel for processing by a dispatch function. This will read +// configs in the following order: +// +// 1. Configs saved on disk, if any +// 2. Configs embedded in the binary according to the specified name, if any. +// 3. Configs fetched remotely, and those will be piped back over and over +// again as the remote configs change (but only if they change). +// +// pipeConfig returns a function that can be used to stop polling +func pipeConfig(opts *options) (stop func()) { + stopCh := make(chan bool) + + // lastCfg is accessed by both the current goroutine when dispatching + // saved or embedded configs, and in a separate goroutine for polling + // for remote configs. There should never be mutual access by these + // goroutines, however, since the polling routine is started after the prior + // calls to dispatch return. + var lastCfg interface{} + dispatch := func(cfg interface{}, src Source) { + a := lastCfg + b := yamlRoundTrip(cfg) + if reflect.DeepEqual(a, b) { + log.Debug("Config unchanged, ignoring") + } else { + log.Debug("Dispatching updated config") + opts.dispatch(cfg, src) + lastCfg = b + } + } + + configPath := filepath.Join(opts.saveDir, opts.name) + + log.Tracef("Obfuscating %v", opts.obfuscate) + conf := newConfig(configPath, opts) + + if saved, proxyErr := conf.saved(); proxyErr != nil { + log.Debugf("Could not load stored config %v", proxyErr) + if embedded, errr := conf.embedded(opts.embeddedData); errr != nil { + log.Errorf("Could not load embedded config %v", errr) + } else { + log.Debugf("Sending embedded config for %v", opts.name) + dispatch(embedded, Embedded) + } + } else { + log.Debugf("Sending saved config for %v", opts.name) + dispatch(saved, Saved) + } + + // Now continually poll for new configs and pipe them back to the dispatch + // function. + if !opts.sticky { + fetcher := newFetcher(opts.userConfig, opts.rt, opts.originURL) + go conf.poll(stopCh, func(cfg interface{}) { + dispatch(cfg, Fetched) + }, fetcher, opts.sleep) + } else { + log.Debugf("Using sticky config") + } + + return func() { + close(stopCh) + } +} + +func yamlRoundTrip(o interface{}) interface{} { + if o == nil { + return nil + } + var or interface{} + t := reflect.TypeOf(o) + if t.Kind() == reflect.Ptr { + or = reflect.New(t.Elem()).Interface() + } else { + or = reflect.New(t).Interface() + } + b, err := yaml.Marshal(o) + if err != nil { + log.Errorf("Unable to yaml round trip (marshal): %v %v", o, err) + return o + } + err = yaml.Unmarshal(b, or) + if err != nil { + log.Errorf("Unable to yaml round trip (unmarshal): %v %v", o, err) + return o + } + return or +} + +// newConfig create a new ProxyConfig instance that saves and looks for +// saved data at the specified path. +func newConfig(filePath string, + opts *options, +) Config { + cfg := &config{ + filePath: filePath, + obfuscate: opts.obfuscate, + saveChan: make(chan interface{}), + unmarshaler: opts.unmarshaler, + } + if cfg.unmarshaler == nil { + cfg.unmarshaler = func([]byte) (interface{}, error) { + return nil, errors.New("No unmarshaler") + } + } + + // Start separate go routine that saves newly fetched proxies to disk. + go cfg.save(opts.onSaveError) + return cfg +} + +func (conf *config) saved() (interface{}, error) { + infile, err := os.Open(conf.filePath) + if err != nil { + err = fmt.Errorf("Unable to open config file %v for reading: %v", conf.filePath, err) + log.Error(err.Error()) + return nil, err + } + defer infile.Close() + + var in io.Reader = infile + if conf.obfuscate { + in = rot13.NewReader(infile) + } + + bytes, err := ioutil.ReadAll(in) + if err != nil { + err = fmt.Errorf("Error reading config from %v: %v", conf.filePath, err) + log.Error(err.Error()) + return nil, err + } + if len(bytes) == 0 { + return nil, fmt.Errorf("Config exists but is empty at %v", conf.filePath) + } + + log.Debugf("Returning saved config at %v", conf.filePath) + return conf.unmarshaler(bytes) +} + +func (conf *config) embedded(data []byte) (interface{}, error) { + return conf.unmarshaler(data) +} + +func (conf *config) poll(stopCh chan bool, dispatch func(interface{}), fetcher Fetcher, defaultSleep func() time.Duration) { + for { + sleepDuration := conf.fetchConfig(stopCh, dispatch, fetcher) + if sleepDuration == noSleep { + sleepDuration = defaultSleep() + } + select { + case <-stopCh: + log.Debug("Stopping polling") + return + case <-time.After(sleepDuration): + continue + } + } +} + +func (conf *config) fetchConfig(stopCh chan bool, dispatch func(interface{}), fetcher Fetcher) time.Duration { + if bytes, sleepTime, err := fetcher.fetch(); err != nil { + log.Errorf("Error fetching config: %v", err) + return sleepTime + } else if bytes == nil { + // This is what fetcher returns for not-modified. + log.Debug("Ignoring not modified response") + return sleepTime + } else if cfg, err := conf.unmarshaler(bytes); err != nil { + log.Errorf("Error fetching config: %v", err) + return sleepTime + } else { + log.Debugf("Fetched config!") + + // Push these to channels to avoid race conditions that might occur if + // we did these on go routines, for example. + conf.saveChan <- cfg + log.Debugf("Sent to save chan") + dispatch(cfg) + return sleepTime + } +} + +func (conf *config) save(onError func(error)) { + for { + in := <-conf.saveChan + if err := conf.saveOne(in); err != nil && onError != nil { + // Handle the error in a goroutine to avoid blocking the save loop. + go onError(err) + } + } +} + +func (conf *config) saveOne(in interface{}) error { + op := ops.Begin("save_config") + defer op.End() + return op.FailIf(conf.doSaveOne(in)) +} + +func (conf *config) doSaveOne(in interface{}) error { + bytes, err := yaml.Marshal(in) + if err != nil { + return fmt.Errorf("Unable to marshal config yaml: %v", err) + } + + outfile, err := os.OpenFile(conf.filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("Unable to open file %v for writing: %v", conf.filePath, err) + } + defer outfile.Close() + + var out io.Writer = outfile + if conf.obfuscate { + out = rot13.NewWriter(outfile) + } + _, err = out.Write(bytes) + if err != nil { + return fmt.Errorf("Unable to write yaml to file %v: %v", conf.filePath, err) + } + log.Debugf("Wrote file at %v", conf.filePath) + return nil +} diff --git a/config_v2/config_test.go b/config_v2/config_test.go new file mode 100644 index 0000000000..ea0f0bb30e --- /dev/null +++ b/config_v2/config_test.go @@ -0,0 +1,257 @@ +package config + +import ( + "errors" + "io/ioutil" + "net/http" + "os" + "testing" + "time" + + "github.com/getlantern/fronted" + "github.com/getlantern/golog" + "github.com/stretchr/testify/assert" + + "github.com/getlantern/flashlight/chained" + "github.com/getlantern/flashlight/common" + "github.com/getlantern/flashlight/embeddedconfig" +) + +// TestInvalidFile test an empty or malformed config file +func TestInvalidFile(t *testing.T) { + logger := golog.LoggerFor("config-test") + + tmpfile, err := ioutil.TempFile("", "invalid-test-file") + if err != nil { + logger.Fatal(err) + } + defer os.Remove(tmpfile.Name()) // clean up + + configPath := tmpfile.Name() + + logger.Debugf("path: %v", configPath) + conf := newConfig(configPath, &options{}) + _, proxyErr := conf.saved() + assert.Error(t, proxyErr, "should get error if config file is empty") + + tmpfile.WriteString("content: anything") + tmpfile.Sync() + var expectedError = errors.New("invalid content") + conf = newConfig(configPath, &options{ + unmarshaler: func([]byte) (interface{}, error) { + return nil, expectedError + }, + }) + _, proxyErr = conf.saved() + assert.Equal(t, expectedError, proxyErr, + "should get application specific unmarshal error") +} + +// TestObfuscated tests reading obfuscated global config from disk +func TestObfuscated(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + file := inTempDir("global.yaml") + globalConfig := newGlobalConfig(t) + writeObfuscatedConfig(t, globalConfig, file) + + config := newConfig(file, &options{ + obfuscate: true, + unmarshaler: newGlobalUnmarshaler(nil), + }) + + conf, err := config.saved() + assert.Nil(t, err) + + cfg := conf.(*Global) + + // Just make sure it's legitimately reading the config. + assert.True(t, len(cfg.Client.MasqueradeSets) > 1) + }) +} + +// TestSaved tests reading stored proxies from disk +func TestSaved(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + file := inTempDir("proxies.yaml") + proxiesConfig := newProxiesConfig(t) + writeObfuscatedConfig(t, proxiesConfig, file) + + cfg := newConfig(file, &options{ + obfuscate: true, + unmarshaler: newProxiesUnmarshaler(), + }) + + pr, err := cfg.saved() + assert.Nil(t, err) + + proxies := pr.(map[string]*chained.ChainedServerInfo) + chained := proxies["fallback-104.236.192.114"] + assert.True(t, chained != nil) + assert.Equal(t, "104.236.192.114:443", chained.Addr) + }) +} + +// TestEmbedded tests reading stored proxies from disk +func TestEmbedded(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + file := inTempDir("proxies.yaml") + + cfg := newConfig(file, &options{ + unmarshaler: newProxiesUnmarshaler(), + }) + + _, err := cfg.embedded(embeddedconfig.Proxies) + assert.NotNil(t, err) + }) +} + +func TestPollProxies(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + fronted.ConfigureForTest(t) + + file := inTempDir("proxies.yaml") + proxyConfig := newProxiesConfig(t) + writeObfuscatedConfig(t, proxyConfig, file) + + proxyChan := make(chan interface{}) + cfg := newConfig(file, &options{ + unmarshaler: newProxiesUnmarshaler(), + }) + var fi os.FileInfo + var err error + for i := 1; i <= 400; i++ { + fi, err = os.Stat(file) + if err == nil { + break + } + time.Sleep(200 * time.Millisecond) + } + if !assert.Nil(t, err) { + return + } + + mtime := fi.ModTime() + os.Remove(file) + + proxyConfigURLs, _ := startConfigServer(t, proxyConfig) + fetcher := newFetcher(newTestUserConfig(), &http.Transport{}, proxyConfigURLs) + dispatch := func(cfg interface{}) { + proxyChan <- cfg + } + go cfg.poll(nil, dispatch, fetcher, func() time.Duration { return 1 * time.Hour }) + proxies := (<-proxyChan).(map[string]*chained.ChainedServerInfo) + + assert.True(t, len(proxies) > 0) + for _, val := range proxies { + assert.True(t, val != nil) + assert.True(t, len(val.Addr) > 6) + } + + for i := 1; i <= 400; i++ { + fi, err = os.Stat(file) + if err == nil && fi != nil && fi.ModTime().After(mtime) { + //log.Debugf("Got newer mod time?") + break + } + time.Sleep(50 * time.Millisecond) + } + + fi, err = os.Stat(file) + + assert.NotNil(t, fi) + assert.Nil(t, err, "Got error: %v", err) + + assert.True(t, fi.ModTime().After(mtime)) + }) +} + +// TestProductionGlobal validates certain properties of the live production global config +func TestProductionGlobal(t *testing.T) { + + testURL := common.GlobalURL // this should always point to the live production configuration (not staging etc) + + expectedProviders := map[string]bool{ + "cloudfront": true, + "akamai": true, + } + + f := newFetcher(newTestUserConfig(), &http.Transport{}, testURL) + + cfgBytes, _, err := f.fetch() + if !assert.NoError(t, err, "Error fetching global config from %s", testURL) { + return + } + + unmarshal := newGlobalUnmarshaler(nil) + cfgIf, err := unmarshal(cfgBytes) + if !assert.NoError(t, err, "Error unmarshaling global config from %s", testURL) { + return + } + + cfg, ok := cfgIf.(*Global) + if !assert.True(t, ok, "Unexpected configuration type returned from %s", testURL) { + return + } + + defaultMasq := cfg.Client.MasqueradeSets["cloudfront"] + assert.True(t, len(defaultMasq) > 500, "global config %s should have a large number of default masquerade sets for cloudfront (found %d)", testURL, len(defaultMasq)) + + if !assert.NotNil(t, cfg.Client.Fronted, "global config %s missing fronted section!", testURL) { + return + } + + for pid := range expectedProviders { + provider := cfg.Client.Fronted.Providers[pid] + if !assert.NotNil(t, provider, "global config %s missing expected fronted provider %s", testURL, pid) { + continue + } + assert.True(t, len(provider.Masquerades) > 100, "global config %s provider %s had only %d masquerades!", testURL, pid, len(provider.Masquerades)) + assert.True(t, len(provider.HostAliases) > 0, "global config %s provider %s has no host aliases?", testURL, pid) + + } + + for pid := range cfg.Client.Fronted.Providers { + assert.True(t, expectedProviders[pid], "global config %s had unexpected provider %s (update expected list?)", testURL, pid) + } +} + +func TestPollIntervals(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + fronted.ConfigureForTest(t) + + file := inTempDir("global.yaml") + globalConfig := newGlobalConfig(t) + writeObfuscatedConfig(t, globalConfig, file) + + cfg := newConfig(file, &options{ + unmarshaler: newGlobalUnmarshaler(nil), + }) + var err error + for i := 1; i <= 400; i++ { + _, err = os.Stat(file) + if err == nil { + break + } + time.Sleep(200 * time.Millisecond) + } + if !assert.Nil(t, err) { + return + } + + os.Remove(file) + + configURLs, reqCount := startConfigServer(t, globalConfig) + pollInterval := 500 * time.Millisecond + waitTime := pollInterval*2 + (200 * time.Millisecond) + + fetcher := newFetcher(newTestUserConfig(), &http.Transport{}, configURLs) + dispatch := func(cfg interface{}) {} + + stopChan := make(chan bool) + go cfg.poll(stopChan, dispatch, fetcher, func() time.Duration { return pollInterval }) + time.Sleep(waitTime) + close(stopChan) + + assert.Equal(t, 3, int(reqCount()), "should have fetched config every %v", pollInterval) + }) +} diff --git a/config_v2/configchecker/main.go b/config_v2/configchecker/main.go new file mode 100644 index 0000000000..c70a14e793 --- /dev/null +++ b/config_v2/configchecker/main.go @@ -0,0 +1,148 @@ +// This program is a simple config checker tool for proxies and global configs. It prints out basic +// statistics and round-trips the low-cardinality portions of the config in order to make sure the +// YAML parses as expected. The tool can either read a local config file (assumed to be plain text) +// or a remote http(s) URL (assumed to be gzipped). +// +// Examples: +// +// go run main.go proxies ~/Library/Application\ Support/Lantern/proxies.yaml +// go run main.go global ~/Library/Application\ Support/Lantern/global.yaml +// go run main.go global https://globalconfig.flashlightproxy.com/global.yaml.gz +// +package main + +import ( + "compress/gzip" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + + "github.com/getlantern/flashlight/chained" + "github.com/getlantern/flashlight/config" + "github.com/getlantern/flashlight/domainrouting" + "github.com/getlantern/yaml" +) + +func main() { + if len(os.Args) < 3 { + fail("Please specify a format ('proxies' or 'global') and a filename or http(s) url for the config to check") + } + + format := os.Args[1] + target := os.Args[2] + + u, err := url.Parse(target) + if err != nil { + fail("Unable to parse config url %v: %v", target, err) + } + + var bytes []byte + var readErr error + + switch u.Scheme { + case "": + bytes, readErr = ioutil.ReadFile(target) + case "http", "https": + bytes, readErr = readRemote(target) + default: + fail("Unrecognized url scheme: %v", u.Scheme) + } + + if readErr != nil { + fail("Unable to read %v: %v", target, readErr) + } + + log("Checking %v config at %v", format, target) + + switch format { + case "proxies": + parseProxies(bytes) + case "global": + parseGlobal(bytes) + default: + fail("Unknown format %v, please specify either 'proxies' or 'global'", format) + } +} + +func readRemote(target string) ([]byte, error) { + resp, err := http.Get(target) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Bad response status: %d", resp.StatusCode) + } + gzr, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + return ioutil.ReadAll(gzr) +} + +func parseProxies(bytes []byte) { + cfg := make(map[string]*chained.ChainedServerInfo) + err := yaml.Unmarshal(bytes, cfg) + if err != nil { + fail("Unable to parse proxies config: %v", err) + } + log("Number of proxies: %d", len(cfg)) + out, err := yaml.Marshal(cfg) + if err != nil { + fail("Unable to marshal proxies config") + } + + log("------ Round-tripped YAML ------") + os.Stdout.Write(out) +} + +func parseGlobal(bytes []byte) { + cfg := &config.Global{} + err := yaml.Unmarshal(bytes, cfg) + if err != nil { + fail("Unable to parse global config: %v", err) + } + + log("Number of Proxied Sites: %d", len(cfg.ProxiedSites.Cloud)) + + var direct, proxied int + for _, rule := range cfg.DomainRoutingRules { + switch rule { + case domainrouting.Direct: + direct++ + case domainrouting.Proxy: + proxied++ + } + } + log("Domainrouting direct: %d", direct) + log("Domainrouting proxied: %d", proxied) + + for name, provider := range cfg.Client.FrontedProviders() { + log("Masquerades for %v: %d", name, len(provider.Masquerades)) + } + + // Clear out high cardinality data before marshaling + cfg.ProxiedSites = nil + cfg.DomainRoutingRules = nil + cfg.NamedDomainRoutingRules = nil + cfg.Client.Fronted.Providers = nil + + out, err := yaml.Marshal(cfg) + if err != nil { + fail("Unable to marshal global config") + } + + log("------ Round-tripped YAML ------") + os.Stdout.Write(out) +} + +func log(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", args...) +} + +func fail(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", args...) + os.Exit(1) +} diff --git a/config_v2/embedded_global_test.go b/config_v2/embedded_global_test.go new file mode 100644 index 0000000000..b57d82b07d --- /dev/null +++ b/config_v2/embedded_global_test.go @@ -0,0 +1,22 @@ +package config + +import ( + "testing" + + "github.com/getlantern/flashlight/embeddedconfig" + "github.com/stretchr/testify/assert" +) + +func TestEmbeddedGlobal(t *testing.T) { + + globalFunc := newGlobalUnmarshaler(make(map[string]interface{})) + + global, err := globalFunc(embeddedconfig.Global) + assert.NoError(t, err) + + gl := global.(*Global) + assert.True(t, len(gl.Client.Fronted.Providers["akamai"].Masquerades) > 20) + assert.True(t, len(gl.Client.Fronted.Providers["cloudfront"].Masquerades) > 20) + assert.Containsf(t, gl.Client.Fronted.Providers["cloudfront"].HostAliases, "replica-search.lantern.io", "embedded global config does not contain replica-search cloudfront fronted provider") + assert.Containsf(t, gl.Client.Fronted.Providers["akamai"].HostAliases, "replica-search.lantern.io", "embedded global config does not contain replica-search akamai fronted provider") +} diff --git a/config_v2/features.go b/config_v2/features.go new file mode 100644 index 0000000000..7d811274c6 --- /dev/null +++ b/config_v2/features.go @@ -0,0 +1,408 @@ +package config + +import ( + "fmt" + "math/rand" + "reflect" + "regexp" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + + "github.com/blang/semver" + + "github.com/getlantern/errors" + "github.com/getlantern/flashlight/common" +) + +const ( + FeatureAuth = "auth" + FeatureProxyBench = "proxybench" + FeaturePingProxies = "pingproxies" + FeatureTrafficLog = "trafficlog" + FeatureNoBorda = "noborda" + FeatureNoProbeProxies = "noprobeproxies" + FeatureNoShortcut = "noshortcut" + FeatureNoDetour = "nodetour" + FeatureNoHTTPSEverywhere = "nohttpseverywhere" + FeatureReplica = "replica" + FeatureProxyWhitelistedOnly = "proxywhitelistedonly" + FeatureTrackYouTube = "trackyoutube" + FeatureGoogleSearchAds = "googlesearchads" + FeatureYinbiWallet = "yinbiwallet" + FeatureYinbi = "yinbi" + FeatureAnalytics = "analytics" +) + +var ( + // to have stable calculation of fraction until the client restarts. + randomFloat = rand.Float64() + + errAbsentOption = errors.New("option is absent") + errMalformedOption = errors.New("malformed option") +) + +// FeatureOptions is an interface implemented by all feature options +type FeatureOptions interface { + fromMap(map[string]interface{}) error +} + +type AnalyticsProvider struct { + SampleRate float32 + Endpoint string + Config map[string]interface{} +} + +// AnalyticsOptions is the configuration for analytics providers such as Google Analytics or Matomo. +type AnalyticsOptions struct { + // Providers maps provider names to their sampling rates. + Providers map[string]*AnalyticsProvider +} + +const GA = "ga" +const MATOMO = "matomo" + +func (ao *AnalyticsOptions) fromMap(m map[string]interface{}) error { + return mapstructure.Decode(m, &ao) +} + +func (ao *AnalyticsOptions) GetProvider(key string) *AnalyticsProvider { + return ao.Providers[key] +} + +type ReplicaOptionsRoot struct { + // This is the default. + ReplicaOptions `mapstructure:",squash"` + // Options tailored to country. This could be used to pattern match any arbitrary string really. + // mapstructure should ignore the field name. + ByCountry map[string]ReplicaOptions `mapstructure:",remain"` + // Deprecated. An unmatched country uses the embedded ReplicaOptions.ReplicaRustEndpoint. + // Removing this will break unmarshalling config. + ReplicaRustDefaultEndpoint string + // Deprecated. Use ByCountry.ReplicaRustEndpoint. + ReplicaRustEndpoints map[string]string +} + +func (ro *ReplicaOptionsRoot) fromMap(m map[string]interface{}) error { + return mapstructure.Decode(m, ro) +} + +type ReplicaOptions struct { + // Use infohash and old-style prefixing simultaneously for now. Later, the old-style can be removed. + WebseedBaseUrls []string + Trackers []string + StaticPeerAddrs []string + // Merged with the webseed URLs when the metadata and data buckets are merged. + MetadataBaseUrls []string + // The replica-rust endpoint to use. There's only one because object uploads and ownership are + // fixed to a specific bucket, and replica-rust endpoints are 1:1 with a bucket. + ReplicaRustEndpoint string + // A set of info hashes (20 bytes, hex-encoded) to which proxies should announce themselves. + ProxyAnnounceTargets []string + // A set of info hashes where p2p-proxy peers can be found. + ProxyPeerInfoHashes []string +} + +func (ro *ReplicaOptions) GetWebseedBaseUrls() []string { + return ro.WebseedBaseUrls +} + +func (ro *ReplicaOptions) GetTrackers() []string { + return ro.Trackers +} + +func (ro *ReplicaOptions) GetStaticPeerAddrs() []string { + return ro.StaticPeerAddrs +} + +func (ro *ReplicaOptions) GetMetadataBaseUrls() []string { + return ro.MetadataBaseUrls +} + +func (ro *ReplicaOptions) GetReplicaRustEndpoint() string { + return ro.ReplicaRustEndpoint +} + +type GoogleSearchAdsOptions struct { + Pattern string `mapstructure:"pattern"` + BlockFormat string `mapstructure:"block_format"` + AdFormat string `mapstructure:"ad_format"` + Partners map[string][]PartnerAd `mapstructure:"partners"` +} + +type PartnerAd struct { + Name string + URL string + Campaign string + Description string + Keywords []*regexp.Regexp + Probability float32 +} + +func (o *GoogleSearchAdsOptions) fromMap(m map[string]interface{}) error { + // since keywords can be regexp and we don't want to compile them each time we compare, define a custom decode hook + // that will convert string to regexp and error out on syntax issues + config := &mapstructure.DecoderConfig{ + DecodeHook: func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { + if t != reflect.TypeOf(regexp.Regexp{}) { + return data, nil + } + r, err := regexp.Compile(fmt.Sprintf("%v", data)) + if err != nil { + return nil, err + } + return r, nil + }, + Result: o, + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(m) +} + +type PingProxiesOptions struct { + Interval time.Duration +} + +func (o *PingProxiesOptions) fromMap(m map[string]interface{}) error { + interval, err := durationFromMap(m, "interval") + if err != nil { + return err + } + o.Interval = interval + return nil +} + +// TrafficLogOptions represents options for github.com/getlantern/trafficlog-flashlight. +type TrafficLogOptions struct { + // Size of the traffic log's packet buffers (if enabled). + CaptureBytes int + SaveBytes int + + // How far back to go when attaching packets to an issue report. + CaptureSaveDuration time.Duration + + // Whether to overwrite the traffic log binary. This may result in users being re-prompted for + // their passwords. The binary will never be overwritten if the existing binary matches the + // embedded version. + Reinstall bool + + // The minimum amount of time to wait before re-prompting the user since the last time we failed + // to install the traffic log. The most likely reason for a failed install is denial of + // permission by the user. A value of 0 means we never re-attempt installation. + WaitTimeSinceFailedInstall time.Duration + + // The number of times installation can fail before we give up on this client. A value of zero + // is equivalent to a value of one. + FailuresThreshold int + + // After this amount of time has elapsed, the failure count is reset and a user may be + // re-prompted to install the traffic log. + TimeBeforeFailureReset time.Duration + + // The number of times a user must deny permission for the traffic log before we stop asking. A + // value of zero is equivalent to a value of one. + UserDenialThreshold int + + // After this amount of time has elapsed, the user denial count is reset and a user may be + // re-prompted to install the traffic log. + TimeBeforeDenialReset time.Duration +} + +func (o *TrafficLogOptions) fromMap(m map[string]interface{}) error { + var err error + o.CaptureBytes, err = intFromMap(m, "capturebytes") + if err != nil { + return errors.New("error unmarshaling 'capturebytes': %v", err) + } + o.SaveBytes, err = intFromMap(m, "savebytes") + if err != nil { + return errors.New("error unmarshaling 'savebytes': %v", err) + } + o.CaptureSaveDuration, err = durationFromMap(m, "capturesaveduration") + if err != nil { + return errors.New("error unmarshaling 'capturesaveduration': %v", err) + } + o.Reinstall, err = boolFromMap(m, "reinstall") + if err != nil { + return errors.New("error unmarshaling 'reinstall': %v", err) + } + o.WaitTimeSinceFailedInstall, err = durationFromMap(m, "waittimesincefailedinstall") + if err != nil { + return errors.New("error unmarshaling 'waittimesincefailedinstall': %v", err) + } + o.UserDenialThreshold, err = intFromMap(m, "userdenialthreshold") + if err != nil { + return errors.New("error unmarshaling 'userdenialthreshold': %v", err) + } + o.TimeBeforeDenialReset, err = durationFromMap(m, "timebeforedenialreset") + if err != nil { + return errors.New("error unmarshaling 'timebeforedenialreset': %v", err) + } + return nil +} + +// ClientGroup represents a subgroup of Lantern clients chosen randomly or +// based on certain criteria on which features can be selectively turned on. +type ClientGroup struct { + // A label so that the group can be referred to when collecting/analyzing + // metrics. Better to be unique and meaningful. + Label string + // UserFloor and UserCeil defines the range of user IDs so that with + // precision p, any user ID u satisfies floor*p <= u%p < ceil*p belongs to + // the group. Precision is expressed in the code and can be changed freely. + // + // For example, given floor = 0.1 and ceil = 0.2, it matches user IDs end + // between 100 and 199 if precision is 1000, and IDs end between 1000 and + // 1999 if precision is 10000. + // + // Range: 0-1. When both are omitted, all users fall within the range. + UserFloor float64 + UserCeil float64 + // The application the feature applies to. Defaults to all applications. + Application string + // A semantic version range which only Lantern versions falls within is consided. + // Defaults to all versions. + VersionConstraints string + // Comma separated list of platforms the group includes. + // Defaults to all platforms. + Platforms string + // Only include Lantern Free clients. + FreeOnly bool + // Only include Lantern Pro clients. + ProOnly bool + // Comma separated list of countries the group includes. + // Defaults to all countries. + GeoCountries string + // Random fraction of clients to include from the final set where all other + // criteria match. + // + // Range: 0-1. Defaults to 1. + Fraction float64 +} + +// Validate checks if the ClientGroup fields are valid and do not conflict with +// each other. +func (g ClientGroup) Validate() error { + if g.UserFloor < 0 || g.UserFloor > 1.0 { + return errors.New("Invalid UserFloor") + } + if g.UserCeil < 0 || g.UserCeil > 1.0 { + return errors.New("Invalid UserCeil") + } + if g.UserCeil < g.UserFloor { + return errors.New("Invalid user range") + } + if g.Fraction < 0 || g.Fraction > 1.0 { + return errors.New("Invalid Fraction") + } + if g.FreeOnly && g.ProOnly { + return errors.New("Both FreeOnly and ProOnly is set") + } + if g.VersionConstraints != "" { + _, err := semver.ParseRange(g.VersionConstraints) + if err != nil { + return fmt.Errorf("error parsing version constraints: %v", err) + } + } + return nil +} + +//Includes checks if the ClientGroup includes the user, device and country +//combination, assuming the group has been validated. +func (g ClientGroup) Includes(appName string, userID int64, isPro bool, geoCountry string) bool { + if g.UserCeil > 0 { + // Unknown user ID doesn't belong to any user range + if userID == 0 { + return false + } + percision := 1000.0 + remainder := userID % int64(percision) + if remainder < int64(g.UserFloor*percision) || remainder >= int64(g.UserCeil*percision) { + return false + } + } + if g.FreeOnly && isPro { + return false + } + if g.ProOnly && !isPro { + return false + } + if g.Application != "" && strings.ToLower(g.Application) != strings.ToLower(appName) { + return false + } + if g.VersionConstraints != "" { + expectedRange, err := semver.ParseRange(g.VersionConstraints) + if err != nil { + return false + } + if !expectedRange(semver.MustParse(common.Version)) { + return false + } + } + if g.Platforms != "" && !csvContains(g.Platforms, common.Platform) { + return false + } + if g.GeoCountries != "" && !csvContains(g.GeoCountries, geoCountry) { + return false + } + if g.Fraction > 0 && randomFloat >= g.Fraction { + return false + } + return true +} + +func csvContains(csv, s string) bool { + fields := strings.Split(csv, ",") + for _, f := range fields { + if strings.EqualFold(s, strings.TrimSpace(f)) { + return true + } + } + return false +} + +func boolFromMap(m map[string]interface{}, name string) (bool, error) { + v, exists := m[name] + if !exists { + return false, errAbsentOption + } + b, ok := v.(bool) + if !ok { + return false, errMalformedOption + } + return b, nil +} + +func intFromMap(m map[string]interface{}, name string) (int, error) { + v, exists := m[name] + if !exists { + return 0, errAbsentOption + } + i, ok := v.(int) + if !ok { + return 0, errMalformedOption + } + return i, nil +} + +func durationFromMap(m map[string]interface{}, name string) (time.Duration, error) { + v, exists := m[name] + if !exists { + return 0, errAbsentOption + } + s, ok := v.(string) + if !ok { + return 0, errMalformedOption + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, errMalformedOption + } + return d, nil +} diff --git a/config_v2/features_test.go b/config_v2/features_test.go new file mode 100644 index 0000000000..e1320ceb06 --- /dev/null +++ b/config_v2/features_test.go @@ -0,0 +1,201 @@ +package config + +import ( + "bytes" + "strings" + "testing" + "text/template" + "time" + + "github.com/getlantern/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/getlantern/flashlight/common" + "github.com/getlantern/flashlight/embeddedconfig" +) + +func TestValidate(t *testing.T) { + assert.NoError(t, ClientGroup{}.Validate(), "zero value should be valid") + assert.NoError(t, ClientGroup{UserFloor: 0.9, UserCeil: 0.98}.Validate(), "valid user range") + assert.Error(t, ClientGroup{UserFloor: -1.0}.Validate(), "invalid user floor") + assert.Error(t, ClientGroup{UserFloor: 1.09}.Validate(), "invalid user floor") + assert.Error(t, ClientGroup{UserFloor: 0.1, UserCeil: 0}.Validate(), "invalid user range") + assert.Error(t, ClientGroup{Fraction: 1.1}.Validate(), "invalid fraction") + assert.Error(t, ClientGroup{FreeOnly: true, ProOnly: true}.Validate(), "conflict user status requirements") + assert.NoError(t, ClientGroup{VersionConstraints: ">3.2.1 <= 9.2.0 "}.Validate(), "compound version constraits") + assert.NoError(t, ClientGroup{VersionConstraints: "<3.2.1 || >= 9.2.0 "}.Validate(), "compound version constraits") +} + +func TestIncludes(t *testing.T) { + assert.True(t, ClientGroup{}.Includes(common.DefaultAppName, 0, true, "whatever"), "zero value should include all combinations") + assert.True(t, ClientGroup{}.Includes(common.DefaultAppName, 111, false, "whatever"), "zero value should include all combinations") + assert.True(t, ClientGroup{UserCeil: 0.12}.Includes(common.DefaultAppName, 111, false, "whatever"), "match user range") + assert.False(t, ClientGroup{UserCeil: 0.11}.Includes(common.DefaultAppName, 111, false, "whatever"), "user range does not match") + assert.False(t, ClientGroup{UserCeil: 0.11}.Includes(common.DefaultAppName, 0, false, "whatever"), "unknown user ID should not belong to any user range") + + assert.True(t, ClientGroup{FreeOnly: true}.Includes(common.DefaultAppName, 111, false, "whatever"), "user status met") + assert.False(t, ClientGroup{ProOnly: true}.Includes(common.DefaultAppName, 111, false, "whatever"), "user status unmet") + assert.True(t, ClientGroup{ProOnly: true}.Includes(common.DefaultAppName, 111, true, "whatever"), "user status met") + assert.False(t, ClientGroup{FreeOnly: true}.Includes(common.DefaultAppName, 111, true, "whatever"), "user status unmet") + + // The default AppName is "Default" + assert.True(t, ClientGroup{Application: (common.DefaultAppName)}.Includes(common.DefaultAppName, 111, true, "whatever"), "application met, case insensitive") + assert.True(t, ClientGroup{Application: strings.ToUpper(common.DefaultAppName)}.Includes(common.DefaultAppName, 111, true, "whatever"), "application met, case insensitive") + assert.False(t, ClientGroup{Application: "Beam"}.Includes(common.DefaultAppName, 111, true, "whatever"), "application unmet, case insensitive") + assert.False(t, ClientGroup{Application: "beam"}.Includes(common.DefaultAppName, 111, true, "whatever"), "application unmet, case insensitive") + + // The client version is 9999.99.99-dev when in development mode + assert.True(t, ClientGroup{VersionConstraints: "> 5.1.0"}.Includes(common.DefaultAppName, 111, true, "whatever"), "version met") + assert.True(t, ClientGroup{VersionConstraints: "> 5.1.0 < 10000.0.0"}.Includes(common.DefaultAppName, 111, true, "whatever"), "version met") + assert.False(t, ClientGroup{VersionConstraints: "< 5.1.0"}.Includes(common.DefaultAppName, 111, true, "whatever"), "version unmet") + + // Platforms tests are likely run + assert.True(t, ClientGroup{Platforms: "linux,darwin,windows"}.Includes(common.DefaultAppName, 111, true, "whatever"), "platform met") + // Platforms tests are unlikely run + assert.False(t, ClientGroup{Platforms: "ios,android"}.Includes(common.DefaultAppName, 111, true, "whatever"), "platform unmet") + + assert.True(t, ClientGroup{GeoCountries: "ir , cn"}.Includes(common.DefaultAppName, 111, true, "IR"), "country met") + assert.False(t, ClientGroup{GeoCountries: "us"}.Includes(common.DefaultAppName, 111, true, "IR"), "country unmet") + + // Fraction calculation should be stable + g := ClientGroup{Fraction: 0.1} + hits := 0 + for i := 0; i < 1000; i++ { + if g.Includes(common.DefaultAppName, 111, true, "whatever") { + hits++ + } + } + if randomFloat >= 0.1 { + assert.Equal(t, 0, hits) + } else { + assert.Equal(t, 1000, hits) + } +} + +func TestUnmarshalFeaturesEnabled(t *testing.T) { + yml := ` +featuresenabled: + replica: + - userfloor: 0 + userceil: 0.2 + versionconstraints: ">3.0.0" + geocountries: us,cn,au,ir + - versionconstraints: "=9999.99.99-dev" + proonly: true +` + gl := NewGlobal() + if !assert.NoError(t, yaml.Unmarshal([]byte(yml), gl)) { + return + } + assert.True(t, gl.FeatureEnabled(FeatureReplica, common.DefaultAppName, 111, false, "au"), "met the first group") + assert.True(t, gl.FeatureEnabled(FeatureReplica, common.DefaultAppName, 111, true, ""), "met the second group") + assert.False(t, gl.FeatureEnabled(FeatureReplica, common.DefaultAppName, 211, false, "au"), "unmet both groups") + assert.False(t, gl.FeatureEnabled(FeatureReplica, common.DefaultAppName, 111, false, ""), "unmet both groups") +} + +func TestUnmarshalFeatureOptions(t *testing.T) { + yml := ` +featureoptions: + trafficlog: + capturebytes: 1 + savebytes: 2 + capturesaveduration: 5m + reinstall: true + waittimesincefailedinstall: 24h + userdenialthreshold: 3 + timebeforedenialreset: 2160h + pingproxies: + interval: 1h +` + gl := NewGlobal() + require.NoError(t, yaml.Unmarshal([]byte(yml), gl)) + + var opts TrafficLogOptions + require.NoError(t, gl.UnmarshalFeatureOptions(FeatureTrafficLog, &opts)) + + require.Equal(t, 1, opts.CaptureBytes) + require.Equal(t, 2, opts.SaveBytes) + require.Equal(t, 5*time.Minute, opts.CaptureSaveDuration) + require.Equal(t, true, opts.Reinstall) + require.Equal(t, 24*time.Hour, opts.WaitTimeSinceFailedInstall) + require.Equal(t, 3, opts.UserDenialThreshold) + require.Equal(t, 2160*time.Hour, opts.TimeBeforeDenialReset) + + var opts2 PingProxiesOptions + require.NoError(t, gl.UnmarshalFeatureOptions(FeaturePingProxies, &opts2)) + require.Equal(t, time.Hour, opts2.Interval) +} + +func TestUnmarshalAnalyticsOptions(t *testing.T) { + yml := ` +featureoptions: + analytics: + providers: + ga: + endpoint: "https://ssl.google-analytics.com/collect" + samplerate: 1.0 + config: + k1: 2 + k2: 3 + matomo: + samplerate: 0.1 + config: + idsite: 1 + token_auth: "418290ccds0d01" +` + gl := NewGlobal() + require.NoError(t, yaml.Unmarshal([]byte(yml), gl)) + + var opts AnalyticsOptions + require.NoError(t, gl.UnmarshalFeatureOptions(FeatureAnalytics, &opts)) + log.Debugf("%+v", opts) + + mat := opts.GetProvider(MATOMO) + ga := opts.GetProvider(GA) + require.Equal(t, float32(0.1), mat.SampleRate) + require.Equal(t, 2, ga.Config["k1"]) + require.Equal(t, "https://ssl.google-analytics.com/collect", ga.Endpoint) + + require.Equal(t, 1, mat.Config["idsite"]) + require.Nil(t, mat.Config["k1"]) +} + +func TestReplicaByCountry(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fos := getReplicaOptionsRoot(require) + assert.Contains(fos.ByCountry, "RU") + assert.NotContains(fos.ByCountry, "AU") + assert.NotEmpty(fos.ByCountry) + globalTrackers := fos.Trackers + assert.NotEmpty(globalTrackers) + // Check the countries pull in the trackers using the anchor. Just change this if they stop + // using the same trackers. I really don't want this to break out the gate is all. + assert.Equal(fos.ByCountry["CN"].Trackers, globalTrackers) + assert.Equal(fos.ByCountry["RU"].Trackers, globalTrackers) + assert.Equal(fos.ByCountry["IR"].Trackers, globalTrackers) +} + +func getReplicaOptionsRoot(require *require.Assertions) (fos ReplicaOptionsRoot) { + var w bytes.Buffer + // We could write into a pipe, but that requires concurrency and we're old-school in tests. + require.NoError(template.Must(template.New("").Parse(embeddedconfig.GlobalTemplate)).Execute(&w, nil)) + var g Global + require.NoError(yaml.Unmarshal(w.Bytes(), &g)) + require.NoError(g.UnmarshalFeatureOptions(FeatureReplica, &fos)) + return +} + +func TestReplicaProxying(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fos := getReplicaOptionsRoot(require) + numInfohashes := len(fos.ProxyAnnounceTargets) + // The default is to announce as a proxy. + assert.True(numInfohashes > 0) + // The default is not to look for proxies + assert.Empty(fos.ProxyPeerInfoHashes) + // Iran looks for peers from the default countries. + assert.Len(fos.ByCountry["IR"].ProxyPeerInfoHashes, numInfohashes) +} diff --git a/config_v2/fetcher.go b/config_v2/fetcher.go new file mode 100644 index 0000000000..6db0ff4db0 --- /dev/null +++ b/config_v2/fetcher.go @@ -0,0 +1,162 @@ +package config + +import ( + "compress/gzip" + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/getlantern/detour" + + "github.com/getlantern/flashlight/common" + "github.com/getlantern/flashlight/ops" +) + +var ( + forceCountry atomic.Value +) + +// ForceCountry forces config fetches to pretend client is running in the +// given countryCode (e.g. 'cn') +func ForceCountry(countryCode string) { + countryCode = strings.ToLower(countryCode) + log.Debugf("Forcing config country to %v", countryCode) + forceCountry.Store(countryCode) +} + +// Fetcher is an interface for fetching config updates. +type Fetcher interface { + fetch() ([]byte, time.Duration, error) +} + +// fetcher periodically fetches the latest cloud configuration. +type fetcher struct { + lastCloudConfigETag map[string]string + user common.UserConfig + rt http.RoundTripper + originURL string +} + +var noSleep = 0 * time.Second + +// newFetcher creates a new configuration fetcher with the specified +// interface for obtaining the user ID and token if those are populated. +func newFetcher(conf common.UserConfig, rt http.RoundTripper, originURL string) Fetcher { + log.Debugf("Will poll for config at %v", originURL) + + // Force detour to whitelist chained domain + u, err := url.Parse(originURL) + if err != nil { + log.Fatalf("Unable to parse chained cloud config URL: %v", err) + } + detour.ForceWhitelist(u.Host) + + return &fetcher{ + lastCloudConfigETag: map[string]string{}, + user: conf, + rt: rt, + originURL: originURL, + } +} + +func (cf *fetcher) fetch() ([]byte, time.Duration, error) { + op, ctx := ops.BeginWithNewBeam("fetch_config", context.Background()) + defer op.End() + result, sleep, err := cf.doFetch(ctx, op) + return result, sleep, op.FailIf(err) +} + +func (cf *fetcher) doFetch(ctx context.Context, op *ops.Op) ([]byte, time.Duration, error) { + log.Debugf("Fetching cloud config from %v", cf.originURL) + + sleepTime := noSleep + url := cf.originURL + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, sleepTime, fmt.Errorf("Unable to construct request for cloud config at %s: %s", url, err) + } + if cf.lastCloudConfigETag[url] != "" { + // Don't bother fetching if unchanged + req.Header.Set(common.IfNoneMatchHeader, cf.lastCloudConfigETag[url]) + } + + req.Header.Set("Accept", "application/x-gzip") + // Prevents intermediate nodes (domain-fronters) from caching the content + req.Header.Set("Cache-Control", "no-cache") + common.AddCommonHeaders(cf.user, req) + + _forceCountry := forceCountry.Load() + if _forceCountry != nil { + countryCode := _forceCountry.(string) + log.Debugf("Forcing config country to %v", countryCode) + req.Header.Set(common.ClientCountryHeader, countryCode) + } + + // make sure to close the connection after reading the Body + // this prevents the occasional EOFs errors we're seeing with + // successive requests + req.Close = true + + resp, err := cf.rt.RoundTrip(req.WithContext(ctx)) + if err != nil { + return nil, sleepTime, fmt.Errorf("Unable to fetch cloud config at %s: %s", url, err) + } + + sleepVal := resp.Header.Get("X-Lantern-Config-Sleep") + if sleepVal != "" { + seconds, err := strconv.ParseInt(sleepVal, 10, 64) + if err != nil { + log.Errorf("Could not parse sleep val: %v", err) + } else { + sleepTime = time.Duration(seconds) * time.Second + } + } + + dump, dumperr := httputil.DumpResponse(resp, false) + if dumperr != nil { + log.Errorf("Could not dump response: %v", dumperr) + } else { + log.Debugf("Response headers from %v:\n%v", cf.originURL, string(dump)) + } + defer func() { + if closeerr := resp.Body.Close(); closeerr != nil { + log.Errorf("Error closing response body: %v", closeerr) + } + }() + + if resp.StatusCode == 304 { + op.Set("config_changed", false) + log.Debug("Config unchanged in cloud") + return nil, sleepTime, nil + } else if resp.StatusCode != 200 { + op.HTTPStatusCode(resp.StatusCode) + if dumperr != nil { + return nil, sleepTime, fmt.Errorf("Bad config response code: %v", resp.StatusCode) + } + return nil, sleepTime, fmt.Errorf("Bad config resp:\n%v", string(dump)) + } + + op.Set("config_changed", true) + cf.lastCloudConfigETag[url] = resp.Header.Get(common.EtagHeader) + gzReader, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, sleepTime, fmt.Errorf("Unable to open gzip reader: %s", err) + } + + defer func() { + if err := gzReader.Close(); err != nil { + log.Errorf("Unable to close gzip reader: %v", err) + } + }() + + log.Debugf("Fetched cloud config") + body, err := ioutil.ReadAll(gzReader) + return body, sleepTime, err +} diff --git a/config_v2/fetcher_test.go b/config_v2/fetcher_test.go new file mode 100644 index 0000000000..c157934e47 --- /dev/null +++ b/config_v2/fetcher_test.go @@ -0,0 +1,54 @@ +package config + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/getlantern/flashlight/common" +) + +func newTestUserConfig() *common.UserConfigData { + return common.NewUserConfigData(common.DefaultAppName, "deviceID", 10, "token", nil, "en-US") +} + +// TestFetcher actually fetches a config file over the network. +func TestFetcher(t *testing.T) { + defer deleteGlobalConfig() + + // This will actually fetch the cloud config over the network. + rt := &http.Transport{} + configFetcher := newFetcher(newTestUserConfig(), rt, common.GlobalURL) + + bytes, _, err := configFetcher.fetch() + assert.Nil(t, err) + assert.True(t, len(bytes) > 200) +} + +// TestStagingSetup tests to make sure our staging config flag sets the +// appropriate URLs for staging servers. +func TestStagingSetup(t *testing.T) { + flags := make(map[string]interface{}) + flags["staging"] = false + + rt := &http.Transport{} + + var fetch *fetcher + fetch = newFetcher(newTestUserConfig(), rt, common.ProxiesURL).(*fetcher) + + assert.Equal(t, "http://config.getiantem.org/proxies.yaml.gz", fetch.originURL) + + url := common.ProxiesURL + + // Blank flags should mean we use the default + flags["cloudconfig"] = "" + fetch = newFetcher(newTestUserConfig(), rt, url).(*fetcher) + + assert.Equal(t, "http://config.getiantem.org/proxies.yaml.gz", fetch.originURL) + + stagingURL := common.ProxiesStagingURL + flags["staging"] = true + fetch = newFetcher(newTestUserConfig(), rt, stagingURL).(*fetcher) + assert.Equal(t, "http://config-staging.getiantem.org/proxies.yaml.gz", fetch.originURL) +} diff --git a/config_v2/global.go b/config_v2/global.go new file mode 100644 index 0000000000..3224bd7308 --- /dev/null +++ b/config_v2/global.go @@ -0,0 +1,152 @@ +package config + +import ( + "crypto/x509" + "errors" + "time" + + "github.com/getlantern/flashlight/browsers/simbrowser" + "github.com/getlantern/flashlight/domainrouting" + "github.com/getlantern/fronted" + "github.com/getlantern/keyman" +) + +// Global contains general configuration for Lantern either set globally via +// the cloud, in command line flags, or in local customizations during +// development. +type Global struct { + Version int + CloudConfigCA string + // AutoUpdateCA is the CA key to pin for auto-updates. + AutoUpdateCA string + UpdateServerURL string + BordaReportInterval time.Duration + BordaSamplePercentage float64 + // ReportIssueEmail is the recipient of the email sent when the user + // reports issue. + ReportIssueEmail string + + // AdSettings are the settings to use for showing ads to mobile clients + AdSettings *AdSettings + + Client *ClientConfig + + // ProxiedSites are domains that get routed through Lantern rather than accessed directly. + // This has been deprecated in favor of more precise DomainRoutingRules (see below). + // The client will continue to honor ProxiedSites configuration for now. + ProxiedSites *domainrouting.ProxiedSitesConfig + + // DomainRoutingRules specifies routing rules for specific domains, such as forcing proxing, forcing direct dials, etc. + DomainRoutingRules domainrouting.Rules + + // NamedDomainRoutingRules specifies routing rules for specific domains, grouped by name. + NamedDomainRoutingRules map[string]domainrouting.Rules + + // TrustedCAs are trusted CAs for domain fronting domains only. + TrustedCAs []*fronted.CA + + // GlobalConfigPollInterval sets interval at which to poll for global config + GlobalConfigPollInterval time.Duration + + // ProxyConfigPollInterval sets interval at which to poll for proxy config + ProxyConfigPollInterval time.Duration + + // FeaturesEnabled specifies which optional feature is enabled for certain + // groups of clients. + FeaturesEnabled map[string][]*ClientGroup + // FeatureOptions is a generic way to specify options for optional + // features. It's up to the feature code to handle the raw JSON message. + FeatureOptions map[string]map[string]interface{} + + // Market share data used by the simbrowser package when picking a browser to simulate. + GlobalBrowserMarketShareData simbrowser.MarketShareData + RegionalBrowserMarketShareData map[simbrowser.CountryCode]simbrowser.MarketShareData +} + +// NewGlobal creates a new global config with otherwise nil values set. +func NewGlobal() *Global { + return &Global{ + Client: NewClientConfig(), + ProxiedSites: &domainrouting.ProxiedSitesConfig{}, + } +} + +// FeatureEnabled checks if the feature is enabled given the client properties. +func (cfg *Global) FeatureEnabled(feature string, appName string, userID int64, isPro bool, + geoCountry string) bool { + enabled, _ := cfg.FeatureEnabledWithLabel(feature, appName, userID, isPro, geoCountry) + log.Tracef("Feature %v enabled for user %v in country %v?: %v", feature, userID, geoCountry, enabled) + return enabled +} + +// FeatureEnabledWithLabel is the same as FeatureEnabled but also returns the +// label of the first matched ClientGroup if the feature is enabled. +func (cfg *Global) FeatureEnabledWithLabel(feature string, appName string, userID int64, isPro bool, + geoCountry string) (enabled bool, label string) { + groups, exists := cfg.FeaturesEnabled[feature] + if !exists { + return false, "" + } + for _, g := range groups { + if g.Includes(appName, userID, isPro, geoCountry) { + return true, g.Label + } + } + return false, "" +} + +func (cfg *Global) UnmarshalFeatureOptions(feature string, opts FeatureOptions) error { + m, exists := cfg.FeatureOptions[feature] + if !exists { + return errAbsentOption + } + return opts.fromMap(m) +} + +// TrustedCACerts returns a certificate pool containing the TrustedCAs from this +// config. +func (cfg *Global) TrustedCACerts() (pool *x509.CertPool, err error) { + certs := make([]string, 0, len(cfg.TrustedCAs)) + for _, ca := range cfg.TrustedCAs { + certs = append(certs, ca.Cert) + } + pool, err = keyman.PoolContainingCerts(certs...) + if err != nil { + log.Errorf("Could not create pool %v", err) + } + return +} + +// applyFlags updates this config from any command-line flags that were passed +// in. +func (cfg *Global) applyFlags(flags map[string]interface{}) { + // Visit all flags that have been set and copy to config + for key, value := range flags { + switch key { + case "cloudconfigca": + cfg.CloudConfigCA = value.(string) + case "borda-report-interval": + cfg.BordaReportInterval = value.(time.Duration) + case "borda-sample-percentage": + cfg.BordaSamplePercentage = value.(float64) + } + } +} + +func (cfg *Global) validate() error { + err := cfg.Client.Validate() + if err != nil { + return err + } + if len(cfg.TrustedCAs) == 0 { + return errors.New("no trusted CAs") + } + for _, groups := range cfg.FeaturesEnabled { + for _, g := range groups { + if err := g.Validate(); err != nil { + return err + } + } + } + return nil +} diff --git a/config_v2/initializer.go b/config_v2/initializer.go new file mode 100644 index 0000000000..5a5b60ef88 --- /dev/null +++ b/config_v2/initializer.go @@ -0,0 +1,236 @@ +package config + +import ( + "errors" + "net/http" + "sync" + "time" + + "github.com/getlantern/golog" + "github.com/getlantern/yaml" + + "github.com/getlantern/flashlight/chained" + "github.com/getlantern/flashlight/common" + "github.com/getlantern/flashlight/embeddedconfig" +) + +var ( + log = golog.LoggerFor("flashlight.config") + + // DefaultProxyConfigPollInterval determines how frequently to fetch proxies.yaml + DefaultProxyConfigPollInterval = 1 * time.Minute + + // ForceProxyConfigPollInterval overrides how frequently to fetch proxies.yaml if set (does not honor values from global.yaml) + ForceProxyConfigPollInterval = 0 * time.Second + + // DefaultGlobalConfigPollInterval determines how frequently to fetch global.yaml + DefaultGlobalConfigPollInterval = 1 * time.Hour +) + +// Init determines the URLs at which to fetch proxy and global config and +// passes those to InitWithURLs, which initializes the config setup for both +// fetching per-user proxies as well as the global config. It returns a function +// that can be used to stop the reading of configs. +func Init( + configDir string, flags map[string]interface{}, userConfig common.UserConfig, + proxiesDispatch func(interface{}, Source), onProxiesSaveError func(error), + origGlobalDispatch func(interface{}, Source), onGlobalSaveError func(error), + rt http.RoundTripper) (stop func()) { + + staging := isStaging(flags) + proxyConfigURL := checkOverrides(flags, getProxyURL(staging), "proxies.yaml.gz") + globalConfigURL := checkOverrides(flags, getGlobalURL(staging), "global.yaml.gz") + + return InitWithURLs( + configDir, flags, userConfig, proxiesDispatch, onProxiesSaveError, + origGlobalDispatch, onGlobalSaveError, proxyConfigURL, globalConfigURL, rt) +} + +type cfgWithSource struct { + cfg interface{} + src Source +} + +// InitWithURLs initializes the config setup for both fetching per-user proxies +// as well as the global config given a set of URLs for fetching proxy and +// global config. It returns a function that can be used to stop the reading of +// configs. +func InitWithURLs( + configDir string, flags map[string]interface{}, userConfig common.UserConfig, + origProxiesDispatch func(interface{}, Source), onProxiesSaveError func(error), + origGlobalDispatch func(interface{}, Source), onGlobalSaveError func(error), + proxyURL string, globalURL string, rt http.RoundTripper) (stop func()) { + + var mx sync.RWMutex + globalConfigPollInterval := DefaultGlobalConfigPollInterval + proxyConfigPollInterval := DefaultProxyConfigPollInterval + if ForceProxyConfigPollInterval > 0 { + proxyConfigPollInterval = ForceProxyConfigPollInterval + } + + globalDispatchCh := make(chan cfgWithSource) + proxiesDispatchCh := make(chan cfgWithSource) + go func() { + for c := range globalDispatchCh { + origGlobalDispatch(c.cfg, c.src) + } + }() + go func() { + for c := range proxiesDispatchCh { + origProxiesDispatch(c.cfg, c.src) + } + }() + + globalDispatch := func(cfg interface{}, src Source) { + globalConfig, ok := cfg.(*Global) + if ok { + mx.Lock() + if globalConfig.GlobalConfigPollInterval > 0 { + globalConfigPollInterval = globalConfig.GlobalConfigPollInterval + } + if ForceProxyConfigPollInterval == 0 && globalConfig.ProxyConfigPollInterval > 0 { + proxyConfigPollInterval = globalConfig.ProxyConfigPollInterval + } + mx.Unlock() + } + // Rather than call `origGlobalDispatch` here, we are calling it in a + // separate goroutine (initiated above) that listens for messages on + // `globalDispatchCh`. This (a) avoids blocking Lantern startup when + // applying new configuration and (b) allows us to serialize application of + // config changes. + globalDispatchCh <- cfgWithSource{cfg, src} + } + + proxiesDispatch := func(cfg interface{}, src Source) { + proxiesDispatchCh <- cfgWithSource{cfg, src} + } + + // These are the options for fetching the per-user proxy config. + proxyOptions := &options{ + saveDir: configDir, + onSaveError: onProxiesSaveError, + obfuscate: obfuscate(flags), + name: "proxies.yaml", + originURL: proxyURL, + userConfig: userConfig, + unmarshaler: newProxiesUnmarshaler(), + dispatch: proxiesDispatch, + embeddedData: embeddedconfig.Proxies, + sleep: func() time.Duration { + mx.RLock() + defer mx.RUnlock() + return proxyConfigPollInterval + }, + sticky: isSticky(flags), + rt: rt, + } + + stopProxies := pipeConfig(proxyOptions) + + // These are the options for fetching the global config. + globalOptions := &options{ + saveDir: configDir, + onSaveError: onGlobalSaveError, + obfuscate: obfuscate(flags), + name: "global.yaml", + originURL: globalURL, + userConfig: userConfig, + unmarshaler: newGlobalUnmarshaler(flags), + dispatch: globalDispatch, + embeddedData: embeddedconfig.Global, + sleep: func() time.Duration { + mx.RLock() + defer mx.RUnlock() + return globalConfigPollInterval + }, + sticky: isSticky(flags), + rt: rt, + } + + stopGlobal := pipeConfig(globalOptions) + + return func() { + log.Debug("*************** Stopping Config") + stopProxies() + stopGlobal() + } +} + +func newGlobalUnmarshaler(flags map[string]interface{}) func(bytes []byte) (interface{}, error) { + return func(bytes []byte) (interface{}, error) { + gl := NewGlobal() + gl.applyFlags(flags) + if err := yaml.Unmarshal(bytes, gl); err != nil { + return nil, err + } + if err := gl.validate(); err != nil { + return nil, err + } + return gl, nil + } +} + +func newProxiesUnmarshaler() func(bytes []byte) (interface{}, error) { + return func(bytes []byte) (interface{}, error) { + servers := make(map[string]*chained.ChainedServerInfo) + if err := yaml.Unmarshal(bytes, servers); err != nil { + return nil, err + } + if len(servers) == 0 { + return nil, errors.New("No chained server") + } + return servers, nil + } +} + +func obfuscate(flags map[string]interface{}) bool { + return flags["readableconfig"] == nil || !flags["readableconfig"].(bool) +} + +func isStaging(flags map[string]interface{}) bool { + return checkBool(flags, "staging") +} + +func isSticky(flags map[string]interface{}) bool { + return checkBool(flags, "stickyconfig") +} + +func checkBool(flags map[string]interface{}, key string) bool { + if s, ok := flags[key].(bool); ok { + return s + } + return false +} + +func checkOverrides(flags map[string]interface{}, + url string, name string) string { + if s, ok := flags["cloudconfig"].(string); ok { + if len(s) > 0 { + log.Debugf("Overridding config URL from the command line '%v'", s) + return s + "/" + name + } + } + return url +} + +// getProxyURL returns the proxy URL to use depending on whether or not +// we're in staging. +func getProxyURL(staging bool) string { + if staging { + log.Debug("Will obtain proxies.yaml from staging service") + return common.ProxiesStagingURL + } + log.Debug("Will obtain proxies.yaml from production service") + return common.ProxiesURL +} + +// getGlobalURL returns the global URL to use depending on whether or not +// we're in staging. +func getGlobalURL(staging bool) string { + if staging { + log.Debug("Will obtain global.yaml from staging service") + return common.GlobalStagingURL + } + log.Debugf("Will obtain global.yaml from production service at %v", common.GlobalURL) + return common.GlobalURL +} diff --git a/config_v2/initializer_test.go b/config_v2/initializer_test.go new file mode 100644 index 0000000000..e48be851a3 --- /dev/null +++ b/config_v2/initializer_test.go @@ -0,0 +1,127 @@ +package config + +import ( + "net/http" + "net/url" + "testing" + "time" + + "github.com/getlantern/eventual" + "github.com/getlantern/flashlight/chained" + "github.com/getlantern/flashlight/common" + "github.com/stretchr/testify/assert" +) + +// TestInit tests initializing configs. +func TestInit(t *testing.T) { + defer deleteGlobalConfig() + + flags := make(map[string]interface{}) + flags["staging"] = true + + gotProxies := eventual.NewValue() + gotGlobal := eventual.NewValue() + + // Note these dispatch functions will receive multiple configs -- local ones, + // embedded ones, and remote ones. + proxiesDispatch := func(cfg interface{}, src Source) { + proxies := cfg.(map[string]*chained.ChainedServerInfo) + assert.True(t, len(proxies) > 0) + gotProxies.Set(true) + } + globalDispatch := func(cfg interface{}, src Source) { + global := cfg.(*Global) + assert.True(t, len(global.Client.MasqueradeSets) > 1) + gotGlobal.Set(true) + } + stop := Init( + ".", flags, newTestUserConfig(), proxiesDispatch, nil, globalDispatch, nil, &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + // the same token should also be configured on staging + // config-server, staging proxies and staging DDF distributions. + req.Header.Add(common.CfgSvrAuthTokenHeader, "staging-token") + return nil, nil + }, + }) + defer stop() + + _, valid := gotProxies.Get(time.Second * 12) + assert.True(t, valid, "Should have got proxies config in a reasonable time") + _, valid = gotGlobal.Get(time.Second * 12) + assert.True(t, valid, "Should have got global config in a reasonable time") +} + +// TestInitWithURLs tests that proxy and global configs are fetched at the +// correct polling intervals. +func TestInitWithURLs(t *testing.T) { + withTempDir(t, func(inTempDir func(string) string) { + globalConfig := newGlobalConfig(t) + proxiesConfig := newProxiesConfig(t) + + globalConfig.GlobalConfigPollInterval = 3 * time.Second + globalConfig.ProxyConfigPollInterval = 1 * time.Second + + // ensure a `global.yaml` exists in order to avoid fetching embedded config + writeObfuscatedConfig(t, globalConfig, inTempDir("global.yaml")) + + // set up 2 servers: + // 1. one that serves up the global config and + // 2. one that serves up the proxy config + // each should track the number of requests made to it + + // set up servers to serve global config and count number of requests + globalConfigURL, globalReqCount := startConfigServer(t, globalConfig) + + // set up servers to serve global config and count number of requests + proxyConfigURL, proxyReqCount := startConfigServer(t, proxiesConfig) + + // set up and call InitWithURLs + flags := make(map[string]interface{}) + flags["staging"] = true + + proxiesDispatch := func(interface{}, Source) {} + globalDispatch := func(interface{}, Source) {} + stop := InitWithURLs( + inTempDir("."), flags, newTestUserConfig(), + proxiesDispatch, nil, + globalDispatch, nil, + proxyConfigURL, globalConfigURL, &http.Transport{}) + defer stop() + + // sleep some amount + time.Sleep(7 * time.Second) + // in 7 sec, should have made: + // 1 + (7 / 3) = 3 global requests + // 1 + (7 / 1) = 8 proxy requests + // We provide a little leeway in the checks below to account for possible delays in CI. + + // test that proxy & config servers were called the correct number of times + assert.GreaterOrEqual(t, 3, int(globalReqCount()), "should have fetched global config every %v", globalConfig.GlobalConfigPollInterval) + assert.GreaterOrEqual(t, 7, int(proxyReqCount()), "should have fetched proxy config every %v", globalConfig.ProxyConfigPollInterval) + }) +} + +func TestStaging(t *testing.T) { + flags := make(map[string]interface{}) + flags["staging"] = true + + assert.True(t, isStaging(flags)) + + flags["staging"] = false + + assert.False(t, isStaging(flags)) +} + +// TestOverrides tests url override flags +func TestOverrides(t *testing.T) { + url := "host" + flags := make(map[string]interface{}) + out := checkOverrides(flags, url, "name") + + assert.Equal(t, "host", out) + + flags["cloudconfig"] = "test" + out = checkOverrides(flags, url, "name") + + assert.Equal(t, "test/name", out) +} diff --git a/embeddedconfig/global.yaml.tmpl b/embeddedconfig/global.yaml.tmpl index d8923cf5e1..b2cde33c73 100644 --- a/embeddedconfig/global.yaml.tmpl +++ b/embeddedconfig/global.yaml.tmpl @@ -121,6 +121,12 @@ featureoptions: trafficlog: capturebytes: 10485760 savebytes: 10485760 + analytics: + providers: + matomo: + samplerate: 0.1 + config: + idsite: 1 replica: # Uses ISO 3166 country codes # https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes