diff --git a/internal/cmd/dismantle/backendclient/backendclient.go b/internal/cmd/dismantle/backendclient/backendclient.go new file mode 100644 index 0000000000..096c52c37a --- /dev/null +++ b/internal/cmd/dismantle/backendclient/backendclient.go @@ -0,0 +1,67 @@ +package backendclient + +import ( + "context" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/ooapi" +) + +type Config struct { + KVStore model.KeyValueStore + HTTPClient model.HTTPClient + Logger model.Logger + UserAgent string + + // optional fields + BaseURL *url.URL + ProxyURL *url.URL +} + +type Client struct { + endpoint *httpapi.Endpoint +} + +func New(config *Config) *Client { + baseURL := "https://api.ooni.io/" + if config.BaseURL != nil { + baseURL = config.BaseURL.String() + } + endpoint := &httpapi.Endpoint{ + BaseURL: baseURL, + HTTPClient: config.HTTPClient, + Host: "", + Logger: config.Logger, + UserAgent: config.UserAgent, + } + backendClient := &Client{ + endpoint: endpoint, + } + return backendClient +} + +func (c *Client) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + return httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) +} + +func (c *Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + panic("not implemented") +} + +func (c *Client) FetchTorTargets( + ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) { + panic("not implemented") +} + +func (c *Client) Submit(ctx context.Context, m *model.Measurement) error { + req := &model.OOAPICollectorUpdateRequest{ + Format: "json", + Content: m, + } + descriptor := newSubmitDescriptor(req, m.ReportID) + _, err := httpapi.Call(ctx, descriptor, c.endpoint) + return err +} diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go new file mode 100644 index 0000000000..b7f392fc8e --- /dev/null +++ b/internal/cmd/dismantle/backendclient/measurement.go @@ -0,0 +1,57 @@ +package backendclient + +import ( + "fmt" + "runtime" + "time" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" +) + +const dateFormat = "2006-01-02 15:04:05" + +func NewMeasurement( + location *geolocate.Results, + testName string, + testVersion string, + testStartTime time.Time, + reportID string, + softwareName string, + softwareVersion string, + input string, +) *model.Measurement { + utctimenow := time.Now().UTC() + m := &model.Measurement{ + DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, + Input: model.MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: model.DefaultProbeIP, + ProbeASN: location.ASNString(), + ProbeCC: location.CountryCode, + ProbeNetworkName: location.NetworkName, + ReportID: reportID, + ResolverASN: fmt.Sprintf("AS%d", location.ResolverASN), // XXX + ResolverIP: location.ResolverIP, + ResolverNetworkName: location.ResolverNetworkName, + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TestName: testName, + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: testVersion, + } + m.AddAnnotation("architecture", runtime.GOARCH) + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified) + m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) + m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) + m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) + return m +} diff --git a/internal/cmd/dismantle/backendclient/submitter.go b/internal/cmd/dismantle/backendclient/submitter.go new file mode 100644 index 0000000000..8bfed04b2d --- /dev/null +++ b/internal/cmd/dismantle/backendclient/submitter.go @@ -0,0 +1,34 @@ +package backendclient + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func newSubmitDescriptor( + req *model.OOAPICollectorUpdateRequest, reportID string) *httpapi.Descriptor[ + *model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse] { + rawBody, err := json.Marshal(req) + runtimex.PanicOnError(err, "json.Marshal failed") + return &httpapi.Descriptor[*model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]{ + Accept: httpapi.ApplicationJSON, + Authorization: "", + AcceptEncodingGzip: false, + ContentType: httpapi.ApplicationJSON, + LogBody: true, + MaxBodySize: 0, + Method: http.MethodPost, + Request: &httpapi.RequestDescriptor[*model.OOAPICollectorUpdateRequest]{ + Body: rawBody, + }, + Response: &httpapi.JSONResponseDescriptor[model.OOAPICollectorUpdateResponse]{}, + Timeout: 0, + URLPath: fmt.Sprintf("/report/%s", reportID), + URLQuery: nil, + } +} diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go new file mode 100644 index 0000000000..32c18b1ef7 --- /dev/null +++ b/internal/cmd/dismantle/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/backendclient" + "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/sessionhttpclient" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/logx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionresolver" + "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/version" +) + +func main() { + const softwareName = "dismantle" + const softwareVersion = "0.1.0-dev" + userAgent := fmt.Sprintf( + "%s/%s ooniprobe-engine/%s", + softwareName, softwareVersion, + version.Version, + ) + + logHandler := logx.NewHandlerWithDefaultSettings() + logHandler.Emoji = true + logger := &log.Logger{Level: log.InfoLevel, Handler: logHandler} + progressBar := model.NewPrinterCallbacks(logger) + counter := bytecounter.New() + home := filepath.Join(os.Getenv("HOME"), ".miniooni") + statedir := filepath.Join(home, "kvstore2") + ctx := context.Background() + tunnelDir := filepath.Join(home, "tunnel") + runtimex.Try0(os.MkdirAll(tunnelDir, 0700)) + + kvstore := runtimex.Try1(kvstore.NewFS(statedir)) + + tunnelConfig := &tunnel.Config{ + Name: "tor", + TunnelDir: tunnelDir, + Logger: logger, + } + tunnel, _ := runtimex.Try2(tunnel.Start(ctx, tunnelConfig)) + defer tunnel.Stop() + proxyURL := tunnel.SOCKS5ProxyURL() + + sessionResolver := &sessionresolver.Resolver{ + ByteCounter: counter, + KVStore: kvstore, + Logger: logger, + ProxyURL: proxyURL, + } + defer sessionResolver.CloseIdleConnections() + + geolocateConfig := &geolocate.Config{ + Resolver: sessionResolver, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + } + geolocateTask := geolocate.NewTask(*geolocateConfig) // XXX + location := runtimex.Try1(geolocateTask.Run(ctx)) + logger.Infof("%+v", location) + + sessionHTTPClientConfig := &sessionhttpclient.Config{ + ByteCounter: counter, + Logger: logger, + Resolver: sessionResolver, + ProxyURL: proxyURL, + } + sessionHTTPClient := sessionhttpclient.New(sessionHTTPClientConfig) + defer sessionHTTPClient.CloseIdleConnections() + + backendClientConfig := &backendclient.Config{ + KVStore: kvstore, + HTTPClient: sessionHTTPClient, + Logger: logger, + UserAgent: userAgent, + BaseURL: nil, + ProxyURL: proxyURL, + } + backendClient := backendclient.New(backendClientConfig) + + checkInConfig := &model.OOAPICheckInConfig{ + Charging: false, + OnWiFi: false, + Platform: platform.Name(), + ProbeASN: location.ASNString(), + ProbeCC: location.CountryCode, + RunType: "manual", + SoftwareName: softwareName, + SoftwareVersion: softwareName, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: []string{}, + }, + } + checkInResult := runtimex.Try1(backendClient.CheckIn(ctx, checkInConfig)) + logger.Infof("%+v", checkInResult) + + runtimex.Assert(checkInResult.Tests.WebConnectivity != nil, "no web connectivity info") + reportID := checkInResult.Tests.WebConnectivity.ReportID + + experimentSession := &experimentSession{ + httpClient: sessionHTTPClient, + location: location, + logger: logger, + testHelpers: checkInResult.Conf.TestHelpers, + userAgent: userAgent, + } + + testStartTime := time.Now() + for _, input := range checkInResult.Tests.WebConnectivity.URLs { + cfg := &webconnectivitylte.Config{} + runner := webconnectivitylte.NewExperimentMeasurer(cfg) + measurement := backendclient.NewMeasurement( + location, runner.ExperimentName(), runner.ExperimentVersion(), + testStartTime, reportID, softwareName, softwareVersion, input.URL, + ) + args := &model.ExperimentArgs{ + Callbacks: progressBar, + Measurement: measurement, + Session: experimentSession, + } + if err := runner.Run(ctx, args); err != nil { + logger.Warnf("runner.Run failed: %s", err.Error()) + } + if err := backendClient.Submit(ctx, measurement); err != nil { + logger.Warnf("backendClient.Submit failed: %s", err.Error()) + } + log.Infof("measurement URL: %s", makeExplorerURL(reportID, input.URL)) + } +} + +func makeExplorerURL(reportID, input string) string { + query := url.Values{} + query.Add("input", input) + explorerURL := &url.URL{ + Scheme: "https", + Host: "explorer.ooni.org", + Path: fmt.Sprintf("/measurement/%s", reportID), + RawQuery: query.Encode(), + Fragment: "", + RawFragment: "", + } + return explorerURL.String() +} diff --git a/internal/cmd/dismantle/session.go b/internal/cmd/dismantle/session.go new file mode 100644 index 0000000000..1f2131eb57 --- /dev/null +++ b/internal/cmd/dismantle/session.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +type experimentSession struct { + httpClient model.HTTPClient + location *geolocate.Results + logger model.Logger + testHelpers map[string][]model.OOAPIService + userAgent string +} + +var _ model.ExperimentSession = &experimentSession{} + +// DefaultHTTPClient implements model.ExperimentSession +func (es *experimentSession) DefaultHTTPClient() model.HTTPClient { + return es.httpClient +} + +// FetchPsiphonConfig implements model.ExperimentSession +func (es *experimentSession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// FetchTorTargets implements model.ExperimentSession +func (es *experimentSession) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// GetTestHelpersByName implements model.ExperimentSession +func (es *experimentSession) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { + value, found := es.testHelpers[name] + return value, found +} + +// Logger implements model.ExperimentSession +func (es *experimentSession) Logger() model.Logger { + return es.logger +} + +// ProbeCC implements model.ExperimentSession +func (es *experimentSession) ProbeCC() string { + return es.location.CountryCode +} + +// ResolverIP implements model.ExperimentSession +func (es *experimentSession) ResolverIP() string { + return es.location.ResolverIP +} + +// TempDir implements model.ExperimentSession +func (es *experimentSession) TempDir() string { + panic("unimplemented") // FIXME +} + +// TorArgs implements model.ExperimentSession +func (es *experimentSession) TorArgs() []string { + panic("unimplemented") // FIXME +} + +// TorBinary implements model.ExperimentSession +func (es *experimentSession) TorBinary() string { + panic("unimplemented") // FIXME +} + +// TunnelDir implements model.ExperimentSession +func (es *experimentSession) TunnelDir() string { + panic("unimplemented") // FIXME +} + +// UserAgent implements model.ExperimentSession +func (es *experimentSession) UserAgent() string { + return es.userAgent +} diff --git a/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go b/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go new file mode 100644 index 0000000000..c7b5598170 --- /dev/null +++ b/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go @@ -0,0 +1,34 @@ +// Package sessionhttpclient creates an HTTP client for +// a measurement session. We will use this client for +// communicating with the OONI backend. +package sessionhttpclient + +import ( + "net/url" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Config contains config for creating a new session HTTP client. +type Config struct { + ByteCounter *bytecounter.Counter + Logger model.Logger + Resolver model.Resolver + + // optional fields + ProxyURL *url.URL +} + +// New creates a new HTTP client to be used during a measurement +// session to communicate with the OONI backend. +func New(config *Config) model.HTTPClient { + dialer := netxlite.NewDialerWithResolver(config.Logger, config.Resolver) + dialer = netxlite.MaybeWrapWithProxyDialer(dialer, config.ProxyURL) + handshaker := netxlite.NewTLSHandshakerStdlib(config.Logger) + tlsDialer := netxlite.NewTLSDialer(dialer, handshaker) + txp := netxlite.NewHTTPTransport(config.Logger, dialer, tlsDialer) + txp = bytecounter.MaybeWrapHTTPTransport(txp, config.ByteCounter) + return netxlite.NewHTTPClient(txp) +} diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 925b112741..dd2ca3f8c8 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -87,6 +87,9 @@ type OOAPICheckInResult struct { type OOAPICheckInResultConfig struct { // Features contains feature flags. Features map[string]bool `json:"features"` + + // TestHelpers contains test-helpers information. + TestHelpers map[string][]OOAPIService `json:"test_helpers"` } // OOAPICheckReportIDResponse is the check-report-id API response.