[TUF autoupdater] Remove osquery client dependency (#1178)

pkg/autoupdate/tuf/autoupdate.go

@@ -6,6 +6,7 @@ package tuf
 import (
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
+	"github.com/kolide/kit/version"
 	"github.com/kolide/launcher/pkg/agent/types"
 	client "github.com/theupdateframework/go-tuf/client"
 	filejsonstore "github.com/theupdateframework/go-tuf/client/filejsonstore"
@@ -47,18 +49,24 @@ type ReleaseFileCustomMetadata struct {
 
 type librarian interface {
 	Available(binary autoupdatableBinary, targetFilename string) bool
-	AddToLibrary(binary autoupdatableBinary, targetFilename string, targetMetadata data.TargetFileMeta) error
-	TidyLibrary()
+	AddToLibrary(binary autoupdatableBinary, currentVersion string, targetFilename string, targetMetadata data.TargetFileMeta) error
+	TidyLibrary(binary autoupdatableBinary, currentVersion string)
+}
+
+type querier interface {
+	Query(query string) ([]map[string]string, error)
 }
 
 type TufAutoupdater struct {
-	metadataClient *client.Client
-	libraryManager librarian
-	channel        string
-	checkInterval  time.Duration
-	store          types.KVStore // stores autoupdater errors for kolide_tuf_autoupdater_errors table
-	interrupt      chan struct{}
-	logger         log.Logger
+	metadataClient         *client.Client
+	libraryManager         librarian
+	osquerier              querier // used to query for current running osquery version
+	osquerierRetryInterval time.Duration
+	channel                string
+	checkInterval          time.Duration
+	store                  types.KVStore // stores autoupdater errors for kolide_tuf_autoupdater_errors table
+	interrupt              chan struct{}
+	logger                 log.Logger
 }
 
 type TufAutoupdaterOption func(*TufAutoupdater)
@@ -69,14 +77,16 @@ func WithLogger(logger log.Logger) TufAutoupdaterOption {
 	}
 }
 
-func NewTufAutoupdater(k types.Knapsack, metadataHttpClient *http.Client,
-	mirrorHttpClient *http.Client, osquerier querier, opts ...TufAutoupdaterOption) (*TufAutoupdater, error) {
+func NewTufAutoupdater(k types.Knapsack, metadataHttpClient *http.Client, mirrorHttpClient *http.Client,
+	osquerier querier, opts ...TufAutoupdaterOption) (*TufAutoupdater, error) {
 	ta := &TufAutoupdater{
-		channel:       k.UpdateChannel(),
-		interrupt:     make(chan struct{}),
-		checkInterval: k.AutoupdateInterval(),
-		store:         k.AutoupdateErrorsStore(),
-		logger:        log.NewNopLogger(),
+		channel:                k.UpdateChannel(),
+		interrupt:              make(chan struct{}),
+		checkInterval:          k.AutoupdateInterval(),
+		store:                  k.AutoupdateErrorsStore(),
+		osquerier:              osquerier,
+		osquerierRetryInterval: 1 * time.Minute,
+		logger:                 log.NewNopLogger(),
 	}
 
 	for _, opt := range opts {
@@ -92,9 +102,9 @@ func NewTufAutoupdater(k types.Knapsack, metadataHttpClient *http.Client,
 	// If the update directory wasn't set by a flag, use the default location of <launcher root>/updates.
 	updateDirectory := k.UpdateDirectory()
 	if updateDirectory == "" {
-		updateDirectory = filepath.Join(k.RootDirectory(), "updates")
+		updateDirectory = DefaultLibraryDirectory(k.RootDirectory())
 	}
-	ta.libraryManager, err = newUpdateLibraryManager(k.MirrorServerURL(), mirrorHttpClient, updateDirectory, osquerier, ta.logger)
+	ta.libraryManager, err = newUpdateLibraryManager(k.MirrorServerURL(), mirrorHttpClient, updateDirectory, ta.logger)
 	if err != nil {
 		return nil, fmt.Errorf("could not init update library manager: %w", err)
 	}
@@ -138,13 +148,17 @@ func LocalTufDirectory(rootDirectory string) string {
 	return filepath.Join(rootDirectory, tufDirectoryName)
 }
 
+func DefaultLibraryDirectory(rootDirectory string) string {
+	return filepath.Join(rootDirectory, "updates")
+}
+
 // Execute is the TufAutoupdater run loop. It periodically checks to see if a new release
 // has been published; less frequently, it removes old/outdated TUF errors from the bucket
 // we store them in.
 func (ta *TufAutoupdater) Execute() (err error) {
 	// For now, tidy the library on startup. In the future, we will tidy the library
 	// earlier, after version selection.
-	ta.libraryManager.TidyLibrary()
+	ta.tidyLibrary()
 
 	checkTicker := time.NewTicker(ta.checkInterval)
 	defer checkTicker.Stop()
@@ -171,6 +185,54 @@ func (ta *TufAutoupdater) Interrupt(_ error) {
 	ta.interrupt <- struct{}{}
 }
 
+// tidyLibrary gets the current running version for each binary (so that the current version is not removed)
+// and then asks the update library manager to tidy the update library.
+func (ta *TufAutoupdater) tidyLibrary() {
+	for _, binary := range binaries {
+		// Get the current running version to preserve it when tidying the available updates
+		currentVersion, err := ta.currentRunningVersion(binary)
+		if err != nil {
+			level.Debug(ta.logger).Log("msg", "could not get current running version", "binary", binary, "err", err)
+			continue
+		}
+
+		ta.libraryManager.TidyLibrary(binary, currentVersion)
+	}
+}
+
+// currentRunningVersion returns the current running version of the given binary.
+// It will perform retries for osqueryd.
+func (ta *TufAutoupdater) currentRunningVersion(binary autoupdatableBinary) (string, error) {
+	switch binary {
+	case binaryLauncher:
+		launcherVersion := version.Version().Version
+		if launcherVersion == "unknown" {
+			return "", errors.New("unknown launcher version")
+		}
+		return launcherVersion, nil
+	case binaryOsqueryd:
+		// The osqueryd client may not have initialized yet, so retry the version
+		// check a couple times before giving up
+		osquerydVersionCheckRetries := 5
+		var err error
+		for i := 0; i < osquerydVersionCheckRetries; i += 1 {
+			var resp []map[string]string
+			resp, err = ta.osquerier.Query("SELECT version FROM osquery_info;")
+			if err == nil && len(resp) > 0 {
+				if osquerydVersion, ok := resp[0]["version"]; ok {
+					return osquerydVersion, nil
+				}
+			}
+			err = fmt.Errorf("error querying for osquery_info: %w; rows returned: %d", err, len(resp))
+
+			time.Sleep(ta.osquerierRetryInterval)
+		}
+		return "", err
+	default:
+		return "", fmt.Errorf("cannot determine current running version for unexpected binary %s", binary)
+	}
+}
+
 // checkForUpdate fetches latest metadata from the TUF server, then checks to see if there's
 // a new release that we should download. If so, it will add the release to our updates library.
 func (ta *TufAutoupdater) checkForUpdate() error {
@@ -239,11 +301,20 @@ func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets dat
 		return "", fmt.Errorf("could not find release: %w", err)
 	}
 
+	// Get the current running version if available -- don't error out if we can't
+	// get it, since the worst case is that we download an update whose version matches
+	// our install version.
+	var currentVersion string
+	currentVersion, _ = ta.currentRunningVersion(binary)
+	if currentVersion == versionFromTarget(binary, release) {
+		return "", nil
+	}
+
 	if ta.libraryManager.Available(binary, release) {
 		return "", nil
 	}
 
-	if err := ta.libraryManager.AddToLibrary(binary, release, releaseMetadata); err != nil {
+	if err := ta.libraryManager.AddToLibrary(binary, currentVersion, release, releaseMetadata); err != nil {
 		return "", fmt.Errorf("could not add release %s for binary %s to library: %w", release, binary, err)
 	}
 

pkg/autoupdate/tuf/autoupdate_test.go

@@ -13,13 +13,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/Masterminds/semver"
 	"github.com/go-kit/kit/log"
 	"github.com/kolide/launcher/pkg/agent/storage"
 	storageci "github.com/kolide/launcher/pkg/agent/storage/ci"
 	"github.com/kolide/launcher/pkg/agent/types"
 	typesmocks "github.com/kolide/launcher/pkg/agent/types/mocks"
 	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/kolide/launcher/pkg/threadsafebuffer"
+	mock "github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
@@ -73,9 +75,10 @@ func TestExecute(t *testing.T) {
 	mockKnapsack.On("TufServerURL").Return(tufServerUrl)
 	mockKnapsack.On("UpdateDirectory").Return("")
 	mockKnapsack.On("MirrorServerURL").Return("https://example.com")
+	mockQuerier := newMockQuerier(t)
 
 	// Set up autoupdater
-	autoupdater, err := NewTufAutoupdater(mockKnapsack, http.DefaultClient, http.DefaultClient, newMockQuerier(t))
+	autoupdater, err := NewTufAutoupdater(mockKnapsack, http.DefaultClient, http.DefaultClient, mockQuerier)
 	require.NoError(t, err, "could not initialize new TUF autoupdater")
 
 	// Confirm we pulled all config items as expected
@@ -99,14 +102,19 @@ func TestExecute(t *testing.T) {
 	launcherMetadata, err := autoupdater.metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, binaryLauncher, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher")
 
-	// Expect that we attempt to update the library
+	// Expect that we attempt to tidy the library first before running execute loop
 	mockLibraryManager := NewMocklibrarian(t)
 	autoupdater.libraryManager = mockLibraryManager
-	mockLibraryManager.On("TidyLibrary").Return().Once()
+	currentLauncherVersion := "" // cannot determine using version package in test
+	currentOsqueryVersion := "1.1.1"
+	mockQuerier.On("Query", mock.Anything).Return([]map[string]string{{"version": currentOsqueryVersion}}, nil)
+	mockLibraryManager.On("TidyLibrary", binaryOsqueryd, mock.Anything).Return().Once()
+
+	// Expect that we attempt to update the library
 	mockLibraryManager.On("Available", binaryOsqueryd, fmt.Sprintf("osqueryd-%s.tar.gz", testReleaseVersion)).Return(false)
 	mockLibraryManager.On("Available", binaryLauncher, fmt.Sprintf("launcher-%s.tar.gz", testReleaseVersion)).Return(false)
-	mockLibraryManager.On("AddToLibrary", binaryOsqueryd, fmt.Sprintf("osqueryd-%s.tar.gz", testReleaseVersion), osquerydMetadata).Return(nil)
-	mockLibraryManager.On("AddToLibrary", binaryLauncher, fmt.Sprintf("launcher-%s.tar.gz", testReleaseVersion), launcherMetadata).Return(nil)
+	mockLibraryManager.On("AddToLibrary", binaryOsqueryd, currentOsqueryVersion, fmt.Sprintf("osqueryd-%s.tar.gz", testReleaseVersion), osquerydMetadata).Return(nil)
+	mockLibraryManager.On("AddToLibrary", binaryLauncher, currentLauncherVersion, fmt.Sprintf("launcher-%s.tar.gz", testReleaseVersion), launcherMetadata).Return(nil)
 
 	// Let the autoupdater run for a bit
 	go autoupdater.Execute()
@@ -132,6 +140,59 @@ func TestExecute(t *testing.T) {
 	require.Contains(t, logLines[len(logLines)-1], "received interrupt, stopping")
 }
 
+func Test_currentRunningVersion_launcher_errorWhenVersionIsNotSet(t *testing.T) {
+	t.Parallel()
+
+	mockQuerier := newMockQuerier(t)
+	autoupdater := &TufAutoupdater{
+		logger:    log.NewNopLogger(),
+		osquerier: mockQuerier,
+	}
+
+	// In test, version.Version() returns `unknown` for everything, which is not something
+	// that the semver library can parse. So we only expect an error here.
+	launcherVersion, err := autoupdater.currentRunningVersion("launcher")
+	require.Error(t, err, "expected an error fetching current running version of launcher")
+	require.Equal(t, "", launcherVersion)
+}
+
+func Test_currentRunningVersion_osqueryd(t *testing.T) {
+	t.Parallel()
+
+	mockQuerier := newMockQuerier(t)
+	autoupdater := &TufAutoupdater{
+		logger:    log.NewNopLogger(),
+		osquerier: mockQuerier,
+	}
+
+	// Expect to return one row containing the version
+	expectedOsqueryVersion, err := semver.NewVersion("5.10.12")
+	require.NoError(t, err)
+	mockQuerier.On("Query", mock.Anything).Return([]map[string]string{{"version": expectedOsqueryVersion.Original()}}, nil).Once()
+
+	osqueryVersion, err := autoupdater.currentRunningVersion("osqueryd")
+	require.NoError(t, err, "expected no error fetching current running version of osqueryd")
+	require.Equal(t, expectedOsqueryVersion.Original(), osqueryVersion)
+}
+
+func Test_currentRunningVersion_osqueryd_handlesQueryError(t *testing.T) {
+	t.Parallel()
+
+	mockQuerier := newMockQuerier(t)
+	autoupdater := &TufAutoupdater{
+		logger:                 log.NewNopLogger(),
+		osquerier:              mockQuerier,
+		osquerierRetryInterval: 1 * time.Millisecond,
+	}
+
+	// Expect to return an error (five times, since we perform retries)
+	mockQuerier.On("Query", mock.Anything).Return(make([]map[string]string, 0), errors.New("test osqueryd querying error"))
+
+	osqueryVersion, err := autoupdater.currentRunningVersion("osqueryd")
+	require.Error(t, err, "expected an error returning osquery version when querying osquery fails")
+	require.Equal(t, "", osqueryVersion)
+}
+
 func Test_storeError(t *testing.T) {
 	t.Parallel()
 
@@ -149,16 +210,21 @@ func Test_storeError(t *testing.T) {
 	mockKnapsack.On("TufServerURL").Return(testTufServer.URL)
 	mockKnapsack.On("UpdateDirectory").Return("")
 	mockKnapsack.On("MirrorServerURL").Return("https://example.com")
+	mockQuerier := newMockQuerier(t)
 
-	autoupdater, err := NewTufAutoupdater(mockKnapsack, http.DefaultClient, http.DefaultClient, newMockQuerier(t))
+	autoupdater, err := NewTufAutoupdater(mockKnapsack, http.DefaultClient, http.DefaultClient, mockQuerier)
 	require.NoError(t, err, "could not initialize new TUF autoupdater")
 
 	// Confirm we pulled all config items as expected
 	mockKnapsack.AssertExpectations(t)
 
 	mockLibraryManager := NewMocklibrarian(t)
 	autoupdater.libraryManager = mockLibraryManager
-	mockLibraryManager.On("TidyLibrary").Return().Once()
+	mockQuerier.On("Query", mock.Anything).Return([]map[string]string{{"version": "1.1.1"}}, nil).Once()
+
+	// We only expect TidyLibrary to run for osqueryd, since we can't get the current running version
+	// for launcher in tests.
+	mockLibraryManager.On("TidyLibrary", binaryOsqueryd, mock.Anything).Return().Once()
 
 	// Set the check interval to something short so we can accumulate some errors
 	autoupdater.checkInterval = 1 * time.Second
@@ -187,6 +253,7 @@ func Test_storeError(t *testing.T) {
 	require.Greater(t, errorCount, 0, "TUF autoupdater did not record error counts")
 
 	mockLibraryManager.AssertExpectations(t)
+	mockQuerier.AssertExpectations(t)
 }
 
 func Test_cleanUpOldErrors(t *testing.T) {