diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 6e35736d8..a80b49547 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -365,7 +365,7 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err metadataClient := http.DefaultClient metadataClient.Timeout = 1 * time.Minute mirrorClient := http.DefaultClient - mirrorClient.Timeout = 5 * time.Minute // gives us extra time to avoid a timeout on download + mirrorClient.Timeout = 8 * time.Minute // gives us extra time to avoid a timeout on download tufAutoupdater, err := tuf.NewTufAutoupdater( k, metadataClient, diff --git a/docs/architecture/library_lookup.md b/docs/architecture/library_lookup.md new file mode 100644 index 000000000..60f39a021 --- /dev/null +++ b/docs/architecture/library_lookup.md @@ -0,0 +1,23 @@ +## Library lookup + +When launcher looks for the version to run for itself or for osquery, it first +looks through local TUF metadata to see if it knows what version to run for its +given release channel. If it does, and the version is already downloaded, it +will run that version. + +Otherwise, it will look for the most recent version downloaded to its update +library. + +```mermaid +flowchart TB + A[Library lookup] --> B{Do we have a local TUF repo?} + B ---->|No| C[Get most recent version from update library] + C --> D[Return path to most recent version of executable] + D --> H[End] + B -->|Yes| E{release.json target metadata exists?} + E -->|No| C + E --> |Yes| F{Target indicated by release.json\nis downloaded to update library?} + F --> |Yes| G[Return path to selected executable in update library] + F --> |No| C + G --> H +``` diff --git a/pkg/autoupdate/tuf/autoupdate.go b/pkg/autoupdate/tuf/autoupdate.go index a08ebc2c8..8ea4215cf 100644 --- a/pkg/autoupdate/tuf/autoupdate.go +++ b/pkg/autoupdate/tuf/autoupdate.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "os" + "path" "path/filepath" "runtime" "strconv" @@ -102,7 +103,7 @@ func NewTufAutoupdater(k types.Knapsack, metadataHttpClient *http.Client, mirror // If the update directory wasn't set by a flag, use the default location of /updates. updateDirectory := k.UpdateDirectory() if updateDirectory == "" { - updateDirectory = DefaultLibraryDirectory(k.RootDirectory()) + updateDirectory = defaultLibraryDirectory(k.RootDirectory()) } ta.libraryManager, err = newUpdateLibraryManager(k.MirrorServerURL(), mirrorHttpClient, updateDirectory, ta.logger) if err != nil { @@ -148,7 +149,7 @@ func LocalTufDirectory(rootDirectory string) string { return filepath.Join(rootDirectory, tufDirectoryName) } -func DefaultLibraryDirectory(rootDirectory string) string { +func defaultLibraryDirectory(rootDirectory string) string { return filepath.Join(rootDirectory, "updates") } @@ -296,7 +297,7 @@ func (ta *TufAutoupdater) checkForUpdate() error { // downloadUpdate will download a new release for the given binary, if available from TUF // and not already downloaded. func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets data.TargetFiles) (string, error) { - release, releaseMetadata, err := ta.findRelease(binary, targets) + release, releaseMetadata, err := findRelease(binary, targets, ta.channel) if err != nil { return "", fmt.Errorf("could not find release: %w", err) } @@ -322,12 +323,12 @@ func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets dat } // findRelease checks the latest data from TUF (in `targets`) to see whether a new release -// has been published for our channel. If it has, it returns the target for that release +// has been published for the given channel. If it has, it returns the target for that release // and its associated metadata. -func (ta *TufAutoupdater) findRelease(binary autoupdatableBinary, targets data.TargetFiles) (string, data.TargetFileMeta, error) { +func findRelease(binary autoupdatableBinary, targets data.TargetFiles, channel string) (string, data.TargetFileMeta, error) { // First, find the target that the channel release file is pointing to var releaseTarget string - targetReleaseFile := fmt.Sprintf("%s/%s/%s/release.json", binary, runtime.GOOS, ta.channel) + targetReleaseFile := path.Join(string(binary), runtime.GOOS, channel, "release.json") for targetName, target := range targets { if targetName != targetReleaseFile { continue @@ -357,7 +358,7 @@ func (ta *TufAutoupdater) findRelease(binary autoupdatableBinary, targets data.T return filepath.Base(releaseTarget), target, nil } - return "", data.TargetFileMeta{}, fmt.Errorf("could not find metadata for release target %s for binary %s", targetReleaseFile, binary) + return "", data.TargetFileMeta{}, fmt.Errorf("could not find metadata for release target %s for binary %s", releaseTarget, binary) } // storeError saves errors that occur during the periodic check for updates, so that they diff --git a/pkg/autoupdate/tuf/channel_version.go b/pkg/autoupdate/tuf/channel_version.go new file mode 100644 index 000000000..af39e6cd3 --- /dev/null +++ b/pkg/autoupdate/tuf/channel_version.go @@ -0,0 +1,35 @@ +package tuf + +import ( + "fmt" + "net/http" + + "github.com/kolide/launcher/pkg/agent" +) + +// GetChannelVersionFromTufServer returns the tagged version of the given binary for the given release channel. +// It is intended for use in e.g. packaging and `launcher doctor`, where we want to determine the correct +// version tag for the channel but do not necessarily have access to a local TUF repo. +func GetChannelVersionFromTufServer(binary, channel, tufServerUrl string) (string, error) { + tempTufRepoDir, err := agent.MkdirTemp("temp-tuf") + if err != nil { + return "", fmt.Errorf("could not make temporary directory: %w", err) + } + + tempTufClient, err := initMetadataClient(tempTufRepoDir, tufServerUrl, http.DefaultClient) + if err != nil { + return "", fmt.Errorf("could not init metadata client: %w", err) + } + + targets, err := tempTufClient.Update() + if err != nil { + return "", fmt.Errorf("could not update targets: %w", err) + } + + releaseTarget, _, err := findRelease(autoupdatableBinary(binary), targets, channel) + if err != nil { + return "", fmt.Errorf("could not find release: %w", err) + } + + return versionFromTarget(autoupdatableBinary(binary), releaseTarget), nil +} diff --git a/pkg/autoupdate/tuf/library_lookup.go b/pkg/autoupdate/tuf/library_lookup.go new file mode 100644 index 000000000..3042830b5 --- /dev/null +++ b/pkg/autoupdate/tuf/library_lookup.go @@ -0,0 +1,90 @@ +package tuf + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/kolide/launcher/pkg/autoupdate" +) + +type BinaryUpdateInfo struct { + Path string + Version string +} + +// CheckOutLatest returns the path to the latest downloaded executable for our binary, as well +// as its version. +func CheckOutLatest(binary autoupdatableBinary, rootDirectory string, updateDirectory string, channel string, logger log.Logger) (*BinaryUpdateInfo, error) { + if updateDirectory == "" { + updateDirectory = defaultLibraryDirectory(rootDirectory) + } + + update, err := findExecutableFromRelease(binary, LocalTufDirectory(rootDirectory), channel, updateDirectory) + if err == nil { + return update, nil + } + + level.Debug(logger).Log("msg", "could not find executable from release", "err", err) + + // If we can't find the specific release version that we should be on, then just return the executable + // with the most recent version in the library + return mostRecentVersion(binary, updateDirectory) +} + +// findExecutableFromRelease looks at our local TUF repository to find the release for our +// given channel. If it's already downloaded, then we return its path and version. +func findExecutableFromRelease(binary autoupdatableBinary, tufRepositoryLocation string, channel string, baseUpdateDirectory string) (*BinaryUpdateInfo, error) { + // Initialize a read-only TUF metadata client to parse the data we already have downloaded about releases. + metadataClient, err := readOnlyTufMetadataClient(tufRepositoryLocation) + if err != nil { + return nil, errors.New("could not initialize TUF client, cannot find release") + } + + // From already-downloaded metadata, look for the release version + targets, err := metadataClient.Targets() + if err != nil { + return nil, fmt.Errorf("could not get target: %w", err) + } + + targetName, _, err := findRelease(binary, targets, channel) + if err != nil { + return nil, fmt.Errorf("could not find release: %w", err) + } + + targetPath, targetVersion := pathToTargetVersionExecutable(binary, targetName, baseUpdateDirectory) + if autoupdate.CheckExecutable(context.TODO(), targetPath, "--version") != nil { + return nil, fmt.Errorf("version %s from target %s either not yet downloaded or corrupted: %w", targetVersion, targetName, err) + } + + return &BinaryUpdateInfo{ + Path: targetPath, + Version: targetVersion, + }, nil +} + +// mostRecentVersion returns the path to the most recent, valid version available in the library for the +// given binary, along with its version. +func mostRecentVersion(binary autoupdatableBinary, baseUpdateDirectory string) (*BinaryUpdateInfo, error) { + // Pull all available versions from library + validVersionsInLibrary, _, err := sortedVersionsInLibrary(binary, baseUpdateDirectory) + if err != nil { + return nil, fmt.Errorf("could not get sorted versions in library for %s: %w", binary, err) + } + + // No valid versions in the library + if len(validVersionsInLibrary) < 1 { + return nil, errors.New("no versions in library") + } + + // Versions are sorted in ascending order -- return the last one + mostRecentVersionInLibraryRaw := validVersionsInLibrary[len(validVersionsInLibrary)-1] + versionDir := filepath.Join(updatesDirectory(binary, baseUpdateDirectory), mostRecentVersionInLibraryRaw) + return &BinaryUpdateInfo{ + Path: executableLocation(versionDir, binary), + Version: mostRecentVersionInLibraryRaw, + }, nil +} diff --git a/pkg/autoupdate/tuf/library_lookup_test.go b/pkg/autoupdate/tuf/library_lookup_test.go new file mode 100644 index 000000000..cfd020544 --- /dev/null +++ b/pkg/autoupdate/tuf/library_lookup_test.go @@ -0,0 +1,163 @@ +package tuf + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/go-kit/kit/log" + tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" + "github.com/stretchr/testify/require" +) + +func TestCheckOutLatest_withTufRepository(t *testing.T) { + t.Parallel() + + for _, binary := range binaries { + binary := binary + t.Run(string(binary), func(t *testing.T) { + t.Parallel() + + // Set up an update library + rootDir := t.TempDir() + updateDir := defaultLibraryDirectory(rootDir) + + // Set up a local TUF repo + tufDir := LocalTufDirectory(rootDir) + require.NoError(t, os.MkdirAll(tufDir, 488)) + testReleaseVersion := "1.0.30" + expectedTargetName := fmt.Sprintf("%s-%s.tar.gz", binary, testReleaseVersion) + tufci.SeedLocalTufRepo(t, testReleaseVersion, rootDir) + + // Create a corresponding downloaded target + executablePath, executableVersion := pathToTargetVersionExecutable(binary, expectedTargetName, updateDir) + require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755)) + tufci.CopyBinary(t, executablePath) + require.NoError(t, os.Chmod(executablePath, 0755)) + + // Make a more recent version that we should ignore since it isn't the release version + tooRecentTarget := fmt.Sprintf("%s-2.1.1.tar.gz", binary) + tooRecentPath, _ := pathToTargetVersionExecutable(binary, tooRecentTarget, updateDir) + require.NoError(t, os.MkdirAll(filepath.Dir(tooRecentPath), 0755)) + tufci.CopyBinary(t, tooRecentPath) + require.NoError(t, os.Chmod(tooRecentPath, 0755)) + + // Check it + latest, err := CheckOutLatest(binary, rootDir, "", "stable", log.NewNopLogger()) + require.NoError(t, err, "unexpected error on checking out latest") + require.Equal(t, executablePath, latest.Path) + require.Equal(t, executableVersion, latest.Version) + }) + } +} + +func TestCheckOutLatest_withoutTufRepository(t *testing.T) { + t.Parallel() + + for _, binary := range binaries { + binary := binary + t.Run(string(binary), func(t *testing.T) { + t.Parallel() + + // Set up an update library, but no TUF repo + rootDir := t.TempDir() + updateDir := defaultLibraryDirectory(rootDir) + target := fmt.Sprintf("%s-1.1.1.tar.gz", binary) + executablePath, executableVersion := pathToTargetVersionExecutable(binary, target, updateDir) + require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755)) + tufci.CopyBinary(t, executablePath) + require.NoError(t, os.Chmod(executablePath, 0755)) + _, err := os.Stat(executablePath) + require.NoError(t, err, "did not make test binary") + + // Check it + latest, err := CheckOutLatest(binary, rootDir, "", "stable", log.NewNopLogger()) + require.NoError(t, err, "unexpected error on checking out latest") + require.Equal(t, executablePath, latest.Path) + require.Equal(t, executableVersion, latest.Version) + }) + } +} + +func Test_mostRecentVersion(t *testing.T) { + t.Parallel() + + for _, binary := range binaries { + binary := binary + t.Run(string(binary), func(t *testing.T) { + t.Parallel() + + // Create update directories + testBaseDir := t.TempDir() + + // Now, create a version in the update library + firstVersionTarget := fmt.Sprintf("%s-2.2.3.tar.gz", binary) + firstVersionPath, _ := pathToTargetVersionExecutable(binary, firstVersionTarget, testBaseDir) + require.NoError(t, os.MkdirAll(filepath.Dir(firstVersionPath), 0755)) + tufci.CopyBinary(t, firstVersionPath) + require.NoError(t, os.Chmod(firstVersionPath, 0755)) + + // Create an even newer version in the update library + secondVersionTarget := fmt.Sprintf("%s-2.5.3.tar.gz", binary) + secondVersionPath, secondVersion := pathToTargetVersionExecutable(binary, secondVersionTarget, testBaseDir) + require.NoError(t, os.MkdirAll(filepath.Dir(secondVersionPath), 0755)) + tufci.CopyBinary(t, secondVersionPath) + require.NoError(t, os.Chmod(secondVersionPath, 0755)) + + latest, err := mostRecentVersion(binary, testBaseDir) + require.NoError(t, err, "did not expect error getting most recent version") + require.Equal(t, secondVersionPath, latest.Path) + require.Equal(t, secondVersion, latest.Version) + }) + } +} + +func Test_mostRecentVersion_DoesNotReturnInvalidExecutables(t *testing.T) { + t.Parallel() + + for _, binary := range binaries { + binary := binary + t.Run(string(binary), func(t *testing.T) { + t.Parallel() + + // Create update directories + testBaseDir := t.TempDir() + + // Now, create a version in the update library + firstVersionTarget := fmt.Sprintf("%s-2.2.3.tar.gz", binary) + firstVersionPath, firstVersion := pathToTargetVersionExecutable(binary, firstVersionTarget, testBaseDir) + require.NoError(t, os.MkdirAll(filepath.Dir(firstVersionPath), 0755)) + tufci.CopyBinary(t, firstVersionPath) + require.NoError(t, os.Chmod(firstVersionPath, 0755)) + + // Create an even newer, but also corrupt, version in the update library + secondVersionTarget := fmt.Sprintf("%s-2.1.12.tar.gz", binary) + secondVersionPath, _ := pathToTargetVersionExecutable(binary, secondVersionTarget, testBaseDir) + require.NoError(t, os.MkdirAll(filepath.Dir(secondVersionPath), 0755)) + os.WriteFile(secondVersionPath, []byte{}, 0755) + + latest, err := mostRecentVersion(binary, testBaseDir) + require.NoError(t, err, "did not expect error getting most recent version") + require.Equal(t, firstVersionPath, latest.Path) + require.Equal(t, firstVersion, latest.Version) + }) + } +} + +func Test_mostRecentVersion_ReturnsErrorOnNoUpdatesDownloaded(t *testing.T) { + t.Parallel() + + for _, binary := range binaries { + binary := binary + t.Run(string(binary), func(t *testing.T) { + t.Parallel() + + // Create update directories + testBaseDir := t.TempDir() + + _, err := mostRecentVersion(binary, testBaseDir) + require.Error(t, err, "should have returned error when there are no available updates") + }) + } +} diff --git a/pkg/autoupdate/tuf/library_manager.go b/pkg/autoupdate/tuf/library_manager.go index 35c4f4a92..452b75785 100644 --- a/pkg/autoupdate/tuf/library_manager.go +++ b/pkg/autoupdate/tuf/library_manager.go @@ -58,7 +58,7 @@ func newUpdateLibraryManager(mirrorUrl string, mirrorClient *http.Client, baseDi // Create the update library for _, binary := range binaries { - if err := os.MkdirAll(ulm.updatesDirectory(binary), 0755); err != nil { + if err := os.MkdirAll(updatesDirectory(binary, baseDir), 0755); err != nil { return nil, fmt.Errorf("could not make updates directory for %s: %w", binary, err) } } @@ -67,20 +67,22 @@ func newUpdateLibraryManager(mirrorUrl string, mirrorClient *http.Client, baseDi } // updatesDirectory returns the update library location for the given binary. -func (ulm *updateLibraryManager) updatesDirectory(binary autoupdatableBinary) string { - return filepath.Join(ulm.baseDir, string(binary)) +func updatesDirectory(binary autoupdatableBinary, baseUpdateDirectory string) string { + return filepath.Join(baseUpdateDirectory, string(binary)) } // Available determines if the given target is already available in the update library. func (ulm *updateLibraryManager) Available(binary autoupdatableBinary, targetFilename string) bool { - executablePath := ulm.PathToTargetVersionExecutable(binary, targetFilename) + executablePath, _ := pathToTargetVersionExecutable(binary, targetFilename, ulm.baseDir) return autoupdate.CheckExecutable(context.TODO(), executablePath, "--version") == nil } -// PathToTargetVersionExecutable returns the path to the executable for the desired version. -func (ulm *updateLibraryManager) PathToTargetVersionExecutable(binary autoupdatableBinary, targetFilename string) string { - versionDir := filepath.Join(ulm.updatesDirectory(binary), versionFromTarget(binary, targetFilename)) - return executableLocation(versionDir, binary) +// pathToTargetVersionExecutable returns the path to the executable for the desired target, +// along with its version. +func pathToTargetVersionExecutable(binary autoupdatableBinary, targetFilename string, baseUpdateDirectory string) (string, string) { + targetVersion := versionFromTarget(binary, targetFilename) + versionDir := filepath.Join(updatesDirectory(binary, baseUpdateDirectory), targetVersion) + return executableLocation(versionDir, binary), targetVersion } // AddToLibrary adds the given target file to the library for the given binary, @@ -185,7 +187,7 @@ func (ulm *updateLibraryManager) moveVerifiedUpdate(binary autoupdatableBinary, } // All good! Shelve it in the library under its version - newUpdateDirectory := filepath.Join(ulm.updatesDirectory(binary), targetVersion) + newUpdateDirectory := filepath.Join(updatesDirectory(binary, ulm.baseDir), targetVersion) if err := os.Rename(stagedVersionedDirectory, newUpdateDirectory); err != nil { return fmt.Errorf("could not move staged target %s from %s to %s: %w", targetFilename, stagedVersionedDirectory, newUpdateDirectory, err) } @@ -195,7 +197,7 @@ func (ulm *updateLibraryManager) moveVerifiedUpdate(binary autoupdatableBinary, // removeUpdate removes a given version from the given binary's update library. func (ulm *updateLibraryManager) removeUpdate(binary autoupdatableBinary, binaryVersion string) { - directoryToRemove := filepath.Join(ulm.updatesDirectory(binary), binaryVersion) + directoryToRemove := filepath.Join(updatesDirectory(binary, ulm.baseDir), binaryVersion) if err := os.RemoveAll(directoryToRemove); err != nil { level.Debug(ulm.logger).Log("msg", "could not remove update", "err", err, "directory", directoryToRemove) } else { @@ -242,7 +244,7 @@ func (ulm *updateLibraryManager) tidyUpdateLibrary(binary autoupdatableBinary, c const numberOfVersionsToKeep = 3 - versionsInLibrary, invalidVersionsInLibrary, err := ulm.sortedVersionsInLibrary(binary) + versionsInLibrary, invalidVersionsInLibrary, err := sortedVersionsInLibrary(binary, ulm.baseDir) if err != nil { level.Debug(ulm.logger).Log("msg", "could not get versions in library to tidy update library", "err", err) return @@ -279,8 +281,8 @@ func (ulm *updateLibraryManager) tidyUpdateLibrary(binary autoupdatableBinary, c // sortedVersionsInLibrary looks through the update library for the given binary to validate and sort all // available versions. It returns a sorted list of the valid versions, a list of invalid versions, and // an error only when unable to glob for versions. -func (ulm *updateLibraryManager) sortedVersionsInLibrary(binary autoupdatableBinary) ([]string, []string, error) { - rawVersionsInLibrary, err := filepath.Glob(filepath.Join(ulm.updatesDirectory(binary), "*")) +func sortedVersionsInLibrary(binary autoupdatableBinary, baseUpdateDirectory string) ([]string, []string, error) { + rawVersionsInLibrary, err := filepath.Glob(filepath.Join(updatesDirectory(binary, baseUpdateDirectory), "*")) if err != nil { return nil, nil, fmt.Errorf("could not glob for updates in library: %w", err) } @@ -295,7 +297,7 @@ func (ulm *updateLibraryManager) sortedVersionsInLibrary(binary autoupdatableBin continue } - versionDir := filepath.Join(ulm.updatesDirectory(binary), rawVersion) + versionDir := filepath.Join(updatesDirectory(binary, baseUpdateDirectory), rawVersion) if err := autoupdate.CheckExecutable(context.TODO(), executableLocation(versionDir, binary), "--version"); err != nil { invalidVersions = append(invalidVersions, rawVersion) continue diff --git a/pkg/autoupdate/tuf/library_manager_test.go b/pkg/autoupdate/tuf/library_manager_test.go index 81c94e571..2f95b578d 100644 --- a/pkg/autoupdate/tuf/library_manager_test.go +++ b/pkg/autoupdate/tuf/library_manager_test.go @@ -43,12 +43,10 @@ func Test_newUpdateLibraryManager(t *testing.T) { require.True(t, launcherDownloadDir.IsDir(), "launcher download dir is not a directory") } -func TestPathToTargetVersionExecutable(t *testing.T) { +func Test_pathToTargetVersionExecutable(t *testing.T) { t.Parallel() - testBaseDir := filepath.Join(t.TempDir(), "updates") - testLibrary, err := newUpdateLibraryManager("", nil, testBaseDir, log.NewNopLogger()) - require.NoError(t, err, "expected no error when creating library") + testBaseDir := defaultLibraryDirectory(t.TempDir()) testVersion := "1.0.7-30-abcdabcd" testTargetFilename := fmt.Sprintf("launcher-%s.tar.gz", testVersion) @@ -59,8 +57,9 @@ func TestPathToTargetVersionExecutable(t *testing.T) { expectedPath = expectedPath + ".exe" } - actualPath := testLibrary.PathToTargetVersionExecutable(binaryLauncher, testTargetFilename) + actualPath, actualVersion := pathToTargetVersionExecutable(binaryLauncher, testTargetFilename, testBaseDir) require.Equal(t, expectedPath, actualPath, "path mismatch") + require.Equal(t, testVersion, actualVersion, "version mismatch") } func TestAvailable(t *testing.T) { @@ -76,7 +75,7 @@ func TestAvailable(t *testing.T) { // Set up valid "osquery" executable runningOsqueryVersion := "5.5.7" runningTarget := fmt.Sprintf("osqueryd-%s.tar.gz", runningOsqueryVersion) - executablePath := testLibrary.PathToTargetVersionExecutable(binaryOsqueryd, runningTarget) + executablePath, _ := pathToTargetVersionExecutable(binaryOsqueryd, runningTarget, testBaseDir) tufci.CopyBinary(t, executablePath) require.NoError(t, os.Chmod(executablePath, 0755)) @@ -147,10 +146,10 @@ func TestAddToLibrary(t *testing.T) { wg.Wait() // Confirm the update was downloaded - dirInfo, err := os.Stat(filepath.Join(testLibraryManager.updatesDirectory(tt.binary), testReleaseVersion)) + dirInfo, err := os.Stat(filepath.Join(updatesDirectory(tt.binary, testBaseDir), testReleaseVersion)) require.NoError(t, err, "checking that update was downloaded") require.True(t, dirInfo.IsDir()) - executableInfo, err := os.Stat(executableLocation(filepath.Join(testLibraryManager.updatesDirectory(tt.binary), testReleaseVersion), tt.binary)) + executableInfo, err := os.Stat(executableLocation(filepath.Join(updatesDirectory(tt.binary, testBaseDir), testReleaseVersion), tt.binary)) require.NoError(t, err, "checking that downloaded update includes executable") require.False(t, executableInfo.IsDir()) @@ -185,7 +184,7 @@ func TestAddToLibrary_alreadyRunning(t *testing.T) { } // Make sure our update directory exists - require.NoError(t, os.MkdirAll(testLibraryManager.updatesDirectory(binary), 0755)) + require.NoError(t, os.MkdirAll(updatesDirectory(binary, testBaseDir), 0755)) // Set the current running version to the version we're going to request to download currentRunningVersion := "4.3.2" @@ -196,7 +195,7 @@ func TestAddToLibrary_alreadyRunning(t *testing.T) { require.NoError(t, testLibraryManager.AddToLibrary(binary, currentRunningVersion, targetFilename, data.TargetFileMeta{}), "expected no error on adding already-downloaded version to library") // Confirm the requested version was not downloaded - _, err := os.Stat(filepath.Join(testLibraryManager.updatesDirectory(binary), currentRunningVersion)) + _, err := os.Stat(filepath.Join(updatesDirectory(binary, testBaseDir), currentRunningVersion)) require.True(t, os.IsNotExist(err), "should not have downloaded currently-running version") }) } @@ -225,11 +224,11 @@ func TestAddToLibrary_alreadyAdded(t *testing.T) { } // Make sure our update directory exists - require.NoError(t, os.MkdirAll(testLibraryManager.updatesDirectory(binary), 0755)) + require.NoError(t, os.MkdirAll(updatesDirectory(binary, testBaseDir), 0755)) // Ensure that a valid update already exists in that directory for the specified version testVersion := "2.2.2" - executablePath := executableLocation(filepath.Join(testLibraryManager.updatesDirectory(binary), testVersion), binary) + executablePath := executableLocation(filepath.Join(updatesDirectory(binary, testBaseDir), testVersion), binary) tufci.CopyBinary(t, executablePath) require.NoError(t, os.Chmod(executablePath, 0755)) _, err := os.Stat(executablePath) @@ -316,7 +315,7 @@ func TestAddToLibrary_verifyStagedUpdate_handlesInvalidFiles(t *testing.T) { require.Equal(t, 0, len(downloadMatches), "unexpected files found in staged updates directory: %+v", downloadMatches) // Confirm the update was not added to the library - updateMatches, err := filepath.Glob(filepath.Join(testLibraryManager.updatesDirectory(tt.binary), "*")) + updateMatches, err := filepath.Glob(filepath.Join(updatesDirectory(tt.binary, testBaseDir), "*")) require.NoError(t, err, "checking that updates directory does not contain any updates") require.Equal(t, 0, len(updateMatches), "unexpected files found in updates directory: %+v", updateMatches) }) @@ -491,14 +490,14 @@ func TestTidyLibrary(t *testing.T) { require.NoError(t, err, "creating fake download file") f1.Close() - // Confirm we made the files - matches, err := filepath.Glob(filepath.Join(testLibraryManager.stagingDir, "*")) - require.NoError(t, err, "could not glob for files in staged osqueryd download dir") - require.Equal(t, 1, len(matches)) + // Confirm we made the staging files + stagingMatches, err := filepath.Glob(filepath.Join(testLibraryManager.stagingDir, "*")) + require.NoError(t, err, "could not glob for files in staged download dir") + require.Equal(t, 1, len(stagingMatches)) // Set up existing versions for test for existingVersion, isExecutable := range tt.existingVersions { - executablePath := executableLocation(filepath.Join(testLibraryManager.updatesDirectory(binary), existingVersion), binary) + executablePath := executableLocation(filepath.Join(updatesDirectory(binary, testBaseDir), existingVersion), binary) if !isExecutable && runtime.GOOS == "windows" { // We check file extension .exe to confirm executable on Windows, so trim the extension // if this test does not expect the file to be executable. @@ -512,6 +511,11 @@ func TestTidyLibrary(t *testing.T) { } } + // Confirm we made the update files + updateMatches, err := filepath.Glob(filepath.Join(updatesDirectory(binary, testBaseDir), "*")) + require.NoError(t, err, "could not glob for directories in updates dir") + require.Equal(t, len(tt.existingVersions), len(updateMatches)) + // Tidy the library testLibraryManager.TidyLibrary(binary, tt.currentlyRunningVersion) @@ -524,14 +528,14 @@ func TestTidyLibrary(t *testing.T) { // Confirm that the versions we expect are still there for _, expectedPreservedVersion := range tt.expectedPreservedVersions { - info, err := os.Stat(filepath.Join(testLibraryManager.updatesDirectory(binary), expectedPreservedVersion)) + info, err := os.Stat(filepath.Join(updatesDirectory(binary, testBaseDir), expectedPreservedVersion)) require.NoError(t, err, "could not stat update dir that was expected to exist: %s", expectedPreservedVersion) require.True(t, info.IsDir()) } // Confirm all other versions were removed for _, expectedRemovedVersion := range tt.expectedRemovedVersions { - _, err := os.Stat(filepath.Join(testLibraryManager.updatesDirectory(binary), expectedRemovedVersion)) + _, err := os.Stat(filepath.Join(updatesDirectory(binary, testBaseDir), expectedRemovedVersion)) require.Error(t, err, "expected version to be removed: %s", expectedRemovedVersion) require.True(t, os.IsNotExist(err)) } @@ -578,12 +582,8 @@ func Test_sortedVersionsInLibrary(t *testing.T) { require.NoError(t, autoupdate.CheckExecutable(context.TODO(), executablePath, "--version"), "binary created for test is corrupt") } - // Set up test library - testLibrary, err := newUpdateLibraryManager("", nil, testBaseDir, log.NewNopLogger()) - require.NoError(t, err, "unexpected error creating new read-only library") - // Get sorted versions - validVersions, invalidVersions, err := testLibrary.sortedVersionsInLibrary(binaryLauncher) + validVersions, invalidVersions, err := sortedVersionsInLibrary(binaryLauncher, testBaseDir) require.NoError(t, err, "expected no error on sorting versions in library") // Confirm invalid versions are the ones we expect diff --git a/pkg/autoupdate/tuf/read_only_tuf_client.go b/pkg/autoupdate/tuf/read_only_tuf_client.go new file mode 100644 index 000000000..cce524121 --- /dev/null +++ b/pkg/autoupdate/tuf/read_only_tuf_client.go @@ -0,0 +1,78 @@ +package tuf + +import ( + "encoding/json" + "fmt" + "io" + "os" + + client "github.com/theupdateframework/go-tuf/client" + filejsonstore "github.com/theupdateframework/go-tuf/client/filejsonstore" +) + +// readOnlyTufMetadataClient returns a metadata client that can read already-downloaded +// metadata but will not download new metadata. +func readOnlyTufMetadataClient(tufRepositoryLocation string) (*client.Client, error) { + // Initialize a read-only TUF metadata client to parse the data we already have about releases + if _, err := os.Stat(tufRepositoryLocation); os.IsNotExist(err) { + return nil, fmt.Errorf("local TUF dir doesn't exist, cannot create read-only client: %w", err) + } + + localStore, err := newReadOnlyLocalStore(tufRepositoryLocation) + if err != nil { + return nil, fmt.Errorf("could not initialize read-only local TUF store: %w", err) + } + + metadataClient := client.NewClient(localStore, newNoopRemoteStore()) + if err := metadataClient.Init(rootJson); err != nil { + return nil, fmt.Errorf("failed to initialize read-only TUF client with root JSON: %w", err) + } + + return metadataClient, nil +} + +// Wraps TUF's FileJSONStore to be read-only +type readOnlyLocalStore struct { + localFilestore *filejsonstore.FileJSONStore +} + +func newReadOnlyLocalStore(tufRepositoryLocation string) (*readOnlyLocalStore, error) { + localStore, err := filejsonstore.NewFileJSONStore(tufRepositoryLocation) + if err != nil { + return nil, fmt.Errorf("could not initialize local TUF store: %w", err) + } + return &readOnlyLocalStore{ + localFilestore: localStore, + }, nil +} + +func (r *readOnlyLocalStore) Close() error { + return r.localFilestore.Close() +} + +func (r *readOnlyLocalStore) GetMeta() (map[string]json.RawMessage, error) { + return r.localFilestore.GetMeta() +} + +func (r *readOnlyLocalStore) SetMeta(name string, meta json.RawMessage) error { + return nil +} + +func (r *readOnlyLocalStore) DeleteMeta(name string) error { + return nil +} + +// Satisfies TUF's RemoteStore interface for our read-only TUF client +type noopRemoteStore struct{} + +func newNoopRemoteStore() *noopRemoteStore { + return &noopRemoteStore{} +} + +func (n *noopRemoteStore) GetMeta(name string) (stream io.ReadCloser, size int64, err error) { + return nil, 0, nil +} + +func (n *noopRemoteStore) GetTarget(path string) (stream io.ReadCloser, size int64, err error) { + return nil, 0, nil +}