Merge remote-tracking branch 'upstream/main' into becca/patchelf

cmd/launcher/interactive.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/kolide/launcher/cmd/launcher/internal"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/agent"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	"github.com/kolide/launcher/pkg/launcher"
 	"github.com/kolide/launcher/pkg/osquery/interactive"
 )

cmd/launcher/launcher.go

@@ -33,13 +33,13 @@ import (
 	"github.com/kolide/launcher/ee/control/consumers/notificationconsumer"
 	desktopRunner "github.com/kolide/launcher/ee/desktop/runner"
 	"github.com/kolide/launcher/ee/localserver"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/agent"
 	"github.com/kolide/launcher/pkg/agent/flags"
 	"github.com/kolide/launcher/pkg/agent/knapsack"
 	"github.com/kolide/launcher/pkg/agent/storage"
 	agentbbolt "github.com/kolide/launcher/pkg/agent/storage/bbolt"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	"github.com/kolide/launcher/pkg/backoff"
 	"github.com/kolide/launcher/pkg/contexts/ctxlog"
 	"github.com/kolide/launcher/pkg/debug"

cmd/launcher/main.go

@@ -16,8 +16,8 @@ import (
 	"github.com/kolide/kit/env"
 	"github.com/kolide/kit/logutil"
 	"github.com/kolide/kit/version"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	"github.com/kolide/launcher/pkg/contexts/ctxlog"
 	"github.com/kolide/launcher/pkg/execwrapper"
 	"github.com/kolide/launcher/pkg/launcher"

pkg/autoupdate/tuf/autoupdate_test.go → ee/tuf/autoupdate_test.go

@@ -15,11 +15,11 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/go-kit/kit/log"
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/kolide/launcher/pkg/agent/storage"
 	storageci "github.com/kolide/launcher/pkg/agent/storage/ci"
 	"github.com/kolide/launcher/pkg/agent/types"
 	typesmocks "github.com/kolide/launcher/pkg/agent/types/mocks"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/kolide/launcher/pkg/threadsafebuffer"
 	mock "github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"

pkg/autoupdate/tuf/library_lookup_test.go → ee/tuf/library_lookup_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/kit/log"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/stretchr/testify/require"
 )
 

pkg/autoupdate/tuf/library_manager_test.go → ee/tuf/library_manager_test.go

@@ -13,8 +13,8 @@ import (
 	"testing"
 
 	"github.com/go-kit/kit/log"
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/stretchr/testify/require"
 	"github.com/theupdateframework/go-tuf/data"
 )

pkg/autoupdate/tuf/library_manager_windows_test.go → ee/tuf/library_manager_windows_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/kit/log"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/stretchr/testify/require"
 	"github.com/theupdateframework/go-tuf/data"
 	"golang.org/x/sys/windows"

pkg/agent/knapsack/knapsack.go

@@ -8,11 +8,11 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/kolide/kit/ulid"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/agent/flags/keys"
 	"github.com/kolide/launcher/pkg/agent/storage"
 	"github.com/kolide/launcher/pkg/agent/types"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	"github.com/kolide/launcher/pkg/log/multislogger"
 	"go.etcd.io/bbolt"
 )

pkg/debug/checkups/tuf.go

@@ -12,8 +12,8 @@ import (
 	"runtime"
 
 	"github.com/go-kit/kit/log"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/agent/types"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 )
 
 type (

pkg/debug/checkups/tuf_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	typesmocks "github.com/kolide/launcher/pkg/agent/types/mocks"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/stretchr/testify/require"
 )
 

pkg/osquery/runtime/runner.go

@@ -12,8 +12,8 @@ import (
 	"time"
 
 	"github.com/go-kit/kit/log/level"
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/autoupdate"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	"github.com/kolide/launcher/pkg/backoff"
 	"github.com/kolide/launcher/pkg/contexts/ctxlog"
 	"github.com/kolide/launcher/pkg/osquery/runtime/history"

pkg/osquery/tables/tufinfo/release_version.go

@@ -12,8 +12,8 @@ import (
 	"github.com/osquery/osquery-go/plugin/table"
 	"github.com/theupdateframework/go-tuf/data"
 
+	"github.com/kolide/launcher/ee/tuf"
 	"github.com/kolide/launcher/pkg/agent/types"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 )
 
 const tufReleaseVersionTableName = "kolide_tuf_release_version"

pkg/osquery/tables/tufinfo/release_version_test.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/kolide/launcher/ee/tuf"
+	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/kolide/launcher/pkg/agent/types/mocks"
-	"github.com/kolide/launcher/pkg/autoupdate/tuf"
-	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/osquery/osquery-go/gen/osquery"
 
 	"github.com/stretchr/testify/require"

pkg/sendbuffer/sendbuffer.go

@@ -16,31 +16,32 @@ type sender interface {
 }
 
 var (
-	defaultMaxSize     = 128 * 1024
-	defaultMaxSendSize = 8 * 1024
+	defaultMaxSizeBytes  = 512 * 1024
+	defaultSendSizeBytes = 8 * 1024
 )
 
 type SendBuffer struct {
-	logs                              [][]byte
-	size, maxStorageSize, maxSendSize int
-	writeMutex, sendMutex             sync.Mutex
-	logger                            log.Logger
-	sender                            sender
-	sendInterval                      time.Duration
-	isSending                         bool
+	logs                                        [][]byte
+	size, maxStorageSizeBytes, maxSendSizeBytes int
+	sendMutex                                   sync.Mutex
+	writeMutex                                  sync.RWMutex
+	logger                                      log.Logger
+	sender                                      sender
+	sendInterval                                time.Duration
+	isSending                                   bool
 }
 
 type option func(*SendBuffer)
 
-func WithMaxStorageSize(maxSize int) option {
+func WithMaxStorageSizeBytes(maxSize int) option {
 	return func(sb *SendBuffer) {
-		sb.maxStorageSize = maxSize
+		sb.maxStorageSizeBytes = maxSize
 	}
 }
 
-func WithMaxSendSize(sendSize int) option {
+func WithMaxSendSizeBytes(sendSize int) option {
 	return func(sb *SendBuffer) {
-		sb.maxSendSize = sendSize
+		sb.maxSendSizeBytes = sendSize
 	}
 }
 
@@ -59,12 +60,12 @@ func WithSendInterval(sendInterval time.Duration) option {
 
 func New(sender sender, opts ...option) *SendBuffer {
 	sb := &SendBuffer{
-		maxStorageSize: defaultMaxSize,
-		maxSendSize:    defaultMaxSendSize,
-		sender:         sender,
-		sendInterval:   1 * time.Minute,
-		logger:         log.NewNopLogger(),
-		isSending:      false,
+		maxStorageSizeBytes: defaultMaxSizeBytes,
+		maxSendSizeBytes:    defaultSendSizeBytes,
+		sender:              sender,
+		sendInterval:        1 * time.Minute,
+		logger:              log.NewNopLogger(),
+		isSending:           false,
 	}
 
 	for _, opt := range opts {
@@ -85,27 +86,27 @@ func (sb *SendBuffer) Write(in []byte) (int, error) {
 	}
 
 	// if the single data piece is larger than the max send size, drop it and log
-	if len(in) > sb.maxSendSize {
+	if len(in) > sb.maxSendSizeBytes {
 		sb.logger.Log(
 			"msg", "dropped data because element greater than max send size",
-			"size_of_data", len(in),
-			"max_send_size", sb.maxSendSize,
+			"size_of_data_bytes", len(in),
+			"max_send_size_bytes", sb.maxSendSizeBytes,
 			"head", string(in)[0:minInt(len(in), 100)],
 		)
 		return len(in), nil
 	}
 
 	// if we are full, something has backed up
 	// purge everything
-	if len(in)+sb.size > sb.maxStorageSize {
+	if len(in)+sb.size > sb.maxStorageSizeBytes {
 		sb.deleteLogs(len(sb.logs))
 
 		sb.logger.Log(
 			"msg", "reached capacity, dropping all data and starting over",
-			"size_of_data", len(in),
-			"buffer_size", sb.size,
-			"size_plus_data", sb.size+len(in),
-			"max_size", sb.maxStorageSize,
+			"size_of_data_bytes", len(in),
+			"buffer_size_bytes", sb.size,
+			"size_plus_data_bytes", sb.size+len(in),
+			"max_size", sb.maxStorageSizeBytes,
 		)
 	}
 
@@ -161,7 +162,8 @@ func (sb *SendBuffer) sendAndPurge() error {
 	defer sb.sendMutex.Unlock()
 
 	toSendBuff := &bytes.Buffer{}
-	if err := sb.flushToWriter(toSendBuff); err != nil {
+	lastKey, err := sb.copyLogs(toSendBuff, sb.maxSendSizeBytes)
+	if err != nil {
 		return err
 	}
 
@@ -170,34 +172,46 @@ func (sb *SendBuffer) sendAndPurge() error {
 	}
 
 	if err := sb.sender.Send(toSendBuff); err != nil {
-		sb.logger.Log("msg", "failed to send, dropping data", "err", err)
+		sb.logger.Log("msg", "failed to send, will retry", "err", err)
+		return nil
 	}
+	// TODO: populate logs with device data (id, serial, munemo, orgid) when we
+	// get first set of control data with device info before shipping
+
+	// testing on a new enrollment in debug mode, log size hit 130K bytes
+	// before enrollment completed and was able to ship logs
+	// 2023-11-16
+	sb.writeMutex.Lock()
+	defer sb.writeMutex.Unlock()
+	sb.deleteLogs(lastKey)
 
 	return nil
 }
 
-func (sb *SendBuffer) flushToWriter(w io.Writer) error {
-	sb.writeMutex.Lock()
-	defer sb.writeMutex.Unlock()
+// copyLogs writes to the provided writer, peeking at the size of each log
+// before for copying and returning when the next log would exceed the maxSize,
+// it's up to the caller to delete any copied logs
+func (sb *SendBuffer) copyLogs(w io.Writer, maxSizeBytes int) (int, error) {
+	sb.writeMutex.RLock()
+	defer sb.writeMutex.RUnlock()
 
 	size := 0
-	removeDataKeysToIndex := 0
+	lastLogIndex := 0
 
 	for i := 0; i < len(sb.logs); i++ {
-		if len(sb.logs[i])+size > sb.maxSendSize {
+		if len(sb.logs[i])+size > maxSizeBytes {
 			break
 		}
 
 		if _, err := w.Write(sb.logs[i]); err != nil {
-			return err
+			return 0, err
 		}
 
 		size += len(sb.logs[i])
-		removeDataKeysToIndex++
+		lastLogIndex++
 	}
 
-	sb.deleteLogs(removeDataKeysToIndex)
-	return nil
+	return lastLogIndex, nil
 }
 
 func (sb *SendBuffer) deleteLogs(toIndex int) {

pkg/sendbuffer/sendbuffer_test.go

@@ -67,8 +67,8 @@ func TestSendBuffer(t *testing.T) {
 
 			sb := New(
 				&testSender{lastReceived: lastReceivedData, t: t},
-				WithMaxStorageSize(tt.maxStorageSize),
-				WithMaxSendSize(tt.maxSendSize),
+				WithMaxStorageSizeBytes(tt.maxStorageSize),
+				WithMaxSendSizeBytes(tt.maxSendSize),
 			)
 
 			requireStoreSizeEqualsHttpBufferReportedSize(t, sb)
@@ -141,7 +141,7 @@ func TestSendBufferConcurrent(t *testing.T) {
 			testSender := &testSender{lastReceived: &bytes.Buffer{}, t: t}
 			sb := New(
 				testSender,
-				WithMaxSendSize(tt.maxSendSize),
+				WithMaxSendSizeBytes(tt.maxSendSize),
 				// run interval in background quickly
 				WithSendInterval(1*time.Millisecond),
 			)