diff --git a/cmd/launcher/interactive.go b/cmd/launcher/interactive.go index f82d1c5c9..a74102ad2 100644 --- a/cmd/launcher/interactive.go +++ b/cmd/launcher/interactive.go @@ -10,9 +10,9 @@ import ( "github.com/go-kit/kit/log" "github.com/kolide/launcher/cmd/launcher/internal" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/agent" "github.com/kolide/launcher/pkg/autoupdate" - "github.com/kolide/launcher/pkg/autoupdate/tuf" "github.com/kolide/launcher/pkg/launcher" "github.com/kolide/launcher/pkg/osquery/interactive" ) diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 6b9e9907c..b77a02450 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -33,13 +33,13 @@ import ( "github.com/kolide/launcher/ee/control/consumers/notificationconsumer" desktopRunner "github.com/kolide/launcher/ee/desktop/runner" "github.com/kolide/launcher/ee/localserver" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/agent" "github.com/kolide/launcher/pkg/agent/flags" "github.com/kolide/launcher/pkg/agent/knapsack" "github.com/kolide/launcher/pkg/agent/storage" agentbbolt "github.com/kolide/launcher/pkg/agent/storage/bbolt" "github.com/kolide/launcher/pkg/autoupdate" - "github.com/kolide/launcher/pkg/autoupdate/tuf" "github.com/kolide/launcher/pkg/backoff" "github.com/kolide/launcher/pkg/contexts/ctxlog" "github.com/kolide/launcher/pkg/debug" diff --git a/cmd/launcher/main.go b/cmd/launcher/main.go index 26a6ed4d7..3c677d4f8 100644 --- a/cmd/launcher/main.go +++ b/cmd/launcher/main.go @@ -16,8 +16,8 @@ import ( "github.com/kolide/kit/env" "github.com/kolide/kit/logutil" "github.com/kolide/kit/version" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/autoupdate" - "github.com/kolide/launcher/pkg/autoupdate/tuf" "github.com/kolide/launcher/pkg/contexts/ctxlog" "github.com/kolide/launcher/pkg/execwrapper" "github.com/kolide/launcher/pkg/launcher" diff --git a/pkg/autoupdate/tuf/assets/tuf/root.json b/ee/tuf/assets/tuf/root.json similarity index 100% rename from pkg/autoupdate/tuf/assets/tuf/root.json rename to ee/tuf/assets/tuf/root.json diff --git a/pkg/autoupdate/tuf/autoupdate.go b/ee/tuf/autoupdate.go similarity index 100% rename from pkg/autoupdate/tuf/autoupdate.go rename to ee/tuf/autoupdate.go diff --git a/pkg/autoupdate/tuf/autoupdate_test.go b/ee/tuf/autoupdate_test.go similarity index 99% rename from pkg/autoupdate/tuf/autoupdate_test.go rename to ee/tuf/autoupdate_test.go index d8119b62c..b45a6246f 100644 --- a/pkg/autoupdate/tuf/autoupdate_test.go +++ b/ee/tuf/autoupdate_test.go @@ -15,11 +15,11 @@ import ( "github.com/Masterminds/semver" "github.com/go-kit/kit/log" + tufci "github.com/kolide/launcher/ee/tuf/ci" "github.com/kolide/launcher/pkg/agent/storage" storageci "github.com/kolide/launcher/pkg/agent/storage/ci" "github.com/kolide/launcher/pkg/agent/types" typesmocks "github.com/kolide/launcher/pkg/agent/types/mocks" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" "github.com/kolide/launcher/pkg/threadsafebuffer" mock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/autoupdate/tuf/ci/testdata/README.md b/ee/tuf/ci/testdata/README.md similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/README.md rename to ee/tuf/ci/testdata/README.md diff --git a/pkg/autoupdate/tuf/ci/testdata/darwin_launcher.tar.gz b/ee/tuf/ci/testdata/darwin_launcher.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/darwin_launcher.tar.gz rename to ee/tuf/ci/testdata/darwin_launcher.tar.gz diff --git a/pkg/autoupdate/tuf/ci/testdata/darwin_osqueryd.tar.gz b/ee/tuf/ci/testdata/darwin_osqueryd.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/darwin_osqueryd.tar.gz rename to ee/tuf/ci/testdata/darwin_osqueryd.tar.gz diff --git a/pkg/autoupdate/tuf/ci/testdata/linux_launcher.tar.gz b/ee/tuf/ci/testdata/linux_launcher.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/linux_launcher.tar.gz rename to ee/tuf/ci/testdata/linux_launcher.tar.gz diff --git a/pkg/autoupdate/tuf/ci/testdata/linux_osqueryd.tar.gz b/ee/tuf/ci/testdata/linux_osqueryd.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/linux_osqueryd.tar.gz rename to ee/tuf/ci/testdata/linux_osqueryd.tar.gz diff --git a/pkg/autoupdate/tuf/ci/testdata/windows_launcher.tar.gz b/ee/tuf/ci/testdata/windows_launcher.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/windows_launcher.tar.gz rename to ee/tuf/ci/testdata/windows_launcher.tar.gz diff --git a/pkg/autoupdate/tuf/ci/testdata/windows_osqueryd.tar.gz b/ee/tuf/ci/testdata/windows_osqueryd.tar.gz similarity index 100% rename from pkg/autoupdate/tuf/ci/testdata/windows_osqueryd.tar.gz rename to ee/tuf/ci/testdata/windows_osqueryd.tar.gz diff --git a/pkg/autoupdate/tuf/ci/tuf_client.go b/ee/tuf/ci/tuf_client.go similarity index 100% rename from pkg/autoupdate/tuf/ci/tuf_client.go rename to ee/tuf/ci/tuf_client.go diff --git a/pkg/autoupdate/tuf/ci/tuf_server.go b/ee/tuf/ci/tuf_server.go similarity index 100% rename from pkg/autoupdate/tuf/ci/tuf_server.go rename to ee/tuf/ci/tuf_server.go diff --git a/pkg/autoupdate/tuf/ci/valid_executable.go b/ee/tuf/ci/valid_executable.go similarity index 100% rename from pkg/autoupdate/tuf/ci/valid_executable.go rename to ee/tuf/ci/valid_executable.go diff --git a/pkg/autoupdate/tuf/errors.go b/ee/tuf/errors.go similarity index 100% rename from pkg/autoupdate/tuf/errors.go rename to ee/tuf/errors.go diff --git a/pkg/autoupdate/tuf/finalize_linux.go b/ee/tuf/finalize_linux.go similarity index 100% rename from pkg/autoupdate/tuf/finalize_linux.go rename to ee/tuf/finalize_linux.go diff --git a/pkg/autoupdate/tuf/finalize_other.go b/ee/tuf/finalize_other.go similarity index 100% rename from pkg/autoupdate/tuf/finalize_other.go rename to ee/tuf/finalize_other.go diff --git a/pkg/autoupdate/tuf/library_lock.go b/ee/tuf/library_lock.go similarity index 100% rename from pkg/autoupdate/tuf/library_lock.go rename to ee/tuf/library_lock.go diff --git a/pkg/autoupdate/tuf/library_lookup.go b/ee/tuf/library_lookup.go similarity index 100% rename from pkg/autoupdate/tuf/library_lookup.go rename to ee/tuf/library_lookup.go diff --git a/pkg/autoupdate/tuf/library_lookup_test.go b/ee/tuf/library_lookup_test.go similarity index 99% rename from pkg/autoupdate/tuf/library_lookup_test.go rename to ee/tuf/library_lookup_test.go index c94507666..55a09f73d 100644 --- a/pkg/autoupdate/tuf/library_lookup_test.go +++ b/ee/tuf/library_lookup_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/go-kit/kit/log" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" + tufci "github.com/kolide/launcher/ee/tuf/ci" "github.com/stretchr/testify/require" ) diff --git a/pkg/autoupdate/tuf/library_manager.go b/ee/tuf/library_manager.go similarity index 100% rename from pkg/autoupdate/tuf/library_manager.go rename to ee/tuf/library_manager.go diff --git a/pkg/autoupdate/tuf/library_manager_test.go b/ee/tuf/library_manager_test.go similarity index 99% rename from pkg/autoupdate/tuf/library_manager_test.go rename to ee/tuf/library_manager_test.go index f949f179b..dd1a7868a 100644 --- a/pkg/autoupdate/tuf/library_manager_test.go +++ b/ee/tuf/library_manager_test.go @@ -13,8 +13,8 @@ import ( "testing" "github.com/go-kit/kit/log" + tufci "github.com/kolide/launcher/ee/tuf/ci" "github.com/kolide/launcher/pkg/autoupdate" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" "github.com/stretchr/testify/require" "github.com/theupdateframework/go-tuf/data" ) diff --git a/pkg/autoupdate/tuf/library_manager_windows_test.go b/ee/tuf/library_manager_windows_test.go similarity index 98% rename from pkg/autoupdate/tuf/library_manager_windows_test.go rename to ee/tuf/library_manager_windows_test.go index 51bfe7a49..759975e84 100644 --- a/pkg/autoupdate/tuf/library_manager_windows_test.go +++ b/ee/tuf/library_manager_windows_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/go-kit/kit/log" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" + tufci "github.com/kolide/launcher/ee/tuf/ci" "github.com/stretchr/testify/require" "github.com/theupdateframework/go-tuf/data" "golang.org/x/sys/windows" diff --git a/pkg/autoupdate/tuf/mock_librarian_test.go b/ee/tuf/mock_librarian_test.go similarity index 100% rename from pkg/autoupdate/tuf/mock_librarian_test.go rename to ee/tuf/mock_librarian_test.go diff --git a/pkg/autoupdate/tuf/mock_querier_test.go b/ee/tuf/mock_querier_test.go similarity index 100% rename from pkg/autoupdate/tuf/mock_querier_test.go rename to ee/tuf/mock_querier_test.go diff --git a/pkg/autoupdate/tuf/mockery.md b/ee/tuf/mockery.md similarity index 100% rename from pkg/autoupdate/tuf/mockery.md rename to ee/tuf/mockery.md diff --git a/pkg/autoupdate/tuf/read_only_tuf_client.go b/ee/tuf/read_only_tuf_client.go similarity index 100% rename from pkg/autoupdate/tuf/read_only_tuf_client.go rename to ee/tuf/read_only_tuf_client.go diff --git a/pkg/autoupdate/tuf/util_darwin.go b/ee/tuf/util_darwin.go similarity index 100% rename from pkg/autoupdate/tuf/util_darwin.go rename to ee/tuf/util_darwin.go diff --git a/pkg/autoupdate/tuf/util_linux.go b/ee/tuf/util_linux.go similarity index 100% rename from pkg/autoupdate/tuf/util_linux.go rename to ee/tuf/util_linux.go diff --git a/pkg/autoupdate/tuf/util_test.go b/ee/tuf/util_test.go similarity index 100% rename from pkg/autoupdate/tuf/util_test.go rename to ee/tuf/util_test.go diff --git a/pkg/autoupdate/tuf/util_windows.go b/ee/tuf/util_windows.go similarity index 100% rename from pkg/autoupdate/tuf/util_windows.go rename to ee/tuf/util_windows.go diff --git a/pkg/agent/knapsack/knapsack.go b/pkg/agent/knapsack/knapsack.go index 00cda2c90..4e40656af 100644 --- a/pkg/agent/knapsack/knapsack.go +++ b/pkg/agent/knapsack/knapsack.go @@ -8,11 +8,11 @@ import ( "github.com/go-kit/kit/log" "github.com/kolide/kit/ulid" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/agent/flags/keys" "github.com/kolide/launcher/pkg/agent/storage" "github.com/kolide/launcher/pkg/agent/types" "github.com/kolide/launcher/pkg/autoupdate" - "github.com/kolide/launcher/pkg/autoupdate/tuf" "github.com/kolide/launcher/pkg/log/multislogger" "go.etcd.io/bbolt" ) diff --git a/pkg/debug/checkups/tuf.go b/pkg/debug/checkups/tuf.go index cd7727f59..c97af2361 100644 --- a/pkg/debug/checkups/tuf.go +++ b/pkg/debug/checkups/tuf.go @@ -12,8 +12,8 @@ import ( "runtime" "github.com/go-kit/kit/log" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/agent/types" - "github.com/kolide/launcher/pkg/autoupdate/tuf" ) type ( diff --git a/pkg/debug/checkups/tuf_test.go b/pkg/debug/checkups/tuf_test.go index 3f80564c5..97e282621 100644 --- a/pkg/debug/checkups/tuf_test.go +++ b/pkg/debug/checkups/tuf_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + tufci "github.com/kolide/launcher/ee/tuf/ci" typesmocks "github.com/kolide/launcher/pkg/agent/types/mocks" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" "github.com/stretchr/testify/require" ) diff --git a/pkg/osquery/runtime/runner.go b/pkg/osquery/runtime/runner.go index 476ecbc57..d999294e2 100644 --- a/pkg/osquery/runtime/runner.go +++ b/pkg/osquery/runtime/runner.go @@ -12,8 +12,8 @@ import ( "time" "github.com/go-kit/kit/log/level" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/autoupdate" - "github.com/kolide/launcher/pkg/autoupdate/tuf" "github.com/kolide/launcher/pkg/backoff" "github.com/kolide/launcher/pkg/contexts/ctxlog" "github.com/kolide/launcher/pkg/osquery/runtime/history" diff --git a/pkg/osquery/tables/tufinfo/release_version.go b/pkg/osquery/tables/tufinfo/release_version.go index 9c8cb4092..a03a65fb9 100644 --- a/pkg/osquery/tables/tufinfo/release_version.go +++ b/pkg/osquery/tables/tufinfo/release_version.go @@ -12,8 +12,8 @@ import ( "github.com/osquery/osquery-go/plugin/table" "github.com/theupdateframework/go-tuf/data" + "github.com/kolide/launcher/ee/tuf" "github.com/kolide/launcher/pkg/agent/types" - "github.com/kolide/launcher/pkg/autoupdate/tuf" ) const tufReleaseVersionTableName = "kolide_tuf_release_version" diff --git a/pkg/osquery/tables/tufinfo/release_version_test.go b/pkg/osquery/tables/tufinfo/release_version_test.go index ecc1861a1..802547a8e 100644 --- a/pkg/osquery/tables/tufinfo/release_version_test.go +++ b/pkg/osquery/tables/tufinfo/release_version_test.go @@ -9,9 +9,9 @@ import ( "time" "github.com/google/uuid" + "github.com/kolide/launcher/ee/tuf" + tufci "github.com/kolide/launcher/ee/tuf/ci" "github.com/kolide/launcher/pkg/agent/types/mocks" - "github.com/kolide/launcher/pkg/autoupdate/tuf" - tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci" "github.com/osquery/osquery-go/gen/osquery" "github.com/stretchr/testify/require" diff --git a/pkg/sendbuffer/sendbuffer.go b/pkg/sendbuffer/sendbuffer.go index 09cec52ca..b2e3d006e 100644 --- a/pkg/sendbuffer/sendbuffer.go +++ b/pkg/sendbuffer/sendbuffer.go @@ -16,31 +16,32 @@ type sender interface { } var ( - defaultMaxSize = 128 * 1024 - defaultMaxSendSize = 8 * 1024 + defaultMaxSizeBytes = 512 * 1024 + defaultSendSizeBytes = 8 * 1024 ) type SendBuffer struct { - logs [][]byte - size, maxStorageSize, maxSendSize int - writeMutex, sendMutex sync.Mutex - logger log.Logger - sender sender - sendInterval time.Duration - isSending bool + logs [][]byte + size, maxStorageSizeBytes, maxSendSizeBytes int + sendMutex sync.Mutex + writeMutex sync.RWMutex + logger log.Logger + sender sender + sendInterval time.Duration + isSending bool } type option func(*SendBuffer) -func WithMaxStorageSize(maxSize int) option { +func WithMaxStorageSizeBytes(maxSize int) option { return func(sb *SendBuffer) { - sb.maxStorageSize = maxSize + sb.maxStorageSizeBytes = maxSize } } -func WithMaxSendSize(sendSize int) option { +func WithMaxSendSizeBytes(sendSize int) option { return func(sb *SendBuffer) { - sb.maxSendSize = sendSize + sb.maxSendSizeBytes = sendSize } } @@ -59,12 +60,12 @@ func WithSendInterval(sendInterval time.Duration) option { func New(sender sender, opts ...option) *SendBuffer { sb := &SendBuffer{ - maxStorageSize: defaultMaxSize, - maxSendSize: defaultMaxSendSize, - sender: sender, - sendInterval: 1 * time.Minute, - logger: log.NewNopLogger(), - isSending: false, + maxStorageSizeBytes: defaultMaxSizeBytes, + maxSendSizeBytes: defaultSendSizeBytes, + sender: sender, + sendInterval: 1 * time.Minute, + logger: log.NewNopLogger(), + isSending: false, } for _, opt := range opts { @@ -85,11 +86,11 @@ func (sb *SendBuffer) Write(in []byte) (int, error) { } // if the single data piece is larger than the max send size, drop it and log - if len(in) > sb.maxSendSize { + if len(in) > sb.maxSendSizeBytes { sb.logger.Log( "msg", "dropped data because element greater than max send size", - "size_of_data", len(in), - "max_send_size", sb.maxSendSize, + "size_of_data_bytes", len(in), + "max_send_size_bytes", sb.maxSendSizeBytes, "head", string(in)[0:minInt(len(in), 100)], ) return len(in), nil @@ -97,15 +98,15 @@ func (sb *SendBuffer) Write(in []byte) (int, error) { // if we are full, something has backed up // purge everything - if len(in)+sb.size > sb.maxStorageSize { + if len(in)+sb.size > sb.maxStorageSizeBytes { sb.deleteLogs(len(sb.logs)) sb.logger.Log( "msg", "reached capacity, dropping all data and starting over", - "size_of_data", len(in), - "buffer_size", sb.size, - "size_plus_data", sb.size+len(in), - "max_size", sb.maxStorageSize, + "size_of_data_bytes", len(in), + "buffer_size_bytes", sb.size, + "size_plus_data_bytes", sb.size+len(in), + "max_size", sb.maxStorageSizeBytes, ) } @@ -161,7 +162,8 @@ func (sb *SendBuffer) sendAndPurge() error { defer sb.sendMutex.Unlock() toSendBuff := &bytes.Buffer{} - if err := sb.flushToWriter(toSendBuff); err != nil { + lastKey, err := sb.copyLogs(toSendBuff, sb.maxSendSizeBytes) + if err != nil { return err } @@ -170,34 +172,46 @@ func (sb *SendBuffer) sendAndPurge() error { } if err := sb.sender.Send(toSendBuff); err != nil { - sb.logger.Log("msg", "failed to send, dropping data", "err", err) + sb.logger.Log("msg", "failed to send, will retry", "err", err) + return nil } + // TODO: populate logs with device data (id, serial, munemo, orgid) when we + // get first set of control data with device info before shipping + + // testing on a new enrollment in debug mode, log size hit 130K bytes + // before enrollment completed and was able to ship logs + // 2023-11-16 + sb.writeMutex.Lock() + defer sb.writeMutex.Unlock() + sb.deleteLogs(lastKey) return nil } -func (sb *SendBuffer) flushToWriter(w io.Writer) error { - sb.writeMutex.Lock() - defer sb.writeMutex.Unlock() +// copyLogs writes to the provided writer, peeking at the size of each log +// before for copying and returning when the next log would exceed the maxSize, +// it's up to the caller to delete any copied logs +func (sb *SendBuffer) copyLogs(w io.Writer, maxSizeBytes int) (int, error) { + sb.writeMutex.RLock() + defer sb.writeMutex.RUnlock() size := 0 - removeDataKeysToIndex := 0 + lastLogIndex := 0 for i := 0; i < len(sb.logs); i++ { - if len(sb.logs[i])+size > sb.maxSendSize { + if len(sb.logs[i])+size > maxSizeBytes { break } if _, err := w.Write(sb.logs[i]); err != nil { - return err + return 0, err } size += len(sb.logs[i]) - removeDataKeysToIndex++ + lastLogIndex++ } - sb.deleteLogs(removeDataKeysToIndex) - return nil + return lastLogIndex, nil } func (sb *SendBuffer) deleteLogs(toIndex int) { diff --git a/pkg/sendbuffer/sendbuffer_test.go b/pkg/sendbuffer/sendbuffer_test.go index 77e4a4763..3c4abcb74 100644 --- a/pkg/sendbuffer/sendbuffer_test.go +++ b/pkg/sendbuffer/sendbuffer_test.go @@ -67,8 +67,8 @@ func TestSendBuffer(t *testing.T) { sb := New( &testSender{lastReceived: lastReceivedData, t: t}, - WithMaxStorageSize(tt.maxStorageSize), - WithMaxSendSize(tt.maxSendSize), + WithMaxStorageSizeBytes(tt.maxStorageSize), + WithMaxSendSizeBytes(tt.maxSendSize), ) requireStoreSizeEqualsHttpBufferReportedSize(t, sb) @@ -141,7 +141,7 @@ func TestSendBufferConcurrent(t *testing.T) { testSender := &testSender{lastReceived: &bytes.Buffer{}, t: t} sb := New( testSender, - WithMaxSendSize(tt.maxSendSize), + WithMaxSendSizeBytes(tt.maxSendSize), // run interval in background quickly WithSendInterval(1*time.Millisecond), )