Expose instance status

cmd/launcher/launcher.go

@@ -389,6 +389,7 @@ func runLauncher(ctx context.Context, cancel func(), multiSlogger, systemMultiSl
 		osqueryruntime.WithAugeasLensFunction(augeas.InstallLenses),
 	)
 	runGroup.Add("osqueryRunner", osqueryRunner.Run, osqueryRunner.Interrupt)
+	k.SetInstanceQuerier(osqueryRunner)
 
 	versionInfo := version.Version()
 	k.SystemSlogger().Log(ctx, slog.LevelInfo,

ee/agent/knapsack/knapsack.go

@@ -39,9 +39,9 @@ type knapsack struct {
 
 	slogger, systemSlogger *multislogger.MultiSlogger
 
+	querier types.InstanceQuerier
+
 	// This struct is a work in progress, and will be iteratively added to as needs arise.
-	// Some potential future additions include:
-	// Querier
 }
 
 func New(stores map[storage.Store]types.KVStore, flags types.Flags, db *bbolt.DB, slogger, systemSlogger *multislogger.MultiSlogger) *knapsack {
@@ -87,6 +87,18 @@ func (k *knapsack) AddSlogHandler(handler ...slog.Handler) {
 	k.systemSlogger.AddHandler(handler...)
 }
 
+// Osquery instance querier
+func (k *knapsack) SetInstanceQuerier(q types.InstanceQuerier) {
+	k.querier = q
+}
+
+func (k *knapsack) InstanceStatuses() map[string]types.InstanceStatus {
+	if k.querier == nil {
+		return nil
+	}
+	return k.querier.InstanceStatuses()
+}
+
 // BboltDB interface methods
 func (k *knapsack) BboltDB() *bbolt.DB {
 	return k.db

ee/agent/types/knapsack.go

@@ -9,6 +9,8 @@ type Knapsack interface {
 	BboltDB
 	Flags
 	Slogger
+	InstanceQuerier
+	SetInstanceQuerier(q InstanceQuerier)
 	// LatestOsquerydPath finds the path to the latest osqueryd binary, after accounting for updates.
 	LatestOsquerydPath(ctx context.Context) string
 	// ReadEnrollSecret returns the enroll secret value, checking in various locations.

ee/agent/types/querier.go

@@ -0,0 +1,14 @@
+package types
+
+type InstanceStatus string
+
+const (
+	InstanceStatusNotStarted InstanceStatus = "not_started"
+	InstanceStatusHealthy    InstanceStatus = "healthy"
+	InstanceStatusUnhealthy  InstanceStatus = "unhealthy"
+)
+
+// InstanceQuerier is implemented by pkg/osquery/runtime/runner.go
+type InstanceQuerier interface {
+	InstanceStatuses() map[string]InstanceStatus
+}

ee/debug/checkups/osquery.go

@@ -14,10 +14,11 @@ import (
 )
 
 type osqueryCheckup struct {
-	k              types.Knapsack
-	status         Status
-	executionTimes map[string]any // maps command to how long it took to run, in ms
-	summary        string
+	k                types.Knapsack
+	status           Status
+	executionTimes   map[string]any // maps command to how long it took to run, in ms
+	instanceStatuses map[string]types.InstanceStatus
+	summary          string
 }
 
 func (o *osqueryCheckup) Name() string {
@@ -40,6 +41,8 @@ func (o *osqueryCheckup) Run(ctx context.Context, extraWriter io.Writer) error {
 		return fmt.Errorf("running launcher interactive: %w", err)
 	}
 
+	o.instanceStatuses = o.k.InstanceStatuses()
+
 	return nil
 }
 
@@ -102,5 +105,8 @@ func (o *osqueryCheckup) Summary() string {
 }
 
 func (o *osqueryCheckup) Data() any {
-	return o.executionTimes
+	return map[string]any{
+		"execution_time":    o.executionTimes,
+		"instance_statuses": o.instanceStatuses,
+	}
 }

ee/localserver/request-id.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/kolide/kit/ulid"
+	"github.com/kolide/launcher/ee/agent/types"
 	"github.com/kolide/launcher/pkg/traces"
 )
 
@@ -30,6 +31,7 @@ type (
 
 	status struct {
 		EnrollmentStatus string
+		InstanceStatuses map[string]types.InstanceStatus
 	}
 )
 
@@ -82,6 +84,7 @@ func (ls *localServer) requestIdHandlerFunc(w http.ResponseWriter, r *http.Reque
 		Origin:    r.Header.Get("Origin"),
 		Status: status{
 			EnrollmentStatus: string(enrollmentStatus),
+			InstanceStatuses: ls.knapsack.InstanceStatuses(),
 		},
 	}
 	response.identifiers = ls.identifiers

ee/localserver/request-id_test.go

@@ -27,6 +27,7 @@ func Test_localServer_requestIdHandler(t *testing.T) {
 	mockKnapsack.On("ConfigStore").Return(storageci.NewStore(t, multislogger.NewNopLogger(), storage.ConfigStore.String()))
 	mockKnapsack.On("KolideServerURL").Return("localhost")
 	mockKnapsack.On("CurrentEnrollmentStatus").Return(types.Enrolled, nil)
+	mockKnapsack.On("InstanceStatuses").Return(map[string]types.InstanceStatus{"default": types.InstanceStatusHealthy})
 
 	var logBytes bytes.Buffer
 	slogger := slog.New(slog.NewJSONHandler(&logBytes, &slog.HandlerOptions{

pkg/osquery/runtime/runner.go

@@ -315,3 +315,26 @@ func (r *Runner) Healthy() error {
 
 	return nil
 }
+
+func (r *Runner) InstanceStatuses() map[string]types.InstanceStatus {
+	r.instanceLock.Lock()
+	defer r.instanceLock.Unlock()
+
+	instanceStatuses := make(map[string]types.InstanceStatus)
+	for _, registrationId := range r.registrationIds {
+		instance, ok := r.instances[registrationId]
+		if !ok {
+			instanceStatuses[registrationId] = types.InstanceStatusNotStarted
+			continue
+		}
+
+		if err := instance.Healthy(); err != nil {
+			instanceStatuses[registrationId] = types.InstanceStatusUnhealthy
+			continue
+		}
+
+		instanceStatuses[registrationId] = types.InstanceStatusHealthy
+	}
+
+	return instanceStatuses
+}

pkg/osquery/runtime/runtime_helpers.go

@@ -30,6 +30,6 @@ func platformArgs() []string {
 	return nil
 }
 
-func isExitOk(err error) bool {
+func isExitOk(_ error) bool {
 	return false
 }

pkg/osquery/runtime/runtime_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kolide/launcher/ee/agent/storage"
 	storageci "github.com/kolide/launcher/ee/agent/storage/ci"
 	"github.com/kolide/launcher/ee/agent/storage/inmemory"
+	"github.com/kolide/launcher/ee/agent/types"
 	typesMocks "github.com/kolide/launcher/ee/agent/types/mocks"
 	kolidelog "github.com/kolide/launcher/ee/log/osquerylogs"
 	"github.com/kolide/launcher/pkg/backoff"
@@ -389,6 +390,13 @@ func TestMultipleInstances(t *testing.T) {
 	require.NotEmpty(t, runner.instances[extraRegistrationId].stats.StartTime, "start time should be added to secondary instance stats on start up")
 	require.NotEmpty(t, runner.instances[extraRegistrationId].stats.ConnectTime, "connect time should be added to secondary instance stats on start up")
 
+	// Confirm instance statuses are reported correctly
+	instanceStatuses := runner.InstanceStatuses()
+	require.Contains(t, instanceStatuses, defaultRegistrationId)
+	require.Equal(t, instanceStatuses[defaultRegistrationId], types.InstanceStatusHealthy)
+	require.Contains(t, instanceStatuses, extraRegistrationId)
+	require.Equal(t, instanceStatuses[extraRegistrationId], types.InstanceStatusHealthy)
+
 	waitShutdown(t, runner, logBytes)
 
 	// Confirm both instances exited