new(pkg,cmd): added new stats subcmd.

README.md

@@ -7,6 +7,7 @@ Right now, the tool implements:
 * configs generation (comprehensive of automatic generation from kernel-crawler output)
 * configs cleanup
 * configs validation
+* configs stats
 
 This is enough to port [`update-dbg` image](https://github.com/falcosecurity/test-infra/tree/master/images/update-dbg) to make use of this tool instead of the currently used bash scripts.  
 First benchmarks showed a tremendous perf improvement: old update-dbg scripts took around 50m on my laptop for a single driverversion. The new tool takes ~10s.  
@@ -29,11 +30,12 @@ Available Commands:
   completion  Generate the autocompletion script for the specified shell
   generate    Generate new dbg configs
   help        Help about any command
+  stats       Fetch stats about configs
   validate    Validate dbg configs
 
 Flags:
   -a, --architecture string           architecture to run against. (default "x86_64")
-      --driver-version strings        driver versions to generate configs against.
+      --driver-version strings        driver versions to run against.
       --dry-run                       enable dry-run mode.
   -h, --help                          help for dbg-go
   -l, --log-level string              set log verbosity. (default "INFO")

cmd/root.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/fededp/dbg-go/cmd/cleanup"
 	"github.com/fededp/dbg-go/cmd/generate"
+	"github.com/fededp/dbg-go/cmd/stats"
 	"github.com/fededp/dbg-go/cmd/validate"
 	"github.com/fededp/dbg-go/pkg/root"
 	"github.com/fededp/dbg-go/pkg/utils"
@@ -73,7 +74,7 @@ func init() {
 	flags.StringP("log-level", "l", logger.LevelInfo.String(), "set log verbosity.")
 	flags.String("repo-root", cwd, "test-infra repository root path.")
 	flags.StringP("architecture", "a", utils.FromDebArch(runtime.GOARCH), "architecture to run against.")
-	flags.StringSlice("driver-version", nil, "driver versions to generate configs against.")
+	flags.StringSlice("driver-version", nil, "driver versions to run against.")
 	flags.String("target-kernelrelease", "",
 		`target kernel release to work against. By default tool will work on any kernel release. Can be a regex.`)
 	flags.String("target-kernelversion", "",
@@ -97,6 +98,7 @@ func init() {
 	rootCmd.AddCommand(generate.Cmd)
 	rootCmd.AddCommand(cleanup.Cmd)
 	rootCmd.AddCommand(validate.Cmd)
+	rootCmd.AddCommand(stats.Cmd)
 }
 
 func initLogger(subcmd string) error {

cmd/stats/stats.go

@@ -0,0 +1,19 @@
+package stats
+
+import (
+	"github.com/fededp/dbg-go/pkg/root"
+	"github.com/fededp/dbg-go/pkg/stats"
+	"github.com/spf13/cobra"
+)
+
+var (
+	Cmd = &cobra.Command{
+		Use:   "stats",
+		Short: "Fetch stats about configs",
+		RunE:  execute,
+	}
+)
+
+func execute(c *cobra.Command, args []string) error {
+	return stats.Run(stats.Options{Options: root.LoadRootOptions()})
+}

go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/falcosecurity/driverkit v0.14.1-0.20230828134718-835307efe091
+	github.com/olekukonko/tablewriter v0.0.4
 	github.com/ompluscator/dynamic-struct v1.4.0
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.16.0
@@ -23,12 +24,14 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.1 // indirect

go.sum

@@ -158,10 +158,15 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
 github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/ompluscator/dynamic-struct v1.4.0 h1:I/Si9LZtItSwiTMe7vosEuIu2TKdOvWbE3R/lokpN4Q=
 github.com/ompluscator/dynamic-struct v1.4.0/go.mod h1:ADQ1+6Ox1D+ntuNwTHyl1NvpAqY2lBXPSPbcO4CJdeA=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -178,6 +183,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=

pkg/cleanup/cleanup.go

@@ -3,10 +3,8 @@ package cleanup
 import (
 	"fmt"
 	"github.com/fededp/dbg-go/pkg/root"
-	"github.com/fededp/dbg-go/pkg/validate"
 	logger "log/slog"
 	"os"
-	"path/filepath"
 )
 
 func Run(opts Options) error {
@@ -41,27 +39,8 @@ func cleanupFolder(opts Options, driverVersion string) error {
 }
 
 func cleanupMatchingConfigs(opts Options, driverVersion string) error {
-	configNameGlob := validate.ConfGlobFromDistro(opts.Target)
-
-	configPath := fmt.Sprintf(root.ConfigPathFmt,
-		opts.RepoRoot,
-		driverVersion,
-		opts.Architecture,
-		configNameGlob)
-
-	files, err := filepath.Glob(configPath)
-	if err != nil {
-		return err
-	}
-	for _, f := range files {
-		logger.Info("removing file", "path", f)
-		if opts.DryRun {
-			logger.Info("skipping because of dry-run.")
-			continue
-		}
-		if err = os.Remove(f); err != nil {
-			return err
-		}
-	}
-	return nil
+	opts.DriverVersion = []string{driverVersion} // locally overwrite driverVersions to only match current driverVersion
+	return root.LoopConfigsFiltered(opts.Options, "removing file", func(driverVersion, configPath string) error {
+		return os.Remove(configPath)
+	})
 }

pkg/cleanup/cleanup_test.go

@@ -196,7 +196,7 @@ func TestCleanupFiltered(t *testing.T) {
 			// then, for each parsed logged line, check if it contains one of the requested string by the test.
 			// Count all "containing" lines; they must match total lines logged (that have a "config:" key).
 			type MessageJSON struct {
-				Path string `json:"path,omitempty"`
+				Path string `json:"config,omitempty"`
 			}
 			var messageJSON MessageJSON
 			scanner := bufio.NewScanner(&buf)
@@ -219,6 +219,7 @@ func TestCleanupFiltered(t *testing.T) {
 			if found != lines {
 				t.Errorf("wrong number of printed lines; expected %d, found %d", lines, found)
 			}
+			buf.Reset()
 		})
 	}
 

pkg/root/types.go

@@ -1,8 +1,10 @@
 package root
 
 import (
+	"fmt"
 	"github.com/spf13/viper"
 	logger "log/slog"
+	"strings"
 )
 
 type Target struct {
@@ -15,6 +17,29 @@ func (t Target) IsSet() bool {
 	return t.Distro != "" && t.KernelRelease != "" && t.KernelVersion != ""
 }
 
+func (t Target) toGlob() string {
+	// Empty filters fallback at ".*" since we are using a regex match below
+	if t.Distro == "" {
+		t.Distro = "*"
+	} else {
+		dkDistro, found := SupportedDistros[t.Distro]
+		if found {
+			// Filenames use driverkit lowercase target, instead of the kernel-crawler naming.
+			t.Distro = dkDistro
+		} else {
+			// Perhaps a regex? ToLower and pray
+			t.Distro = strings.ToLower(t.Distro)
+		}
+	}
+	if t.KernelRelease == "" {
+		t.KernelRelease = "*"
+	}
+	if t.KernelVersion == "" {
+		t.KernelVersion = "*"
+	}
+	return fmt.Sprintf("%s_%s_%s.yaml", t.Distro, t.KernelRelease, t.KernelVersion)
+}
+
 type Options struct {
 	DryRun        bool
 	RepoRoot      string

pkg/root/utils.go

@@ -0,0 +1,36 @@
+package root
+
+import (
+	"fmt"
+	logger "log/slog"
+	"path/filepath"
+)
+
+type ConfigLooper func(driverVersion, configPath string) error
+
+func LoopConfigsFiltered(opts Options, message string, worker ConfigLooper) error {
+	configNameGlob := opts.Target.toGlob()
+	for _, driverVersion := range opts.DriverVersion {
+		configPath := fmt.Sprintf(ConfigPathFmt,
+			opts.RepoRoot,
+			driverVersion,
+			opts.Architecture,
+			configNameGlob)
+		configs, err := filepath.Glob(configPath)
+		if err != nil {
+			return err
+		}
+		for _, config := range configs {
+			logger.Info(message, "config", config)
+			if opts.DryRun {
+				logger.Info("skipping because of dry-run.")
+				return nil
+			}
+			err = worker(driverVersion, config)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/stats/stats.go

@@ -0,0 +1,84 @@
+package stats
+
+import (
+	"github.com/fededp/dbg-go/pkg/root"
+	"github.com/fededp/dbg-go/pkg/validate"
+	"github.com/olekukonko/tablewriter"
+	"gopkg.in/yaml.v3"
+	"log"
+	logger "log/slog"
+	"os"
+	"strconv"
+)
+
+func Run(opts Options) error {
+	logger.Info("fetching stats from existing config files")
+	driverStatsByVersion := make(map[string]driverStats)
+	totalDriverStats := driverStats{}
+	err := root.LoopConfigsFiltered(opts.Options, "computing stats", func(driverVersion, configPath string) error {
+		dStats := driverStatsByVersion[driverVersion]
+		err := getConfigStats(&dStats, configPath)
+		driverStatsByVersion[driverVersion] = dStats
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	table := tablewriter.NewWriter(log.Default().Writer())
+	table.SetHeader([]string{"Version", "Modules", "Probes", "Headers", "KernelConfigData"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+
+	data := make([]string, 5)
+	for key, stat := range driverStatsByVersion {
+		data[0] = key
+		data[1] = strconv.FormatInt(stat.NumModules, 10)
+		data[2] = strconv.FormatInt(stat.NumProbes, 10)
+		data[3] = strconv.FormatInt(stat.NumHeaders, 10)
+		data[4] = strconv.FormatInt(stat.NumKernelConfigDatas, 10)
+		table.Append(data)
+
+		totalDriverStats.NumModules += stat.NumModules
+		totalDriverStats.NumProbes += stat.NumProbes
+		totalDriverStats.NumHeaders += stat.NumHeaders
+		totalDriverStats.NumKernelConfigDatas += stat.NumKernelConfigDatas
+	}
+	data[0] = "TOTALS"
+	data[1] = strconv.FormatInt(totalDriverStats.NumModules, 10)
+	data[2] = strconv.FormatInt(totalDriverStats.NumProbes, 10)
+	data[3] = strconv.FormatInt(totalDriverStats.NumHeaders, 10)
+	data[4] = strconv.FormatInt(totalDriverStats.NumKernelConfigDatas, 10)
+	table.Append(data)
+	table.Render() // Send output
+
+	return nil
+}
+
+func getConfigStats(dStats *driverStats, configPath string) error {
+	configData, err := os.ReadFile(configPath)
+	if err != nil {
+		return err
+	}
+	var driverkitYaml validate.DriverkitYaml
+	err = yaml.Unmarshal(configData, &driverkitYaml)
+	if err != nil {
+		return err
+	}
+
+	logger.Debug("fetching stats", "parsedConfig", driverkitYaml)
+
+	if driverkitYaml.Output.Probe != "" {
+		dStats.NumProbes++
+	}
+	if driverkitYaml.Output.Module != "" {
+		dStats.NumModules++
+	}
+	if len(driverkitYaml.KernelUrls) > 0 {
+		dStats.NumHeaders++
+	}
+	if driverkitYaml.KernelConfigData != "" {
+		dStats.NumKernelConfigDatas++
+	}
+	return nil
+}