Merge pull request #235 from kairos-io/2281-no-format-custom-partitio…

…ning

2281 - target manually partitioned disk

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/mudler/go-nodepair v0.0.0-20221223092639-ba399a66fdfb
 	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5
 	github.com/mudler/go-processmanager v0.0.0-20230818213616-f204007f963c
-	github.com/mudler/yip v1.5.0
+	github.com/mudler/yip v1.5.1-0.20240410092330-4ebbc7582ee9
 	github.com/nxadm/tail v1.4.11
 	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
@@ -37,9 +37,9 @@ require (
 	github.com/spf13/viper v1.18.2
 	github.com/twpayne/go-vfs/v4 v4.3.0 // v5 requires go1.20
 	github.com/urfave/cli/v2 v2.27.1
-	golang.org/x/net v0.19.0
+	golang.org/x/net v0.21.0
 	golang.org/x/oauth2 v0.15.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.18.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/mount-utils v0.27.4
 )
@@ -110,7 +110,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
-	github.com/itchyny/gojq v0.12.13 // indirect
+	github.com/itchyny/gojq v0.12.15 // indirect
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/jaypipes/pcidb v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -155,7 +155,7 @@ require (
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/qeesung/image2ascii v1.0.1 // indirect
 	github.com/rancher-sandbox/linuxkit v1.0.2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -196,12 +196,12 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/image v0.0.0-20191206065243-da761ea9ff43 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

internal/agent/install.go

@@ -121,7 +121,7 @@ func Install(sourceImgURL string, dir ...string) error {
 			return err
 		}
 
-		if cc.Install.Reboot == false && cc.Install.Poweroff == false {
+		if !cc.Install.Reboot && !cc.Install.Poweroff {
 			pterm.DefaultInteractiveContinue.Show("Installation completed, press enter to go back to the shell.")
 			svc, err := machine.Getty(1)
 			if err == nil {
@@ -194,7 +194,7 @@ func Install(sourceImgURL string, dir ...string) error {
 
 	// If neither reboot and poweroff are enabled let the user insert enter to go back to a new shell
 	// This is helpful to see the installation messages instead of just cleaning the screen with a new tty
-	if cc.Install.Reboot == false && cc.Install.Poweroff == false {
+	if !cc.Install.Reboot && !cc.Install.Poweroff {
 		pterm.DefaultInteractiveContinue.Show("Installation completed, press enter to go back to the shell.")
 
 		utils.Prompt("") //nolint:errcheck
@@ -213,10 +213,6 @@ func RunInstall(c *config.Config) error {
 	utils.SetEnv(c.Env)
 	utils.SetEnv(c.Install.Env)
 
-	if c.Install.Device == "" || c.Install.Device == "auto" {
-		c.Install.Device = detectDevice()
-	}
-
 	// UKI path. Check if we are on UKI AND if we are running off a cd, otherwise it makes no sense to run the install
 	// From the installed system
 	if internalutils.IsUkiWithFs(c.Fs) {
@@ -266,9 +262,6 @@ func runInstall(c *config.Config) error {
 		return err
 	}
 
-	// TODO: This should not be neccessary
-	installSpec.NoFormat = c.Install.NoFormat
-
 	// Set our cloud-init to the file we just created
 	f, err := dumpCCStringToFile(c)
 	if err == nil {

internal/agent/interactive_install.go

@@ -112,23 +112,6 @@ func promptToUnstructured(p events.YAMLPrompt, unstructuredYAML map[string]inter
 	return unstructuredYAML, nil
 }
 
-func detectDevice() string {
-	preferedDevice := "/dev/sda"
-	maxSize := float64(0)
-
-	block, err := ghw.Block()
-	if err == nil {
-		for _, disk := range block.Disks {
-			size := float64(disk.SizeBytes) / float64(GiB)
-			if size > maxSize {
-				maxSize = size
-				preferedDevice = "/dev/" + disk.Name
-			}
-		}
-	}
-	return preferedDevice
-}
-
 func InteractiveInstall(debug, spawnShell bool, sourceImgURL string) error {
 	var sshUsers []string
 	bus.Manager.Initialize()

pkg/action/install.go

@@ -144,13 +144,24 @@ func (i InstallAction) Run() (err error) {
 		}
 	}
 
-	// Check no-format flag
 	if i.spec.NoFormat {
+		i.cfg.Logger.Infof("NoFormat is true, skipping format and partitioning")
 		// Check force flag against current device
 		labels := []string{i.spec.Active.Label, i.spec.Recovery.Label}
 		if e.CheckActiveDeployment(labels) && !i.spec.Force {
 			return fmt.Errorf("use `force` flag to run an installation over the current running deployment")
 		}
+
+		if i.spec.Target == "" || i.spec.Target == "auto" {
+			// This needs to run after the pre-install stage to give the user the
+			// opportunity to prepare the target disk in the pre-install stage.
+			device, err := config.DetectPreConfiguredDevice(i.cfg.Logger)
+			if err != nil {
+				return fmt.Errorf("no target device specified and no device found: %s", err)
+			}
+			i.cfg.Logger.Infof("No target device specified, using pre-configured device: %s", device)
+			i.spec.Target = device
+		}
 	} else {
 		// Deactivate any active volume on target
 		err = e.DeactivateDevices()

pkg/config/config.go

@@ -35,7 +35,7 @@ const (
 type Install struct {
 	Auto                   bool              `yaml:"auto,omitempty"`
 	Reboot                 bool              `yaml:"reboot,omitempty"`
-	NoFormat               bool              `yaml:"no_format,omitempty"`
+	NoFormat               bool              `yaml:"no-format,omitempty"`
 	Device                 string            `yaml:"device,omitempty"`
 	Poweroff               bool              `yaml:"poweroff,omitempty"`
 	GrubOptions            map[string]string `yaml:"grub_options,omitempty"`
@@ -353,8 +353,8 @@ func scan(result *Config, opts ...collector.Option) (c *Config, err error) {
 	genericConfig, err := collector.Scan(o, FilterKeys)
 	if err != nil {
 		return result, err
-
 	}
+
 	result.Config = *genericConfig
 	configStr, err := genericConfig.String()
 	if err != nil {

pkg/config/spec.go

@@ -18,14 +18,16 @@ package config
 
 import (
 	"fmt"
-	sdkTypes "github.com/kairos-io/kairos-sdk/types"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 
+	sdkTypes "github.com/kairos-io/kairos-sdk/types"
+
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/jaypipes/ghw"
 	"golang.org/x/sys/unix"
 
 	"github.com/kairos-io/kairos-agent/v2/pkg/constants"
@@ -37,6 +39,14 @@ import (
 	"github.com/spf13/viper"
 )
 
+const (
+	_ = 1 << (10 * iota)
+	KiB
+	MiB
+	GiB
+	TiB
+)
+
 // NewInstallSpec returns an InstallSpec struct all based on defaults and basic host checks (e.g. EFI vs BIOS)
 func NewInstallSpec(cfg *Config) (*v1.InstallSpec, error) {
 	var firmware string
@@ -107,6 +117,7 @@ func NewInstallSpec(cfg *Config) (*v1.InstallSpec, error) {
 		Active:    activeImg,
 		Recovery:  recoveryImg,
 		Passive:   passiveImg,
+		NoFormat:  cfg.Install.NoFormat,
 	}
 
 	// Get the actual source size to calculate the image size and partitions size
@@ -556,16 +567,11 @@ func ReadInstallSpecFromConfig(c *Config) (*v1.InstallSpec, error) {
 		return &v1.InstallSpec{}, err
 	}
 	installSpec := sp.(*v1.InstallSpec)
-	// Workaround!
-	// If we set the "auto" for the device in the cloudconfig the value will be proper in the Config.Install.Device
-	// But on the cloud-config it will still appear as "auto" as we dont modify that
-	// Unfortunately as we load the full cloud-config and unmarshall it into our spec, we cannot infer from there
-	// What device was choosen, and re-choosing again could lead to different results
-	// So instead we do the check here and override the installSpec.Target with the Config.Install.Device
-	// as its the soonest we have access to both
-	if installSpec.Target == "auto" {
-		installSpec.Target = c.Install.Device
+
+	if (installSpec.Target == "" || installSpec.Target == "auto") && !installSpec.NoFormat {
+		installSpec.Target = detectLargestDevice()
 	}
+
 	return installSpec, nil
 }
 
@@ -656,16 +662,11 @@ func ReadUkiInstallSpecFromConfig(c *Config) (*v1.InstallUkiSpec, error) {
 		return &v1.InstallUkiSpec{}, err
 	}
 	installSpec := sp.(*v1.InstallUkiSpec)
-	// Workaround!
-	// If we set the "auto" for the device in the cloudconfig the value will be proper in the Config.Install.Device
-	// But on the cloud-config it will still appear as "auto" as we dont modify that
-	// Unfortunately as we load the full cloud-config and unmarshall it into our spec, we cannot infer from there
-	// What device was choosen, and re-choosing again could lead to different results
-	// So instead we do the check here and override the installSpec.Target with the Config.Install.Device
-	// as its the soonest we have access to both
-	if installSpec.Target == "auto" {
-		installSpec.Target = c.Install.Device
+
+	if (installSpec.Target == "" || installSpec.Target == "auto") && !installSpec.NoFormat {
+		installSpec.Target = detectLargestDevice()
 	}
+
 	return installSpec, nil
 }
 
@@ -969,3 +970,41 @@ func unmarshallFullSpec(r *Config, subkey string, sp v1.Spec) error {
 
 	return nil
 }
+
+// detectLargestDevice returns the largest disk found
+func detectLargestDevice() string {
+	preferedDevice := "/dev/sda"
+	maxSize := float64(0)
+
+	block, err := ghw.Block()
+	if err == nil {
+		for _, disk := range block.Disks {
+			size := float64(disk.SizeBytes) / float64(GiB)
+			if size > maxSize {
+				maxSize = size
+				preferedDevice = "/dev/" + disk.Name
+			}
+		}
+	}
+	return preferedDevice
+}
+
+// DetectPreConfiguredDevice returns a disk that has partitions labeled with
+// Kairos labels. It can be used to detect a pre-configured device.
+func DetectPreConfiguredDevice(logger sdkTypes.KairosLogger) (string, error) {
+	block, err := ghw.Block()
+	if err != nil {
+		logger.Errorf("failed getting block devices: %s", err.Error())
+		return "", err
+	}
+
+	for _, disk := range block.Disks {
+		for _, p := range disk.Partitions {
+			if p.FilesystemLabel == "COS_STATE" {
+				return filepath.Join("/", "dev", disk.Name), nil
+			}
+		}
+	}
+
+	return "", nil
+}

pkg/types/v1/config.go

@@ -510,6 +510,7 @@ type InstallUkiSpec struct {
 	PowerOff        bool                `yaml:"poweroff,omitempty" mapstructure:"poweroff"`
 	Partitions      ElementalPartitions `yaml:"partitions,omitempty" mapstructure:"partitions"`
 	ExtraPartitions PartitionList       `yaml:"extra-partitions,omitempty" mapstructure:"extra-partitions"`
+	NoFormat        bool                `yaml:"no-format,omitempty" mapstructure:"no-format"`
 	CloudInit       []string            `yaml:"cloud-init,omitempty" mapstructure:"cloud-init"`
 	SkipEntries     []string            `yaml:"skip-entries,omitempty" mapstructure:"skip-entries"`
 }

pkg/uki/install.go

@@ -40,17 +40,32 @@ func (i *InstallAction) Run() (err error) {
 		i.cfg.Logger.Errorf("running kairos-uki-install.pre hook script: %s", err.Error())
 	}
 
-	// Deactivate any active volume on target
-	err = e.DeactivateDevices()
-	if err != nil {
-		i.cfg.Logger.Errorf("deactivating devices: %s", err.Error())
-		return err
-	}
-	// Partition device
-	err = e.PartitionAndFormatDevice(i.spec)
-	if err != nil {
-		i.cfg.Logger.Errorf("partitioning and formating devices: %s", err.Error())
-		return err
+	if i.spec.NoFormat {
+		i.cfg.Logger.Infof("NoFormat is true, skipping format and partitioning")
+		if i.spec.Target == "" || i.spec.Target == "auto" {
+			// This needs to run after the pre-install stage to give the user the
+			// opportunity to prepare the target disk in the pre-install stage.
+			device, err := config.DetectPreConfiguredDevice(i.cfg.Logger)
+			if err != nil {
+				return fmt.Errorf("no target device specified and no device found: %s", err)
+			}
+			i.cfg.Logger.Infof("No target device specified, using pre-configured device: %s", device)
+			i.spec.Target = device
+		}
+	} else {
+		// Deactivate any active volume on target
+		err = e.DeactivateDevices()
+		if err != nil {
+			i.cfg.Logger.Errorf("deactivating devices: %s", err.Error())
+			return err
+		}
+
+		// Partition device
+		err = e.PartitionAndFormatDevice(i.spec)
+		if err != nil {
+			i.cfg.Logger.Errorf("partitioning and formating devices: %s", err.Error())
+			return err
+		}
 	}
 
 	err = e.MountPartitions(i.spec.GetPartitions().PartitionsByMountPoint(false))