From 8fb98fb6c89224f9b0dab8751bef47956d55905d Mon Sep 17 00:00:00 2001 From: Joe Skazinski Date: Wed, 25 May 2022 13:09:29 -0600 Subject: [PATCH 1/2] fix: refresh dm paths and increase attempts and 30s delay plus additional debug Signed-off-by: Joe Skazinski --- iscsi/iscsi.go | 68 +++++++++++++++++++++++++++------------------- iscsi/multipath.go | 8 ++++++ 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/iscsi/iscsi.go b/iscsi/iscsi.go index 3dcda02..ff2371f 100644 --- a/iscsi/iscsi.go +++ b/iscsi/iscsi.go @@ -16,7 +16,11 @@ import ( "time" ) -const defaultPort = "3260" +const ( + defaultPort = "3260" + maxMultipathAttempts = 10 + multipathDelay = 30 +) var ( debug *log.Logger @@ -211,37 +215,42 @@ func getMultipathDisk(path string) (string, error) { // check to see if any have an entry under /sys/block/dm-*/slaves matching // the device the symlink was pointing at attempts := 1 - dmPaths, err := filepath.Glob("/sys/block/dm-*") - for attempts < 4 { - debug.Printf("[%d] dmPaths=%v", attempts, dmPaths) - if err != nil { - debug.Printf("Glob error: %s", err) - return "", err - } - if len(dmPaths) > 0 { - break - } - time.Sleep(1 * time.Second) - attempts++ + var dmPaths []string + + for attempts < (maxMultipathAttempts + 1) { + var err error dmPaths, err = filepath.Glob("/sys/block/dm-*") - } - for _, dmPath := range dmPaths { - sdevices, err := filepath.Glob(filepath.Join(dmPath, "slaves", "*")) - debug.Printf("dmPath=%v/slaves/*, sdevices=%v", dmPath, sdevices) + debug.Printf("[%d] refresh dmPaths [%d] %v", attempts, len(dmPaths), dmPaths) if err != nil { debug.Printf("Glob error: %s", err) + return "", err } - for _, spath := range sdevices { - s := filepath.Base(spath) - debug.Printf("Basepath: %s", s) - if sdevice == s { - // We've found a matching entry, return the path for the - // dm-* device it was found under - p := filepath.Join("/dev", filepath.Base(dmPath)) - debug.Printf("Found matching multipath device (%s) under dm-* device path (%s), (%v)", sdevice, dmPath, p) - return p, nil + + for _, dmPath := range dmPaths { + sdevices, err := filepath.Glob(filepath.Join(dmPath, "slaves", "*")) + // debug.Printf(".. dmPath=%v, sdevices=[%d]%v", dmPath, len(sdevices), sdevices) + if err != nil { + debug.Printf("Glob error: %s", err) + } + for _, spath := range sdevices { + s := filepath.Base(spath) + // debug.Printf(".. Basepath: %s", s) + if sdevice == s { + // We've found a matching entry, return the path for the + // dm-* device it was found under + p := filepath.Join("/dev", filepath.Base(dmPath)) + debug.Printf("Found matching multipath device (%s) under dm-* device path (%s) p (%v)", sdevice, dmPath, p) + return p, nil + } } } + + // Force a reload of all existing multipath maps + output, err := execCommand("multipath", "-r").CombinedOutput() + debug.Printf("## multipath -r: output=%v, err=%v", output, err) + + time.Sleep(multipathDelay * time.Second) + attempts++ } debug.Printf("Couldn't find dm-* path for path: %s, found non dm-* path: %s", path, devicePath) return "", fmt.Errorf("couldn't find dm-* path for path: %s, found non dm-* path: %s", path, devicePath) @@ -350,7 +359,7 @@ func Connect(c *Connector) (string, error) { } if lastErr != nil { - debug.Printf("Last error occured during iscsi init: \n%v", lastErr) + debug.Printf("Last error occurred during iscsi init: \n%v", lastErr) } for i, path := range devicePaths { @@ -465,11 +474,14 @@ func PersistConnector(c *Connector, filePath string) error { //file := path.Join("mnt", c.VolumeName+".json") f, err := os.Create(filePath) if err != nil { + debug.Printf("ERROR: creating iscsi persistence file %s: %s\n", filePath, err) return fmt.Errorf("error creating iscsi persistence file %s: %s", filePath, err) } defer f.Close() encoder := json.NewEncoder(f) + debug.Printf("ConnectorFromFile (write): file=%v, c=%+v\n", filePath, c) if err = encoder.Encode(c); err != nil { + debug.Printf("ERROR: error encoding connector: %v\n", err) return fmt.Errorf("error encoding connector: %v", err) } return nil @@ -490,7 +502,7 @@ func GetConnectorFromFile(filePath string) (*Connector, error) { return &Connector{}, err } - debug.Printf("data: %+v\n", data) + debug.Printf("ConnectorFromFile (read): file=%s, %+v\n", filePath, data) return &data, nil diff --git a/iscsi/multipath.go b/iscsi/multipath.go index c5e1d2b..64d1d1a 100644 --- a/iscsi/multipath.go +++ b/iscsi/multipath.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "syscall" "time" ) @@ -108,7 +109,13 @@ func RemoveAndClear(device string) error { debug.Printf("Remove and clear multipath device (%s)\n", device) // Remove device-mapper logical device + debug.Printf("dmsetup remove -f %s\n", device) if output, err := execCommand("dmsetup", "remove", "-f", device).CombinedOutput(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if ok && exitErr.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() != 0 { + debug.Printf("ERROR: dmsetup remove -f ExitStatus: %d, err=%v\n", exitErr.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), err) + } + return fmt.Errorf("device-mapper could not remove device: %s (%v)", output, err) } @@ -116,6 +123,7 @@ func RemoveAndClear(device string) error { if _, e := os.Stat(device); os.IsNotExist(e) { debug.Printf("device-mapper logical device %q was removed\n", device) } else { + debug.Printf("dmsetup clear %s\n", device) if output, err := execCommand("dmsetup", "clear", device).CombinedOutput(); err != nil { return fmt.Errorf("device-mapper could not clear device: %s (%v)", output, err) } From 4fb6a1f7acb4f573cfcad17ef1307835787b6bd2 Mon Sep 17 00:00:00 2001 From: Joe Skazinski Date: Thu, 26 May 2022 20:37:48 -0600 Subject: [PATCH 2/2] fix: reduce multipath delay to 10s, use ExitCode Signed-off-by: Joe Skazinski --- iscsi/iscsi.go | 7 ++++--- iscsi/multipath.go | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/iscsi/iscsi.go b/iscsi/iscsi.go index ff2371f..5c237a9 100644 --- a/iscsi/iscsi.go +++ b/iscsi/iscsi.go @@ -19,7 +19,7 @@ import ( const ( defaultPort = "3260" maxMultipathAttempts = 10 - multipathDelay = 30 + multipathDelay = 10 ) var ( @@ -162,7 +162,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int, err = nil if deviceTransport == "tcp" { _, err = osStat(*devicePath) - debug.Printf("os stat device: exist %v device %v", !os.IsNotExist(err), *devicePath) + debug.Printf("[%d] os stat device: exist %v device %v", i, !os.IsNotExist(err), *devicePath) if err != nil && !os.IsNotExist(err) { debug.Printf("Error attempting to stat device: %s", err.Error()) return false, err @@ -171,7 +171,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int, } } else { - debug.Printf("filepathGlob: %s", *devicePath) + debug.Printf("[%d] filepathGlob: %s", i, *devicePath) fpath, _ := filepathGlob(*devicePath) if fpath == nil { err = os.ErrNotExist @@ -191,6 +191,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int, } time.Sleep(time.Second * time.Duration(intervalSeconds)) } + debug.Printf("device does NOT exist [%d*%ds] (%v)", maxRetries, intervalSeconds, *devicePath) return false, err } diff --git a/iscsi/multipath.go b/iscsi/multipath.go index 64d1d1a..4680002 100644 --- a/iscsi/multipath.go +++ b/iscsi/multipath.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "syscall" "time" ) @@ -112,8 +111,8 @@ func RemoveAndClear(device string) error { debug.Printf("dmsetup remove -f %s\n", device) if output, err := execCommand("dmsetup", "remove", "-f", device).CombinedOutput(); err != nil { exitErr, ok := err.(*exec.ExitError) - if ok && exitErr.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() != 0 { - debug.Printf("ERROR: dmsetup remove -f ExitStatus: %d, err=%v\n", exitErr.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), err) + if ok && exitErr.ExitCode() != 0 { + debug.Printf("ERROR: dmsetup remove -f ExitCode: %d, err=%v\n", exitErr.ExitCode(), err) } return fmt.Errorf("device-mapper could not remove device: %s (%v)", output, err)