Merge branches 'pub/testutil-pkgdata', 'pub/fsutil-explicit-parents' …

…and 'pub/fsutil-dir-clean' into pub/extract-proper-parent-modes

internal/deb/extract.go

@@ -189,6 +189,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error {
 				err := fsutil.Create(&fsutil.CreateOptions{
 					Path: filepath.Join(options.TargetDir, sourcePath),
 					Mode: tarHeader.FileInfo().Mode(),
+					Dirs: true,
 				})
 				if err != nil {
 					return err
@@ -231,6 +232,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error {
 				Mode: tarHeader.FileInfo().Mode(),
 				Data: pathReader,
 				Link: tarHeader.Linkname,
+				Dirs: true,
 			})
 			if err != nil {
 				return err

internal/fsutil/create.go

@@ -13,10 +13,16 @@ type CreateOptions struct {
 	Mode fs.FileMode
 	Data io.Reader
 	Link string
+	Dirs bool // create missing parent directories with 0755 mode
 }
 
 func Create(o *CreateOptions) error {
 	var err error
+	if o.Dirs {
+		if err := os.MkdirAll(filepath.Dir(o.Path), 0755); err != nil {
+			return err
+		}
+	}
 	switch o.Mode & fs.ModeType {
 	case 0:
 		err = createFile(o)
@@ -32,11 +38,7 @@ func Create(o *CreateOptions) error {
 
 func createDir(o *CreateOptions) error {
 	debugf("Creating directory: %s (mode %#o)", o.Path, o.Mode)
-	err := os.MkdirAll(filepath.Dir(o.Path), 0755)
-	if err != nil {
-		return err
-	}
-	err = os.Mkdir(o.Path, o.Mode)
+	err := os.Mkdir(o.Path, o.Mode)
 	if os.IsExist(err) {
 		err = os.Chmod(o.Path, o.Mode)
 	}
@@ -45,10 +47,6 @@ func createDir(o *CreateOptions) error {
 
 func createFile(o *CreateOptions) error {
 	debugf("Writing file: %s (mode %#o)", o.Path, o.Mode)
-	err := os.MkdirAll(filepath.Dir(o.Path), 0755)
-	if err != nil && !os.IsExist(err) {
-		return err
-	}
 	file, err := os.OpenFile(o.Path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, o.Mode)
 	if err != nil {
 		return err
@@ -63,10 +61,6 @@ func createFile(o *CreateOptions) error {
 
 func createSymlink(o *CreateOptions) error {
 	debugf("Creating symlink: %s => %s", o.Path, o.Link)
-	err := os.MkdirAll(filepath.Dir(o.Path), 0755)
-	if err != nil && !os.IsExist(err) {
-		return err
-	}
 	fileinfo, err := os.Lstat(o.Path)
 	if err == nil {
 		if (fileinfo.Mode() & os.ModeSymlink) != 0 {

internal/fsutil/create_test.go

@@ -23,6 +23,7 @@ var createTests = []createTest{{
 		Path: "foo/bar",
 		Data: bytes.NewBufferString("data1"),
 		Mode: 0444,
+		Dirs: true,
 	},
 	result: map[string]string{
 		"/foo/":    "dir 0755",
@@ -33,6 +34,7 @@ var createTests = []createTest{{
 		Path: "foo/bar",
 		Link: "../baz",
 		Mode: fs.ModeSymlink,
+		Dirs: true,
 	},
 	result: map[string]string{
 		"/foo/":    "dir 0755",
@@ -42,6 +44,7 @@ var createTests = []createTest{{
 	options: fsutil.CreateOptions{
 		Path: "foo/bar",
 		Mode: fs.ModeDir | 0444,
+		Dirs: true,
 	},
 	result: map[string]string{
 		"/foo/":     "dir 0755",
@@ -55,6 +58,12 @@ var createTests = []createTest{{
 	result: map[string]string{
 		"/tmp/":     "dir 01775",
 	},
+}, {
+	options: fsutil.CreateOptions{
+		Path: "foo/bar",
+		Mode: fs.ModeDir | 0775,
+	},
+	error: `.*: no such file or directory`,
 }}
 
 func (s *S) TestCreate(c *C) {

internal/fsutil/path.go

@@ -0,0 +1,47 @@
+package fsutil
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// These functions exists because we work with slash terminated directory paths
+// that come from deb package tarballs but standard library path functions
+// treat slash terminated paths as unclean.
+
+// Like filepath.Dir() but trailing slash on input is ignored and the return
+// value always includes trailing slash.
+//
+// Comparison of filepath.Dir() with fsutil.Dir():
+//
+//	filepath.Dir("/foo/bar/")	== "/foo/bar"
+//	filepath.Dir("/foo/bar")	== "/foo"
+//
+//	fsutil.Dir("/foo/bar")		== "/foo/"
+//	fsutil.Dir("/foo/bar/")		== "/foo/"
+func Dir(path string) string {
+	parent := filepath.Dir(filepath.Clean(path))
+	if parent != "/" {
+		parent += "/"
+	}
+	return parent
+}
+
+// Like filepath.Clean() but keeps trailing slash.
+//
+// Comparison of filepath.Clean() with fsutil.Clean():
+//
+//	filepath.Clean("/foo/bar")	== "/foo/bar"
+//	filepath.Clean("/foo/bar/")	== "/foo/bar"
+//	filepath.Clean("/foo/bar/.//)	== "/foo/bar"
+//
+//	fsutil.Clean("/foo/bar")	== "/foo/bar"
+//	fsutil.Clean("/foo/bar/")	== "/foo/bar/"
+//	fsutil.Clean("/foo/bar/.//")	== "/foo/bar/"
+func Clean(path string) string {
+	clean := filepath.Clean(path)
+	if strings.HasSuffix(path, "/") {
+		clean += "/"
+	}
+	return clean
+}

internal/fsutil/path_test.go

@@ -0,0 +1,52 @@
+package fsutil_test
+
+import (
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/fsutil"
+)
+
+var dirTests = []struct {
+	input  string
+	output string
+}{
+	{"/a/b/c", "/a/b/"},
+	{"/a/b/c/", "/a/b/"},
+	{"/a/b/c//", "/a/b/"},
+	{"/a/b/c/././", "/a/b/"},
+	{"/a/b/c/.././", "/a/"},
+	{"/a/b//c", "/a/b/"},
+	{"/a/b/./c", "/a/b/"},
+	{"a/b/./c", "a/b/"},
+	{"./a/b/./c", "a/b/"},
+	{"/", "/"},
+	{"///.///", "/"},
+	{".", "./"},
+	{"", "./"},
+}
+
+var cleanTests = []struct {
+	input  string
+	output string
+}{
+	{"/a/b/c", "/a/b/c"},
+	{"/a/b/c/", "/a/b/c/"},
+	{"/a/b/c/.//", "/a/b/c/"},
+	{"/a/b//./c", "/a/b/c"},
+	{"/a/b//./c/", "/a/b/c/"},
+	{"/a/b/.././c/", "/a/c/"},
+}
+
+func (s *S) TestDir(c *C) {
+	for _, t := range dirTests {
+		c.Logf("%s => %s", t.input, t.output)
+		c.Assert(fsutil.Dir(t.input), Equals, t.output)
+	}
+}
+
+func (s *S) TestClean(c *C) {
+	for _, t := range cleanTests {
+		c.Logf("%s => %s", t.input, t.output)
+		c.Assert(fsutil.Clean(t.input), Equals, t.output)
+	}
+}

internal/setup/fetch.go

@@ -143,6 +143,7 @@ func extractTar(dataReader io.Reader, targetDir string) error {
 			Mode: tarHeader.FileInfo().Mode(),
 			Data: tarReader,
 			Link: tarHeader.Linkname,
+			Dirs: true,
 		})
 		if err != nil {
 			return err

internal/slicer/slicer.go

@@ -36,21 +36,15 @@ func Run(options *RunOptions) error {
 		if path[0] != '/' {
 			panic("bug: tried to add relative path to known paths")
 		}
-		cleanPath := filepath.Clean(path)
-		slashPath := cleanPath
-		if path[len(path)-1] == '/' && cleanPath != "/" {
-			slashPath += "/"
-		}
+		path = fsutil.Clean(path)
 		for {
-			if _, ok := knownPaths[slashPath]; ok {
+			if _, ok := knownPaths[path]; ok {
 				break
 			}
-			knownPaths[slashPath] = true
-			cleanPath = filepath.Dir(cleanPath)
-			if cleanPath == "/" {
+			knownPaths[path] = true
+			if path = fsutil.Dir(path); path == "/" {
 				break
 			}
-			slashPath = cleanPath + "/"
 		}
 	}
 
@@ -113,7 +107,7 @@ func Run(options *RunOptions) error {
 					hasCopyright = true
 				}
 			} else {
-				targetDir := filepath.Dir(strings.TrimRight(targetPath, "/")) + "/"
+				targetDir := fsutil.Dir(targetPath)
 				if targetDir == "" || targetDir == "/" {
 					continue
 				}
@@ -216,6 +210,7 @@ func Run(options *RunOptions) error {
 				Mode: tarHeader.FileInfo().Mode(),
 				Data: fileContent,
 				Link: linkTarget,
+				Dirs: true,
 			})
 			if err != nil {
 				return err