fix: skip dirs with no permissions for release-1.0 (#4125)

.github/scripts/setup-hdfs.sh

@@ -90,5 +90,6 @@ echo "hello world" > /tmp/testfile
 cd ~/app/hadoop-${HADOOP_VERSION}/bin
 ./hdfs dfs -put /tmp/testfile /
 ./hdfs dfs -rm /testfile
+./hdfs dfs -chmod 777 /
 
-echo "hdfs started successfully"
+echo "hdfs started successfully"

.github/workflows/dockerfile-sftp

@@ -1,5 +1,4 @@
-FROM debian:stretch-slim
-RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
+FROM debian:stable-slim
 RUN apt-get clean
 RUN apt-get update
 
@@ -9,5 +8,7 @@ RUN mkdir /run/sshd
 RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
 RUN echo "root:password"|chpasswd
+RUN useradd -m testUser1
+RUN echo "testUser1:password"|chpasswd
 EXPOSE 22
 CMD    ["/usr/sbin/sshd", "-D"]

.github/workflows/unittests.yml

@@ -39,8 +39,8 @@ jobs:
       GLUSTER_VOLUME: jfstest/gv0
       DISPLAY_PROGRESSBAR: false
       HDFS_ADDR: localhost:8020
-      SFTP_HOST: localhost:2222:/upload/
-      SFTP_USER: root
+      SFTP_HOST: localhost:2222:/home/testUser1/upload/
+      SFTP_USER: testUser1
       SFTP_PASS: password
       WEBDAV_TEST_BUCKET: 127.0.0.1:9007
       TIKV_ADDR: 127.0.0.1

pkg/object/file.go

@@ -220,6 +220,10 @@ func walk(path string, info os.FileInfo, isSymlink bool, walkFn WalkFunc) error
 		if err == nil {
 			err = walk(p, in, e.isSymlink, walkFn)
 		}
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", path, err)
+			break
+		}
 		if err != nil && err != filepath.SkipDir && !os.IsNotExist(err) {
 			return err
 		}
@@ -338,6 +342,10 @@ func (d *filestore) ListAll(prefix, marker string) (<-chan Object, error) {
 			}
 
 			if err != nil {
+				if os.IsPermission(err) {
+					logger.Warnf("skip %s: %s", path, err)
+					return nil
+				}
 				if os.IsNotExist(err) {
 					logger.Warnf("skip not exist file or directory: %s", path)
 					return nil

pkg/object/filesystem_test.go

@@ -63,7 +63,7 @@ func TestHDFS2(t *testing.T) {
 	if os.Getenv("HDFS_ADDR") == "" {
 		t.Skip()
 	}
-	dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "", "", "")
+	dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "testUser1", "", "")
 	testFileSystem(t, dfs)
 }
 
@@ -124,6 +124,28 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		t.Fatalf("testKeysEqual fail: %s", err)
 	}
 
+	if ss, ok := s.(FileSystem); ok {
+		for _, mode := range []uint32{0022, 0122, 0422} {
+			t.Logf("test mode %o", os.FileMode(mode))
+			err := ss.Chmod("x/", os.FileMode(mode))
+			if err != nil {
+				t.Fatalf("chmod %ofailed: %s", mode, err)
+			}
+			objs, err = listAll(s, "x", "", 100)
+			if err != nil {
+				t.Fatalf("list failed: %s mode %o", err, mode)
+			}
+			expectedKeys = []string{"x/", "xy.txt", "xyz/", "xyz/xyz.txt"}
+			if err = testKeysEqual(objs, expectedKeys); err != nil {
+				t.Fatalf("testKeysEqual fail: %s mode %o", err, mode)
+			}
+			err = ss.Chmod("x/", os.FileMode(0777))
+			if err != nil {
+				t.Fatalf("chmod %o failed: %s", mode, err)
+			}
+		}
+	}
+
 	objs, err = listAll(s, "xy", "", 100)
 	if err != nil {
 		t.Fatalf("list failed: %s", err)

pkg/object/hdfs.go

@@ -233,6 +233,10 @@ func (h *hdfsclient) ListAll(prefix, marker string) (<-chan Object, error) {
 	go func() {
 		_ = h.walk(root, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
+				if os.IsPermission(err) {
+					logger.Warnf("skip %s: %s", path, err)
+					return nil
+				}
 				if err == io.EOF {
 					err = nil // ignore
 				} else {

pkg/object/webdav.go

@@ -240,6 +240,10 @@ func (w *webdav) ListAll(prefix, marker string) (<-chan Object, error) {
 		defer close(listed)
 		_ = w.Walk(root, func(path string, info fs.FileInfo, err error) error {
 			if err != nil {
+				if gowebdav.IsErrCode(err, http.StatusForbidden) {
+					logger.Warnf("skip %s: %s", path, err)
+					return nil
+				}
 				if gowebdav.IsErrNotFound(err) {
 					logger.Warnf("skip not exist file or directory: %s", path)
 					return nil