Fix file path appended with slash issue (#1837)

littlezhou · Jun 21, 2018 · 266dbea · 266dbea
1 parent e8f2c8c
commit 266dbea
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 9 deletions.
diff --git a/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java
@@ -378,20 +378,23 @@ private void insertDeleteDiff(String path, boolean isDir) throws MetaStoreExcept
 
   private void insertDeleteDiff(String path) throws MetaStoreException {
     // TODO: remove "/" appended in src or dest in backup_file table
+    String pathWithSlash;
     if (!path.endsWith("/")) {
-      path = path + "/";
+      pathWithSlash = path + "/";
+    } else {
+      pathWithSlash = path;
     }
-    if (inBackup(path)) {
-      List<BackUpInfo> backUpInfos = metaStore.getBackUpInfoBySrc(path);
+    if (inBackup(pathWithSlash)) {
+      List<BackUpInfo> backUpInfos = metaStore.getBackUpInfoBySrc(pathWithSlash);
       for (BackUpInfo backUpInfo : backUpInfos) {
-        String destPath = path.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest());
+        String destPath = pathWithSlash.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest());
         try {
           // tackle root path case
           URI namenodeUri = new URI(destPath);
           String root = "hdfs://" + namenodeUri.getHost() + ":"
               + String.valueOf(namenodeUri.getPort());
           if (destPath.equals(root) || destPath.equals(root + "/") || destPath.equals("/")) {
-            for (String srcFilePath : getFilesUnderDir(path)) {
+            for (String srcFilePath : getFilesUnderDir(pathWithSlash)) {
               FileDiff fileDiff = new FileDiff(FileDiffType.DELETE);
               fileDiff.setSrc(srcFilePath);
               String destFilePath = srcFilePath.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest());
@@ -400,8 +403,9 @@ private void insertDeleteDiff(String path) throws MetaStoreException {
             }
           } else {
             FileDiff fileDiff = new FileDiff(FileDiffType.DELETE);
+            // use the path getting from event with no slash appended
             fileDiff.setSrc(path);
-            //put sync's dest path in parameter for delete use
+            // put sync's dest path in parameter for delete use
             fileDiff.getParameters().put("-dest", destPath);
             metaStore.insertFileDiff(fileDiff);
           }
@@ -417,14 +421,32 @@ private List<String> getFilesUnderDir(String dir) throws MetaStoreException {
       dir = dir + "/";
     }
     List<String> fileList = new ArrayList<>();
-    List<FileInfo> fileInfos = metaStore.getFilesByPrefix(dir);
+    List<String> subdirList = new ArrayList<>();
+    // get fileInfo in asc order of path to guarantee that
+    // the subdir is tackled prior to files or dirs under it
+    List<FileInfo> fileInfos = metaStore.getFilesByPrefixInOrder(dir);
     for (FileInfo fileInfo : fileInfos) {
-      // To avoid deleting subdir before deleting the file under it
-      if (fileInfo.isdir()) {
+      // just delete subdir instead of deleting all files under it
+      if (isUnderDir(fileInfo.getPath(), subdirList)) {
         continue;
       }
       fileList.add(fileInfo.getPath());
+      if (fileInfo.isdir()) {
+        subdirList.add(fileInfo.getPath());
+      }
     }
     return fileList;
   }
+
+  private boolean isUnderDir(String path, List<String> dirs) {
+    if (dirs.isEmpty()) {
+      return false;
+    }
+    for (String subdir : dirs) {
+      if (path.startsWith(subdir)) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java
@@ -288,6 +288,17 @@ public List<FileInfo> getFilesByPrefix(String path) throws MetaStoreException {
     }
   }
 
+  public List<FileInfo> getFilesByPrefixInOrder(String path) throws MetaStoreException {
+    updateCache();
+    try {
+      return fileInfoDao.getFilesByPrefixInOrder(path);
+    } catch (EmptyResultDataAccessException e) {
+      return new ArrayList<>();
+    } catch (Exception e) {
+      throw new MetaStoreException(e);
+    }
+  }
+
   public List<FileInfo> getFilesByPaths(Collection<String> paths)
       throws MetaStoreException {
     try {

diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java
@@ -57,6 +57,12 @@ public List<FileInfo> getFilesByPrefix(String path) {
         new FileInfoDao.FileInfoRowMapper(), path + "%");
   }
 
+  public List<FileInfo> getFilesByPrefixInOrder(String path) {
+    JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
+    return jdbcTemplate.query("SELECT * FROM file WHERE path LIKE ? ORDER BY path ASC",
+        new FileInfoDao.FileInfoRowMapper(), path + "%");
+  }
+
   public List<FileInfo> getFilesByPaths(Collection<String> paths) {
     NamedParameterJdbcTemplate namedParameterJdbcTemplate =
         new NamedParameterJdbcTemplate(dataSource);