diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index 2cc8aff0d43..9eb0c119f26 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -378,20 +378,23 @@ private void insertDeleteDiff(String path, boolean isDir) throws MetaStoreExcept private void insertDeleteDiff(String path) throws MetaStoreException { // TODO: remove "/" appended in src or dest in backup_file table + String pathWithSlash; if (!path.endsWith("/")) { - path = path + "/"; + pathWithSlash = path + "/"; + } else { + pathWithSlash = path; } - if (inBackup(path)) { - List backUpInfos = metaStore.getBackUpInfoBySrc(path); + if (inBackup(pathWithSlash)) { + List backUpInfos = metaStore.getBackUpInfoBySrc(pathWithSlash); for (BackUpInfo backUpInfo : backUpInfos) { - String destPath = path.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest()); + String destPath = pathWithSlash.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest()); try { // tackle root path case URI namenodeUri = new URI(destPath); String root = "hdfs://" + namenodeUri.getHost() + ":" + String.valueOf(namenodeUri.getPort()); if (destPath.equals(root) || destPath.equals(root + "/") || destPath.equals("/")) { - for (String srcFilePath : getFilesUnderDir(path)) { + for (String srcFilePath : getFilesUnderDir(pathWithSlash)) { FileDiff fileDiff = new FileDiff(FileDiffType.DELETE); fileDiff.setSrc(srcFilePath); String destFilePath = srcFilePath.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest()); @@ -400,8 +403,9 @@ private void insertDeleteDiff(String path) throws MetaStoreException { } } else { FileDiff fileDiff = new FileDiff(FileDiffType.DELETE); + // use the path getting from event with no slash appended fileDiff.setSrc(path); - //put sync's dest path in parameter for delete use + // put sync's dest path in parameter for delete use fileDiff.getParameters().put("-dest", destPath); metaStore.insertFileDiff(fileDiff); } @@ -417,14 +421,32 @@ private List getFilesUnderDir(String dir) throws MetaStoreException { dir = dir + "/"; } List fileList = new ArrayList<>(); - List fileInfos = metaStore.getFilesByPrefix(dir); + List subdirList = new ArrayList<>(); + // get fileInfo in asc order of path to guarantee that + // the subdir is tackled prior to files or dirs under it + List fileInfos = metaStore.getFilesByPrefixInOrder(dir); for (FileInfo fileInfo : fileInfos) { - // To avoid deleting subdir before deleting the file under it - if (fileInfo.isdir()) { + // just delete subdir instead of deleting all files under it + if (isUnderDir(fileInfo.getPath(), subdirList)) { continue; } fileList.add(fileInfo.getPath()); + if (fileInfo.isdir()) { + subdirList.add(fileInfo.getPath()); + } } return fileList; } + + private boolean isUnderDir(String path, List dirs) { + if (dirs.isEmpty()) { + return false; + } + for (String subdir : dirs) { + if (path.startsWith(subdir)) { + return true; + } + } + return false; + } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java index e93003c2a5c..36f2fe12382 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java @@ -288,6 +288,17 @@ public List getFilesByPrefix(String path) throws MetaStoreException { } } + public List getFilesByPrefixInOrder(String path) throws MetaStoreException { + updateCache(); + try { + return fileInfoDao.getFilesByPrefixInOrder(path); + } catch (EmptyResultDataAccessException e) { + return new ArrayList<>(); + } catch (Exception e) { + throw new MetaStoreException(e); + } + } + public List getFilesByPaths(Collection paths) throws MetaStoreException { try { diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java index cc6e8fcdc8c..852d1c4ddb9 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java @@ -57,6 +57,12 @@ public List getFilesByPrefix(String path) { new FileInfoDao.FileInfoRowMapper(), path + "%"); } + public List getFilesByPrefixInOrder(String path) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + return jdbcTemplate.query("SELECT * FROM file WHERE path LIKE ? ORDER BY path ASC", + new FileInfoDao.FileInfoRowMapper(), path + "%"); + } + public List getFilesByPaths(Collection paths) { NamedParameterJdbcTemplate namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(dataSource);