[ADH-4064] Preserve file attributes in sync and distcp actions

smart-action/src/main/java/org/smartdata/action/SyncAction.java

@@ -27,13 +27,16 @@
 @ActionSignature(
     actionId = "sync",
     displayName = "sync",
-    usage = SyncAction.SRC + " $src" + SyncAction.DEST + " $dest"
+    usage = SyncAction.SRC + " $src "
+        + SyncAction.DEST + " $dest "
+        + SyncAction.PRESERVE + " $attributes"
 )
 public class SyncAction extends SmartAction {
   // related to fileDiff.src
   public static final String SRC = "-src";
   // related to remote cluster and fileDiff.src
   public static final String DEST = "-dest";
+  public static final String PRESERVE = "-preserve";
 
   @Override
   protected void execute() throws Exception {

smart-common/src/main/java/org/smartdata/model/FileInfoUpdate.java → smart-common/src/main/java/org/smartdata/model/FileInfoDiff.java

@@ -20,12 +20,10 @@
 
 import java.util.Objects;
 
-public class FileInfoUpdate {
+public class FileInfoDiff {
   private String path;
   private Long length;
-  private Boolean isdir;
   private Short blockReplication;
-  private Long blocksize;
   private Long modificationTime;
   private Long accessTime;
   private Short permission;
@@ -37,7 +35,7 @@ public String getPath() {
     return path;
   }
 
-  public FileInfoUpdate setPath(String path) {
+  public FileInfoDiff setPath(String path) {
     this.path = path;
     return this;
   }
@@ -46,43 +44,25 @@ public Long getLength() {
     return length;
   }
 
-  public FileInfoUpdate setLength(long length) {
+  public FileInfoDiff setLength(Long length) {
     this.length = length;
     return this;
   }
 
-  public Boolean getIsdir() {
-    return isdir;
-  }
-
-  public FileInfoUpdate setIsdir(boolean isdir) {
-    this.isdir = isdir;
-    return this;
-  }
-
   public Short getBlockReplication() {
     return blockReplication;
   }
 
-  public FileInfoUpdate setBlockReplication(short blockReplication) {
+  public FileInfoDiff setBlockReplication(Short blockReplication) {
     this.blockReplication = blockReplication;
     return this;
   }
 
-  public Long getBlocksize() {
-    return blocksize;
-  }
-
-  public FileInfoUpdate setBlocksize(long blocksize) {
-    this.blocksize = blocksize;
-    return this;
-  }
-
   public Long getModificationTime() {
     return modificationTime;
   }
 
-  public FileInfoUpdate setModificationTime(long modificationTime) {
+  public FileInfoDiff setModificationTime(Long modificationTime) {
     this.modificationTime = modificationTime;
     return this;
   }
@@ -91,7 +71,7 @@ public Long getAccessTime() {
     return accessTime;
   }
 
-  public FileInfoUpdate setAccessTime(long accessTime) {
+  public FileInfoDiff setAccessTime(Long accessTime) {
     this.accessTime = accessTime;
     return this;
   }
@@ -100,7 +80,7 @@ public Short getPermission() {
     return permission;
   }
 
-  public FileInfoUpdate setPermission(short permission) {
+  public FileInfoDiff setPermission(Short permission) {
     this.permission = permission;
     return this;
   }
@@ -109,7 +89,7 @@ public String getOwner() {
     return owner;
   }
 
-  public FileInfoUpdate setOwner(String owner) {
+  public FileInfoDiff setOwner(String owner) {
     this.owner = owner;
     return this;
   }
@@ -118,7 +98,7 @@ public String getGroup() {
     return group;
   }
 
-  public FileInfoUpdate setGroup(String group) {
+  public FileInfoDiff setGroup(String group) {
     this.group = group;
     return this;
   }
@@ -127,7 +107,7 @@ public Byte getErasureCodingPolicy() {
     return erasureCodingPolicy;
   }
 
-  public FileInfoUpdate setErasureCodingPolicy(byte erasureCodingPolicy) {
+  public FileInfoDiff setErasureCodingPolicy(Byte erasureCodingPolicy) {
     this.erasureCodingPolicy = erasureCodingPolicy;
     return this;
   }
@@ -140,12 +120,10 @@ public boolean equals(Object o) {
     if (o == null || getClass() != o.getClass()) {
       return false;
     }
-    FileInfoUpdate that = (FileInfoUpdate) o;
+    FileInfoDiff that = (FileInfoDiff) o;
     return Objects.equals(path, that.path)
         && Objects.equals(length, that.length)
-        && Objects.equals(isdir, that.isdir)
         && Objects.equals(blockReplication, that.blockReplication)
-        && Objects.equals(blocksize, that.blocksize)
         && Objects.equals(modificationTime, that.modificationTime)
         && Objects.equals(accessTime, that.accessTime)
         && Objects.equals(permission, that.permission)
@@ -156,18 +134,16 @@ public boolean equals(Object o) {
 
   @Override
   public int hashCode() {
-    return Objects.hash(path, length, isdir, blockReplication,
-        blocksize, modificationTime, accessTime, permission, owner, group, erasureCodingPolicy);
+    return Objects.hash(path, length, blockReplication,
+        modificationTime, accessTime, permission, owner, group, erasureCodingPolicy);
   }
 
   @Override
   public String toString() {
-    return "FileInfoUpdate{"
+    return "FileInfoDiff{"
         + "path='" + path + '\''
         + ", length=" + length
-        + ", isdir=" + isdir
         + ", blockReplication=" + blockReplication
-        + ", blocksize=" + blocksize
         + ", modificationTime=" + modificationTime
         + ", accessTime=" + accessTime
         + ", permission=" + permission

smart-common/src/main/java/org/smartdata/utils/ConfigUtil.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.smartdata.utils;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.smartdata.SmartConstants.DISTRIBUTED_FILE_SYSTEM;
+import static org.smartdata.SmartConstants.FS_HDFS_IMPL;
+import static org.smartdata.SmartConstants.SMART_FILE_SYSTEM;
+
+public class ConfigUtil {
+  public static Configuration toRemoteClusterConfig(Configuration configuration) {
+    Configuration remoteConfig = new Configuration(configuration);
+    if (SMART_FILE_SYSTEM.equals(remoteConfig.get(FS_HDFS_IMPL))) {
+      remoteConfig.set(FS_HDFS_IMPL, DISTRIBUTED_FILE_SYSTEM);
+    }
+
+    return remoteConfig;
+  }
+}

smart-engine/src/main/java/org/smartdata/server/engine/rule/FileCopyDrPlugin.java

@@ -17,10 +17,11 @@
  */
 package org.smartdata.server.engine.rule;
 
-
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.action.SyncAction;
+import org.smartdata.hdfs.action.CopyFileAction;
 import org.smartdata.metastore.MetaStore;
 import org.smartdata.metastore.MetaStoreException;
 import org.smartdata.model.BackUpInfo;
@@ -43,8 +44,8 @@
 import static org.smartdata.utils.StringUtil.ssmPatternsToRegex;
 
 public class FileCopyDrPlugin implements RuleExecutorPlugin {
-  private MetaStore metaStore;
-  private Map<Long, List<BackUpInfo>> backups = new HashMap<>();
+  private final MetaStore metaStore;
+  private final Map<Long, List<BackUpInfo>> backups = new HashMap<>();
   private static final Logger LOG =
       LoggerFactory.getLogger(FileCopyDrPlugin.class.getName());
 
@@ -63,6 +64,9 @@ public void onNewRuleExecutor(final RuleInfo ruleInfo, TranslateResult tResult)
     CmdletDescriptor des = tResult.getCmdDescriptor();
     for (int i = 0; i < des.getActionSize(); i++) {
       if (des.getActionName(i).equals("sync")) {
+        String rawPreserveArg = des.getActionArgs(i).get(SyncAction.PRESERVE);
+        // fail fast if preserve arg is not valid
+        validatePreserveArg(rawPreserveArg);
 
         List<String> statements = tResult.getSqlStatements();
         String before = statements.get(statements.size() - 1);
@@ -169,4 +173,14 @@ public void onRuleExecutorExit(final RuleInfo ruleInfo) {
       }
     }
   }
+
+  private void validatePreserveArg(String rawPreserveArg) {
+    if (StringUtils.isBlank(rawPreserveArg)) {
+      return;
+    }
+
+    for (String attribute: rawPreserveArg.split(",")) {
+      CopyFileAction.PreserveAttribute.validate(attribute);
+    }
+  }
 }

smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MultiClusterHarness.java

@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.smartdata.hdfs;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.smartdata.hdfs.MultiClusterHarness.TestType.INTER_CLUSTER;
+import static org.smartdata.hdfs.MultiClusterHarness.TestType.INTRA_CLUSTER;
+
+/**
+ * A MiniCluster for action test.
+ */
+@RunWith(Parameterized.class)
+public abstract class MultiClusterHarness extends MiniClusterHarness {
+
+  public enum TestType {
+    INTRA_CLUSTER,
+    INTER_CLUSTER
+  }
+
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+  @Parameterized.Parameter()
+  public TestType testType;
+
+  protected MiniDFSCluster anotherCluster;
+  protected DistributedFileSystem anotherDfs;
+  protected DFSClient anotherDfsClient;
+
+  @Parameterized.Parameters(name = "Test type - {0}")
+  public static Object[] parameters() {
+    return new Object[] {INTRA_CLUSTER, INTER_CLUSTER};
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    if (testType == INTRA_CLUSTER) {
+      anotherDfs = dfs;
+      anotherDfsClient = dfsClient;
+      return;
+    }
+    Configuration clusterConfig = new Configuration(smartContext.getConf());
+    clusterConfig.set("hdfs.minidfs.basedir", tmpFolder.newFolder().getAbsolutePath());
+    anotherCluster = createCluster(clusterConfig);
+    anotherDfs = anotherCluster.getFileSystem();
+    anotherDfsClient = anotherDfs.getClient();
+  }
+
+  @After
+  public void shutdown() throws IOException {
+    if (anotherCluster != null) {
+      anotherCluster.shutdown(true);
+    }
+  }
+
+  protected Path anotherClusterPath(String parent, String child) {
+    return anotherDfs.makeQualified(new Path(parent, child));
+  }
+
+  protected String pathToActionArg(Path path) {
+    return testType == TestType.INTER_CLUSTER ? path.toString() : path.toUri().getPath();
+  }
+}