HADOOP-19085. Compatibility Benchmark over HCFS Implementations

Contributed by Han Liu
vinayakumarb · Mar 17, 2024 · 4d88f98 · 4d88f98
1 parent 783cc3e
commit 4d88f98
Show file tree

Hide file tree

Showing 66 changed files with 6,127 additions and 0 deletions.
diff --git a/hadoop-tools/hadoop-compat-bench/pom.xml b/hadoop-tools/hadoop-compat-bench/pom.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.5.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-compat-bench</artifactId>
+  <version>3.5.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <description>Apache Hadoop Compatibility</description>
+  <name>Apache Hadoop Compatibility Benchmark</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <!-- Should we keep this -->
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!-- For test -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-client</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <mainClass>org.apache.hadoop.fs.compat.HdfsCompatTool</mainClass>
+            </manifest>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+            <configuration>
+              <archive>
+                <manifest>
+                  <mainClass>org.apache.hadoop.fs.compat.hdfs.HdfsCompatMiniCluster</mainClass>
+                </manifest>
+              </archive>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+        </configuration>
+      </plugin>
+    </plugins>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+      </resource>
+      <resource>
+        <directory>shell</directory>
+      </resource>
+    </resources>
+  </build>
+</project>
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/attr.t b/hadoop-tools/hadoop-compat-bench/shell/cases/attr.t
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+hadoop fs -put "${localDir}/dat" "${baseDir}/"
+
+echo "1..10"
+
+# 1. chown
+hadoop fs -chown "hadoop-compat-bench-user" "${baseDir}/dat"
+expect_out "chown" "user:hadoop-compat-bench-user" hadoop fs -stat "user:%u" "${baseDir}/dat"
+
+# 2. chgrp
+hadoop fs -chgrp "hadoop-compat-bench-group" "${baseDir}/dat"
+expect_out "chgrp" "group:hadoop-compat-bench-group" hadoop fs -stat "group:%g" "${baseDir}/dat"
+
+# 3. chmod
+hadoop fs -chmod 777 "${baseDir}/dat"
+expect_out "chmod" "perm:777" hadoop fs -stat "perm:%a" "${baseDir}/dat"
+
+# 4. touch
+hadoop fs -touch -m -t "20000615:000000" "${baseDir}/dat"
+expect_out "touch" "date:2000-06-.*" hadoop fs -stat "date:%y" "${baseDir}/dat"
+
+# 5. setfattr
+expect_ret "setfattr" 0 hadoop fs -setfattr -n "user.key" -v "value" "${baseDir}/dat"
+
+# 6. getfattr
+expect_out "getfattr" ".*value.*" hadoop fs -getfattr -n "user.key" "${baseDir}/dat"
+
+# 7. setfacl
+expect_ret "setfacl" 0 hadoop fs -setfacl -m "user:foo:---" "${baseDir}/dat"
+
+# 8. getfacl
+expect_out "getfacl" ".*foo.*" hadoop fs -getfacl "${baseDir}/dat"
+
+# 9. setrep
+hadoop fs -setrep 1 "${baseDir}/dat"
+expect_out "setrep" "replication:1" hadoop fs -stat "replication:%r" "${baseDir}/dat"
+
+# 10. checksum
+expect_ret "checksum" 0 hadoop fs -checksum "${baseDir}/dat"  # TODO
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/concat.t b/hadoop-tools/hadoop-compat-bench/shell/cases/concat.t
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+hadoop fs -put "${localDir}/dat" "${baseDir}/src1"
+hadoop fs -put "${localDir}/dat" "${baseDir}/src2"
+
+echo "1..3"
+
+# 1. touchz
+hadoop fs -touchz "${baseDir}/dat"
+expect_out "touchz" "size:0" hadoop fs -stat "size:%b" "${baseDir}/dat"
+
+# 2. concat
+expect_ret "concat" 0 hadoop fs -concat "${baseDir}/dat" "${baseDir}/src1" "${baseDir}/src2"
+# expect_out "size:26" hadoop fs -stat "size:%b" "${baseDir}/dat"
+
+# 3. getmerge
+hadoop fs -getmerge "${baseDir}" "${localDir}/merged"
+expect_ret "getmerge" 0 test -s "${localDir}/merged"
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/copy.t b/hadoop-tools/hadoop-compat-bench/shell/cases/copy.t
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+
+echo "1..3"
+
+# 1. copyFromLocal
+expect_ret "copyFromLocal" 0 hadoop fs -copyFromLocal "${localDir}/dat" "${baseDir}/"
+
+# 2. cp
+hadoop fs -cp "${baseDir}/dat" "${baseDir}/dat2"
+expect_ret "cp" 0 hadoop fs -test -f "${baseDir}/dat2"
+
+# 3. copyToLocal
+hadoop fs -copyToLocal "${baseDir}/dat2" "${localDir}/"
+expect_ret "copyToLocal" 0 test -f "${localDir}/dat2"
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/directory.t b/hadoop-tools/hadoop-compat-bench/shell/cases/directory.t
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+hadoop fs -put "${localDir}/dat" "${baseDir}/"
+
+echo "1..8"
+
+# 1. mkdir
+expect_ret "mkdir" 0 hadoop fs -mkdir -p "${baseDir}/dir/sub"
+
+# 2. ls
+expect_lines "ls" 2 ".*dat.*" ".*dir.*" hadoop fs -ls "${baseDir}"
+
+# 3. lsr
+expect_lines "lsr" 3 ".*dat.*" ".*dir.*" ".*sub.*" hadoop fs -lsr "${baseDir}"
+
+# 4. count
+expect_out "count" ".*13.*" hadoop fs -count "${baseDir}"
+
+# 5. du
+expect_out "du" ".*13.*" hadoop fs -du "${baseDir}"
+
+# 6. dus
+expect_out "dus" ".*13.*" hadoop fs -dus "${baseDir}"
+
+# 7. df
+expect_ret "df" 0 hadoop fs -df "${baseDir}"
+
+# 8. find
+expect_out "find" ".*dat.*" hadoop fs -find "${baseDir}" -name "dat" -print
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/fileinfo.t b/hadoop-tools/hadoop-compat-bench/shell/cases/fileinfo.t
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+hadoop fs -put "${localDir}/dat" "${baseDir}/"
+
+echo "1..2"
+
+# 1. stat
+expect_out "stat" "size:13" hadoop fs -stat "size:%b" "${baseDir}/dat"
+
+# 2. test
+expect_ret "test" 0 hadoop fs -test -f "${baseDir}/dat"
diff --git a/hadoop-tools/hadoop-compat-bench/shell/cases/move.t b/hadoop-tools/hadoop-compat-bench/shell/cases/move.t
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. $(dirname "$0")/../misc.sh
+
+echo "Hello World!" > "${localDir}/dat"
+
+echo "1..2"
+
+# 1. moveFromLocal
+expect_ret "moveFromLocal" 0 hadoop fs -moveFromLocal "${localDir}/dat" "${baseDir}/"
+
+# 2. mv
+hadoop fs -mv "${baseDir}/dat" "${baseDir}/dat2"
+expect_ret "mv" 0 hadoop fs -test -f "${baseDir}/dat2"
+
+# moveToLocal is not achieved on HDFS
+# hadoop fs -moveToLocal "${baseDir}/dat2" "${localDir}/"
+# expect_ret "moveToLocal" 0 test -f "${localDir}/dat2"