fixed failed UT and robot test

ChenSammi · Dec 4, 2024 · 87c61a5 · 87c61a5
1 parent a6bea06
commit 87c61a5
Show file tree

Hide file tree

Showing 21 changed files with 74 additions and 86 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,7 +31,7 @@ env:
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
   TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-  HADOOP_IMAGE: ghcr.io/apache/hadoop
+  HADOOP_IMAGE: ghcr.io/chensammi/hadoop
   OZONE_IMAGE: ghcr.io/apache/ozone
   OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
   OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}

diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
@@ -23,9 +23,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.hdds.conf.Config;
 import org.apache.hadoop.hdds.conf.ConfigGroup;
 import org.apache.hadoop.hdds.conf.ConfigType;

diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
@@ -426,7 +426,7 @@ private void acquireClient() throws IOException {
     // fall back to acquire GRPC client
     if (xceiverClientFactory != null && xceiverClientGrpc == null) {
       try {
-        xceiverClientGrpc = xceiverClientFactory.acquireClientForReadData(pipeline, false);
+        xceiverClientGrpc = xceiverClientFactory.acquireClientForReadData(pipeline);
       } catch (IOException ioe) {
         LOG.warn("Failed to acquire client for pipeline {}, block {}", pipeline, blockID);
         throw ioe;

diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainSocketFactory.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainSocketFactory.java
@@ -26,7 +26,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.Timer;
@@ -153,12 +152,6 @@ private DomainSocketFactory(ConfigurationSource conf) {
         LOG.warn(FEATURE + " cannot be used because " + nativeLibraryLoadFailureReason);
         pathInfo = PathInfo.DISABLED;
       } else {
-        File file = new File(domainSocketPath);
-        if (file.exists()) {
-          throw new IllegalArgumentException(FEATURE + " is enabled but "
-              + OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH + " is an existing " +
-              (file.isDirectory() ? "directory" : "file"));
-        }
         pathInfo = PathInfo.VALID;
         isEnabled = true;
         timer = new Timer(DomainSocketFactory.class.getSimpleName() + "-Timer");
@@ -274,9 +267,11 @@ public Timer getTimer() {
   }
 
   public static synchronized void close() {
-    if (instance.getTimer() != null) {
-      instance.getTimer().cancel();
+    if (instance != null) {
+      if (instance.getTimer() != null) {
+        instance.getTimer().cancel();
+      }
+      DomainSocketFactory.instance = null;
     }
-    DomainSocketFactory.instance = null;
   }
 }
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2003,7 +2003,16 @@
     <value>30s</value>
     <tag>OZONE, CLIENT, MANAGEMENT</tag>
     <description>
-      Timeout for ozone grpc client during read.
+      Timeout for ozone grpc and short-circuit client during read.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.client.write.timeout</name>
+    <value>30s</value>
+    <tag>OZONE, CLIENT, MANAGEMENT</tag>
+    <description>
+      Timeout for ozone short-circuit client during write.
     </description>
   </property>
 

diff --git a/...ice/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/Receiver.java b/...ice/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/Receiver.java
@@ -296,7 +296,7 @@ private boolean isSupportedCmdType(ContainerProtos.Type type) {
     return type == ContainerProtos.Type.GetBlock || type == ContainerProtos.Type.Echo;
   }
 
-  class TaskEntry {
+  static  class TaskEntry {
     private ContainerCommandRequestProto request;
     private ContainerCommandResponseProto response;
     private FileInputStream fis;

diff --git a/.../org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java b/.../org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java
@@ -229,7 +229,7 @@ public void run() {
         LOG.info("XceiverServerDomainSocket is closed", ace);
       } catch (IOException ie) {
         // usually when the xceiver count limit is hit.
-        LOG.warn("Got an exception. Peer {}", peer.toString(), ie);
+        LOG.warn("Got an exception. Peer {}", peer, ie);
         IOUtils.closeQuietly(peer);
       } catch (OutOfMemoryError ie) {
         IOUtils.closeQuietly(peer);

diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh
@@ -161,10 +161,10 @@ download_hadoop_aws() {
   if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then
     mkdir -p "${dir}"
     if [[ ! -f "${dir}.tar.gz" ]]; then
-      local url="https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz"
+      local url="https://github.com/ChenSammi/hadoop/archive/refs/tags/release-${HADOOP_VERSION}-RC0.tar.gz"
       echo "Downloading Hadoop from ${url}"
       curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1
     fi
-    tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1
+    tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*/hadoop-tools/hadoop-aws' || return 1
   fi
 }
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
@@ -15,7 +15,6 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env
@@ -15,7 +15,6 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -16,7 +16,7 @@
 
 HDDS_VERSION=${hdds.version}
 HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
@@ -16,7 +16,7 @@
 
 HDDS_VERSION=${hdds.version}
 HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -16,7 +16,7 @@
 
 HDDS_VERSION=${hdds.version}
 HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -18,7 +18,7 @@ HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
 OZONE_IMAGE=${docker.ozone.image}
 OZONE_IMAGE_FLAVOR="${docker.ozone.image.flavor}"
diff --git a/hadoop-ozone/dist/src/main/smoketest/.env b/hadoop-ozone/dist/src/main/smoketest/.env
@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-HADOOP_VERSION=${hadoop.version}
+HADOOP_VERSION=3.4.1
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
@@ -42,13 +42,14 @@ Create bucket with invalid bucket name
     ${result} =         Execute AWSS3APICli and checkrc         create-bucket --bucket invalid_bucket_${randStr}   255
                         Should contain          ${result}           InvalidBucketName
 
-Create new bucket and check no group ACL
+Create new bucket and check default group ACL
     ${bucket} =         Create bucket
     ${acl} =            Execute     ozone sh bucket getacl s3v/${bucket}
     ${group} =          Get Regexp Matches   ${acl}     "GROUP"
     IF      '${group}' is not '[]'
         ${json} =           Evaluate    json.loads('''${acl}''')    json
         # make sure this check is for group acl
         Should contain      ${json}[1][type]       GROUP
-        Should contain      ${json}[1][aclList]    NONE
+        Should contain      ${json}[1][aclList]    READ
+        Should contain      ${json}[1][aclList]    LIST
     END