[HUDI-312] Make docker hdfs cluster ephemeral. This is needed to fix flakiness in integration tests. Also, Fix DeltaStreamer hanging issue due to uncaught exception

docker/compose/docker-compose_hadoop284_hive233_spark231.yml

@@ -21,8 +21,6 @@ services:
     image: apachehudi/hudi-hadoop_2.8.4-namenode:latest
     hostname: namenode
     container_name: namenode
-    volumes:
-      - /tmp/hadoop_name:/hadoop/dfs/name
     environment:
       - CLUSTER_NAME=hudi_hadoop284_hive232_spark231
     ports:
@@ -57,8 +55,6 @@ services:
       retries: 3
     depends_on:
       - namenode
-    volumes:
-      - /tmp/hadoop_data:/hadoop/dfs/data
 
   historyserver:
     image: apachehudi/hudi-hadoop_2.8.4-history:latest

docker/hoodie/hadoop/base/Dockerfile

@@ -24,7 +24,7 @@ USER root
 ENV LANG C.UTF-8
 
 ARG HADOOP_VERSION=2.8.4 
-ARG HADOOP_URL=https://www.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
+ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
 ENV HADOOP_VERSION ${HADOOP_VERSION}
 ENV HADOOP_URL ${HADOOP_URL}
 

docker/setup_demo.sh

@@ -18,16 +18,10 @@
 # limitations under the License.
 ################################################################################
 
-# Create host mount directory and copy
-mkdir -p /tmp/hadoop_name
-mkdir -p /tmp/hadoop_data
-
 WS_ROOT=`dirname $PWD`
 # restart cluster
 HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml down
 HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml pull
-rm -rf /tmp/hadoop_data/*
-rm -rf /tmp/hadoop_name/*
 sleep 5
 HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml up -d
 sleep 15