feat(cli): 增加配置发布
将hadoop配置分发到各个主机上
This commit is contained in:
@@ -3,6 +3,9 @@
|
||||
root_path=$(dirname $(cd $(dirname $0); pwd))
|
||||
|
||||
clusters=(
|
||||
b2a4
|
||||
b2b1
|
||||
b2b5
|
||||
b2b12
|
||||
b9b9
|
||||
)
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://b1</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload.buffer</name>
|
||||
<value>disk</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.multipart.size</name>
|
||||
<value>67108864</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.trash.interval</name>
|
||||
<value>360</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.acl</name>
|
||||
<value>sasl:nn:rwcda</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.quorum</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.signature.secret.file</name>
|
||||
<value>/etc/security/http_secret</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.type</name>
|
||||
<value>simple</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-headers</name>
|
||||
<value>X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-methods</name>
|
||||
<value>GET,PUT,POST,OPTIONS,HEAD,DELETE</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-origins</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.max-age</name>
|
||||
<value>1800</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.filter.initializers</name>
|
||||
<value>org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.rpc.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.auth_to_local</name>
|
||||
<value>RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/
|
||||
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
|
||||
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
|
||||
RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/
|
||||
RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/
|
||||
RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/
|
||||
RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/
|
||||
RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/
|
||||
RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/
|
||||
RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/
|
||||
DEFAULT</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authentication</name>
|
||||
<value>kerberos</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authorization</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.instrumentation.requires.admin</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codec.lzo.class</name>
|
||||
<value>com.hadoop.compression.lzo.LzoCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codecs</name>
|
||||
<value>org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.file.buffer.size</name>
|
||||
<value>131072</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.serializations</name>
|
||||
<value>org.apache.hadoop.io.serializer.WritableSerialization</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connect.max.retries</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connection.maxidletime</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.idlethreshold</name>
|
||||
<value>8000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.server.tcpnodelay</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobtracker.webinterface.trusted</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.fallback-to-simple-auth-allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.hdfs.impl.disable.cache</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
@@ -1,698 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>dfs.block.access.token.enable</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blockreport.initialDelay</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blocksize</name>
|
||||
<value>134217728</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
|
||||
<value>4096</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.retry.policy.enabled</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.cluster.administrators</name>
|
||||
<value> hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.content-summary.limit</name>
|
||||
<value>5000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.data.transfer.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.address</name>
|
||||
<value>0.0.0.0:1019</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.balance.bandwidthPerSec</name>
|
||||
<value>6250000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir.perm</name>
|
||||
<value>750</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.du.reserved</name>
|
||||
<value>26405499904</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.failed.volumes.tolerated</name>
|
||||
<value>2</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.http.address</name>
|
||||
<value>0.0.0.0:1022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.https.address</name>
|
||||
<value>0.0.0.0:50475</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.ipc.address</name>
|
||||
<value>0.0.0.0:8010</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.kerberos.principal</name>
|
||||
<value>dn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/dn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.max.transfer.threads</name>
|
||||
<value>16384</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.domain.socket.path</name>
|
||||
<value>/var/lib/hadoop-hdfs/dn_socket</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.encrypt.data.transfer.cipher.suites</name>
|
||||
<value>AES/CTR/NoPadding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.automatic-failover.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.fencing.methods</name>
|
||||
<value>shell(/bin/true)</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b2</name>
|
||||
<value>nn3,nn4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.heartbeat.interval</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.hosts.exclude</name>
|
||||
<value>/etc/hadoop/conf/dfs.exclude</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.http.policy</name>
|
||||
<value>HTTP_ONLY</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.https.port</name>
|
||||
<value>50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.internal.nameservices</name>
|
||||
<value>b1,b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b1</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b2</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.http-address</name>
|
||||
<value>0.0.0.0:8480</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.https-address</name>
|
||||
<value>0.0.0.0:8481</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.principal</name>
|
||||
<value>jn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/jn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.accesstime.precision</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.acls.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.audit.log.async</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.read.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.write.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.dir</name>
|
||||
<value>/data/hadoop/hdfs/namesecondary</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.edits.dir</name>
|
||||
<value>${dfs.namenode.checkpoint.dir}</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.period</name>
|
||||
<value>21600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.txns</name>
|
||||
<value>1000000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fslock.fair</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.handler.count</name>
|
||||
<value>200</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.principal</name>
|
||||
<value>nn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/nn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.max.extra.edits.segments.retained</name>
|
||||
<value>180</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir</name>
|
||||
<value>/data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir.restore</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.num.extra.edits.retained</name>
|
||||
<value>18000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.safemode.threshold-pct</name>
|
||||
<value>0.99</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b1</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b2</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.stale.datanode.interval</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
|
||||
<value>3600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.write.stale.datanode.ratio</name>
|
||||
<value>1.0f</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.nameservices</name>
|
||||
<value>b1,b2,b3,b4,a3,a4,f1,d2,e1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.ContentSummary.subAccess</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.superusergroup</name>
|
||||
<value>hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication.max</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.webhdfs.enabled</name>
|
||||
<value>true</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.permissions.umask-mode</name>
|
||||
<value>022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.caller.context.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>manage.include.files</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.exports.allowed.hosts</name>
|
||||
<value>* rw</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.file.dump.dir</name>
|
||||
<value>/tmp/.hdfs-nfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.datanode-restart.timeout</name>
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.f1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.f1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.d2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.d2</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.e1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.e1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,262 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://b1</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload.buffer</name>
|
||||
<value>disk</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.multipart.size</name>
|
||||
<value>67108864</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.trash.interval</name>
|
||||
<value>360</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.acl</name>
|
||||
<value>sasl:nn:rwcda</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.quorum</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.signature.secret.file</name>
|
||||
<value>/etc/security/http_secret</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.type</name>
|
||||
<value>simple</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-headers</name>
|
||||
<value>X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-methods</name>
|
||||
<value>GET,PUT,POST,OPTIONS,HEAD,DELETE</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-origins</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.max-age</name>
|
||||
<value>1800</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.filter.initializers</name>
|
||||
<value>org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.rpc.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.auth_to_local</name>
|
||||
<value>RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/
|
||||
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
|
||||
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
|
||||
RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/
|
||||
RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/
|
||||
RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/
|
||||
RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/
|
||||
RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/
|
||||
RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/
|
||||
RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/
|
||||
DEFAULT</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authentication</name>
|
||||
<value>kerberos</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authorization</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.instrumentation.requires.admin</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codec.lzo.class</name>
|
||||
<value>com.hadoop.compression.lzo.LzoCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codecs</name>
|
||||
<value>org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.file.buffer.size</name>
|
||||
<value>131072</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.serializations</name>
|
||||
<value>org.apache.hadoop.io.serializer.WritableSerialization</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connect.max.retries</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connection.maxidletime</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.idlethreshold</name>
|
||||
<value>8000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.server.tcpnodelay</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobtracker.webinterface.trusted</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.fallback-to-simple-auth-allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.hdfs.impl.disable.cache</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
@@ -1,698 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>dfs.block.access.token.enable</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blockreport.initialDelay</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blocksize</name>
|
||||
<value>134217728</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
|
||||
<value>4096</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.retry.policy.enabled</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.cluster.administrators</name>
|
||||
<value> hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.content-summary.limit</name>
|
||||
<value>5000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.data.transfer.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.address</name>
|
||||
<value>0.0.0.0:1019</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.balance.bandwidthPerSec</name>
|
||||
<value>6250000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir.perm</name>
|
||||
<value>750</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.du.reserved</name>
|
||||
<value>26405499904</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.failed.volumes.tolerated</name>
|
||||
<value>2</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.http.address</name>
|
||||
<value>0.0.0.0:1022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.https.address</name>
|
||||
<value>0.0.0.0:50475</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.ipc.address</name>
|
||||
<value>0.0.0.0:8010</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.kerberos.principal</name>
|
||||
<value>dn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/dn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.max.transfer.threads</name>
|
||||
<value>16384</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.domain.socket.path</name>
|
||||
<value>/var/lib/hadoop-hdfs/dn_socket</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.encrypt.data.transfer.cipher.suites</name>
|
||||
<value>AES/CTR/NoPadding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.automatic-failover.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.fencing.methods</name>
|
||||
<value>shell(/bin/true)</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b2</name>
|
||||
<value>nn3,nn4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.heartbeat.interval</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.hosts.exclude</name>
|
||||
<value>/etc/hadoop/conf/dfs.exclude</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.http.policy</name>
|
||||
<value>HTTP_ONLY</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.https.port</name>
|
||||
<value>50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.internal.nameservices</name>
|
||||
<value>b1,b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b1</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b2</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.http-address</name>
|
||||
<value>0.0.0.0:8480</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.https-address</name>
|
||||
<value>0.0.0.0:8481</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.principal</name>
|
||||
<value>jn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/jn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.accesstime.precision</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.acls.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.audit.log.async</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.read.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.write.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.dir</name>
|
||||
<value>/data/hadoop/hdfs/namesecondary</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.edits.dir</name>
|
||||
<value>${dfs.namenode.checkpoint.dir}</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.period</name>
|
||||
<value>21600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.txns</name>
|
||||
<value>1000000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fslock.fair</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.handler.count</name>
|
||||
<value>200</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.principal</name>
|
||||
<value>nn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/nn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.max.extra.edits.segments.retained</name>
|
||||
<value>180</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir</name>
|
||||
<value>/data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir.restore</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.num.extra.edits.retained</name>
|
||||
<value>18000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.safemode.threshold-pct</name>
|
||||
<value>0.99</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b1</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b2</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.stale.datanode.interval</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
|
||||
<value>3600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.write.stale.datanode.ratio</name>
|
||||
<value>1.0f</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.nameservices</name>
|
||||
<value>b1,b2,b3,b4,a3,a4,f1,d2,e1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.ContentSummary.subAccess</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.superusergroup</name>
|
||||
<value>hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication.max</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.webhdfs.enabled</name>
|
||||
<value>true</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.permissions.umask-mode</name>
|
||||
<value>022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.caller.context.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>manage.include.files</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.exports.allowed.hosts</name>
|
||||
<value>* rw</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.file.dump.dir</name>
|
||||
<value>/tmp/.hdfs-nfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.datanode-restart.timeout</name>
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.f1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.f1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.d2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.d2</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.e1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.e1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,5 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://b2</value>
|
||||
@@ -32,7 +25,7 @@
|
||||
<name>fs.trash.interval</name>
|
||||
<value>4320</value>
|
||||
</property>
|
||||
|
||||
|
||||
<property>
|
||||
<name>fs.trash.checkpoint.interval</name>
|
||||
<value>360</value>
|
||||
@@ -50,7 +43,7 @@
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.quorum</name>
|
||||
<value>b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181</value>
|
||||
<value>a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -160,7 +153,7 @@
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.hosts</name>
|
||||
<value>*</value>
|
||||
<value>a4m1.hdp.dc,a4m2.hdp.dc</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -170,10 +163,10 @@
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.auth_to_local</name>
|
||||
<value>RULE:[1:$1@$0](hbase-b5@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-b5@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-b5@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-b5@ECLD.COM)s/.*/yarn-ats/
|
||||
<value>RULE:[1:$1@$0](hbase-a4@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-a4@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-a4@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-a4@ECLD.COM)s/.*/yarn-ats/
|
||||
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
|
||||
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
|
||||
@@ -254,14 +247,4 @@ DEFAULT</value>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.fallback-to-simple-auth-allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.hdfs.impl.disable.cache</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
@@ -1,4 +1,5 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>dfs.block.access.token.enable</name>
|
||||
@@ -16,7 +17,7 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b5</name>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
@@ -62,7 +63,7 @@
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data,[DISK]file:///data13/hadoop/hdfs/data,[DISK]file:///data14/hadoop/hdfs/data,[DISK]file:///data15/hadoop/hdfs/data,[DISK]file:///data16/hadoop/hdfs/data,[DISK]file:///data17/hadoop/hdfs/data,[DISK]file:///data18/hadoop/hdfs/data,[DISK]file:///data19/hadoop/hdfs/data,[DISK]file:///data20/hadoop/hdfs/data,[DISK]file:///data21/hadoop/hdfs/data,[DISK]file:///data22/hadoop/hdfs/data,[DISK]file:///data23/hadoop/hdfs/data,[DISK]file:///data24/hadoop/hdfs/data</value>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
@@ -133,7 +134,7 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b5</name>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
|
||||
@@ -159,11 +160,11 @@
|
||||
|
||||
<property>
|
||||
<name>dfs.internal.nameservices</name>
|
||||
<value>b5</value>
|
||||
<value>a4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b5</name>
|
||||
<name>dfs.journalnode.edits.dir.a4</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
@@ -248,23 +249,23 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:50070</value>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:50470</value>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:50470</value>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -304,13 +305,13 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:8020</value>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:8020</value>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -319,8 +320,8 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b5</name>
|
||||
<value>qjournal://b5m1.hdp.dc:8485;b5m2.hdp.dc:8485;b5m3.hdp.dc:8485/b5</value>
|
||||
<name>dfs.namenode.shared.edits.dir.a4</name>
|
||||
<value>qjournal://a4m1.hdp.dc:8485;a4m2.hdp.dc:8485;a4m3.hdp.dc:8485/a4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -340,7 +341,7 @@
|
||||
|
||||
<property>
|
||||
<name>dfs.nameservices</name>
|
||||
<value>b5,b1,b2,b3,b4,a3,a4,f1,e1,d2</value>
|
||||
<value>a4,a3,b3,b1,b2,b4,f1,d2,b5,e1,a6</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -414,37 +415,13 @@
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
<property>
|
||||
<name>dfs.permissions</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
<name>dfs.namenode.inode.attributes.provider.class</name>
|
||||
<value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -454,19 +431,19 @@
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50470</value>
|
||||
@@ -487,19 +464,19 @@
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50470</value>
|
||||
@@ -578,6 +555,39 @@
|
||||
<value>b1m6.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.f1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
@@ -644,6 +654,44 @@
|
||||
<value>d2m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b5</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b5</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b5.nn1</name>
|
||||
<value>b5m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b5.nn2</name>
|
||||
<value>b5m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.txns</name>
|
||||
<value>15000000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.e1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
@@ -678,36 +726,36 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<name>dfs.client.failover.proxy.provider.a6</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<name>dfs.ha.namenodes.a6</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
<name>dfs.namenode.http-address.a6.nn1</name>
|
||||
<value>a6m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
<name>dfs.namenode.http-address.a6.nn2</name>
|
||||
<value>a6m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
<name>dfs.namenode.https-address.a6.nn1</name>
|
||||
<value>a6m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
<name>dfs.namenode.https-address.a6.nn2</name>
|
||||
<value>a6m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
<name>dfs.namenode.rpc-address.a6.nn1</name>
|
||||
<value>a6m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
<name>dfs.namenode.rpc-address.a6.nn2</name>
|
||||
<value>a6m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
</configuration>
|
||||
@@ -1,4 +1,5 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-origins</name>
|
||||
@@ -53,12 +54,12 @@
|
||||
|
||||
<property>
|
||||
<name>hadoop.registry.system.accounts</name>
|
||||
<value>sasl:yarn,sasl:jhs,sasl:hdfs-b1,sasl:rm,sasl:hive,sasl:spark</value>
|
||||
<value>sasl:yarn,sasl:jhs,sasl:hdfs-a4,sasl:rm,sasl:hive,sasl:spark</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.registry.zk.quorum</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
<value>a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -105,15 +106,25 @@
|
||||
<name>yarn.log-aggregation.retain-seconds</name>
|
||||
<value>2592000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-formats</name>
|
||||
<value>TFile</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-controller.TFile.class</name>
|
||||
<value>org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log.server.url</name>
|
||||
<value>http://b1m4.hdp.dc:19888/jobhistory/logs</value>
|
||||
<value>http://a4m3.hdp.dc:19888/jobhistory/logs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log.server.web-service.url</name>
|
||||
<value>http://b1m4.hdp.dc:8188/ws/v1/applicationhistory</value>
|
||||
<value>http://a4m3.hdp.dc:8188/ws/v1/applicationhistory</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -146,6 +157,16 @@
|
||||
<value>mapreduce_shuffle,spark_shuffle,timeline_collector,sparkv2_shuffle</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services.sparkv2_shuffle.class</name>
|
||||
<value>org.apache.spark.network.yarn.YarnShuffleService</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services.sparkv2_shuffle.classpath</name>
|
||||
<value>/usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
|
||||
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
|
||||
@@ -166,16 +187,6 @@
|
||||
<value>org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services.sparkv2_shuffle.class</name>
|
||||
<value>org.apache.spark.network.yarn.YarnShuffleService</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services.sparkv2_shuffle.classpath</name>
|
||||
<value>/usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.bind-host</name>
|
||||
<value>0.0.0.0</value>
|
||||
@@ -198,7 +209,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.delete.debug-delay-sec</name>
|
||||
<value>86400</value>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -248,7 +259,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.local-dirs</name>
|
||||
<value>/data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local</value>
|
||||
<value>/data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -273,7 +284,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.log-dirs</name>
|
||||
<value>/data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log</value>
|
||||
<value>/data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -288,7 +299,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.recovery.dir</name>
|
||||
<value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
|
||||
<value>/data3/var/log/hadoop-yarn/nodemanager/recovery-state</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -313,42 +324,42 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-plugins</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-plugins.gpu.docker-plugin</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource.cpu-vcores</name>
|
||||
<value>39</value>
|
||||
<value>96</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource.memory-mb</name>
|
||||
<value>184320</value>
|
||||
<value>348160</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
|
||||
<value>80</value>
|
||||
<value>96</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -380,7 +391,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.privileged-containers.acl</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -415,12 +426,12 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.address</name>
|
||||
<value>b1m2.hdp.dc:8050</value>
|
||||
<value>a4m1.hdp.dc:8050</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.admin.address</name>
|
||||
<value>b1m2.hdp.dc:8141</value>
|
||||
<value>a4m1.hdp.dc:8141</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -480,17 +491,17 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.hostname</name>
|
||||
<value>b1m2.hdp.dc</value>
|
||||
<value>a4m1.hdp.dc</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.hostname.rm1</name>
|
||||
<value>b1m2.hdp.dc</value>
|
||||
<value>a4m1.hdp.dc</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.hostname.rm2</name>
|
||||
<value>b1m3.hdp.dc</value>
|
||||
<value>a4m2.hdp.dc</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -540,17 +551,17 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.proxyuser.*.groups</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.proxyuser.*.hosts</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.proxyuser.*.users</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -560,22 +571,22 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.resource-tracker.address</name>
|
||||
<value>b1m2.hdp.dc:8025</value>
|
||||
<value>a4m1.hdp.dc:8025</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
|
||||
<value>b1m2.hdp.dc:8025</value>
|
||||
<value>a4m1.hdp.dc:8025</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
|
||||
<value>b1m3.hdp.dc:8025</value>
|
||||
<value>a4m2.hdp.dc:8025</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.scheduler.address</name>
|
||||
<value>b1m2.hdp.dc:8030</value>
|
||||
<value>a4m1.hdp.dc:8030</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -610,17 +621,17 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.address</name>
|
||||
<value>b1m2.hdp.dc:8088</value>
|
||||
<value>a4m1.hdp.dc:8088</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.address.rm1</name>
|
||||
<value>b1m2.hdp.dc:8088</value>
|
||||
<value>a4m1.hdp.dc:8088</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.address.rm2</name>
|
||||
<value>b1m3.hdp.dc:8088</value>
|
||||
<value>a4m2.hdp.dc:8088</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -635,17 +646,17 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.https.address</name>
|
||||
<value>b1m2.hdp.dc:8090</value>
|
||||
<value>a4m1.hdp.dc:8090</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.https.address.rm1</name>
|
||||
<value>b1m2.hdp.dc:8090</value>
|
||||
<value>a4m1.hdp.dc:8090</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.https.address.rm2</name>
|
||||
<value>b1m3.hdp.dc:8090</value>
|
||||
<value>a4m2.hdp.dc:8090</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -675,7 +686,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.zk-address</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
<value>a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -740,7 +751,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.address</name>
|
||||
<value>b1m4.hdp.dc:10200</value>
|
||||
<value>a4m3.hdp.dc:10200</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -750,7 +761,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.client.max-retries</name>
|
||||
<value>30</value>
|
||||
<value>180</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -760,7 +771,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.enabled</name>
|
||||
<value>false</value>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -790,7 +801,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -835,12 +846,12 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.cookie.domain</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.cookie.path</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -850,7 +861,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.kerberos.name.rules</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -860,37 +871,37 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.proxyuser.*.groups</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.proxyuser.*.hosts</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.proxyuser.*.users</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.signature.secret</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.signature.secret.file</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.signer.secret.provider</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.signer.secret.provider.object</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -900,7 +911,7 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.http-authentication.token.validity</name>
|
||||
<value></value>
|
||||
<value/>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -920,12 +931,12 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.leveldb-state-store.path</name>
|
||||
<value>/data/hadoop/yarn/timeline</value>
|
||||
<value>/data2/hadoop/yarn/timeline</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
|
||||
<value>/data/hadoop/yarn/timeline</value>
|
||||
<value>/data2/hadoop/yarn/timeline</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -955,12 +966,12 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.reader.webapp.address</name>
|
||||
<value>b1m4.hdp.dc:8198</value>
|
||||
<value>a4m3.hdp.dc:8198</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.reader.webapp.https.address</name>
|
||||
<value>b1m4.hdp.dc:8199</value>
|
||||
<value>a4m3.hdp.dc:8199</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -1000,12 +1011,12 @@
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.webapp.address</name>
|
||||
<value>b1m4.hdp.dc:8188</value>
|
||||
<value>a4m3.hdp.dc:8188</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.timeline-service.webapp.https.address</name>
|
||||
<value>b1m4.hdp.dc:8190</value>
|
||||
<value>a4m3.hdp.dc:8190</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
@@ -1018,9 +1029,18 @@
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.authorization-provider</name>
|
||||
<value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.max-completed-applications</name>
|
||||
<value>10000</value>
|
||||
<name>yarn.nodemanager.numa-awareness.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
<property>
|
||||
<name>yarn.nodemanager.numa-awareness.read-topology</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
@@ -1,7 +1,5 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
@@ -32,7 +30,7 @@
|
||||
<name>fs.trash.interval</name>
|
||||
<value>360</value>
|
||||
</property>
|
||||
|
||||
|
||||
<property>
|
||||
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
|
||||
<value>120</value>
|
||||
|
||||
@@ -105,7 +105,17 @@
|
||||
<name>yarn.log-aggregation.retain-seconds</name>
|
||||
<value>2592000</value>
|
||||
</property>
|
||||
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-formats</name>
|
||||
<value>TFile</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-controller.TFile.class</name>
|
||||
<value>org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log.server.url</name>
|
||||
<value>http://b1m4.hdp.dc:19888/jobhistory/logs</value>
|
||||
|
||||
@@ -105,6 +105,16 @@
|
||||
<name>yarn.log-aggregation.retain-seconds</name>
|
||||
<value>172800</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-formats</name>
|
||||
<value>TFile</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-controller.TFile.class</name>
|
||||
<value>org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log.server.url</name>
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
|
||||
@@ -105,7 +105,17 @@
|
||||
<name>yarn.log-aggregation.retain-seconds</name>
|
||||
<value>604800</value>
|
||||
</property>
|
||||
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-formats</name>
|
||||
<value>TFile</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log-aggregation.file-controller.TFile.class</name>
|
||||
<value>org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.log.server.url</name>
|
||||
<value>http://b5m1.hdp.dc:19888/jobhistory/logs</value>
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://b2</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload.buffer</name>
|
||||
<value>disk</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.multipart.size</name>
|
||||
<value>67108864</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.trash.interval</name>
|
||||
<value>4320</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.trash.checkpoint.interval</name>
|
||||
<value>360</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.acl</name>
|
||||
<value>sasl:nn:rwcda</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.quorum</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.signature.secret.file</name>
|
||||
<value>/etc/security/http_secret</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.type</name>
|
||||
<value>simple</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-headers</name>
|
||||
<value>X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-methods</name>
|
||||
<value>GET,PUT,POST,OPTIONS,HEAD,DELETE</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-origins</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.max-age</name>
|
||||
<value>1800</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.filter.initializers</name>
|
||||
<value>org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.rpc.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.auth_to_local</name>
|
||||
<value>RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/
|
||||
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
|
||||
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
|
||||
RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/
|
||||
RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/
|
||||
RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/
|
||||
RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/
|
||||
RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/
|
||||
RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/
|
||||
RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/
|
||||
DEFAULT</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authentication</name>
|
||||
<value>kerberos</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authorization</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.instrumentation.requires.admin</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codec.lzo.class</name>
|
||||
<value>com.hadoop.compression.lzo.LzoCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codecs</name>
|
||||
<value>org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.file.buffer.size</name>
|
||||
<value>131072</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.serializations</name>
|
||||
<value>org.apache.hadoop.io.serializer.WritableSerialization</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connect.max.retries</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connection.maxidletime</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.idlethreshold</name>
|
||||
<value>8000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.server.tcpnodelay</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobtracker.webinterface.trusted</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.fallback-to-simple-auth-allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.hdfs.impl.disable.cache</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
@@ -1,698 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>dfs.block.access.token.enable</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blockreport.initialDelay</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blocksize</name>
|
||||
<value>134217728</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
|
||||
<value>4096</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.retry.policy.enabled</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.cluster.administrators</name>
|
||||
<value> hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.content-summary.limit</name>
|
||||
<value>5000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.data.transfer.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.address</name>
|
||||
<value>0.0.0.0:1019</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.balance.bandwidthPerSec</name>
|
||||
<value>6250000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir.perm</name>
|
||||
<value>750</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.du.reserved</name>
|
||||
<value>26405499904</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.failed.volumes.tolerated</name>
|
||||
<value>2</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.http.address</name>
|
||||
<value>0.0.0.0:1022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.https.address</name>
|
||||
<value>0.0.0.0:50475</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.ipc.address</name>
|
||||
<value>0.0.0.0:8010</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.kerberos.principal</name>
|
||||
<value>dn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/dn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.max.transfer.threads</name>
|
||||
<value>16384</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.domain.socket.path</name>
|
||||
<value>/var/lib/hadoop-hdfs/dn_socket</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.encrypt.data.transfer.cipher.suites</name>
|
||||
<value>AES/CTR/NoPadding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.automatic-failover.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.fencing.methods</name>
|
||||
<value>shell(/bin/true)</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b2</name>
|
||||
<value>nn3,nn4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.heartbeat.interval</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.hosts.exclude</name>
|
||||
<value>/etc/hadoop/conf/dfs.exclude</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.http.policy</name>
|
||||
<value>HTTP_ONLY</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.https.port</name>
|
||||
<value>50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.internal.nameservices</name>
|
||||
<value>b1,b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b1</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b2</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.http-address</name>
|
||||
<value>0.0.0.0:8480</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.https-address</name>
|
||||
<value>0.0.0.0:8481</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.principal</name>
|
||||
<value>jn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/jn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.accesstime.precision</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.acls.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.audit.log.async</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.read.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.write.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.dir</name>
|
||||
<value>/data/hadoop/hdfs/namesecondary</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.edits.dir</name>
|
||||
<value>${dfs.namenode.checkpoint.dir}</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.period</name>
|
||||
<value>21600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.txns</name>
|
||||
<value>1000000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fslock.fair</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.handler.count</name>
|
||||
<value>200</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.principal</name>
|
||||
<value>nn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/nn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.max.extra.edits.segments.retained</name>
|
||||
<value>180</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir</name>
|
||||
<value>/data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir.restore</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.num.extra.edits.retained</name>
|
||||
<value>18000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.safemode.threshold-pct</name>
|
||||
<value>0.99</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b1</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b2</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.stale.datanode.interval</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
|
||||
<value>3600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.write.stale.datanode.ratio</name>
|
||||
<value>1.0f</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.nameservices</name>
|
||||
<value>b1,b2,b3,b4,a3,a4,f1,d2,e1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.ContentSummary.subAccess</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.superusergroup</name>
|
||||
<value>hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication.max</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.webhdfs.enabled</name>
|
||||
<value>true</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.permissions.umask-mode</name>
|
||||
<value>022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.caller.context.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>manage.include.files</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.exports.allowed.hosts</name>
|
||||
<value>* rw</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.file.dump.dir</name>
|
||||
<value>/tmp/.hdfs-nfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.datanode-restart.timeout</name>
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.f1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.f1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.d2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.d2</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.e1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.e1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,262 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<xi:include href="/usr/lib/edp/hadoop-3.2.2/etc/hadoop/viewfs-mount-table.xml"/>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://b1</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.fast.upload.buffer</name>
|
||||
<value>disk</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.s3a.multipart.size</name>
|
||||
<value>67108864</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.trash.interval</name>
|
||||
<value>360</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.acl</name>
|
||||
<value>sasl:nn:rwcda</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ha.zookeeper.quorum</name>
|
||||
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.signature.secret.file</name>
|
||||
<value>/etc/security/http_secret</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.authentication.type</name>
|
||||
<value>simple</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-headers</name>
|
||||
<value>X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-methods</name>
|
||||
<value>GET,PUT,POST,OPTIONS,HEAD,DELETE</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.allowed-origins</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.cross-origin.max-age</name>
|
||||
<value>1800</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.http.filter.initializers</name>
|
||||
<value>org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hdfs.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.hive.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.HTTP.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.iap.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.livy.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.groups</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.proxyuser.yarn.hosts</name>
|
||||
<value>*</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.rpc.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.auth_to_local</name>
|
||||
<value>RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/
|
||||
RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/
|
||||
RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/
|
||||
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
|
||||
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
|
||||
RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/
|
||||
RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/
|
||||
RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/
|
||||
RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/
|
||||
RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/
|
||||
RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/
|
||||
RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/
|
||||
RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/
|
||||
RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/
|
||||
RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/
|
||||
DEFAULT</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authentication</name>
|
||||
<value>kerberos</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.authorization</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.security.instrumentation.requires.admin</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codec.lzo.class</name>
|
||||
<value>com.hadoop.compression.lzo.LzoCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.compression.codecs</name>
|
||||
<value>org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.file.buffer.size</name>
|
||||
<value>131072</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>io.serializations</name>
|
||||
<value>org.apache.hadoop.io.serializer.WritableSerialization</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connect.max.retries</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.connection.maxidletime</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.idlethreshold</name>
|
||||
<value>8000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.server.tcpnodelay</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobtracker.webinterface.trusted</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.client.fallback-to-simple-auth-allowed</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.hdfs.impl.disable.cache</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
@@ -1,698 +0,0 @@
|
||||
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
|
||||
<property>
|
||||
<name>dfs.block.access.token.enable</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blockreport.initialDelay</name>
|
||||
<value>120</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.blocksize</name>
|
||||
<value>134217728</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
|
||||
<value>4096</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.retry.policy.enabled</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.cluster.administrators</name>
|
||||
<value> hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.content-summary.limit</name>
|
||||
<value>5000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.data.transfer.protection</name>
|
||||
<value>authentication,privacy</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.address</name>
|
||||
<value>0.0.0.0:1019</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.balance.bandwidthPerSec</name>
|
||||
<value>6250000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir.perm</name>
|
||||
<value>750</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.du.reserved</name>
|
||||
<value>26405499904</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.failed.volumes.tolerated</name>
|
||||
<value>2</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.http.address</name>
|
||||
<value>0.0.0.0:1022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.https.address</name>
|
||||
<value>0.0.0.0:50475</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.ipc.address</name>
|
||||
<value>0.0.0.0:8010</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.kerberos.principal</name>
|
||||
<value>dn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/dn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.max.transfer.threads</name>
|
||||
<value>16384</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.domain.socket.path</name>
|
||||
<value>/var/lib/hadoop-hdfs/dn_socket</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.encrypt.data.transfer.cipher.suites</name>
|
||||
<value>AES/CTR/NoPadding</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.automatic-failover.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.fencing.methods</name>
|
||||
<value>shell(/bin/true)</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b2</name>
|
||||
<value>nn3,nn4</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.heartbeat.interval</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.hosts.exclude</name>
|
||||
<value>/etc/hadoop/conf/dfs.exclude</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.http.policy</name>
|
||||
<value>HTTP_ONLY</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.https.port</name>
|
||||
<value>50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.internal.nameservices</name>
|
||||
<value>b1,b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b1</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.edits.dir.b2</name>
|
||||
<value>/data2/hadoop/hdfs/journal</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.http-address</name>
|
||||
<value>0.0.0.0:8480</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.https-address</name>
|
||||
<value>0.0.0.0:8481</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.kerberos.principal</name>
|
||||
<value>jn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.journalnode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/jn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.accesstime.precision</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.acls.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.audit.log.async</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.read.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.avoid.write.stale.datanode</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.dir</name>
|
||||
<value>/data/hadoop/hdfs/namesecondary</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.edits.dir</name>
|
||||
<value>${dfs.namenode.checkpoint.dir}</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.period</name>
|
||||
<value>21600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.checkpoint.txns</name>
|
||||
<value>1000000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fslock.fair</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.handler.count</name>
|
||||
<value>200</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50070</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:50470</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.kerberos.principal</name>
|
||||
<value>nn/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.keytab.file</name>
|
||||
<value>/etc/security/keytabs/nn.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.max.extra.edits.segments.retained</name>
|
||||
<value>180</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir</name>
|
||||
<value>/data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir.restore</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.num.extra.edits.retained</name>
|
||||
<value>18000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn1</name>
|
||||
<value>b1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b1.nn2</name>
|
||||
<value>b1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn3</name>
|
||||
<value>b1m5.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b2.nn4</name>
|
||||
<value>b1m6.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.safemode.threshold-pct</name>
|
||||
<value>0.99</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b1</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.shared.edits.dir.b2</name>
|
||||
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.stale.datanode.interval</name>
|
||||
<value>30000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
|
||||
<value>3600</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.write.stale.datanode.ratio</name>
|
||||
<value>1.0f</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.nameservices</name>
|
||||
<value>b1,b2,b3,b4,a3,a4,f1,d2,e1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.ContentSummary.subAccess</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions.superusergroup</name>
|
||||
<value>hdfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>3</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.replication.max</name>
|
||||
<value>50</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.keytab</name>
|
||||
<value>/etc/security/keytabs/spnego.service.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.web.authentication.kerberos.principal</name>
|
||||
<value>HTTP/_HOST@ECLD.COM</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.webhdfs.enabled</name>
|
||||
<value>true</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.permissions.umask-mode</name>
|
||||
<value>022</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hadoop.caller.context.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>manage.include.files</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.exports.allowed.hosts</name>
|
||||
<value>* rw</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>nfs.file.dump.dir</name>
|
||||
<value>/tmp/.hdfs-nfs</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.datanode-restart.timeout</name>
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn1</name>
|
||||
<value>a4m1.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a4.nn2</name>
|
||||
<value>a4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.a3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.a3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn1</name>
|
||||
<value>a3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.a3.nn2</name>
|
||||
<value>a3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b3</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b3</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn1</name>
|
||||
<value>b3m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b3.nn2</name>
|
||||
<value>b3m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.b4</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.b4</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn1</name>
|
||||
<value>b4m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.b4.nn2</name>
|
||||
<value>b4m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.f1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.f1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn1</name>
|
||||
<value>f1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.f1.nn2</name>
|
||||
<value>f1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.d2</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.d2</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn1</name>
|
||||
<value>d2m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.d2.nn2</name>
|
||||
<value>d2m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.failover.proxy.provider.e1</name>
|
||||
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.ha.namenodes.e1</name>
|
||||
<value>nn1,nn2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.http-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50070</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.https-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:50470</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn1</name>
|
||||
<value>e1m2.hdp.dc:8020</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.rpc-address.e1.nn2</name>
|
||||
<value>e1m3.hdp.dc:8020</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,7 @@ public class RuntimeInfo {
|
||||
private String jarPath;
|
||||
private String jdkPath;
|
||||
private String logPath;
|
||||
private String confPath;
|
||||
private String dataPath;
|
||||
private String kerberosKeytabPath;
|
||||
private LokiInfo loki;
|
||||
@@ -52,6 +53,14 @@ public class RuntimeInfo {
|
||||
this.logPath = logPath;
|
||||
}
|
||||
|
||||
public String getConfPath() {
|
||||
return confPath;
|
||||
}
|
||||
|
||||
public void setConfPath(String confPath) {
|
||||
this.confPath = confPath;
|
||||
}
|
||||
|
||||
public String getDataPath() {
|
||||
return dataPath;
|
||||
}
|
||||
@@ -123,6 +132,7 @@ public class RuntimeInfo {
|
||||
", jarPath='" + jarPath + '\'' +
|
||||
", jdkPath='" + jdkPath + '\'' +
|
||||
", logPath='" + logPath + '\'' +
|
||||
", confPath='" + confPath + '\'' +
|
||||
", dataPath='" + dataPath + '\'' +
|
||||
", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
|
||||
", loki=" + loki +
|
||||
@@ -130,6 +140,7 @@ public class RuntimeInfo {
|
||||
", connectorZkUrl='" + connectorZkUrl + '\'' +
|
||||
", hudi=" + hudi +
|
||||
", security=" + security +
|
||||
", yarn=" + yarn +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ public class ServiceInfo {
|
||||
private List<String> groups = new ArrayList<>();
|
||||
private Integer replicas = 0;
|
||||
private String sourceJar;
|
||||
private List<String> classpath = new ArrayList<>();
|
||||
private Map<String, Object> environments = new HashMap<>();
|
||||
private Map<String, Object> arguments = new HashMap<>();
|
||||
|
||||
@@ -60,6 +61,14 @@ public class ServiceInfo {
|
||||
this.sourceJar = sourceJar;
|
||||
}
|
||||
|
||||
public List<String> getClasspath() {
|
||||
return classpath;
|
||||
}
|
||||
|
||||
public void setClasspath(List<String> classpath) {
|
||||
this.classpath = classpath;
|
||||
}
|
||||
|
||||
public Map<String, Object> getEnvironments() {
|
||||
return environments;
|
||||
}
|
||||
@@ -84,6 +93,7 @@ public class ServiceInfo {
|
||||
", groups=" + groups +
|
||||
", replicas=" + replicas +
|
||||
", sourceJar='" + sourceJar + '\'' +
|
||||
", classpath=" + classpath +
|
||||
", environments=" + environments +
|
||||
", arguments=" + arguments +
|
||||
'}';
|
||||
|
||||
@@ -17,10 +17,16 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.list.MutableList;
|
||||
import org.eclipse.collections.api.map.MutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
@@ -120,13 +126,12 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String deployScript = deployTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getHostnameIp)
|
||||
.sorted((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp()))
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getHostnameIp)
|
||||
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
|
||||
.put("selectedHosts", selectedHosts)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
.put("classpath", String.join(":", serviceInfo.getClasspath()))
|
||||
.put("arguments", serviceInfo.getArguments())
|
||||
.put("environments", serviceInfo.getEnvironments())
|
||||
.build());
|
||||
@@ -141,10 +146,8 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String stopScript = stopTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getIp)
|
||||
.sorted(Comparator.naturalOrder())
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getIp)
|
||||
.toSortedList(Comparator.naturalOrder()))
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
.put("arguments", serviceInfo.getArguments())
|
||||
@@ -161,10 +164,8 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String logScript = logTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getIp)
|
||||
.sorted(Comparator.naturalOrder())
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getIp)
|
||||
.toSortedList(Comparator.naturalOrder()))
|
||||
.put("selectedHosts", selectedHosts)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
@@ -179,29 +180,56 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
Files.write(logScriptFile, logScript.getBytes());
|
||||
}
|
||||
|
||||
ImmutableList<String> clusters = Lists.mutable.of(runtimeInfo.getYarn().getSyncClusters().split(","))
|
||||
.withAll(Lists.mutable.of(runtimeInfo.getYarn().getCompactionClusters().split(",")))
|
||||
.collect(n -> runtimeInfo.getYarn().getStoreCluster() + n)
|
||||
.distinct()
|
||||
.toImmutable();
|
||||
Template downloadConfigTemplate = engine.getTemplate("cloud/download-config.ftl");
|
||||
String downloadConfigScript = downloadConfigTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("classpath", clusters)
|
||||
.build());
|
||||
Path downloadConfigScriptFile = Paths.get(root.toString(), "download-config.sh");
|
||||
Files.deleteIfExists(downloadConfigScriptFile);
|
||||
Files.write(downloadConfigScriptFile, downloadConfigScript.getBytes());
|
||||
|
||||
Template deployConfigTemplate = engine.getTemplate("cloud/deploy-config.ftl");
|
||||
String deployConfigScript = deployConfigTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("hosts", hostInfoList
|
||||
.collect(HostInfoWrapper::getHostnameIp)
|
||||
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
|
||||
.build());
|
||||
Path deployConfigScriptFile = Paths.get(root.toString(), "deploy-config.sh");
|
||||
Files.deleteIfExists(deployConfigScriptFile);
|
||||
Files.write(deployConfigScriptFile, deployConfigScript.getBytes());
|
||||
|
||||
ClassPathResource stopFile = new ClassPathResource("script/stop.sh");
|
||||
String stopScript = new String(IoUtil.readBytes(stopFile.getInputStream()));
|
||||
Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
|
||||
|
||||
Map<String, List<ServiceInfoWrapper>> groups = new HashMap<>();
|
||||
MutableMap<String, MutableList<ServiceInfoWrapper>> groups = Maps.mutable.empty();
|
||||
for (ServiceInfoWrapper service : serviceInfoList) {
|
||||
service.getGroups().add(0, "all");
|
||||
for (String group : service.getGroups()) {
|
||||
if (!groups.containsKey(group)) {
|
||||
groups.put(group, new ArrayList<>());
|
||||
groups.put(group, Lists.mutable.empty());
|
||||
}
|
||||
List<ServiceInfoWrapper> list = groups.get(group);
|
||||
list.add(service);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, List<ServiceInfoWrapper>> entry : groups.entrySet()) {
|
||||
for (Map.Entry<String, MutableList<ServiceInfoWrapper>> entry : groups.entrySet()) {
|
||||
String group = entry.getKey();
|
||||
List<ServiceInfoWrapper> infos = entry.getValue();
|
||||
MutableList<ServiceInfoWrapper> infos = entry.getValue();
|
||||
|
||||
Template batchDeployTemplate = engine.getTemplate("cloud/batch-deploy.ftl");
|
||||
String batchDeployScript = batchDeployTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList()))
|
||||
.put("services", infos.collect(ServiceInfoWrapper::getName))
|
||||
.build());
|
||||
Path batchDeployScriptFile = Paths.get(
|
||||
root.toString(),
|
||||
@@ -213,7 +241,7 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
Template batchStopTemplate = engine.getTemplate("cloud/batch-stop.ftl");
|
||||
String batchStopScript = batchStopTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList()))
|
||||
.put("services", infos.collect(ServiceInfoWrapper::getName))
|
||||
.build());
|
||||
Path batchStopScriptFile = Paths.get(
|
||||
root.toString(),
|
||||
|
||||
@@ -49,6 +49,10 @@ public class ServiceInfoWrapper {
|
||||
return serviceInfo.getSourceJar();
|
||||
}
|
||||
|
||||
public List<String> getClasspath() {
|
||||
return serviceInfo.getClasspath();
|
||||
}
|
||||
|
||||
public Map<String, Object> getEnvironments() {
|
||||
return environments;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ deploy:
|
||||
jar-path: /data/datalake/jars
|
||||
jdk-path: /opt/jdk8u252-b09/bin/java
|
||||
log-path: /data/datalake/logs
|
||||
conf-path: /data/datalake/conf
|
||||
data-path: /data/datalake/data
|
||||
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
|
||||
loki:
|
||||
|
||||
@@ -4,6 +4,7 @@ deploy:
|
||||
jar-path: /apps/datalake/hudi/jars
|
||||
jdk-path: /opt/jdk1.8.0_162/bin/java
|
||||
log-path: /apps/datalake/hudi/logs
|
||||
conf-path: /apps/datalake/hudi/conf
|
||||
data-path: /apps/datalake/hudi/data
|
||||
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
|
||||
loki:
|
||||
|
||||
@@ -28,6 +28,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b1
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -49,6 +51,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b5
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -70,6 +74,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2a4
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -91,6 +97,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b12
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -154,6 +162,8 @@ deploy:
|
||||
- "service"
|
||||
source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
|
||||
replicas: 5
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b12
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
<#list hosts as host>
|
||||
echo ${host.ip} ${host.hostname}
|
||||
ssh ${host.ip} 'bash -s' < ${currentPath}/download-config.sh
|
||||
</#list>
|
||||
@@ -24,7 +24,7 @@ hostname=`ssh $host 'hostname'`
|
||||
hostname_full=`ssh $host 'hostname -f'`
|
||||
ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
|
||||
start_time=`date +%Y%m%d%H%M%S`
|
||||
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
|
||||
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -Xbootclasspath/a:${classpath} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
|
||||
</#if>
|
||||
echo ''
|
||||
</#list>
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
config_names=(
|
||||
<#list classpath as cp>
|
||||
${cp}
|
||||
</#list>
|
||||
)
|
||||
|
||||
for config_name in <#noparse>${config_names[@]}</#noparse>
|
||||
do
|
||||
conf_path=${runtime.confPath}/<#noparse>${config_name}</#noparse>
|
||||
tar_path=<#noparse>${conf_path}</#noparse>.tar.gz
|
||||
rm -rf <#noparse>${conf_path}</#noparse>
|
||||
mkdir -p <#noparse>${conf_path}</#noparse>
|
||||
curl -s ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/<#noparse>${config_name}</#noparse>.tar.gz -o <#noparse>${tar_path}</#noparse>
|
||||
tar -zxf <#noparse>${tar_path}</#noparse> -C <#noparse>${conf_path}</#noparse>
|
||||
rm -rf <#noparse>${tar_path}</#noparse>
|
||||
done
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.cli.runner;
|
||||
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-02-27
|
||||
*/
|
||||
public class UrlTest {
|
||||
public static void main(String[] args) {
|
||||
System.out.println(URLUtil.toURI("ftp://yyy:QeY!68)4nH1@132.121.122.15:2222/b9b9.tar.gz").getPath());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user