flinkx on yarn perjob

[zyf-proj]

flinkx run with flink on yarn perjob mode, the command and job config as follows. it does not throw an exception, and blocked in YarnClusterDescriptor.deployJobCluster. what is the reason of this probolem.

bin/flinkx -mode yarnPer -job ./job/demo.json -pluginRoot /home/yifanzhang/flinkx/plugins -flinkconf /home/yifanzhang/flink-1.8.1/conf -yarnconf /opt/cloudera/parcels/CDH/lib/hadoop-yarn/etc/hadoop -flinkLibJar /home/yifanzhang/flink/lib

{
"job" : {
"content" : [ {
"reader" : {
"parameter" : {
"groupId" : "default",
"topic" : "test",
"consumerSettings" : {
"zookeeper.connect" : "10.126.144.37:2181,10.126.144.38:2181,10.126.144.39:2181/cp3",
"bootstrap.servers" : "10.126.144.34:9092,10.126.144.35:9092,10.126.144.36:9092",
"auto.commit.interval.ms" : "1000",
"auto.offset.reset" : "latest"
}
},
"name" : "kafka11reader",
"type" : 14
},
"writer" : {
"parameter" : {
"path" : "/tmp",
"fileName" : "test",
"hadoopConfig" : {
"dfs.ha.namenodes.ns1" : "namenode252,namenode369",
"fs.defaultFS" : "hdfs:///idc-nn",
"dfs.namenode.rpc-address.idc-nn.namenode252" : "shoyi04plpebdp015.mcd.com.cn:8020",
"dfs.client.failover.proxy.provider.idc-nn" : "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"dfs.namenode.rpc-address.idc-nn.namenode369" : "shoyi05plpebdp019.mcd.com.cn:8022",
"dfs.nameservices" : "idc-nn",
"fs.hdfs.impl.disable.cache" : "true",
"fs.hdfs.impl" : "org.apache.hadoop.hdfs.DistributedFileSystem",
"hadoop.security.authorization": "true",
"hadoop.security.authentication": "Kerberos",
"dfs.namenode.kerberos.principal": "yifanzhang",
"dfs.namenode.keytab.file": "/home/yifanzhang/yifanzhang.keytab",
"java.security.krb5.conf": "/etc/krb5.conf"
},
"column" : [ {
"name" : "id",
"index" : 0,
"type" : "string"
} ],
"defaultFS" : "hdfs:///idc-nn",
"writeMode" : "APPEND",
"fieldDelimiter" : ",",
"fileType" : "orc",
"charsetName" : "utf-8"
},
"name" : "hdfswriter",
"type" : 6
}
} ],
"setting" : {
"restore" : {
"isRestore" : true,
"isStream" : true
},
"errorLimit" : {
},
"speed" : {
"bytes" : 2097152,
"channel" : 3
}
}
}
}

[kanata163]

往kerberos集群提交需要指定kerberos相关的启动参数

[FlechazoW]

This issue has been too long to review it and reopen new one if necessary.