大数据异常问题集锦

大数据异常问题集锦 未知错误1

org.apache.hadoop.hdfs.qjournal.client.QuorumException: Could not format one or more JournalNodes. 1 exceptions thrown:
192.168.141.104:8485: Directory /var/bdsp/hadoop/ha/qjm/bdsp is in an inconsistent state: Can't format the storage directory because the current directory is not empty.
	at org.apache.hadoop.hdfs.server.common.Storage$StorageDirectory.checkEmptyCurrent(Storage.java:622)
	at org.apache.hadoop.hdfs.server.common.Storage$StorageDirectory.analyzeStorage(Storage.java:706)
	at org.apache.hadoop.hdfs.qjournal.server.JNStorage.format(JNStorage.java:219)
	at org.apache.hadoop.hdfs.qjournal.server.Journal.format(Journal.java:258)
	at org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer.format(JournalNodeRpcServer.java:183)
	at org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolServerSideTranslatorPB.format(QJournalProtocolServerSideTranslatorPB.java:150)
	at org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$QJournalProtocolService.callBlockingMethod(QJournalProtocolProtos.java:28972)
	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1086)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1029)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:957)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2957)

	at org.apache.hadoop.hdfs.qjournal.client.QuorumException.create(QuorumException.java:81)
	at org.apache.hadoop.hdfs.qjournal.client.QuorumCall.rethrowException(QuorumCall.java:305)
	at org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.format(QuorumJournalManager.java:264)
	at org.apache.hadoop.hdfs.server.namenode.FSEditLog.formatNonFileJournals(FSEditLog.java:429)
	at org.apache.hadoop.hdfs.server.namenode.FSImage.format(FSImage.java:189)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:1230)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1673)
	at org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:1783)
2021-10-30 02:29:11,528 INFO util.ExitUtil: Exiting with status 1: org.apache.hadoop.hdfs.qjournal.client.QuorumException: Could not format one or more JournalNodes. 1 exceptions thrown:
192.168.141.104:8485: Directory /var/bdsp/hadoop/ha/qjm/bdsp is in an inconsistent state: Can't format the storage directory because the current directory is not empty.
	at org.apache.hadoop.hdfs.server.common.Storage$StorageDirectory.checkEmptyCurrent(Storage.java:622)
	at org.apache.hadoop.hdfs.server.common.Storage$StorageDirectory.analyzeStorage(Storage.java:706)
	at org.apache.hadoop.hdfs.qjournal.server.JNStorage.format(JNStorage.java:219)
	at org.apache.hadoop.hdfs.qjournal.server.Journal.format(Journal.java:258)
	at org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer.format(JournalNodeRpcServer.java:183)
	at org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolServerSideTranslatorPB.format(QJournalProtocolServerSideTranslatorPB.java:150)
	at org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$QJournalProtocolService.callBlockingMethod(QJournalProtocolProtos.java:28972)
	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1086)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1029)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:957)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2957)

2021-10-30 02:29:11,530 INFO namenode.NameNode: SHUTDOWN_MSG: 

未知错误1 原先配置

fs.defaultFS
hdfs://bdsp


hadoop.tmp.dir
/var/bdsp/hadoop/ha


hadoop.http.staticuser.user
root


ha.zookeeper.quorum
nodez001:2181,nodez002:2181,nodez003:2181

## hdfs-site.xml 

dfs.nameservices
bdsp


dfs.ha.namenodes.bdsp
nn1,nn2


dfs.namenode.rpc-address.bdsp.nn1
nodez001:8020


dfs.namenode.rpc-address.bdsp.nn2
nodez002:8020


dfs.namenode.http-address.bdsp.nn1
nodez001:9870


dfs.namenode.http-address.bdsp.nn2
nodez002:9870


dfs.namenode.shared.edits.dir
qjournal://nodez001:8485;nodez002:8485;nodez003:8485/bdsp


dfs.journalnode.edits.dir
/var/bdsp/hadoop/ha/qjm


dfs.client.failover.proxy.provider.bdsp
org.apache.hadoop.hdfs.server.namenode.ha.configuredFailoverProxyProvider

   
dfs.ha.fencing.methods
sshfence
shell(/bin/true)


dfs.ha.fencing.ssh.private-key-files
/root/.ssh/id_rsa

 
dfs.ha.automatic-failover.enabled
true

解决方法

清空所有节点上的logs(${HADOOP_HOME}/logs/)和 data（/var/bdsp/hadoop/ha/）中数据