大数据组件安装记录：HadoopHBasezookeeperkafka

大数据组件安装记录：Hadoop/HBase/zookeeper/kafka 安装记录 ubuntu 环境 ~/.bashrc或者/etc/profile

export JAVA_HOME=/usr/local/jdk1.8.0_291
export PATH=${JAVA_HOME}/bin:${PATH}

export M2_HOME=/usr/local/maven
export PATH=${M2_HOME}/bin:${PATH}

export SCALA_HOME=/usr/local/scala
export PATH=$PATH:$SCALA_HOME/bin

export KAFKA_HOME=/usr/local/bigdata/kafka
export PATH=$PATH:$KAFKA_HOME/bin

export HADOOP_HOME=/usr/local/bigdata/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

export SPARK_HOME=/usr/local/spark
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

export ZK_HOME=/usr/local/bigdata/zookeeper
export PATH=$PATH:$ZK_HOME/bin

export Hbase_HOME=/usr/local/bigdata/hbase
export PATH=$PATH:$Hbase_HOME/bin

配置JDK和Maven

版本：
JDK： 1.8
Maven： 1.8
配置环境变量：
export JAVA_HOME=/usr/local/jdk1.8.0_291
export PATH=${JAVA_HOME}/bin:${PATH}

export M2_HOME=/usr/local/maven
export PATH=${M2_HOME}/bin:${PATH}

Hadoop 版本 hadoop-2.10.1

// 进入hadoop安装目录
sudo vim etc/hadoop/hadoop-env.sh
// 设置JAVA_HOME 参数 根据自身电脑的Java位置
export JAVA_HOME=/usr/local/jdk1.8.0_291
//保存退出

//配置 etc/hadoop/core-site.xml

        
        hadoop.tmp.dir
        file:/usr/local/hadoop/tmp
        Abase for other temporary directories.
    
    
        fs.defaultFS
        hdfs://localhost:9000
    

// 配置etc/hadoop/hdfs-site.xml

        
        dfs.replication
        1
    
    
        dfs.namenode.name.dir
        file:/usr/local/hadoop/tmp/dfs/name
    
    
        dfs.datanode.data.dir
        file:/usr/local/hadoop/tmp/dfs/data
    

//配置ssh免密码登录
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
//格式化文件系统 不要重复格式化，会导致Datanode节点启动不了
bin/hdfs namenode -format
// 启动hadoop服务
sbin/start-dfs.sh

// 如果出现 The authenticity of host 'master (192.168.5.133)' can‘t be established.
#打开文件
vi /etc/ssh/ssh_config
#在最后面添加如下语句即可
StrictHostKeyChecking no
UserKnownHostsFile /dev/null

zookeeper

//配置zookeeper
//创建data 存放数据
mkdir data
//编辑配置文件
sudo mv conf/zoo_sample.cfg conf/zoo.cfg
sudo vim conf/zoo.cfg
//配置如下
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/usr/local/bigdata/zookeeper/data
# the port at which the clients will connect
clientPort=2181

//启动zookeeper
sudo bash bin/zkServer.sh start

//failed to start 
mvn package -Dmaven.test.skip=true

Hbase

# 下载：https://hbase.apache.org/downloads.html
sudo tar -vxzf hbase-2.4.2-bin.tar.gz
cp -a hbase-2.4.2 /usr/local/hbase-2.4.2
# 配置hbase-env.sh
sudo vim conf/hbase-env.sh
//设置如下参数
export JAVA_HOME=/usr/local/jdk1.8.0_291
export Hbase_MANAGES_ZK=false

// 配置habse-site.xml 
sudo vim conf/hbase-site.xml 
//配置如下
 
    hbase.rootdir
    hdfs://hadoop:9000/hbase
  
  
     hbase.cluster.distributed
     true
  

  
    hbase.zookeeper.property.clientPort
    2181
  
  
     hbase.zookeeper.quorum
     hadoop
  
	
    hbase.zookeeper.property.dataDir
    /usr/local/bigdata/zookeeper/zoo_data
  
# 启动Hbase
./bin/start-hbase.sh

Kafka

# 配置kafka 
# sudo vim config/server.properties
broker.id=0
port=9092
log.dirs=/usr/local/bigdata/kafka/kafkalog
# 启动Kafka
sudo bash bin/kafka-server-start.sh config/server.properties