- 基础软件安装
- 安装部署
- PostgreSQL (8.2.15+) or MySQL (5.7系列) : 两者任选其一即可
- JDK (1.8+) : 必装,请安装好后在/etc/profile下配置 JAVA_HOME 及 PATH 变量
- ZooKeeper (3.4.6+) :必装
- Hadoop (2.6+) or MinIO :选装,如果需要用到资源上传功能,可以选择上传到Hadoop or MinIO上
注意:DolphinScheduler本身不依赖Hadoop、Hive、Spark,仅是会调用他们的Client,用于对应任务的提交。
安装部署- 下载上传安装包
DolphinScheduler下载
[root@hadoop103 module]# mkdir /opt/module/dolphinscheduler [root@hadoop103 module]# cd dolphinscheduler/ [root@hadoop103 dolphinscheduler]# tar -zxvf apache-dolphinscheduler-incubating-1.3.3-dolphinscheduler-bin.tar.gz -C /opt/module/dolphinscheduler [root@hadoop103 dolphinscheduler]# mv apache-dolphinscheduler-incubating-1.3.3-dolphinscheduler-bin dolphinscheduler-bin
- 创建部署用户
分别在hadoop101,hadoop102,hadoop103三台机器创建dolphinscheduler用户。
[root@hadoop101 ~]# useradd dolphinscheduler [root@hadoop101 ~]# echo "dolphinscheduler123" | passwd --stdin dolphinscheduler Changing password for user dolphinscheduler. passwd: all authentication tokens updated successfully. [root@hadoop101 ~]# echo 'dolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers [root@hadoop101 ~]# sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers hadoop102 ,hadoop103 *** 作步骤同上
- 配置hosts映射,并配置ssh免密登录
[root@hadoop103 ~]$ su dolphinscheduler [dolphinscheduler@hadoop103 ~]$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa [dolphinscheduler@hadoop103 ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys [dolphinscheduler@hadoop103 ~]$ chmod 600 ~/.ssh/authorized_keys [root@hadoop103 module]# for ip in hadoop101 hadoop102; > do > ssh-copy-id $ip > done
- 数据库初始化
选择MySQL作为dolphinscheduler的数据库添加mysql-connector-java驱动包到dolphinscheduler的lib目录下。MySQL安装在hadoop101,进入到数据库,创建dolphinscheduer所需数据库。
[root@hadoop101 ~]# mysql -uroot -p****** mysql> CREATE DATAbase dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; ## 密码自定义 mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%' IDENTIFIED BY '******'; mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost' IDENTIFIED BY '******'; mysql> flush privileges;
- 创建表和导入基础数据
[root@hadoop103 module]# cd /opt/module/dolphinscheduler/dolphinscheduler-bin/ [root@hadoop103 dolphinscheduler-bin]# cd conf/ [root@hadoop103 conf]# vim datasource.properties 将postgresql的配置注释,并添加mysql地址 #spring.datasource.driver-class-name=org.postgresql.Driver #spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:mysql://hadoop101:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true spring.datasource.username=dophinscheduler spring.datasource.password=****** 导入脚本,导脚本之前确保配置了JAVA_HOME [root@hadoop103 conf]# cd .. [root@hadoop103 dolphinscheduler-bin]# sh script/create-dolphinscheduler.sh
- 修改运行参数
修改dolphinscheduler_env.sh的环境变量,没有用到的可以注释或者忽略,JAVA_HOME和PATH是必须配置的。
[root@hadoop103 dolphinscheduler-bin]# vim conf/env/dolphinscheduler_env.sh export HADOOP_HOME=/opt/module/hadoop-3.1.3 export HADOOP_CONF_DIR=/opt/module/hadoop-3.1.3/etc/hadoop #export SPARK_HOME1=/opt/soft/spark1 export SPARK_HOME2=/opt/module/spark-3.0.0-bin-hadoop2.7 #export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/module/jdk1.8.0_211 export HIVE_HOME=/opt/module/apache-hive-3.1.2-bin #export Flink_HOME=/opt/soft/flink #export DATAX_HOME=/opt/soft/datax/bin/datax.py export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$Flink_HOME/bin:$DATAX_HOME:$PATH
将jdk软链到/usr/bin/java下
[root@hadoop103 dolphinscheduler-bin]# ln -s /opt/module/jdk1.8.0_211/bin/java /usr/bin/java
- 修改conf/config/install_config.conf各参数
dbtype="mysql" dbhost="hadoop101:3306" username="dolphinscheduler" dbname="dolphinscheduler" password="123456" zkQuorum="hadoop101:2181,hadoop102:2181,hadoop103:2181" # ds安装目录 不同于/opt/module/dolphinscheduler installPath="/opt/module/ds" deployUser="dolphinscheduler" mailServerHost="smtp.qq.com" mailServerPort="25" # sender,配置了和mailUser一样就行 mailSender="[email protected]" # user mailUser="[email protected]" #邮箱密码 mailPassword="xxxxxx" #starttlsEnable和sslEnable不能同时为true starttlsEnable="true" sslEnable="false" sslTrust="smtp.qq.com" resourceStorageType="HDFS" defaultFS="hdfs://mycluster" #resourcemanager HA对应的地址 yarnHaIps="hadoop101,hadoop103" #因为使用了resourcemaanger Ha所以保持默认,如果是单resourcemanager配置对应ip singleYarnIp="yarnIp1" #资源上传根路径,支持hdfs和s3 resourceUploadPath="/data/dolphinscheduler" hdfsRootUser="hdfs" #需要部署ds的机器 ips="hadoop101,hadoop102,hadoop103" sshPort="22" #指定master masters="hadoop101" #指定workers,并且可以指定组名,default为默认组名 workers="hadoop102:default,hadoop103:default" #报警服务器地址 alertServer="hadoop102" #后台api服务器地址 apiServers="hadoop101"
- 如果需要将资源上传到HDFS功能,并且开启了NAMENODE HA则需要将配置文件复制到/opt/module/dolphinscheduler/conf下。非NAMENODE HA则可以忽略
[root@hadoop103 dolphinscheduler-bin]# cp /opt/module/hadoop-3.1.3/etc/hadoop/core-site.xml /opt/module/dolphinscheduler/dolphinscheduler-bin/conf/ [root@hadoop103 dolphinscheduler-bin]#/opt/module/dolphinscheduler/dolphinscheduler-bin/conf/ cp /opt/module/hadoop-3.1.3/etc/hadoop/hdfs-site.xml /opt/module/dolphinscheduler/dolphinscheduler-bin/conf/
- 一键部署
切换用户,执行脚本
[root@hadoop103 module]# chown -R dolphinscheduler:dolphinscheduler dolphinscheduler/ [root@hadoop103 dolphinscheduler]# su dolphinscheduler [dolphinscheduler@hadoop103 dolphinscheduler]$ cd dolphinscheduler-bin/ [dolphinscheduler@hadoop103 dolphinscheduler-bin]$ pwd /opt/module/dolphinscheduler/dolphinscheduler-bin
因为alert服务需要访问mysql,所以hadoop102上也需要mysql驱动包
[root@hadoop101 mysql-connector-java-5.1.27]# scp mysql-connector-java-5.1.27-bin.jar hadoop102:/usr/share/java [root@hadoop102 java]# mv mysql-connector-java-5.1.27-bin.jar mysql-connector-java.jar
一键部署
[root@hadoop103 dolphinscheduler-bin]# sh install.sh
MasterServer ----- master服务 WorkerServer ----- worker服务 LoggerServer ----- logger服务 ApiApplicationServer ----- api服务 alertServer ----- alert服务
- 根据配置文件访问http地址:hadoop101:12345/dolphinscheduler
参考资料:尚硅谷
欢迎分享,转载请注明来源:内存溢出
评论列表(0条)