ICode9

精准搜索请尝试: 精确搜索
首页 > 数据库> 文章详细

dolphinscheduler最新版2.0.2+mysql8.0.16部署(一)

2022-01-26 09:02:06  阅读:671  来源: 互联网

标签:opt bin 2.0 16 dolphin 192.168 mysql 最新版 dolphinscheduler


1. 下载dolphinscheduler

下载地址https://dolphinscheduler.apache.org/en-us/download/download.html

下载版本为:wps79CF.tmp_thumb[2]src源码,需要重新编译

查看pom.xml对应的组件版本,这里不介绍编译方式,下次用到介绍

<curator.version>4.3.0</curator.version>

<zookeeper.version>3.4.14</zookeeper.version>

<spring.version>5.3.12</spring.version>

<spring.boot.version>2.5.6</spring.boot.version>

<java.version>1.8</java.version>

<logback.version>1.2.3</logback.version>

<hadoop.version>2.7.3</hadoop.version>

<quartz.version>2.3.0</quartz.version>

<jackson.version>2.10.5</jackson.version>

<mybatis-plus.version>3.2.0</mybatis-plus.version>

<mybatis.spring.version>2.0.1</mybatis.spring.version>

<cron.utils.version>9.1.3</cron.utils.version>

<druid.version>1.2.4</druid.version>

<h2.version>1.4.200</h2.version>

<commons.codec.version>1.11</commons.codec.version>

<commons.logging.version>1.1.1</commons.logging.version>

<httpclient.version>4.4.1</httpclient.version>

<httpcore.version>4.4.1</httpcore.version>

<junit.version>4.12</junit.version>

<mysql.connector.version>8.0.16</mysql.connector.version>

<slf4j.api.version>1.7.5</slf4j.api.version>

<slf4j.log4j12.version>1.7.5</slf4j.log4j12.version>

<commons.collections.version>3.2.2</commons.collections.version>

<commons.httpclient>3.0.1</commons.httpclient>

<commons.beanutils.version>1.9.4</commons.beanutils.version>

<commons.configuration.version>1.10</commons.configuration.version>

<commons.lang.version>2.6</commons.lang.version>

<commons.email.version>1.5</commons.email.version>

<poi.version>4.1.2</poi.version>

<javax.servlet.api.version>3.1.0</javax.servlet.api.version>

<commons.collections4.version>4.1</commons.collections4.version>

<guava.version>24.1-jre</guava.version>

<postgresql.version>42.2.5</postgresql.version>

<hive.jdbc.version>2.1.0</hive.jdbc.version>

<commons.io.version>2.4</commons.io.version>

<oshi.core.version>3.9.1</oshi.core.version>

<clickhouse.jdbc.version>0.1.52</clickhouse.jdbc.version>

<mssql.jdbc.version>6.1.0.jre8</mssql.jdbc.version>

<presto.jdbc.version>0.238.1</presto.jdbc.version>

<spotbugs.version>3.1.12</spotbugs.version>

<checkstyle.version>3.1.2</checkstyle.version>

<zookeeper.version>3.4.14</zookeeper.version>

<curator.test>2.12.0</curator.test>

<frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>

<maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>

<maven-assembly-plugin.version>3.1.0</maven-assembly-plugin.version>

<maven-release-plugin.version>2.5.3</maven-release-plugin.version>

<maven-javadoc-plugin.version>2.10.3</maven-javadoc-plugin.version>

<maven-source-plugin.version>2.4</maven-source-plugin.version>

<maven-surefire-plugin.version>2.22.1</maven-surefire-plugin.version>

<maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>

<rpm-maven-plugion.version>2.2.0</rpm-maven-plugion.version>

<jacoco.version>0.8.4</jacoco.version>

<jcip.version>1.0</jcip.version>

<maven.deploy.skip>false</maven.deploy.skip>

<cobertura-maven-plugin.version>2.7</cobertura-maven-plugin.version>

<mockito.version>2.21.0</mockito.version>

<powermock.version>2.0.2</powermock.version>

<servlet-api.version>2.5</servlet-api.version>

<swagger.version>1.9.3</swagger.version>

<springfox.version>2.9.2</springfox.version>

<swagger-models.version>1.5.24</swagger-models.version>

<guava-retry.version>2.0.0</guava-retry.version>

<protostuff.version>1.7.2</protostuff.version>

<reflections.version>0.9.12</reflections.version>

<byte-buddy.version>1.9.16</byte-buddy.version>

<java-websocket.version>1.5.1</java-websocket.version>

<py4j.version>0.10.9</py4j.version>

<auto-service.version>1.0.1</auto-service.version>

<jacoco.skip>false</jacoco.skip>

<netty.version>4.1.53.Final</netty.version>

<maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>

2. 创建应用用户及免密

以dolphin-server为server机,以下需要在每台机器创建用户

# 创建用户需使用root登录,设置部署用户名,请自行修改,后面以dolphinscheduler为例

useradd dolphin;

# 设置用户密码,请自行修改,后面以dolphinscheduler123为例

echo "dolphin" | passwd --stdin dolphin

# 配置sudo免密

echo 'dolphin  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers

在84机器上执行

#add ip hostname

vi /etc/hosts

198.28.1.82 ht-28-tdh82

198.28.1.83 ht-28-tdh83

198.28.1.84 ht-28-tdh84

#同步dolphin-server上的/etc/hosts到所有部署机器

for ip in dolphin-82 dolphin-83;    

do

sudo scp -r /etc/hosts  $ip:/etc/          #在运行中需要输入root密码

done

#在dolphin-84上,切换到部署用户并配置ssh本机免密登录

su - dolphin;

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

chmod 600 ~/.ssh/authorized_keys

for ip in ht-28-tdh82 ht-28-tdh83;     #请将此处ds2 ds3替换为自己要部署的机器的hostname

do

ssh-copy-id  $ip   #该操作执行过程中需要手动输入dolphinscheduler用户的密码

done

3. Mysql部署

下载mysql 版本 (mysql-8.0.16-linux-glibc2.12-x86_64.tar)

wps79E0.tmp_thumb[1]

cd /opt/dolphin

tar -xvf mysql-8.0.16-linux-glibc2.12-x86_64.tar

tar -xvJf mysql-8.0.16-linux-glibc2.12-x86_64.tar.xz

# 建立软链接,便于以后版本升级

ln -s mysql-8.0.16-linux-glibc2.12-x86_64 mysql

创建临时目录、数据目录和日志目录

mkdir -p /opt/dolphin/mysql/3306/data

mkdir -p /opt/dolphin/mysql/3306/log

mkdir -p /opt/dolphin/mysql/3306/tmp

# 创建配置文件-root下

cd /etc

# 在my.cnf文件中添加对应的配置项

cp my.cnf my.cnf.bak20220125
vi my.cnf

[client] # 客户端设置,即客户端默认的连接参数

port = 3306 # 默认连接端口

socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 用于本地连接的socket套接字,mysqld守护进程生成了这个文件

[mysqld] # 服务端基本设置

# 基础设置

server-id = 1 # Mysql服务的唯一编号 每个mysql服务Id需唯一

port = 3306 # MySQL监听端口

basedir = /opt/dolphin/mysql # MySQL安装根目录

datadir = /opt/dolphin/mysql/3306/data # MySQL数据文件所在位置

tmpdir = /opt/dolphin/mysql/3306/tmp # 临时目录,比如load data infile会用到

socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 为MySQL客户端程序和服务器之间的本地通讯指定一个套接字文件

pid-file = /opt/dolphin/mysql/3306/log/mysql.pid # pid文件所在目录

#skip_name_resolve = 1 # 只能用IP地址检查客户端的登录,不用主机名

character-set-server = utf8mb4 # 数据库默认字符集,主流字符集支持一些特殊表情符号(特殊表情符占用4个字节)

transaction_isolation = READ-COMMITTED # 事务隔离级别,默认为可重复读,MySQL默认可重复读级别

collation-server = utf8mb4_general_ci # 数据库字符集对应一些排序等规则,注意要和character-set-server对应

init_connect='SET NAMES utf8mb4' # 设置client连接mysql时的字符集,防止乱码

lower_case_table_names = 1 # 是否对sql语句大小写敏感,1表示不敏感

max_connections = 400 # 最大连接数

max_connect_errors = 1000 # 最大错误连接数

explicit_defaults_for_timestamp = true # TIMESTAMP如果没有显示声明NOT NULL,允许NULL值

max_allowed_packet = 128M # SQL数据包发送的大小,如果有BLOB对象建议修改成1G

interactive_timeout = 1800 # MySQL连接闲置超过一定时间后(单位:秒)将会被强行关闭

wait_timeout = 1800 # MySQL默认的wait_timeout值为8个小时, interactive_timeout参数需要同时配置才能生效

tmp_table_size = 16M # 内部内存临时表的最大值 ,设置成128M;比如大数据量的group by ,order by时可能用到临时表;超过了这个值将写入磁盘,系统IO压力增大

安装MySQL数据库

# 进入MySQL的bin目录

cd /opt/dolphin/mysql/bin

# 初始化数据库,并指定启动mysql的用户

./mysqld --initialize --user=dolphin

生成随机密码

root@localhost: ,8v_<fe;,.cP

设置开机自启动服务

# 复制启动脚本到资源目录

cp /opt/dolphin/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld

# 增加mysqld服务控制脚本执行权限

chmod +x /etc/rc.d/init.d/mysqld

# 将mysqld服务加入到系统服务

chkconfig --add mysqld

# 检查mysqld服务是否已经生效

chkconfig --list mysqld

# 切换至mysql用户,启动mysql

su - dolphin

service mysqld start

wps79E1.tmp_thumb[1]

配置环境变量

# 修改配置文件,增加

cp .bash_profile .bash_profile_bak20220125

vi .bash_profile

export PATH=$PATH:/opt/dolphin/mysql/bin

# 立即生效

source .bash_profile

登陆,修改密码

# 登陆mysql

./mysql -uroot -p -S/opt/dolphin/mysql/3306/tmp/mysql.sock

报错:ERROR 2059 (HY000): Authentication plugin 'caching_sha2_password' cannot be loaded: /usr/lib64/mysql/plugin/caching_sha2_password.so: cannot open shared object file: No such file or directory

修改

vi /etc/my.cnf

[mysqld]

default_authentication_plugin=mysql_native_password

重启mysql

service mysqld restart

# 修改root用户密码

alter user 'root'@'localhost'IDENTIFIED BY '123456';

4. 数据库初始化

· 进入数据库,默认数据库是PostgreSQL,如选择Mysql的话,后续需要添加mysql-connector-java驱动包到DolphinScheduler的lib目录下,这里以mysql为例

mysql -uroot -p

· 进入数据库命令行窗口后,执行数据库初始化命令,设置访问账号和密码。

mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

Query OK, 1 row affected (0.00 sec)

mysql> create user 'dolphin'@'localhost' identified by 'dolphin';

mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphin'@'localhost';

Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> flush privileges;

Query OK, 0 rows affected (0.00 sec)

5. Ds部署

root下面创建目录

mkdir -p /opt/dolphin;

赋权

chown -R dolphin:dolphin /opt/dolphin

cd /opt/dolphin;

tar -zxvf apache-dolphinscheduler-2.0.2-bin.tar.gz -C /opt/dolphin;

#在ds1上,修改目录权限,使得部署用户对dolphin-backend目录有操作权限

chown -R dolphin:dolphin apache-dolphinscheduler-2.0.2-bin

· 修改 conf 目录下 application-mysql.yaml 中的下列配置

vi conf/application-mysql.yaml

spring:

datasource:

driver-class-name: com.mysql.jdbc.Driver

url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

username: dolphin

password: dolphin

hikari:

connection-test-query: select 1

minimum-idle: 5

auto-commit: true

validation-timeout: 3000

pool-name: DolphinScheduler

maximum-pool-size: 50

connection-timeout: 30000

idle-timeout: 600000

leak-detection-threshold: 0

initialization-fail-timeout: 1

还需要手动添加 [ mysql-connector-java 驱动 jar ] 包到 lib 目录下,这里下载的是mysql-connector-java-8.0.16.jar

· 修改并保存完后,执行 script 目录下的创建表及导入基础数据脚本

sh script/create-dolphinscheduler.sh

发现报错,报出来数据库不对为h2,原因是因为初始化数据脚本是按照

/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/conf/config/install_config.conf

去读取配置

可以看下create-dolphinscheduler.sh文件

里面加载了install_config.conf这个文件

#

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements. See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

# ---------------------------------------------------------

# INSTALL MACHINE

# ---------------------------------------------------------

# A comma separated list of machine hostname or IP would be installed DolphinScheduler,

# including master, worker, api, alert. If you want to deploy in pseudo-distributed

# mode, just write a pseudo-distributed hostname

# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"

ips="ht-28-tdh82,ht-28-tdh83"

# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine

# modify it if you use different ssh port

sshPort="22"

# A comma separated list of machine hostname or IP would be installed Master server, it

# must be a subset of configuration `ips`.

# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"

masters="ht-28-tdh84"

# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a

# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts

# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"

workers="ht-28-tdh82:default,ht-28-tdh83:default"

# A comma separated list of machine hostname or IP would be installed Alert server, it

# must be a subset of configuration `ips`.

# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"

alertServer="ht-28-tdh84"

# A comma separated list of machine hostname or IP would be installed API server, it

# must be a subset of configuration `ips`.

# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"

apiServers="ht-28-tdh84"

# A comma separated list of machine hostname or IP would be installed Python gateway server, it

# must be a subset of configuration `ips`.

# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"

pythonGatewayServers="ht-28-tdh84"

# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.

# Do not set this configuration same as the current path (pwd)

installPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin"

# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`

# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs

# to be created by this user

deployUser="dolphin"

# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.

dataBasedirPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/tmp"

# ---------------------------------------------------------

# DolphinScheduler ENV

# ---------------------------------------------------------

# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler

# and this configuration only support one parameter so far.

javaHome="/usr/java/jdk1.8.0_144"

# DolphinScheduler API service port, also this is your DolphinScheduler UI component's URL port, default value is 12345

apiServerPort="12345"

# ---------------------------------------------------------

# Database

# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.

# ---------------------------------------------------------

# The type for the metadata database

# Supported values: ``postgresql``, ``mysql`, `h2``.

DATABASE_TYPE=${DATABASE_TYPE:-"mysql"}

# Spring datasource url, following <HOST>:<PORT>/<database>?<parameter> format, If you using mysql, you could use jdbc

# string jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 as example

SPRING_DATASOURCE_URL=${SPRING_DATASOURCE_URL:-"jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8"}

# Spring datasource username

SPRING_DATASOURCE_USERNAME=${SPRING_DATASOURCE_USERNAME:-"dolphin"}

# Spring datasource password

SPRING_DATASOURCE_PASSWORD=${SPRING_DATASOURCE_PASSWORD:-"dolphin"}

# ---------------------------------------------------------

# Registry Server

# ---------------------------------------------------------

# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency

registryPluginName="zookeeper"

# Registry Server address.

registryServers="198.28.1.81:2181,198.28.1.82:2181,198.28.1.83:2181"

# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.

zkRoot="/dolphinscheduler"

# ---------------------------------------------------------

# Worker Task Server

# ---------------------------------------------------------

# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.

taskPluginDir="lib/plugin/task"

# resource storage type: HDFS, S3, NONE

resourceStorageType="NONE"

# resource store on HDFS/S3 path, resource file will store to this hdfs path, self configuration, please make sure the directory exists on hdfs and has read write permissions.

"/dolphinscheduler" is recommended

resourceUploadPath="/dolphinscheduler"

# if resourceStorageType is HDFS,defaultFS write namenode address,HA, you need to put core-site.xml and hdfs-site.xml in the conf directory.

# if S3,write S3 address,HA,for example :s3a://dolphinscheduler,

# Note,S3 be sure to create the root directory /dolphinscheduler

defaultFS="hdfs://198.28.1.81:8020"

# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore

s3Endpoint="http://192.168.xx.xx:9010"

s3AccessKey="xxxxxxxxxx"

s3SecretKey="xxxxxxxxxx"

# resourcemanager port, the default value is 8088 if not specified

resourceManagerHttpAddressPort="8088"

# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single node, keep this value empty

yarnHaIps="192.168.xx.xx,192.168.xx.xx"

# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single node, you only need to replace 'yarnIp1' to actual re

sourcemanager hostname

singleYarnIp="yarnIp1"

# who has permission to create directory under HDFS/S3 root path

# Note: if kerberos is enabled, please config hdfsRootUser=

hdfsRootUser="hdfs"

# kerberos config

# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore

kerberosStartUp="false"

# kdc krb5 config file path

krb5ConfPath="$installPath/conf/krb5.conf"

# keytab username,watch out the @ sign should followd by \\

keytabUserName="hdfs"

# username keytab path

keytabPath="$installPath/conf/hdfs.keytab"

# kerberos expire time, the unit is hour

kerberosExpireTime="2"

# use sudo or not

sudoEnable="true"

# worker tenant auto create

workerTenantAutoCreate="false"

2.0.2版本的ds存在一个bug就是ips的配置中不能输入server节点,因为server节点安装的时候会scp文件到ips里面的机器,导致server节点自己scp自己导致文件夹丢失

可以查看根目录install.sh

存在wps79F1.tmp_thumb[1]我们查看该文件内容如下

wps79F2.tmp_thumb[1]

可以发现scp是通过循环ips里面的机器scp的,有点问题

环境变量env/dolphinscheduler_env.sh

#

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements. See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

export HADOOP_HOME=/opt/soft/hadoop

export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop

export SPARK_HOME1=/opt/soft/spark1

export SPARK_HOME2=/opt/soft/spark2

export PYTHON_HOME=/opt/soft/python

export JAVA_HOME=/usr/java/jdk1.8.0_144

export HIVE_HOME=/opt/soft/hive

export FLINK_HOME=/opt/soft/flink

export DATAX_HOME=/opt/soft/datax

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

执行根目录install.sh安装部署

sh install.sh

会出现启动成功的状态

访问页面

http://198.28.1.84:12345/dolphinscheduler

wps79F3.tmp_thumb[1]

下次说下功能使用吧,谢谢

标签:opt,bin,2.0,16,dolphin,192.168,mysql,最新版,dolphinscheduler
来源: https://www.cnblogs.com/wang3680/p/15845378.html

本站声明: 1. iCode9 技术分享网(下文简称本站)提供的所有内容,仅供技术学习、探讨和分享;
2. 关于本站的所有留言、评论、转载及引用,纯属内容发起人的个人观点,与本站观点和立场无关;
3. 关于本站的所有言论和文字,纯属内容发起人的个人观点,与本站观点和立场无关;
4. 本站文章均是网友提供,不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属;如您发现该文章侵犯了您的权益,可联系我们第一时间进行删除;
5. 本站为非盈利性的个人网站,所有内容不会用来进行牟利,也不会利用任何形式的广告来间接获益,纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。

专注分享技术,共同学习,共同进步。侵权联系[81616952@qq.com]

Copyright (C)ICode9.com, All Rights Reserved.

ICode9版权所有