88 lines
2.5 KiB
MySQL
88 lines
2.5 KiB
MySQL
|
|
|||
|
|
|
|||
|
|
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
|
|||
|
|
更新镜像源
|
|||
|
|
清除缓存：yum clean all
|
|||
|
|
生成缓存：yum makecache
|
|||
|
|
yum -y makecache update
|
|||
|
|
|
|||
|
|
|
|||
|
|
yum install net-tools - y
|
|||
|
|
netstat -tlnp
|
|||
|
|
|
|||
|
|
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
|
|||
|
|
|
|||
|
|
vi /etc/profile
|
|||
|
|
export JAVA_HOME=/opt/jdk8
|
|||
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|||
|
|
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
|
|||
|
|
|
|||
|
|
/opt/dolphinscheduler-3.2.2/api-server/libs/
|
|||
|
|
|
|||
|
|
|
|||
|
|
export PYTHON_LAUNCHER=/usr/bin/python3
|
|||
|
|
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
|
|||
|
|
export DATAX_LAUNCHER=/opt/datax/bin/datax.py
|
|||
|
|
export PATH=$PYTHON_LAUNCHER:$JAVA_HOME/bin:$DATAX_LAUNCHER:$PATH
|
|||
|
|
|
|||
|
|
|
|||
|
|
python /opt/datax/bin/datax.py /opt/datax/job/job.json
|
|||
|
|
|
|||
|
|
|
|||
|
|
http://yin520.cn:12345/dolphinscheduler/ui
|
|||
|
|
yinzhou
|
|||
|
|
dolphinscheduler996
|
|||
|
|
|
|||
|
|
https://note.youdao.com/s/WuYHv1cU
|
|||
|
|
这是文档,它可以做sql任务调度,还可以做数据同步,任务监控,还有你上次说的数据质量管理(方式是去跑指定sql) 这个也集成了
|
|||
|
|
|
|||
|
|
|
|||
|
|
开源dolphinscheduler调度器搭建,流程整理
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 解压
|
|||
|
|
tar -xvzf apache-dolphinscheduler-*-bin.tar.gz
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 创建用户需使用 root 登录
|
|||
|
|
useradd dolphinscheduler
|
|||
|
|
|
|||
|
|
# 添加密码
|
|||
|
|
echo "dolphinscheduler" | passwd --stdin dolphinscheduler
|
|||
|
|
|
|||
|
|
# 配置 sudo 免密
|
|||
|
|
sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
|||
|
|
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
|||
|
|
|
|||
|
|
# 修改目录权限，使得部署用户对二进制包解压后的 apache-dolphinscheduler-*-bin 目录有操作权限
|
|||
|
|
chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
|
|||
|
|
chmod -R 755 apache-dolphinscheduler-*-bin
|
|||
|
|
|
|||
|
|
su dolphinscheduler
|
|||
|
|
|
|||
|
|
# 一键停止集群所有服务
|
|||
|
|
bash /opt/dolphinscheduler-3.2.2/bin/stop-all.sh
|
|||
|
|
|
|||
|
|
# 一键开启集群所有服务
|
|||
|
|
bash /opt/dolphinscheduler-3.2.2/bin/start-all.sh
|
|||
|
|
|
|||
|
|
|
|||
|
|
bash /opt/dolphinscheduler-3.2.2/bin/dolphinscheduler-daemon.sh stop master-server
|
|||
|
|
bash /opt/dolphinscheduler-3.2.2/bin/dolphinscheduler-daemon.sh start master-server
|
|||
|
|
|
|||
|
|
|
|||
|
|
pip3 install virtualenv -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|||
|
|
virtualenv -p python3 /opt/my_python_env
|
|||
|
|
source /opt/job/.venv/Scripts/activate
|
|||
|
|
deactivate
|
|||
|
|
|
|||
|
|
|
|||
|
|
cd /opt/job/ && source /opt/my_python_env/bin/activate
|
|||
|
|
|
|||
|
|
source /opt/job/.venv/Scripts/activate
|
|||
|
|
pip3 install -r /opt/job/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|||
|
|
|
|||
|
|
python3 -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|||
|
|
|
|||
|
|
pip install psycopg2==2.9.6 -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|||
|
|
pip3 install wheel -i https://pypi.tuna.tsinghua.edu.cn/simple
|