88 lines
2.5 KiB
SQL
88 lines
2.5 KiB
SQL
|
||
|
||
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
|
||
更新镜像源
|
||
清除缓存：yum clean all
|
||
生成缓存：yum makecache
|
||
yum -y makecache update
|
||
|
||
|
||
yum install net-tools - y
|
||
netstat -tlnp
|
||
|
||
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
|
||
|
||
vi /etc/profile
|
||
export JAVA_HOME=/opt/jdk8
|
||
export PATH=$JAVA_HOME/bin:$PATH
|
||
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
|
||
|
||
/opt/dolphinscheduler-3.2.2/api-server/libs/
|
||
|
||
|
||
export PYTHON_LAUNCHER=/usr/bin/python3
|
||
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
|
||
export DATAX_LAUNCHER=/opt/datax/bin/datax.py
|
||
export PATH=$PYTHON_LAUNCHER:$JAVA_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||
|
||
|
||
python /opt/datax/bin/datax.py /opt/datax/job/job.json
|
||
|
||
|
||
http://yin520.cn:12345/dolphinscheduler/ui
|
||
yinzhou
|
||
dolphinscheduler996
|
||
|
||
https://note.youdao.com/s/WuYHv1cU
|
||
这是文档,它可以做sql任务调度,还可以做数据同步,任务监控,还有你上次说的数据质量管理(方式是去跑指定sql) 这个也集成了
|
||
|
||
|
||
开源dolphinscheduler调度器搭建,流程整理
|
||
|
||
|
||
# 解压
|
||
tar -xvzf apache-dolphinscheduler-*-bin.tar.gz
|
||
|
||
|
||
# 创建用户需使用 root 登录
|
||
useradd dolphinscheduler
|
||
|
||
# 添加密码
|
||
echo "dolphinscheduler" | passwd --stdin dolphinscheduler
|
||
|
||
# 配置 sudo 免密
|
||
sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||
|
||
# 修改目录权限，使得部署用户对二进制包解压后的 apache-dolphinscheduler-*-bin 目录有操作权限
|
||
chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
|
||
chmod -R 755 apache-dolphinscheduler-*-bin
|
||
|
||
su dolphinscheduler
|
||
|
||
# 一键停止集群所有服务
|
||
bash /opt/dolphinscheduler-3.2.2/bin/stop-all.sh
|
||
|
||
# 一键开启集群所有服务
|
||
bash /opt/dolphinscheduler-3.2.2/bin/start-all.sh
|
||
|
||
|
||
bash /opt/dolphinscheduler-3.2.2/bin/dolphinscheduler-daemon.sh stop master-server
|
||
bash /opt/dolphinscheduler-3.2.2/bin/dolphinscheduler-daemon.sh start master-server
|
||
|
||
|
||
pip3 install virtualenv -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||
virtualenv -p python3 /opt/my_python_env
|
||
source /opt/job/.venv/Scripts/activate
|
||
deactivate
|
||
|
||
|
||
cd /opt/job/ && source /opt/my_python_env/bin/activate
|
||
|
||
source /opt/job/.venv/Scripts/activate
|
||
pip3 install -r /opt/job/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||
|
||
python3 -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||
|
||
pip install psycopg2==2.9.6 -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||
pip3 install wheel -i https://pypi.tuna.tsinghua.edu.cn/simple |