Centos7部署AirFlow
1.準備環境
1.1.安裝Python3
yum install epel-release -y
yum install https://centos7.iuscommunity.org/ius-release.rpm -y
yum install python36u -y
ln -s /bin/python3.6 /bin/python3
yum install python36u-pip -y
ln -s /bin/pip3.6 /bin/pip3
1.2.換pip國內源
mkdir ~/.pip
vim ~/.pip/pip.conf
[global]
index-url = https://pypi.tuna.tsinghua.edu.cn/simple
1.3.配置虛擬環境
mkdir -p /usr/local/thirdparty/
cd /usr/local/thirdparty/
python3 -m venv airflow_env
source /usr/local/thirdparty/airflow_env/bin/activate
1.4.安裝操作系統依賴
yum install -y gcc gcc-c++ python-devel python36u-devel
2.安裝AirFlow
- 安裝
export SLUGIFY_USES_TEXT_UNIDECODE=yes
pip install apache-airflow[all]
- 啓動測試
mkdir /usr/local/airflow
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb
# 開啓兩個終端,第一個執行
airflow webserver -p 8080
# 第二個執行
airflow scheduler
瀏覽器訪問http://ip:8080查看服務是否啓動成功
3.Systemd部署
3.1.創建MySQL庫和用戶
mysql> CREATE DATABASE airflow;
mysql> GRANT all privileges on airflow.* TO 'airflow'@'localhost' IDENTIFIED BY 'airflow';
mysql> FLUSH PRIVILEGES;
3.2.創建airflow系統用戶
useradd airflow
3.3.修改配置
vim /usr/local/airflow/airflow.cfg
# [core]部分修改
# 替換成mysql配置
sql_alchemy_conn = mysql://airflow:[email protected]:3306/airflow
# 移除示例Dags
load_examples = False
# [webserver] 部分修改
# 開啓認證
authenticate = True
# 添加認證後端(該項配置需要新增)
auth_backend = airflow.contrib.auth.backends.password_auth
# 按用戶過濾
filter_by_owner = True
3.4.修改/usr/local/airflow屬主
chown -R airflow:airflow /usr/local/airflow
3.5.配置systemd相關服務文件
- 獲取文件
GitHub獲取airflow源碼,啓動腳本在scripts/systemd
目錄下
獲取systemd
目錄到/root
下
cd /root/systemd
- 配置
airflow
文件
vim airflow
替換配置
AIRFLOW_CONFIG=/usr/local/airflow/airflow.cfg
AIRFLOW_HOME=/usr/local/airflow
- 配置
airflow-webserver.service
文件
vim airflow-webserver.service
替換啓動腳本成如下
ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow webserver --pid /run/airflow/webserver.pid'
- 配置
airflow-scheduler.service
文件
vim airflow-scheduler.service
替換啓動腳本成如下
ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow scheduler'
- 配置
airflow-flower.service
文件
vim airflow-flower.service
替換啓動腳本成如下
ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow flower'
- 配置
airflow-worker.service
文件
vim airflow-worker.service
替換啓動腳本成如下
ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow worker'
- 移動相關文件到對應目錄
mkdir /run/airflow
chown -R airflow:airflow /run/airflow
cp airflow /etc/sysconfig/
cp airflow.conf /etc/tmpfiles.d/
cp airflow-*.service /usr/lib/systemd/system/
3.6.初始化db
source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb
3.7.創建用戶
source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
vim user_manger.py
複製如下內容
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
from airflow import models, settings
from airflow.contrib.auth.backends.password_auth import PasswordUser
class UserManager(object):
def __init__(self):
self.user = models.User()
self.passwd_user = PasswordUser(self.user)
self.session = settings.Session()
def add_user(self, username, passwd, email=None, is_super=False):
self.passwd_user.username = username
self.passwd_user.password = passwd
if email:
self.passwd_user.email = email
if is_super is True:
self.passwd_user.superuser = True
self.session.add(self.passwd_user)
self.session.commit()
def change_passwd(self, username, new_passwd=None, email=None, superuser=None):
if not any([new_passwd, email, superuser]):
return
info = {}
if new_passwd:
self.passwd_user.password = new_passwd
info['password'] = self.passwd_user.password
if email:
info['email'] = email
if superuser is True:
info['superuser'] = True
if superuser is False:
info['superuser'] = False
try:
self.session.query(PasswordUser).filter(PasswordUser.username == username).update(info)
except Exception as e:
pass
else:
self.session.commit()
def __del__(self):
self.session.close()
if __name__ == '__main__':
manager = UserManager()
manager.add_user('admin', 'admin', 'admin@localhost', True)
# manager.change_passwd('admin', 'admin', 'admin@localhost', True)
python user_manger.py
3.8.啓動服務
systemctl start airflow-webserver.service
systemctl start airflow-scheduler.service
3.9.設置開機啓動
systemctl enable airflow-webserver.service
systemctl enable airflow-scheduler.service
4.訪問web
瀏覽器訪問http://ip:8080
admin/admin