ceph集羣 自動化部署腳本

cat >> ceph-config.yml <<EOF
# for all nodes
all: &all
  - mon0
  - osd0
  - osd1
  - osd2

# for mon and mgr nodes
mons: &mons
  - mon0
  - osd0
  - osd1

# for osd node
osds:
  - osd0
  - osd1
  - osd2

# for cephfs node
mds:
  *mons

# for admin node
admin:
  *mons

# for client node
clients:
  - k8s1

disks:
  osd0: &osd0
    - /dev/sdb
    - /dev/sdc
  osd1:
    *osd0
  osd2:
    *osd0

global_config: |
  #public network = 192.168.42.0/24
  mgr initial modules = prometheus
  mon allow pool delete = true
  max_open_files = 131072
  osd_journal_size = 10000
  osd_pool_default_size = 3
  osd_pool_default_min_size = 1
EOF

cat >> ceph-auto-deploy.py <<EOF
#!/usr/bin/env python
# coding=utf-8

from executor import execute
import sys, os
import argparse
import json
import yaml
from whichcraft import which

cluster_dir = './myceph'
ceph_config_yml = os.getcwd() + '/ceph-config.yml'

os.environ['CEPH_DEPLOY_REPO_URL'] = 'https://mirrors.aliyun.com/ceph/rpm-mimic/el7/'
os.environ['CEPH_DEPLOY_GPG_URL'] = 'https://mirrors.aliyun.com/ceph/keys/release.asc'

def get_config():
    with open(ceph_config_yml, 'r') as f:
        config = yaml.load(f, Loader=yaml.Loader)
    return config

# 參數hosts 是all mons osds mds admin 中的一個
def get_nodes(hosts):
    config = get_config()
    return ' '.join(config[hosts])

def get_global_config():
    config = get_config()
    return config['global_config']

def get_disks():
    config = get_config()
    return config['disks']

# 安裝epel
def install_epel(hosts):
    config = get_config()
    # 先安裝epel
    for node in config[hosts]:
        #execute('ssh {node} sudo yum -y install https://mirrors.aliyun.com/centos/7/extras/x86_64/Packages/epel-release-7-11.noarch.rpm'.format(node=node), check=False)
        execute('ssh {node} sudo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo'.format(node=node), check=False)
        execute('ssh {node} sudo yum repolist'.format(node=node), check=False)
        execute('ssh {node} sudo yum yum-plugin-priorities'.format(node=node), check=False)

# 對節點進行ceph的安裝
def install_ceph(hosts):
    config = get_config()
    nodes = ' '.join(config[hosts])
    # 安裝ceph 包
    if not execute('ceph-deploy --overwrite-conf install {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)

# 對clients 節點進行ceph的安裝
def install_clients():
    config = get_config()
    nodes = ' '.join(config['clients'])
    # 安裝ceph 包
    if not execute('ceph-deploy --overwrite-conf install --cli {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)
    # 安裝 ceph-fuse
    for node in config['clients']:
        execute('ssh {node} sudo yum -y install ceph-fuse'.format(node=node), check=False)

# 將節點配置成管理節點
def admin_ceph():
    nodes = get_nodes('admin')
    if not execute('ceph-deploy --overwrite-conf admin {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)

# 添加mgr
def add_mgr():
    nodes = get_nodes('mons')
    if not execute('ceph-deploy --overwrite-conf mgr create {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)

# 添加osd
def add_osd():
    disks = get_disks()
    for node, devs in disks.items():
        for dev in devs:
        if not execute('ceph-deploy --overwrite-conf osd create {node} --data {dev}'.format(node=node, dev=dev), check=False):
                sys.exit(1)

# 創建pool
# app：有三種類型 cephfs, rbd, rgw
def add_pool(poolName, app, pgNum):
    config = get_config()
    node = config['admin'][0]
    if which('ceph'):
        execute('ceph osd pool create {poolName} {pgNum}'.format(poolName=poolName, pgNum=pgNum), check=False, sudo=True)
    execute('ceph osd pool application enable {poolName} {app}'.format(poolName=poolName, app=app), check=False, sudo=True)
    execute('ceph auth get-or-create client.{poolName} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool={poolName}" -o ceph.client.{poolName}.keyring'.format(poolName=poolName), check=False, sudo=True)
    else:
        execute('ssh {node} sudo ceph osd pool create {poolName} {pgNum}'.format(node=node, poolName=poolName, pgNum=pgNum), check=False)
    execute('ssh {node} sudo ceph osd pool application enable {poolName} {app}'.format(node=node, poolName=poolName, app=app), check=False)
        if app == 'rbd':
        execute('ssh {node} sudo ceph auth get-or-create client.{poolName} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool={poolName}" -o ceph.client.{poolName}.keyring'.format(node=node, poolName=poolName), check=False)

# 生成cephfs 文件系統
def gen_cephfs_filesystem():
    add_pool('cephfs_data', 'cephfs', '64')
    add_pool('cephfs_metadata', 'cephfs', '64')
    config = get_config()
    node = config['admin'][0]
    if which('ceph'):
        execute('ceph fs new data cephfs_metadata cephfs_data', check=False, sudo=True)
    else:
        execute('ssh {node} sudo ceph fs new data cephfs_metadata cephfs_data'.format(node=node), check=False)

# 添加mds
def add_mds():
    nodes = get_nodes('mds')
    if not execute('ceph-deploy --overwrite-conf mds create {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)

# 刪除集羣
def purge_cluster():
    nodes = get_nodes('all')
    if not execute('ceph-deploy purge {nodes}'.format(nodes=nodes), check=False):
        sys.exit(1)

# 創建集羣
def create_cluster():
    # 對所有的節點安裝epel 軟件包
    print('=='*10 + 'Install epel' + '=='*10)
    install_epel('all')

    print('=='*10 + 'Init ceph cluster....' + '=='*10)
    mons_node = get_nodes('mons')
    if not os.path.exists(cluster_dir):
        os.makedirs(cluster_dir)
    os.chdir(cluster_dir)
    global_config = get_global_config()
    if execute('ceph-deploy --overwrite-conf new {mons_node}'.format(mons_node=mons_node), check=False):
        with open('ceph.conf', 'a') as f:
            f.write(global_config)

    # 對所有的節點安裝ceph 軟件包
    print('=='*10 + 'Install ceph package....' + '=='*10)
    install_ceph('all')

    # 初始化monitor
    print('=='*10 + 'Init ceph monitor....' + '=='*10)
    execute('ceph-deploy --overwrite-conf mon create-initial', check=False)

    # 配置管理節點
    print('=='*10 + 'Config admin....' + '=='*10)
    admin_ceph()

    # 部署mgr服務
    print('=='*10 + 'Deploy mgr....' + '=='*10)
    add_mgr()

    # 部署osd服務
    print('=='*10 + 'Deploy osd....' + '=='*10)
    add_osd()

    # 創建k8s pool
    print('=='*10 + 'Create pool....' + '=='*10)
    add_pool('kube','rbd','128')

    # 部署mds服務
    print('=='*10 + 'Deploy mds....' + '=='*10)
    add_mds()

    # 生成cephfs 文件系統
    print('=='*10 + 'Create cephfs filesystem....' + '=='*10)
    gen_cephfs_filesystem()

def parse_arguments(args=None):
    parser = argparse.ArgumentParser(description='Used to deploy ceph cluster')
    parser.add_argument('-d', '--deploy', dest='deploy', action="store_true", help='Deploy ceph cluster')
    parser.add_argument('-p', '--purge', dest='purge', action="store_true", help='Purge ceph cluster')
    parser.add_argument('-c', '--client', dest='client', action="store_true", help='Install ceph package for client node')
    try:
        args = parser.parse_args(args=args)
    except IOError as msg:
        parser.error(str(msg))
    if len(sys.argv) == 1:
        parser.print_help(sys.stderr)
        sys.exit(1)
    return args

def main(arg):
    args = parse_arguments(arg)
    if args.deploy:
    create_cluster()
    elif args.purge:
    purge_cluster()
    elif args.client:
        install_clients()

if __name__ == '__main__':
    main(sys.argv[1:])
EOF