ceph角色分配
172.31.68.241 | admin-node/deph-deploy/mon/mgr/mds/rgw |
172.31.68.242 | osd.0/mon |
172.31.68.243 | osd.1/mon |
配置ssh無密碼登錄
admin-node要可以無密碼ssh登錄osd機器,如果是普通用戶,則要分配sudo權限,如下:
useradd -d /home/cephadmin -m cephadmin
echo "cephadmin ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephadmin
chmod 0440 /etc/sudoers.d/cephadmin
快速安裝
ceph-deploy new ceph1
ceph-deploy install ceph1 ceph2 ceph3
ceph-deploy --overwrite-conf mon create-initial
ceph-deploy admin ceph1 ceph2 ceph3
ceph-deploy mgr create ceph1
ceph-deploy osd create --data /dev/vdb1 ceph2
ceph-deploy osd create --data /dev/vdb1 ceph3
ceph health
ceph -s
擴展
ceph-deploy mds create ceph1
ceph-deploy mon add ceph2
ceph-deploy mon add ceph3
ceph quorum_status --format json-pretty
安裝rgw
ceph-deploy rgw create ceph1
調整配置
[client.rgw.ceph1]
rgw_frontends = "civetweb port=8080"
ceph-deploy --overwrite-conf admin ceph1 ceph2 ceph3 ceph4
systemctl restart [email protected]
curl http://172.31.68.241:8080 -I
模擬client
apt-get install ceph
ceph-deploy admin ceph4
對象存儲
echo 'hello ceph oject storage' > testfile.txt
創建pool
ceph osd pool create mytest 8
上傳文件
rados put test-object-1 testfile.txt --pool=mytest
rados -p mytest ls
獲取文件
rados get test-object-1 testfile.txt.1 --pool=mytest
查看映射位置
ceph osd map mytest test-object-1
刪除文件
rados rm test-object-1 --pool=mytest
刪除pool
ceph osd pool rm mytest mytest --yes-i-really-really-mean-it
塊存儲
admin上執行:ceph osd pool create rdb 8
admin上執行:rbd pool init rdb
admin上執行:
rbd create foo --size 512 --image-feature layering -p rdb
rbd map foo --name client.admin -p rdb
cephfs
admin:ceph osd pool create cephfs_data 4
admin:ceph osd pool create cephfs_metadata 4
admin:ceph osd lspools
admin:ceph fs new cephfs cephfs_metadata cephfs_data
admin.secret內容爲ceph.client.admin.keyring內容的一部分
AQDhRX1baLeFFxAAskNapEuyipJ7SqS7Q1mh/Q==
內核級別掛載方法:
mkdir /mnt/mycephfs
mount -t ceph 172.31.68.241:6789,172.31.68.242:6789,172.31.68.243:6789:/ /mnt/mycephfs -o name=admin,secretfile=admin.secret
實驗
cd /mnt/mycephfs
echo 'hello ceph CephFS' > hello.txt
cd ~
卸載
umount -lf /mnt/mycephfs
rm -rf /mnt/mycephfs
用戶級別掛載:
mkdir /mnt/mycephfs
ceph-fuse -m 172.31.68.241:6789 /mnt/mycephfs
S3 存儲
ceph osd pool create .rgw 8 8
ceph osd pool create .rgw.root 8 8
ceph osd pool create .rgw.control 8 8
ceph osd pool create .rgw.gc 8 8
ceph osd pool create .rgw.buckets 8 8
ceph osd pool create .rgw.buckets.index 8 8
ceph osd pool create .rgw.buckets.extra 8 8
ceph osd pool create .log 8 8
ceph osd pool create .intent-log 8 8
ceph osd pool create .usage 8 8
ceph osd pool create .users 8 8
ceph osd pool create .users.email 8 8
ceph osd pool create .users.swift 8 8
ceph osd pool create .users.uid 8 8
vm外掛磁盤
cd /opt/vm/data_image
qemu-img create -f qcow2 ubuntu16.04-2-data.img 2G
qemu-img create -f qcow2 ubuntu16.04-3-data.img 2G
virsh attach-disk [--domin] $DOMIN [--source] $SOURCEFILE [--target] $TARGET --subdriver qcow2 --config --live
virsh attach-disk Ubuntu16.04-2 /opt/vm/data_image/ubuntu16.04-2-data.img vdb --subdriver qcow2
virsh attach-disk Ubuntu16.04-3 /opt/vm/data_image/ubuntu16.04-3-data.img vdb --subdriver qcow2
清除安裝包
ceph-deploy purge ceph1 ceph2 ceph3
清除配置信息
ceph-deploy purgedata ceph1 ceph2 ceph3
ceph-deploy forgetkeys
每個節點刪除殘留的配置文件
rm -rf /var/lib/ceph/osd/*
rm -rf /var/lib/ceph/mon/*
rm -rf /var/lib/ceph/mds/*
rm -rf /var/lib/ceph/bootstrap-mds/*
rm -rf /var/lib/ceph/bootstrap-osd/*
rm -rf /var/lib/ceph/bootstrap-mon/*
rm -rf /var/lib/ceph/tmp/*
rm -rf /etc/ceph/*
rm -rf /var/run/ceph/*
清除lvm配置
vgscan
vgdisplay -v
lvremove
vgremove
pvremove