作者：張華 發表於：2020-02-27
版權聲明：可以任意轉載，轉載時請務必以超鏈接形式標明文章原始出處和作者信息及本版權聲明
juju deploy -n 3 rabbitmq-server
rabbitmq-server/0* -> 10.5.0.12 -> juju-3f992e-ha-0
rabbitmq-server/1 -> 10.5.0.7 -> juju-3f992e-ha-1
rabbitmq-server/2 -> 10.5.0.10 -> juju-3f992e-ha-2
#https://www.centos.bz/2019/06/linux%E4%B8%8B%E6%90%AD%E5%BB%BAhaproxypacemakercorosync%E9%9B%86%E7%BE%A4/
#run the following commands in 3 units
sudo apt install pacemaker corosync heartbeat crmsh cluster-glue resource-agents pcs -y
HOSTNAME=$(hostname)
cat << EOF | sudo tee -a /etc/rabbitmq/rabbitmq-env.conf
RABBITMQ_NODENAME=rabbit@$HOSTNAME
EOF
cat << EOF | sudo tee /etc/corosync/corosync.conf
totem {
version: 2
secauth: off
cluster_name: quqicluster
transport: udpu
token: 55000
rrp_problem_count_timeout: 110000
rrp_problem_count_threshold: 30
token_retransmits_before_loss_const: 30
}
nodelist {
node {
ring0_addr: juju-3f992e-ha-0
nodeid: 1
}
node {
ring0_addr: juju-3f992e-ha-1
nodeid: 2
}
node {
ring0_addr: juju-3f992e-ha-2
nodeid: 3
}
}
quorum {
provider: corosync_votequorum
}
logging {
fileline: off
to_stderr: no
to_logfile: yes
logfile: /var/log/corosync.log
to_syslog: yes
syslog_facility: daemon
debug: on
timestamp: on
logger_subsys {
subsys: QUORUM
debug: off
}
}
EOF
#sudo systemctl restart corosync
sudo systemctl restart pcsd
sudo systemctl enable pcsd
#configure node authentication
sudo passwd hacluster #change the password to 'password' for the user hacluster
sudo pcs cluster auth -u hacluster juju-3f992e-ha-0 juju-3f992e-ha-1 juju-3f992e-ha-2
#create the cluster
sudo pcs cluster setup --force --name quqicluster juju-3f992e-ha-0 juju-3f992e-ha-1 juju-3f992e-ha-2
sudo pcs cluster start --all
sudo pcs cluster enable --all
sudo pcs status
sudo pcs status corosync
sudo pcs cluster status
sudo corosync-cfgtool -s
sudo corosync-cmapctl | grep members
sudo pcs property set stonith-enabled=false #Resource start-up disabled since no STONITH resources have been defined
#sudo pcs property set no-quorum-policy=ignore
sudo crm_verify -L -V
sudo pcs resource delete rabbitmq-clone --force
sudo pcs resource create rabbitmq-clone ocf:heartbeat:rabbitmq-cluster --clone
sudo pcs resource update rabbitmq-clone op monitor interval=30 timeout=120
sudo pcs resource update rabbitmq-clone op start interval=0 timeout=100
sudo pcs resource update rabbitmq-clone op stop interval=0 timeout=90
sudo pcs resource update rabbitmq-clone meta ordered=true interleave=true
sudo pcs resource show rabbitmq-clone
sudo pcs resource
sudo pcs config show
sudo pcs status
sudo rabbitmqctl cluster_status
sudo crm resource restart rabbitmq-clone
sudo crm resource clean rabbitmq-clone
#use crm instead of pcs - https://www.sebastien-han.fr/blog/2012/09/16/pacemaker-load-balancing-with-clone/
wget https://raw.githubusercontent.com/ClusterLabs/crmsh/master/contrib/bash_completion.sh
sudo cp bash_completion.sh /etc/bash_completion.d/crmsh
source /etc/bash_completion.d/crmsh
sudo crm configure show
sudo crm ra list ocf |grep rabbit
sudo crm ra meta rabbitmq-cluster
sudo crm configure property stonith-enabled=false
sudo crm configure primitive rabbitmq ocf:heartbeat:rabbitmq-cluster \
op monitor interval=30 timeout=120 \
op start interval=0 timeout=100 \
op stop interval=0 timeout=90
sudo crm configure clone rabbitmq-clone rabbitmq \
meta ordered=true interleave=true
ubuntu@juju-3f992e-ha-2:~$ sudo pcs resource show rabbitmq-clone
Clone: rabbitmq-clone
Meta Attrs: ordered=true interleave=true
Resource: rabbitmq (class=ocf provider=heartbeat type=rabbitmq-cluster)
Operations: monitor interval=30 timeout=120 (rabbitmq-monitor-30)
start interval=0 timeout=100 (rabbitmq-start-0)
stop interval=0 timeout=90 (rabbitmq-stop-0)
Clone: rabbitmq-clone
Meta Attrs: interleave=true ordered=true
Resource: rabbitmq (class=ocf provider=heartbeat type=rabbitmq-cluster)
Attributes: set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"
Operations: start interval=0s timeout=100 (rabbitmq-start-interval-0s)
stop interval=0s timeout=90 (rabbitmq-stop-interval-0s)
monitor interval=30 timeout=120 (rabbitmq-monitor-interval-30)
haproxy vip
cat << EOF | sudo tee -a /etc/haproxy/haproxy.cfg
listen httpd
bind 10.5.0.122:5672
balance source
option tcpka
option httpchk
option tcplog
server juju-3f992e-ha-0 10.5.0.12:5672 check inter 2000 rise 2 fall 5
server juju-3f992e-ha-1 10.5.0.7:5672 check inter 2000 rise 2 fall 5
server juju-3f992e-ha-2 10.5.0.10:5672 check inter 2000 rise 2 fall 5
EOF
sudo pcs resource create vip ocf:heartbeat:IPaddr2 ip=10.5.0.122 cidr_netmask=24 nic=ens33 op monitor interval=3s
sudo pcs resource create lb-haproxy systemd:haproxy --clone
sudo pcs constraint colocation add lb-haproxy-clone with vip #make haproxy and vip in the same nodes
sudo pcs constraint order start vip then lb-haproxy-clone kind=Optional #make first start vip before starting haproxy
sudo pcs resource
ubuntu@juju-3f992e-ha-2:~$ sudo pcs cluster setup --force --name quqicluster juju-3f992e-ha-0 juju-3f992e-ha-1 juju-3f992e-ha-2
Destroying cluster on nodes: juju-3f992e-ha-0, juju-3f992e-ha-1, juju-3f992e-ha-2...
juju-3f992e-ha-2: Stopping Cluster (pacemaker)...
juju-3f992e-ha-0: Stopping Cluster (pacemaker)...
juju-3f992e-ha-1: Stopping Cluster (pacemaker)...
juju-3f992e-ha-0: Successfully destroyed cluster
juju-3f992e-ha-2: Successfully destroyed cluster
juju-3f992e-ha-1: Successfully destroyed cluster
Sending 'pacemaker_remote authkey' to 'juju-3f992e-ha-0', 'juju-3f992e-ha-1', 'juju-3f992e-ha-2'
juju-3f992e-ha-2: successful distribution of the file 'pacemaker_remote authkey'
juju-3f992e-ha-1: successful distribution of the file 'pacemaker_remote authkey'
juju-3f992e-ha-0: successful distribution of the file 'pacemaker_remote authkey'
Sending cluster config files to the nodes...
juju-3f992e-ha-0: Succeeded
juju-3f992e-ha-1: Succeeded
juju-3f992e-ha-2: Succeeded
Synchronizing pcsd certificates on nodes juju-3f992e-ha-0, juju-3f992e-ha-1, juju-3f992e-ha-2...
juju-3f992e-ha-2: Success
juju-3f992e-ha-1: Success
juju-3f992e-ha-0: Success
Restarting pcsd on the nodes in order to reload the certificates...
juju-3f992e-ha-2: Success
juju-3f992e-ha-0: Success
juju-3f992e-ha-1: Success
Invalid IPC credentials
#because the cluster is managed by pcs rather than corosync so this error will happen, and the following workaround will not work as well
cat << EOF | sudo tee /etc/corosync/uidgid.d/hacluster
uidgid {
uid: hacluster
gid: haclient
}
EOF