redhat6.5 Tomcat+Nginx+Memcacheed集羣部署

主機環境 redhat6.5 64位

實驗環境 服務端1 ip172.25.29.1   nginx

服務端2 ip 172.25.29.2    tomcat+memcached

服務端3 ip 172.25.29.3    tomcat+memcaceed

安裝包  jdk-7u79-linux-x64.tar.gz

apache-tomcat-7.0.37.tar.gz

nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz

asm-3.2.jar

kryo-1.04.jar

kryo-serializers-0.10.jar

memcached-session-manager-1.6.3.jar

memcached-session-manager-tc7-1.6.3.jar

minlog-1.2.jar

msm-kryo-serializer-1.6.3.jar

reflectasm-1.01.jar

spymemcached-2.7.3.jar

防火牆狀態關閉

 1.jdk安裝、環境配置及測試服務端2

1.解壓、作軟鏈接

[root@server2mnt]# tar zxf jdk-7u79-linux-x64.tar.gz -C /usr/local/  #解壓jdk

[root@server2mnt]# cd /usr/local/    #切換到解壓目錄

[root@server2local]# ls

bin  etc games  include  jdk1.7.0_79 lib  lib64  libexec sbin  share  src

[root@server2local]# ln -s jdk1.7.0_79/ java      #作軟鏈接

[root@server2local]# ll

total44

drwxr-xr-x.2 root root 4096 Jun 28  2011 bin

drwxr-xr-x.2 root root 4096 Jun 28  2011 etc

drwxr-xr-x.2 root root 4096 Jun 28  2011 games

drwxr-xr-x.2 root root 4096 Jun 28  2011 include

lrwxrwxrwx.1 root root   12 Sep 24 10:50 java ->jdk1.7.0_79/    #查看

drwxr-xr-x.8 uucp  143 4096 Apr 11  2015 jdk1.7.0_79

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib64

drwxr-xr-x.2 root root 4096 Jun 28  2011 libexec

drwxr-xr-x.2 root root 4096 Jun 28  2011 sbin

drwxr-xr-x.5 root root 4096 Aug  8 21:38 share

drwxr-xr-x.2 root root 4096 Jun 28  2011 src

2.將jdk添加到環境變量

[root@server2local]# vim /etc/profile   

 export JAVA_HOME=/usr/local/java

 exportCLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib

 export PATH=$PATH:$JAVA_HOME/bin

[root@server2local]# source /etc/profile

3.測試

[root@server2local]# echo $JAVA_HOME    #測試添加環境變量是否成功

/usr/local/java

[root@server2local]# echo $CLASSPATH

.:/usr/local/java/lib:/usr/local/java/jre/lib

[root@server2local]# echo $JAVA_HOME     

/usr/local/java

[root@server2local]# cd java       #測試java的環境是否配置成功

[root@server2java]# vim test.java    #寫個簡單的測試頁

 public class test{

 public static void main(String[] args)

          {

                   System.out.println("Helloworld!");

          }

   }

[root@server2java]# javac test.java    #鏈接生成test.class文件

[root@server2mnt]# java test    #執行

Helloworld!

 2.tomcat的安裝服務端2

1.解壓、作軟鏈接

[root@server2mnt]# tar zxf apache-tomcat-7.0.8.tar.gz -C /usr/local/  #解壓

[root@server2mnt]# cd /usr/local/

[root@server2local]# ls

apache-tomcat-7.0.8  etc   include  jdk1.7.0_79  lib64   sbin   src

bin                  games  java    lib          libexec  share

[root@server2local]# ln -s apache-tomcat-7.0.8/ tomcat   #作軟鏈接

[root@server2local]# ll

total48

drwxr-xr-x.9 root root 4096 Sep 24 11:11 apache-tomcat-7.0.8

drwxr-xr-x.2 root root 4096 Jun 28  2011 bin

drwxr-xr-x.2 root root 4096 Jun 28  2011 etc

drwxr-xr-x.2 root root 4096 Jun 28  2011 games

drwxr-xr-x.2 root root 4096 Jun 28  2011 include

lrwxrwxrwx.1 root root   12 Sep 24 10:50 java ->jdk1.7.0_79/

drwxr-xr-x.8 uucp  143 4096 Sep 24 11:02 jdk1.7.0_79

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib64

drwxr-xr-x.2 root root 4096 Jun 28  2011 libexec

drwxr-xr-x.2 root root 4096 Jun 28  2011 sbin

drwxr-xr-x.5 root root 4096 Aug  8 21:38 share

drwxr-xr-x.2 root root 4096 Jun 28  2011 src

lrwxrwxrwx.1 root root   20 Sep 24 11:12 tomcat-> apache-tomcat-7.0.8/     #查看

[root@server2local]# cd tomcat/bin

[root@server2bin]# ./startup.sh     #開啓tomcat

 2.測試

[root@server2bin]#cd ..

[root@server2tomcat]# vim webapps/ROOT/test.jsp    #寫測試儀頁

  1 server2-The Time is <%=newjava.util.Date()%>

#測試 172.25.29.2:8080

 

172.25.29.2:8080/test.jsp

在服務端3上進行同樣的配置也可以用scp把服務端2上java和tomcat目錄傳過去如下

[root@server2local]# scp -r java/ tomcat/ 172.25.29.3:/usr/local/

在進行系統環境里加上java的配置啓動tomcat即可

 

3.Nginx添加sticky  (服務端1)

1.nginx負載均衡已經配置好了

在前面的博客裏已經寫過nginx源碼安裝這裏就不再重複了也可以參考前面的博客

[root@server1~]# cd /usr/local/lnmp/nginx/conf  

[root@server1conf]# vim nginx.conf

upstream wen {

               server 172.25.29.2:8080;    #輪詢機制

                server 172.25.29.3:8080;

       }

        location / {

           root   html;

            index  index.html index.jspindex.php index.htm;   #默認發佈目錄

        }

 

       location ~ \.jsp$ {

           proxy_pass   http://wen;

       }

[root@server1conf]# nginx -t    #檢測

nginx:the configuration file /usr/local/lnmp/nginx/conf/nginx.conf syntax is ok

nginx:configuration file /usr/local/lnmp/nginx/conf/nginx.conf test is successful

[root@server1conf]# nginx -s reload    #刷新

測試 172.25.29.1

 

172.25.29.1/test.jsp

刷新之後

也就是server2 和server3相互交替

 

2.改變負載均衡機制爲sticky

由於nginx是輪詢機制如果在訪問頁面如填寫信息時突然卡住刷新之後就會跳到另一個server上就得重新開始填寫。但是Nginx裏有很多算法其中的ip_hash也可以防止這些問題。使用ip_hash有一個問題是當客戶端和服務器之間使用cdn內容分發系統高速緩存時客戶端的訪問到達cdn由cdn訪問服務器識別的ip是cdn的ip那麼將集中於訪問後臺的一臺服務器(x相當於DDOS***)會加速服務器的損壞。那麼爲了防止這些問題就有用下面的算法sticky不是nginx裏自帶的nginx-sticky-module爲 nginx 的第三方模塊,使 nginx 支持 sticky 模式,所以需要將包加入配置、重新編譯、安裝nginx

1.重新源碼安裝nginx添加一個模塊

[root@server1local]# nginx -s stop    #關閉nginx

[root@server1mnt]# tar zxf nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz #解壓

[root@server1mnt]# ls

nginx-1.8.1.tar.gz

nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d

nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz

 [[email protected]]# make clean    #清除上一次的緩存文件

rm-rf Makefile objs

[[email protected]]# ./configure --prefix=/usr/local/lnmp/nginx   #重新配置--with-http_ssl_module--with-http_stub_status_module--add-module=/mnt/nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d

[[email protected]]# make     #編譯、鏈接

[[email protected]]# make install   #安裝

[[email protected]]#cd /usr/local/lnmp/nginx/conf

[root@server1conf]# vim nginx.conf

        upstream westos{

                 sticky;     #使用sticky

                 server 172.25.29.2:8080;

                 server 172.25.29.3:8080;

[root@server1local]# nginx -t   #檢測nginx文件裏是否有錯誤

nginx:the configuration file /usr/local/lnmp/nginx/conf/nginx.conf syntax is ok

nginx:configuration file /usr/local/lnmp/nginx/conf/nginx.conf test is successful

[root@server1local]# nginx   #啓動

2.測試

測試 172.25.29.1/test.jsp

刷新之後結果不變還是

 

4交叉存儲避免單點故障添加memcached服務端2

配置好之後如果一臺服務器宕機了那麼這臺服務器正在運行的業務將直接結束正在存儲的數據丟失。爲了防止這些問題Tomcat1將session存儲到Tomcat2的memcached中當Tomcat2的memcached不可用時Tomcat1將session存儲到自己的memcached上Tomcat2則正好相反。使用這種配置就避免了單點故障。

1.安裝、開啓memcached、寫測試頁

[root@server2ROOT]# yum install -y memcached      #安裝memcached

[root@server2ROOT]# /etc/init.d/memcached start      #開啓memcached

Startingmemcached:                                        [  OK  ]

[root@server2tomcat]# bin/shutdown.sh   #關閉Tomcat

[root@server2ROOT]# vim test.jsp    #寫一個jsp的測試頁面

 <%@ page contentType="text/html;charset=GBK" %>

 <%@ page import="java.util.*"%>

 <html><head><title>Cluster AppTest</title></head>

 <body>

  Server Info:

  <%

  out.println(request.getLocalAddr() + ": " + request.getLocalPort()+"<br>");%    >

  <%

  out.println("<br> ID " + session.getId()+"<br>");

  String dataName =request.getParameter("dataName");

  if (dataName != null &&dataName.length() > 0) {

  String dataValue =request.getParameter("dataValue");

  session.setAttribute(dataName, dataValue);

  }

  out.print("<b>Sessionlist</b>");

  Enumeration e =session.getAttributeNames();

  while (e.hasMoreElements()) {

  String name = (String)e.nextElement();

  String value =session.getAttribute(name).toString();

  out.println( name + " = " +value+"<br>");

  System.out.println( name + " = "+ value);

  }

  %>

  <form action="test.jsp"method="POST">

  name:<input type=text size=20name="dataName">

  <br>

  key:<input type=text size=20name="dataValue">

 <br>

  <input type=submit>

  </form>

  </body>

  </html>

 [root@server2ROOT]# cd /usr/local/tomcat/lib

[root@server2mnt]# ls /mnt/update/      #先前下載好的包

asm-3.2.jar                              minlog-1.2.jar

kryo-1.04.jar                           msm-kryo-serializer-1.6.3.jar

kryo-serializers-0.10.jar                reflectasm-1.01.jar

memcached-session-manager-1.6.3.jar      spymemcached-2.7.3.jar

memcached-session-manager-tc7-1.6.3.jar

[root@server2mnt]# cd /usr/local/tomcat/lib/

[root@server2lib]# mv /mnt/update/* .    #將包移動到指定路徑

[root@server2lib]# ls

annotations-api.jar                 memcached-session-manager-tc7-1.6.3.jar

asm-3.2.jar                          minlog-1.2.jar

catalina-ant.jar                    msm-kryo-serializer-1.6.3.jar

catalina-ha.jar                      reflectasm-1.01.jar

catalina.jar                         servlet-api.jar

catalina-tribes.jar                  spymemcached-2.7.3.jar

ecj-4.2.1.jar                        tomcat-api.jar

el-api.jar                           tomcat-coyote.jar

jasper-el.jar                        tomcat-dbcp.jar

jasper.jar                           tomcat-i18n-es.jar

jsp-api.jar                          tomcat-i18n-fr.jar

kryo-1.04.jar                        tomcat-i18n-ja.jar

kryo-serializers-0.10.jar            tomcat-jdbc.jar

memcached-session-manager-1.6.3.jar  tomcat-util.jar

[root@server2lib]# cd ..

UsingCATALINA_BASE:   /usr/local/tomcat

UsingCATALINA_HOME:   /usr/local/tomcat

UsingCATALINA_TMPDIR: /usr/local/tomcat/temp

UsingJRE_HOME:        /usr/local/java

UsingCLASSPATH:       /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar

[root@server2tomcat]# vim conf/context.xml    #添加節點及節點失效後該怎麼存儲

  <ManagerclassName="de.javakaffee.web.msm.MemcachedBackupSessionManager"

 memcachedNodes="n1:172.25.29.2:11211,n2:172.25.29.3:11211"     #結點

  failoverNodes="n1"           #當n2失效時存儲到n1上

 requestUriIgnorePattern=".*\.(ico|png|gif|jpg|css|js)$"

 transcoderFactoryClass="de.javakaffee.web.msm.serializer.kryo.KryoTranscoderF    actory"

  />

[root@server2tomcat]# bin/startup.sh   #開啓Tomcat

服務端3和服務端2的配置基本相同唯一不同的是上面提到的/usr/local/tomcat/conf/context.xml裏的36行把“n1”改成“n2”可以用scp傳過去再做修改。

 2.測試

測試 172.25.29.1/test.jsp

添加用戶

添加完成

添加3個用戶

                              

[root@server2tomcat]# tail -f logs/catalina.out      #查看日誌

Sep28, 2016 11:39:53 PM de.javakaffee.web.msm.serializer.kryo.KryoTranscoder<init>

INFO:Starting with initialBufferSize 102400 and maxBufferSize 2048000

Sep28, 2016 11:39:53 PM de.javakaffee.web.msm.MemcachedSessionServicestartInternal

INFO:MemcachedSessionService finished initialization, sticky true, operation timeout1000, with node ids [n2] and failover node ids [n1]

Sep28, 2016 11:39:53 PM org.apache.coyote.AbstractProtocol start

INFO:Starting ProtocolHandler ["http-bio-8080"]

Sep28, 2016 11:39:53 PM org.apache.coyote.AbstractProtocol start

INFO:Starting ProtocolHandler ["ajp-bio-8009"]

Sep28, 2016 11:39:53 PM org.apache.catalina.startup.Catalina start

INFO:Server startup in 1921 ms

user1= 111

user2= 222

user1= 111           #上面創建的用戶

user2= 222

user1= 111

user3= 333

^C

[root@server2tomcat]# telnet 172.25.29.3 11211  #遠程登陸服務端3查看是否是否寫進入11211是memcached的端口號

Trying172.25.29.3...

Connectedto 172.25.29.3.

Escapecharacter is '^]'.

get50B9BAB1CBB21C9BF884CC3613560752-n2   #get後面的是上面截圖裏的ID

VALUE50B9BAB1CBB21C9BF884CC3613560752-n2 2048 125

[1]WWqt2Wqt01WqtWqt#50B9BAB1CBB21C9BF884CC3613560752-n2

user2

222user1

111user3

333       #大概可以看出來添加的用戶

END

quit

Connectionclosed by foreign host.

 [root@server2tomcat]# bin/shutdown.sh  #關閉Tomcat1

UsingCATALINA_BASE:   /usr/local/tomcat

UsingCATALINA_HOME:   /usr/local/tomcat

UsingCATALINA_TMPDIR: /usr/local/tomcat/temp

UsingJRE_HOME:        /usr/local/java

UsingCLASSPATH:      /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar

 

當關閉Tomcat1續輸入數據時並沒有中斷而是跳到Tomcat2上且數據並未丟失

 

[root@server2tomcat]# telnet 172.25.29.3 11211     #遠程登陸服務端3

Trying172.25.29.3...

Connectedto 172.25.29.3.

Escapecharacter is '^]'.

get50B9BAB1CBB21C9BF884CC3613560752-n2

VALUE50B9BAB1CBB21C9BF884CC3613560752-n2 2048 137

[1]WWqt2Wqu01WquWqu4#50B9BAB1CBB21C9BF884CC3613560752-n2

---

user2  222user1  111user4  444user3  333   #大概可以看到數據還是寫入服務端3

END

quit

Connectionclosed by foreign host.