目錄
五、DefaultServerMessageListenerImpl
一、概述
seata的事務協調器TC(即DefaultCoordinator類)需要發送rpc請求至RM，進行branchCommit和branchRollback。
持有的ServerMessageSender的具體實現即是RpcServer，它是通過netty打開8091默認端口，啓動服務，接受請求
和發送信息。它的類框架圖如下所示，繼承了netty提供的ChannelDuplexHandler類，它提供對網絡讀寫數據和網絡
連接端口的攔截方法。實現ServerMessageSender接口，提供發送請求至RM方法。
二、AbstractRpcRemoting
AbstractRpcRemoting抽象類繼承ChannelDuplexHandler類和實現Disposable接口的destroy方法。它是seata框架應用於所有netty網絡的公共抽象類，有提供給RM和TM的rpc客戶端實現類和提供給TC的rpc服務端實現類。此處我們只分析與RpcServer
的相關方法。
public abstract class AbstractRpcRemoting extends ChannelDuplexHandler implements Disposable {
protected final ScheduledExecutorService timerExecutor = new ScheduledThreadPoolExecutor(1,
new NamedThreadFactory("timeoutChecker", 1, true));
//初始化， 啓動異步超時線程池timer，定時檢測MessageFuture隊列中的future是否超時，如果超時則移除
public void init() {
timerExecutor.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
List<MessageFuture> timeoutMessageFutures = new ArrayList<MessageFuture>(futures.size());
for (MessageFuture future : futures.values()) {
if (future.isTimeout()) {
timeoutMessageFutures.add(future);
}
}
for (MessageFuture messageFuture : timeoutMessageFutures) {
futures.remove(messageFuture.getRequestMessage().getId());
messageFuture.setResultMessage(null);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("timeout clear future : " + messageFuture.getRequestMessage().getBody());
}
}
nowMills = System.currentTimeMillis();
}
}, TIMEOUT_CHECK_INTERNAL, TIMEOUT_CHECK_INTERNAL, TimeUnit.MILLISECONDS);
}
// 銷燬線程池，回收資源
public void destroy() {
timerExecutor.shutdown();
messageExecutor.shutdown();
}
// 繼承channelWritabilityChanged方法，檢測channel通過是否可寫，如果可寫，則可寫鎖釋放
public void channelWritabilityChanged(ChannelHandlerContext ctx) {
synchronized (lock) {
if (ctx.channel().isWritable()) {
lock.notifyAll();
}
}
ctx.fireChannelWritabilityChanged();
}
// 發送異步請求數據
protected Object sendAsyncRequestWithResponse(String address, Channel channel, Object msg, long timeout) throws
TimeoutException {
if (timeout <= 0) {
throw new FrameworkException("timeout should more than 0ms");
}
return sendAsyncRequest(address, channel, msg, timeout);
}
// 發送異步請求數據
private Object sendAsyncRequest(String address, Channel channel, Object msg, long timeout)
throws TimeoutException {
if (channel == null) {
LOGGER.warn("sendAsyncRequestWithResponse nothing, caused by null channel.");
return null;
}
// 創建rpcMessage對象，messageType爲RESQUEST_ONEWAY,只請求不需要對方響應
final RpcMessage rpcMessage = new RpcMessage();
rpcMessage.setId(getNextMessageId());
rpcMessage.setMessageType(ProtocolConstants.MSGTYPE_RESQUEST_ONEWAY);
rpcMessage.setCodec(ProtocolConstants.CONFIGURED_CODEC);
rpcMessage.setCompressor(ProtocolConstants.CONFIGURED_COMPRESSOR);
rpcMessage.setBody(msg);
// 創建messageFuture對象，放入futures超時檢測隊列中
final MessageFuture messageFuture = new MessageFuture();
messageFuture.setRequestMessage(rpcMessage);
messageFuture.setTimeout(timeout);
futures.put(rpcMessage.getId(), messageFuture);
if (address != null) {
ConcurrentHashMap<String, BlockingQueue<RpcMessage>> map = basketMap;
BlockingQueue<RpcMessage> basket = map.get(address);
if (basket == null) {
map.putIfAbsent(address, new LinkedBlockingQueue<>());
basket = map.get(address);
}
basket.offer(rpcMessage);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("offer message: " + rpcMessage.getBody());
}
if (!isSending) {
synchronized (mergeLock) {
mergeLock.notifyAll();
}
}
} else {
// 服務器端發送address爲null
ChannelFuture future;
// 檢測channel是否可寫
channelWriteableCheck(channel, msg);
// 直接向channel寫數據
future = channel.writeAndFlush(rpcMessage);
// netty的ChannelFuture添加監聽器
future.addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) {
// 操作成功時，判斷ChannelFuture是否成功
if (!future.isSuccess()) {
// ChannelFuture失敗，設置原因並移除超時檢測隊列
MessageFuture messageFuture = futures.remove(rpcMessage.getId());
if (messageFuture != null) {
messageFuture.setResultMessage(future.cause());
}
// 最後銷燬channel
destroyChannel(future.channel());
}
}
});
}
// 等待timeout時間大於0
if (timeout > 0) {
try {
// 等待獲取
return messageFuture.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception exx) {
LOGGER.error("wait response error:" + exx.getMessage() + ",ip:" + address + ",request:" + msg);
if (exx instanceof TimeoutException) {
throw (TimeoutException)exx;
} else {
throw new RuntimeException(exx);
}
}
} else {
// 直接返回null
return null;
}
}
// 根據對方的請求request發送響應數據
// messageType爲HEARTBEAT_RESPONSE或則RESPONSE
protected void sendResponse(RpcMessage request, Channel channel, Object msg) {
RpcMessage rpcMessage = new RpcMessage();
rpcMessage.setMessageType(msg instanceof HeartbeatMessage ?
ProtocolConstants.MSGTYPE_HEARTBEAT_RESPONSE :
ProtocolConstants.MSGTYPE_RESPONSE);
rpcMessage.setCodec(request.getCodec()); // same with request
rpcMessage.setCompressor(request.getCompressor());
rpcMessage.setBody(msg);
rpcMessage.setId(request.getId());
channelWriteableCheck(channel, msg);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("send response:" + rpcMessage.getBody() + ",channel:" + channel);
}
channel.writeAndFlush(rpcMessage);
}
// nettyChannel讀取數據時攔截方法
@Override
public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exception {
// 如果msg是RpcMessage對象，則做相應處理
if (msg instanceof RpcMessage) {
final RpcMessage rpcMessage = (RpcMessage)msg;
// 獲取消息類型，如果是請求和只請求類型，則做處理
if (rpcMessage.getMessageType() == ProtocolConstants.MSGTYPE_RESQUEST
|| rpcMessage.getMessageType() == ProtocolConstants.MSGTYPE_RESQUEST_ONEWAY) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String.format("%s msgId:%s, body:%s", this, rpcMessage.getId(), rpcMessage.getBody()));
}
try {
// 使用消息處理器異步處理請求
AbstractRpcRemoting.this.messageExecutor.execute(new Runnable() {
@Override
public void run() {
try {
// 調用子類dispatch處理請求
dispatch(rpcMessage, ctx);
} catch (Throwable th) {
LOGGER.error(FrameworkErrorCode.NetDispatch.getErrCode(), th.getMessage(), th);
}
}
});
} catch (RejectedExecutionException e) {
LOGGER.error(FrameworkErrorCode.ThreadPoolFull.getErrCode(),
"thread pool is full, current max pool size is " + messageExecutor.getActiveCount());
// 報錯如果允許dumpStack，則調用jstack pid至日誌記錄
if (allowDumpStack) {
String name = ManagementFactory.getRuntimeMXBean().getName();
String pid = name.split("@")[0];
int idx = new Random().nextInt(100);
try {
Runtime.getRuntime().exec("jstack " + pid + " >d:/" + idx + ".log");
} catch (IOException exx) {
LOGGER.error(exx.getMessage());
}
allowDumpStack = false;
}
}
} else {
// 或者是心跳檢測或則響應的類型
// 已經響應了，從超時檢測隊列futures中移除
MessageFuture messageFuture = futures.remove(rpcMessage.getId());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String
.format("%s msgId:%s, future :%s, body:%s", this, rpcMessage.getId(), messageFuture,
rpcMessage.getBody()));
}
// 如果隊列中有messageFuture設置返回的結果給messageFuture
if (messageFuture != null) {
messageFuture.setResultMessage(rpcMessage.getBody());
} else {
try {
AbstractRpcRemoting.this.messageExecutor.execute(new Runnable() {
@Override
public void run() {
try {
// messageFuture爲null，調用dispatch方法
dispatch(rpcMessage, ctx);
} catch (Throwable th) {
LOGGER.error(FrameworkErrorCode.NetDispatch.getErrCode(), th.getMessage(), th);
}
}
});
} catch (RejectedExecutionException e) {
LOGGER.error(FrameworkErrorCode.ThreadPoolFull.getErrCode(),
"thread pool is full, current max pool size is " + messageExecutor.getActiveCount());
}
}
}
}
}
// 通道移除捕獲攔截，打印日誌，銷燬channel
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
LOGGER.error(FrameworkErrorCode.ExceptionCaught.getErrCode(),
ctx.channel() + " connect exception. " + cause.getMessage(),
cause);
try {
destroyChannel(ctx.channel());
} catch (Exception e) {
LOGGER.error("", "close channel" + ctx.channel() + " fail.", e);
}
}
// 根據RpcMessage請求，子類實現分配處理
public abstract void dispatch(RpcMessage request, ChannelHandlerContext ctx);
}
三、AbstractRpcRemotingServer
AbstractRpcRemotingServer繼承AbstractRpcRemoting類，構建netty服務器，即爲seata的TC協調器的網絡核心。
提供創建serverBootstrap，綁定端口，啓動服務，銷燬服務器方法。
public abstract class AbstractRpcRemotingServer extends AbstractRpcRemoting implements RemotingServer {
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRpcRemotingServer.class);
private final ServerBootstrap serverBootstrap;
private final EventLoopGroup eventLoopGroupWorker;
private final EventLoopGroup eventLoopGroupBoss;
private final NettyServerConfig nettyServerConfig;
private int listenPort;
private String host;
private final AtomicBoolean initialized = new AtomicBoolean(false);
public void setListenPort(int listenPort) {
if (listenPort <= 0) {
throw new IllegalArgumentException("listen port: " + listenPort + " is invalid!");
}
this.listenPort = listenPort;
}
public void setHost(String host) {
if (!NetUtil.isValidIp(host, true)) {
throw new IllegalArgumentException("host: " + host + " is invalid!");
}
this.host = host;
}
public AbstractRpcRemotingServer(final NettyServerConfig nettyServerConfig) {
this(nettyServerConfig, null);
}
public AbstractRpcRemotingServer(final NettyServerConfig nettyServerConfig,
final ThreadPoolExecutor messageExecutor, final ChannelHandler... handlers) {
super(messageExecutor);
this.serverBootstrap = new ServerBootstrap();
this.nettyServerConfig = nettyServerConfig;
if (NettyServerConfig.enableEpoll()) {
this.eventLoopGroupBoss = new EpollEventLoopGroup(nettyServerConfig.getBossThreadSize(),
new NamedThreadFactory(nettyServerConfig.getBossThreadPrefix(), nettyServerConfig.getBossThreadSize()));
this.eventLoopGroupWorker = new EpollEventLoopGroup(nettyServerConfig.getServerWorkerThreads(),
new NamedThreadFactory(nettyServerConfig.getWorkerThreadPrefix(),
nettyServerConfig.getServerWorkerThreads()));
} else {
this.eventLoopGroupBoss = new NioEventLoopGroup(nettyServerConfig.getBossThreadSize(),
new NamedThreadFactory(nettyServerConfig.getBossThreadPrefix(), nettyServerConfig.getBossThreadSize()));
this.eventLoopGroupWorker = new NioEventLoopGroup(nettyServerConfig.getServerWorkerThreads(),
new NamedThreadFactory(nettyServerConfig.getWorkerThreadPrefix(),
nettyServerConfig.getServerWorkerThreads()));
}
if (null != handlers) {
channelHandlers = handlers;
}
// init listenPort in constructor so that getListenPort() will always get the exact port
setListenPort(nettyServerConfig.getDefaultListenPort());
}
@Override
public void start() {
this.serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupWorker)
.channel(nettyServerConfig.SERVER_CHANNEL_CLAZZ)
.option(ChannelOption.SO_BACKLOG, nettyServerConfig.getSoBackLogSize())
.option(ChannelOption.SO_REUSEADDR, true)
.childOption(ChannelOption.SO_KEEPALIVE, true)
.childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSendBufSize())
.childOption(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketResvBufSize())
.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK,
new WriteBufferWaterMark(nettyServerConfig.getWriteBufferLowWaterMark(),
nettyServerConfig.getWriteBufferHighWaterMark()))
.localAddress(new InetSocketAddress(listenPort))
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ch.pipeline().addLast(new IdleStateHandler(nettyServerConfig.getChannelMaxReadIdleSeconds(), 0, 0))
.addLast(new ProtocolV1Decoder())
.addLast(new ProtocolV1Encoder());
if (null != channelHandlers) {
addChannelPipelineLast(ch, channelHandlers);
}
}
});
if (nettyServerConfig.isEnableServerPooledByteBufAllocator()) {
this.serverBootstrap.childOption(ChannelOption.ALLOCATOR, NettyServerConfig.DIRECT_BYTE_BUF_ALLOCATOR);
}
try {
ChannelFuture future = this.serverBootstrap.bind(host, listenPort).sync();
LOGGER.info("Server started ... ");
RegistryFactory.getInstance().register(new InetSocketAddress(XID.getIpAddress(), XID.getPort()));
initialized.set(true);
future.channel().closeFuture().sync();
} catch (Exception exx) {
throw new RuntimeException(exx);
}
}
@Override
public void shutdown() {
try {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Shuting server down. ");
}
if (initialized.get()) {
RegistryFactory.getInstance().unregister(new InetSocketAddress(XID.getIpAddress(), XID.getPort()));
RegistryFactory.getInstance().close();
//wait a few seconds for server transport
TimeUnit.SECONDS.sleep(nettyServerConfig.getServerShutdownWaitTime());
}
this.eventLoopGroupBoss.shutdownGracefully();
this.eventLoopGroupWorker.shutdownGracefully();
} catch (Exception exx) {
LOGGER.error(exx.getMessage());
}
}
@Override
public void destroyChannel(String serverAddress, Channel channel) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("will destroy channel:" + channel + ",address:" + serverAddress);
}
channel.disconnect();
channel.close();
}
}
四、RpcServer
RpcServer實現ServerMessageSender接口，提供sendResponse，sendSyncRequest方法。
public class RpcServer extends AbstractRpcRemotingServer implements ServerMessageSender {
protected ServerMessageListener serverMessageListener;
private TransactionMessageHandler transactionMessageHandler;
private RegisterCheckAuthHandler checkAuthHandler;
// 構造方法，傳入messageExecutor
public RpcServer(ThreadPoolExecutor messageExecutor) {
// 調用父類方法構造netty服務器
super(new NettyServerConfig(), messageExecutor);
}
// 初始化
public void init() {
// 調用父類方法，綁定端口，啓動服務器
super.init();
setChannelHandlers(RpcServer.this);
DefaultServerMessageListenerImpl defaultServerMessageListenerImpl = new DefaultServerMessageListenerImpl(
transactionMessageHandler);
defaultServerMessageListenerImpl.init();
defaultServerMessageListenerImpl.setServerMessageSender(this);
this.setServerMessageListener(defaultServerMessageListenerImpl);
super.start();
}
// 銷燬rpcServer
public void destroy() {
super.destroy();
super.shutdown();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("destroyed rpcServer");
}
}
// 根據RM的請求發送響應信息
public void sendResponse(RpcMessage request, Channel channel, Object msg) {
Channel clientChannel = channel;
if (!(msg instanceof HeartbeatMessage)) {
clientChannel = ChannelManager.getSameClientChannel(channel);
}
if (clientChannel != null) {
// 調用父類發送Response
super.sendResponse(request, clientChannel, msg);
} else {
throw new RuntimeException("channel is error. channel:" + clientChannel);
}
}
@Override
// 向RM發送同步請求，branchCommit或者branchRollback
public Object sendSyncRequest(String resourceId, String clientId, Object message,
long timeout) throws TimeoutException {
Channel clientChannel = ChannelManager.getChannel(resourceId, clientId);
if (clientChannel == null) {
throw new RuntimeException("rm client is not connected. dbkey:" + resourceId
+ ",clientId:" + clientId);
}
// 調用父類發送
return sendAsyncRequestWithResponse(null, clientChannel, message, timeout);
}
// 分派客戶端請求數據
public void dispatch(RpcMessage request, ChannelHandlerContext ctx) {
Object msg = request.getBody();
if (msg instanceof RegisterRMRequest) {
// 如果它是註冊請求信息，則RmMessage監聽事件處理
serverMessageListener.onRegRmMessage(request, ctx, this,
checkAuthHandler);
} else {
// 如果是channel已經註冊，則調用TrxMessage監聽事件處理
if (ChannelManager.isRegistered(ctx.channel())) {
serverMessageListener.onTrxMessage(request, ctx, this);
} else {
// 如果沒有註冊，關閉ChannelHandlerContext
try {
closeChannelHandlerContext(ctx);
} catch (Exception exx) {
LOGGER.error(exx.getMessage());
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info(String.format("close a unhandled connection! [%s]", ctx.channel().toString()));
}
}
}
}
// 調用父類讀取通道數據之前進行判斷和監聽
public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exception {
if (msg instanceof RpcMessage) {
RpcMessage rpcMessage = (RpcMessage) msg;
debugLog("read:" + rpcMessage.getBody().toString());
if (rpcMessage.getBody() instanceof RegisterTMRequest) {
serverMessageListener.onRegTmMessage(rpcMessage, ctx, this, checkAuthHandler);
return;
}
if (rpcMessage.getBody() == HeartbeatMessage.PING) {
serverMessageListener.onCheckMessage(rpcMessage, ctx, this);
return;
}
}
super.channelRead(ctx, msg);
}
}
五、DefaultServerMessageListenerImpl
DefaultServerMessageListenerImpl監聽器，處理rpc收到消息請求。
public class DefaultServerMessageListenerImpl implements ServerMessageListener {
public DefaultServerMessageListenerImpl(TransactionMessageHandler transactionMessageHandler) {
this.transactionMessageHandler = transactionMessageHandler;
}
// 處理事務信息請求
public void onTrxMessage(RpcMessage request, ChannelHandlerContext ctx, ServerMessageSender sender) {
Object message = request.getBody();
RpcContext rpcContext = ChannelManager.getContextFromIdentified(ctx.channel());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"server received:" + message + ",clientIp:" + NetUtil.toIpAddress(ctx.channel().remoteAddress())
+ ",vgroup:" + rpcContext.getTransactionServiceGroup());
} else {
messageStrings.offer(
message + ",clientIp:" + NetUtil.toIpAddress(ctx.channel().remoteAddress()) + ",vgroup:" + rpcContext
.getTransactionServiceGroup());
}
if (!(message instanceof AbstractMessage)) { return; }
if (message instanceof MergedWarpMessage) {
AbstractResultMessage[] results = new AbstractResultMessage[((MergedWarpMessage)message).msgs.size()];
for (int i = 0; i < results.length; i++) {
final AbstractMessage subMessage = ((MergedWarpMessage)message).msgs.get(i);
results[i] = transactionMessageHandler.onRequest(subMessage, rpcContext);
}
MergeResultMessage resultMessage = new MergeResultMessage();
resultMessage.setMsgs(results);
sender.sendResponse(request, ctx.channel(), resultMessage);
} else if (message instanceof AbstractResultMessage) {
transactionMessageHandler.onResponse((AbstractResultMessage)message, rpcContext);
}
}
// 處理RM註冊請求
public void onRegRmMessage(RpcMessage request, ChannelHandlerContext ctx,
ServerMessageSender sender, RegisterCheckAuthHandler checkAuthHandler) {
RegisterRMRequest message = (RegisterRMRequest) request.getBody();
boolean isSuccess = false;
try {
if (null == checkAuthHandler || checkAuthHandler.regResourceManagerCheckAuth(message)) {
ChannelManager.registerRMChannel(message, ctx.channel());
Version.putChannelVersion(ctx.channel(), message.getVersion());
isSuccess = true;
}
} catch (Exception exx) {
isSuccess = false;
LOGGER.error(exx.getMessage());
}
sender.sendResponse(request, ctx.channel(), new RegisterRMResponse(isSuccess));
if (LOGGER.isInfoEnabled()) {
LOGGER.info("rm register success,message:" + message + ",channel:" + ctx.channel());
}
}
// 處理TM註冊請求
public void onRegTmMessage(RpcMessage request, ChannelHandlerContext ctx,
ServerMessageSender sender, RegisterCheckAuthHandler checkAuthHandler) {
RegisterTMRequest message = (RegisterTMRequest) request.getBody();
String ipAndPort = NetUtil.toStringAddress(ctx.channel().remoteAddress());
Version.putChannelVersion(ctx.channel(), message.getVersion());
boolean isSuccess = false;
try {
if (null == checkAuthHandler || checkAuthHandler.regTransactionManagerCheckAuth(message)) {
ChannelManager.registerTMChannel(message, ctx.channel());
Version.putChannelVersion(ctx.channel(), message.getVersion());
isSuccess = true;
if (LOGGER.isInfoEnabled()) {
LOGGER.info(String
.format("checkAuth for client:%s vgroup:%s ok", ipAndPort,
message.getTransactionServiceGroup()));
}
}
} catch (Exception exx) {
isSuccess = false;
LOGGER.error(exx.getMessage());
}
//FIXME please add success or fail
sender.sendResponse(request, ctx.channel(),
new RegisterTMResponse(isSuccess));
}
}
