【微服務】- Hystrix請求執行流程分析

hystrix 請求執行流程源碼分析

HystrixCommandAspect：所有註解爲HystrixCommand或HystrixCollapser的Http請求方法被攔截，整個請求封裝成Command在Hystrix流轉，分爲Observable和非Observable類型， 底層都是採用RxJava1.x實現異步請求處理，請求執行類型分爲：SYNCHRONOUS、ASYNCHRONOUS、OBSERVABLE， OBSERVABLE分爲冷(toObservable)和熱(observe)兩種類型，冷即RxJava Observable被訂閱的時候纔會發送數據，re是不等訂閱就發送數據， 通過HystrixCommand的observableExecutionMode屬性決定，默認eagle,即熱Observable

---

舉例：正常請求攔截（非Observable同步執行類型）

1. 實際業務處理：com.netflix.hystrix.AbstractCommand.applyHystrixSemantics

private Observable<R> applyHystrixSemantics(final AbstractCommand<R> _cmd) {
  
    // 標識準備開始執行Command,可以直接實現HystrixCommandExecutionHook監聽command執行生命週期活動
    executionHook.onStart(_cmd);

    /* 斷路器決定是否放開請求：根據斷路器的3種中斷來判斷 */
    if (circuitBreaker.allowRequest()) {
        // 獲取信號量組件：默認線程池返回TryableSemaphoreNoOp.DEFAULT，若設置成信號量機制則返回TryableSemaphoreActual，併發數又配置：execution.isolation.semaphore.maxConcurrentRequests 決定
        final TryableSemaphore executionSemaphore = getExecutionSemaphore();

        //創建信號量是否釋放標識：原子性
        final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false);

        // 創建RxJava函數，主要用於根據semaphoreHasBeenReleased的信號量釋放標識根據CAS操作釋放信號量
        final Action0 singleSemaphoreRelease = new Action0() {
            @Override
            public void call() {
                if (semaphoreHasBeenReleased.compareAndSet(false, true)) {
                    executionSemaphore.release();
                }
            }
        };

        // 創建RxJava函數： 主要用於通知異常事件, 默認HystrixEventNotifierDefault，do nothing
        final Action1<Throwable> markExceptionThrown = new Action1<Throwable>() {
            @Override
            public void call(Throwable t) {
                eventNotifier.markEvent(HystrixEventType.EXCEPTION_THROWN, commandKey);
            }
        };

        // 嘗試獲取信號量，類似AQS共享鎖實現，state,標識共享的資源個數，即Hystrix信號量機制最大的請求併發數
        // 信號量機制：TryableSemaphoreActual
        // 線程池機制：TryableSemaphoreNoOp
        if (executionSemaphore.tryAcquire()) {
            try {
                /* 記錄command處理開始時間 */
                executionResult = executionResult.setInvocationStartTime(System.currentTimeMillis());

                // 真正執行具體的command業務邏輯，並利用RxJava監聽事件：Error 錯誤監聽，Terminate：命令執行完成監聽(成功/失敗)、unsubscribe 取消訂閱事件釋放信號量
                return executeCommandAndObserve(_cmd)
                        .doOnError(markExceptionThrown)
                        .doOnTerminate(singleSemaphoreRelease)
                        .doOnUnsubscribe(singleSemaphoreRelease);
            } catch (RuntimeException e) {
                return Observable.error(e);
            }
        } else {
            // 獲取信號量失敗，則觸發熔斷
            return handleSemaphoreRejectionViaFallback();
        }
    } else {
        // 斷路器打開，所有請求都被拒絕，觸發熔斷操作
        return handleShortCircuitViaFallback();
    }
}

2. 斷路器狀態（打開、關閉、半開）

斷路器狀態：

1. 打開： 斷路器打開，決絕所有請求
2. 關閉： 斷路器關閉，接收所有請求
3. 半開： 只放開1個請求進入，測試目標服務是否可訪問，若可行則斷路器關閉，接收所有請求

源碼：com.netflix.hystrix.HystrixCircuitBreaker.HystrixCircuitBreakerImpl.allowRequest

@Override
public boolean allowRequest() {
    // 斷路器強制打開： 拒絕所有請求
    if (properties.circuitBreakerForceOpen().get()) {
        // properties have asked us to force the circuit open so we will allow NO requests
        return false;
    }

    // 斷路器關閉：返回true,允許放入請求
    if (properties.circuitBreakerForceClosed().get()) {
        //實際檢測斷路器狀態，若單個時間窗口最大請求數 > circuitBreaker.requestVolumeThreshold 或 錯誤請求百分比 > circuitBreaker.errorThresholdPercentage
        // circuitOpen通過CAS操作標識斷路器打開，circuitOpenedOrLastTestedTime 更新最新斷路器打開時間，方便下個時間窗口嘗試放入請求
        isOpen();
        return true;
    }

    // 半開狀態：斷路器打開，允許放入1個請求去測試
    return !isOpen() || allowSingleTest();
}

嘗試放入單個請求進入處理

 public boolean allowSingleTest() {
    // 獲取最新斷路器打開時間
    long timeCircuitOpenedOrWasLastTested = circuitOpenedOrLastTestedTime.get();
    
    // 斷路器打開、而且過了1個時間窗口，則放入1個請求進入，返回true,標識整個Http請求流程可繼續執行
    // 時間窗口時間：circuitBreaker.sleepWindowInMilliseconds
    // 1) if the circuit is open
    // 2) and it's been longer than 'sleepWindow' since we opened the circuit
    if (circuitOpen.get() && System.currentTimeMillis() > timeCircuitOpenedOrWasLastTested + properties.circuitBreakerSleepWindowInMilliseconds().get()) {
       
        // CAS更新斷路器打開時間未保證只有1個請求通過斷路器（原子性）
        if (circuitOpenedOrLastTestedTime.compareAndSet(timeCircuitOpenedOrWasLastTested, System.currentTimeMillis())) {
            return true;
        }
    }
    return false;
}

3. Hystrix 請求處理邏輯控制

com.netflix.hystrix.AbstractCommand.executeCommandAndObserve
- markEmits: RxJava 發射數據監聽，主要用於數據統計
- markOnCompleted： RxJava發射數據完成監聽，即Hystrix請求處理完成監聽
- handleFallback： 執行過程出現異常，進行服務降級處理（RejectedExecutionException、HystrixTimeoutException、HystrixBadRequestException、其他通用異常處理）
- setRequestContext： 設置請求上下文，類似：Zuul RequestContext，Http請求環境

com.netflix.hystrix.AbstractCommand.executeCommandWithSpecifiedIsolation： 按照隔離級別不同進行不同業務處理

com.netflix.hystrix.AbstractCommand.executeCommandWithSpecifiedIsolation

private Observable<R> executeCommandWithSpecifiedIsolation(final AbstractCommand<R> _cmd) {
        // 線程池隔離級別
        if (properties.executionIsolationStrategy().get() == ExecutionIsolationStrategy.THREAD) {
            // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE)
            return Observable.defer(new Func0<Observable<R>>() {
                @Override
                public Observable<R> call() {
                    executionResult = executionResult.setExecutionOccurred();

                    // command狀態週期：NOT_STARTED, OBSERVABLE_CHAIN_CREATED, USER_CODE_EXECUTED, UNSUBSCRIBED, TERMINAL
                    // 更新command狀態：OBSERVABLE_CHAIN_CREATED --> USER_CODE_EXECUTED, 標識要準備執行代碼，若狀態不對則RxJava拋出錯誤，Hystrix直接服務熔斷處理
                    if (!commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.USER_CODE_EXECUTED)) {
                        return Observable.error(new IllegalStateException("execution attempted while in state : " + commandState.get().name()));
                    }
                    
                    // Hystrix 指標器設置監聽的command的相關數據： commandKey(默認方法名)
                    metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.THREAD);
                    
                    // 超時檢測
                    if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) {
                        // the command timed out in the wrapping thread so we will return immediately
                        // and not increment any of the counters below or other such logic
                        return Observable.error(new RuntimeException("timed out before executing run()"));
                    }

                    // 線程狀態更新：NOT_USING_THREAD --> STARTED, 標識業務線程準備運行
                    if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.STARTED)) {
                        //HystrixCountera統計併發線程數 + 1
                        HystrixCounters.incrementGlobalConcurrentThreads();

                        // 線程池標記請求：計數器+1，標識線程池運行線程數
                        threadPool.markThreadExecution();

                        // 保存正在運行的command,本質存在stack中，後續執行完成後從stack pop
                        endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey());

                        // 創建執行結果，標識線程狀態正在執行
                        executionResult = executionResult.setExecutedInThread();
                        /**
                         * If any of these hooks throw an exception, then it appears as if the actual execution threw an error
                         */
                        try {
                            // 事件監聽鉤子函數
                            executionHook.onThreadStart(_cmd);
                            executionHook.onRunStart(_cmd);
                            executionHook.onExecutionStart(_cmd);

                            // 獲取用戶執行任務的Observable
                            return getUserExecutionObservable(_cmd);
                        } catch (Throwable ex) {
                            return Observable.error(ex);
                        }
                    } else {
                        //command has already been unsubscribed, so return immediately
                        return Observable.error(new RuntimeException("unsubscribed before executing run()"));
                    }
                }
            }).doOnTerminate(new Action0() { //任務完成終止操作
                @Override
                public void call() {
                    if (threadState.compareAndSet(ThreadState.STARTED, ThreadState.TERMINAL)) {
                        handleThreadEnd(_cmd);
                    }
                    if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.TERMINAL)) {
                        //if it was never started and received terminal, then no need to clean up (I don't think this is possible)
                    }
                    //if it was unsubscribed, then other cleanup handled it
                }
            }).doOnUnsubscribe(new Action0() { // 取消訂閱操作
                @Override
                public void call() {
                    if (threadState.compareAndSet(ThreadState.STARTED, ThreadState.UNSUBSCRIBED)) {
                        handleThreadEnd(_cmd);
                    }
                    if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.UNSUBSCRIBED)) {
                        //if it was never started and was cancelled, then no need to clean up
                    }
                    //if it was terminal, then other cleanup handled it
                }
            }).subscribeOn(threadPool.getScheduler(new Func0<Boolean>() { // 訂閱操作
                @Override
                public Boolean call() {
                    return properties.executionIsolationThreadInterruptOnTimeout().get() && _cmd.isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT;
                }
            }));
         // 信號量隔離級別
        } else {
            return Observable.defer(new Func0<Observable<R>>() {
                @Override
                public Observable<R> call() {
                    executionResult = executionResult.setExecutionOccurred();
                    if (!commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.USER_CODE_EXECUTED)) {
                        return Observable.error(new IllegalStateException("execution attempted while in state : " + commandState.get().name()));
                    }

                    metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.SEMAPHORE);
                    // semaphore isolated
                    // store the command that is being run
                    endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey());
                    try {
                        executionHook.onRunStart(_cmd);
                        executionHook.onExecutionStart(_cmd);
                        return getUserExecutionObservable(_cmd);  //the getUserExecutionObservable method already wraps sync exceptions, so this shouldn't throw
                    } catch (Throwable ex) {
                        //If the above hooks throw, then use that as the result of the run method
                        return Observable.error(ex);
                    }
                }
            });
        }
    }

4. 用戶代碼執行具體業務

com.netflix.hystrix.AbstractCommand.getUserExecutionObservable

 private Observable<R> getUserExecutionObservable(final AbstractCommand<R> _cmd) {
    Observable<R> userObservable;

    try {
        // 獲取用戶userObservable， single類型，返回單值，本質由用戶代碼執行
        userObservable = getExecutionObservable();
    } catch (Throwable ex) {
        // the run() method is a user provided implementation so can throw instead of using Observable.onError
        // so we catch it here and turn it into Observable.error
        userObservable = Observable.error(ex);
    }

    return userObservable
            .lift(new ExecutionHookApplication(_cmd))
            .lift(new DeprecatedOnRunHookApplication(_cmd));
}

com.netflix.hystrix.HystrixCommand.getExecutionObservable: 重寫getExecutionObservable()

final protected Observable<R> getExecutionObservable() {
    return Observable.defer(new Func0<Observable<R>>() {
        @Override
        public Observable<R> call() {
            try {
                // 返回單值Observable: toObservable().toBlocking().toFuture(); 觸發emit操作（com.netflix.hystrix.HystrixCommand.queue）
                return Observable.just(run());
            } catch (Throwable ex) {
                return Observable.error(ex);
            }
        }
    }).doOnSubscribe(new Action0() {
        @Override
        public void call() {
            // Save thread on which we get subscribed so that we can interrupt it later if needed
            executionThread.set(Thread.currentThread());
        }
    });
}

run實現:com.netflix.hystrix.contrib.javanica.command.GenericCommand.run

@Override
protected Object run() throws Exception {
    LOGGER.debug("execute command: {}", getCommandKey().name());
    return process(new Action() {
        @Override
        Object execute() {
            //MethodExecutionAction：普通方法執行，非懶加載機制
            return getCommandAction().execute(getExecutionType());
        }
    });
}

com.netflix.hystrix.contrib.javanica.command.MethodExecutionAction.executeWithArgs

 @Override
public Object executeWithArgs(ExecutionType executionType, Object[] args) throws CommandActionExecutionException {
    if(ExecutionType.ASYNCHRONOUS == executionType){
        Closure closure = AsyncClosureFactory.getInstance().createClosure(metaHolder, method, object, args);
        return executeClj(closure.getClosureObj(), closure.getClosureMethod());
    }

    return execute(object, method, args);
}

// 反射機制回調，直接執行斷路器切入點的具體業務邏輯，涉及服務底層由Ribbon負責與目標服務交互
 private Object execute(Object o, Method m, Object... args) throws CommandActionExecutionException {
    Object result = null;
    try {
        m.setAccessible(true); // suppress Java language access
        if (isCompileWeaving() && metaHolder.getAjcMethod() != null) {
            result = invokeAjcMethod(metaHolder.getAjcMethod(), o, metaHolder, args);
        } else {
            result = m.invoke(o, args);
        }
    } catch (IllegalAccessException e) {
        propagateCause(e);
    } catch (InvocationTargetException e) {
        propagateCause(e);
    }
    return result;
}