Flink 累加器 實例

每個並行的任務實例下有各自的累加器，獲取最終值時會把各個並行任務的累加器的值求和。（一個並行度對應一個累加器）

版本: flink1.9.2    java1.8

map並行度1:

package Counter;

import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.accumulators.IntCounter;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

/**
 * @Author you guess
 * @Date 2020/6/25 12:30
 * @Version 1.0
 * @Desc
 */
public class CounterTest1 {

    public static void main(String[] args) throws Exception {
        //獲取執行環境
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //數據來源
        DataStream<String> source1 = env.fromElements("a", "b", "c");

        //operate
        source1.map(new RichMapFunction<String, String>() {

            //第一步：定義累加器
            IntCounter numLines = new IntCounter();

            @Override
            public void open(Configuration parameters) throws Exception {

                //第二步：註冊累加器
                getRuntimeContext().addAccumulator("num-lines", numLines);
            }

            @Override
            public String map(String s) throws Exception {
                //第三步：累加
                numLines.add(1);
                System.out.println("LocalValue:" + numLines.getLocalValue()+",線程id#"+Thread.currentThread().getId());
                return s;
            }

        }).setParallelism(1);

        //數據去向
        source1.print();

        //執行
        JobExecutionResult exe1 = env.execute("Flink CounterTest1 Demo");

        //第四步：結束後輸出總量；如果不需要結束後持久化，可以省去，因爲在flinkUI中可以看到
        String total = exe1.getAccumulatorResult("num-lines").toString();
        System.out.println("total:" + total);
    }//main

}
/*
輸出：
LocalValue:1,線程id#79
LocalValue:2,線程id#79
LocalValue:3,線程id#79
1> c
11> a
12> b
total:3
*/

 

map並行度不設置（默認是12）:

package Counter;

import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.accumulators.IntCounter;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

/**
 * @Author you guess
 * @Date 2020/6/25 12:30
 * @Version 1.0
 * @Desc
 */
public class CounterTest1 {

    public static void main(String[] args) throws Exception {
        //獲取執行環境
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //數據來源
        DataStream<String> source1 = env.fromElements("a", "b", "c");

        //operate
        source1.map(new RichMapFunction<String, String>() {

            //第一步：定義累加器
            IntCounter numLines = new IntCounter();

            @Override
            public void open(Configuration parameters) throws Exception {

                //第二步：註冊累加器
                getRuntimeContext().addAccumulator("num-lines", numLines);
            }

            @Override
            public String map(String s) throws Exception {
                //第三步：累加
                numLines.add(1);
                System.out.println("LocalValue:" + numLines.getLocalValue()+",線程id#"+Thread.currentThread().getId());
                return s;
            }

        });

        //數據去向
        source1.print();

        //執行
        JobExecutionResult exe1 = env.execute("Flink CounterTest1 Demo");

        //第四步：結束後輸出總量；如果不需要結束後持久化，可以省去，因爲在flinkUI中可以看到
        String total = exe1.getAccumulatorResult("num-lines").toString();
        System.out.println("total:" + total);
    }//main

}
/*
輸出：
7> b
LocalValue:1,線程id#76
LocalValue:1,線程id#75
6> a
LocalValue:1,線程id#74
8> c
total:3
*/