目錄
一、spark streaming編程步驟
- 構建streaming context（ssc）
- 指定輸入源構建Dstream
- 對Dstream進行transformations與actions
- ssc.start AND ssc.awaitTermination
二、spark streaming官方示例流程研究
1、NetworkWordCount
源代碼：NetworkWordCount.scala
package com.sm.spark.streaming
import org.apache.spark.SparkConf
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* 統計單詞數量
* 數據源：socket
* 輸出：控制檯print
*/
object NetworkWordCount {
def main(args: Array[String]): Unit = {
if (args.length < 2) {
System.err.println("Usage: NetworkWordCount <hostname> <port>")
System.exit(1)
}
StreamingExamples.setStreamingLogLevels()
val sparkConf = new SparkConf().setAppName("NetworkWordCount").setMaster("local[2]")
val ssc = new StreamingContext(sparkConf, Seconds(15))
val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK)
val words = lines.flatMap(_.split(" "))
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}
源代碼：StreamingExamples.scala
package com.sm.spark.streaming
import org.apache.log4j.{Level, Logger}
object StreamingExamples /*extends Logging*/ {
def setStreamingLogLevels(): Unit = {
val log4jInitialized = Logger.getRootLogger.getAllAppenders.hasMoreElements
if (!log4jInitialized) {
Logger.getRootLogger.setLevel(Level.WARN)
}
}
}
2、本地運行
a.設置參數：localhost 9999
b.終端鍵入nc -lk 9999，並輸入文字進行測試
c.查看程序執行日誌
/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/bin/java "-javaagent:/Applications/IntelliJ IDEA.app/Contents/lib/idea_rt.jar=58311:/Applications/IntelliJ IDEA.app/Contents/bin" -Dfile.encoding=UTF-8 -classpath /Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/deploy.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/javaws.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/jfxswt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/management-agent.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/plugin.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/ant-javafx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/dt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/javafx-mx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/jconsole.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/packager.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/sa-jdi.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_211.jdk/Contents/Home/lib/tools.jar:/Users/chengwanliu/IdeaProjects/spark-streaming-demo/target/classes:/Users/chengwanliu/.ivy2/cache/org.scala-lang/scala-reflect/jars/scala-reflect-2.11.12.jar:/Users/chengwanliu/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.12.jar:/Users/chengwanliu/.ivy2/cache/org.scala-lang/scala-reflect/srcs/scala-reflect-2.11.12-sources.jar:/Users/chengwanliu/.ivy2/cache/org.scala-lang/scala-library/srcs/scala-library-2.11.12-sources.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-core_2.11/2.1.1/spark-core_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/avro/avro-mapred/1.7.7/avro-mapred-1.7.7-hadoop2.jar:/Users/chengwanliu/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7.jar:/Users/chengwanliu/.m2/repository/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/Users/chengwanliu/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7-tests.jar:/Users/chengwanliu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/chengwanliu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/chengwanliu/.m2/repository/com/twitter/chill_2.11/0.8.0/chill_2.11-0.8.0.jar:/Users/chengwanliu/.m2/repository/com/esotericsoftware/kryo-shaded/3.0.3/kryo-shaded-3.0.3.jar:/Users/chengwanliu/.m2/repository/com/esotericsoftware/minlog/1.3.0/minlog-1.3.0.jar:/Users/chengwanliu/.m2/repository/org/objenesis/objenesis/2.1/objenesis-2.1.jar:/Users/chengwanliu/.m2/repository/com/twitter/chill-java/0.8.0/chill-java-0.8.0.jar:/Users/chengwanliu/.m2/repository/org/apache/xbean/xbean-asm5-shaded/4.4/xbean-asm5-shaded-4.4.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-client/2.2.0/hadoop-client-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-common/2.2.0/hadoop-common-2.2.0.jar:/Users/chengwanliu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/chengwanliu/.m2/repository/org/apache/commons/commons-math/2.1/commons-math-2.1.jar:/Users/chengwanliu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/chengwanliu/.m2/repository/commons-io/commons-io/2.1/commons-io-2.1.jar:/Users/chengwanliu/.m2/repository/commons-lang/commons-lang/2.5/commons-lang-2.5.jar:/Users/chengwanliu/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/chengwanliu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/chengwanliu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/chengwanliu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/chengwanliu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/chengwanliu/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-auth/2.2.0/hadoop-auth-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/chengwanliu/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.2.0/hadoop-hdfs-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.2.0/hadoop-mapreduce-client-app-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.2.0/hadoop-mapreduce-client-common-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.2.0/hadoop-yarn-client-2.2.0.jar:/Users/chengwanliu/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/Users/chengwanliu/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/chengwanliu/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.2.0/hadoop-yarn-server-common-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.2.0/hadoop-mapreduce-client-shuffle-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.2.0/hadoop-yarn-api-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.2.0/hadoop-mapreduce-client-core-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.2.0/hadoop-yarn-common-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.2.0/hadoop-mapreduce-client-jobclient-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/hadoop/hadoop-annotations/2.2.0/hadoop-annotations-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-launcher_2.11/2.1.1/spark-launcher_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-network-common_2.11/2.1.1/spark-network-common_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/Users/chengwanliu/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.6.5/jackson-annotations-2.6.5.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-network-shuffle_2.11/2.1.1/spark-network-shuffle_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-unsafe_2.11/2.1.1/spark-unsafe_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/net/java/dev/jets3t/jets3t/0.7.1/jets3t-0.7.1.jar:/Users/chengwanliu/.m2/repository/commons-codec/commons-codec/1.3/commons-codec-1.3.jar:/Users/chengwanliu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/chengwanliu/.m2/repository/org/apache/curator/curator-recipes/2.4.0/curator-recipes-2.4.0.jar:/Users/chengwanliu/.m2/repository/org/apache/curator/curator-framework/2.4.0/curator-framework-2.4.0.jar:/Users/chengwanliu/.m2/repository/org/apache/curator/curator-client/2.4.0/curator-client-2.4.0.jar:/Users/chengwanliu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.5/zookeeper-3.4.5.jar:/Users/chengwanliu/.m2/repository/com/google/guava/guava/14.0.1/guava-14.0.1.jar:/Users/chengwanliu/.m2/repository/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/Users/chengwanliu/.m2/repository/org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar:/Users/chengwanliu/.m2/repository/org/apache/commons/commons-math3/3.4.1/commons-math3-3.4.1.jar:/Users/chengwanliu/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/chengwanliu/.m2/repository/org/slf4j/slf4j-api/1.7.16/slf4j-api-1.7.16.jar:/Users/chengwanliu/.m2/repository/org/slf4j/jul-to-slf4j/1.7.16/jul-to-slf4j-1.7.16.jar:/Users/chengwanliu/.m2/repository/org/slf4j/jcl-over-slf4j/1.7.16/jcl-over-slf4j-1.7.16.jar:/Users/chengwanliu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/chengwanliu/.m2/repository/org/slf4j/slf4j-log4j12/1.7.16/slf4j-log4j12-1.7.16.jar:/Users/chengwanliu/.m2/repository/com/ning/compress-lzf/1.0.3/compress-lzf-1.0.3.jar:/Users/chengwanliu/.m2/repository/org/xerial/snappy/snappy-java/1.1.2.6/snappy-java-1.1.2.6.jar:/Users/chengwanliu/.m2/repository/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar:/Users/chengwanliu/.m2/repository/org/roaringbitmap/RoaringBitmap/0.5.11/RoaringBitmap-0.5.11.jar:/Users/chengwanliu/.m2/repository/commons-net/commons-net/2.2/commons-net-2.2.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar:/Users/chengwanliu/.m2/repository/org/json4s/json4s-jackson_2.11/3.2.11/json4s-jackson_2.11-3.2.11.jar:/Users/chengwanliu/.m2/repository/org/json4s/json4s-core_2.11/3.2.11/json4s-core_2.11-3.2.11.jar:/Users/chengwanliu/.m2/repository/org/json4s/json4s-ast_2.11/3.2.11/json4s-ast_2.11-3.2.11.jar:/Users/chengwanliu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.6/paranamer-2.6.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/scalap/2.11.0/scalap-2.11.0.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/scala-compiler/2.11.0/scala-compiler-2.11.0.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/modules/scala-xml_2.11/1.0.1/scala-xml_2.11-1.0.1.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/core/jersey-client/2.22.2/jersey-client-2.22.2.jar:/Users/chengwanliu/.m2/repository/javax/ws/rs/javax.ws.rs-api/2.0.1/javax.ws.rs-api-2.0.1.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/hk2-api/2.4.0-b34/hk2-api-2.4.0-b34.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/hk2-utils/2.4.0-b34/hk2-utils-2.4.0-b34.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/external/aopalliance-repackaged/2.4.0-b34/aopalliance-repackaged-2.4.0-b34.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/external/javax.inject/2.4.0-b34/javax.inject-2.4.0-b34.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/hk2-locator/2.4.0-b34/hk2-locator-2.4.0-b34.jar:/Users/chengwanliu/.m2/repository/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/core/jersey-common/2.22.2/jersey-common-2.22.2.jar:/Users/chengwanliu/.m2/repository/javax/annotation/javax.annotation-api/1.2/javax.annotation-api-1.2.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/bundles/repackaged/jersey-guava/2.22.2/jersey-guava-2.22.2.jar:/Users/chengwanliu/.m2/repository/org/glassfish/hk2/osgi-resource-locator/1.0.1/osgi-resource-locator-1.0.1.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/core/jersey-server/2.22.2/jersey-server-2.22.2.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/media/jersey-media-jaxb/2.22.2/jersey-media-jaxb-2.22.2.jar:/Users/chengwanliu/.m2/repository/javax/validation/validation-api/1.1.0.Final/validation-api-1.1.0.Final.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/containers/jersey-container-servlet/2.22.2/jersey-container-servlet-2.22.2.jar:/Users/chengwanliu/.m2/repository/org/glassfish/jersey/containers/jersey-container-servlet-core/2.22.2/jersey-container-servlet-core-2.22.2.jar:/Users/chengwanliu/.m2/repository/io/netty/netty-all/4.0.42.Final/netty-all-4.0.42.Final.jar:/Users/chengwanliu/.m2/repository/io/netty/netty/3.8.0.Final/netty-3.8.0.Final.jar:/Users/chengwanliu/.m2/repository/com/clearspring/analytics/stream/2.7.0/stream-2.7.0.jar:/Users/chengwanliu/.m2/repository/io/dropwizard/metrics/metrics-core/3.1.2/metrics-core-3.1.2.jar:/Users/chengwanliu/.m2/repository/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar:/Users/chengwanliu/.m2/repository/io/dropwizard/metrics/metrics-json/3.1.2/metrics-json-3.1.2.jar:/Users/chengwanliu/.m2/repository/io/dropwizard/metrics/metrics-graphite/3.1.2/metrics-graphite-3.1.2.jar:/Users/chengwanliu/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.6.5/jackson-databind-2.6.5.jar:/Users/chengwanliu/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.6.5/jackson-core-2.6.5.jar:/Users/chengwanliu/.m2/repository/com/fasterxml/jackson/module/jackson-module-scala_2.11/2.6.5/jackson-module-scala_2.11-2.6.5.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/scala-reflect/2.11.7/scala-reflect-2.11.7.jar:/Users/chengwanliu/.m2/repository/com/fasterxml/jackson/module/jackson-module-paranamer/2.6.5/jackson-module-paranamer-2.6.5.jar:/Users/chengwanliu/.m2/repository/org/apache/ivy/ivy/2.4.0/ivy-2.4.0.jar:/Users/chengwanliu/.m2/repository/oro/oro/2.0.8/oro-2.0.8.jar:/Users/chengwanliu/.m2/repository/net/razorvine/pyrolite/4.13/pyrolite-4.13.jar:/Users/chengwanliu/.m2/repository/net/sf/py4j/py4j/0.10.4/py4j-0.10.4.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-tags_2.11/2.1.1/spark-tags_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/commons/commons-crypto/1.0.0/commons-crypto-1.0.0.jar:/Users/chengwanliu/.m2/repository/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-sql_2.11/2.1.1/spark-sql_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/com/univocity/univocity-parsers/2.2.1/univocity-parsers-2.2.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-sketch_2.11/2.1.1/spark-sketch_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-catalyst_2.11/2.1.1/spark-catalyst_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/codehaus/janino/janino/3.0.0/janino-3.0.0.jar:/Users/chengwanliu/.m2/repository/org/codehaus/janino/commons-compiler/3.0.0/commons-compiler-3.0.0.jar:/Users/chengwanliu/.m2/repository/org/antlr/antlr4-runtime/4.5.3/antlr4-runtime-4.5.3.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-column/1.8.1/parquet-column-1.8.1.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-common/1.8.1/parquet-common-1.8.1.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-encoding/1.8.1/parquet-encoding-1.8.1.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-hadoop/1.8.1/parquet-hadoop-1.8.1.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-format/2.3.0-incubating/parquet-format-2.3.0-incubating.jar:/Users/chengwanliu/.m2/repository/org/apache/parquet/parquet-jackson/1.8.1/parquet-jackson-1.8.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-streaming_2.11/2.1.1/spark-streaming_2.11-2.1.1.jar:/Users/chengwanliu/.m2/repository/org/apache/spark/spark-streaming-kafka-0-10_2.11/2.1.0/spark-streaming-kafka-0-10_2.11-2.1.0.jar:/Users/chengwanliu/.m2/repository/org/apache/kafka/kafka_2.11/0.10.0.1/kafka_2.11-0.10.0.1.jar:/Users/chengwanliu/.m2/repository/com/101tec/zkclient/0.8/zkclient-0.8.jar:/Users/chengwanliu/.m2/repository/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar:/Users/chengwanliu/.m2/repository/org/scala-lang/modules/scala-parser-combinators_2.11/1.0.4/scala-parser-combinators_2.11-1.0.4.jar:/Users/chengwanliu/.m2/repository/org/apache/kafka/kafka-clients/0.10.0.1/kafka-clients-0.10.0.1.jar com.sm.spark.streaming.NetworkWordCount localhost 9999
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
19/10/04 15:30:44 INFO SparkContext: Running Spark version 2.1.1
19/10/04 15:30:44 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/10/04 15:30:44 INFO SecurityManager: Changing view acls to: chengwanliu
19/10/04 15:30:44 INFO SecurityManager: Changing modify acls to: chengwanliu
19/10/04 15:30:44 INFO SecurityManager: Changing view acls groups to:
19/10/04 15:30:44 INFO SecurityManager: Changing modify acls groups to:
19/10/04 15:30:44 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(chengwanliu); groups with view permissions: Set(); users with modify permissions: Set(chengwanliu); groups with modify permissions: Set()
19/10/04 15:30:45 INFO Utils: Successfully started service 'sparkDriver' on port 58315.
19/10/04 15:30:45 INFO SparkEnv: Registering MapOutputTracker
19/10/04 15:30:45 INFO SparkEnv: Registering BlockManagerMaster
19/10/04 15:30:45 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
19/10/04 15:30:45 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
19/10/04 15:30:45 INFO DiskBlockManager: Created local directory at /private/var/folders/jx/jkb9c3v92m51kfc2rrvbdbxw0000gn/T/blockmgr-1185888d-50e7-4ce7-9c76-d844dfa9d6d3
19/10/04 15:30:45 INFO MemoryStore: MemoryStore started with capacity 4.1 GB
19/10/04 15:30:45 INFO SparkEnv: Registering OutputCommitCoordinator
19/10/04 15:30:45 INFO Utils: Successfully started service 'SparkUI' on port 4040.
19/10/04 15:30:45 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://192.168.1.121:4040
19/10/04 15:30:45 INFO Executor: Starting executor ID driver on host localhost
19/10/04 15:30:45 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58317.
19/10/04 15:30:45 INFO NettyBlockTransferService: Server created on 192.168.1.121:58317
19/10/04 15:30:45 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
19/10/04 15:30:45 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.121, 58317, None)
19/10/04 15:30:45 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.121:58317 with 4.1 GB RAM, BlockManagerId(driver, 192.168.1.121, 58317, None)
19/10/04 15:30:45 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.121, 58317, None)
19/10/04 15:30:45 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.121, 58317, None)
19/10/04 15:30:45 INFO ReceiverTracker: Starting 1 receivers
19/10/04 15:30:45 INFO ReceiverTracker: ReceiverTracker started
19/10/04 15:30:45 INFO SocketInputDStream: Slide time = 15000 ms
19/10/04 15:30:45 INFO SocketInputDStream: Storage level = Serialized 1x Replicated
19/10/04 15:30:45 INFO SocketInputDStream: Checkpoint interval = null
19/10/04 15:30:45 INFO SocketInputDStream: Remember interval = 15000 ms
19/10/04 15:30:45 INFO SocketInputDStream: Initialized and validated org.apache.spark.streaming.dstream.SocketInputDStream@45077323
19/10/04 15:30:45 INFO FlatMappedDStream: Slide time = 15000 ms
19/10/04 15:30:45 INFO FlatMappedDStream: Storage level = Serialized 1x Replicated
19/10/04 15:30:45 INFO FlatMappedDStream: Checkpoint interval = null
19/10/04 15:30:45 INFO FlatMappedDStream: Remember interval = 15000 ms
19/10/04 15:30:45 INFO FlatMappedDStream: Initialized and validated org.apache.spark.streaming.dstream.FlatMappedDStream@64112ccd
19/10/04 15:30:45 INFO MappedDStream: Slide time = 15000 ms
19/10/04 15:30:45 INFO MappedDStream: Storage level = Serialized 1x Replicated
19/10/04 15:30:45 INFO MappedDStream: Checkpoint interval = null
19/10/04 15:30:45 INFO MappedDStream: Remember interval = 15000 ms
19/10/04 15:30:45 INFO MappedDStream: Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@3d4fe8c
19/10/04 15:30:45 INFO ShuffledDStream: Slide time = 15000 ms
19/10/04 15:30:45 INFO ShuffledDStream: Storage level = Serialized 1x Replicated
19/10/04 15:30:45 INFO ShuffledDStream: Checkpoint interval = null
19/10/04 15:30:45 INFO ShuffledDStream: Remember interval = 15000 ms
19/10/04 15:30:45 INFO ShuffledDStream: Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@1320795a
19/10/04 15:30:45 INFO ForEachDStream: Slide time = 15000 ms
19/10/04 15:30:45 INFO ForEachDStream: Storage level = Serialized 1x Replicated
19/10/04 15:30:45 INFO ForEachDStream: Checkpoint interval = null
19/10/04 15:30:45 INFO ForEachDStream: Remember interval = 15000 ms
19/10/04 15:30:45 INFO ForEachDStream: Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@6d408e76
19/10/04 15:30:45 INFO RecurringTimer: Started timer for JobGenerator at time 1570174260000
19/10/04 15:30:45 INFO JobGenerator: Started JobGenerator at 1570174260000 ms
19/10/04 15:30:45 INFO JobScheduler: Started JobScheduler
19/10/04 15:30:45 INFO ReceiverTracker: Receiver 0 started
19/10/04 15:30:45 INFO StreamingContext: StreamingContext started
19/10/04 15:30:45 INFO DAGScheduler: Got job 0 (start at NetworkWordCount.scala:33) with 1 output partitions
19/10/04 15:30:45 INFO DAGScheduler: Final stage: ResultStage 0 (start at NetworkWordCount.scala:33)
19/10/04 15:30:45 INFO DAGScheduler: Parents of final stage: List()
19/10/04 15:30:45 INFO DAGScheduler: Missing parents: List()
19/10/04 15:30:45 INFO DAGScheduler: Submitting ResultStage 0 (Receiver 0 ParallelCollectionRDD[0] at makeRDD at ReceiverTracker.scala:620), which has no missing parents
19/10/04 15:30:46 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 34.2 KB, free 4.1 GB)
19/10/04 15:30:46 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 11.4 KB, free 4.1 GB)
19/10/04 15:30:46 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.121:58317 (size: 11.4 KB, free: 4.1 GB)
19/10/04 15:30:46 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:996
19/10/04 15:30:46 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (Receiver 0 ParallelCollectionRDD[0] at makeRDD at ReceiverTracker.scala:620)
19/10/04 15:30:46 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
19/10/04 15:30:46 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, PROCESS_LOCAL, 6810 bytes)
19/10/04 15:30:46 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
19/10/04 15:30:46 INFO RecurringTimer: Started timer for BlockGenerator at time 1570174246200
19/10/04 15:30:46 INFO BlockGenerator: Started BlockGenerator
19/10/04 15:30:46 INFO BlockGenerator: Started block pushing thread
19/10/04 15:30:46 INFO ReceiverTracker: Registered receiver for stream 0 from 192.168.1.121:58315
19/10/04 15:30:46 INFO ReceiverSupervisorImpl: Starting receiver 0
19/10/04 15:30:46 INFO SocketReceiver: Connecting to localhost:9999
19/10/04 15:30:46 INFO SocketReceiver: Connected to localhost:9999
19/10/04 15:30:46 INFO ReceiverSupervisorImpl: Called receiver 0 onStart
19/10/04 15:30:46 INFO ReceiverSupervisorImpl: Waiting for receiver to be stopped
19/10/04 15:30:48 INFO MemoryStore: Block input-0-1570174248000 stored as values in memory (estimated size 72.0 B, free 4.1 GB)
19/10/04 15:30:48 INFO BlockManagerInfo: Added input-0-1570174248000 in memory on 192.168.1.121:58317 (size: 72.0 B, free: 4.1 GB)
19/10/04 15:30:48 INFO BlockGenerator: Pushed block input-0-1570174248000
19/10/04 15:31:00 INFO JobScheduler: Added jobs for time 1570174260000 ms
19/10/04 15:31:00 INFO JobScheduler: Starting job streaming job 1570174260000 ms.0 from job set of time 1570174260000 ms
19/10/04 15:31:00 INFO SparkContext: Starting job: print at NetworkWordCount.scala:30
19/10/04 15:31:00 INFO DAGScheduler: Registering RDD 3 (map at NetworkWordCount.scala:29)
19/10/04 15:31:00 INFO DAGScheduler: Got job 1 (print at NetworkWordCount.scala:30) with 1 output partitions
19/10/04 15:31:00 INFO DAGScheduler: Final stage: ResultStage 2 (print at NetworkWordCount.scala:30)
19/10/04 15:31:00 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 1)
19/10/04 15:31:00 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 1)
19/10/04 15:31:00 INFO DAGScheduler: Submitting ShuffleMapStage 1 (MapPartitionsRDD[3] at map at NetworkWordCount.scala:29), which has no missing parents
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 2.7 KB, free 4.1 GB)
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 1690.0 B, free 4.1 GB)
19/10/04 15:31:00 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.1.121:58317 (size: 1690.0 B, free: 4.1 GB)
19/10/04 15:31:00 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:996
19/10/04 15:31:00 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[3] at map at NetworkWordCount.scala:29)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
19/10/04 15:31:00 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, executor driver, partition 0, ANY, 6489 bytes)
19/10/04 15:31:00 INFO Executor: Running task 0.0 in stage 1.0 (TID 1)
19/10/04 15:31:00 INFO BlockManager: Found block input-0-1570174248000 locally
19/10/04 15:31:00 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 1586 bytes result sent to driver
19/10/04 15:31:00 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 41 ms on localhost (executor driver) (1/1)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
19/10/04 15:31:00 INFO DAGScheduler: ShuffleMapStage 1 (map at NetworkWordCount.scala:29) finished in 0.045 s
19/10/04 15:31:00 INFO DAGScheduler: looking for newly runnable stages
19/10/04 15:31:00 INFO DAGScheduler: running: Set(ResultStage 0)
19/10/04 15:31:00 INFO DAGScheduler: waiting: Set(ResultStage 2)
19/10/04 15:31:00 INFO DAGScheduler: failed: Set()
19/10/04 15:31:00 INFO DAGScheduler: Submitting ResultStage 2 (ShuffledRDD[4] at reduceByKey at NetworkWordCount.scala:29), which has no missing parents
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 2.8 KB, free 4.1 GB)
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1716.0 B, free 4.1 GB)
19/10/04 15:31:00 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.1.121:58317 (size: 1716.0 B, free: 4.1 GB)
19/10/04 15:31:00 INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:996
19/10/04 15:31:00 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 2 (ShuffledRDD[4] at reduceByKey at NetworkWordCount.scala:29)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Adding task set 2.0 with 1 tasks
19/10/04 15:31:00 INFO TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, localhost, executor driver, partition 0, ANY, 6377 bytes)
19/10/04 15:31:00 INFO Executor: Running task 0.0 in stage 2.0 (TID 2)
19/10/04 15:31:00 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
19/10/04 15:31:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms
19/10/04 15:31:00 INFO Executor: Finished task 0.0 in stage 2.0 (TID 2). 1867 bytes result sent to driver
19/10/04 15:31:00 INFO TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 22 ms on localhost (executor driver) (1/1)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool
19/10/04 15:31:00 INFO DAGScheduler: ResultStage 2 (print at NetworkWordCount.scala:30) finished in 0.023 s
19/10/04 15:31:00 INFO DAGScheduler: Job 1 finished: print at NetworkWordCount.scala:30, took 0.096842 s
19/10/04 15:31:00 INFO SparkContext: Starting job: print at NetworkWordCount.scala:30
19/10/04 15:31:00 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 151 bytes
19/10/04 15:31:00 INFO DAGScheduler: Got job 2 (print at NetworkWordCount.scala:30) with 1 output partitions
19/10/04 15:31:00 INFO DAGScheduler: Final stage: ResultStage 4 (print at NetworkWordCount.scala:30)
19/10/04 15:31:00 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 3)
19/10/04 15:31:00 INFO DAGScheduler: Missing parents: List()
19/10/04 15:31:00 INFO DAGScheduler: Submitting ResultStage 4 (ShuffledRDD[4] at reduceByKey at NetworkWordCount.scala:29), which has no missing parents
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 2.8 KB, free 4.1 GB)
19/10/04 15:31:00 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 1716.0 B, free 4.1 GB)
19/10/04 15:31:00 INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.1.121:58317 (size: 1716.0 B, free: 4.1 GB)
19/10/04 15:31:00 INFO SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:996
19/10/04 15:31:00 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[4] at reduceByKey at NetworkWordCount.scala:29)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Adding task set 4.0 with 1 tasks
19/10/04 15:31:00 INFO TaskSetManager: Starting task 0.0 in stage 4.0 (TID 3, localhost, executor driver, partition 1, PROCESS_LOCAL, 6377 bytes)
19/10/04 15:31:00 INFO Executor: Running task 0.0 in stage 4.0 (TID 3)
19/10/04 15:31:00 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 1 blocks
19/10/04 15:31:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
19/10/04 15:31:00 INFO Executor: Finished task 0.0 in stage 4.0 (TID 3). 1718 bytes result sent to driver
19/10/04 15:31:00 INFO TaskSetManager: Finished task 0.0 in stage 4.0 (TID 3) in 5 ms on localhost (executor driver) (1/1)
19/10/04 15:31:00 INFO TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool
19/10/04 15:31:00 INFO DAGScheduler: ResultStage 4 (print at NetworkWordCount.scala:30) finished in 0.005 s
19/10/04 15:31:00 INFO DAGScheduler: Job 2 finished: print at NetworkWordCount.scala:30, took 0.012146 s
-------------------------------------------
Time: 1570174260000 ms
-------------------------------------------
(7777,1)