spark高可用， yarn

1.配置spark-env.sh

# 配置大哥;在二哥上面,MASTER_PORT=指的是自己
SPARK_MASTER_HOST=hadoop102
# 設置zookeepr,不能換行
SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=hadoop101:2181,hadoop102:2181,hadoop103:2181 -Dspark.deploy.zookeeper.dir=/spark"
# 告訴Spark,hadoop放到哪裏面了,前題是每一個Spark,的服務器上都裝有hadoop
HADOOP_CONF_DIR=/data/hadoop/hadoop-3.2.1/etc/hadoop/

配置二哥MASTER_PORT=指的是自己
SPARK_MASTER_HOST=hadoop101

2.配置slaves

#配置的小弟
hadoop103
hadoop104

3.啓動

1. 啓動zookeeper

bin/zkServer.sh

2. 啓動hadoop

sbin/start-all.sh

3. 啓動spark

sbin/start-all.sh

停止spark

sbin/stop-all.sh

注意查看spark的web端的時候，由於啓動了zookeeper，spark的端口與zookeeper的端口衝突，spark會把端口改成8081，在logs日誌中會顯示

# 查看網絡的狀態;是zookeeper佔用了8080端口
netstat -anp|grep 8080

啓動二哥在hadoop101上啓動

sbin/start-master.sh

4.運行示例

# --master:把任務提交給大哥;
# --executor-memory:指定executor的內存;
# --executor-cores:指定executor的cpu,核心
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
100000

# --master:把任務提交給大哥;
# --executor-memory:指定executor的內存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000

# --driver-cores:適應於集羣模式;2核
# --driver-memory:適應於集羣模式;500m;
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--driver-cores 2 \
--driver-memory 500m \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
100

# --master:把任務提交給大哥;
# --executor-memory:指定executor的內存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master yarn \
--name myPi \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000

集羣提交

# --master:把任務提交給大哥;
# --executor-memory:指定executor的內存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master yarn \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000