spark讀取hive和寫入hive

1.導入maven

<properties>
        <spark.version>2.1.1</spark.version>
        <scala.version>2.11.8</scala.version>
</properties>
<dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
</dependencies>

2.讀hive數據

//創建SparkSession
val sparkSession: SparkSession = SparkSession.builder()
								.appName("MemberLogETL")
								.master("local[*]")
								.enableHiveSupport()
								.getOrCreate()
//導入隱式轉換
import sparkSession.implicits._
//切換數據庫
spark.sql("use db_hive_edu");
//讀取用戶表所有數據
Dataset<Row> data = spark.sql("select * from user");
//打印數據
data.show();
//關閉
sparkSession.close()

3.寫數據到hive

//正常寫入，表存在會報錯
Dataset.write.saveAsTable("dwd_member")
//覆蓋寫入
Dataset.write.mode(SaveMode.Overwrite).insertInto("dwd_member")

不使用自帶hive需要配置：hive-site.xml