spark pom文件 胖廋包結合

背景

平時會有同僚（尤其是新手），一直很疑惑，本地開發的spark應用都運行得好好的，一旦編譯文件之後就通不過，可能會有出現各種坑，其中就是使用sdt方式很容易導致項目使用的scala版本與本地編譯的版本起衝突
這裏使用maven-shade-plugin 用來編譯同時會生成兩個包

1. 其中 feature-1.0-SNAPSHOT.jar 爲胖包，是包含所有依賴的，可以在集羣中直接執行
2. original-feature-1.0-SNAPSHOT.jar 是隻有項目的class文件，即廋包，可以加快執行，在集羣中需要部署相關的依賴

pom.xml 以供參考

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.homewell.feature</groupId>
  <artifactId>feature</artifactId>
  <version>1.0-SNAPSHOT</version>
  <inceptionYear>2019</inceptionYear>
  <properties>
    <scala.version>2.11.8</scala.version>
    <spark.version>2.2.1</spark.version>
    <hbase.version>1.2.0</hbase.version>
    <hbase-spark.version>1.2.0-cdh5.9.0</hbase-spark.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <java.version>1.8</java.version>
    <cdh.spark.version>2.2.1</cdh.spark.version>
  </properties>

  <repositories>
    <!--阿里的地址-->
    <repository>
      <id>aliyun</id>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
    </repository>

    <repository>
      <id>cdh</id>
      <name>Cloudera Rel Repository</name>
      <url>https://repository.cloudera.com/content/repositories/releases/</url>
    </repository>

  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </pluginRepository>
  </pluginRepositories>

  <dependencies>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>

    <!--<dependency>-->
      <!--<groupId>org.specs</groupId>-->
      <!--<artifactId>specs</artifactId>-->
      <!--<version>1.2.5</version>-->
      <!--<scope>test</scope>-->
    <!--</dependency>-->



    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <version>${spark.version}</version>
    </dependency>


    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-client</artifactId>
      <version>1.2.0-cdh5.9.0</version>
    <!--<groupId>org.apache.hadoop</groupId>-->
    <!--<artifactId>hadoop-client</artifactId>-->
    <!--<version>2.6.0-cdh5.9.0</version>-->
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-server</artifactId>
      <version>1.2.0-cdh5.9.0</version>
      <!--<groupId>org.apache.hadoop</groupId>-->
      <!--<artifactId>hadoop-client</artifactId>-->
      <!--<version>2.6.0-cdh5.9.0</version>-->
    </dependency>

    <!-- 開發spark-sql的核心依賴 -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.11</artifactId>
      <version>${spark.version}</version>
    </dependency>

    <!--<dependency>-->
      <!--<groupId>org.apache.hbase</groupId>-->
      <!--<artifactId>hbase-spark</artifactId>-->
      <!--<version>${hbase-spark.version}</version>-->
    <!--</dependency>-->
    <!--<dependency>-->
      <!--<artifactId>hbase</artifactId>-->
      <!--<groupId>org.apache.hbase</groupId>-->
      <!--<version>1.2.0-cdh5.9.0</version>-->
    <!--</dependency>-->

    <!-- https://mvnrepository.com/artifact/com.hortonworks/shc-core -->
    <!--<dependency>-->
      <!--<groupId>com.hortonworks</groupId>-->
      <!--<artifactId>shc-core</artifactId>-->
      <!--<version>1.1.1-2.1-s_2.11</version>-->
    <!--</dependency>-->


  </dependencies>

  <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <testSourceDirectory>src/test/scala</testSourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <version>2.10.1</version>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <scalaVersion>${scala.version}</scalaVersion>
          <args>
            <arg>-target:jvm-1.8</arg>
          </args>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
          <encoding>UTF-8</encoding>
          <!--<com-->
          <compilerArguments>

            <extdirs>
              src/main/resource/lib
              <!-- 自定義包-->
              <!--<dependency>-->
              <!--<groupId>com.homewell.sparkjob2</groupId>-->
              <!--<artifactId>hlog</artifactId>-->
              <!--<version>1.0.0</version>-->
              <!--<scope>system</scope>-->
              <!--<systemPath>E:/microservicep/deep_learning/spark/sparkkmeans/src/main/resources/lib/spark-core_2.11-1.5.2.logging.jar</systemPath>-->
              <!--</dependency>-->
            </extdirs>
          </compilerArguments>
        </configuration>

      </plugin>
      <!--<plugin>-->
      <!--<groupId>org.apache.maven.plugins</groupId>-->
      <!--<artifactId>maven-assembly-plugin</artifactId>-->
      <!--<version>3.1.0</version>-->
      <!--&lt;!&ndash;<version>2.2-beta-5</version>&ndash;&gt;-->
      <!--<configuration>-->
      <!--<descriptorRefs>-->
      <!--<descriptorRef>jar-with-dependencies</descriptorRef>-->
      <!--</descriptorRefs>-->
      <!--<archive>-->
      <!--<manifest>-->
      <!--&lt;!&ndash;<addClasspath>true</addClasspath>&ndash;&gt;-->
      <!--<mainClass>com.homewell.sparkjob.avgstd.AvgStdApp</mainClass>-->
      <!--</manifest>-->
      <!--</archive>-->
      <!--</configuration>-->

      <!--<executions>-->
      <!--<execution>-->
      <!--<id>make-assembly</id>-->
      <!--<phase>package</phase>-->
      <!--<goals>-->
      <!--<goal>single</goal>-->
      <!--</goals>-->
      <!--</execution>-->
      <!--</executions>-->
      <!--</plugin>-->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-eclipse-plugin</artifactId>
        <configuration>
          <downloadSources>true</downloadSources>
          <buildcommands>
            <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
          </buildcommands>
          <additionalProjectnatures>
            <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
          </additionalProjectnatures>
          <classpathContainers>
            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
            <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
          </classpathContainers>
        </configuration>
      </plugin>
      <!--<plugin>-->
      <!--<groupId>org.apache.maven.plugins</groupId>-->
      <!--<artifactId>maven-compiler-plugin</artifactId>-->
      <!--<version>2.5.1</version>-->
      <!--</plugin>-->
      <!--<plugin>-->
      <!--<groupId>org.apache.maven.plugins</groupId>-->
      <!--<artifactId>maven-resources-plugin</artifactId>-->
      <!--<version>2.3</version>-->
      <!--</plugin>-->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.1.1</version>
        <configuration>

          <artifactSet>
            <!--<includes>-->
            <!--<include>redis.clients:jedis</include>-->
            <!--<include>org.apache.commons:commons-pool2</include>-->
            <!--</includes>-->
            <includes>
              <!--<include>org.apache.spark.Logging.Scala</include>-->
            </includes>
          </artifactSet>
          <filters>
            <filter>
              <artifact>*:*</artifact>
              <excludes>
                <exclude>META-INF/*.SF</exclude>
                <exclude>META-INF/*.DSA</exclude>
                <exclude>META-INF/*.RSA</exclude>
              </excludes>
            </filter>
          </filters>
          <!--<relocations>-->
          <!--<relocation>-->
          <!--<pattern>redis</pattern>-->
          <!--<shadedPattern>scyuan.maven.shaded.redis</shadedPattern>-->
          <!--</relocation>-->
          <!--<relocation>-->
          <!--<pattern>org.apache.commons</pattern>-->
          <!--<shadedPattern>scyuan.maven.shaded.org.apache.commons</shadedPattern>-->
          <!--</relocation>-->
          <!--</relocations>-->
        </configuration>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
    </plugins>

  </build>
  <reporting>
    <plugins>
      <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <configuration>
          <scalaVersion>${scala.version}</scalaVersion>
        </configuration>
      </plugin>

    </plugins>
  </reporting>
</project>