pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements.  See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License.  You may obtain a copy of the License at
~
~    http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>org.apache.spark</groupId>
  <artifactId>spark-streaming-sql-kinesis-connector_2.12</artifactId>
  <version>1.1.0</version>
  <packaging>jar</packaging>
  <name>Spark Structured Streaming Kinesis Connector</name>
  <description>Connector to read from and write into Kinesis from Structured Streaming Applications</description>


  <properties>
    <spark.version>3.4.1</spark.version>
    <scala.binary.version>2.12</scala.binary.version>
    <fasterxml.jackson.version>2.14.3</fasterxml.jackson.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <aws.sdkv2.version>2.19.33</aws.sdkv2.version>
    <aws.kpl.version>0.15.7</aws.kpl.version>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
  </properties>

  <dependencyManagement>
    <dependencies>
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>bom</artifactId>
        <version>${aws.sdkv2.version}</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>
    </dependencies>
  </dependencyManagement>

  <dependencies>

    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>kinesis</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>dynamodb</artifactId>
    </dependency>

    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>netty-nio-client</artifactId>
    </dependency>

    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>sts</artifactId>
    </dependency>

    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>apache-client</artifactId>
      <exclusions>
        <exclusion>
          <groupId>commons-logging</groupId>
          <artifactId>commons-logging</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-tags_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-aws</artifactId>
      <version>3.2.4</version>
      <scope>test</scope>
    </dependency>
<!--    <dependency>-->
<!--      <groupId>com.amazonaws</groupId>-->
<!--      <artifactId>amazon-kinesis-client</artifactId>-->
<!--      <version>1.14.8</version>-->
<!--    </dependency>-->
    <dependency>
      <groupId>com.amazonaws</groupId>
      <artifactId>aws-java-sdk</artifactId>
      <version>1.12.382</version>
    </dependency>
    <dependency>
      <groupId>com.amazonaws</groupId>
      <artifactId>amazon-kinesis-producer</artifactId>
      <version>${aws.kpl.version}</version>
    </dependency>
    <dependency>
      <groupId>com.google.protobuf</groupId>
      <artifactId>protobuf-java</artifactId>
      <version>3.21.7</version>
      <!--
         We are being explicit about version here and overriding the
         spark default of 2.5.0 because KCL appears to have introduced
         a dependency on protobuf 2.6.1 somewhere between KCL 1.4.0 and 1.6.1.
       -->
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
      <version>${fasterxml.jackson.version}</version>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-annotations</artifactId>
      <version>${fasterxml.jackson.version}</version>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
      <version>${fasterxml.jackson.version}</version>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.dataformat</groupId>
      <artifactId>jackson-dataformat-cbor</artifactId>
      <version>${fasterxml.jackson.version}</version>
    </dependency>
    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-core</artifactId>
      <version>4.8.0</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalacheck</groupId>
      <artifactId>scalacheck_${scala.binary.version}</artifactId>
      <version>1.16.0</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.binary.version}</artifactId>
      <version>3.2.13</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-reload4j</artifactId>
      <version>2.0.6</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-slf4j-impl</artifactId>
      <version>2.19.0</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-slf4j2-impl</artifactId>
      <version>2.19.0</version>
      <scope>test</scope>
    </dependency>
    
    <!--
      This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
      them will yield errors.
    -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-tags_${scala.binary.version}</artifactId>
      <type>test-jar</type>
      <scope>test</scope>
      <version>${spark.version}</version>
    </dependency>

    <dependency>
      <groupId>com.amazonaws</groupId>
      <artifactId>amazon-kinesis-aggregator</artifactId>
      <version>1.0.3</version>
      <exclusions>
        <exclusion>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-s3</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-cloudwatch</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-dynamodb</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-kinesis</artifactId>
        </exclusion>
      </exclusions>
      <scope>test</scope>
    </dependency>
  </dependencies>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.scalatest</groupId>
          <artifactId>scalatest-maven-plugin</artifactId>
          <version>2.0.0</version>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>net.alchim31.maven</groupId>
          <artifactId>scala-maven-plugin</artifactId>
          <version>4.3.0</version>
          <executions>
            <execution>
              <id>compile</id>
              <goals>
                <goal>compile</goal>
                <goal>add-source</goal>
<!--                <goal>doc-jar</goal>-->
              </goals>
              <phase>compile</phase>
            </execution>
            <execution>
              <id>test-compile</id>
              <goals>
                <goal>testCompile</goal>
              </goals>
              <phase>test-compile</phase>
            </execution>
            <execution>
              <phase>process-resources</phase>
              <goals>
                <goal>compile</goal>
              </goals>
            </execution>
          </executions>
          <configuration>
            <args>
              <arg>-nobootcp</arg>
            </args>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-jar-plugin</artifactId>
          <version>3.1.0</version>
          <executions>
            <execution>
              <goals>
                <goal>test-jar</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>3.8.1</version>
          <executions>
            <execution>
              <phase>compile</phase>
              <goals>
                <goal>compile</goal>
              </goals>
            </execution>
          </executions>
          <configuration>
            <source>${maven.compiler.source}</source>
            <target>${maven.compiler.target}</target>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-shade-plugin</artifactId>
          <version>3.2.1</version>
          <executions>
            <execution>
              <phase>package</phase>
              <goals>
                <goal>shade</goal>
              </goals>
              <configuration>
                <artifactSet>
                  <includes>
                    <include>com.amazonaws:amazon-kinesis-producer:*</include>
                    <include>com.amazonaws:aws-java-sdk-kinesis:*</include>
                    <include>com.amazonaws:aws-java-sdk-core:*</include>
                    <include>software.amazon.glue:schema-registry-serde:*</include>
                    <include>software.amazon.awssdk:sts:*</include>
                    <include>software.amazon.awssdk:kinesis:*</include>
                    <include>software.amazon.awssdk:dynamodb:*</include>
                    <include>software.amazon.awssdk:sdk-core:*</include>
                    <include>software.amazon.awssdk:utils:*</include>
                    <include>software.amazon.awssdk:annotations:*</include>
                    <include>software.amazon.awssdk:apache-client:*</include>
                    <include>software.amazon.awssdk:arns:*</include>
                    <include>software.amazon.awssdk:auth:*</include>
                    <include>software.amazon.eventstream:eventstream:*</include>
                    <include>software.amazon.awssdk:aws-cbor-protocol:*</include>
                    <include>software.amazon.awssdk:aws-core:*</include>
                    <include>software.amazon.awssdk:aws-query-protocol:*</include>
                    <include>software.amazon.awssdk:aws-json-protocol:*</include>
                    <include>software.amazon.awssdk:endpoints-spi:*</include>
                    <include>software.amazon.awssdk:http-client-spi:*</include>
                    <include>software.amazon.awssdk:json-utils:*</include>
                    <include>software.amazon.awssdk:metrics-spi:*</include>
                    <include>software.amazon.awssdk:netty-nio-client:*</include>
                    <include>software.amazon.awssdk:profiles:*</include>
                    <include>software.amazon.awssdk:protocol-core:*</include>
                    <include>software.amazon.awssdk:regions:*</include>
                    <include>software.amazon.awssdk:third-party-jackson-core:*</include>
                    <include>software.amazon.awssdk:third-party-jackson-dataformat-cbor:*</include>
                    <include>io.dropwizard.metrics:metrics-core:*</include>
                    <include>org.reactivestreams:reactive-streams:*</include>
                    <include>io.netty:netty-common:*</include>
                    <include>io.netty:netty-buffer:*</include>
                    <include>io.netty:netty-codec:*</include>
                    <include>io.netty:netty-codec-http:*</include>
                    <include>io.netty:netty-codec-http2:*</include>
                    <include>io.netty:netty-handler:*</include>
                    <include>io.netty:netty-resolver:*</include>
                    <include>io.netty:netty-transport:*</include>
                    <include>io.netty:netty-transport-classes-epoll:*</include>
                    <include>io.netty:netty-transport-native-unix-common:*</include>
                    <include>com.fasterxml.jackson.core:*</include>
                    <include>com.fasterxml.jackson.dataformat:*</include>
                    <include>com.google.protobuf:*:*</include>
                    <include>com.google.guava:*:*</include>
                  </includes>
                </artifactSet>
                <relocations>
                  <relocation>
                    <pattern>com.fasterxml.jackson</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.com.fasterxml.jackson</shadedPattern>
                    <includes>
                      <include>com.fasterxml.jackson.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>com.amazonaws.services</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.com.amazonaws.services</shadedPattern>
                    <includes>
                      <include>com.amazonaws.services.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>com.amazonaws</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.com.amazonaws</shadedPattern>
                    <includes>
                      <include>com.amazonaws.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>software.amazon.awssdk</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.software.amazon.awssdk</shadedPattern>
                    <includes>
                      <include>software.amazon.awssdk.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>software.amazon.eventstream</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.software.amazon.eventstream</shadedPattern>
                    <includes>
                      <include>software.amazon.eventstream.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>com.google.protobuf</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.google.protobuf</shadedPattern>
                    <includes>
                      <include>com.google.protobuf.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>com.google.common</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.google.common</shadedPattern>
                    <includes>
                      <include>com.google.common.**</include>
                    </includes>
                  </relocation>
                  <relocation>
                    <pattern>com.codahale.metrics</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.com.codahale.metrics</shadedPattern>
                  </relocation>
                  <relocation>
                    <pattern>org.reactivestreams</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.org.reactivestreams</shadedPattern>
                  </relocation>
                  <relocation>
                    <pattern>io.netty</pattern>
                    <shadedPattern>org.apache.spark.sql.connector.kinesis.shaded.io.netty</shadedPattern>
                  </relocation>
                </relocations>
                <filters>
                  <filter>
                    <artifact>*:*</artifact>
                    <excludes>
                      <exclude>META-INF/LICENSE*</exclude>
                      <exclude>META-INF/NOTICE*</exclude>
                      <exclude>META-INF/DEPENDENCIES</exclude>
                      <exclude>META-INF/maven/**</exclude>
                      <exclude>META-INF/*.xml</exclude>
                      <exclude>META-INF/*.SF</exclude>
                      <exclude>META-INF/*.DSA</exclude>
                      <exclude>META-INF/*.RSA</exclude>
                      <exclude>models/*</exclude>
                      <exclude>.gitkeep</exclude>
                    </excludes>
                  </filter>
                </filters>
                <transformers>
                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                  <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                    <resource>log4j.properties</resource>
                  </transformer>
                  <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
                  <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer" />
                </transformers>
              </configuration>
            </execution>
          </executions>
        </plugin>
      </plugins>
    </pluginManagement>
    <!-- Enable surefire and scalatest in all children, in one place: -->
    <plugins>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest-maven-plugin</artifactId>
      </plugin>
    </plugins>

  <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
  <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
  </build>

  <profiles>
    <profile>
      <id>release</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-source-plugin</artifactId>
            <version>3.2.1</version>
            <executions>
              <execution>
                <id>attach-sources</id>
                <goals>
                  <goal>jar-no-fork</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-gpg-plugin</artifactId>
            <version>1.5</version>
            <executions>
              <execution>
                <id>sign-artifacts</id>
                <phase>verify</phase>
                <goals>
                  <goal>sign</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-javadoc-plugin</artifactId>
            <version>2.10.1</version>
            <executions>
              <execution>
                <id>create-javadoc-jar</id>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-release-plugin</artifactId>
            <version>2.5.1</version>
            <configuration>
              <autoVersionSubmodules>true</autoVersionSubmodules>
            </configuration>
          </plugin>
        </plugins>
      </build>
    </profile>
    <profile>
      <id>default-test</id>
      <activation>
        <activeByDefault>true</activeByDefault>
      </activation>
      <build>
        <plugins>
          <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
          </plugin>
          <plugin>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest-maven-plugin</artifactId>
            <configuration>
              <reportsDirectory>${project.build.directory}/unittest-reports</reportsDirectory>
              <junitxml>.</junitxml>
              <filereports>WDF UnitTestSuite.txt</filereports>
              <tagsToExclude>org.apache.spark.sql.connector.kinesis.it.IntegrationTestSuite</tagsToExclude>
            </configuration>
            <executions>
              <execution>
                <id>test</id>
                <goals>
                  <goal>test</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
    <profile>
      <id>integration-test</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <build>
        <plugins>
          <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
          </plugin>
          <plugin>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest-maven-plugin</artifactId>
            <configuration>
              <reportsDirectory>${project.build.directory}/integrationtest-reports</reportsDirectory>
              <junitxml>.</junitxml>
              <filereports>WDF IntegrationTestSuite.txt</filereports>
              <tagsToInclude>org.apache.spark.sql.connector.kinesis.it.IntegrationTestSuite</tagsToInclude>
            </configuration>
            <executions>
              <execution>
                <id>test</id>
                <goals>
                  <goal>test</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
    <profile>
      <id>java-8-compatible</id>
      <activation>
        <jdk>[9,)</jdk>
      </activation>
      <properties>
        <maven.compiler.release>8</maven.compiler.release>
      </properties>
      <build>
        <plugins>
          <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
          </plugin>
          <plugin>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest-maven-plugin</artifactId>
            <configuration>
              <reportsDirectory>${project.build.directory}/unittest-reports</reportsDirectory>
              <junitxml>.</junitxml>
              <filereports>WDF UnitTestSuite.txt</filereports>
              <tagsToExclude>org.apache.spark.sql.connector.kinesis.it.IntegrationTestSuite</tagsToExclude>
            </configuration>
            <executions>
              <execution>
                <id>test</id>
                <goals>
                  <goal>test</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
    
  </profiles>

  <distributionManagement>
    <snapshotRepository>
      <id>ossrh</id>
      <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
    </snapshotRepository>
    <repository>
      <id>ossrh</id>
      <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
    </repository>
  </distributionManagement>

  <reporting>
    <plugins>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>cobertura-maven-plugin</artifactId>
        <version>2.7</version>
      </plugin>
    </plugins>
  </reporting>
</project>