when I run my tests in Intellij idea choosing code coverage tool as JaCoCo and include my packages I see I get 80% above coverage in the report but when I run it using maven command line I get 0% in JaCoCo report below are two questions.
can I see what command Intellij Idea Ultimate version is using to run my unit tests with code coverage ?
Why my maven command mvn clean test jacoco:report is showing my coverage percentage as 0%.
This is a Scala maven project.
My POM.xml file:-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.blah</groupId>
<artifactId>blah</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.blah.com </sonar.host.url>
<sonar.projectKey>blah</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
</properties>
<scm>
<connection>scm:git:https://github.cloud.blah.com/BB/blah-project</connection>
<url>https://github.cloud.blah.com/BB/blah-project</url>
<developerConnection>scm:git:https://github.cloud.blah.com/BB/blah-project
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>blah-company One Artifactory-releases</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>blah-company One Artifactory-snapshots</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<!-- setting cobertura and jacoco -->
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<scope>test</scope>
</dependency>
<!--
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>io.github.embeddedkafka</groupId>
<artifactId>embedded-kafka_2.11</artifactId>
<version>2.4.0</version>
</dependency>
https://github.com/scallop/scallop
<dependency>
<groupId>org.rogach</groupId>
<artifactId>scallop</artifactId>
<version>4.0.1</version>
</dependency>
https://github.com/ben-manes/caffeine
https://code.google.com/archive/p/findbugs-tutorials/
//find bugs
-->
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<dataFile>${project.build.directory}/jacoco.exec</dataFile>
<outputDirectory>${project.build.directory}/jacoco-ut</outputDirectory>
</configuration>
</execution>
</executions>
<configuration/>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<minimumCoverage>0</minimumCoverage>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>
Thanks
Assuming that you are using JaCoCo with cobertura coverage you need to declare the dependencies and the plugin to run the command mvn cobertura:cobertura.
<dependencies>
<!-- setting cobertura and jacoco -->
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.7</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- setting cobertura and jacoco plugin -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.7</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
<check />
</configuration>
</plugin>
<plugins>
</build>
Create a codecov.yml on the root of your project.
coverage:
precision: 2
round: nearest
range: "40...70"
Then use the comands to generate the report and upload it. Note that you also need to export your secret token.
export CODECOV_TOKEN=your_secret_token
mvn cobertura:cobertura
bash <(curl -s https://codecov.io/bash)
I end up using scoverage to create code coverage and code coverage check , below is my pom.xml file.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.company.apollo</groupId>
<artifactId>myproject</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.company.com </sonar.host.url>
<sonar.projectKey>myproject</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
<stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
<grpc.version>1.24.0</grpc.version>
<protobuf.version>3.6.1</protobuf.version>
<guava.version>26.0-android</guava.version>
<spark.version>2.4.5</spark.version>
</properties>
<scm>
<connection>scm:git:https://github.cloud.company.com/blah</connection>
<url>https://github.cloud.company.com/blah</url>
<developerConnection>scm:git:https://github.cloud.company.com/blah
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>Company Artifactory-releases</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>Company Artifactory-snapshots</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<dependency>
<groupId>com.company.onelake</groupId>
<artifactId>onelake-s3-client_hadoop-2.8_2.11</artifactId>
<version>1.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-producer-spark</artifactId>
<version>${stream-poc.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-sink-kafka-deserializer</artifactId>
<version>${stream-poc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>os-lib_2.11</artifactId>
<version>0.2.9</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>com</pattern>
<shadedPattern>shaded.com</shadedPattern>
<includes>
<include>com.google.common.**</include>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
</relocations>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
<excludedPackages>
misc
</excludedPackages>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>
ok I end up using scoverage for generating code coverage report and checks.
Code Coverage Check:-
mvn scoverage:check
mvn scoverage:report
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.company.apollo</groupId>
<artifactId>myproject</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.company.com </sonar.host.url>
<sonar.projectKey>myproject</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
<stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
<grpc.version>1.24.0</grpc.version>
<protobuf.version>3.6.1</protobuf.version>
<guava.version>26.0-android</guava.version>
<spark.version>2.4.5</spark.version>
</properties>
<scm>
<connection>scm:git:https://github.cloud.company.com/blah</connection>
<url>https://github.cloud.company.com/blah</url>
<developerConnection>scm:git:https://github.cloud.company.com/blah
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>Company Artifactory-releases</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>Company Artifactory-snapshots</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<dependency>
<groupId>com.company.onelake</groupId>
<artifactId>onelake-s3-client_hadoop-2.8_2.11</artifactId>
<version>1.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-producer-spark</artifactId>
<version>${stream-poc.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-sink-kafka-deserializer</artifactId>
<version>${stream-poc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>os-lib_2.11</artifactId>
<version>0.2.9</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>com</pattern>
<shadedPattern>shaded.com</shadedPattern>
<includes>
<include>com.google.common.**</include>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
</relocations>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
<excludedPackages>
misc
</excludedPackages>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>
Related
This is my pom.xml:
<?xml version='1.0' encoding='UTF-8'?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>default</groupId>
<artifactId>factory-servicing</artifactId>
<packaging>jar</packaging>
<description>factory-servicing</description>
<version>0.1</version>
<name>factory-servicing</name>
<organization>
<name>default</name>
</organization>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
</dependency>
<dependency>
<groupId>org.scalikejdbc</groupId>
<artifactId>scalikejdbc_2.12</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.2</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.12</artifactId>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-core_${scala.compat.version}</artifactId>
<version>${spec2.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-junit_${scala.compat.version}</artifactId>
<version>${spec2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.3.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.21.0</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>TestSuiteReport.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>uber-jar</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.dep.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.dep.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
</dependency>
<dependency>
<groupId>org.scalikejdbc</groupId>
<artifactId>scalikejdbc_2.12</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.2</version>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
As you can see, for the outer dependencies stage, I don't have anything scoped to provided. This is because I need all libraries to run my test cases.
However, for the package stage's dependencies, which uses the maven-shade-plugin, I'm providing the spark dependencies as provided, whereas the rest of the libraries from the outer dependencies remain unchanged. This is because for Spark we will be providing the JARs, but for the rest we still need it in the packaged jar.
Then when I run mvn package, it generates a JAR file, which I go on to extract. However, on extracting the JAR file, I only see the class files that I had written in my project, and I don't see any of the other libraries that are not scoped to provided.
How do I ensure that the non-provided libraries are also included as a part of the mvn package command execution?
I'm running scala/spark application on Databricks cluster (it tries to read data from an Event Hub topic using EventHub schemaregistry) and getting the following error:
22/02/17 07:36:14 ERROR JacksonVersion: Version '1.0.19-SNAPSHOT' of package 'jackson-dataformat-xml' is not supported (older than earliest supported version - `2.10.0`), please upgrade.
22/02/17 07:36:14 ERROR JacksonVersion: Version '1.0.19-SNAPSHOT' of package 'jackson-datatype-jsr310' is not supported (older than earliest supported version - `2.10.0`), please upgrade.
22/02/17 07:36:14 INFO JacksonVersion: Package versions: jackson-annotations=2.12.3, jackson-core=2.12.3, jackson-databind=2.12.3, jackson-dataformat-xml=1.0.19-SNAPSHOT, jackson-datatype-jsr310=1.0.19-SNAPSHOT, azure-core=1.8.1, Troubleshooting version conflicts: https://aka.ms/azsdk/java/dependency/troubleshoot
22/02/17 07:36:14 ERROR ScalaDriverLocal: User Code Stack Trace:
java.lang.NoSuchMethodError: reactor.netty.http.client.HttpClient.resolver(Lio/netty/resolver/AddressResolverGroup;)Lreactor/netty/transport/ClientTransport;
Looks that maven is trying to use main artificat version (1.0.19-SNAPSHOT) to 'jackson-dataformat-xml' and 'jackson-datatype-jsr310':
22/02/17 07:36:14 ERROR JacksonVersion: Version '1.0.19-SNAPSHOT' of package 'jackson-dataformat-xml' is not supported (older than earliest supported version - `2.10.0`), please upgrade.
22/02/17 07:36:14 ERROR JacksonVersion: Version '1.0.19-SNAPSHOT' of package 'jackson-datatype-jsr310' is not supported (older than earliest supported version - `2.10.0`), please upgrade.
The project uses Databricks 10.3 runtime and this pom:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>group</groupId>
<artifactId>parent</artifactId>
<version>3.1-SNAPSHOT</version>
</parent>
<artifactId>ingestion</artifactId>
<version>${revision}</version>
<packaging>jar</packaging>
<name>ingestion</name>
<organization>
<name>company</name>
</organization>
<properties>
<revision>1.0.19-SNAPSHOT</revision>
<encoding>UTF-8</encoding>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.compact.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.compact.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.compact.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_${scala.compact.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.delta</groupId>
<artifactId>delta-core_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.compact.version}</artifactId>
<version>3.2.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-scala_${scala.compact.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-json_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>dbutils-api_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>com.github.mrpowers</groupId>
<artifactId>spark-daria_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>com.github.mrpowers</groupId>
<artifactId>spark-fast-tests_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
<groupId>io.circe</groupId>
<artifactId>circe-yaml_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>io.circe</groupId>
<artifactId>circe-generic_${scala.compact.version}</artifactId>
</dependency>
<dependency>
<groupId>io.jvm.uuid</groupId>
<artifactId>scala-uuid_2.12</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
<version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-data-schemaregistry</artifactId>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
</resource>
<resource>
<directory>src/main/resources-filtered</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<jvmArgs>
<jvmArg>-Xms64m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
</jvmArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<manifestEntries>
<Implementation-Title>${project.artifactId}</Implementation-Title>
<Implementation-Version>${project.version}</Implementation-Version>
<Specification-Vendor>${project.groupId}</Specification-Vendor>
<Specification-Title>${project.artifactId}</Specification-Title>
<Implementation-Vendor-Id>${project.groupId}</Implementation-Vendor-Id>
<Specification-Version>${project.version}</Specification-Version>
<Implementation-Vendor>${project.groupId}</Implementation-Vendor>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.2.7</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<finalName>${project.artifactId}-${project.version}-assembly</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifestEntries>
<Implementation-Title>${project.artifactId}</Implementation-Title>
<Implementation-Version>${project.version}</Implementation-Version>
<Specification-Vendor>${project.groupId}</Specification-Vendor>
<Specification-Title>${project.artifactId}</Specification-Title>
<Implementation-Vendor-Id>${project.groupId}</Implementation-Vendor-Id>
<Specification-Version>${project.version}</Specification-Version>
<Implementation-Vendor>${project.groupId}</Implementation-Vendor>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
and this is the parent project:
<groupId>group</groupId>
<artifactId>parent</artifactId>
<version>3.1-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<scala.version>2.12.14</scala.version>
<scala.compact.version>2.12</scala.compact.version>
<spark.version>3.2.1</spark.version>
<databricks.dbutils.api.version>0.0.5</databricks.dbutils.api.version>
<delta.version>1.1.0</delta.version>
<postgresql.version>42.2.9</postgresql.version>
<typesafe.config.version>1.4.0</typesafe.config.version>
<snakeyaml.version>1.25</snakeyaml.version>
<spray-json.version>1.3.5</spray-json.version>
<h2.version>1.4.200</h2.version>
<mockito.version>1.16.46</mockito.version>
<scalacheck.version>1.14.0</scalacheck.version>
<adal4j.version>1.6.6</adal4j.version>
<scalastyle.version>0.8.0</scalastyle.version>
<maven-surefire-plugin.version>2.7</maven-surefire-plugin.version>
<scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
<spark-daria.version>1.2.3</spark-daria.version>
<spark-fast-tests.version>1.1.0</spark-fast-tests.version>
<kafka.version>3.1.0</kafka.version>
<scala-maven-plugin.version>4.4.0</scala-maven-plugin.version>
<cats-core.version>2.0.0</cats-core.version>
<pureconfig_2.11.version>0.12.3</pureconfig_2.11.version>
<jgitflow-maven-plugin.version>1.0-m5.1</jgitflow-maven-plugin.version>
<joda-time.version>2.10.6</joda-time.version>
<mssql-jdbc.version>8.2.2.jre8</mssql-jdbc.version>
<circe-yaml.artifact>circe-yaml_2.12</circe-yaml.artifact>
<circe-yaml.version>0.13.1</circe-yaml.version>
<circe-generic.artifact>circe-generic_2.12</circe-generic.artifact>
<circe-generic.version>0.13.0</circe-generic.version>
<scala-uuid_2.12.version>0.3.1</scala-uuid_2.12.version>
<spark-testing-base.version>0.14.0</spark-testing-base.version>
<scalatest.version>3.0.5</scalatest.version>
<httpclient.version>4.5.6</httpclient.version>
<azure-identity.version>1.4.0</azure-identity.version>
<azure-core.version>1.23.0</azure-core.version>
<azure-data-schemaregistry.version>1.0.0</azure-data-schemaregistry.version>
<log4j-core.version>2.17.1</log4j-core.version>
<commons-cli.version>1.2</commons-cli.version>
<reactor-netty.version>1.0.15</reactor-netty.version>
<azure-core-http-netty.version>1.11.7</azure-core-http-netty.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j-core.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.compact.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.compact.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.compact.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.compact.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>${kafka.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_${scala.compact.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>adal4j</artifactId>
<version>${adal4j.version}</version>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>dbutils-api_2.12</artifactId>
<version>${databricks.dbutils.api.version}</version>
<exclusions>
<exclusion>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>${joda-time.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.delta</groupId>
<artifactId>delta-core_${scala.compact.version}</artifactId>
<version>${delta.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>${typesafe.config.version}</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency>
<dependency>
<groupId>io.circe</groupId>
<artifactId>${circe-yaml.artifact}</artifactId>
<version>${circe-yaml.version}</version>
</dependency>
<dependency>
<groupId>io.circe</groupId>
<artifactId>${circe-generic.artifact}</artifactId>
<version>${circe-generic.version}</version>
</dependency>
<dependency>
<groupId>io.jvm.uuid</groupId>
<artifactId>scala-uuid_2.12</artifactId>
<version>${scala-uuid_2.12.version}</version>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-json_${scala.compact.version}</artifactId>
<version>${spray-json.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-scala_${scala.compact.version}</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.typelevel</groupId>
<artifactId>cats-core_2.12</artifactId>
<version>${cats-core.version}</version>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.compact.version}</artifactId>
<version>${scalacheck.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_${scala.compact.version}
<version>2.4.5_${spark-testing-base.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.compact.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.mrpowers</groupId>
<artifactId>spark-fast-tests_${scala.compact.version}</artifactId>
<version>${spark-fast-tests.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.mrpowers</groupId>
<artifactId>spark-daria_${scala.compact.version}</artifactId>
<version>${spark-daria.version}</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
<version>${azure-identity.version}</version>
<exclusions>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core</artifactId>
<version>${azure-core.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</exclusion>
<exclusion>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
<version>${azure-core-http-netty.version}</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-data-schemaregistry</artifactId>
<version>${azure-data-schemaregistry.version}</version>
<exclusions>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>external.atlassian.jgitflow</groupId>
<artifactId>jgitflow-maven-plugin</artifactId>
<version>${jgitflow-maven-plugin.version}</version>
<configuration>
<pushFeatures>true</pushFeatures>
<pushHotfixes>true</pushHotfixes>
<pushReleases>true</pushReleases>
<enableSshAgent>true</enableSshAgent>
<noDeploy>true</noDeploy>
<flowInitContext>
<masterBranchName>master</masterBranchName>
<developBranchName>develop</developBranchName>
<featureBranchPrefix>feature/</featureBranchPrefix>
<releaseBranchPrefix>release/</releaseBranchPrefix>
<hotfixBranchPrefix>hotfix/</hotfixBranchPrefix>
<versionTagPrefix>${project.artifactId}-</versionTagPrefix>
</flowInitContext>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>${scala-maven-plugin.version}</version>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmArgs>
<jvmArg>-Xms1024M</jvmArg>
<jvmArg>-Xmx2048M</jvmArg>
<jvmArg>-XX:+CMSClassUnloadingEnabled</jvmArg>
</jvmArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
<version>${scalastyle.version}</version>
<configuration>
<verbose>true</verbose>
<failOnViolation>true</failOnViolation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<failOnWarning>false</failOnWarning>
<sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
<testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
<configLocation>${project.basedir}/scalastyle_config.xml</configLocation>
<outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
<outputEncoding>UTF-8</outputEncoding>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>dl.binatry.com</id>
<url>http://dl.bintray.com/spark-packages/maven</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
When I run it locally it works fine, the issue arrives when trying to run it in the cluster.
I try to exclude some conflict versions artificats but the issue persists.
Any help?
I have created a simple Spark application in Scala which is running fine locally. I have used Maven as a build tool and am packaging the JAR file using shade plugin. The directory structure looks like this:
I am using the following pom.xml :
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>cl.aman.fund</groupId>
<artifactId>sym_data_decryptor</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<encoding>UTF-8</encoding>
<scala.version>2.11.12</scala.version>
</properties>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<resources>
<resource>
<directory>/src/resources/</directory>
<filtering>false</filtering>
<includes>
<include>hbase-site.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<!-- see http://davidb.github.com/scala-maven-plugin -->
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.3.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<!-- <arg>-make:transitive</arg> -->
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.5</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<shadedArtifactAttached>true</shadedArtifactAttached>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<!-- Scala and Spark dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.scala-logging</groupId>
<artifactId>scala-logging_2.11</artifactId>
<version>3.9.2</version>
</dependency>
<!-- Spark avro -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>2.4.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcs-connector</artifactId>
<version>hadoop2-1.9.17</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>bigquery-connector</artifactId>
<version>hadoop2-0.13.9</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
<version>5.2.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-xml</artifactId>
<version>2.11.0-M4</version>
</dependency>
<dependency>
<groupId>org.scalactic</groupId>
<artifactId>scalactic_2.11</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock-scalatest-support_2.11</artifactId>
<version>3.6.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-native_2.11</artifactId>
<version>3.6.6</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.13.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api-scala_2.12</artifactId>
<version>11.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_2.11</artifactId>
<version>1.14.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Command used to submit job on Dataproc cluster:
gcloud dataproc jobs submit spark \
--cluster <cluster_name> \
--region <region> \
--class cl.aman.symphony.commons.DecryptorApplication \
--jars gs://<bucket_name>/sym_data_decryptor-0.0.1-SNAPSHOT-jar-with-dependencies.jar \
-- \
"my_Input"
Error:
Exception in thread "main" java.lang.NoSuchMethodError: org.json4s.Serialization$class.read(Lorg/json4s/Serialization;Ljava/lang/String;Lorg/json4s/Formats;Lscala/reflect/Manifest;)Ljava/lang/Object;
at org.json4s.native.Serialization$.read(Serialization.scala:32)
at cl.falabella.symphony.commons.DecryptorApplication$.makeRequest(DecryptorApplication.scala:44)
at cl.falabella.symphony.commons.DecryptorApplication$.main(DecryptorApplication.scala:65)
at cl.falabella.symphony.commons.DecryptorApplication.main(DecryptorApplication.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:890)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:192)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:217)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Can someone help me with the Scala JAR packaging ?
You are using json4s-native_2.11 3.6.6 but spark-core 2.4.4 which is core of Spark framework uses json4s in version 3.5.3 which makes json4s core incompatible. I recommend switching to json4s version 3.5.3
Your plugin for dependencies might be wrong set.
Mine has some differences and works perfectly.
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
Remember to add in your spark-submit "-jar-with-dependencies" at your jar name.
I am getting this error when running the Spotify Spark Bigquery connector on Qubole data platform. I do see the BigQueryUtils class in my jar but still it throws this error:
Exception in thread "main"
org.spark-project.guava.util.concurrent.ExecutionError:
java.lang.NoSuchMethodError:
com.google.cloud.hadoop.io.bigquery.BigQueryUtils.waitForJobCompletion
Attaching the pom below...
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xyz.abc.google.TestProject</groupId>
<artifactId>edesem-google-TestProject</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<gpg.skip>true</gpg.skip>
<!-- Keep in sync with google-api-client dependency -->
<apache.httpcomponents.version>4.0.1</apache.httpcomponents.version>
</properties>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.3.1</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<!-- Maven Shade Plugin -->
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>edesem-google-TestProject</finalName>
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>log4j.properties</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.xyz.abc.bigquery.TestProjectBQClient</mainClass>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>org.eclipse.jetty</pattern>
<shadedPattern>org.spark-project.jetty</shadedPattern>
<includes>
<include>org.eclipse.jetty.**</include>
</includes>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.spark-project.guava</shadedPattern>
<excludes>
<exclude>com/google/common/base/Absent*</exclude>
<exclude>com/google/common/base/Function</exclude>
<exclude>com/google/common/base/Optional*</exclude>
<exclude>com/google/common/base/Present*</exclude>
<exclude>com/google/common/base/Supplier</exclude>
</excludes>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.6</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>2.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>2.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>spark-avro_2.10</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>bigquery-connector</artifactId>
<version>0.10.2-hadoop2</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.21</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.3</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcs-connector</artifactId>
<version>1.8.0-hadoop2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/util-hadoop -->
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util-hadoop</artifactId>
<version>1.8.0-hadoop2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcsio -->
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcsio</artifactId>
<version>1.8.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util</artifactId>
<version>1.8.0</version>
<exclusions>
<exclusion>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client-java6</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.8.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.6-jre</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-bigquery -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigquery</artifactId>
<version>1.23.0</version>
</dependency>
</dependencies>
</project>
I figured the main issue for me was the big query connector config in the cluster. I added the jar to the classpath and it fixed the issue. Below instructions as per Google's documentation.
https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md#add-the-connector-jar-to-hadoops-classpath
Add the connector jar to Hadoop's classpath
Placing the connector jar in the appropriate subdirectory of the Hadoop installation may be effective to have Hadoop load the jar. However, to be certain that the jar is loaded, add HADOOP_CLASSPATH=$HADOOP_CLASSPATH:</path/to/gcs-connector-jar> to hadoop-env.sh in the Hadoop configuration directory.
This is because you are using com.google.cloud.bigdataoss:bigquery-connector:0.10.2-hadoop2 BigQuery connector version which is incompatible with com.google.cloud:google-cloud-bigquery:1.23.0 library version.
You need to upgrade com.google.cloud.bigdataoss:bigquery-connector to at least 0.11.0 version, and make it consistent with versions of other com.google.cloud.bigdataoss dependencies (in your case it will be 0.12.0 version), i.e. they all should be from the same release that are listed here: https://github.com/GoogleCloudPlatform/bigdata-interop/releases
eclipse build failed and i got like this:-
Failed to execute goal on project : Could not resolve dependencies for project com.ericsson:JCA-CA-SADM:jar:1.0.0: Failed to collect dependencies at org.apache.maven.plugins:maven-resources-plugin:jar:2.5 -> org.apache.maven:maven-project:jar:2.0.6 -> org.apache.maven:maven-artifact-manager:jar:2.0.6 -> org.apache.maven.wagon:wagon-provider-api:jar:1.0-beta-2: Failed to read artifact descriptor for org.apache.maven.wagon:wagon-provider-api:jar:1.0-beta-2: Could not transfer artifact org.apache.maven.wagon:wagon:pom:1.0-beta-2 from/to public (http://repo1.maven.org/maven2/): Access denied to http://repo1.maven.org/maven2/org/apache/maven/wagon/wagon/1.0-beta-2/wagon-1.0-beta-2.pom. Error code 403, Forbidden -> [Help 1]
this is my pom.xml
http://maven.apache.org/maven-v4_0_0.xsd">
4.0.0
<!-- <parent> <groupId>com.ericsson</groupId> <artifactId>ca</artifactId>
<version>1.0.87-SNAPSHOT</version> <relativePath>../..</relativePath> </parent> -->
<parent>
<groupId>com.ericsson</groupId>
<artifactId>ca</artifactId>
<version>1.0.125</version>
<relativePath>../..</relativePath>
</parent>
<artifactId>JCA-CA-SADM</artifactId>
<name>JCA-CA-SADM</name>
<version>1.0.0</version>
<description>This is SADM Connection</description>
<!-- <description>This component is an example how to use the Customer Adatation
Support API. It provides an example on how to develop a new outbound connector.</description> -->
<!-- Skips the enforcer rules from parent pom -->
<properties>
<enforcer.skip>true</enforcer.skip>
<cxf.version>2.2.3</cxf.version>
</properties>
<!-- ========================================== = The DEPENDENCIES section
========================================== -->
<dependencies>
<dependency>
<groupId>javax.resource</groupId>
<artifactId>connector-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.axis</groupId>
<artifactId>axis</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>javax.xml</groupId>
<artifactId>jaxrpc-api</artifactId>
<version>1.1</version>
</dependency>
<!-- <dependency> <groupId>com.ericsson</groupId> <artifactId>Test-HTTP-Server</artifactId>
<version>1.0.87-SNAPSHOT</version> <scope>test</scope> </dependency> -->
<!-- Jetty is needed if you're are not using the CXFServlet -->
<!-- <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-frontend-jaxws</artifactId>
<version>${cxf.version}</version> </dependency> <dependency> <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId> <version>${cxf.version}</version>
</dependency> <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-transports-http-jetty</artifactId>
<version>${cxf.version}</version> </dependency> -->
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>META-INF/ra.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>com.ericsson.mvn</groupId>
<artifactId>tar-manifest-generator</artifactId>
<version>1.2</version>
<executions>
<execution>
<goals>
<goal>build</goal>
</goals>
</execution>
</executions>
<configuration>
<id>CXC1736209</id>
<version>${project.version}</version>
<name>${project.artifactId}</name>
</configuration>
</plugin>
<!-- <plugin> ADDED START <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId> <executions> The m2e eclipse
plugin complains but it's valid and works cmd line <execution> <id>copy-dependencies</id>
<phase>package</phase> <goals> <goal>copy-dependencies</goal> </goals> <configuration>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
<includeScope>runtime</includeScope> <excludeTransitive>true</excludeTransitive>
</configuration> </execution> </executions> </plugin>ADDED STOP -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
<executions>
<execution>
<id>default-testCompile</id>
<phase>test-compile</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
<execution>
<id>default-compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-rar-plugin</artifactId>
<configuration>
<raXmlFile>src/main/resources/META-INF/ra.xml</raXmlFile>
<includeJar>true</includeJar>
<rarSourceDirectory>target/classes</rarSourceDirectory>
<archive>
<addMavenDescriptor>false</addMavenDescriptor>
<index>true</index>
</archive>
</configuration>
<executions>
<execution>
<id>rar</id>
<goals>
<goal>rar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>ant-fix-crlf</id>
<phase>package</phase>
<configuration>
<tasks>
<fixcrlf srcdir="${project.basedir}" includes="*.spec"
eol="unix" />
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>Assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
com.ericsson.mvn
</groupId>
<artifactId>
tar-manifest-generator
</artifactId>
<versionRange>
[1.2,)
</versionRange>
<goals>
<goal>build</goal>
</goals>
</pluginExecutionFilter>
<action>
<ignore></ignore>
</action>
</pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
this is the parent pom:-
http://maven.apache.org/maven-v4_0_0.xsd">
4.0.0
<groupId>com.ericsson</groupId>
<artifactId>ca</artifactId>
<name>Customer Adaptation</name>
<packaging>pom</packaging>
<version>1.0.125</version>
<description>This component is an example how to use the Customer Adaptation Support API. It provides an example on how to develop a JDV that is distributable, i.e. only active on a fixed set of nodes in a cluster.</description>
<repositories>
<repository>
<id>project.local</id>
<name>project</name>
<url>file:${project.basedir}/tempRepo</url>
</repository>
</repositories>
<profiles>
<profile>
<id>buildCAWizard</id>
<modules>
<module>Tools/CA-Wizard</module>
</modules>
</profile>
</profiles>
<modules>
<module>JDV-CUSTOM1-P/JDV-Custom1-Provisioning</module>
<module>JDV-CUSTOM1-Conf/JDV-Custom1-Conf</module>
<module>JDV-CUSTOM3-P/JDV-Custom3-Service-Provisioning</module>
<module>JDV-CUSTOM3-P/JDV-Custom3-Resource-Provisioning</module>
<module>JCA-CUSTOM3/JCA-Custom3/</module>
<module>JDV-CUSTOM4-P/JDV-Custom4-Provisioning</module>
<module>JDV-CUSTOM5-P/JDV-Custom5-VoLTE-Subscriber-View</module>
<module>DVE-Custom-Cluster-Strategy</module>
<module>Tools/Test-HTTP-Server</module>
<module>Tools/CA-CodeGenerator</module>
<module>Tools/CA-Wizard</module>
<module>Assembly_Deployables</module>
<module>Assembly_SourceCode</module>
<module>Assembly_Plugin</module>
<module>Assembly_Repository</module>
<module>Assembly_CustomAdaptation</module>
</modules>
<properties> <!-- Latest released CF version, should NOT be a snapshot -->
<common.components.version>2.297</common.components.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<!-- ========================================== = The BUILD section ========================================== -->
<build>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.xsl</include>
<include>**/*.xml</include>
<include>**/*.xsd</include>
<include>**/*.properties</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>META-INF/**/*</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<verbose>true</verbose>
<fork>true</fork>
<encoding>ISO-8859-1</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<verify>true</verify>
<finalName>${project.name}</finalName>
<archive>
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<addDefaultImplementationEntries>
true
</addDefaultImplementationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-rar-plugin</artifactId>
<version>2.3</version>
<configuration>
<rarSourceDirectory>${basedir}/target/classes</rarSourceDirectory>
<finalName>${project.name}</finalName>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.14.1</version>
<configuration>
<argLine>-XX:-UseSplitVerifier</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.0.1</version>
<executions>
<execution>
<id>enforce-no-snapshots</id>
<phase>none</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireReleaseDeps>
<message>!! No SNAPSHOT VERSIONS are allowed when building a
delivery !!</message>
</requireReleaseDeps>
</rules>
<fail>true</fail>
</configuration>
</execution>
<execution>
<id>enforce-repository</id>
<phase>package</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireProperty>
<property>maven.repo.local</property>
<message>You must set a maven.repo.local property to -Dmaven.repo.local=repository</message>
<regex>repository</regex>
<regexMessage>You must set a maven.repo.local property to -Dmaven.repo.local=repository</regexMessage>
</requireProperty>
</rules>
<fail>true</fail>
</configuration>
</execution>
</executions>
</plugin>
<!--plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>resolve</goal>
</goals>
<configuration>
<outputDirectory>${project.basedir}/myRepo/</outputDirectory>
</configuration>
</execution>
<execution>
<id>go-offline</id>
<phase>package</phase>
<goals>
<goal>go-offline</goal>
</goals>
<configuration>
<includeGroupIds>com.ericsson</includeGroupIds>
</configuration>
</execution>
</executions>
</plugin-->
</plugins>
</build>
<!-- ==========================================
= The DEPENDENCIES section
========================================== -->
<dependencies>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>bom</artifactId>
<type>pom</type>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-JDV-Public</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>Utils-PAS</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>API-PAS</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>JavaUtils</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.context</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>TEST-Common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-PAS</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<!-- ==========================================
= The DEPENDENCY MANAGEMENT section
========================================== -->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>bom</artifactId>
<type>pom</type>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>javax.resource</groupId>
<artifactId>connector-api</artifactId>
<version>1.5</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-JDV-Public</artifactId>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.6.4</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>Utils-PAS</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>API-PAS</artifactId>
<version>1.17</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>JavaUtils</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.core</artifactId>
<version>3.0.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.context</artifactId>
<version>3.0.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.9.5</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>TEST-Common</artifactId>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-PAS</artifactId>
<version>${common.components.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<scm>
<connection>scm:git:ssh://gerrit.epk.ericsson.se:29418/provisioning/prov-ca1.git</connection>
<tag>ca-1.0.125</tag>