Error while using Bigquery connector - scala

I am getting this error when running the Spotify Spark Bigquery connector on Qubole data platform. I do see the BigQueryUtils class in my jar but still it throws this error:
Exception in thread "main"
org.spark-project.guava.util.concurrent.ExecutionError:
java.lang.NoSuchMethodError:
com.google.cloud.hadoop.io.bigquery.BigQueryUtils.waitForJobCompletion
Attaching the pom below...
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xyz.abc.google.TestProject</groupId>
<artifactId>edesem-google-TestProject</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<gpg.skip>true</gpg.skip>
<!-- Keep in sync with google-api-client dependency -->
<apache.httpcomponents.version>4.0.1</apache.httpcomponents.version>
</properties>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.3.1</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<!-- Maven Shade Plugin -->
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>edesem-google-TestProject</finalName>
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>log4j.properties</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.xyz.abc.bigquery.TestProjectBQClient</mainClass>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>org.eclipse.jetty</pattern>
<shadedPattern>org.spark-project.jetty</shadedPattern>
<includes>
<include>org.eclipse.jetty.**</include>
</includes>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.spark-project.guava</shadedPattern>
<excludes>
<exclude>com/google/common/base/Absent*</exclude>
<exclude>com/google/common/base/Function</exclude>
<exclude>com/google/common/base/Optional*</exclude>
<exclude>com/google/common/base/Present*</exclude>
<exclude>com/google/common/base/Supplier</exclude>
</excludes>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.6</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>2.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>2.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>spark-avro_2.10</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>bigquery-connector</artifactId>
<version>0.10.2-hadoop2</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.21</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.3</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcs-connector</artifactId>
<version>1.8.0-hadoop2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/util-hadoop -->
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util-hadoop</artifactId>
<version>1.8.0-hadoop2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcsio -->
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcsio</artifactId>
<version>1.8.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util</artifactId>
<version>1.8.0</version>
<exclusions>
<exclusion>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client-java6</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.8.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.6-jre</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-bigquery -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigquery</artifactId>
<version>1.23.0</version>
</dependency>
</dependencies>
</project>

I figured the main issue for me was the big query connector config in the cluster. I added the jar to the classpath and it fixed the issue. Below instructions as per Google's documentation.
https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md#add-the-connector-jar-to-hadoops-classpath
Add the connector jar to Hadoop's classpath
Placing the connector jar in the appropriate subdirectory of the Hadoop installation may be effective to have Hadoop load the jar. However, to be certain that the jar is loaded, add HADOOP_CLASSPATH=$HADOOP_CLASSPATH:</path/to/gcs-connector-jar> to hadoop-env.sh in the Hadoop configuration directory.

This is because you are using com.google.cloud.bigdataoss:bigquery-connector:0.10.2-hadoop2 BigQuery connector version which is incompatible with com.google.cloud:google-cloud-bigquery:1.23.0 library version.
You need to upgrade com.google.cloud.bigdataoss:bigquery-connector to at least 0.11.0 version, and make it consistent with versions of other com.google.cloud.bigdataoss dependencies (in your case it will be 0.12.0 version), i.e. they all should be from the same release that are listed here: https://github.com/GoogleCloudPlatform/bigdata-interop/releases

Related

Unable to see libraries downloaded from mvn in packaged JAR

This is my pom.xml:
<?xml version='1.0' encoding='UTF-8'?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>default</groupId>
<artifactId>factory-servicing</artifactId>
<packaging>jar</packaging>
<description>factory-servicing</description>
<version>0.1</version>
<name>factory-servicing</name>
<organization>
<name>default</name>
</organization>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
</dependency>
<dependency>
<groupId>org.scalikejdbc</groupId>
<artifactId>scalikejdbc_2.12</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.2</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.12</artifactId>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-core_${scala.compat.version}</artifactId>
<version>${spec2.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-junit_${scala.compat.version}</artifactId>
<version>${spec2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.3.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.21.0</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>TestSuiteReport.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>uber-jar</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.dep.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.dep.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
</dependency>
<dependency>
<groupId>org.scalikejdbc</groupId>
<artifactId>scalikejdbc_2.12</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.2</version>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
As you can see, for the outer dependencies stage, I don't have anything scoped to provided. This is because I need all libraries to run my test cases.
However, for the package stage's dependencies, which uses the maven-shade-plugin, I'm providing the spark dependencies as provided, whereas the rest of the libraries from the outer dependencies remain unchanged. This is because for Spark we will be providing the JARs, but for the rest we still need it in the packaged jar.
Then when I run mvn package, it generates a JAR file, which I go on to extract. However, on extracting the JAR file, I only see the class files that I had written in my project, and I don't see any of the other libraries that are not scoped to provided.
How do I ensure that the non-provided libraries are also included as a part of the mvn package command execution?

Intellij Idea Code Coverage Vs Maven Jacoco

when I run my tests in Intellij idea choosing code coverage tool as JaCoCo and include my packages I see I get 80% above coverage in the report but when I run it using maven command line I get 0% in JaCoCo report below are two questions.
can I see what command Intellij Idea Ultimate version is using to run my unit tests with code coverage ?
Why my maven command mvn clean test jacoco:report is showing my coverage percentage as 0%.
This is a Scala maven project.
My POM.xml file:-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.blah</groupId>
<artifactId>blah</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.blah.com </sonar.host.url>
<sonar.projectKey>blah</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
</properties>
<scm>
<connection>scm:git:https://github.cloud.blah.com/BB/blah-project</connection>
<url>https://github.cloud.blah.com/BB/blah-project</url>
<developerConnection>scm:git:https://github.cloud.blah.com/BB/blah-project
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>blah-company One Artifactory-releases</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>blah-company One Artifactory-snapshots</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<!-- setting cobertura and jacoco -->
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<scope>test</scope>
</dependency>
<!--
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>io.github.embeddedkafka</groupId>
<artifactId>embedded-kafka_2.11</artifactId>
<version>2.4.0</version>
</dependency>
https://github.com/scallop/scallop
<dependency>
<groupId>org.rogach</groupId>
<artifactId>scallop</artifactId>
<version>4.0.1</version>
</dependency>
https://github.com/ben-manes/caffeine
https://code.google.com/archive/p/findbugs-tutorials/
//find bugs
-->
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<dataFile>${project.build.directory}/jacoco.exec</dataFile>
<outputDirectory>${project.build.directory}/jacoco-ut</outputDirectory>
</configuration>
</execution>
</executions>
<configuration/>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<minimumCoverage>0</minimumCoverage>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>
Thanks
Assuming that you are using JaCoCo with cobertura coverage you need to declare the dependencies and the plugin to run the command mvn cobertura:cobertura.
   <dependencies>
        <!-- setting cobertura and jacoco -->
        <dependency>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>cobertura-maven-plugin</artifactId>
            <version>2.7</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
<build>
       <plugins>
            <!-- setting cobertura and jacoco plugin -->
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>cobertura-maven-plugin</artifactId>
                <version>2.7</version>
                <configuration>
                    <formats>
                        <format>html</format>
                        <format>xml</format>
                    </formats>
                    <check />
                </configuration>
            </plugin>
<plugins>
</build>
Create a codecov.yml on the root of your project.
coverage:
  precision: 2
  round: nearest
  range: "40...70"
Then use the comands to generate the report and upload it. Note that you also need to export your secret token.
export CODECOV_TOKEN=your_secret_token
mvn cobertura:cobertura
bash <(curl -s https://codecov.io/bash)
I end up using scoverage to create code coverage and code coverage check , below is my pom.xml file.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.company.apollo</groupId>
<artifactId>myproject</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.company.com </sonar.host.url>
<sonar.projectKey>myproject</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
<stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
<grpc.version>1.24.0</grpc.version>
<protobuf.version>3.6.1</protobuf.version>
<guava.version>26.0-android</guava.version>
<spark.version>2.4.5</spark.version>
</properties>
<scm>
<connection>scm:git:https://github.cloud.company.com/blah</connection>
<url>https://github.cloud.company.com/blah</url>
<developerConnection>scm:git:https://github.cloud.company.com/blah
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>Company Artifactory-releases</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>Company Artifactory-snapshots</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<dependency>
<groupId>com.company.onelake</groupId>
<artifactId>onelake-s3-client_hadoop-2.8_2.11</artifactId>
<version>1.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-producer-spark</artifactId>
<version>${stream-poc.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-sink-kafka-deserializer</artifactId>
<version>${stream-poc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>os-lib_2.11</artifactId>
<version>0.2.9</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>com</pattern>
<shadedPattern>shaded.com</shadedPattern>
<includes>
<include>com.google.common.**</include>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
</relocations>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
<excludedPackages>
misc
</excludedPackages>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>
ok I end up using scoverage for generating code coverage report and checks.
Code Coverage Check:-
mvn scoverage:check
mvn scoverage:report
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.company.apollo</groupId>
<artifactId>myproject</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.company.com </sonar.host.url>
<sonar.projectKey>myproject</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
<stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
<grpc.version>1.24.0</grpc.version>
<protobuf.version>3.6.1</protobuf.version>
<guava.version>26.0-android</guava.version>
<spark.version>2.4.5</spark.version>
</properties>
<scm>
<connection>scm:git:https://github.cloud.company.com/blah</connection>
<url>https://github.cloud.company.com/blah</url>
<developerConnection>scm:git:https://github.cloud.company.com/blah
</developerConnection>
</scm>
<distributionManagement>
<repository>
<id>central</id>
<name>Company Artifactory-releases</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>Company Artifactory-snapshots</name>
<url>https://artifactory.cloud.company.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<dependency>
<groupId>com.company.onelake</groupId>
<artifactId>onelake-s3-client_hadoop-2.8_2.11</artifactId>
<version>1.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-producer-spark</artifactId>
<version>${stream-poc.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.company.sdp</groupId>
<artifactId>stream-poc-sink-kafka-deserializer</artifactId>
<version>${stream-poc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>os-lib_2.11</artifactId>
<version>0.2.9</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>com</pattern>
<shadedPattern>shaded.com</shadedPattern>
<includes>
<include>com.google.common.**</include>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
</relocations>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
<excludedPackages>
misc
</excludedPackages>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>

How to generate source code from avro schema in maven(eclipse)?

I know this is not the first time this question has been asked. As i have gone through several links, but unable to understand why avro schema is not generating the source code when no error is being thrown in maven(eclipse). I have tried deleting 'pluginManagement' from pom.xml file but it throws error.
Tried deleting .m2 repo,mvn clean, mvn install, nothing seems to work. Not sure what is the issue.
Also, could you please tell me how to add 'fieldVisibility=private' argument while generating the source code from the command line.
Please guide! Thank you!
This is the pom.xml file
http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
<groupId>kafka</groupId>
<artifactId>ProducerConsumer</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>ProducerConsumer</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.8.2</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-tools</artifactId>
<version>1.8.2</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.19</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.11.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.11.2</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven/</url>
</repository>
</repositories>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.9.2</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>schema</goal>
</goals>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
<outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
<fieldVisibility>PRIVATE</fieldVisibility>
<includes>
<include>**/*.avro</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
I would suggest to use the correct setup like this:
There are two things: you are using not the avro-maven-plugin and also you have configured it in the pluginManagement instead of in using correctly located in <build><plugins>...</plugins></build>.
<project..>
<build>
<plugins>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
<version>1.9.2</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>schema</goal>
</goals>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
<outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

Unable to run a Spark Scala JAR on GCP Dataproc

I have created a simple Spark application in Scala which is running fine locally. I have used Maven as a build tool and am packaging the JAR file using shade plugin. The directory structure looks like this:
I am using the following pom.xml :
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>cl.aman.fund</groupId>
<artifactId>sym_data_decryptor</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<encoding>UTF-8</encoding>
<scala.version>2.11.12</scala.version>
</properties>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<resources>
<resource>
<directory>/src/resources/</directory>
<filtering>false</filtering>
<includes>
<include>hbase-site.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<!-- see http://davidb.github.com/scala-maven-plugin -->
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.3.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<!-- <arg>-make:transitive</arg> -->
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.5</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<shadedArtifactAttached>true</shadedArtifactAttached>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<!-- Scala and Spark dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.scala-logging</groupId>
<artifactId>scala-logging_2.11</artifactId>
<version>3.9.2</version>
</dependency>
<!-- Spark avro -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>2.4.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcs-connector</artifactId>
<version>hadoop2-1.9.17</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>bigquery-connector</artifactId>
<version>hadoop2-0.13.9</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
<version>5.2.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-xml</artifactId>
<version>2.11.0-M4</version>
</dependency>
<dependency>
<groupId>org.scalactic</groupId>
<artifactId>scalactic_2.11</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock-scalatest-support_2.11</artifactId>
<version>3.6.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-native_2.11</artifactId>
<version>3.6.6</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.13.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api-scala_2.12</artifactId>
<version>11.0</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_2.11</artifactId>
<version>1.14.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>2.4.4</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Command used to submit job on Dataproc cluster:
gcloud dataproc jobs submit spark \
--cluster <cluster_name> \
--region <region> \
--class cl.aman.symphony.commons.DecryptorApplication \
--jars gs://<bucket_name>/sym_data_decryptor-0.0.1-SNAPSHOT-jar-with-dependencies.jar \
-- \
"my_Input"
Error:
Exception in thread "main" java.lang.NoSuchMethodError: org.json4s.Serialization$class.read(Lorg/json4s/Serialization;Ljava/lang/String;Lorg/json4s/Formats;Lscala/reflect/Manifest;)Ljava/lang/Object;
at org.json4s.native.Serialization$.read(Serialization.scala:32)
at cl.falabella.symphony.commons.DecryptorApplication$.makeRequest(DecryptorApplication.scala:44)
at cl.falabella.symphony.commons.DecryptorApplication$.main(DecryptorApplication.scala:65)
at cl.falabella.symphony.commons.DecryptorApplication.main(DecryptorApplication.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:890)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:192)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:217)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Can someone help me with the Scala JAR packaging ?
You are using json4s-native_2.11 3.6.6 but spark-core 2.4.4 which is core of Spark framework uses json4s in version 3.5.3 which makes json4s core incompatible. I recommend switching to json4s version 3.5.3
Your plugin for dependencies might be wrong set.
Mine has some differences and works perfectly.
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
Remember to add in your spark-submit "-jar-with-dependencies" at your jar name.

Failed to execute goal on project..Could not resolve dependencies for

eclipse build failed and i got like this:-
Failed to execute goal on project : Could not resolve dependencies for project com.ericsson:JCA-CA-SADM:jar:1.0.0: Failed to collect dependencies at org.apache.maven.plugins:maven-resources-plugin:jar:2.5 -> org.apache.maven:maven-project:jar:2.0.6 -> org.apache.maven:maven-artifact-manager:jar:2.0.6 -> org.apache.maven.wagon:wagon-provider-api:jar:1.0-beta-2: Failed to read artifact descriptor for org.apache.maven.wagon:wagon-provider-api:jar:1.0-beta-2: Could not transfer artifact org.apache.maven.wagon:wagon:pom:1.0-beta-2 from/to public (http://repo1.maven.org/maven2/): Access denied to http://repo1.maven.org/maven2/org/apache/maven/wagon/wagon/1.0-beta-2/wagon-1.0-beta-2.pom. Error code 403, Forbidden -> [Help 1]
this is my pom.xml
http://maven.apache.org/maven-v4_0_0.xsd">
4.0.0
<!-- <parent> <groupId>com.ericsson</groupId> <artifactId>ca</artifactId>
<version>1.0.87-SNAPSHOT</version> <relativePath>../..</relativePath> </parent> -->
<parent>
<groupId>com.ericsson</groupId>
<artifactId>ca</artifactId>
<version>1.0.125</version>
<relativePath>../..</relativePath>
</parent>
<artifactId>JCA-CA-SADM</artifactId>
<name>JCA-CA-SADM</name>
<version>1.0.0</version>
<description>This is SADM Connection</description>
<!-- <description>This component is an example how to use the Customer Adatation
Support API. It provides an example on how to develop a new outbound connector.</description> -->
<!-- Skips the enforcer rules from parent pom -->
<properties>
<enforcer.skip>true</enforcer.skip>
<cxf.version>2.2.3</cxf.version>
</properties>
<!-- ========================================== = The DEPENDENCIES section
========================================== -->
<dependencies>
<dependency>
<groupId>javax.resource</groupId>
<artifactId>connector-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.axis</groupId>
<artifactId>axis</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>javax.xml</groupId>
<artifactId>jaxrpc-api</artifactId>
<version>1.1</version>
</dependency>
<!-- <dependency> <groupId>com.ericsson</groupId> <artifactId>Test-HTTP-Server</artifactId>
<version>1.0.87-SNAPSHOT</version> <scope>test</scope> </dependency> -->
<!-- Jetty is needed if you're are not using the CXFServlet -->
<!-- <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-frontend-jaxws</artifactId>
<version>${cxf.version}</version> </dependency> <dependency> <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId> <version>${cxf.version}</version>
</dependency> <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-transports-http-jetty</artifactId>
<version>${cxf.version}</version> </dependency> -->
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>META-INF/ra.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>com.ericsson.mvn</groupId>
<artifactId>tar-manifest-generator</artifactId>
<version>1.2</version>
<executions>
<execution>
<goals>
<goal>build</goal>
</goals>
</execution>
</executions>
<configuration>
<id>CXC1736209</id>
<version>${project.version}</version>
<name>${project.artifactId}</name>
</configuration>
</plugin>
<!-- <plugin> ADDED START <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId> <executions> The m2e eclipse
plugin complains but it's valid and works cmd line <execution> <id>copy-dependencies</id>
<phase>package</phase> <goals> <goal>copy-dependencies</goal> </goals> <configuration>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
<includeScope>runtime</includeScope> <excludeTransitive>true</excludeTransitive>
</configuration> </execution> </executions> </plugin>ADDED STOP -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
<executions>
<execution>
<id>default-testCompile</id>
<phase>test-compile</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
<execution>
<id>default-compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-rar-plugin</artifactId>
<configuration>
<raXmlFile>src/main/resources/META-INF/ra.xml</raXmlFile>
<includeJar>true</includeJar>
<rarSourceDirectory>target/classes</rarSourceDirectory>
<archive>
<addMavenDescriptor>false</addMavenDescriptor>
<index>true</index>
</archive>
</configuration>
<executions>
<execution>
<id>rar</id>
<goals>
<goal>rar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>ant-fix-crlf</id>
<phase>package</phase>
<configuration>
<tasks>
<fixcrlf srcdir="${project.basedir}" includes="*.spec"
eol="unix" />
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>Assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
com.ericsson.mvn
</groupId>
<artifactId>
tar-manifest-generator
</artifactId>
<versionRange>
[1.2,)
</versionRange>
<goals>
<goal>build</goal>
</goals>
</pluginExecutionFilter>
<action>
<ignore></ignore>
</action>
</pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
this is the parent pom:-
http://maven.apache.org/maven-v4_0_0.xsd">
4.0.0
<groupId>com.ericsson</groupId>
<artifactId>ca</artifactId>
<name>Customer Adaptation</name>
<packaging>pom</packaging>
<version>1.0.125</version>
<description>This component is an example how to use the Customer Adaptation Support API. It provides an example on how to develop a JDV that is distributable, i.e. only active on a fixed set of nodes in a cluster.</description>
<repositories>
<repository>
<id>project.local</id>
<name>project</name>
<url>file:${project.basedir}/tempRepo</url>
</repository>
</repositories>
<profiles>
<profile>
<id>buildCAWizard</id>
<modules>
<module>Tools/CA-Wizard</module>
</modules>
</profile>
</profiles>
<modules>
<module>JDV-CUSTOM1-P/JDV-Custom1-Provisioning</module>
<module>JDV-CUSTOM1-Conf/JDV-Custom1-Conf</module>
<module>JDV-CUSTOM3-P/JDV-Custom3-Service-Provisioning</module>
<module>JDV-CUSTOM3-P/JDV-Custom3-Resource-Provisioning</module>
<module>JCA-CUSTOM3/JCA-Custom3/</module>
<module>JDV-CUSTOM4-P/JDV-Custom4-Provisioning</module>
<module>JDV-CUSTOM5-P/JDV-Custom5-VoLTE-Subscriber-View</module>
<module>DVE-Custom-Cluster-Strategy</module>
<module>Tools/Test-HTTP-Server</module>
<module>Tools/CA-CodeGenerator</module>
<module>Tools/CA-Wizard</module>
<module>Assembly_Deployables</module>
<module>Assembly_SourceCode</module>
<module>Assembly_Plugin</module>
<module>Assembly_Repository</module>
<module>Assembly_CustomAdaptation</module>
</modules>
<properties> <!-- Latest released CF version, should NOT be a snapshot -->
<common.components.version>2.297</common.components.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<!-- ========================================== = The BUILD section ========================================== -->
<build>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.xsl</include>
<include>**/*.xml</include>
<include>**/*.xsd</include>
<include>**/*.properties</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>META-INF/**/*</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<verbose>true</verbose>
<fork>true</fork>
<encoding>ISO-8859-1</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<verify>true</verify>
<finalName>${project.name}</finalName>
<archive>
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<addDefaultImplementationEntries>
true
</addDefaultImplementationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-rar-plugin</artifactId>
<version>2.3</version>
<configuration>
<rarSourceDirectory>${basedir}/target/classes</rarSourceDirectory>
<finalName>${project.name}</finalName>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.14.1</version>
<configuration>
<argLine>-XX:-UseSplitVerifier</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.0.1</version>
<executions>
<execution>
<id>enforce-no-snapshots</id>
<phase>none</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireReleaseDeps>
<message>!! No SNAPSHOT VERSIONS are allowed when building a
delivery !!</message>
</requireReleaseDeps>
</rules>
<fail>true</fail>
</configuration>
</execution>
<execution>
<id>enforce-repository</id>
<phase>package</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireProperty>
<property>maven.repo.local</property>
<message>You must set a maven.repo.local property to -Dmaven.repo.local=repository</message>
<regex>repository</regex>
<regexMessage>You must set a maven.repo.local property to -Dmaven.repo.local=repository</regexMessage>
</requireProperty>
</rules>
<fail>true</fail>
</configuration>
</execution>
</executions>
</plugin>
<!--plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>resolve</goal>
</goals>
<configuration>
<outputDirectory>${project.basedir}/myRepo/</outputDirectory>
</configuration>
</execution>
<execution>
<id>go-offline</id>
<phase>package</phase>
<goals>
<goal>go-offline</goal>
</goals>
<configuration>
<includeGroupIds>com.ericsson</includeGroupIds>
</configuration>
</execution>
</executions>
</plugin-->
</plugins>
</build>
<!-- ==========================================
= The DEPENDENCIES section
========================================== -->
<dependencies>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>bom</artifactId>
<type>pom</type>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-JDV-Public</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>Utils-PAS</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>API-PAS</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>JavaUtils</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.context</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>TEST-Common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-PAS</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<!-- ==========================================
= The DEPENDENCY MANAGEMENT section
========================================== -->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>bom</artifactId>
<type>pom</type>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>javax.resource</groupId>
<artifactId>connector-api</artifactId>
<version>1.5</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-JDV-Public</artifactId>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.6.4</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>Utils-PAS</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>API-PAS</artifactId>
<version>1.17</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>JavaUtils</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.core</artifactId>
<version>3.0.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.context</artifactId>
<version>3.0.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.9.5</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>TEST-Common</artifactId>
<version>${common.components.version}</version>
</dependency>
<dependency>
<groupId>com.ericsson</groupId>
<artifactId>LIB-PAS</artifactId>
<version>${common.components.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<scm>
<connection>scm:git:ssh://gerrit.epk.ericsson.se:29418/provisioning/prov-ca1.git</connection>
<tag>ca-1.0.125</tag>