I'm trying to build a Docker image using sbt-native-packager with the following build.sbt (trying to publish the image to a local repository)
val sparkVersion = "2.4.5"
scalaVersion in ThisBuild := "2.12.0"
val sparkLibs = Seq(
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion
)
// JAR build settings
lazy val commonSettings = Seq(
organization := "dzlab",
version := "0.1",
scalaSource in Compile := baseDirectory.value / "src",
scalaSource in Test := baseDirectory.value / "test",
resourceDirectory in Test := baseDirectory.value / "test" / "resources",
javacOptions ++= Seq(),
scalacOptions ++= Seq(
"-deprecation",
"-feature",
"-language:implicitConversions",
"-language:postfixOps"
),
libraryDependencies ++= sparkLibs
)
// Docker Image build settings
dockerBaseImage := "gcr.io/spark-operator/spark:v" + sparkVersion
lazy val root = (project in file("."))
.enablePlugins(
DockerPlugin,
JavaAppPackaging
)
.settings(
name := "spark-k8s",
commonSettings,
dockerAliases ++= Seq(
dockerAlias.value.withRegistryHost(Some("localhost:5000"))
),
mainClass in (Compile, run) := Some("dzlab.SparkJob")
)
SBT and the packager versions
$ cat project/plugins.sbt
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.0")
$ cat project/build.properties
sbt.version=0.13.18
When I try to run the packager
$ sbt docker:publish
[info] Loading global plugins from /Users/dzlab/.sbt/0.13/plugins
[info] Loading project definition from /Users/dzlab/Projects/spark-k8s/project
/Users/dzlab/Projects/spark-k8s/build.sbt:39: error: not found: value dockerAliases
dockerAliases ++= Seq(
^
sbt.compiler.EvalException: Type error in expression
[error] sbt.compiler.EvalException: Type error in expression
[error] Use 'last' for the full log.
Project loading failed: (r)etry, (q)uit, (l)ast, or (i)gnore? q
It does not recognize dockerAliases not sure why as it is part of the publishing settings.
What is the proper way to set the Docker registry??
Your sbt-native-packager version is hopelessly outdated, as is your sbt version. That SettingKey doesn't exist in that version.
Compare: sbt-native-packager 1.0 vs. sbt-native-packager 1.7.4
Related
build.sbt
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.11.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-core
ThisBuild / libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.8"
lazy val root = (project in file("."))
.settings(
name := "Spark_Practice"
)
Error:
scalac: Error: Error compiling the sbt component
'compiler-interface-2.11.8-62.0' sbt.internal.inc.CompileFailed: Error
compiling the sbt component 'compiler-interface-2.11.8-62.0' at
sbt.internal.inc.AnalyzingCompiler$.handleCompilationError$1(AnalyzingCompiler.scala:436)
at
sbt.internal.inc.AnalyzingCompiler$.$anonfun$compileSources$5(AnalyzingCompiler.scala:453)
at
sbt.internal.inc.AnalyzingCompiler$.$anonfun$compileSources$5$adapted(AnalyzingCompiler.scala:448)
at sbt.io.IO$.withTemporaryDirectory(IO.scala:490) at
sbt.io.IO$.withTemporaryDirectory(IO.scala:500) at
sbt.internal.inc.AnalyzingCompiler$.$anonfun$compileSources$2(AnalyzingCompiler.scala:448)
at
sbt.internal.inc.AnalyzingCompiler$.$anonfun$compileSources$2$adapted(AnalyzingCompiler.scala:440)
at sbt.io.IO$.withTemporaryDirectory(IO.scala:490) at
sbt.io.IO$.withTemporaryDirectory(IO.scala:500) at
sbt.internal.inc.AnalyzingCompiler$.compileSources(AnalyzingCompiler.scala:440)
at
org.jetbrains.jps.incremental.scala.local.CompilerFactoryImpl$.org$jetbrains$jps$incremental$scala$local$CompilerFactoryImpl$$getOrCompileInterfaceJar(CompilerFactoryImpl.scala:163)
at
org.jetbrains.jps.incremental.scala.local.CompilerFactoryImpl.$anonfun$getScalac$1(CompilerFactoryImpl.scala:61)
at scala.Option.map(Option.scala:242) at
org.jetbrains.jps.incremental.scala.local.CompilerFactoryImpl.getScalac(CompilerFactoryImpl.scala:54)
at
org.jetbrains.jps.incremental.scala.local.CompilerFactoryImpl.createCompiler(CompilerFactoryImpl.scala:23)
at
org.jetbrains.jps.incremental.scala.local.CachingFactory.$anonfun$createCompiler$3(CachingFactory.scala:24)
at
org.jetbrains.jps.incremental.scala.local.Cache.$anonfun$getOrUpdate$2(Cache.scala:20)
at scala.Option.getOrElse(Option.scala:201) at
org.jetbrains.jps.incremental.scala.local.Cache.getOrUpdate(Cache.scala:19)
at
org.jetbrains.jps.incremental.scala.local.CachingFactory.createCompiler(CachingFactory.scala:24)
at
org.jetbrains.jps.incremental.scala.local.LocalServer.doCompile(LocalServer.scala:43)
at
org.jetbrains.jps.incremental.scala.local.LocalServer.compile(LocalServer.scala:30)
at
org.jetbrains.jps.incremental.scala.remote.Main$.compileLogic(Main.scala:209)
at
org.jetbrains.jps.incremental.scala.remote.Main$.$anonfun$handleCommand$1(Main.scala:192)
at
org.jetbrains.jps.incremental.scala.remote.Main$.decorated$1(Main.scala:182)
at
org.jetbrains.jps.incremental.scala.remote.Main$.handleCommand(Main.scala:189)
at
org.jetbrains.jps.incremental.scala.remote.Main$.serverLogic(Main.scala:165)
at
org.jetbrains.jps.incremental.scala.remote.Main$.nailMain(Main.scala:105)
at
org.jetbrains.jps.incremental.scala.remote.Main.nailMain(Main.scala)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:577) at
com.facebook.nailgun.NGSession.runImpl(NGSession.java:312) at
com.facebook.nailgun.NGSession.run(NGSession.java:198)
There is no caliban.federation for scala 3 yet.
My question is what is a correct way to use it along with scala 3 libraries?
For now I have such a dependencies in my build.sbt:
lazy val `bookings` =
project
.in(file("."))
.settings(
scalaVersion := "3.0.1",
name := "bookings"
)
.settings(commonSettings)
.settings(dependencies)
lazy val dependencies = Seq(
libraryDependencies ++= Seq(
"com.github.ghostdogpr" %% "caliban-zio-http" % "1.1.0"
),
libraryDependencies ++= Seq(
org.scalatest.scalatest,
org.scalatestplus.`scalacheck-1-15`,
).map(_ % Test),
libraryDependencies +=
("com.github.ghostdogpr" %% "caliban-federation" % "1.1.0")
.cross(CrossVersion.for3Use2_13)
But when I'm trying to build it, it's erroring:
[error] (update) Conflicting cross-version suffixes in:
dev.zio:zio-query,
org.scala-lang.modules:scala-collection-compat,
dev.zio:zio-stacktracer,
dev.zio:izumi-reflect,
com.github.ghostdogpr:caliban-macros,
dev.zio:izumi-reflect-thirdparty-boopickle-shaded,
dev.zio:zio,
com.github.ghostdogpr:caliban,
dev.zio:zio-streams
I get the following error on IntelliJ why I run sbt sync
[error] stack trace is suppressed; run 'last ProjectRef(uri("file:/Users/tushar/Documents/Projects/zio-rocksdb/"), "zio-rocksdb") / updateSbtClassifiers' for the full output
[error] (ProjectRef(uri("file:/Users/tushar/Documents/Projects/zio-rocksdb/"), "zio-rocksdb") / updateSbtClassifiers) java.lang.IllegalArgumentException: Cannot add dependency 'com.typesafe#ssl-config-core_2.12;0.4.0' to configuration 'default' of module dev.zio#zio-rocksdb$sbt_2.12;0.2.0+3-114b41b9+20200418-1131 because this configuration doesn't exist!
[error] Total time: 2 s, completed 18-Apr-2020, 11:31:05 AM
build.sbt
name := "scala-interview-scheduler"
version := "0.1"
scalaVersion := "2.13.1"
// ZIO Core
lazy val zioVersion = "1.0.0-RC18"
// Project Scheduler
lazy val scheduler =
(project in file("scheduler"))
// Project Storage
lazy val storage = (project in file("storage"))
.settings(
libraryDependencies ++= Seq(
"io.suzaku" %% "boopickle" % "1.3.1"
)
)
.dependsOn(ProjectRef(file("../zio-rocksdb"), "zio-rocksdb"))
// Project Program
lazy val program = (project in file("program"))
.dependsOn(scheduler)
// Testing
ThisBuild / libraryDependencies ++= Seq(
"dev.zio" %% "zio" % zioVersion,
"dev.zio" %% "zio-test" % zioVersion % "test",
"dev.zio" %% "zio-test-sbt" % zioVersion % "test"
)
ThisBuild / testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework")
// Global Options
ThisBuild / scalacOptions ++= Seq(
"-language:postfixOps",
"-language:implicitConversions",
"-feature"
)
What does it mean?
I am new to scala. I am using sbt assembly to create a fat jar. My program reads input files. I kept my files under src/main/resources folder.But I am getting java.io.FileNotFoundException
I dont know how to specify the path? I will delpoying the jar on the server.
Here is my sbt build file
lazy val commonSettings = Seq(
organization := "com.insnapinc",
version := "0.1.0",
scalaVersion := "2.11.4"
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
name := "memcache-client"
)
libraryDependencies ++= Seq (
"org.scalaj" %% "scalaj-http" % "1.1.4"
,"org.json4s" %% "json4s-native" % "3.2.10"
,"org.scalatest" % "scalatest_2.11" % "2.2.4" % "test"
)
/* assembly plugin */
mainClass in AssemblyKeys.assembly := Some("com.insnap.memcache.MemcacheTest")
assemblySettings
test in AssemblyKeys.assembly := {}
Currently I'm running a fairly middle-sized project with the Play Framework and sbt/scala. However, sometimes it takes long time to compile the project, like 4 classes taking something like 120 seconds, while most of the time the project takes just a bunch of seconds for everything!
However mostly it looks that it takes a while longer for a bunch of files when refactoring some things inside my Database Access and Services, while creating private / public functions of slick code without return codes.
Still is there anything to do against this ?
It just looks like sbt is hanging while doing:
[info] Loading project definition from /Users/schmitch/projects/envisia/envisia-erp-loki/project
[info] Set current project to envisia-erp-loki (in build file:/Users/schmitch/projects/envisia/envisia-erp-loki/)
[info] Compiling 4 Scala sources to /Users/schmitch/projects/envisia/envisia-erp-loki/modules/utils/target/scala-2.11/classes...
while utils is a sbt subproject. Any idea's why it takes so long? and mostly while that only happens sometimes?
Here is my sbt file:
name := "envisia-erp-loki"
version := "1.0.1-SNAPSHOT"
lazy val utils = project in file("modules/utils")
lazy val migrator = (project in file("modules/migrator"))
.enablePlugins(PlayScala).disablePlugins(PlayLayoutPlugin)
.dependsOn(utils).aggregate(utils)
lazy val codegen = (project in file("modules/codegen"))
.dependsOn(utils).aggregate(utils)
lazy val auth = (project in file("modules/auth"))
.enablePlugins(PlayScala).disablePlugins(PlayLayoutPlugin)
.dependsOn(utils).aggregate(utils)
lazy val pdfgen = project in file("modules/pdfgen")
lazy val root = (project in file("."))
.enablePlugins(PlayScala, DebianPlugin)
.dependsOn(utils).aggregate(utils)
.dependsOn(auth).aggregate(auth)
.dependsOn(pdfgen).aggregate(pdfgen)
scalaVersion in ThisBuild := "2.11.6"
libraryDependencies ++= Seq(
filters,
cache,
ws,
specs2 % Test,
"com.typesafe.play" %% "play-slick" % "1.0.0",
"com.typesafe.play" %% "play-slick-evolutions" % "1.0.0",
"com.typesafe.play" %% "play-mailer" % "3.0.1",
"org.elasticsearch" % "elasticsearch" % "1.5.2",
"org.postgresql" % "postgresql" % "9.4-1201-jdbc41",
"com.chuusai" %% "shapeless" % "2.2.0",
"com.nulab-inc" %% "play2-oauth2-provider" % "0.15.0",
"io.github.nremond" %% "pbkdf2-scala" % "0.4",
"com.google.code.findbugs" % "jsr305" % "3.0.0"
)
resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases"
resolvers ++= Seq(
Resolver.sonatypeRepo("releases"),
Resolver.sonatypeRepo("snapshots")
)
// Faster compilations:
sources in(Compile, doc) := Seq.empty
publishArtifact in(Compile, packageDoc) := false
// "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0"
//pipelineStages := Seq(rjs)
// Play provides two styles of routers, one expects its actions to be injected, the
// other, legacy style, accesses its actions statically.
routesGenerator := InjectedRoutesGenerator
JsEngineKeys.engineType := JsEngineKeys.EngineType.Node
// Do not add API Doc
sources in(Compile, doc) := Seq.empty
publishArtifact in(Compile, packageDoc) := false
// RpmPlugin
maintainer in Linux := "Christian Schmitt <c.schmitt#envisia.de>"
packageSummary in Linux := "Envisia ERP Server 3.0"
packageDescription := "This is the new Envisia ERP Server it will be as standalone as possible"
import com.typesafe.sbt.packager.archetypes.ServerLoader.Systemd
serverLoading in Debian := Systemd
scalacOptions in ThisBuild ++= Seq(
"-target:jvm-1.8",
"-encoding", "UTF-8",
"-deprecation", // warning and location for usages of deprecated APIs
"-feature", // warning and location for usages of features that should be imported explicitly
"-unchecked", // additional warnings where generated code depends on assumptions
"-Xlint", // recommended additional warnings
"-Ywarn-adapted-args", // Warn if an argument list is modified to match the receiver
"-Ywarn-value-discard", // Warn when non-Unit expression results are unused
"-Ywarn-inaccessible",
"-Ywarn-dead-code"
)
updateOptions in ThisBuild := updateOptions.value.withCachedResolution(true)