Details
Description
Installed: Spark 2.4.3 (checked on both Mac OS and Windows)
build.sbt:
name := "test-submit" version := "1" scalaVersion := "2.12.8" libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.3"
SubmitApp.scala:
import org.apache.spark.sql.SparkSession object SubmitApp { def main(args: Array[String]): Unit = { val spark = SparkSession.builder().master("local").getOrCreate() spark.sparkContext.parallelize(Seq(1, 2, 3)).collect().foreach(println) } }
Commands to reproduce:
$ sbt package $ spark-submit --class <package-to>.SubmitApp ./target/scala-2.12/test-submit_2.12-1.jar
Expected result: spark-submit executes successfully.
Actual result: spark-submit fails to execute with the following error:
Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.intArrayOps([I)[I at <package-to>.SubmitApp$.main(SubmitApp.scala:8) at <package-to>.SubmitApp.main(SubmitApp.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)