write hive data problem
happyboy20143 opened this issue · 1 comments
when I use spark-submit to write som data into hive,I got some error as follows:
java.lang.NoSuchMethodError: org.apache.spark.sql.internal.SharedState.externalCatalog()Lorg/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener;
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.overwriteExternalTable(FileOutputWriter.scala:87)
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.save(FileOutputWriter.scala:123)
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.write(FileOutputWriter.scala:76)
at com.yotpo.metorikku.metric.Metric.com$yotpo$metorikku$metric$Metric$$writeBatch(Metric.scala:102)
at com.yotpo.metorikku.metric.Metric$$anonfun$write$1.apply(Metric.scala:156)
at com.yotpo.metorikku.metric.Metric$$anonfun$write$1.apply(Metric.scala:134)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.metric.Metric.write(Metric.scala:134)
at com.yotpo.metorikku.metric.MetricSet$$anonfun$run$1.apply(MetricSet.scala:50)
at com.yotpo.metorikku.metric.MetricSet$$anonfun$run$1.apply(MetricSet.scala:45)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.metric.MetricSet.run(MetricSet.scala:45)
at com.yotpo.metorikku.Metorikku$$anonfun$runMetrics$1.apply(Metorikku.scala:47)
at com.yotpo.metorikku.Metorikku$$anonfun$runMetrics$1.apply(Metorikku.scala:45)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.Metorikku$.runMetrics(Metorikku.scala:45)
at com.yotpo.metorikku.Metorikku$.delayedEndpoint$com$yotpo$metorikku$Metorikku$1(Metorikku.scala:19)
at com.yotpo.metorikku.Metorikku$delayedInit$body.apply(Metorikku.scala:9)
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
at scala.App$class.main(App.scala:76)
at com.yotpo.metorikku.Metorikku$.main(Metorikku.scala:9)
at com.yotpo.metorikku.Metorikku.main(Metorikku.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:920)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:195)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:220)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:140)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
could somebody can help me solve this problem? my config file is under examples directory,and not get changed
I've known this issue,it's the cluster spark version is in compatible with the jar spark version