YotpoLtd/metorikku

write hive data problem

happyboy20143 opened this issue · 1 comments

when I use spark-submit to write som data into hive,I got some error as follows:
java.lang.NoSuchMethodError: org.apache.spark.sql.internal.SharedState.externalCatalog()Lorg/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener;
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.overwriteExternalTable(FileOutputWriter.scala:87)
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.save(FileOutputWriter.scala:123)
at com.yotpo.metorikku.output.writers.file.FileOutputWriter.write(FileOutputWriter.scala:76)
at com.yotpo.metorikku.metric.Metric.com$yotpo$metorikku$metric$Metric$$writeBatch(Metric.scala:102)
at com.yotpo.metorikku.metric.Metric$$anonfun$write$1.apply(Metric.scala:156)
at com.yotpo.metorikku.metric.Metric$$anonfun$write$1.apply(Metric.scala:134)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.metric.Metric.write(Metric.scala:134)
at com.yotpo.metorikku.metric.MetricSet$$anonfun$run$1.apply(MetricSet.scala:50)
at com.yotpo.metorikku.metric.MetricSet$$anonfun$run$1.apply(MetricSet.scala:45)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.metric.MetricSet.run(MetricSet.scala:45)
at com.yotpo.metorikku.Metorikku$$anonfun$runMetrics$1.apply(Metorikku.scala:47)
at com.yotpo.metorikku.Metorikku$$anonfun$runMetrics$1.apply(Metorikku.scala:45)
at scala.collection.immutable.List.foreach(List.scala:381)
at com.yotpo.metorikku.Metorikku$.runMetrics(Metorikku.scala:45)
at com.yotpo.metorikku.Metorikku$.delayedEndpoint$com$yotpo$metorikku$Metorikku$1(Metorikku.scala:19)
at com.yotpo.metorikku.Metorikku$delayedInit$body.apply(Metorikku.scala:9)
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
at scala.App$class.main(App.scala:76)
at com.yotpo.metorikku.Metorikku$.main(Metorikku.scala:9)
at com.yotpo.metorikku.Metorikku.main(Metorikku.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:920)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:195)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:220)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:140)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
could somebody can help me solve this problem? my config file is under examples directory,and not get changed

I've known this issue,it's the cluster spark version is in compatible with the jar spark version