Issues running inside Scala notebook on databricks
Closed this issue · 1 comments
luisvicenteatprima commented
I'm having problems with this bit of code:
import za.co.absa.abris.avro.functions.from_avro
import za.co.absa.abris.config.{AbrisConfig, FromAvroConfig}
val schemaRegistryKey = ""//dbutils.secrets.get("", "")
val schemaRegistrySecret = ""//dbutils.secrets.get("", "")
val schemaRegistryConfig = Map(
AbrisConfig.SCHEMA_REGISTRY_URL -> "",
"confluent.schema.registry.basic.auth.credentials.source" -> "USER_INFO",
"confluent.schema.registry.basic.auth.user.info" -> s"$schemaRegistryKey:$schemaRegistrySecret",
)
val abrisConfig: FromAvroConfig = AbrisConfig.fromConfluentAvro.downloadReaderSchemaByLatestVersion.andTopicNameStrategy(topic).usingSchemaRegistry(schemaRegistryConfig)The last line triggers a:
Caused by: java.lang.ClassNotFoundException: org.apache.kafka.common.config.AbstractConfig
at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at za.co.absa.abris.avro.registry.ConfluentRegistryClient$.za$co$absa$abris$avro$registry$ConfluentRegistryClient$$createClient(ConfluentRegistryClient.scala:31)
at za.co.absa.abris.avro.registry.ConfluentRegistryClient.<init>(ConfluentRegistryClient.scala:25)
at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.$anonfun$getOrCreateRegistryClient$1(SchemaManagerFactory.scala:73)
at scala.collection.concurrent.TrieMap.getOrElseUpdate(TrieMap.scala:895)
at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.getOrCreateRegistryClient(SchemaManagerFactory.scala:53)
at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.create(SchemaManagerFactory.scala:49)
at za.co.absa.abris.config.FromSchemaDownloadingConfigFragment.usingSchemaRegistry(Config.scala:250)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:13)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:60)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:62)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw.<init>(command-3834262893998825:64)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw.<init>(command-3834262893998825:66)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw.<init>(command-3834262893998825:68)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read.<init>(command-3834262893998825:70)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$.<init>(command-3834262893998825:74)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$.<clinit>(command-3834262893998825)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval$.$print$lzycompute(<notebook>:7)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval$.$print(<notebook>:6)
at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval.$print(<notebook>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:747)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1020)
at scala.tools.nsc.interpreter.IMain.$anonfun$interpret$1(IMain.scala:568)
at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36)
at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116)
at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:41)
at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:567)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:594)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:564)
at com.databricks.backend.daemon.driver.DriverILoop.execute(DriverILoop.scala:219)
at com.databricks.backend.daemon.driver.ScalaDriverLocal.$anonfun$repl$1(ScalaDriverLocal.scala:225)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at com.databricks.backend.daemon.driver.DriverLocal$TrapExitInternal$.trapExit(DriverLocal.scala:1013)
at com.databricks.backend.daemon.driver.DriverLocal$TrapExit$.apply(DriverLocal.scala:966)
at com.databricks.backend.daemon.driver.ScalaDriverLocal.repl(ScalaDriverLocal.scala:225)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$13(DriverLocal.scala:644)
at com.databricks.logging.Log4jUsageLoggingShim$.$anonfun$withAttributionContext$1(Log4jUsageLoggingShim.scala:33)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:94)
at com.databricks.logging.Log4jUsageLoggingShim$.withAttributionContext(Log4jUsageLoggingShim.scala:31)
at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:205)
at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:204)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:59)
at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:240)
at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:240)
at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:225)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:59)
at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:621)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:615)
at scala.util.Try$.apply(Try.scala:213)
at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:607)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:526)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:561)
at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:431)
at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:374)
at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:225)
at java.lang.Thread.run(Thread.java:750)
```
This happens with their runtimes that support Spark 3.3.2 and 3.2.1; the library is properly installed on startup. Has you ever seen something like that?
luisvicenteatprima commented
Just replying to myself... you need to install org.apache.kafka:kafka-clients:6.2.1-ccs