holdenk/spark-testing-base

NoSuchMethodError: org.scalatest.Assertions.assertionsHelper

Opened this issue · 4 comments

remisharoon commented

I am getting this error : "NoSuchMethodError: org.scalatest.Assertions.assertionsHelper" when calling "assertDataFrameApproximateEquals"

  test("Test getFeaturesDf") {

    var inputFilepath = classOf[PredictionReportTest].getResource("/reports/predictionreport/processLowerGranularDataset_input.csv").getPath
    val df = spark.read.option("header",true).csv(inputFilepath)
    
    val trainingFieldNames = "hour,day".split(",")
    val targetFieldName = "ModuleConsumedWatt"

    val featuresDF =  PredictionReportFunctions.getFeaturesDf(df, targetFieldName, trainingFieldNames)

    var featuresFilepath = classOf[PredictionReportTest].getResource("/reports/predictionreport/featuresDF.csv").getPath
    val featuresDFExpected = spark.read.option("header",true).csv(featuresFilepath)

    assertDataFrameApproximateEquals(featuresDFExpected, featuresDF, 0.1)
  }

throws error:

An exception or error caused a run to abort: org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper; 
java.lang.NoSuchMethodError: org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper;
	at com.holdenkarau.spark.testing.TestSuite$class.assert(TestSuite.scala:13)
	at analytics.spark.reports.PredictionReportTest.assert(PredictionReportTest.scala:8)
	at com.holdenkarau.spark.testing.DataFrameSuiteBaseLike$class.assertDataFrameApproximateEquals(DataFrameSuiteBase.scala:133)
	at analytics.spark.reports.PredictionReportTest.assertDataFrameApproximateEquals(PredictionReportTest.scala:8)
	at analytics.spark.reports.PredictionReportTest$$anonfun$2.apply$mcV$sp(PredictionReportTest.scala:40)
	at analytics.spark.reports.PredictionReportTest$$anonfun$2.apply(PredictionReportTest.scala:19)
	at analytics.spark.reports.PredictionReportTest$$anonfun$2.apply(PredictionReportTest.scala:19)
	at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
	at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
	at org.scalatest.Transformer.apply(Transformer.scala:22)
	at org.scalatest.Transformer.apply(Transformer.scala:20)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:189)
	at org.scalatest.TestSuite$class.withFixture(TestSuite.scala:196)
	at org.scalatest.funsuite.AnyFunSuite.withFixture(AnyFunSuite.scala:1562)
	at org.scalatest.funsuite.AnyFunSuiteLike$class.invokeWithFixture$1(AnyFunSuiteLike.scala:186)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$runTest$1.apply(AnyFunSuiteLike.scala:199)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$runTest$1.apply(AnyFunSuiteLike.scala:199)
	at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
	at org.scalatest.funsuite.AnyFunSuiteLike$class.runTest(AnyFunSuiteLike.scala:199)
	at analytics.spark.reports.PredictionReportTest.org$scalatest$BeforeAndAfter$$super$runTest(PredictionReportTest.scala:8)
	at org.scalatest.BeforeAndAfter$class.runTest(BeforeAndAfter.scala:213)
	at analytics.spark.reports.PredictionReportTest.runTest(PredictionReportTest.scala:8)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$runTests$1.apply(AnyFunSuiteLike.scala:232)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$runTests$1.apply(AnyFunSuiteLike.scala:232)
	at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
	at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
	at scala.collection.immutable.List.foreach(List.scala:381)
	at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
	at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
	at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
	at org.scalatest.funsuite.AnyFunSuiteLike$class.runTests(AnyFunSuiteLike.scala:232)
	at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1562)
	at org.scalatest.Suite$class.run(Suite.scala:1112)
	at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1562)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$run$1.apply(AnyFunSuiteLike.scala:236)
	at org.scalatest.funsuite.AnyFunSuiteLike$$anonfun$run$1.apply(AnyFunSuiteLike.scala:236)
	at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
	at org.scalatest.funsuite.AnyFunSuiteLike$class.run(AnyFunSuiteLike.scala:236)
	at analytics.spark.reports.PredictionReportTest.org$scalatest$BeforeAndAfter$$super$run(PredictionReportTest.scala:8)
	at org.scalatest.BeforeAndAfter$class.run(BeforeAndAfter.scala:273)
	at analytics.spark.reports.PredictionReportTest.org$scalatest$BeforeAndAfterAll$$super$run(PredictionReportTest.scala:8)
	at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213)
	at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210)
	at analytics.spark.reports.PredictionReportTest.run(PredictionReportTest.scala:8)
	at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
	at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1314)
	at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1308)
	at scala.collection.immutable.List.foreach(List.scala:381)
	at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1308)
	at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:972)
	at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:971)
	at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1474)
	at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
	at org.scalatest.tools.Runner$.run(Runner.scala:798)
	at org.scalatest.tools.Runner.run(Runner.scala)
	at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:131)
	at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.intellij.rt.execution.CommandLineWrapper.main(CommandLineWrapper.java:67)
hagerf commented

What version of ScalaTest are you using? I had the same issue and changing to the version used in this lib i.e. to 3.0.5 solved the issue.

  • 👍6
tsaluszewski commented

What version of ScalaTest are you using? I had the same issue and changing to the version used in this lib i.e. to 3.0.5 solved the issue.

Confirmed, referencing scalatest 3.0.5 helped!

val testDependencies = Seq( "org.scalatest" %% "scalatest" % "3.0.5" % Test, "org.scalacheck" %% "scalacheck" % "1.14.2" % Test, "com.holdenkarau" %% "spark-testing-base" % "2.4.5_0.14.0" % Test)

johnsonjsyuen commented

I also ran into this issue when using latest scalatest ( "org.scalatest" %% "scalatest" % "3.2.3").

The highest version I have tested as working is 3.0.9.

3.1.x already fails.

drobert commented

It seems to be an issue with the way assertions are written in TestSuite.scala, around the use of org.scalatest.Assertions.assert.

I've worked around this by 'forking' the function and replacing the use of these assertions with direct ones like assert(a === b), etc, with changes highlighted by comments:

// return typed changed from `Unit` to `Assertion`
def assertDataFrameApproximateEqualsPatched(
    expected: DataFrame, result: DataFrame, tol: Double): Assertion = {

    // was: assert(expected.schema, result.schema)
    assert(expected.schema === result.schema)

    // in-lined from https://github.com/holdenk/spark-testing-base/blob/0ffd8e208db7b438d11960109ae77ebf5688c06f/core/src/main/2.0/scala/com/holdenkarau/spark/testing/DataFrameSuiteBase.scala#L207
    def zipWithIndex[U](rdd: RDD[U]) = {
      rdd.zipWithIndex().map { case (row, idx) => (idx, row) }
    }

    try {
      expected.rdd.cache
      result.rdd.cache
      // was: assert("Length not Equal", expected.rdd.count, result.rdd.count)
      assert(expected.rdd.count === result.rdd.count, "Length not Equal")

      val expectedIndexValue = zipWithIndex(expected.rdd)
      val resultIndexValue = zipWithIndex(result.rdd)

      val unequalRDD = expectedIndexValue.join(resultIndexValue).filter { case (idx, (r1, r2)) =>
        !(r1.equals(r2) || DataFrameSuiteBase.approxEquals(r1, r2, tol))
      }

      // was: assertEmpty(unequalRDD.take(maxUnequalRowsToShow))
      assert(unequalRDD.take(maxUnequalRowsToShow).isEmpty)
    } finally {
      expected.rdd.unpersist()
      result.rdd.unpersist()
    }
  }

So, basically just not using the TestSuite assert helper functions anymore

  • 👍1