sbt assembly
The Jar will be created as spark-prediction-assembly-1.0.jar at [project-root-dir]/target/scala-2.11/
/Developer/spark-2.0.0-bin-hadoop2.7/bin/spark-submit --driver-memory 4G --executor-memory 4G \
--master local \
--class com.datalogs.main.Main [project-root-dir]/target/scala-2.11/spark-prediction-assembly-1.0.jar \
--csv-dir "[input-data-dir]" \
--feature-dir "[dir-to-store-constructed-features]" \
--pipeline-stage 3
The command line parameters to run these program are -
1. "**--driver-memory**" and "**--executor-memory**" are optional and find details at http://spark.apache.org/docs/latest/configuration.html.
2. "**--master**" is required and can be set at "**local**" for local installation of spark or "**yarn**" (e.g. --master yarn)
3. "**--class**" is required to specify the fully-qualified-name of the "main" method in JAR and the location of the mortality_prediction-assembly-1.0.jar file.
4. "**--csv-dir**" is required to specify the location of MIMIC and other input files as required by this program.
e.g. --csv-dir "[sourcecode]/MIMIC/input" (please don't put "/" at the end)
5. "**--feature-dir**" is required to specify the location of the program generated features file(s).
e.g. --feature-dir "[sourcecode]/features" (please don't put "/" at the end)
6. "**--pipeline-stage**" is required to specify the workflow steps that the program will execute. There are 4 different options -
--pipeline-stage 1 => For constructing features only.
--pipeline-stage 2 => To run the all the models using the feature previous created
--pipeline-stage 3 => To construct features and run models in one step (together)
println("Welcome to the Scala worksheet")
88 + 30
var x1 = 4 x1 = 6
def size = 2
5 * size
def square(x: Double) = x * x
square(2)
square(square(4))
def sumOfSquare(x: Double, y: Double): Double = square(x) + square(y)
sumOfSquare(3, 4+5)
/////////////////////Function by Name vs Value/////////////////////////////////////
def first(x: Int, y: Int) = x
first(1, 2)
def constOne(x: Int, y: => Int) = 1
constOne(1+2, 1)
////////////////////////Function calling Function///////////////////////////////
def abs(x: Double) = if (x < 0) -x else x
def sqrtIter(guess: Double, x: Double): Double = if (isGoodEnough(guess, x)) guess else sqrtIter(improve(guess, x), x)
def improve(guess: Double, x: Double) =
(guess + x / guess) / 2
def isGoodEnough(guess: Double, x: Double) =
abs(guess * guess - x)/x < 0.001
def sqrt(x: Double) = sqrtIter(1.0, x)
sqrt(4) sqrt(2) sqrt(1e-6) sqrt(1e6)
//////////////////////Function inside function////////////////////////////////
def sqrt1(x: Double) = {
def sqrtIter(guess: Double): Double =
if (isGoodEnough(guess)) guess
else sqrtIter(improve(guess))
def improve(guess: Double) =
(guess + x / guess) / 2
def isGoodEnough(guess: Double) =
abs(guess * guess - x)/x < 0.001
sqrtIter(1.0)
} sqrt1(9)
//////////////////Tail recursion ///////////////////////////////////////
def gcd(a: Int, b: Int): Int = if (b == 0) a else gcd(b, a % b)
def factorial(n: Int): Int = if (n == 0) 1 else n * factorial(n - 1)
factorial(4)
def factorial1(n: Int): Int = {
def loop(acc: Int, n: Int): Int =
if (n==0) acc
else loop(acc * n, n-1)
loop(1, n)
}
factorial1(4)
////////////////////Function passing Func//////////////////////////////////
def id(x: Int): Int = x def cube(x: Int): Int = x * x * x
def sum(f: Int => Int, a: Int, b: Int): Int = if (a > b) 0 else f(a) + sum(f, a+1, b)
def sumInts(a: Int, b: Int) = sum(id, a, b) def sumCubes(a: Int, b: Int) = sum(cube, a, b)
//////////////////Anonymous function///////////////////////////////////////
def sumInts1(a: Int, b: Int) = sum(x=>x, a, b) def sumCubes1(a: Int, b: Int) = sum(x => x * x * x, a, b)
/////////////////////Func Return Func////////////////////////////////////
def sum1(f: Int => Int): (Int, Int) => Int = { def sumF(a: Int, b: Int): Int = if (a > b) 0 else f(a) + sumF(a+1, b)
sumF
}
def sumInts2 = sum1(x => x) def sumCubes2 = sum1(x => x * x * x)
sumCubes2(1, 10) sum1(cube)(1, 10) sum1(x => x * x * x)(1, 10)
/////////////////////Object orientation////////////////////////////////////
class Rational(x: Int, y: Int) {
private def gcd(a: Int, b: Int): Int
= if (b == 0) a else gcd(b, a % b)
private val g = gcd(x, y)
def numer = x/g
def denom =y/g
def add(that: Rational) = new Rational(numer * that.denom + that.numer * denom, denom * that.denom)
def +(r: Rational) =
new Rational(numer * r.denom + r.numer * denom, denom * r.denom)
}
val x = new Rational(1, 2) val y = new Rational(1, 3)
val w = x.add(y) w.numer
val z = x add y z.numer
val u = x + y u.numer
/////////////////////Polymorphism////////////////////////////////////
abstract class IntSet { def incl(x: Int): IntSet def contains(x: Int): Boolean def foo = 1 }
class Empty extends IntSet { def contains(x: Int): Boolean = false def incl(x: Int): IntSet = new NonEmpty(x, new Empty, new Empty) }
class NonEmpty(elem: Int, left: IntSet, right: IntSet) extends IntSet {
def contains(x: Int): Boolean =
if (x < elem) left contains x
else if (x > elem) right contains x
else true
def incl(x: Int): IntSet =
if (x < elem) new NonEmpty(elem, left incl x, right)
else if (x > elem) new NonEmpty(elem, left, right incl x)
else this
override def foo = 2
}
object Empty extends IntSet { def contains(x: Int): Boolean = false def incl(x: Int): IntSet = new NonEmpty(x, Empty, Empty) }
object Hello { def main(args: Array[String]) = println("hello world!") }
val ne = new Empty() trait List1[T] { def isEmpty: Boolean def head: T def tail: List1[T] }
class Cons[T](val head: T, val tail: List1[T]) extends List1[T] { def isEmpty = false }
class Nil[T] extends List1[T] { def isEmpty = true def head = throw new NoSuchElementException("Nil.head") def tail = throw new NoSuchElementException("Nil.tail") }
val nil = new NilInt nil.isEmpty
/////////////////////Patern Matching//////////////////////////////
trait Expr { def isNumber: Boolean def isSum: Boolean def numValue: Int def leftOp: Expr def rightOp: Expr def eval: Int } case class Number(n: Int) extends Expr { def isNumber: Boolean = true def isSum: Boolean = false def numValue: Int = n def leftOp: Expr = throw new Error("Number.leftOp") def rightOp: Expr = throw new Error("Number.rightOp") def eval: Int = n } case class Sum(e1: Expr, e2: Expr) extends Expr { def isNumber: Boolean = false def isSum: Boolean = true def numValue: Int = throw new Error("Sum.numValue") def leftOp: Expr = e1 def rightOp: Expr = e2 def eval: Int = e1.eval + e2.eval }
val num1 = new Number(5) val num2 = new Number(6)
val sum11 = new Sum(num1, num2) sum11.eval
def eval(e: Expr): Int = { if (e.isNumber) e.numValue else if (e.isSum) eval(e.leftOp) + eval(e.rightOp) else throw new Error("Unknown expression " + e) }
eval(new Sum(num1, num2))
def eval1(e: Expr): Int = e match { case Number(n) => n case Sum(e1, e2) => eval(e1) + eval(e2) }
eval1(new Sum(num1, num2))
////////////////////////// MERGE SORT ///////////////////////////////////// def msort(xs: List[Int]): List[Int] = { val n = xs.length/2 if (n == 0) xs else { //def merge(xs: List[Int], ys: List[Int]) = ??? val (fst, snd) = xs splitAt n merge(msort(fst), msort(snd)) } } def merge(xs: List[Int], ys: List[Int]): List[Int] = { xs match { case Nil => ys case x :: xs1 => ys match { case Nil => xs case y :: ys1 => if (x < y) x :: merge(xs1, ys) else y :: merge(xs, ys1) } } }
msort(List(1,4,6,2))