diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala index 26021917d..2c8b80603 100644 --- a/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala +++ b/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala @@ -36,7 +36,7 @@ object PathPlanners_Simple { val result = candidates .minBy { v => - val indices = v.map(_.sortIndex(depthField, ordinalField)) + val indices = v.map(_.sortIndex(schema.sortIndices: _*)) indices } @@ -58,7 +58,7 @@ object PathPlanners_Simple { val dataRows = tuple._2 val firstDataRow = dataRows.head - val sortEvs = firstDataRow.sortIndex(depthField, ordinalField) + val sortEvs = firstDataRow.sortIndex(schema.sortIndices: _*) 0 -> sortEvs } result diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala index 9f2b1a124..492461679 100644 --- a/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala +++ b/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala @@ -368,7 +368,7 @@ case class FetchedDataset( traces: HasTraceSet, keyBy: Trace => Any = identity, genPartitioner: GenPartitioner = spooky.conf.localityPartitioner, - depthField: Field = null, + depthField: Field = null, // TODO: Some of them has to be moved upwards range: Range = spooky.conf.exploreRange, pathPlanning: PathPlanning = spooky.conf.explorePathPlanning, epochSize: Int = spooky.conf.exploreEpochSize, diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala index 823568724..9747ced94 100644 --- a/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala +++ b/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala @@ -39,7 +39,7 @@ case class SpookySchema( // lazy val evictTransientFields: SpookySchema = filterFields(_.isNonTransient) - lazy val structFields: Seq[StructField] = fieldTypes.toSeq + @transient lazy val structFields: Seq[StructField] = fieldTypes.toSeq .map { tuple => StructField( tuple._1.name, @@ -47,7 +47,7 @@ case class SpookySchema( ) } - lazy val structType: StructType = { + @transient lazy val structType: StructType = { StructType(structFields) } @@ -56,6 +56,8 @@ case class SpookySchema( fieldTypes = fieldTypes -- field ) + @transient lazy val sortIndices: List[Field] = fields.filter(_.isSortIndex) + def newResolver: Resolver = new Resolver() class Resolver extends Serializable { diff --git a/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala b/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala index f443ca558..b7898441c 100644 --- a/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala +++ b/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala @@ -1,5 +1,6 @@ package com.tribbloids.spookystuff.execution +import ai.acyclic.prover.commons.debug.print_@ import ai.acyclic.prover.commons.function.Impl import com.tribbloids.spookystuff.actions.{Trace, Wget} import com.tribbloids.spookystuff.extractors.impl.Lit @@ -31,7 +32,7 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture { S"h1".text ~ 'header ) - println(explored.plan.toString) + print_@(explored.plan.toString) } it("should create a new beaconRDD if its upstream doesn't have one") { @@ -92,12 +93,14 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture { .fetch { Wget('_) } - .explore(S"root directory URI".text)( + .explore( + S"root directory URI".text + )( Wget('A), depthField = 'depth, range = range ) - .fork(S"root file")( + .fork(S"root file", ordinalField = 'index)( A"name".text into 'leaf, A"URI".text ~ 'fullPath ) @@ -113,32 +116,26 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture { lazy val `-1..` = computeFrom(-1 to bigInt) - it("from 0") { + it("from -1") { - `0..`.mkString("\n") + `-1..`.mkString("\n") .shouldBe( """ - |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv] - |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv] - |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf] - |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml] - |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json] + |[/tmp/spookystuff/resources/testutils/dir,null,null,null,null] + |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(0),ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv] + |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(1),ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv] + |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(0, 0),ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf] + |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(0, 0, 0),ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml] + |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(0, 0, 0, 0),ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json] |""".stripMargin ) } - it("from -1") { + it("from 0") { - `-1..`.mkString("\n") + `0..`.mkString("\n") .shouldBe( - """ - |[/tmp/spookystuff/resources/testutils/dir,null,null,null] - |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv] - |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv] - |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf] - |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml] - |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json] - |""".stripMargin + `-1..`.slice(1, bigInt).mkString("\n") ) }