Merge branch 'master' of https://github.com/actionml/template-scala-p…

…arallel-universal-recommendation

engine.json.minimum

@@ -16,7 +16,6 @@
     "spark.kryo.referenceTracking": "false",
     "spark.kryoserializer.buffer.mb": "300",
     "spark.kryoserializer.buffer": "300m",
-    "spark.executor.memory": "4g",
     "es.index.auto.create": "true"
   },
   "algorithms": [

engine.json.spark-tuning

@@ -0,0 +1,38 @@
+{
+  "comment":" This config file uses default settings for all but the required values see README.md for docs",
+  "id": "default",
+  "description": "Default settings",
+  "engineFactory": "org.template.RecommendationEngine",
+  "datasource": {
+    "params" : {
+      "name": "sample-handmade-data.txt",
+      "appName": "handmade",
+      "eventNames": ["purchase", "view"]
+    }
+  },
+  "sparkConf": {
+    "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
+    "spark.kryo.registrator": "org.apache.mahout.sparkbindings.io.MahoutKryoRegistrator",
+    "spark.kryo.referenceTracking": "false",
+    "spark.kryoserializer.buffer": "300m",
+    "spark.executor.memory": "4g",
+    "spark.executor.cores": "2",
+    "spark.task.cpus": "2",
+    "spark.default.parallelism": "16",
+    "es.index.auto.create": "true"
+  },
+  "algorithms": [
+    {
+      "comment": "simplest setup where all values are default, popularity based backfill, must add eventsNames",
+      "name": "ur",
+      "params": {
+        "appName": "handmade",
+        "indexName": "urindex",
+        "typeName": "items",
+        "comment": "must have data for the first event or the model will not build, other events are optional",
+        "eventNames": ["purchase", "view"]
+      }
+    }
+  ]
+}
+

src/main/scala/DataSource.scala

@@ -65,7 +65,7 @@ class DataSource(val dsp: DataSourceParams)
       appName = dsp.appName,
       entityType = Some("user"),
       eventNames = Some(eventNames),
-      targetEntityType = Some(Some("item")))(sc)
+      targetEntityType = Some(Some("item")))(sc).repartition(sc.defaultParallelism)
 
     // now separate the events by event name
     val actionRDDs = eventNames.map { eventName =>