From 1ae526a9f7aca366340d72437ae0d1959fd52d5e Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 12 Jul 2015 23:05:17 +0530 Subject: [PATCH 01/31] Adding leaflet for map --- zeppelin-web/bower.json | 1 + zeppelin-web/src/app/notebook/paragraph/paragraph.html | 4 ++++ zeppelin-web/src/index.html | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index 04bdbf86b70..f233a64c05d 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -27,6 +27,7 @@ "lodash": "~3.9.3", "angular-filter": "~0.5.4", "ngtoast": "~1.5.5" + "leaflet": "~0.7.3" }, "devDependencies": { "angular-mocks": "1.3.8" diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 4d0f4510fc7..b0d28a860fb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -129,6 +129,10 @@ ng-class="{'active': isGraphMode('scatterChart')}" ng-click="setGraphMode('scatterChart', true)"> + diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index ac16ea76d88..b1d902fbabe 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -43,6 +43,8 @@ + + @@ -115,6 +117,9 @@ + + + From 67cc563d1a4b30e5d675cdcc3b92f8d507b00ad3 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 12 Jul 2015 23:38:08 +0530 Subject: [PATCH 02/31] Adding mapping for UI --- .../src/app/notebook/paragraph/paragraph.controller.js | 7 +++++++ zeppelin-web/src/app/notebook/paragraph/paragraph.html | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 08eb657b9c8..6a229791989 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -730,6 +730,10 @@ angular.module('zeppelinWebApp') } }; + $scope.defaults= { + scrollWheelZoom: false + }; + $scope.setGraphMode = function(type, emit, refresh) { if (emit) { setNewMode(type); @@ -742,6 +746,9 @@ angular.module('zeppelinWebApp') if (!type || type === 'table') { setTable($scope.paragraph.result, refresh); } + if (!type || type === 'mapChart') { + //setting the map + } else { setD3Chart(type, $scope.paragraph.result, refresh); } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index b0d28a860fb..305964617cf 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -337,6 +337,11 @@ id="p{{paragraph.id}}_scatterChart"> + +
+ +
Date: Mon, 13 Jul 2015 00:25:02 +0530 Subject: [PATCH 03/31] Adding map chart for zeppelin --- zeppelin-web/bower.json | 5 +++-- zeppelin-web/src/app/app.js | 3 ++- .../src/app/notebook/paragraph/paragraph.controller.js | 7 ++++++- zeppelin-web/src/app/notebook/paragraph/paragraph.html | 4 ++-- zeppelin-web/src/index.html | 4 ++-- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index f233a64c05d..2a5f986fbe5 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -26,8 +26,9 @@ "highlightjs": "~8.4.0", "lodash": "~3.9.3", "angular-filter": "~0.5.4", - "ngtoast": "~1.5.5" - "leaflet": "~0.7.3" + "ngtoast": "~1.5.5", + "leaflet": "~0.7.3", + "angular-leaflet-directive": "~0.8.5" }, "devDependencies": { "angular-mocks": "1.3.8" diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index b4a769bab91..9a255c1abf8 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -31,7 +31,8 @@ angular.module('zeppelinWebApp', [ 'monospaced.elastic', 'puElasticInput', 'xeditable', - 'ngToast' + 'ngToast', + 'leaflet-directive' ]) .filter('breakFilter', function() { return function (text) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 6a229791989..460a9e1ccfa 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -747,7 +747,12 @@ angular.module('zeppelinWebApp') setTable($scope.paragraph.result, refresh); } if (!type || type === 'mapChart') { - //setting the map + //setting the map : To-DO Moving the map for separate file +/* angular.extend($scope, { + defaults: { + scrollWheelZoom: false + } + });*/ } else { setD3Chart(type, $scope.paragraph.result, refresh); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 305964617cf..b7878ef1c8e 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -134,7 +134,7 @@ ng-click="setGraphMode('mapChart', true)">
- @@ -340,7 +340,7 @@
- +
diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index b1d902fbabe..08d862d51aa 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -44,7 +44,7 @@ - + @@ -119,7 +119,7 @@ - + From 19d5ec86431911b06fa5744625c45dd4a58dc790 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 19 Jul 2015 20:29:04 +0530 Subject: [PATCH 04/31] adding sample tutorial for notbook on map --- notebook/2AU6K8KS2/note.json | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 notebook/2AU6K8KS2/note.json diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json new file mode 100644 index 00000000000..a6068cff2c6 --- /dev/null +++ b/notebook/2AU6K8KS2/note.json @@ -0,0 +1,136 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true, + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437313582566_-967040274", + "id": "20150719-191622_9501079", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Jul 19, 2015 7:16:22 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:26 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(5).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437314214306_141157507", + "id": "20150719-192654_12434929", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[14] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[17] at map at \u003cconsole\u003e:39\n" + }, + "dateCreated": "Jul 19, 2015 7:26:54 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect * from myMap", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437316940442_271264979", + "id": "20150719-201220_10279868", + "result": { + "code": "ERROR", + "type": "TEXT", + "msg": "org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 4.0 failed 1 times, most recent failure: Lost task 0.0 in stage 4.0 (TID 4, localhost): java.lang.ArrayIndexOutOfBoundsException: 5\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:44)\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:40)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$1.next(Iterator.scala:853)\n\tat scala.collection.Iterator$$anon$1.head(Iterator.scala:840)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:42)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:37)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:64)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n" + }, + "dateCreated": "Jul 19, 2015 8:12:20 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "ERROR", + "progressUpdateIntervalMs": 500 + }, + { + "text": "", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437317266677_-261952669", + "id": "20150719-201746_23407076", + "result": { + "code": "SUCCESS", + "type": "TEXT" + }, + "dateCreated": "Jul 19, 2015 8:17:46 PM", + "dateStarted": "Jul 19, 2015 8:20:32 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Map Visualization Tutorial", + "id": "2AU6K8KS2", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file From 0c87778712bd1cbc1aa976a46b74eb9cf7cdb5de Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 19 Jul 2015 20:32:15 +0530 Subject: [PATCH 05/31] adding sql --- notebook/2AU6K8KS2/note.json | 51 ++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json index a6068cff2c6..ece8c513e91 100644 --- a/notebook/2AU6K8KS2/note.json +++ b/notebook/2AU6K8KS2/note.json @@ -34,7 +34,7 @@ "progressUpdateIntervalMs": 500 }, { - "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(5).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(4).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", "config": { "colWidth": 12.0, "graph": { @@ -56,11 +56,11 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[14] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[17] at map at \u003cconsole\u003e:39\n" + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[37] at parallelize at \u003cconsole\u003e:47\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[40] at map at \u003cconsole\u003e:47\n" }, "dateCreated": "Jul 19, 2015 7:26:54 PM", - "dateStarted": "Jul 19, 2015 8:20:26 PM", - "dateFinished": "Jul 19, 2015 8:20:32 PM", + "dateStarted": "Jul 19, 2015 8:31:04 PM", + "dateFinished": "Jul 19, 2015 8:31:07 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -69,13 +69,36 @@ "config": { "colWidth": 12.0, "graph": { - "mode": "table", + "mode": "mapChart", "height": 300.0, "optionOpen": false, - "keys": [], - "values": [], + "keys": [ + { + "name": "Country", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + ], "groups": [], - "scatter": {} + "scatter": { + "xAxis": { + "name": "Country", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + } } }, "settings": { @@ -85,14 +108,14 @@ "jobName": "paragraph_1437316940442_271264979", "id": "20150719-201220_10279868", "result": { - "code": "ERROR", - "type": "TEXT", - "msg": "org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 4.0 failed 1 times, most recent failure: Lost task 0.0 in stage 4.0 (TID 4, localhost): java.lang.ArrayIndexOutOfBoundsException: 5\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:44)\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:40)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$1.next(Iterator.scala:853)\n\tat scala.collection.Iterator$$anon$1.head(Iterator.scala:840)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:42)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:37)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:64)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n" + "code": "SUCCESS", + "type": "TABLE", + "msg": "Country\tName\tlat\tlan\tAltitude\nUnited States\tNew York City\t40.714268\t-74.005974\t2.0\nUnited States\tLos Angeles\t34.052235\t-118.24368\t115.0\nUnited States\tChicago\t41.850033\t-87.650055\t181.0\nUnited States\tHouston\t29.763283\t-95.36327\t15.0\nUnited States\tPhiladelphia\t39.952335\t-75.16379\t8.0\nUnited States\tPhoenix\t33.448376\t-112.074036\t0.0\nUnited States\tSan Antonio\t29.424122\t-98.49363\t196.0\nUnited States\tSan Diego\t32.71533\t-117.15726\t26.0\nUnited States\tDallas\t32.783054\t-96.80666\t137.0\nUnited States\tDetroit\t42.33143\t-83.04575\t181.0\nUnited States\tSan Jose\t37.339386\t-121.89496\t24.0\nUnited States\tJacksonville\t30.332184\t-81.65565\t7.0\nUnited States\tIndianapolis\t39.768375\t-86.15804\t222.0\nUnited States\tColumbus\t39.961174\t-82.998795\t237.0\nUnited States\tSan Francisco\t37.77493\t-122.41942\t60.0\nUnited States\tAustin\t30.267153\t-97.74306\t163.0\nUnited States\tMemphis\t35.149536\t-90.04898\t75.0\nUnited States\tNew South Memphis\t35.086758\t-90.056755\t74.0\nUnited States\tFort Worth\t32.72541\t-97.32085\t199.0\nUnited States\tBaltimore\t39.290386\t-76.61219\t2.0\nUnited States\tCharlotte\t35.227085\t-80.843124\t229.0\nUnited States\tBoston\t42.35843\t-71.05978\t11.0\nUnited States\tMilwaukee\t43.038902\t-87.90647\t195.0\nUnited States\tSouth Boston\t42.33343\t-71.04949\t10.0\nUnited States\tSeattle\t47.60621\t-122.33207\t60.0\nUnited States\tEl Paso\t31.75872\t-106.48693\t1147.0\nUnited States\tDenver\t39.739155\t-104.9847\t1598.0\nUnited States\tWashington\t38.89511\t-77.03637\t11.0\nUnited States\tPortland\t45.523453\t-122.67621\t61.0\nUnited States\tOklahoma City\t35.46756\t-97.516426\t364.0\nUnited States\tNashville\t36.16589\t-86.78445\t151.0\nUnited States\tTucson\t32.221745\t-110.926476\t751.0\nUnited States\tAlbuquerque\t35.08449\t-106.65114\t1510.0\nUnited States\tNew Orleans\t29.954649\t-90.07507\t0.0\nUnited States\tLong Beach\t33.766964\t-118.18923\t1.0\nUnited States\tLas Vegas\t36.17497\t-115.13722\t609.0\nUnited States\tSacramento\t38.581573\t-121.4944\t5.0\nUnited States\tFresno\t36.747726\t-119.77237\t91.0\nUnited States\tCleveland\t41.499496\t-81.69541\t185.0\nUnited States\tNorth Kansas City\t39.130005\t-94.56218\t223.0\nUnited States\tKansas City\t39.099728\t-94.57857\t271.0\nUnited States\tVirginia Beach\t36.85293\t-75.97798\t3.0\nUnited States\tAtlanta\t33.748997\t-84.387985\t305.0\nUnited States\tOakland\t37.80437\t-122.270805\t0.0\nUnited States\tMesa\t33.422268\t-111.82264\t386.0\nUnited States\tTulsa\t36.15398\t-95.992775\t213.0\nUnited States\tOmaha\t41.25861\t-95.93779\t0.0\nUnited States\tMiami\t25.774265\t-80.19366\t2.0\nUnited States\tHonolulu\t21.306944\t-157.85834\t0.0\nUnited States\tMinneapolis\t44.979965\t-93.26384\t259.0\nUnited States\tColorado Springs\t38.83388\t-104.821365\t1829.0\nUnited States\tWichita\t37.692238\t-97.33755\t402.0\nUnited States\tSanta Ana\t33.745575\t-117.867836\t30.0\nUnited States\tWest Raleigh\t35.78682\t-78.663895\t122.0\nUnited States\tArlington\t32.735687\t-97.10806\t184.0\nUnited States\tAnaheim\t33.835293\t-117.914505\t40.0\nUnited States\tTampa\t27.947521\t-82.45843\t1.0\nUnited States\tSaint Louis\t38.627274\t-90.19789\t135.0\nUnited States\tPittsburgh\t40.440624\t-79.99589\t235.0\nUnited States\tToledo\t41.663937\t-83.555214\t191.0\nUnited States\tCincinnati\t39.162003\t-84.45689\t203.0\nUnited States\tRiverside\t33.95335\t-117.39616\t262.0\nUnited States\tBakersfield\t35.37329\t-119.018715\t122.0\nUnited States\tStockton\t37.957703\t-121.29078\t4.0\nUnited States\tNewark\t40.735657\t-74.17236\t16.0\nUnited States\tBuffalo\t42.886448\t-78.87837\t183.0\nUnited States\tCorpus Christi\t27.800583\t-97.39638\t6.0\nUnited States\tAurora\t39.72943\t-104.83192\t1647.0\nUnited States\tRaleigh\t35.772095\t-78.63861\t91.0\nUnited States\tSaint Paul\t44.94441\t-93.09328\t235.0\nUnited States\tLexington-Fayette\t38.0498\t-84.45855\t312.0\nUnited States\tAnchorage\t61.218056\t-149.90028\t0.0\nUnited States\tPlano\t33.019844\t-96.69888\t200.0\nUnited States\tSaint Petersburg\t27.77086\t-82.67927\t14.0\nUnited States\tLouisville\t38.25424\t-85.75941\t128.0\nUnited States\tLincoln\t40.8\t-96.66696\t382.0\nUnited States\tGlendale\t33.53865\t-112.18599\t336.0\nUnited States\tHenderson\t36.0397\t-114.98193\t580.0\nUnited States\tJersey City\t40.728157\t-74.077644\t14.0\nUnited States\tNorfolk\t36.846813\t-76.28522\t3.0\nUnited States\tChandler\t33.30616\t-111.84125\t380.0\nUnited States\tGreensboro\t36.072636\t-79.79198\t244.0\nUnited States\tBirmingham\t33.52066\t-86.80249\t173.0\nUnited States\tFort Wayne\t41.130604\t-85.12886\t243.0\nUnited States\tLexington\t37.98869\t-84.477715\t298.0\nUnited States\tHialeah\t25.857595\t-80.27811\t2.0\nUnited States\tMadison\t43.07305\t-89.40123\t270.0\nUnited States\tBaton Rouge\t30.450747\t-91.15455\t0.0\nUnited States\tGarland\t32.912624\t-96.638885\t165.0\nUnited States\tModesto\t37.639095\t-120.99688\t12.0\nUnited States\tParadise\t36.097195\t-115.14667\t626.0\nUnited States\tChula Vista\t32.640053\t-117.0842\t23.0\nUnited States\tLubbock\t33.57786\t-101.85516\t976.0\nUnited States\tRochester\t43.154785\t-77.615555\t155.0\nUnited States\tLaredo\t27.506407\t-99.507545\t115.0\nUnited States\tAkron\t41.081444\t-81.519005\t290.0\nUnited States\tOrlando\t28.538336\t-81.379234\t34.0\nUnited States\tDurham\t35.994034\t-78.89862\t122.0\nUnited States\tNorth Glendale\t34.160564\t-118.26452\t174.0\nUnited States\tScottsdale\t33.50921\t-111.89903\t386.0\n" }, "dateCreated": "Jul 19, 2015 8:12:20 PM", - "dateStarted": "Jul 19, 2015 8:20:26 PM", - "dateFinished": "Jul 19, 2015 8:20:32 PM", - "status": "ERROR", + "dateStarted": "Jul 19, 2015 8:31:10 PM", + "dateFinished": "Jul 19, 2015 8:31:10 PM", + "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { From f0b54b3f1aefc9e189821ccdaf49eb341d6e62ef Mon Sep 17 00:00:00 2001 From: madhuka Date: Mon, 20 Jul 2015 17:06:04 +0530 Subject: [PATCH 06/31] adding markers --- .../notebook/paragraph/paragraph.controller.js | 18 ++++++++++++++++++ .../src/app/notebook/paragraph/paragraph.html | 3 +-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 460a9e1ccfa..ac7268165db 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -747,6 +747,24 @@ angular.module('zeppelinWebApp') setTable($scope.paragraph.result, refresh); } if (!type || type === 'mapChart') { + //TODO just testing markers + //setMap($scope.paragraph.result, refresh); + $scope.markers = { + SriLanka: {, + lat: 6.923117, + lng: 79.861184, + message: "Sri Lanka", + focus: true, + draggable: true + }, + India: {, + lat: 13.811499, + lng: 77.818405, + message: "India", + focus: false, + draggable: false + } +} //setting the map : To-DO Moving the map for separate file /* angular.extend($scope, { defaults: { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index b7878ef1c8e..8a026d6c121 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -340,8 +340,7 @@
- -
+
Date: Mon, 20 Jul 2015 19:39:55 +0530 Subject: [PATCH 07/31] fixing multi maker --- notebook/2AU6K8KS2/note.json | 22 +++++++++---------- .../paragraph/paragraph.controller.js | 10 ++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json index ece8c513e91..4432bf6e0a9 100644 --- a/notebook/2AU6K8KS2/note.json +++ b/notebook/2AU6K8KS2/note.json @@ -13,7 +13,7 @@ "groups": [], "scatter": {} }, - "editorHide": true, + "editorHide": false, "tableHide": false }, "settings": { @@ -28,8 +28,8 @@ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" }, "dateCreated": "Jul 19, 2015 7:16:22 PM", - "dateStarted": "Jul 19, 2015 8:20:26 PM", - "dateFinished": "Jul 19, 2015 8:20:26 PM", + "dateStarted": "Jul 20, 2015 7:31:02 PM", + "dateFinished": "Jul 20, 2015 7:31:02 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -56,11 +56,11 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[37] at parallelize at \u003cconsole\u003e:47\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[40] at map at \u003cconsole\u003e:47\n" + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[12] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[15] at map at \u003cconsole\u003e:39\n" }, "dateCreated": "Jul 19, 2015 7:26:54 PM", - "dateStarted": "Jul 19, 2015 8:31:04 PM", - "dateFinished": "Jul 19, 2015 8:31:07 PM", + "dateStarted": "Jul 20, 2015 7:31:02 PM", + "dateFinished": "Jul 20, 2015 7:31:05 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -69,7 +69,7 @@ "config": { "colWidth": 12.0, "graph": { - "mode": "mapChart", + "mode": "table", "height": 300.0, "optionOpen": false, "keys": [ @@ -113,8 +113,8 @@ "msg": "Country\tName\tlat\tlan\tAltitude\nUnited States\tNew York City\t40.714268\t-74.005974\t2.0\nUnited States\tLos Angeles\t34.052235\t-118.24368\t115.0\nUnited States\tChicago\t41.850033\t-87.650055\t181.0\nUnited States\tHouston\t29.763283\t-95.36327\t15.0\nUnited States\tPhiladelphia\t39.952335\t-75.16379\t8.0\nUnited States\tPhoenix\t33.448376\t-112.074036\t0.0\nUnited States\tSan Antonio\t29.424122\t-98.49363\t196.0\nUnited States\tSan Diego\t32.71533\t-117.15726\t26.0\nUnited States\tDallas\t32.783054\t-96.80666\t137.0\nUnited States\tDetroit\t42.33143\t-83.04575\t181.0\nUnited States\tSan Jose\t37.339386\t-121.89496\t24.0\nUnited States\tJacksonville\t30.332184\t-81.65565\t7.0\nUnited States\tIndianapolis\t39.768375\t-86.15804\t222.0\nUnited States\tColumbus\t39.961174\t-82.998795\t237.0\nUnited States\tSan Francisco\t37.77493\t-122.41942\t60.0\nUnited States\tAustin\t30.267153\t-97.74306\t163.0\nUnited States\tMemphis\t35.149536\t-90.04898\t75.0\nUnited States\tNew South Memphis\t35.086758\t-90.056755\t74.0\nUnited States\tFort Worth\t32.72541\t-97.32085\t199.0\nUnited States\tBaltimore\t39.290386\t-76.61219\t2.0\nUnited States\tCharlotte\t35.227085\t-80.843124\t229.0\nUnited States\tBoston\t42.35843\t-71.05978\t11.0\nUnited States\tMilwaukee\t43.038902\t-87.90647\t195.0\nUnited States\tSouth Boston\t42.33343\t-71.04949\t10.0\nUnited States\tSeattle\t47.60621\t-122.33207\t60.0\nUnited States\tEl Paso\t31.75872\t-106.48693\t1147.0\nUnited States\tDenver\t39.739155\t-104.9847\t1598.0\nUnited States\tWashington\t38.89511\t-77.03637\t11.0\nUnited States\tPortland\t45.523453\t-122.67621\t61.0\nUnited States\tOklahoma City\t35.46756\t-97.516426\t364.0\nUnited States\tNashville\t36.16589\t-86.78445\t151.0\nUnited States\tTucson\t32.221745\t-110.926476\t751.0\nUnited States\tAlbuquerque\t35.08449\t-106.65114\t1510.0\nUnited States\tNew Orleans\t29.954649\t-90.07507\t0.0\nUnited States\tLong Beach\t33.766964\t-118.18923\t1.0\nUnited States\tLas Vegas\t36.17497\t-115.13722\t609.0\nUnited States\tSacramento\t38.581573\t-121.4944\t5.0\nUnited States\tFresno\t36.747726\t-119.77237\t91.0\nUnited States\tCleveland\t41.499496\t-81.69541\t185.0\nUnited States\tNorth Kansas City\t39.130005\t-94.56218\t223.0\nUnited States\tKansas City\t39.099728\t-94.57857\t271.0\nUnited States\tVirginia Beach\t36.85293\t-75.97798\t3.0\nUnited States\tAtlanta\t33.748997\t-84.387985\t305.0\nUnited States\tOakland\t37.80437\t-122.270805\t0.0\nUnited States\tMesa\t33.422268\t-111.82264\t386.0\nUnited States\tTulsa\t36.15398\t-95.992775\t213.0\nUnited States\tOmaha\t41.25861\t-95.93779\t0.0\nUnited States\tMiami\t25.774265\t-80.19366\t2.0\nUnited States\tHonolulu\t21.306944\t-157.85834\t0.0\nUnited States\tMinneapolis\t44.979965\t-93.26384\t259.0\nUnited States\tColorado Springs\t38.83388\t-104.821365\t1829.0\nUnited States\tWichita\t37.692238\t-97.33755\t402.0\nUnited States\tSanta Ana\t33.745575\t-117.867836\t30.0\nUnited States\tWest Raleigh\t35.78682\t-78.663895\t122.0\nUnited States\tArlington\t32.735687\t-97.10806\t184.0\nUnited States\tAnaheim\t33.835293\t-117.914505\t40.0\nUnited States\tTampa\t27.947521\t-82.45843\t1.0\nUnited States\tSaint Louis\t38.627274\t-90.19789\t135.0\nUnited States\tPittsburgh\t40.440624\t-79.99589\t235.0\nUnited States\tToledo\t41.663937\t-83.555214\t191.0\nUnited States\tCincinnati\t39.162003\t-84.45689\t203.0\nUnited States\tRiverside\t33.95335\t-117.39616\t262.0\nUnited States\tBakersfield\t35.37329\t-119.018715\t122.0\nUnited States\tStockton\t37.957703\t-121.29078\t4.0\nUnited States\tNewark\t40.735657\t-74.17236\t16.0\nUnited States\tBuffalo\t42.886448\t-78.87837\t183.0\nUnited States\tCorpus Christi\t27.800583\t-97.39638\t6.0\nUnited States\tAurora\t39.72943\t-104.83192\t1647.0\nUnited States\tRaleigh\t35.772095\t-78.63861\t91.0\nUnited States\tSaint Paul\t44.94441\t-93.09328\t235.0\nUnited States\tLexington-Fayette\t38.0498\t-84.45855\t312.0\nUnited States\tAnchorage\t61.218056\t-149.90028\t0.0\nUnited States\tPlano\t33.019844\t-96.69888\t200.0\nUnited States\tSaint Petersburg\t27.77086\t-82.67927\t14.0\nUnited States\tLouisville\t38.25424\t-85.75941\t128.0\nUnited States\tLincoln\t40.8\t-96.66696\t382.0\nUnited States\tGlendale\t33.53865\t-112.18599\t336.0\nUnited States\tHenderson\t36.0397\t-114.98193\t580.0\nUnited States\tJersey City\t40.728157\t-74.077644\t14.0\nUnited States\tNorfolk\t36.846813\t-76.28522\t3.0\nUnited States\tChandler\t33.30616\t-111.84125\t380.0\nUnited States\tGreensboro\t36.072636\t-79.79198\t244.0\nUnited States\tBirmingham\t33.52066\t-86.80249\t173.0\nUnited States\tFort Wayne\t41.130604\t-85.12886\t243.0\nUnited States\tLexington\t37.98869\t-84.477715\t298.0\nUnited States\tHialeah\t25.857595\t-80.27811\t2.0\nUnited States\tMadison\t43.07305\t-89.40123\t270.0\nUnited States\tBaton Rouge\t30.450747\t-91.15455\t0.0\nUnited States\tGarland\t32.912624\t-96.638885\t165.0\nUnited States\tModesto\t37.639095\t-120.99688\t12.0\nUnited States\tParadise\t36.097195\t-115.14667\t626.0\nUnited States\tChula Vista\t32.640053\t-117.0842\t23.0\nUnited States\tLubbock\t33.57786\t-101.85516\t976.0\nUnited States\tRochester\t43.154785\t-77.615555\t155.0\nUnited States\tLaredo\t27.506407\t-99.507545\t115.0\nUnited States\tAkron\t41.081444\t-81.519005\t290.0\nUnited States\tOrlando\t28.538336\t-81.379234\t34.0\nUnited States\tDurham\t35.994034\t-78.89862\t122.0\nUnited States\tNorth Glendale\t34.160564\t-118.26452\t174.0\nUnited States\tScottsdale\t33.50921\t-111.89903\t386.0\n" }, "dateCreated": "Jul 19, 2015 8:12:20 PM", - "dateStarted": "Jul 19, 2015 8:31:10 PM", - "dateFinished": "Jul 19, 2015 8:31:10 PM", + "dateStarted": "Jul 20, 2015 7:37:58 PM", + "dateFinished": "Jul 20, 2015 7:37:59 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -143,8 +143,8 @@ "type": "TEXT" }, "dateCreated": "Jul 19, 2015 8:17:46 PM", - "dateStarted": "Jul 19, 2015 8:20:32 PM", - "dateFinished": "Jul 19, 2015 8:20:32 PM", + "dateStarted": "Jul 20, 2015 7:31:05 PM", + "dateFinished": "Jul 20, 2015 7:31:06 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index ac7268165db..8964509e566 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -747,24 +747,22 @@ angular.module('zeppelinWebApp') setTable($scope.paragraph.result, refresh); } if (!type || type === 'mapChart') { - //TODO just testing markers - //setMap($scope.paragraph.result, refresh); $scope.markers = { - SriLanka: {, + SriLanka: { lat: 6.923117, lng: 79.861184, message: "Sri Lanka", focus: true, - draggable: true + draggable: false }, - India: {, + India: { lat: 13.811499, lng: 77.818405, message: "India", focus: false, draggable: false } -} + } //setting the map : To-DO Moving the map for separate file /* angular.extend($scope, { defaults: { From 7b78ff6ef3fea894568baf11b915f4c107be954c Mon Sep 17 00:00:00 2001 From: madhuka Date: Fri, 24 Jul 2015 16:53:34 +0530 Subject: [PATCH 08/31] adding map rendering on dynamic data --- .../paragraph/paragraph.controller.js | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 8964509e566..ae83a11d329 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -747,28 +747,7 @@ angular.module('zeppelinWebApp') setTable($scope.paragraph.result, refresh); } if (!type || type === 'mapChart') { - $scope.markers = { - SriLanka: { - lat: 6.923117, - lng: 79.861184, - message: "Sri Lanka", - focus: true, - draggable: false - }, - India: { - lat: 13.811499, - lng: 77.818405, - message: "India", - focus: false, - draggable: false - } - } - //setting the map : To-DO Moving the map for separate file -/* angular.extend($scope, { - defaults: { - scrollWheelZoom: false - } - });*/ + setMapChart(type, $scope.paragraph.result, refresh); } else { setD3Chart(type, $scope.paragraph.result, refresh); @@ -874,6 +853,33 @@ angular.module('zeppelinWebApp') }; + var setMapChart = function(type, data, refresh) { + if (!$scope.chart[type]) { + + var mapChartModel = function(d) { + var key = d[1].replace("-", "_");; + var obj = {}; + obj[key] = { + lat: parseFloat(d[2]), + lng: parseFloat(d[3]), + message: d[1], + focus: true, + draggable: false + }; + console.log(obj); + return obj; + }; + + var newmarkers = {} + for (var i = 0; i < data.rows.length; i++) { + var row = data.rows[i]; + var rowMarker = mapChartModel(row); + newmarkers = $.extend(newmarkers, rowMarker); + } + } + $scope.markers = newmarkers; + }; + var setD3Chart = function(type, data, refresh) { if (!$scope.chart[type]) { var chart = nv.models[type](); From 75ff02ded6fdca34177016e173c40ee8695d4ca1 Mon Sep 17 00:00:00 2001 From: madhuka Date: Fri, 24 Jul 2015 18:42:03 +0530 Subject: [PATCH 09/31] adding map bounds --- .../notebook/paragraph/paragraph.controller.js | 17 +++++++++++++---- .../src/app/notebook/paragraph/paragraph.html | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index ae83a11d329..ab46e0ff047 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -17,7 +17,7 @@ angular.module('zeppelinWebApp') .controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv) { + $timeout, $compile, websocketMsgSrv, leafletBoundsHelpers) { $scope.paragraph = null; $scope.editor = null; @@ -854,11 +854,16 @@ angular.module('zeppelinWebApp') }; var setMapChart = function(type, data, refresh) { + var latArr = [], + lngArr = [], + newmarkers = {}; if (!$scope.chart[type]) { var mapChartModel = function(d) { - var key = d[1].replace("-", "_");; + var key = d[1].replace('-', '_'); var obj = {}; + latArr.push(Math.round(parseFloat(d[2]))); + lngArr.push(Math.round(parseFloat(d[3]))); obj[key] = { lat: parseFloat(d[2]), lng: parseFloat(d[3]), @@ -866,11 +871,9 @@ angular.module('zeppelinWebApp') focus: true, draggable: false }; - console.log(obj); return obj; }; - var newmarkers = {} for (var i = 0; i < data.rows.length; i++) { var row = data.rows[i]; var rowMarker = mapChartModel(row); @@ -878,6 +881,12 @@ angular.module('zeppelinWebApp') } } $scope.markers = newmarkers; + var bounds = leafletBoundsHelpers.createBoundsFromArray([ + [Math.max.apply(Math, latArr), Math.max.apply(Math, lngArr)], + [Math.min.apply(Math, latArr), Math.min.apply(Math, lngArr)] + ]); + $scope.bounds = bounds; + $scope.center = {}; }; var setD3Chart = function(type, data, refresh) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 8a026d6c121..dd48e35971d 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -340,7 +340,7 @@
- +
Date: Fri, 24 Jul 2015 19:04:20 +0530 Subject: [PATCH 10/31] adding dynamic size feature for map --- zeppelin-web/src/app/notebook/notebook.css | 9 +++++++++ .../src/app/notebook/paragraph/paragraph.controller.js | 4 ++++ zeppelin-web/src/app/notebook/paragraph/paragraph.html | 5 +++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/notebook.css b/zeppelin-web/src/app/notebook/notebook.css index 0d3de89b64d..00bf86757e4 100644 --- a/zeppelin-web/src/app/notebook/notebook.css +++ b/zeppelin-web/src/app/notebook/notebook.css @@ -456,6 +456,15 @@ margin-bottom: 5px !important; } +.angular-leaflet-map { + height: 300px; + width: 100%; +} + +.graphContainer .mapChart { + overflow: hidden; +} + .allFields { margin-bottom: 10px; } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index ab46e0ff047..04fdc612420 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -887,6 +887,10 @@ angular.module('zeppelinWebApp') ]); $scope.bounds = bounds; $scope.center = {}; + + // set map chart height + var height = $scope.paragraph.config.graph.height; + $('#p'+$scope.paragraph.id+'_mapChart').height(height); }; var setD3Chart = function(type, data, refresh) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index dd48e35971d..7478e870aa3 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -339,8 +339,9 @@
- + id="p{{paragraph.id}}_mapChart" + class="mapChart"> +
Date: Fri, 24 Jul 2015 20:38:07 +0530 Subject: [PATCH 11/31] adding data validator service --- .../paragraph/paragraph.controller.js | 3 +- .../data-validator/data-validator-factory.js | 43 +++++++++++++++++++ .../data-validator/data-validator-service.js | 24 +++++++++++ zeppelin-web/src/index.html | 2 + 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 zeppelin-web/src/components/data-validator/data-validator-factory.js create mode 100644 zeppelin-web/src/components/data-validator/data-validator-service.js diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 04fdc612420..58f72271af0 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -17,7 +17,7 @@ angular.module('zeppelinWebApp') .controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, leafletBoundsHelpers) { + $timeout, $compile, websocketMsgSrv, leafletBoundsHelpers, dataValidatorSrv) { $scope.paragraph = null; $scope.editor = null; @@ -889,6 +889,7 @@ angular.module('zeppelinWebApp') $scope.center = {}; // set map chart height + dataValidatorSrv.validateMapData(data); var height = $scope.paragraph.config.graph.height; $('#p'+$scope.paragraph.id+'_mapChart').height(height); }; diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js new file mode 100644 index 00000000000..38e19702a98 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { + + var dataValidator = function() { + this.schema = null; + this.error = true; + this.msg = null; + }; + + dataValidator.setDataschema = function(dataschema) { + dataValidator.dataschema = dataschema; + }; + //TODO - data validation process inprogress. + dataValidator.checkData = function(data) { + if (basicCheck(data)) { + dataValidator.error = false; + } + }; + + function basicCheck(data) { + if (data.code) { + return true; + } else { + return false; + } + } + + return dataValidator; +}); diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js new file mode 100644 index 00000000000..7fea3fd4844 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator) { + + this.validateMapData = function(data) { + var validator = dataValidator; + validator.checkData(data); + return validator; + }; + +}); diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 08d862d51aa..3a1d64b481a 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -142,6 +142,8 @@ + + From b501fb829332948a53c5a7760e5662710b7026c5 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 25 Jul 2015 08:45:42 +0530 Subject: [PATCH 12/31] updating the sample --- notebook/2AU6K8KS2/note.json | 124 +++++++++++++++++++++++++++++------ 1 file changed, 103 insertions(+), 21 deletions(-) diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json index 4432bf6e0a9..0af0d4bf661 100644 --- a/notebook/2AU6K8KS2/note.json +++ b/notebook/2AU6K8KS2/note.json @@ -14,7 +14,7 @@ "scatter": {} }, "editorHide": false, - "tableHide": false + "tableHide": true }, "settings": { "params": {}, @@ -28,13 +28,13 @@ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" }, "dateCreated": "Jul 19, 2015 7:16:22 PM", - "dateStarted": "Jul 20, 2015 7:31:02 PM", - "dateFinished": "Jul 20, 2015 7:31:02 PM", + "dateStarted": "Jul 25, 2015 8:43:03 AM", + "dateFinished": "Jul 25, 2015 8:43:03 AM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { - "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(4).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load map data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(4).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use myMap.registerTempTable(\"myMap\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", "config": { "colWidth": 12.0, "graph": { @@ -56,20 +56,20 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[12] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[15] at map at \u003cconsole\u003e:39\n" + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[28] at parallelize at \u003cconsole\u003e:35\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[31] at map at \u003cconsole\u003e:35\n" }, "dateCreated": "Jul 19, 2015 7:26:54 PM", - "dateStarted": "Jul 20, 2015 7:31:02 PM", - "dateFinished": "Jul 20, 2015 7:31:05 PM", + "dateStarted": "Jul 25, 2015 8:42:17 AM", + "dateFinished": "Jul 25, 2015 8:42:24 AM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { - "text": "%sql \nselect * from myMap", + "text": "%sql \nselect * from myMap\nwhere Country \u003d \"${Country\u003d\u0027United States\u0027}\"", "config": { - "colWidth": 12.0, + "colWidth": 6.0, "graph": { - "mode": "table", + "mode": "mapChart", "height": 300.0, "optionOpen": false, "keys": [ @@ -99,11 +99,20 @@ "aggr": "sum" } } - } + }, + "editorHide": true }, "settings": { - "params": {}, - "forms": {} + "params": { + "Country": "United States" + }, + "forms": { + "Country": { + "name": "Country", + "defaultValue": "\u0027United States\u0027", + "hidden": false + } + } }, "jobName": "paragraph_1437316940442_271264979", "id": "20150719-201220_10279868", @@ -113,8 +122,71 @@ "msg": "Country\tName\tlat\tlan\tAltitude\nUnited States\tNew York City\t40.714268\t-74.005974\t2.0\nUnited States\tLos Angeles\t34.052235\t-118.24368\t115.0\nUnited States\tChicago\t41.850033\t-87.650055\t181.0\nUnited States\tHouston\t29.763283\t-95.36327\t15.0\nUnited States\tPhiladelphia\t39.952335\t-75.16379\t8.0\nUnited States\tPhoenix\t33.448376\t-112.074036\t0.0\nUnited States\tSan Antonio\t29.424122\t-98.49363\t196.0\nUnited States\tSan Diego\t32.71533\t-117.15726\t26.0\nUnited States\tDallas\t32.783054\t-96.80666\t137.0\nUnited States\tDetroit\t42.33143\t-83.04575\t181.0\nUnited States\tSan Jose\t37.339386\t-121.89496\t24.0\nUnited States\tJacksonville\t30.332184\t-81.65565\t7.0\nUnited States\tIndianapolis\t39.768375\t-86.15804\t222.0\nUnited States\tColumbus\t39.961174\t-82.998795\t237.0\nUnited States\tSan Francisco\t37.77493\t-122.41942\t60.0\nUnited States\tAustin\t30.267153\t-97.74306\t163.0\nUnited States\tMemphis\t35.149536\t-90.04898\t75.0\nUnited States\tNew South Memphis\t35.086758\t-90.056755\t74.0\nUnited States\tFort Worth\t32.72541\t-97.32085\t199.0\nUnited States\tBaltimore\t39.290386\t-76.61219\t2.0\nUnited States\tCharlotte\t35.227085\t-80.843124\t229.0\nUnited States\tBoston\t42.35843\t-71.05978\t11.0\nUnited States\tMilwaukee\t43.038902\t-87.90647\t195.0\nUnited States\tSouth Boston\t42.33343\t-71.04949\t10.0\nUnited States\tSeattle\t47.60621\t-122.33207\t60.0\nUnited States\tEl Paso\t31.75872\t-106.48693\t1147.0\nUnited States\tDenver\t39.739155\t-104.9847\t1598.0\nUnited States\tWashington\t38.89511\t-77.03637\t11.0\nUnited States\tPortland\t45.523453\t-122.67621\t61.0\nUnited States\tOklahoma City\t35.46756\t-97.516426\t364.0\nUnited States\tNashville\t36.16589\t-86.78445\t151.0\nUnited States\tTucson\t32.221745\t-110.926476\t751.0\nUnited States\tAlbuquerque\t35.08449\t-106.65114\t1510.0\nUnited States\tNew Orleans\t29.954649\t-90.07507\t0.0\nUnited States\tLong Beach\t33.766964\t-118.18923\t1.0\nUnited States\tLas Vegas\t36.17497\t-115.13722\t609.0\nUnited States\tSacramento\t38.581573\t-121.4944\t5.0\nUnited States\tFresno\t36.747726\t-119.77237\t91.0\nUnited States\tCleveland\t41.499496\t-81.69541\t185.0\nUnited States\tNorth Kansas City\t39.130005\t-94.56218\t223.0\nUnited States\tKansas City\t39.099728\t-94.57857\t271.0\nUnited States\tVirginia Beach\t36.85293\t-75.97798\t3.0\nUnited States\tAtlanta\t33.748997\t-84.387985\t305.0\nUnited States\tOakland\t37.80437\t-122.270805\t0.0\nUnited States\tMesa\t33.422268\t-111.82264\t386.0\nUnited States\tTulsa\t36.15398\t-95.992775\t213.0\nUnited States\tOmaha\t41.25861\t-95.93779\t0.0\nUnited States\tMiami\t25.774265\t-80.19366\t2.0\nUnited States\tHonolulu\t21.306944\t-157.85834\t0.0\nUnited States\tMinneapolis\t44.979965\t-93.26384\t259.0\nUnited States\tColorado Springs\t38.83388\t-104.821365\t1829.0\nUnited States\tWichita\t37.692238\t-97.33755\t402.0\nUnited States\tSanta Ana\t33.745575\t-117.867836\t30.0\nUnited States\tWest Raleigh\t35.78682\t-78.663895\t122.0\nUnited States\tArlington\t32.735687\t-97.10806\t184.0\nUnited States\tAnaheim\t33.835293\t-117.914505\t40.0\nUnited States\tTampa\t27.947521\t-82.45843\t1.0\nUnited States\tSaint Louis\t38.627274\t-90.19789\t135.0\nUnited States\tPittsburgh\t40.440624\t-79.99589\t235.0\nUnited States\tToledo\t41.663937\t-83.555214\t191.0\nUnited States\tCincinnati\t39.162003\t-84.45689\t203.0\nUnited States\tRiverside\t33.95335\t-117.39616\t262.0\nUnited States\tBakersfield\t35.37329\t-119.018715\t122.0\nUnited States\tStockton\t37.957703\t-121.29078\t4.0\nUnited States\tNewark\t40.735657\t-74.17236\t16.0\nUnited States\tBuffalo\t42.886448\t-78.87837\t183.0\nUnited States\tCorpus Christi\t27.800583\t-97.39638\t6.0\nUnited States\tAurora\t39.72943\t-104.83192\t1647.0\nUnited States\tRaleigh\t35.772095\t-78.63861\t91.0\nUnited States\tSaint Paul\t44.94441\t-93.09328\t235.0\nUnited States\tLexington-Fayette\t38.0498\t-84.45855\t312.0\nUnited States\tAnchorage\t61.218056\t-149.90028\t0.0\nUnited States\tPlano\t33.019844\t-96.69888\t200.0\nUnited States\tSaint Petersburg\t27.77086\t-82.67927\t14.0\nUnited States\tLouisville\t38.25424\t-85.75941\t128.0\nUnited States\tLincoln\t40.8\t-96.66696\t382.0\nUnited States\tGlendale\t33.53865\t-112.18599\t336.0\nUnited States\tHenderson\t36.0397\t-114.98193\t580.0\nUnited States\tJersey City\t40.728157\t-74.077644\t14.0\nUnited States\tNorfolk\t36.846813\t-76.28522\t3.0\nUnited States\tChandler\t33.30616\t-111.84125\t380.0\nUnited States\tGreensboro\t36.072636\t-79.79198\t244.0\nUnited States\tBirmingham\t33.52066\t-86.80249\t173.0\nUnited States\tFort Wayne\t41.130604\t-85.12886\t243.0\nUnited States\tLexington\t37.98869\t-84.477715\t298.0\nUnited States\tHialeah\t25.857595\t-80.27811\t2.0\nUnited States\tMadison\t43.07305\t-89.40123\t270.0\nUnited States\tBaton Rouge\t30.450747\t-91.15455\t0.0\nUnited States\tGarland\t32.912624\t-96.638885\t165.0\nUnited States\tModesto\t37.639095\t-120.99688\t12.0\nUnited States\tParadise\t36.097195\t-115.14667\t626.0\nUnited States\tChula Vista\t32.640053\t-117.0842\t23.0\nUnited States\tLubbock\t33.57786\t-101.85516\t976.0\nUnited States\tRochester\t43.154785\t-77.615555\t155.0\nUnited States\tLaredo\t27.506407\t-99.507545\t115.0\nUnited States\tAkron\t41.081444\t-81.519005\t290.0\nUnited States\tOrlando\t28.538336\t-81.379234\t34.0\nUnited States\tDurham\t35.994034\t-78.89862\t122.0\nUnited States\tNorth Glendale\t34.160564\t-118.26452\t174.0\nUnited States\tScottsdale\t33.50921\t-111.89903\t386.0\n" }, "dateCreated": "Jul 19, 2015 8:12:20 PM", - "dateStarted": "Jul 20, 2015 7:37:58 PM", - "dateFinished": "Jul 20, 2015 7:37:59 PM", + "dateStarted": "Jul 25, 2015 8:42:17 AM", + "dateFinished": "Jul 25, 2015 8:42:24 AM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect * from myMap\nwhere Altitude \u003e ${Altitude\u003d300}", + "config": { + "colWidth": 6.0, + "graph": { + "mode": "mapChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "Country", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "Country", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + } + }, + "editorHide": true + }, + "settings": { + "params": { + "Altitude": "400" + }, + "forms": { + "Altitude": { + "name": "Altitude", + "defaultValue": "300", + "hidden": false + } + } + }, + "jobName": "paragraph_1437317266677_-261952669", + "id": "20150719-201746_23407076", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "Country\tName\tlat\tlan\tAltitude\nUnited States\tEl Paso\t31.75872\t-106.48693\t1147.0\nUnited States\tDenver\t39.739155\t-104.9847\t1598.0\nUnited States\tTucson\t32.221745\t-110.926476\t751.0\nUnited States\tAlbuquerque\t35.08449\t-106.65114\t1510.0\nUnited States\tLas Vegas\t36.17497\t-115.13722\t609.0\nUnited States\tColorado Springs\t38.83388\t-104.821365\t1829.0\nUnited States\tWichita\t37.692238\t-97.33755\t402.0\nUnited States\tAurora\t39.72943\t-104.83192\t1647.0\nUnited States\tHenderson\t36.0397\t-114.98193\t580.0\nUnited States\tParadise\t36.097195\t-115.14667\t626.0\nUnited States\tLubbock\t33.57786\t-101.85516\t976.0\n" + }, + "dateCreated": "Jul 19, 2015 8:17:46 PM", + "dateStarted": "Jul 25, 2015 8:42:24 AM", + "dateFinished": "Jul 25, 2015 8:42:24 AM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -136,22 +208,32 @@ "params": {}, "forms": {} }, - "jobName": "paragraph_1437317266677_-261952669", - "id": "20150719-201746_23407076", + "jobName": "paragraph_1437793749347_527943396", + "id": "20150725-083909_25279730", "result": { "code": "SUCCESS", "type": "TEXT" }, - "dateCreated": "Jul 19, 2015 8:17:46 PM", - "dateStarted": "Jul 20, 2015 7:31:05 PM", - "dateFinished": "Jul 20, 2015 7:31:06 PM", + "dateCreated": "Jul 25, 2015 8:39:09 AM", + "dateStarted": "Jul 25, 2015 8:42:24 AM", + "dateFinished": "Jul 25, 2015 8:42:24 AM", "status": "FINISHED", "progressUpdateIntervalMs": 500 } ], "name": "Map Visualization Tutorial", "id": "2AU6K8KS2", - "angularObjects": {}, + "angularObjects": { + "2AVFWDXQ8": [], + "2AW52KKF2": [], + "2AWBECUSM": [], + "2ATG5P9DG": [], + "2AUPDKZH1": [], + "2AVUJ5W2S": [], + "2AUSHYMSD": [], + "2ATHTSAJV": [], + "2AUZPG9UC": [] + }, "config": { "looknfeel": "default" }, From 8217b4f6c3efb4e27a005f01ad997b200cd2048e Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 26 Jul 2015 12:56:28 +0530 Subject: [PATCH 13/31] connceting validator services and factory --- .../src/app/notebook/paragraph/paragraph.controller.js | 4 +++- .../components/data-validator/data-validator-factory.js | 8 +++----- .../components/data-validator/data-validator-service.js | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 58f72271af0..dacc467b5d8 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -880,6 +880,9 @@ angular.module('zeppelinWebApp') newmarkers = $.extend(newmarkers, rowMarker); } } + //data model validator + var msg = dataValidatorSrv.validateMapData(data); + $scope.markers = newmarkers; var bounds = leafletBoundsHelpers.createBoundsFromArray([ [Math.max.apply(Math, latArr), Math.max.apply(Math, lngArr)], @@ -889,7 +892,6 @@ angular.module('zeppelinWebApp') $scope.center = {}; // set map chart height - dataValidatorSrv.validateMapData(data); var height = $scope.paragraph.config.graph.height; $('#p'+$scope.paragraph.id+'_mapChart').height(height); }; diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index 38e19702a98..f0013ed5794 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -19,15 +19,13 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { this.schema = null; this.error = true; this.msg = null; + this.checkData = checkData; }; - dataValidator.setDataschema = function(dataschema) { - dataValidator.dataschema = dataschema; - }; //TODO - data validation process inprogress. - dataValidator.checkData = function(data) { + function checkData(data) { if (basicCheck(data)) { - dataValidator.error = false; + this.error = false; } }; diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 7fea3fd4844..664d15ce3ee 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -16,8 +16,10 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator) { this.validateMapData = function(data) { - var validator = dataValidator; + var validator = new dataValidator(); + validator.schema = {type:{},model:{}}; validator.checkData(data); + //var msg = {'error':validator.error, 'msg':validator.msg}; return validator; }; From 10204ab2e1ec8cb658b3e0ef85fb571380587f17 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 26 Jul 2015 17:08:58 +0530 Subject: [PATCH 14/31] adding dataCheckValidator --- .../data-validator/data-validator-factory.js | 42 +++++++++++++++---- .../data-validator/data-validator-service.js | 8 ++-- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index f0013ed5794..fe5a4f607ed 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -22,15 +22,43 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { this.checkData = checkData; }; - //TODO - data validation process inprogress. - function checkData(data) { - if (basicCheck(data)) { - this.error = false; + function checkData(data, schema) { + console.log(schema); + if (basicCheck(data, schema)) { + return false; } - }; + } + + function basicCheck(data, schema) { + if (data.code && data.rows) { + rowCheck(data.rows, 3, schema); + return true; + } else { + return false; + } + } + + function rowCheck(rowData, num, schema) { + if (rowData) { + for (var i = 0; i < rowData.length; i++) { + var row = rowData[i]; + dataCheckValidator(row, schema); + } + return true; + } else { + return false; + } + } - function basicCheck(data) { - if (data.code) { + function dataCheckValidator(record, schema) { + console.log(schema.type.length); + if (record) { + for (var i = 0; i < schema.type.length; i++) { + //TODOto remove logs code inprogress on return msg + console.log('data is validated'); + console.log(record[i]); + console.log(isNaN(record[i]) === (schema.type[i] === 'string')); + } return true; } else { return false; diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 664d15ce3ee..3a7041e7180 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -16,11 +16,11 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator) { this.validateMapData = function(data) { - var validator = new dataValidator(); - validator.schema = {type:{},model:{}}; - validator.checkData(data); + var mapValidator = new dataValidator(); + mapValidator.schema = {type:['string','string','number','number','number']}; + mapValidator.checkData(data,mapValidator.schema); //var msg = {'error':validator.error, 'msg':validator.msg}; - return validator; + return mapValidator; }; }); From fee29557b341136425f8a6d22a6e34122bb75f9b Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 26 Jul 2015 21:24:44 +0530 Subject: [PATCH 15/31] adding schema config and error msg building --- .../paragraph/paragraph.controller.js | 9 ++++- .../data-validator/data-validator-factory.js | 39 ++++++++++++++----- .../data-validator/data-validator-schema.js | 21 ++++++++++ .../data-validator/data-validator-service.js | 8 ++-- zeppelin-web/src/index.html | 1 + 5 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 zeppelin-web/src/components/data-validator/data-validator-schema.js diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index dacc467b5d8..8132f82bcfb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -879,9 +879,16 @@ angular.module('zeppelinWebApp') var rowMarker = mapChartModel(row); newmarkers = $.extend(newmarkers, rowMarker); } - } + } //data model validator var msg = dataValidatorSrv.validateMapData(data); + console.log(msg); + //TODO- warning need to be show in here. currently it is only printing. + //need to know what is the standard way of warning in zepplin. + if(msg.error){ + console.log('Data is not matching for schema'); + console.log(msg.msg); + } $scope.markers = newmarkers; var bounds = leafletBoundsHelpers.createBoundsFromArray([ diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index fe5a4f607ed..22c7c2b35ed 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -15,25 +15,38 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { + var msg = ''; + var errorStatus = true; var dataValidator = function() { this.schema = null; - this.error = true; - this.msg = null; + this.error = getErrorStatus; this.checkData = checkData; + this.getMsg = getMsg; }; function checkData(data, schema) { console.log(schema); if (basicCheck(data, schema)) { - return false; + msg += 'data is exisiting | '; + } else { + msg += 'data does not exisiting | '; } } + function getMsg() { + return msg; + } + + function getErrorStatus() { + return errorStatus; + } + function basicCheck(data, schema) { if (data.code && data.rows) { rowCheck(data.rows, 3, schema); return true; } else { + msg += 'data rows does not exisiting | '; return false; } } @@ -42,10 +55,13 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { if (rowData) { for (var i = 0; i < rowData.length; i++) { var row = rowData[i]; - dataCheckValidator(row, schema); + if (dataCheckValidator(row, schema)) { + msg += 'data record does not mapping to data schema| '; + } } return true; } else { + msg += 'data row does not exisiting | '; return false; } } @@ -54,14 +70,17 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { console.log(schema.type.length); if (record) { for (var i = 0; i < schema.type.length; i++) { - //TODOto remove logs code inprogress on return msg - console.log('data is validated'); - console.log(record[i]); - console.log(isNaN(record[i]) === (schema.type[i] === 'string')); + if (isNaN(record[i]) !== (schema.type[i] === 'string')) { + errorStatus = true; + msg += 'data record ' + (record[i]) + ' is not matching for schema | '; + return true; + } } - return true; - } else { + errorStatus = false; return false; + } else { + msg += 'data record does not exisiting | '; + return true; } } diff --git a/zeppelin-web/src/components/data-validator/data-validator-schema.js b/zeppelin-web/src/components/data-validator/data-validator-schema.js new file mode 100644 index 00000000000..e82b1184fd8 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-schema.js @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp') + .constant('dataModelSchemas', { + 'MapSchema': { + type: ['string', 'string', 'number', 'number', 'number'] + } + }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 3a7041e7180..2bd1adb25f1 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -13,14 +13,14 @@ */ 'use strict'; -angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator) { +angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator, dataModelSchemas) { this.validateMapData = function(data) { var mapValidator = new dataValidator(); - mapValidator.schema = {type:['string','string','number','number','number']}; + mapValidator.schema = dataModelSchemas.MapSchema; mapValidator.checkData(data,mapValidator.schema); - //var msg = {'error':validator.error, 'msg':validator.msg}; - return mapValidator; + var msg = {'error':mapValidator.error(), 'msg':mapValidator.getMsg()}; + return msg; }; }); diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 3a1d64b481a..28b02721b18 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -142,6 +142,7 @@ + From e49aa20531693c77c6a004f0edcac6d01e0a7504 Mon Sep 17 00:00:00 2001 From: madhuka Date: Wed, 29 Jul 2015 23:46:52 +0530 Subject: [PATCH 16/31] adding range validator for map data set --- .../data-validator/data-validator-factory.js | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index 22c7c2b35ed..c50b856eb5e 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -67,13 +67,19 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { } function dataCheckValidator(record, schema) { - console.log(schema.type.length); + //console.log(schema.type.length); if (record) { for (var i = 0; i < schema.type.length; i++) { if (isNaN(record[i]) !== (schema.type[i] === 'string')) { errorStatus = true; msg += 'data record ' + (record[i]) + ' is not matching for schema | '; return true; + }else{ + if(i==2){ + latitudeValidator(record[i]); + }if(i==3){ + longitudeValidator(record[i]); + } } } errorStatus = false; @@ -84,5 +90,35 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { } } + //Latitude measurements range from 0° to (+/–)90°. + function latitudeValidator(record, schema) { + if(record) { + var latitude = parseFloat(record) + if(!(-90 <= latitude <= 90)) { + msg += 'Latitude ' + (record) + ' is not in range | '; + return true; + } + return false; + } else { + msg += 'Latitude record does not exisiting | '; + return true; + } + } + + //Longitude measurements range from 0° to (+/–)180°. + function longitudeValidator(record, schema) { + if(record) { + var longitude = parseFloat(record) + if(!(-180 <= longitude <= 180)) { + msg += 'Longitude' + (record) + ' is not in range | '; + return true; + } + return false; + } else { + msg += 'Longitude record does not exisiting | '; + return true; + } + } + return dataValidator; }); From 077feb82c5d529476271ceed1dd680b56d44e618 Mon Sep 17 00:00:00 2001 From: madhuka Date: Wed, 29 Jul 2015 23:58:36 +0530 Subject: [PATCH 17/31] adding range schema --- .../data-validator/data-validator-factory.js | 12 ++++++------ .../data-validator/data-validator-schema.js | 12 +++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index c50b856eb5e..d626911c432 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -76,9 +76,9 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { return true; }else{ if(i==2){ - latitudeValidator(record[i]); + latitudeValidator(record[i],schema.range); }if(i==3){ - longitudeValidator(record[i]); + longitudeValidator(record[i],schema.range); } } } @@ -94,8 +94,8 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { function latitudeValidator(record, schema) { if(record) { var latitude = parseFloat(record) - if(!(-90 <= latitude <= 90)) { - msg += 'Latitude ' + (record) + ' is not in range | '; + if(!(schema.latitude.low <= latitude <= schema.latitude.high)) { + msg += 'Latitude ' + record + ' is not in range | '; return true; } return false; @@ -109,8 +109,8 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { function longitudeValidator(record, schema) { if(record) { var longitude = parseFloat(record) - if(!(-180 <= longitude <= 180)) { - msg += 'Longitude' + (record) + ' is not in range | '; + if(!(schema.longitude.low <= longitude <= schema.longitude.high)) { + msg += 'Longitude' + record + ' is not in range | '; return true; } return false; diff --git a/zeppelin-web/src/components/data-validator/data-validator-schema.js b/zeppelin-web/src/components/data-validator/data-validator-schema.js index e82b1184fd8..d22374c9537 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-schema.js +++ b/zeppelin-web/src/components/data-validator/data-validator-schema.js @@ -16,6 +16,16 @@ angular.module('zeppelinWebApp') .constant('dataModelSchemas', { 'MapSchema': { - type: ['string', 'string', 'number', 'number', 'number'] + type: ['string', 'string', 'number', 'number', 'number'], + range: { + latitude: { + low: -90, + high: 90 + }, + longitude: { + low: -180, + high: 180 + } + } } }); \ No newline at end of file From 796e3d6d7d0dc956c67753948fa9f9095d3bc1f3 Mon Sep 17 00:00:00 2001 From: madhuka Date: Thu, 30 Jul 2015 00:34:42 +0530 Subject: [PATCH 18/31] adding error msg --- .../data-validator/data-validator-factory.js | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index d626911c432..f265faa8c7c 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { var msg = ''; - var errorStatus = true; + var errorStatus = false; var dataValidator = function() { this.schema = null; this.error = getErrorStatus; @@ -67,7 +67,6 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { } function dataCheckValidator(record, schema) { - //console.log(schema.type.length); if (record) { for (var i = 0; i < schema.type.length; i++) { if (isNaN(record[i]) !== (schema.type[i] === 'string')) { @@ -75,13 +74,16 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { msg += 'data record ' + (record[i]) + ' is not matching for schema | '; return true; }else{ - if(i==2){ - latitudeValidator(record[i],schema.range); - }if(i==3){ - longitudeValidator(record[i],schema.range); + if(i===2){ + errorStatus = !latitudeValidator(record[i],schema.range); + }if(i===3){ + errorStatus = !longitudeValidator(record[i],schema.range); + } + if(errorStatus){ + return true; } } - } + }//end validation on data record errorStatus = false; return false; } else { @@ -93,11 +95,11 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { //Latitude measurements range from 0° to (+/–)90°. function latitudeValidator(record, schema) { if(record) { - var latitude = parseFloat(record) - if(!(schema.latitude.low <= latitude <= schema.latitude.high)) { - msg += 'Latitude ' + record + ' is not in range | '; + var latitude = parseFloat(record); + if(schema.latitude.low <= latitude <= schema.latitude.high) { return true; } + msg += 'Latitude ' + record + ' is not in range | '; return false; } else { msg += 'Latitude record does not exisiting | '; @@ -107,12 +109,13 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { //Longitude measurements range from 0° to (+/–)180°. function longitudeValidator(record, schema) { + console.log(record); if(record) { - var longitude = parseFloat(record) - if(!(schema.longitude.low <= longitude <= schema.longitude.high)) { - msg += 'Longitude' + record + ' is not in range | '; + var longitude = parseFloat(record); + if(schema.longitude.low <= longitude <= schema.longitude.high) { return true; } + msg += 'Longitude' + record + ' is not in range | '; return false; } else { msg += 'Longitude record does not exisiting | '; From 04bd14eabf361f095d01387d2efa613b2ce04c33 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 1 Aug 2015 20:13:59 +0530 Subject: [PATCH 19/31] Generalization data validation and adding map data validation --- .../data-validator/data-validator-factory.js | 55 ++---------- .../data-validator/data-validator-service.js | 10 ++- .../map-data-validator-factory.js | 84 +++++++++++++++++++ 3 files changed, 98 insertions(+), 51 deletions(-) create mode 100644 zeppelin-web/src/components/data-validator/map-data-validator-factory.js diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index f265faa8c7c..f777c368e1a 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -13,20 +13,21 @@ */ 'use strict'; -angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { +angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { var msg = ''; var errorStatus = false; - var dataValidator = function() { - this.schema = null; + var DataValidator = function(schema) { + this.schema = schema; this.error = getErrorStatus; this.checkData = checkData; this.getMsg = getMsg; + this.data = null; }; - function checkData(data, schema) { - console.log(schema); - if (basicCheck(data, schema)) { + + function checkData() { + if (basicCheck(this.data, this.schema)) {// jshint ignore:line msg += 'data is exisiting | '; } else { msg += 'data does not exisiting | '; @@ -73,15 +74,6 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { errorStatus = true; msg += 'data record ' + (record[i]) + ' is not matching for schema | '; return true; - }else{ - if(i===2){ - errorStatus = !latitudeValidator(record[i],schema.range); - }if(i===3){ - errorStatus = !longitudeValidator(record[i],schema.range); - } - if(errorStatus){ - return true; - } } }//end validation on data record errorStatus = false; @@ -92,36 +84,5 @@ angular.module('zeppelinWebApp').factory('dataValidator', function($rootScope) { } } - //Latitude measurements range from 0° to (+/–)90°. - function latitudeValidator(record, schema) { - if(record) { - var latitude = parseFloat(record); - if(schema.latitude.low <= latitude <= schema.latitude.high) { - return true; - } - msg += 'Latitude ' + record + ' is not in range | '; - return false; - } else { - msg += 'Latitude record does not exisiting | '; - return true; - } - } - - //Longitude measurements range from 0° to (+/–)180°. - function longitudeValidator(record, schema) { - console.log(record); - if(record) { - var longitude = parseFloat(record); - if(schema.longitude.low <= longitude <= schema.longitude.high) { - return true; - } - msg += 'Longitude' + record + ' is not in range | '; - return false; - } else { - msg += 'Longitude record does not exisiting | '; - return true; - } - } - - return dataValidator; + return DataValidator; }); diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 2bd1adb25f1..ee9dff744f8 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -13,12 +13,14 @@ */ 'use strict'; -angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, dataValidator, dataModelSchemas) { +angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, dataModelSchemas) { this.validateMapData = function(data) { - var mapValidator = new dataValidator(); - mapValidator.schema = dataModelSchemas.MapSchema; - mapValidator.checkData(data,mapValidator.schema); + var mapValidator = mapdataValidator; + mapValidator.data = data; + mapValidator.checkData(); + mapValidator.checkLatiLong(); + //mapdataValidator.data = data; var msg = {'error':mapValidator.error(), 'msg':mapValidator.getMsg()}; return msg; }; diff --git a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js new file mode 100644 index 00000000000..6b9d1e52562 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('mapdataValidator', function($rootScope, DataValidator, dataModelSchemas) { + + var msg = ''; + var errorStatus = false; + var mapdataValidator = new DataValidator(dataModelSchemas.MapSchema); + mapdataValidator.checkLatiLong = function() { + //TODO add mapDataCheckValidator and latitudeValidator + }; + + + function mapDataCheckValidator(record, schema) { + if (record) { + for (var i = 0; i < schema.type.length; i++) { + if (isNaN(record[i]) !== (schema.type[i] === 'string')) { + errorStatus = true; + msg += 'data record ' + (record[i]) + ' is not matching for schema | '; + return true; + }else{ + if(i===2){ + errorStatus = !latitudeValidator(record[i],schema.range); + }if(i===3){ + errorStatus = !longitudeValidator(record[i],schema.range); + } + if(errorStatus){ + return true; + } + } + }//end validation on data record + errorStatus = false; + return false; + } else { + msg += 'data record does not exisiting | '; + return true; + } + } + + //Latitude measurements range from 0° to (+/–)90°. + function latitudeValidator(record, schema) { + if(record) { + var latitude = parseFloat(record); + if(schema.latitude.low <= latitude <= schema.latitude.high) { + return true; + } + msg += 'Latitude ' + record + ' is not in range | '; + return false; + } else { + msg += 'Latitude record does not exisiting | '; + return true; + } + } + + //Longitude measurements range from 0° to (+/–)180°. + function longitudeValidator(record, schema) { + console.log(record); + if(record) { + var longitude = parseFloat(record); + if(schema.longitude.low <= longitude <= schema.longitude.high) { + return true; + } + msg += 'Longitude' + record + ' is not in range | '; + return false; + } else { + msg += 'Longitude record does not exisiting | '; + return true; + } + } + + return mapdataValidator; +}); From 95c3aafad925efc5505031a0dee8c7e1764eb458 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 1 Aug 2015 20:26:21 +0530 Subject: [PATCH 20/31] adding map validation for home page --- zeppelin-web/src/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 28b02721b18..cb6b8cf7a8e 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -144,6 +144,7 @@ + From 8f82a38e89adf0c079f28158c835ddae0877fdb4 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 1 Aug 2015 23:08:06 +0530 Subject: [PATCH 21/31] improving map data validation --- .../data-validator/data-validator-factory.js | 11 ++- .../map-data-validator-factory.js | 84 +++++++------------ 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index f777c368e1a..6f9ca7e68a9 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -20,12 +20,13 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { var DataValidator = function(schema) { this.schema = schema; this.error = getErrorStatus; + this.setError = setError; this.checkData = checkData; this.getMsg = getMsg; + this.setMsg = setMsg; this.data = null; }; - function checkData() { if (basicCheck(this.data, this.schema)) {// jshint ignore:line msg += 'data is exisiting | '; @@ -38,10 +39,18 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { return msg; } + function setMsg(newMsg) { + msg += newMsg; + } + function getErrorStatus() { return errorStatus; } + function setError() { + errorStatus = true; + } + function basicCheck(data, schema) { if (data.code && data.rows) { rowCheck(data.rows, 3, schema); diff --git a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js index 6b9d1e52562..6494888db9f 100644 --- a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js @@ -13,72 +13,52 @@ */ 'use strict'; -angular.module('zeppelinWebApp').factory('mapdataValidator', function($rootScope, DataValidator, dataModelSchemas) { +angular.module('zeppelinWebApp').factory('mapdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { var msg = ''; var errorStatus = false; - var mapdataValidator = new DataValidator(dataModelSchemas.MapSchema); - mapdataValidator.checkLatiLong = function() { - //TODO add mapDataCheckValidator and latitudeValidator - }; - + var mapSchema = dataModelSchemas.MapSchema; - function mapDataCheckValidator(record, schema) { - if (record) { - for (var i = 0; i < schema.type.length; i++) { - if (isNaN(record[i]) !== (schema.type[i] === 'string')) { - errorStatus = true; - msg += 'data record ' + (record[i]) + ' is not matching for schema | '; - return true; - }else{ - if(i===2){ - errorStatus = !latitudeValidator(record[i],schema.range); - }if(i===3){ - errorStatus = !longitudeValidator(record[i],schema.range); - } - if(errorStatus){ - return true; - } - } - }//end validation on data record - errorStatus = false; - return false; - } else { - msg += 'data record does not exisiting | '; - return true; + var mapdataValidator = new DataValidator(mapSchema); + mapdataValidator.checkLatiLong = function() { + var data = mapdataValidator.data.rows; + msg = ''; + for(var i = 0; i < data.length; i++) { + latitudeValidator(data[i][2], mapSchema.range); + longitudeValidator(data[i][3], mapSchema.range); + if(errorStatus) { + mapdataValidator.setError(); + } } - } + mapdataValidator.setMsg(msg); + return; + }; //Latitude measurements range from 0° to (+/–)90°. function latitudeValidator(record, schema) { - if(record) { - var latitude = parseFloat(record); - if(schema.latitude.low <= latitude <= schema.latitude.high) { - return true; - } - msg += 'Latitude ' + record + ' is not in range | '; - return false; - } else { - msg += 'Latitude record does not exisiting | '; - return true; + var latitude = parseFloat(record); + if(schema.latitude.low < latitude && latitude < schema.latitude.high) { + msg += 'latitudes are ok | '; + return; } + msg += 'Latitude ' + record + ' is not in range | '; + errorStatus = true; + return; } //Longitude measurements range from 0° to (+/–)180°. function longitudeValidator(record, schema) { - console.log(record); - if(record) { - var longitude = parseFloat(record); - if(schema.longitude.low <= longitude <= schema.longitude.high) { - return true; - } - msg += 'Longitude' + record + ' is not in range | '; - return false; - } else { - msg += 'Longitude record does not exisiting | '; - return true; + var longitude = parseFloat(record); + console.log(schema.longitude.low < longitude); + if(schema.longitude.low < longitude && longitude < schema.longitude.high) { + msg += 'longitude are ok | '; + return; } + msg += 'Longitude' + record + ' is not in range | '; + errorStatus = true; + return; } return mapdataValidator; -}); +}); \ No newline at end of file From 34ccd75d3ec495db22b897af9071eb7101d20071 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 1 Aug 2015 23:30:15 +0530 Subject: [PATCH 22/31] clearing code and adding chart data validator factory --- .../chart-data-validator-factory.js | 25 +++++++++++++++++++ .../data-validator/data-validator-factory.js | 20 +++++++-------- .../data-validator/data-validator-service.js | 23 +++++++++++++---- zeppelin-web/src/index.html | 1 + 4 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 zeppelin-web/src/components/data-validator/chart-data-validator-factory.js diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js new file mode 100644 index 00000000000..44d1a8c1880 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('chartdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { + + var simpleChartSchema = dataModelSchemas.SimpleChartSchema; + var chartdataValidator = new DataValidator(simpleChartSchema); + + //Can specify for more deeper level for chart data models + + return chartdataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index 6f9ca7e68a9..759b934f211 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -28,7 +28,7 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { }; function checkData() { - if (basicCheck(this.data, this.schema)) {// jshint ignore:line + if(basicCheck(this.data, this.schema)) { // jshint ignore:line msg += 'data is exisiting | '; } else { msg += 'data does not exisiting | '; @@ -47,12 +47,12 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { return errorStatus; } - function setError() { + function setError() { errorStatus = true; } function basicCheck(data, schema) { - if (data.code && data.rows) { + if(data.code && data.rows) { rowCheck(data.rows, 3, schema); return true; } else { @@ -62,10 +62,10 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { } function rowCheck(rowData, num, schema) { - if (rowData) { - for (var i = 0; i < rowData.length; i++) { + if(rowData) { + for(var i = 0; i < rowData.length; i++) { var row = rowData[i]; - if (dataCheckValidator(row, schema)) { + if(dataCheckValidator(row, schema)) { msg += 'data record does not mapping to data schema| '; } } @@ -77,14 +77,14 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { } function dataCheckValidator(record, schema) { - if (record) { - for (var i = 0; i < schema.type.length; i++) { - if (isNaN(record[i]) !== (schema.type[i] === 'string')) { + if(record) { + for(var i = 0; i < schema.type.length; i++) { + if(isNaN(record[i]) !== (schema.type[i] === 'string')) { errorStatus = true; msg += 'data record ' + (record[i]) + ' is not matching for schema | '; return true; } - }//end validation on data record + } //end validation on data record errorStatus = false; return false; } else { diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index ee9dff744f8..af44e8785ab 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -13,16 +13,29 @@ */ 'use strict'; -angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, dataModelSchemas) { +angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, chartdataValidator, dataModelSchemas) { this.validateMapData = function(data) { - var mapValidator = mapdataValidator; + var mapValidator = mapdataValidator; mapValidator.data = data; mapValidator.checkData(); mapValidator.checkLatiLong(); - //mapdataValidator.data = data; - var msg = {'error':mapValidator.error(), 'msg':mapValidator.getMsg()}; - return msg; + return buildMsg(mapValidator); + }; + + this.validateChartData = function(data) { + var chartValidator = chartdataValidator; + chartValidator.data = data; + chartValidator.checkData(); + return buildMsg(chartValidator); }; + function buildMsg(validator){ + var msg = { + 'error': validator.error(), + 'msg': validator.getMsg() + }; + return msg; + } + }); diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index cb6b8cf7a8e..6e12a77db34 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -145,6 +145,7 @@ + From 62f5fc7604151e6f8ec5c7f10db360d199b19bd9 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 2 Aug 2015 18:19:01 +0530 Subject: [PATCH 23/31] adding scatter data validator and updating schema --- .../chart-data-validator-factory.js | 2 +- .../data-validator/data-validator-schema.js | 9 ++++++++ .../data-validator/data-validator-service.js | 9 +++++++- .../scatter-data-validator-factory.js | 22 +++++++++++++++++++ zeppelin-web/src/index.html | 1 + 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index 44d1a8c1880..a122ae17a24 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( $rootScope, DataValidator, dataModelSchemas) { - var simpleChartSchema = dataModelSchemas.SimpleChartSchema; + var simpleChartSchema = dataModelSchemas.D3ChartDataSchema; var chartdataValidator = new DataValidator(simpleChartSchema); //Can specify for more deeper level for chart data models diff --git a/zeppelin-web/src/components/data-validator/data-validator-schema.js b/zeppelin-web/src/components/data-validator/data-validator-schema.js index d22374c9537..d514a7e15bc 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-schema.js +++ b/zeppelin-web/src/components/data-validator/data-validator-schema.js @@ -27,5 +27,14 @@ angular.module('zeppelinWebApp') high: 180 } } + }, + 'D3ChartDataSchema': { + type: ['number', 'number', 'number'], + }, + 'PieChartSchema': { + type: ['number', 'number'], + }, + 'ScatterDataSchema': { + type: ['number', 'number'], } }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index af44e8785ab..13dea02d2b6 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -13,7 +13,7 @@ */ 'use strict'; -angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, chartdataValidator, dataModelSchemas) { +angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, scatterdataValidator, chartdataValidator, dataModelSchemas) { this.validateMapData = function(data) { var mapValidator = mapdataValidator; @@ -30,6 +30,13 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope return buildMsg(chartValidator); }; + this.validateScatterData = function(data) { + var chartValidator = scatterdataValidator; + chartValidator.data = data; + chartValidator.checkData(); + return buildMsg(chartValidator); + }; + function buildMsg(validator){ var msg = { 'error': validator.error(), diff --git a/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js b/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js new file mode 100644 index 00000000000..416dd3627da --- /dev/null +++ b/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('scatterdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { + + var simpleChartSchema = dataModelSchemas.ScatterDataSchema; + var scatterdataValidator = new DataValidator(simpleChartSchema); + return scatterdataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 6e12a77db34..1d44b5ef6de 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -146,6 +146,7 @@ + From 4c904f0a0c93f63c9ba808a773bee4cb7f3505b5 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 2 Aug 2015 18:19:42 +0530 Subject: [PATCH 24/31] adding data validators for paragraph.controller --- .../notebook/paragraph/paragraph.controller.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 8132f82bcfb..27e6284ebbb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -1308,6 +1308,8 @@ angular.module('zeppelinWebApp') var schema = data.schema; var rows = data.rows; + console.log(rows) + //check for d3s var values = $scope.paragraph.config.graph.values; var concat = function(o, n) { @@ -1404,7 +1406,9 @@ angular.module('zeppelinWebApp') }); }); } - + //TODO adding data validator services + //var msg = dataValidatorSrv.validateChartData(data); + //console.log(msg); // clear aggregation name, if possible var namesWithoutAggr = {}; var colName; @@ -1450,7 +1454,6 @@ angular.module('zeppelinWebApp') } } - return { xLabels : rowIndexValue, d3g : d3g @@ -1540,8 +1543,17 @@ angular.module('zeppelinWebApp') }; } + var msg = dataValidatorSrv.validateScatterData(data); + console.log(msg); + //TODO- warning need to error model or services will needed + if(msg.error){ + console.log('Data is not matching for schema'); + console.log(msg.msg); + } + for (var i = 0; i < data.rows.length; i++) { row = data.rows[i]; + if (xAxis) { xValue = row[xAxis.index]; xValues[i] = xValue; From 0e079ae9106a290fa697571dfdc20b8bee05ca5a Mon Sep 17 00:00:00 2001 From: madhuka Date: Wed, 5 Aug 2015 20:34:32 +0530 Subject: [PATCH 25/31] adding a basic check function for service --- .../data-validator/data-validator-service.js | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 13dea02d2b6..bfc8307e67f 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -17,27 +17,24 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope this.validateMapData = function(data) { var mapValidator = mapdataValidator; - mapValidator.data = data; - mapValidator.checkData(); + doBasicCheck(mapValidator,data); mapValidator.checkLatiLong(); return buildMsg(mapValidator); }; this.validateChartData = function(data) { var chartValidator = chartdataValidator; - chartValidator.data = data; - chartValidator.checkData(); + doBasicCheck(chartValidator,data); return buildMsg(chartValidator); }; this.validateScatterData = function(data) { - var chartValidator = scatterdataValidator; - chartValidator.data = data; - chartValidator.checkData(); - return buildMsg(chartValidator); + var scatterChartValidator = scatterdataValidator; + doBasicCheck(scatterChartValidator,data); + return buildMsg(scatterChartValidator); }; - function buildMsg(validator){ + function buildMsg(validator) { var msg = { 'error': validator.error(), 'msg': validator.getMsg() @@ -45,4 +42,9 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope return msg; } + function doBasicCheck(validator,data) { + validator.data = data; + validator.checkData(); + } + }); From 909e434b611c00bea1325d1999b53fc1eca503b5 Mon Sep 17 00:00:00 2001 From: madhuka Date: Wed, 12 Aug 2015 09:37:33 +0530 Subject: [PATCH 26/31] fixing jshint --- zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 27e6284ebbb..f9d4c2e7d52 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -1308,7 +1308,7 @@ angular.module('zeppelinWebApp') var schema = data.schema; var rows = data.rows; - console.log(rows) + console.log(rows); //check for d3s var values = $scope.paragraph.config.graph.values; From 83304795e2cd3f45dfa629d56b8067f0b311b40a Mon Sep 17 00:00:00 2001 From: madhuka Date: Fri, 14 Aug 2015 23:57:25 +0530 Subject: [PATCH 27/31] adding custom data validator --- .../chart-data-validator-factory.js | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index a122ae17a24..9304894ec2c 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -19,7 +19,68 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( var simpleChartSchema = dataModelSchemas.D3ChartDataSchema; var chartdataValidator = new DataValidator(simpleChartSchema); - //Can specify for more deeper level for chart data models + //overriding the check data + chartdataValidator.checkData = function() { + console.log('over riding happen'); + basicCheck(); + }; + + function rowCheck(dataRows) { + if(dataRows instanceof Object) { + console.log(dataRows) + for(var key in dataRows) { + key + if(dataRows.hasOwnProperty(key)) { + var obj = dataRows[key]; + rowValueCheck(obj); + } + } + } else { + msg += 'dataRows is not a Object | '; + errorStatus = true; + } + } + + function rowValueCheck(record) { + if(record instanceof Object) { + console.log(record) + for(var key in record) { + + var recordValues = record[key]; + for(var dataValues in recordValues) { + if(dataValues.hasOwnProperty(key)) { + var values = values[key]; + console.log(values); + } + } + } + } else { + msg += 'record is not a Object | '; + errorStatus = true; + } + } + + function basicCheck() { + var data = chartdataValidator.data; + if(data.schema && data.rows) { + rowCheck(data.rows); + console.log(data); + return true; + } else { + msg += 'data rows does not exisiting | '; + errorStatus = true; + } + } + + function schemaChecker(record, index) { + if(isNaN(record) !== (schema.type[index] === 'string')) { + errorStatus = true; + msg += 'data record ' + (record[index]) + + ' is not matching for schema | '; + return true; + } + } + return chartdataValidator; }); \ No newline at end of file From 21fad0cd0fde80f53383560beed971f6c4b528fe Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 15 Aug 2015 00:31:31 +0530 Subject: [PATCH 28/31] connecting services and d3 chart validation --- .../paragraph/paragraph.controller.js | 9 +++-- .../chart-data-validator-factory.js | 40 ++++++++++--------- .../data-validator/data-validator-service.js | 1 + 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index f9d4c2e7d52..009a9a29fd7 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -1406,9 +1406,10 @@ angular.module('zeppelinWebApp') }); }); } - //TODO adding data validator services - //var msg = dataValidatorSrv.validateChartData(data); - //console.log(msg); + + var msg = dataValidatorSrv.validateChartData(data); + console.log(msg); + // clear aggregation name, if possible var namesWithoutAggr = {}; var colName; @@ -1542,7 +1543,7 @@ angular.module('zeppelinWebApp') d3g : [] }; } - + console.log(data.rows); var msg = dataValidatorSrv.validateScatterData(data); console.log(msg); //TODO- warning need to error model or services will needed diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index 9304894ec2c..00bfd2af335 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -27,36 +27,38 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( function rowCheck(dataRows) { if(dataRows instanceof Object) { - console.log(dataRows) for(var key in dataRows) { - key + schemaChecker(key, 0); if(dataRows.hasOwnProperty(key)) { var obj = dataRows[key]; rowValueCheck(obj); } } } else { - msg += 'dataRows is not a Object | '; - errorStatus = true; + chartdataValidator.msg += 'dataRows is not a Object | '; + chartdataValidator.errorStatus = true; } } function rowValueCheck(record) { if(record instanceof Object) { - console.log(record) + //console.log(record) for(var key in record) { - var recordValues = record[key]; - for(var dataValues in recordValues) { - if(dataValues.hasOwnProperty(key)) { - var values = values[key]; - console.log(values); + var countKey = 1; + for(var recordKey in recordValues) { + + if(recordValues.hasOwnProperty(recordKey)) { + var values = recordValues[recordKey]; + //console.log(values); + schemaChecker(values, countKey); + countKey++; } } } } else { - msg += 'record is not a Object | '; - errorStatus = true; + chartdataValidator.msg += 'record is not a Object | '; + chartdataValidator.errorStatus = true; } } @@ -67,20 +69,20 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( console.log(data); return true; } else { - msg += 'data rows does not exisiting | '; - errorStatus = true; + chartdataValidator.msg += 'data rows does not exisiting | '; + chartdataValidator.errorStatus = true; } } function schemaChecker(record, index) { - if(isNaN(record) !== (schema.type[index] === 'string')) { - errorStatus = true; - msg += 'data record ' + (record[index]) + + if(isNaN(record) !== (simpleChartSchema.type[index] === 'string')) { + //console.log(record+' is failed '+simpleChartSchema.type[index]); + chartdataValidator.errorStatus = true; + chartdataValidator.msg += 'data record ' + (record[index]) + ' is not matching for schema | '; - return true; } + //console.log(record+' is passed'); } - return chartdataValidator; }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index bfc8307e67f..bfc0798fb0c 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -24,6 +24,7 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope this.validateChartData = function(data) { var chartValidator = chartdataValidator; + //console.log(data); doBasicCheck(chartValidator,data); return buildMsg(chartValidator); }; From 2a8a3e66b7caca412daa25bc7081cd760e088aad Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 15 Aug 2015 14:04:05 +0530 Subject: [PATCH 29/31] completing validations with logs (demo) --- .../paragraph/paragraph.controller.js | 8 +++--- .../chart-data-validator-factory.js | 19 +++++++------- .../data-validator/data-validator-factory.js | 12 ++++++--- .../data-validator/data-validator-service.js | 26 ++++++++++++------- .../map-data-validator-factory.js | 16 +++++------- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 009a9a29fd7..0b4f3a9bda9 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -888,19 +888,21 @@ angular.module('zeppelinWebApp') if(msg.error){ console.log('Data is not matching for schema'); console.log(msg.msg); - } - + }else{ + //drawing map deatils if only data set is validated. $scope.markers = newmarkers; var bounds = leafletBoundsHelpers.createBoundsFromArray([ [Math.max.apply(Math, latArr), Math.max.apply(Math, lngArr)], [Math.min.apply(Math, latArr), Math.min.apply(Math, lngArr)] ]); $scope.bounds = bounds; - $scope.center = {}; + // set map chart height var height = $scope.paragraph.config.graph.height; $('#p'+$scope.paragraph.id+'_mapChart').height(height); + } + $scope.center = {}; }; var setD3Chart = function(type, data, refresh) { diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index 00bfd2af335..09eee51968c 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -35,8 +35,7 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( } } } else { - chartdataValidator.msg += 'dataRows is not a Object | '; - chartdataValidator.errorStatus = true; + markValidationError('dataRows is not a Object | '); } } @@ -57,8 +56,7 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( } } } else { - chartdataValidator.msg += 'record is not a Object | '; - chartdataValidator.errorStatus = true; + markValidationError('record is not a Object | '); } } @@ -69,20 +67,23 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( console.log(data); return true; } else { - chartdataValidator.msg += 'data rows does not exisiting | '; - chartdataValidator.errorStatus = true; + markValidationError('data rows does not exisiting | '); } } function schemaChecker(record, index) { if(isNaN(record) !== (simpleChartSchema.type[index] === 'string')) { //console.log(record+' is failed '+simpleChartSchema.type[index]); - chartdataValidator.errorStatus = true; - chartdataValidator.msg += 'data record ' + (record[index]) + - ' is not matching for schema | '; + markValidationError('data record ' + record + + ' is not matching for schema | '); } //console.log(record+' is passed'); } + function markValidationError(msg) { + chartdataValidator.setMsg(msg); + chartdataValidator.setError(); + } + return chartdataValidator; }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index 759b934f211..afcd75e8826 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -24,6 +24,7 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { this.checkData = checkData; this.getMsg = getMsg; this.setMsg = setMsg; + this.clearMsg = clearMsg; this.data = null; }; @@ -43,6 +44,10 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { msg += newMsg; } + function clearMsg() { + msg = ''; + } + function getErrorStatus() { return errorStatus; } @@ -69,10 +74,8 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { msg += 'data record does not mapping to data schema| '; } } - return true; } else { msg += 'data row does not exisiting | '; - return false; } } @@ -81,7 +84,8 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { for(var i = 0; i < schema.type.length; i++) { if(isNaN(record[i]) !== (schema.type[i] === 'string')) { errorStatus = true; - msg += 'data record ' + (record[i]) + ' is not matching for schema | '; + msg += 'data record ' + (record[i]) + + ' is not matching for schema | '; return true; } } //end validation on data record @@ -94,4 +98,4 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { } return DataValidator; -}); +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index bfc0798fb0c..999fb6c3d93 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -13,25 +13,29 @@ */ 'use strict'; -angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope, mapdataValidator, scatterdataValidator, chartdataValidator, dataModelSchemas) { +angular.module('zeppelinWebApp').service('dataValidatorSrv', function( + $rootScope, mapdataValidator, scatterdataValidator, chartdataValidator, + dataModelSchemas) { this.validateMapData = function(data) { + console.log('map data validator'); var mapValidator = mapdataValidator; - doBasicCheck(mapValidator,data); + doBasicCheck(mapValidator, data); mapValidator.checkLatiLong(); return buildMsg(mapValidator); }; - this.validateChartData = function(data) { + this.validateChartData = function(data) { + console.log('chart data validator'); var chartValidator = chartdataValidator; - //console.log(data); - doBasicCheck(chartValidator,data); + doBasicCheck(chartValidator, data); return buildMsg(chartValidator); }; - this.validateScatterData = function(data) { + this.validateScatterData = function(data) { + console.log('scatter data validator'); var scatterChartValidator = scatterdataValidator; - doBasicCheck(scatterChartValidator,data); + doBasicCheck(scatterChartValidator, data); return buildMsg(scatterChartValidator); }; @@ -43,9 +47,13 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function($rootScope return msg; } - function doBasicCheck(validator,data) { + function doBasicCheck(validator, data) { + validator.clearMsg(); validator.data = data; validator.checkData(); + console.log('doBasicCheck..'); + console.log(validator.getMsg()); + console.log(validator); } -}); +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js index 6494888db9f..350dc232ffd 100644 --- a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js @@ -22,6 +22,7 @@ angular.module('zeppelinWebApp').factory('mapdataValidator', function( var mapdataValidator = new DataValidator(mapSchema); mapdataValidator.checkLatiLong = function() { + errorStatus = false; var data = mapdataValidator.data.rows; msg = ''; for(var i = 0; i < data.length; i++) { @@ -32,7 +33,6 @@ angular.module('zeppelinWebApp').factory('mapdataValidator', function( } } mapdataValidator.setMsg(msg); - return; }; //Latitude measurements range from 0° to (+/–)90°. @@ -40,11 +40,10 @@ angular.module('zeppelinWebApp').factory('mapdataValidator', function( var latitude = parseFloat(record); if(schema.latitude.low < latitude && latitude < schema.latitude.high) { msg += 'latitudes are ok | '; - return; + } else { + msg += 'Latitude ' + record + ' is not in range | '; + errorStatus = true; } - msg += 'Latitude ' + record + ' is not in range | '; - errorStatus = true; - return; } //Longitude measurements range from 0° to (+/–)180°. @@ -53,11 +52,10 @@ angular.module('zeppelinWebApp').factory('mapdataValidator', function( console.log(schema.longitude.low < longitude); if(schema.longitude.low < longitude && longitude < schema.longitude.high) { msg += 'longitude are ok | '; - return; + } else { + msg += 'Longitude' + record + ' is not in range | '; + errorStatus = true; } - msg += 'Longitude' + record + ' is not in range | '; - errorStatus = true; - return; } return mapdataValidator; From e1855c0d4ca94ee77f80f20b3d1694ab164ebf9d Mon Sep 17 00:00:00 2001 From: madhuka Date: Sat, 15 Aug 2015 20:38:48 +0530 Subject: [PATCH 30/31] adding test on data validator --- .../chart-data-validator-factory.js | 3 - .../data-validator/data-validator-factory.js | 3 + .../data-validator/data-validator-service.js | 6 -- .../map-data-validator-factory.js | 1 - zeppelin-web/test/karma.conf.js | 4 ++ .../chart-data-validator-test.js | 67 +++++++++++++++++++ .../data-validator/data-validator-service.js | 35 ++++++++++ .../data-validator/map-data-validator-test.js | 47 +++++++++++++ .../scatter-data-validator-test.js | 63 +++++++++++++++++ 9 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js create mode 100644 zeppelin-web/test/spec/components/data-validator/data-validator-service.js create mode 100644 zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js create mode 100644 zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index 09eee51968c..3e71f53c4e4 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -21,7 +21,6 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( //overriding the check data chartdataValidator.checkData = function() { - console.log('over riding happen'); basicCheck(); }; @@ -49,7 +48,6 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( if(recordValues.hasOwnProperty(recordKey)) { var values = recordValues[recordKey]; - //console.log(values); schemaChecker(values, countKey); countKey++; } @@ -73,7 +71,6 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( function schemaChecker(record, index) { if(isNaN(record) !== (simpleChartSchema.type[index] === 'string')) { - //console.log(record+' is failed '+simpleChartSchema.type[index]); markValidationError('data record ' + record + ' is not matching for schema | '); } diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js index afcd75e8826..6024f6913de 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -62,6 +62,7 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { return true; } else { msg += 'data rows does not exisiting | '; + setError(); return false; } } @@ -75,6 +76,7 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { } } } else { + setError(); msg += 'data row does not exisiting | '; } } @@ -93,6 +95,7 @@ angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { return false; } else { msg += 'data record does not exisiting | '; + setError(); return true; } } diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js index 999fb6c3d93..5c9a0843c83 100644 --- a/zeppelin-web/src/components/data-validator/data-validator-service.js +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -18,7 +18,6 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function( dataModelSchemas) { this.validateMapData = function(data) { - console.log('map data validator'); var mapValidator = mapdataValidator; doBasicCheck(mapValidator, data); mapValidator.checkLatiLong(); @@ -26,14 +25,12 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function( }; this.validateChartData = function(data) { - console.log('chart data validator'); var chartValidator = chartdataValidator; doBasicCheck(chartValidator, data); return buildMsg(chartValidator); }; this.validateScatterData = function(data) { - console.log('scatter data validator'); var scatterChartValidator = scatterdataValidator; doBasicCheck(scatterChartValidator, data); return buildMsg(scatterChartValidator); @@ -51,9 +48,6 @@ angular.module('zeppelinWebApp').service('dataValidatorSrv', function( validator.clearMsg(); validator.data = data; validator.checkData(); - console.log('doBasicCheck..'); - console.log(validator.getMsg()); - console.log(validator); } }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js index 350dc232ffd..9b75b79763c 100644 --- a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js @@ -49,7 +49,6 @@ angular.module('zeppelinWebApp').factory('mapdataValidator', function( //Longitude measurements range from 0° to (+/–)180°. function longitudeValidator(record, schema) { var longitude = parseFloat(record); - console.log(schema.longitude.low < longitude); if(schema.longitude.low < longitude && longitude < schema.longitude.high) { msg += 'longitude are ok | '; } else { diff --git a/zeppelin-web/test/karma.conf.js b/zeppelin-web/test/karma.conf.js index d76f72cb661..b0cb2e61b5a 100644 --- a/zeppelin-web/test/karma.conf.js +++ b/zeppelin-web/test/karma.conf.js @@ -54,10 +54,14 @@ module.exports = function(config) { 'bower_components/lodash/lodash.js', 'bower_components/angular-filter/dist/angular-filter.min.js', 'bower_components/ngtoast/dist/ngToast.js', + 'bower_components/leaflet/dist/leaflet.js', + 'bower_components/leaflet/dist/leaflet-src.js', + 'bower_components/angular-leaflet-directive/dist/angular-leaflet-directive.js', 'bower_components/angular-mocks/angular-mocks.js', // endbower 'src/app/app.js', 'src/app/app.controller.js', + 'src/components/data-validator/*.js', 'src/app/**/*.js', 'test/spec/**/*.js' ], diff --git a/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js new file mode 100644 index 00000000000..fa3f080ecba --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js @@ -0,0 +1,67 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('validateChartData function', function() { + + var mockDataOne = { + 'schema': Object, + 'rows': Object + }; + it('testing with empty data', function() { + expect(chartDataValidatorSrv.validateChartData({}).error).toBe( + true); + expect(chartDataValidatorSrv.validateChartData({}).msg).toBe( + 'data rows does not exisiting | '); + }); + it('testing with data contain empty rows objects', function() { + expect(chartDataValidatorSrv.validateChartData({}).error).toBe( + true); + expect(chartDataValidatorSrv.validateChartData({}).msg).toBe( + 'data rows does not exisiting | '); + }); + //mock correct data set model + var mockRecord = { + 'value(sum)': { + 'count': 1, + 'value': '4' + } + }; + var mockRow = { + '19': mockRecord + }; + mockDataOne.rows = mockRow; + it('testing with validated data', function() { + expect(chartDataValidatorSrv.validateChartData(mockDataOne).error) + .toBe(false); + expect(chartDataValidatorSrv.validateChartData(mockDataOne).msg) + .toBe(''); + }); + //mock incorrect data set model + var incorrectMockRecord = { + 'value(sum)': { + 'count': 1, + 'value': 'wrong' + } + }; + var incorrectMockRow = { + '23': incorrectMockRecord + }; + var mockDataTwo = { + 'schema': Object, + 'rows': incorrectMockRow + }; + it('testing with invalidated data', function() { + expect(chartDataValidatorSrv.validateChartData(mockDataTwo).error) + .toBe(true); + expect(chartDataValidatorSrv.validateChartData(mockDataTwo).msg) + .toBe('data record wrong is not matching for schema | '); + }); + }); +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/data-validator-service.js b/zeppelin-web/test/spec/components/data-validator/data-validator-service.js new file mode 100644 index 00000000000..b2a8ab5bd73 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/data-validator-service.js @@ -0,0 +1,35 @@ +'use strict'; + +describe('Service : DataValidatorSrv', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('Testing API', function() { + + it('to be define', function() { + expect(chartDataValidatorSrv).toBeDefined(); + }); + + it('Services to be define', function() { + expect(chartDataValidatorSrv.validateMapData).toBeDefined(); + expect(chartDataValidatorSrv.validateScatterData).toBeDefined(); + expect(chartDataValidatorSrv.validateChartData).toBeDefined(); + }); + //mock data for testing + var mockData = { + 'schema': Object, + 'rows': Object + }; + it('testing return message format', function() { + expect(chartDataValidatorSrv.validateChartData(mockData).msg) + .toBeDefined(); + expect(chartDataValidatorSrv.validateChartData(mockData).error) + .toBeDefined(); + }); + + }); +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js new file mode 100644 index 00000000000..62659214160 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js @@ -0,0 +1,47 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('validateMapData function', function() { + + var mockDataOne = { + 'msg': 'mock data', + 'code': 'SUCCESS', + 'rows': [ + ['us', 'El paso', '31.758', '-106.4869', '1147'] + ] + }; + //test use case + //testing with correct validated data set then change data and test again + it('testing with validate GIS data and range check', function() { + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + false); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toBe( + 'data is exisiting | latitudes are ok | longitude are ok | '); + //change number to string and test + mockDataOne.rows[0][3] = 'text'; + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + true); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toContain('data record text is not matching for schema'); + //testing for Longitude range + mockDataOne.rows[0][3] = 250; + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + true); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toBe('data is exisiting | latitudes are ok | Longitude250 is not in range | '); + }); + + }); + +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js new file mode 100644 index 00000000000..c8bf2d9acb3 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js @@ -0,0 +1,63 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var validateScatterDataSrv; + beforeEach(inject(function(dataValidatorSrv) { + validateScatterDataSrv = dataValidatorSrv; + })); + + describe('validateScatterData function', function() { + + var mockDataOne = { + 'code': 'SUCCESS', + 'rows': Object + }; + it('testing with empty data', function() { + expect(validateScatterDataSrv.validateScatterData({}).error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData({}).msg).toBe( + 'data rows does not exisiting | data does not exisiting | ' + ); + }); + it('testing with data contain empty rows objects', function() { + //mockDataOne.rows = [[]]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toContain( + 'data is exisiting | '); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toContain( + 'data record does not exisiting | '); + }); + + it('testing with valid data ', function() { + mockDataOne.rows = [ + [12, 34] + ]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + false); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toBe( + 'data is exisiting | '); + }); + + it('testing with string data model', function() { + mockDataOne.rows = [ + ['test', '34'] + ]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toBe( + 'data record test is not matching for schema | data record does not mapping to data schema| data is exisiting | ' + ); + }); + + }); +}); \ No newline at end of file From 58a1de184571f634c3c2ae59af69f087bfa3d13c Mon Sep 17 00:00:00 2001 From: madhuka Date: Fri, 21 Aug 2015 00:52:33 +0530 Subject: [PATCH 31/31] removing console logs --- .../app/notebook/paragraph/paragraph.controller.js | 12 +++++------- .../data-validator/chart-data-validator-factory.js | 4 +--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 0b4f3a9bda9..2a5dbc0c0c0 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -882,11 +882,9 @@ angular.module('zeppelinWebApp') } //data model validator var msg = dataValidatorSrv.validateMapData(data); - console.log(msg); //TODO- warning need to be show in here. currently it is only printing. //need to know what is the standard way of warning in zepplin. if(msg.error){ - console.log('Data is not matching for schema'); console.log(msg.msg); }else{ //drawing map deatils if only data set is validated. @@ -1310,7 +1308,6 @@ angular.module('zeppelinWebApp') var schema = data.schema; var rows = data.rows; - console.log(rows); //check for d3s var values = $scope.paragraph.config.graph.values; @@ -1410,7 +1407,10 @@ angular.module('zeppelinWebApp') } var msg = dataValidatorSrv.validateChartData(data); - console.log(msg); + //Only console print error found + if(msg.error){ + console.log(msg.msg); + } // clear aggregation name, if possible var namesWithoutAggr = {}; @@ -1545,12 +1545,10 @@ angular.module('zeppelinWebApp') d3g : [] }; } - console.log(data.rows); + var msg = dataValidatorSrv.validateScatterData(data); - console.log(msg); //TODO- warning need to error model or services will needed if(msg.error){ - console.log('Data is not matching for schema'); console.log(msg.msg); } diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js index 3e71f53c4e4..ffc2f7c9b53 100644 --- a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -40,7 +40,6 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( function rowValueCheck(record) { if(record instanceof Object) { - //console.log(record) for(var key in record) { var recordValues = record[key]; var countKey = 1; @@ -62,7 +61,6 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( var data = chartdataValidator.data; if(data.schema && data.rows) { rowCheck(data.rows); - console.log(data); return true; } else { markValidationError('data rows does not exisiting | '); @@ -74,7 +72,7 @@ angular.module('zeppelinWebApp').factory('chartdataValidator', function( markValidationError('data record ' + record + ' is not matching for schema | '); } - //console.log(record+' is passed'); + //record is passed } function markValidationError(msg) {