Skip to content

Commit

Permalink
Addresses htm-community#56. Large files will be windowed - whether th…
Browse files Browse the repository at this point in the history 
…ey are local or remote.
  • Loading branch information
@jefffohl
jefffohl committed Jan 6, 2016
1 parent d3dcc8d commit 38cbc58
Show file tree
Hide file tree
Showing 8 changed files with 183 additions and 70 deletions.
2 changes: 1 addition & 1 deletion build/app.js
View file

Large diffs are not rendered by default.

71 changes: 42 additions & 29 deletions build/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,36 +71,49 @@ <h3 class="file-name-label">{{view.loadedFileName}}</h3>
<button class="btn btn-default btn-sm pull-right options-toggle" ng-click="toggleOptions()"><span class="glyphicon glyphicon-triangle-left" ng-show="view.optionsVisible"></span> <span ng-hide="view.optionsVisible">Show</span><span ng-show="view.optionsVisible">Hide</span> Options <span class="glyphicon glyphicon-triangle-right" ng-show="!view.optionsVisible"></span></button>
</div>
</div>
<div class="field-data" ng-class="{'col-md-10' : view.optionsVisible, 'col-md-12' : !view.optionsVisible}">
<table class="table table-condensed">
<thead>
<tr>
<th>Visible</th>
<th>Value</th>
<th>Name</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="field in view.fieldState track by field.name">
<td><input type="checkbox" ng-model="field.visible" ng-click="toggleVisibility(field)"></td>
<td style="color:{{field.color}};">{{field.value}}</td>
<td style="color:{{field.color}};"><div class="span-wrapper" uib-popover="{{field.name}}" popover-trigger="mouseenter" popover-animation="false"><span>{{field.name}}</span></div></td>
</tr>
</tbody>
</table>
<div class="row">
<div class="field-data" ng-class="{'col-md-10' : view.optionsVisible, 'col-md-12' : !view.optionsVisible}">
<table class="table table-condensed">
<thead>
<tr>
<th>Visible</th>
<th>Value</th>
<th>Name</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="field in view.fieldState track by field.name">
<td><input type="checkbox" ng-model="field.visible" ng-click="toggleVisibility(field)"></td>
<td style="color:{{field.color}};">{{field.value}}</td>
<td style="color:{{field.color}};"><div class="span-wrapper" uib-popover="{{field.name}}" popover-trigger="mouseenter" popover-animation="false"><span>{{field.name}}</span></div></td>
</tr>
</tbody>
</table>
</div>
<div class="field-options col-md-2" ng-show="view.optionsVisible">
<table class="table table-condensed" ng-show="view.optionsVisible">
<thead>
<tr>
<th uib-tooltip="Normalize this field with the selected data field" tooltip-placement="left" tooltip-append-to-body="true">Normalize</th>
<th uib-tooltip="Data field to which other fields can be normalized" tooltip-placement="left" tooltip-append-to-body="true">Data</th>
</tr>
</thead>
<tbody>
<tr normalize-fields ng-repeat="field in view.fieldState track by field.name"></tr>
</tbody>
</table>
</div>
</div>
<div class="field-options col-md-2" ng-show="view.optionsVisible">
<table class="table table-condensed" ng-show="view.optionsVisible">
<thead>
<tr>
<th uib-tooltip="Normalize this field with the selected data field" tooltip-placement="left" tooltip-append-to-body="true">Normalize</th>
<th uib-tooltip="Data field to which other fields can be normalized" tooltip-placement="left" tooltip-append-to-body="true">Data</th>
</tr>
</thead>
<tbody>
<tr normalize-fields ng-repeat="field in view.fieldState track by field.name"></tr>
</tbody>
</table>
<div class="windowing row" ng-show="view.windowing.show">
<div class="col-md-9">
<span class="warning">File is over {{view.windowing.threshold|bytes}}. Windowing is enabled.</span>
</div>
<div class="col-md-3" ng-hide="view.windowing.paused">
<button class="btn btn-primary pull-right" ng-disabled="view.windowing.aborted" ng-click="pauseParse()">Pause</button>
</div>
<div class="col-md-3" ng-show="view.windowing.paused">
<button class="btn btn-primary pull-right" ng-disabled="view.windowing.aborted" ng-click="resumeParse()">Resume</button>
</div>
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion build/stylesheet.css
View file

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion client/src/app/appConfig.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,14 @@ angular.module('app').constant('appConfig', {
NONE_VALUE_REPLACEMENT : 0,
// BUFFER:
// buffer size used for DyGraph streaming, default 1000
BUFFER_SIZE : 1000,
BUFFER_SIZE : 10000,
// SLIDING_WINDOW:
// True = each batch existing values are dropped, new BUFFER_SIZE is painted. Graph will "move to the right".
// False = data never dropped, just append. Graph will "shrink". (default)
SLIDING_WINDOW : false,
// MAX_FILE_SIZE:
// Maximum size in bytes, for a file. Over this size, and windowing will automatically occur.
MAX_FILE_SIZE : 5000000,
// LOCAL_CHUNK_SIZE:
// size in bytes of each chunk for the data stream, when reading local files
LOCAL_CHUNK_SIZE : 65536,
Expand Down
73 changes: 64 additions & 9 deletions client/src/app/appCtrl.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,24 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
filePath: "",
loadedFileName: "",
errors: [],
loading: false
loading: false,
windowing : {
threshold : appConfig.MAX_FILE_SIZE,
show : false,
paused : false,
aborted : false
}
};

var loadedCSV = [],
loadedFields = [], //=CSV header (parsed)
backupCSV = [],
timers = {},
useIterationsForTimestamp = false,
iteration = 0;
iteration = 0,
slidingWindow = appConfig.SLIDING_WINDOW,
streamParser = null,
firstDataLoaded = false;

// the "Show/Hide Options" button
$scope.toggleOptions = function() {
Expand All @@ -31,13 +40,25 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
};

$scope.getRemoteFile = function() {
$scope.view.windowing.show = false;
$scope.view.windowing.paused = false;
$scope.view.windowing.aborted = false;
slidingWindow = false;
$scope.$broadcast("fileUploadChange");
$scope.view.loading = true;
// we do a quick test here to see if the server supports the Range header.
// If so, we try to stream. If not, we try to download.
$http.head($scope.view.filePath,{'headers' : {'Range' : 'bytes=0-32'}}).then(function(response){
if(response.status === 206) {
streamRemoteFile($scope.view.filePath);
// now we check to see how big the file is
$http.head($scope.view.filePath).then(function(response){
var contentLength = response.headers('Content-Length');
if (contentLength > appConfig.MAX_FILE_SIZE) {
slidingWindow = true;
$scope.view.windowing.show = true;
}
streamRemoteFile($scope.view.filePath);
});
} else {
downloadFile($scope.view.filePath);
}
Expand All @@ -55,10 +76,35 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
}
};

$scope.abortParse = function() {
if (angular.isDefined(streamParser) && angular.isDefined(streamParser.abort)) {
streamParser.abort();
$scope.view.windowing.paused = false;
$scope.view.windowing.aborted = true;
}
};

$scope.pauseParse = function() {
if (angular.isDefined(streamParser) && angular.isDefined(streamParser.pause)) {
streamParser.pause();
$scope.view.windowing.paused = true;
}
};

$scope.resumeParse = function() {
if (angular.isDefined(streamParser) && angular.isDefined(streamParser.resume)) {
streamParser.resume();
$scope.view.windowing.paused = false;
}
};

$scope.getLocalFile = function(event) {
$scope.view.filePath = event.target.files[0].name;
if (event.target.files[0].size > appConfig.MAX_FILE_SIZE) {
slidingWindow = true;
$scope.view.windowing.show = true;
}
$scope.view.loading = true;
// $scope.view.filePath = "";
streamLocalFile(event.target.files[0]);
};

Expand Down Expand Up @@ -90,7 +136,7 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
}
arr.push(fieldValue);
}
if (appConfig.SLIDING_WINDOW && loadedCSV.length > appConfig.BUFFER_SIZE) {
if (slidingWindow && loadedCSV.length > appConfig.BUFFER_SIZE) {
loadedCSV.shift();
backupCSV.shift();
}
Expand All @@ -104,7 +150,10 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
'file': loadedCSV
});
}
$scope.$apply();
if (!firstDataLoaded) {
$scope.$apply();
firstDataLoaded = true;
}
};

var resetFields = function() {
Expand All @@ -114,10 +163,14 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
$scope.view.dataField = null;
$scope.view.errors.length = 0;
$scope.view.loadedFileName = "";
//$scope.view.windowing.show = false;
//$scope.view.windowing.paused = false;
//$scope.view.windowing.aborted = false;
useIterationsForTimestamp = false;
iteration = 0;
loadedCSV.length = 0;
loadedFields.length = 0;
firstDataLoaded = false;
};

var downloadFile = function(url) {
Expand All @@ -142,8 +195,8 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
$scope.$apply();
},
error: function(error) {
handleError("Could not download file.", "danger");
$scope.view.loading = false;
handleError("Could not download file.", "danger");
}
});
};
Expand All @@ -159,8 +212,9 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
dynamicTyping: true,
worker: false, // multithreaded, !but does NOT work with other libs in app.js or streaming
comments: "#",
chunk: function(chunk) {
chunk: function(chunk, parser) {
if (!firstChunkComplete) {
streamParser = parser;
loadedFields = generateFieldMap(chunk.data[chunk.data.length - 1], appConfig.EXCLUDE_FIELDS);
firstChunkComplete = true;
}
Expand Down Expand Up @@ -192,8 +246,9 @@ angular.module('app').controller('appCtrl', ['$scope', '$http', '$timeout', 'app
dynamicTyping: true,
worker: false, // multithreaded, !but does NOT work with other libs in app.js or streaming
comments: "#",
chunk: function(chunk) {
chunk: function(chunk, parser) {
if (!firstChunkComplete) {
streamParser = parser;
loadedFields = generateFieldMap(chunk.data[chunk.data.length - 1], appConfig.EXCLUDE_FIELDS);
firstChunkComplete = true;
}
Expand Down
9 changes: 9 additions & 0 deletions client/src/app/filters/bytes.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
angular.module('app').filter('bytes', function() {
return function(bytes, precision) {
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
if (typeof precision === 'undefined') precision = 1;
var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
number = Math.floor(Math.log(bytes) / Math.log(1024));
return (bytes / Math.pow(1024, Math.floor(number))).toFixed(precision) + ' ' + units[number];
};
});
71 changes: 42 additions & 29 deletions client/src/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,36 +71,49 @@ <h3 class="file-name-label">{{view.loadedFileName}}</h3>
<button class="btn btn-default btn-sm pull-right options-toggle" ng-click="toggleOptions()"><span class="glyphicon glyphicon-triangle-left" ng-show="view.optionsVisible"></span> <span ng-hide="view.optionsVisible">Show</span><span ng-show="view.optionsVisible">Hide</span> Options <span class="glyphicon glyphicon-triangle-right" ng-show="!view.optionsVisible"></span></button>
</div>
</div>
<div class="field-data" ng-class="{'col-md-10' : view.optionsVisible, 'col-md-12' : !view.optionsVisible}">
<table class="table table-condensed">
<thead>
<tr>
<th>Visible</th>
<th>Value</th>
<th>Name</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="field in view.fieldState track by field.name">
<td><input type="checkbox" ng-model="field.visible" ng-click="toggleVisibility(field)"></td>
<td style="color:{{field.color}};">{{field.value}}</td>
<td style="color:{{field.color}};"><div class="span-wrapper" uib-popover="{{field.name}}" popover-trigger="mouseenter" popover-animation="false"><span>{{field.name}}</span></div></td>
</tr>
</tbody>
</table>
<div class="row">
<div class="field-data" ng-class="{'col-md-10' : view.optionsVisible, 'col-md-12' : !view.optionsVisible}">
<table class="table table-condensed">
<thead>
<tr>
<th>Visible</th>
<th>Value</th>
<th>Name</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="field in view.fieldState track by field.name">
<td><input type="checkbox" ng-model="field.visible" ng-click="toggleVisibility(field)"></td>
<td style="color:{{field.color}};">{{field.value}}</td>
<td style="color:{{field.color}};"><div class="span-wrapper" uib-popover="{{field.name}}" popover-trigger="mouseenter" popover-animation="false"><span>{{field.name}}</span></div></td>
</tr>
</tbody>
</table>
</div>
<div class="field-options col-md-2" ng-show="view.optionsVisible">
<table class="table table-condensed" ng-show="view.optionsVisible">
<thead>
<tr>
<th uib-tooltip="Normalize this field with the selected data field" tooltip-placement="left" tooltip-append-to-body="true">Normalize</th>
<th uib-tooltip="Data field to which other fields can be normalized" tooltip-placement="left" tooltip-append-to-body="true">Data</th>
</tr>
</thead>
<tbody>
<tr normalize-fields ng-repeat="field in view.fieldState track by field.name"></tr>
</tbody>
</table>
</div>
</div>
<div class="field-options col-md-2" ng-show="view.optionsVisible">
<table class="table table-condensed" ng-show="view.optionsVisible">
<thead>
<tr>
<th uib-tooltip="Normalize this field with the selected data field" tooltip-placement="left" tooltip-append-to-body="true">Normalize</th>
<th uib-tooltip="Data field to which other fields can be normalized" tooltip-placement="left" tooltip-append-to-body="true">Data</th>
</tr>
</thead>
<tbody>
<tr normalize-fields ng-repeat="field in view.fieldState track by field.name"></tr>
</tbody>
</table>
<div class="windowing row" ng-show="view.windowing.show">
<div class="col-md-9">
<span class="warning">File is over {{view.windowing.threshold|bytes}}. Windowing is enabled.</span>
</div>
<div class="col-md-3" ng-hide="view.windowing.paused">
<button class="btn btn-primary pull-right" ng-disabled="view.windowing.aborted" ng-click="pauseParse()">Pause</button>
</div>
<div class="col-md-3" ng-show="view.windowing.paused">
<button class="btn btn-primary pull-right" ng-disabled="view.windowing.aborted" ng-click="resumeParse()">Resume</button>
</div>
</div>
</div>
</div>
Expand Down
20 changes: 20 additions & 0 deletions client/src/less/nupic-visualizations.less
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,26 @@
}
}

.windowing {
background-color:lighten(@brand-info, 35%);
border:solid 1px @brand-info;
border-radius:3px;
//margin:10px 5px 20px 55px;
padding:10px;
// margin-left:55px;
.warning {
font-size:110%;
color:darken(@brand-info, 25%);
font-weight:bold;
display:inline-block;
padding-top:5px;
}
div[class*="col-"] {
padding-left:5px;
padding-right:5px;
}
}

.btn-file {
position: relative;
overflow: hidden;
Expand Down

0 comments on commit 38cbc58

Please sign in to comment.