Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resize issue #996

Open
wants to merge 13 commits into
base: spike-sounds
Choose a base branch
from
Open

Resize issue #996

Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3c917af
highlight neuron that is clicking
celiasmith Dec 16, 2017
dae5be6
initial functional version
celiasmith Dec 16, 2017
67eb628
Trigger spike sound on data received
tcstewar Dec 16, 2017
cc0b731
Changed the sound
celiasmith Dec 17, 2017
b9c2896
reset the sound on graph reset
celiasmith Dec 17, 2017
68b6919
click to select neuron permanently
celiasmith Dec 19, 2017
ba52963
removed some scraps
celiasmith Dec 19, 2017
cd7c3a8
Use WebAudio API for playback
astoeckel Dec 20, 2017
d5d9d6b
Fixed drawn width of spike selection indicator
tcstewar May 30, 2018
131d1dd
Fixed old clicks being used to update selector
tcstewar May 30, 2018
f8fb722
fix the resize issue but ended up having another issue
smilejennyyu Sep 19, 2018
b6f8c2a
fixed the resize issue
smilejennyyu Sep 25, 2018
bab89fb
remove the log messages
smilejennyyu Sep 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 186 additions & 2 deletions nengo_gui/static/components/raster.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,61 @@ Nengo.Raster = function(parent, sim, args) {
this.axes2d = new Nengo.TimeAxes(this.div, args);
this.axes2d.scale_y.domain([0, args.n_neurons]);

// Flag for whether or not update code should be changing the highlight
// Both zooming and the simulator time changing cause an update, but the highlight
// should only update when the time is changing
this.neuron_highlight_updates = false;

// Keep track of mouse position TODO: fix this to be not required
this.mouse_position = [0,0];

this.neuron_highlights_g = this.axes2d.svg.append('g')
.attr('class', 'neuron_highlights');

// Index of the neuron that makes a sound when spiking
this.sound_index = -1;
this.draw_clicked = false;

// TODO: put the neuron highlight properties in CSS
this.neuron_highlights_g.append('rect')
.attr('id', 'neuron_highlights_Y')
.attr('stroke', 'black')
.attr('fill', 'black')
.attr('fill-opacity', '0.1')
.attr('stroke-width', '0.5px');

// TODO: have the fonts and colour set appropriately
this.neuron_highlights_g.append('text')
.attr('id', 'neuron_highlights_text')
.style('text-anchor', 'end')
.attr('class', 'graph_text');

this.axes2d.svg
.on('mouseover', function() {
var mouse = d3.mouse(this);
self.neuron_highlight_updates = true;
self.mouse_position = [mouse[0], mouse[1]];
})
.on('mouseleave', function() {
var mouse = d3.mouse(this);
self.neuron_highlight_updates = false;
if (!self.draw_clicked) {
self.neuron_highlights_g.style('display', 'none');
}
self.mouse_position = [mouse[0], mouse[1]];
})
.on('mousemove', function() {
var mouse = d3.mouse(this);
self.neuron_highlight_updates = true;
self.mouse_position = [mouse[0], mouse[1]];
self.update_highlight(mouse, false);
})
.on('click', function() {
var mouse = d3.mouse(this);
self.neuron_highlight_updates = true;
self.mouse_position = [mouse[0], mouse[1]];
self.update_highlight(mouse, true);
})

/** call schedule_update whenever the time is adjusted in the SimControl */
this.sim.div.addEventListener('adjust_time',
Expand Down Expand Up @@ -54,11 +109,62 @@ Nengo.Raster.prototype.constructor = Nengo.Raster;
/**
* Receive new line data from the server
*/
Nengo.Raster.prototype.init_sound = function() {
// A click sound made with Chris' computer. To construct such an array run
// ffmpeg -loglevel quiet -i click.wav -f f32le - |
// LC_ALL=C od -tfF click.raw | cut -c8-
const audio_data = [
-0.0037, -0.0039, -0.0026, -0.0040, -0.0090, -0.0022, 0.0007, -0.0261,
0.0199, 0.0026, -0.0426, 0.0630, -0.0463, -0.0381, 0.0673, -0.0574,
-0.0766, 0.0937, 0.0585, -0.1867, 0.1089, 0.1133, -0.0442, 0.1591,
-0.0507, -0.0150, 0.1503, -0.0294, 0.1143, 0.1090, -0.1606, 0.0321,
-0.0281, -0.1615, 0.1967, -0.0758, -0.6281, -0.5096, -0.3386, 0.0203,
0.1450, -0.2048, -0.0178, 0.1260, -0.0071, 0.0696, 0.1872, 0.2700,
0.2018, 0.1558, 0.0344, 0.0345, 0.3278, 0.1261, 0.0892, 0.2582,
-0.0265, -0.0388, 0.1367, 0.1976, 0.1260, 0.0908, 0.0840, 0.2228,
0.3244, 0.1823, 0.4558, 0.3872, 0.2277, 0.4438, 0.3826, 0.2770,
0.2003, 0.2816, 0.3549, 0.1583, 0.0789, 0.1050, -0.1784, -0.3706,
-0.3919, -0.5127, -0.5320, -0.6877, -0.9420, -0.8644, -0.6729, -0.5241,
-0.4980, -0.5645, -0.4064, -0.2721, -0.2415, -0.1957, -0.0701,
];

// Instantiate the WebAudio context, only one per browser window
if (Nengo.audio_ctx) {
return;
}
Nengo.audio_ctx = new (window.AudioContext || window.webkitAudioContext)();

// Create the AudioBuffer object containing the audio data, transfer the
// data from the above array into the audio buffer object
if (Nengo.audio_ctx) {
Nengo.audio_click_sound = Nengo.audio_ctx.createBuffer(
1, audio_data.length, 44100);
const channel_data = Nengo.audio_click_sound.getChannelData(0);
for (let i = 0; i < audio_data.length; i++) {
// Attenuate the sound to prevent clipping with high rates
channel_data[i] = 0.5 * audio_data[i];
}
}
}

Nengo.Raster.prototype.on_message = function(event) {
var time = new Float32Array(event.data, 0, 1);
var data = new Int16Array(event.data, 4);
this.data_store.push([time[0], data]);
this.schedule_update();

// make a sound if the neuron spiked
if ($.inArray(this.sound_index-1, data) > -1) {
if (!Nengo.audio_ctx) {
this.init_sound();
}
if (Nengo.audio_ctx && Nengo.audio_click_sound) {
const source = Nengo.audio_ctx.createBufferSource();
source.buffer = Nengo.audio_click_sound;
source.connect(Nengo.audio_ctx.destination);
source.start(0);
}
}
}

Nengo.Raster.prototype.set_n_neurons = function(n_neurons) {
Expand All @@ -68,6 +174,69 @@ Nengo.Raster.prototype.set_n_neurons = function(n_neurons) {
this.ws.send('n_neurons:' + n_neurons);
}

Nengo.Raster.prototype.update_highlight = function(mouse, click) {
var self = this;
var x = mouse[0];
var y = mouse[1];

// TODO: I don't like having ifs here, make a smaller rectangle for mouseovers
if (x > this.axes2d.ax_left && x < this.axes2d.ax_right && y > this.axes2d.ax_top && y < this.axes2d.ax_bottom-1) {
var y1 = this.axes2d.scale_y.invert(y);
var new_sound_index = Math.ceil(y1);
var y2 = this.axes2d.scale_y(new_sound_index);
var y3 = this.axes2d.scale_y(new_sound_index-1);

this.neuron_highlights_g.style('display', null);

if (click){
if (new_sound_index == this.sound_index) {
this.draw_clicked = false;
this.sound_index = -1;
} else {
this.draw_clicked = true;
this.sound_index = new_sound_index;
}
}

//draw the currently clicked highlight
if (this.draw_clicked) {

var ys2 = this.axes2d.scale_y(this.sound_index)
var ys3 = this.axes2d.scale_y(this.sound_index-1)

this.neuron_highlights_g.select('#neuron_highlights_Y')
.attr('x', this.axes2d.ax_left)
.attr('y', ys2)
.attr('width', this.axes2d.ax_right - this.axes2d.ax_left)
.attr('height', ys3-ys2);

this.neuron_highlights_g.select('#neuron_highlights_text')
.attr('x', this.axes2d.ax_left - 3)
.attr('y', ys2 + (ys3-ys2)/2 + 3)
.text(function () {
return self.sound_index;
});
} else {
//draw the temporary highlight
this.neuron_highlights_g.select('#neuron_highlights_Y')
.attr('x', this.axes2d.ax_left)
.attr('y', y2)
.attr('width', this.axes2d.ax_right - this.axes2d.ax_left)
.attr('height', y3-y2);

this.neuron_highlights_g.select('#neuron_highlights_text')
.attr('x', this.axes2d.ax_left - 3)
.attr('y', y2 + (y3-y2)/2 + 3)
.text(function () {
return new_sound_index;
});
}

} else {
//this.neuron_highlights_g.style('display', 'none');
//this.sound_index = -1;
}
};

/**
* Redraw the lines and axis due to changed data
Expand All @@ -84,7 +253,6 @@ Nengo.Raster.prototype.update = function() {

/** update the lines */
var shown_data = this.data_store.get_shown_data();

var path = [];
for (var i = 0; i < shown_data[0].length; i++) {
var t = this.axes2d.scale_x(
Expand All @@ -97,7 +265,14 @@ Nengo.Raster.prototype.update = function() {
path.push('M ' + t + ' ' + y1 + 'V' + y2);
}
}

this.path.attr("d", path.join(""));

//** Update the highlight text if the mouse is on top */
if (this.neuron_highlight_updates) {
this.update_highlight(this.mouse_position, false);
}

};

/**
Expand All @@ -121,11 +296,21 @@ Nengo.Raster.prototype.on_resize = function(width, height) {
this.height = height;
this.div.style.width = width;
this.div.style.height= height;

if (this.draw_clicked) {
this.neuron_highlights_g.select('#neuron_highlights_Y')
.attr('width', this.axes2d.ax_right - this.axes2d.ax_left)
.attr('height', this.axes2d.scale_y(this.sound_index-1)-this.axes2d.scale_y(this.sound_index))
.attr('y', this.axes2d.scale_y(this.sound_index));
this.neuron_highlights_g.select('#neuron_highlights_text')
.attr('y', this.axes2d.scale_y(this.sound_index) + (this.axes2d.scale_y(this.sound_index-1)-this.axes2d.scale_y(this.sound_index))/2 + 3);
}
};

Nengo.Raster.prototype.reset = function(event) {
this.data_store.reset();
this.schedule_update();
this.sound_index = -1;
}

Nengo.Raster.prototype.generate_menu = function() {
Expand Down Expand Up @@ -180,4 +365,3 @@ Nengo.Raster.prototype.set_neuron_count = function() {
self.on_resize(w, h);
})
}