Auto-synchro: music-based audio+input latency calibration technique

package.json

@@ -77,7 +77,9 @@
     "debug": "^2.1.1",
     "jquery": "^2.1.3",
     "keytime": "^0.1.0",
+    "once": "^1.3.1",
     "pixi.js": "^2.2.3",
-    "ramda": "^0.9.1"
+    "ramda": "^0.9.1",
+    "vue": "^0.11.4"
   }
 }

src/app/index.js

@@ -6,7 +6,6 @@ import $ from 'jquery'
 
 export function main() {
   co(function*() {
-    console.log(Scintillator)
     let skin      = yield Scintillator.load('/skins/default/skin.xml')
     let context   = new Scintillator.Context(skin)
 

src/auto-synchro/experiment/style.scss

@@ -0,0 +1,27 @@
+
+.experiment {
+  padding: 0 20px;
+  margin: 0 auto;
+  max-width: 720px;
+  h1 {
+    color: #8b8685;
+    font: bold 40px Helvetica, sans-serif;
+  }
+  button {
+    background: #666;
+    border: 3px outset #777;
+    font: bold 30px Verdana, sans-serif;
+    color: #fff;
+    padding: 15px;
+    &:disabled {
+      opacity: 0.4;
+    }
+  }
+  select {
+    background: #666;
+    border: 3px inset #777;
+    font: bold 20px Verdana, sans-serif;
+    color: #fff;
+    padding: 10px;
+  }
+}

src/auto-synchro/experiment/template.jade

@@ -0,0 +1,27 @@
+
+.experiment
+  h1 An Experiment on Audio+Input Latency Calibration for Rhythm Action Games
+  h2 1. Please answer the questionaire and click the button
+  p What kind of audio device you are using?
+  select(v-model='device')
+    option(selected) Built-in Speaker
+    option Earphone
+    option External Speaker
+  p When you are ready...
+  div(v-show='showStart')
+    button(v-on='click:start') Start the Music
+  div(v-show='showLoading')
+    button(disabled) Loading Music (~700kb)
+  div(v-show='showStarted')
+    button(disabled) Playing Music
+  h2 2. Press space bar (mobile phone: tap the screen) when you hear the "kick drum"
+  div(v-show='showCollect')
+    p {{numSamples}} samples recorded.
+    p We need 56 to 84 samples.
+  div(v-show='showSending')
+    h2 3. Sending results to server
+  div(v-show='showThank')
+    h2 4. Finished! Thank you!
+    p This song will loop forever.
+
+

src/auto-synchro/index.js

@@ -0,0 +1,87 @@
+
+import '../polyfill'
+import * as Music from './music'
+import Vue from 'vue'
+import template from './experiment/template.jade'
+import './experiment/style.scss'
+import $ from 'jquery'
+
+export function main() {
+
+  let el = $('<div></div>').appendTo('body')
+
+  function send(device, data) {
+    return new Promise((resolve) => {
+      $.getScript('//www.parsecdn.com/js/parse-1.3.4.min.js', () => {
+        Parse.initialize('JPhEf2EisJuKiuSOD50p6De7ZW7iwKVPakcbMo0h',
+                          'jd0W1cM8Kpn5jTBUp8emJJSimjfYmpP5o0tSpH71')
+        let SurveyResult = Parse.Object.extend('SurveyResult')
+        resolve(new SurveyResult().save({
+          agent: navigator.userAgent,
+          device: device,
+          samples: data,
+        }))
+      })
+    })
+  }
+
+  let data = {
+        showLoading: true,
+        showStart: false,
+        showSending: false,
+        showStarted: false,
+        showThank: false,
+        numSamples: 0,
+        showCollect: true,
+      }
+
+  let play
+
+  Music.load().then(music => {
+    let bound = 56
+    let samples = []
+    data.showLoading = false
+    data.showStart = true
+    play = () => {
+      data.showStart = false
+      data.showStarted = true
+      let remote = music({
+        a() {
+          data.showCollect = false
+          data.showSending = true
+          send(data.device, samples).then(() => {
+            data.showThank = true
+          }, () => { alert('Oops cannot send!') })
+        }
+      })
+      let tap = () => {
+        samples.push(remote.getSample())
+        remote.progress(Math.min(1, samples.length / bound))
+        if (samples.length >= bound) remote.ok()
+        data.numSamples = samples.length
+      }
+      window.addEventListener('keydown', e => {
+        if (e.which !== 32) return
+        e.preventDefault()
+        tap()
+      })
+      window.addEventListener('touchstart', e => {
+        if (e.touches.length !== 1) return
+        e.preventDefault()
+        tap()
+      })
+    }
+  })
+
+  new Vue({
+    el: el[0],
+    template: template(),
+    data: data,
+    methods: {
+      start() {
+        play()
+      }
+    },
+  })
+
+}

src/auto-synchro/music.js

@@ -0,0 +1,127 @@
+
+import download       from '../download'
+import co             from 'co'
+import once           from 'once'
+import SamplingMaster from '../sampling-master'
+import context        from 'audio-context'
+import R              from 'ramda'
+
+/**
+ * Checks whether an audio format is supported.
+ */
+let canPlay = (() => {
+  let dummyAudioTag = document.createElement('audio')
+  return (type) => dummyAudioTag.canPlayType(type) === 'probably'
+})()
+
+/**
+ * The audio format to use (.ogg or .m4a)
+ */
+let audioExt =  once(() =>
+                  canPlay('audio/ogg; codecs="vorbis"') ? '.ogg' : '.m4a')
+
+/**
+ * Loads the files and create a music instance.
+ */
+export function load() {
+  return co(function*() {
+    let master  = new SamplingMaster(context)
+    let sample  =
+          name => download(`/sounds/sync/${name}${audioExt()}`)
+            .as('arraybuffer')
+            .then(buf => master.sample(buf))
+    let samples = R.fromPairs(
+          yield Promise.all(
+            ['bgm', 'intro', 'kick', 'snare'].map(
+              name => sample(name).then(sample => [name, sample]))))
+    return music(master, samples)
+  })
+}
+
+/**
+ * Takes the sample and sequences a music
+ */
+function music(master, samples) {
+  return function play(callbacks) {
+
+    master.unmute()
+
+    let BPM = 148
+    let time = new AudioTime(context, -1)
+
+    let filter = context.createBiquadFilter()
+    filter.type = 'lowpass'
+    filter.frequency.value = 0
+    filter.Q.value = 10
+    filter.connect(context.destination)
+
+    let state = { part2: null }
+
+    let sequence = beatSequencer(BPM, (beat, delay) => {
+      if (beat % 8 !== 7) {
+        samples.kick.play(delay)
+      }
+      if (state.part2 !== null) {
+        beat -= state.part2.begin
+        if (beat % 128 === 0) {
+          samples.bgm.play(delay)
+        }
+      } else {
+        if (beat % 32 === 0) {
+          samples.intro.play(delay, filter)
+        }
+        if (beat % 32 === 31) {
+          if (state.ok === true) {
+            samples.snare.play(delay)
+            state.part2 = { begin: beat + 1}
+            callbacks.a()
+          }
+        }
+      }
+    })
+
+    setInterval(() => sequence(time.t), 33)
+
+    return {
+      ok() {
+        state.ok = true
+      },
+      progress(p) {
+        filter.frequency.value = 20000 * p * p * p
+      },
+      getSample() {
+        let nearestBeat = Math.round(time.t * BPM / 60)
+        let nearestBeatTime = nearestBeat * 60 / BPM
+        return [nearestBeat, time.t - nearestBeatTime]
+      },
+    }
+
+  }
+}
+
+function beatSequencer(bpm, f) {
+  let beat = -1
+  return (time) => {
+    let nowBeat = Math.floor((time + 0.1) * bpm / 60)
+    while (beat < nowBeat) {
+      beat += 1
+      let beatTime = beat * 60 / bpm
+      f(beat, beatTime - time)
+    }
+  }
+}
+
+class AudioTime {
+  constructor(context, leadTime) {
+    this._context = context
+    this._start = context.currentTime
+    this._startTime = leadTime
+  }
+  get t() {
+    return context.currentTime - this._start + this._startTime
+  }
+}
+
+
+
+

src/boot/loader.js

@@ -6,6 +6,7 @@ var modules = {
       app: '../app',
       test: '../test',
       comingSoon: '../coming-soon',
+      sync: '../auto-synchro',
     }
 
 var code = 'module.exports = {'

src/download/index.js

@@ -0,0 +1,17 @@
+
+export function download(url) {
+  return {
+    as(type) {
+      return new Promise((resolve, reject) => {
+        var xh = new XMLHttpRequest()
+        xh.open('GET', url, true)
+        xh.responseType = type
+        xh.onload  = () => resolve(xh.response)
+        xh.onerror = () => reject(new Error(`Unable to download ${url}`))
+        xh.send(null)
+      })
+    }
+  }
+}
+
+export default download

src/read-blob/index.js

@@ -3,7 +3,7 @@ export default readBlob
 
 export function readBlob(blob) {
   return {
-    as: function(type) {
+    as(type) {
       return new Promise(function(resolve, reject) {
         let reader = new FileReader()
         reader.onload = function() {

src/sampling-master/index.js

@@ -82,8 +82,8 @@ class Sample {
     this._buffer = audioBuffer
   }
 
-  play() {
-    return new PlayInstance(this._master, this._buffer)
+  play(delay, node) {
+    return new PlayInstance(this._master, this._buffer, delay, node)
   }
 
   destroy() {
@@ -95,17 +95,17 @@ class Sample {
 
 class PlayInstance {
 
-  constructor(samplingMaster, buffer) {
+  constructor(samplingMaster, buffer, delay, node) {
     this._master = samplingMaster
     let context = samplingMaster.audioContext
     let source = context.createBufferSource()
     source.buffer = buffer
     let gain = context.createGain()
     source.connect(gain)
-    gain.connect(context.destination)
+    gain.connect(node || context.destination)
     this._source = source
     this._gain = gain
-    source.start(0)
+    source.start(delay === 0 ? 0 : Math.max(0, context.currentTime + delay))
     setTimeout(() => this.stop(), buffer.duration * 1000)
     this._master._startPlaying(this)
   }