diff --git a/src/components/ThreeOhThree.tsx b/src/components/ThreeOhThree.tsx index bd63539..cdcfcb7 100644 --- a/src/components/ThreeOhThree.tsx +++ b/src/components/ThreeOhThree.tsx @@ -1,26 +1,35 @@ import useDevice from "@/hooks/useDevice"; -import {useEffect, useState} from "react"; +import dynamic from "next/dynamic"; +import React, {useEffect, useState} from "react"; import compute from "@/shaders/threeOhThree/compute.wgsl"; import {audioContext} from "@/utils/audio/audioTools"; import styled from "styled-components"; +const KnobParamLabel = dynamic(() => import("el-vis-audio").then((mod) => mod.KnobParamLabel), {ssr: false}); + +const chunkDurationSeconds = 0.1; +const numChannels = 2; // currently only two channels allowed (shader uses vec2) +const workgroupSize = 256; +const maxBufferedChunks = 1; + const ThreeOhThree = () => { const [playing, setPlaying] = useState(false); - const {adapter, device, gpu} = useDevice() - //const audioShaderModuleDescriptor = {code: compute}; + const [audioParamBuffer, setAudioParamBuffer] = useState(); + const [partials, setPartials] = useState(256); + const [frequency, setFrequency] = useState(1); + const [timeMod, setTimeMod] = useState(16); + const [timeScale, setTimeScale] = useState(9); + const {adapter, device} = useDevice() + + if (numChannels !== 2) { + throw new Error('Currently the number of channels has to be 2, sorry :/'); + } + useEffect(() => { if (!audioContext || !adapter || !device) return; const audioCtx = audioContext; + async function playSound() { - // CONFIG STUFF - const chunkDurationSeconds = 1; - const numChannels = 2; // currently only two channels allowed (shader uses vec2) - const workgroupSize = 256; - const maxBufferedChunks = 5; - - if (numChannels !== 2) { - throw new Error('Currently the number of channels has to be 2, sorry :/'); - } const chunkNumSamplesPerChannel = audioCtx.sampleRate * chunkDurationSeconds; const chunkNumSamples = numChannels * chunkNumSamplesPerChannel; @@ -40,10 +49,16 @@ const ThreeOhThree = () => { usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }); + const audioParamBuffer = device.createBuffer({ + size: Float32Array.BYTES_PER_ELEMENT * 4, + usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST + }); + const audioShaderModule = device.createShaderModule({ label: "Audio shader", code: compute }); + const pipeline = device.createComputePipeline({ layout: 'auto', compute: { @@ -61,19 +76,16 @@ const ThreeOhThree = () => { entries: [ {binding: 0, resource: {buffer: timeInfoBuffer}}, {binding: 1, resource: {buffer: chunkBuffer}}, + {binding: 2, resource: {buffer: audioParamBuffer}}, ] }); + setAudioParamBuffer(audioParamBuffer); - // CHUNK CREATION - - // state tracking const startTime = performance.now() / 1000.0; let nextChunkOffset = 0.0; - // create sound data on the GPU, read back to CPU and schedule for playback async function createSongChunk() { - // if we've already scheduled `maxBufferedChunks` of sound data for playback, reschedule sound data creation for later const bufferedSeconds = (startTime + nextChunkOffset) - (performance.now() / 1000.0); const numBufferedChunks = Math.floor(bufferedSeconds / chunkDurationSeconds); if (numBufferedChunks > maxBufferedChunks) { @@ -83,13 +95,11 @@ const ThreeOhThree = () => { return; } - // update uniform buffer: set the new chunk's offset in seconds from t = 0 console.log('writing nextChunkOffset', nextChunkOffset); device.queue.writeBuffer(timeInfoBuffer, 0, new Float32Array([nextChunkOffset])); const commandEncoder = device.createCommandEncoder(); - // encode compute pass, i.e., sound chunk creation const pass = commandEncoder.beginComputePass(); pass.setPipeline(pipeline); pass.setBindGroup(0, bindGroup); @@ -98,27 +108,16 @@ const ThreeOhThree = () => { ); pass.end(); - // copy sound chunk to map buffer commandEncoder.copyBufferToBuffer(chunkBuffer, 0, chunkMapBuffer, 0, chunkBufferSize); device.queue.submit([commandEncoder.finish()]); - // after submitting(!) chunk creation & copy commands, map chunkMapBuffer's memory to CPU memory for reading - // Note: a mapped buffer is not allowed to be used in a command encoder. - // To avoid an illegal use of the map buffer in a command encoder (i.e., when copying the data from the storage buffer), - // we wait for the buffer's memory to be mapped. - // In this case, this is okay, because we have a couple of seconds of sound data cached in the audio context's destination, - // so we can easily afford to wait for the GPU commands to finish and the buffer to be mapped. - // However, doing this within the render loop of a real-time renderer is usually a bad idea, since it forces a CPU-GPU sync. - // In such cases, it might be a good idea to have a ring buffer of map-buffers to not use the same map buffer in each frame. await chunkMapBuffer.mapAsync(GPUMapMode.READ, 0, chunkBufferSize); - // when the buffer's memory is mapped, copy it to a JavaScript array and unmap the buffer const chunkData = new Float32Array(chunkNumSamples); chunkData.set(new Float32Array(chunkMapBuffer.getMappedRange())); chunkMapBuffer.unmap(); - // copy chunk data to audio buffer const audioBuffer = audioCtx.createBuffer( numChannels, chunkNumSamplesPerChannel, @@ -136,16 +135,14 @@ const ThreeOhThree = () => { } } - // create new audio source from audio buffer and schedule for execution const audioSource = audioCtx.createBufferSource(); audioSource.buffer = audioBuffer; audioSource.connect(audioCtx.destination); - // (there is some issue with the second chunk's offset - no idea why, music's hard I guess) + audioSource.start(nextChunkOffset); console.log(`created new chunk, starts at ${startTime + nextChunkOffset}`); - // schedule next chunk creation nextChunkOffset += audioSource.buffer.duration; await createSongChunk(); } @@ -160,18 +157,65 @@ const ThreeOhThree = () => { audioCtx.suspend(); } - }, [audioContext, device, adapter, playing]) + useEffect(() => { + if (!audioParamBuffer || !device) return; + device.queue.writeBuffer(audioParamBuffer, 0, new Float32Array([partials, frequency, timeMod, timeScale])); + }, [audioParamBuffer, partials, frequency, timeScale, timeMod, device]); + return ( <> + + + + + + ) } -export const DataDiv = styled.div` - width: 1024px; -`; +const KnobsFlexBox = styled.div` + justify-content: space-evenly; + display: flex; + flex-wrap: wrap; + flex-direction: row; + padding: 10px; + border: 2px solid #ff0000; +`; export default ThreeOhThree \ No newline at end of file diff --git a/src/shaders/conwaysGameAudio/audioCompute.wgsl b/src/shaders/conwaysGameAudio/audioCompute.wgsl index b95a11b..98dae0e 100644 --- a/src/shaders/conwaysGameAudio/audioCompute.wgsl +++ b/src/shaders/conwaysGameAudio/audioCompute.wgsl @@ -14,7 +14,7 @@ struct AudioParam { @binding(2) @group(0) var audio_param: AudioParam; @compute -@workgroup_size(WORKGROUP_SIZE) +@workgroup_size(WORKGROUP_SIZE) // vec3(256, 256, 256) fn synthesize(@builtin(global_invocation_id) global_id: vec3) { let sample = global_id.x; diff --git a/src/shaders/threeOhThree/compute.wgsl b/src/shaders/threeOhThree/compute.wgsl index e4a7e2a..580f233 100644 --- a/src/shaders/threeOhThree/compute.wgsl +++ b/src/shaders/threeOhThree/compute.wgsl @@ -1,14 +1,15 @@ +const PARTIALS: u32 = 256u; +const PI2: f32 = 6.283185307179586476925286766559; override WORKGROUP_SIZE: u32 = 256; override SAMPLING_RATE: f32 = 44100.0; -struct TimeInfo { - // time since song start in seconds - offset: f32, -} +struct TimeInfo { offset: f32 } +struct AudioParam { partials: f32, frequency: f32, timeMod: f32, timeScale: f32 } @group(0) @binding(0) var time_info: TimeInfo; @group(0) @binding(1) var song_chunk: array>; // 2 channel pcm data +@binding(2) @group(0) var audio_param: AudioParam; @compute @workgroup_size(WORKGROUP_SIZE) @@ -21,80 +22,13 @@ fn synthesize(@builtin(global_invocation_id) global_id: vec3) { let t = f32(sample) / SAMPLING_RATE; - song_chunk[sample] = mainSound(time_info.offset + t); -} - -const PI: f32 = 3.141592654; -const TAU: f32 = 6.283185307179586476925286766559; - -// length of string (approx 25 inches, standard guitar string length) -const L: f32 = 0.635; - -// height of pluck (12.5 cm, just a random number to make it clearly audible) -const h: f32 = 0.125; -// position of pluck along string (5 inches from lower bridge) -const d: f32 = 0.15; - -// Damping coefficient (bigger = shorter) -const GAMMA: f32 = 2.5; - -// String stiffness coefficient -const b: f32 = 0.008; - -const MAX_HARMONICS: u32 = 50u; - -// fundamental frequencies of each string -// changing these will "tune" the guitar -const FUNDAMENTAL: array = array( - 329.63, // E4 - 246.94, // B3 - 196.00, // G3 - 146.83, // D3 - 110.00, // A2 - 082.41 // E2 -); - -fn song(time: f32) -> vec2 { - var sig = 0.0; - // for each string - for (var s = 0u; s < 6u; s += 1) { - // repeat at a different offset - let t = (time + f32(s) / 6.) % (8./6.); - - // for each harmonic - for (var n = 0u; n < MAX_HARMONICS; n += 1) { - // amplitude for each harmonic - let a_n: f32 = ((2. * h * L * L) / (PI * PI * d * (L - d) * f32(n+1u) * f32(n+1u))) * sin((f32(n+1u) * PI * d) / L ); - - // frequency for each harmonic - let f_n = f32(n+1u) * FUNDAMENTAL[s] * sqrt(1. + b * b * f32(n+1u) * f32(n+1u)); - - // add value to total sound signal, with exponential falloff - sig += a_n * sin(TAU * f_n * t) * exp(-f32(n+1u) * GAMMA * FUNDAMENTAL[s]/200.0 * t); - } - } - - // x = left channel, y = right channel - return vec2(sig); + song_chunk[sample] = mainSound(time_info.offset + t, audio_param); } -// number of synthesized harmonics (tune for quality/preformance) -const NSPC: u32 = 256u; - -const pi2: f32 = 6.283185307179586476925286766559; - -fn dist(s: f32, d: f32) -> f32 { - return clamp(s * d, -1.0, 1.0); -} - -fn distVec(s: vec2, d: f32) -> vec2 { +fn dist(s: vec2, d: f32) -> vec2 { let distClamp: vec2 = vec2(s * d); let distSig: vec2 = clamp(distClamp, vec2(-1.0), vec2(1.0)); - return distClamp; -} - -fn quan(s: f32, c: f32) -> f32 { - return floor(s / c) * c; + return distSig; } fn _filter(h: f32, cut: f32, res: f32) -> f32 { @@ -108,16 +42,11 @@ fn nse(x: f32) -> f32 { return fract(sin(x * 110.082) * 19871.8972); } -fn nse_slide(x: f32) -> f32 { - let fl: f32 = floor(x); - return mix(nse(fl), nse(fl + 1.0), smoothstep(0.0, 1.0, fract(x))); -} - fn ntof(n: f32) -> f32 { return 440.0 * pow(2.0, (n - 69.0) / 12.0); } -fn synth(tseq: f32, t: f32) -> vec2 { +fn synth(tseq: f32, t: f32, partials: f32, frequency: f32) -> vec2 { var v: vec2 = vec2(0.0); let tnote: f32 = fract(tseq); let dr: f32 = 0.26; @@ -126,143 +55,31 @@ fn synth(tseq: f32, t: f32) -> vec2 { let n: f32 = 20.0 + floor(seqn * 38.0); let f: f32 = ntof(n); let sqr: f32 = smoothstep(0.0, 0.01, abs((t*9.0)%64.0 - 20.0) - 20.0); - let base: f32 = f; + let base: f32 = f * frequency; let flt: f32 = exp(tnote * -1.5) * 50.0 + pow(cos(t * 1.0) * 0.5 + 0.5, 4.0) * 80.0 - 0.0; - for (var i = 0u; i < NSPC; i += 1) { + for (var i = 0u; i < u32(partials); i += 1) { var h: f32 = f32(i + 1); var inten: f32 = 1.0 / h; + inten = mix(inten, inten * (h%2.0), sqr); inten *= exp(-1.0 * max(2.0 - h, 0.0)); inten *= _filter(h, flt, 4.0); - var vx = v.x + (inten * sin((pi2 + 0.01) * (t * base * h))); - var vy = v.y + (inten * sin(pi2 * (t * base * h))); + var vx = v.x + (inten * sin((PI2 + 0.01) * (t * base * h))); + var vy = v.y + (inten * sin(PI2 * (t * base * h))); v = vec2(vx, vy); } let o: f32 = v.x * amp; - return vec2(distVec(v * amp, 2.0)); + return vec2(dist(v * amp, 0.5)); } -fn gate1(t: f32) -> f32 -{ - const stp: f32 = 0.0625; - var v: f32 = abs(t - 0.00 - 0.015) - 0.015; - v = min(v, abs(t - stp*1. - 0.015) - 0.015); - v = min(v, abs(t - stp*2. - 0.015) - 0.015); - v = min(v, abs(t - stp*4. - 0.015) - 0.015); - v = min(v, abs(t - stp*6. - 0.015) - 0.015); - v = min(v, abs(t - stp*8. - 0.05) - 0.05); - v = min(v, abs(t - stp*11. - 0.05) - 0.05); - v = min(v, abs(t - stp*14. - 0.05) - 0.05); - - return smoothstep(0.001, 0.0, v); -} +fn mainSound(time: f32, audio_param: AudioParam) -> vec2 { -fn synth2(time: f32) -> vec2 { - var tb: f32 = ((time * 9.0)%16.0) / 16.0; - var f: f32 = time * pi2 * ntof(87.0 - 12.0 + (tb%4.0)); - var v: f32 = dist(sin(f + sin(f * 0.5)), 5.0) * gate1(tb); - - return vec2(v); -} -fn synth2_echo(time: f32, tb: f32) -> vec2 { - var mx: vec2 = synth2(time) * 0.5;// + synth2(time) * 0.5; - var ec: f32 = 0.3; - var fb: f32 = 0.6; - var et: f32 = 3.0 / 9.0; - var tm: f32 = 2.0 / 9.0; - mx += synth2(time - et) * ec * vec2(1.0, 0.2); - ec *= fb; - et += tm; - mx += synth2(time - et) * ec * vec2(0.2, 1.0); - ec *= fb; - et += tm; - mx += synth2(time - et) * ec * vec2(1.0, 0.2); - ec *= fb; - et += tm; - mx += synth2(time - et) * ec * vec2(0.2, 1.0); - ec *= fb; - et += tm; - return mx; -} - -fn synth1_echo(tb: f32, time: f32) -> vec2 { - var v: vec2; - v = synth(tb, time) * 0.5; - var ec: f32 = 0.4; - var fb: f32 = 0.6; - var et: f32 = 2.0 / 9.0; - var tm: f32 = 2.0 / 9.0; - v += synth(tb, time - et) * ec * vec2(1.0, 0.5); - ec *= fb; - et += tm; - v += synth(tb, time - et).yx * ec * vec2(0.5, 1.0); - ec *= fb; - et += tm; - v += synth(tb, time - et) * ec * vec2(1.0, 0.5); - ec *= fb; - et += tm; - v += synth(tb, time - et).yx * ec * vec2(0.5, 1.0); - ec *= fb; - et += tm; - - return v; -} - -fn mainSound(time: f32) -> vec2 { - var mx: vec2 = vec2(0.0); - var tb: f32 = (time * 9.0)%16.0; - mx = synth1_echo(tb, time) * 0.8 * smoothstep(0.0, 0.01, abs(((time * 9.0)%256.0) + 8.0 - 128.0) - 8.0); - var hi: f32 = 1.0; - var ki: f32 = smoothstep(0.01, 0.0, abs(((time * 9.0)%256.0) - 64.0 - 128.0) - 64.0); - var s2i: f32 = 1.0 - smoothstep(0.01, 0.0, abs(((time * 9.0)%256.0) - 64.0 - 128.0) - 64.0); - hi = ki; - mx += vec2(synth2_echo(time, tb)) * 0.2 * s2i; - - mx = mix(mx, mx * (1.0 - fract(tb / 4.0) * 0.5), ki); - var sc: f32 = sin(pi2 * tb) * 0.4 + 0.6; - - mx = distVec(mx, 1.00); + var tb: f32 = (time * audio_param.timeScale)%audio_param.timeMod; + var mx: vec2 = synth(tb, time, audio_param.partials, audio_param.frequency) * 0.8 * smoothstep(0.0, 0.01, abs(((time * 9.0)%256.0) + 8.0 - 128.0) - 8.0); return vec2(mx); -} - /* -fn mainSound(time: f32) -> vec2 -{ - vec2 mx = vec2(0.0); - - float tb = mod(time * 9.0, 16.0); - - - mx = synth1_echo(tb, time) * 0.8 * smoothstep(0.0, 0.01, abs(mod(time * 9.0, 256.0) + 8.0 - 128.0) - 8.0); - - float hi = 1.0; - float ki = smoothstep(0.01, 0.0, abs(mod(time * 9.0, 256.0) - 64.0 - 128.0) - 64.0); - float s2i = 1.0 - smoothstep(0.01, 0.0, abs(mod(time * 9.0, 256.0) - 64.0 - 128.0) - 64.0); - hi = ki; - - mx += expl(mod(time * 9.0, 64.0) / 4.5) * 0.4 * s2i; - - mx += vec2(hat(tb) * 1.5) * hi; - - //mx += dist(fract(tb / 16.0) * sin(ntof(77.0 - 36.0) * pi2 * time), 8.0) * 0.2; - //mx += expl(tb) * 0.5; - - mx += vec2(synth2_echo(time, tb)) * 0.2 * s2i; - - - mx = mix(mx, mx * (1.0 - fract(tb / 4.0) * 0.5), ki); - float sc = sin(pi2 * tb) * 0.4 + 0.6; - float k = kick(tb, time) * 0.8 * sc * ki;// - kick(tb, time - 0.004) * 0.5 - kick(tb, time - 0.008) * 0.25); - - mx += vec2(k); - - - - mx = dist(mx, 1.00); - - return vec2(mx); -}*/ \ No newline at end of file +} \ No newline at end of file