feat: Real zoom. #1601

ui/src/audio_annotator.tsx

@@ -2,6 +2,7 @@ import * as Fluent from '@fluentui/react'
 import { B, F, Id, Rec, S, U } from 'h2o-wave'
 import React from 'react'
 import { stylesheet } from 'typestyle'
+import { averageChannels } from './parts/audioUtils'
 import { DrawnAnnotation, RangeAnnotator, TimeComponent } from './parts/range_annotator'
 import { Waveform } from './parts/waveform'
 import { AnnotatorTags } from './text_annotator'
@@ -86,7 +87,7 @@ window.AudioContext = window.AudioContext || window.webkitAudioContext
 export const XAudioAnnotator = ({ model }: { model: AudioAnnotator }) => {
   const
     [activeTag, setActiveTag] = React.useState(model.tags[0]?.name),
-    [waveFormData, setWaveFormData] = React.useState<F[] | null>(null),
+    [waveFormData, setWaveFormData] = React.useState<Float32Array | null>(null),
     [isPlaying, setIsPlaying] = React.useState(false),
     [duration, setDuration] = React.useState(0),
     [currentTime, setCurrentTime] = React.useState(0),
@@ -134,21 +135,8 @@ export const XAudioAnnotator = ({ model }: { model: AudioAnnotator }) => {
         setErrMsg('Could not decode audio data. The file is either corrupted or the format is not supported.')
         return
       }
-      const rawData = audioBuffer.getChannelData(0) // We only need to work with one channel of data
-
-      const samples = audioRef.current.parentElement?.clientWidth || 600
-      const blockSize = Math.floor(rawData.length / samples)
-      const filteredData = new Array<U>(samples)
-      for (let i = 0; i < samples; i++) {
-        const blockStart = blockSize * i // the location of the first sample in the block
-        let sum = 0
-        for (let j = 0; j < blockSize; j++) {
-          sum += Math.abs(rawData[blockStart + j]) // find the sum of all the samples in the block
-        }
-        filteredData[i] = sum / blockSize // divide the sum by the block size to get the average
-      }
-      const multiplier = Math.pow(Math.max(...filteredData), -1)
-      setWaveFormData(filteredData.map(n => n * multiplier))
+      const channel2 = audioBuffer.numberOfChannels > 1 ? audioBuffer.getChannelData(1) : undefined
+      setWaveFormData(averageChannels(audioBuffer.getChannelData(0), channel2))
       setDuration(audioBuffer.duration)
       setLoadingMsg('')
     },

ui/src/parts/audioUtils.ts

@@ -0,0 +1,28 @@
+import { F } from "h2o-wave"
+
+export const
+  averageChannels = (channel1: Float32Array, channel2?: Float32Array) => {
+    if (!channel2) return channel1
+    const result = new Float32Array(channel1.length)
+    for (let i = 0; i < channel1.length; i++) {
+      result[i] = (channel1[i] + channel2[i]) / 2
+    }
+    return result
+  },
+  parseAudioData = (samples: F, rawData: F[]) => {
+    if (!samples) return []
+    console.log('parse', samples)
+
+    const blockSize = Math.floor(rawData.length / samples)
+    const filteredData = new Array<F>(samples)
+    for (let i = 0; i < samples; i++) {
+      const blockStart = blockSize * i // the location of the first sample in the block
+      let sum = 0
+      for (let j = 0; j < blockSize; j++) {
+        sum += Math.abs(rawData[blockStart + j]) // find the sum of all the samples in the block
+      }
+      filteredData[i] = sum / blockSize // divide the sum by the block size to get the average
+    }
+    const multiplier = Math.pow(Math.max(...filteredData), -1)
+    return filteredData.map(n => n * multiplier)
+  }

ui/src/parts/range_annotator.tsx

@@ -6,6 +6,7 @@ import { AudioAnnotatorItem, AudioAnnotatorTag } from '../audio_annotator'
 import { isIntersectingRect } from '../image_annotator_rect'
 import { clas, cssVar, cssVarValue } from '../theme'
 import { eventToCursor } from './annotator_utils'
+import { parseAudioData } from './audioUtils'
 
 type RangeAnnotatorProps<T> = {
   activeTag: S
@@ -538,6 +539,14 @@ export const
       [zoom, setZoom] = React.useState({ from: 0, to: 100 }),
       annotatorContainerRef = React.useRef<HTMLDivElement>(null),
       canvasWidth = annotatorContainerRef.current?.getBoundingClientRect().width || 0,
+      parsedAudioData = React.useMemo(() => parseAudioData(canvasWidth, backgroundData as any), [backgroundData, canvasWidth]),
+      parsedZoomAudioData = React.useMemo(() => {
+        return parseAudioData(canvasWidth,
+          backgroundData.slice(
+            Math.ceil(zoom.from / canvasWidth * backgroundData.length),
+            Math.floor(zoom.to / canvasWidth * backgroundData.length)
+          ) as any)
+      }, [backgroundData, canvasWidth, zoom.from, zoom.to]),
       theme = Fluent.useTheme(),
       colorsMap = React.useMemo(() => new Map<S, TagColor>(tags.map(tag => {
         const color = Fluent.getColorFromString(cssVarValue(tag.color))
@@ -668,7 +677,7 @@ export const
           focusAnnotation={focusAnnotation}
           colorsMap={colorsMap}
           setZoom={setZoom}
-        >{onRenderBackground(backgroundData)}</Annotator>
+        >{onRenderBackground(parsedAudioData as any)}</Annotator>
         {needsZoom(duration) && (
           <div ref={annotatorContainerRef}>
             {annotatorContainerRef.current && (
@@ -683,10 +692,7 @@ export const
                 moveOrResizeAnnotation={moveOrResizeAnnotation}
                 focusAnnotation={focusAnnotation}
                 colorsMap={colorsMap}
-              >{onRenderBackground(backgroundData.slice(
-                Math.ceil(zoom.from / canvasWidth * backgroundData.length),
-                Math.floor(zoom.to / canvasWidth * backgroundData.length)
-              ))}</Annotator>
+              >{onRenderBackground(parsedZoomAudioData as any)}</Annotator>
             )}
           </div>
         )}