From d7ed2d3486718d83ea42555946177ed7f3e3ab52 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 3 May 2021 10:27:15 -0600 Subject: [PATCH 1/3] Switch to m.audio with extensible events --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index bab95291bac..981c1499abb 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -27,6 +27,7 @@ import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore"; import {UPDATE_EVENT} from "../../../stores/AsyncStore"; import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import {MsgType} from "matrix-js-sdk/lib/@types/event"; interface IProps { room: Room; @@ -64,8 +65,8 @@ export default class VoiceRecordComposerTile extends React.PureComponent Math.round(v * 1024)), }, + "org.matrix.msc2516.voice": {}, // No content, this is a rendering hint }); await this.disposeRecording(); } From 3f4ee9742abe9543a44ee3caa1dba31e6fb6e475 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 3 May 2021 11:41:14 -0600 Subject: [PATCH 2/3] Simple rendering of voice messages in the timeline Fixes https://github.com/vector-im/element-web/issues/17138 --- res/css/_components.scss | 1 + .../views/messages/_MVoiceMessageBody.scss | 19 ++++ .../views/messages/MVoiceMessageBody.tsx | 106 ++++++++++++++++++ .../views/messages/MVoiceOrAudioBody.tsx | 39 +++++++ src/components/views/messages/MessageEvent.js | 6 +- src/i18n/strings/en_EN.json | 1 + src/voice/Playback.ts | 2 +- 7 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 res/css/views/messages/_MVoiceMessageBody.scss create mode 100644 src/components/views/messages/MVoiceMessageBody.tsx create mode 100644 src/components/views/messages/MVoiceOrAudioBody.tsx diff --git a/res/css/_components.scss b/res/css/_components.scss index 0057f8a8fc5..ec9592f3a1d 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -161,6 +161,7 @@ @import "./views/messages/_MStickerBody.scss"; @import "./views/messages/_MTextBody.scss"; @import "./views/messages/_MVideoBody.scss"; +@import "./views/messages/_MVoiceMessageBody.scss"; @import "./views/messages/_MessageActionBar.scss"; @import "./views/messages/_MessageTimestamp.scss"; @import "./views/messages/_MjolnirBody.scss"; diff --git a/res/css/views/messages/_MVoiceMessageBody.scss b/res/css/views/messages/_MVoiceMessageBody.scss new file mode 100644 index 00000000000..3dfb98f7782 --- /dev/null +++ b/res/css/views/messages/_MVoiceMessageBody.scss @@ -0,0 +1,19 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_MVoiceMessageBody { + display: inline-block; // makes the playback controls magically line up +} diff --git a/src/components/views/messages/MVoiceMessageBody.tsx b/src/components/views/messages/MVoiceMessageBody.tsx new file mode 100644 index 00000000000..4a2a83465d8 --- /dev/null +++ b/src/components/views/messages/MVoiceMessageBody.tsx @@ -0,0 +1,106 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {MatrixEvent} from "matrix-js-sdk/src/models/event"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {Playback} from "../../../voice/Playback"; +import MFileBody from "./MFileBody"; +import InlineSpinner from '../elements/InlineSpinner'; +import {_t} from "../../../languageHandler"; +import {mediaFromContent} from "../../../customisations/Media"; +import {decryptFile} from "../../../utils/DecryptFile"; +import RecordingPlayback from "../voice_messages/RecordingPlayback"; + +interface IProps { + mxEvent: MatrixEvent; +} + +interface IState { + error?: Error; + playback?: Playback; + decryptedBlob?: Blob; +} + +@replaceableComponent("views.messages.MVoiceMessageBody") +export default class MVoiceMessageBody extends React.PureComponent { + constructor(props: IProps) { + super(props); + + this.state = {}; + } + + public async componentDidMount() { + let buffer: ArrayBuffer; + const content = this.props.mxEvent.getContent(); + const media = mediaFromContent(content); + if (media.isEncrypted) { + try { + const blob = await decryptFile(content.file); + buffer = await blob.arrayBuffer(); + this.setState({decryptedBlob: blob}); + } catch (e) { + this.setState({error: e}); + console.warn("Unable to decrypt voice message", e); + return; // stop processing the audio file + } + } else { + try { + buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer()); + } catch (e) { + this.setState({error: e}); + console.warn("Unable to download voice message", e); + return; // stop processing the audio file + } + } + + const waveform = content?.["org.matrix.msc1767.audio"]?.waveform?.map(p => p / 1024); + + // We should have a buffer to work with now: let's set it up + const playback = new Playback(buffer, waveform); + this.setState({playback}); + // Note: the RecordingPlayback component will handle preparing the Playback class for us. + } + + public render() { + if (this.state.error) { + // TODO: @@TR: Verify error state + return ( + + + { _t("Error processing voice message") } + + ); + } + + if (!this.state.playback) { + // TODO: @@TR: Verify loading/decrypting state + return ( + + + + ); + } + + // At this point we should have a playable state + return ( + + + + + ) + } +} diff --git a/src/components/views/messages/MVoiceOrAudioBody.tsx b/src/components/views/messages/MVoiceOrAudioBody.tsx new file mode 100644 index 00000000000..0cebcf3440b --- /dev/null +++ b/src/components/views/messages/MVoiceOrAudioBody.tsx @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {MatrixEvent} from "matrix-js-sdk/src/models/event"; +import MAudioBody from "./MAudioBody"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import SettingsStore from "../../../settings/SettingsStore"; +import MVoiceMessageBody from "./MVoiceMessageBody"; + +interface IProps { + mxEvent: MatrixEvent; +} + +@replaceableComponent("views.messages.MVoiceOrAudioBody") +export default class MVoiceOrAudioBody extends React.PureComponent { + public render() { + const isVoiceMessage = !!this.props.mxEvent.getContent()['org.matrix.msc2516.voice']; + const voiceMessagesEnabled = SettingsStore.getValue("feature_voice_messages"); + if (isVoiceMessage && voiceMessagesEnabled) { + return ; + } else { + return ; + } + } +} diff --git a/src/components/views/messages/MessageEvent.js b/src/components/views/messages/MessageEvent.js index 60f7631c8ea..78e0dc422d7 100644 --- a/src/components/views/messages/MessageEvent.js +++ b/src/components/views/messages/MessageEvent.js @@ -72,12 +72,8 @@ export default class MessageEvent extends React.Component { 'm.emote': sdk.getComponent('messages.TextualBody'), 'm.image': sdk.getComponent('messages.MImageBody'), 'm.file': sdk.getComponent('messages.MFileBody'), - 'm.audio': sdk.getComponent('messages.MAudioBody'), + 'm.audio': sdk.getComponent('messages.MVoiceOrAudioBody'), 'm.video': sdk.getComponent('messages.MVideoBody'), - - // TODO: @@ TravisR: Use labs flag determination. - // MSC: https://github.com/matrix-org/matrix-doc/pull/2516 - 'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'), }; const evTypes = { 'm.sticker': sdk.getComponent('messages.MStickerBody'), diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 5863f2a834d..312e98e6d1d 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1848,6 +1848,7 @@ "%(name)s wants to verify": "%(name)s wants to verify", "You sent a verification request": "You sent a verification request", "Error decrypting video": "Error decrypting video", + "Error processing voice message": "Error processing voice message", "Show all": "Show all", "Reactions": "Reactions", " reacted with %(content)s": " reacted with %(content)s", diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 99b1f628661..49a80a35d17 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -49,7 +49,7 @@ export class Playback extends EventEmitter implements IDestroyable { constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) { super(); this.context = new AudioContext(); - this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES); + this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); From 8abd251ec56f229137cda107da5759ea74fc3927 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 3 May 2021 11:43:46 -0600 Subject: [PATCH 3/3] delib --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 981c1499abb..f0acc4fb092 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -27,7 +27,7 @@ import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore"; import {UPDATE_EVENT} from "../../../stores/AsyncStore"; import RecordingPlayback from "../voice_messages/RecordingPlayback"; -import {MsgType} from "matrix-js-sdk/lib/@types/event"; +import {MsgType} from "matrix-js-sdk/src/@types/event"; interface IProps { room: Room;