Skip to content

Commit

Permalink
Fix VP9 decoding on iOS
Browse files Browse the repository at this point in the history
The current implementation of the VP9 payloader produces payloads that
are not compatible with iOS. This is because the payloader provides
only the muxing strategy called "flexible mode".

According to the VP9 RFC draft, there are two ways to wrap VP9 frames
into RTP packets: the "flexible mode" and the "non-flexible mode", with
the latter being the preferred one for live-streaming applications. In
particular, all browsers encodes VP9 RTP packets in the "non-flexible
mode", while iOS supports decoding RTP packets in this mode only, and
this is probably a problem shared by other implementations.

This patch improves the VP9 payloader by adding support for the
"non-flexible mode". The "flexible mode" is retained and a flag is
provided to perform the switch between the two modes.
  • Loading branch information
aler9 authored and Sean-Der committed Apr 29, 2024
1 parent 12646b6 commit bc5124c
Show file tree
Hide file tree
Showing 5 changed files with 598 additions and 60 deletions.
65 changes: 65 additions & 0 deletions codecs/vp9/bits.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// SPDX-FileCopyrightText: 2023 The Pion community <https://pion.ly>
// SPDX-License-Identifier: MIT

package vp9

import "errors"

var errNotEnoughBits = errors.New("not enough bits")

func hasSpace(buf []byte, pos int, n int) error {
if n > ((len(buf) * 8) - pos) {
return errNotEnoughBits
}
return nil
}

func readFlag(buf []byte, pos *int) (bool, error) {
err := hasSpace(buf, *pos, 1)
if err != nil {
return false, err
}

return readFlagUnsafe(buf, pos), nil
}

func readFlagUnsafe(buf []byte, pos *int) bool {
b := (buf[*pos>>0x03] >> (7 - (*pos & 0x07))) & 0x01
*pos++
return b == 1
}

func readBits(buf []byte, pos *int, n int) (uint64, error) {
err := hasSpace(buf, *pos, n)
if err != nil {
return 0, err
}

return readBitsUnsafe(buf, pos, n), nil
}

func readBitsUnsafe(buf []byte, pos *int, n int) uint64 {
res := 8 - (*pos & 0x07)
if n < res {
v := uint64((buf[*pos>>0x03] >> (res - n)) & (1<<n - 1))
*pos += n
return v
}

v := uint64(buf[*pos>>0x03] & (1<<res - 1))
*pos += res
n -= res

for n >= 8 {
v = (v << 8) | uint64(buf[*pos>>0x03])
*pos += 8
n -= 8
}

if n > 0 {
v = (v << n) | uint64(buf[*pos>>0x03]>>(8-n))
*pos += n
}

return v
}
221 changes: 221 additions & 0 deletions codecs/vp9/header.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
// SPDX-FileCopyrightText: 2023 The Pion community <https://pion.ly>
// SPDX-License-Identifier: MIT

// Package vp9 contains a VP9 header parser.
package vp9

import (
"errors"
)

var (
errInvalidFrameMarker = errors.New("invalid frame marker")
errWrongFrameSyncByte0 = errors.New("wrong frame_sync_byte_0")
errWrongFrameSyncByte1 = errors.New("wrong frame_sync_byte_1")
errWrongFrameSyncByte2 = errors.New("wrong frame_sync_byte_2")
)

// HeaderColorConfig is the color_config member of an header.
type HeaderColorConfig struct {
TenOrTwelveBit bool
BitDepth uint8
ColorSpace uint8
ColorRange bool
SubsamplingX bool
SubsamplingY bool
}

func (c *HeaderColorConfig) unmarshal(profile uint8, buf []byte, pos *int) error {
if profile >= 2 {
var err error
c.TenOrTwelveBit, err = readFlag(buf, pos)
if err != nil {
return err
}

if c.TenOrTwelveBit {
c.BitDepth = 12
} else {
c.BitDepth = 10
}
} else {
c.BitDepth = 8
}

tmp, err := readBits(buf, pos, 3)
if err != nil {
return err
}
c.ColorSpace = uint8(tmp)

if c.ColorSpace != 7 {
var err error
c.ColorRange, err = readFlag(buf, pos)
if err != nil {
return err
}

if profile == 1 || profile == 3 {
err := hasSpace(buf, *pos, 3)
if err != nil {
return err
}

c.SubsamplingX = readFlagUnsafe(buf, pos)
c.SubsamplingY = readFlagUnsafe(buf, pos)
*pos++
} else {
c.SubsamplingX = true
c.SubsamplingY = true
}
} else {
c.ColorRange = true

if profile == 1 || profile == 3 {
c.SubsamplingX = false
c.SubsamplingY = false

err := hasSpace(buf, *pos, 1)
if err != nil {
return err
}
*pos++
}
}

return nil
}

// HeaderFrameSize is the frame_size member of an header.
type HeaderFrameSize struct {
FrameWidthMinus1 uint16
FrameHeightMinus1 uint16
}

func (s *HeaderFrameSize) unmarshal(buf []byte, pos *int) error {
err := hasSpace(buf, *pos, 32)
if err != nil {
return err
}

s.FrameWidthMinus1 = uint16(readBitsUnsafe(buf, pos, 16))
s.FrameHeightMinus1 = uint16(readBitsUnsafe(buf, pos, 16))
return nil
}

// Header is a VP9 Frame header.
// Specification:
// https://storage.googleapis.com/downloads.webmproject.org/docs/vp9/vp9-bitstream-specification-v0.6-20160331-draft.pdf
type Header struct {
Profile uint8
ShowExistingFrame bool
FrameToShowMapIdx uint8
NonKeyFrame bool
ShowFrame bool
ErrorResilientMode bool
ColorConfig *HeaderColorConfig
FrameSize *HeaderFrameSize
}

// Unmarshal decodes a Header.
func (h *Header) Unmarshal(buf []byte) error {
pos := 0

err := hasSpace(buf, pos, 4)
if err != nil {
return err
}

frameMarker := readBitsUnsafe(buf, &pos, 2)
if frameMarker != 2 {
return errInvalidFrameMarker
}

profileLowBit := uint8(readBitsUnsafe(buf, &pos, 1))
profileHighBit := uint8(readBitsUnsafe(buf, &pos, 1))
h.Profile = profileHighBit<<1 + profileLowBit

if h.Profile == 3 {
err = hasSpace(buf, pos, 1)
if err != nil {
return err
}
pos++
}

h.ShowExistingFrame, err = readFlag(buf, &pos)
if err != nil {
return err
}

if h.ShowExistingFrame {
var tmp uint64
tmp, err = readBits(buf, &pos, 3)
if err != nil {
return err
}
h.FrameToShowMapIdx = uint8(tmp)
return nil
}

err = hasSpace(buf, pos, 3)
if err != nil {
return err
}

h.NonKeyFrame = readFlagUnsafe(buf, &pos)
h.ShowFrame = readFlagUnsafe(buf, &pos)
h.ErrorResilientMode = readFlagUnsafe(buf, &pos)

if !h.NonKeyFrame {
err := hasSpace(buf, pos, 24)
if err != nil {
return err
}

frameSyncByte0 := uint8(readBitsUnsafe(buf, &pos, 8))
if frameSyncByte0 != 0x49 {
return errWrongFrameSyncByte0
}

frameSyncByte1 := uint8(readBitsUnsafe(buf, &pos, 8))
if frameSyncByte1 != 0x83 {
return errWrongFrameSyncByte1
}

frameSyncByte2 := uint8(readBitsUnsafe(buf, &pos, 8))
if frameSyncByte2 != 0x42 {
return errWrongFrameSyncByte2
}

h.ColorConfig = &HeaderColorConfig{}
err = h.ColorConfig.unmarshal(h.Profile, buf, &pos)
if err != nil {
return err
}

h.FrameSize = &HeaderFrameSize{}
err = h.FrameSize.unmarshal(buf, &pos)
if err != nil {
return err
}
}

return nil
}

// Width returns the video width.
func (h Header) Width() uint16 {
if h.FrameSize == nil {
return 0
}
return h.FrameSize.FrameWidthMinus1 + 1
}

// Height returns the video height.
func (h Header) Height() uint16 {
if h.FrameSize == nil {
return 0
}
return h.FrameSize.FrameHeightMinus1 + 1
}
85 changes: 85 additions & 0 deletions codecs/vp9/header_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// SPDX-FileCopyrightText: 2023 The Pion community <https://pion.ly>
// SPDX-License-Identifier: MIT

package vp9

import (
"reflect"
"testing"
)

func TestHeaderUnmarshal(t *testing.T) {
cases := []struct {
name string
byts []byte
sh Header
width uint16
height uint16
}{
{
"chrome webrtc",
[]byte{
0x82, 0x49, 0x83, 0x42, 0x00, 0x77, 0xf0, 0x32,
0x34, 0x30, 0x38, 0x24, 0x1c, 0x19, 0x40, 0x18,
0x03, 0x40, 0x5f, 0xb4,
},
Header{
ShowFrame: true,
ColorConfig: &HeaderColorConfig{
BitDepth: 8,
SubsamplingX: true,
SubsamplingY: true,
},
FrameSize: &HeaderFrameSize{
FrameWidthMinus1: 1919,
FrameHeightMinus1: 803,
},
},
1920,
804,
},
{
"vp9 sample",
[]byte{
0x82, 0x49, 0x83, 0x42, 0x40, 0xef, 0xf0, 0x86,
0xf4, 0x04, 0x21, 0xa0, 0xe0, 0x00, 0x30, 0x70,
0x00, 0x00, 0x00, 0x01,
},
Header{
ShowFrame: true,
ColorConfig: &HeaderColorConfig{
BitDepth: 8,
ColorSpace: 2,
SubsamplingX: true,
SubsamplingY: true,
},
FrameSize: &HeaderFrameSize{
FrameWidthMinus1: 3839,
FrameHeightMinus1: 2159,
},
},
3840,
2160,
},
}

for _, ca := range cases {
t.Run(ca.name, func(t *testing.T) {
var sh Header
err := sh.Unmarshal(ca.byts)
if err != nil {
t.Fatal("unexpected error")
}

if !reflect.DeepEqual(ca.sh, sh) {
t.Fatalf("expected %#+v, got %#+v", ca.sh, sh)
}
if ca.width != sh.Width() {
t.Fatalf("unexpected width")
}
if ca.height != sh.Height() {
t.Fatalf("unexpected height")
}
})
}
}
Loading

0 comments on commit bc5124c

Please sign in to comment.