fix: segmentation_upids with certain ISO-8859-1 characters decoded in…

…correctly * Handle ASCII-encoded segmentation UPIDs correctly * fix: move test and more complete fix for ascii chars * chore: cleanup some tmp changes * docs: add some comments around the Latin <-> UTF8 process * docs: Update SegmentationUPID.ASCIIValue() * chore: address feedback --------- Co-authored-by: Sean Bowman <pico303@gmail.com>
Comcast · Aug 2, 2023 · 377cee5 · 377cee5
1 parent 0992f43
commit 377cee5
Show file tree

Hide file tree

Showing 10 changed files with 156 additions and 49 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -9,19 +9,22 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
-  - repo: https://github.com/dnephin/pre-commit-golang
-    rev: v0.5.1
+  - repo: https://github.com/tekwizely/pre-commit-golang
+    rev: v1.0.0-rc.1
     hooks:
       - id: go-fmt
-      - id: go-vet
-      - id: golangci-lint
-      - id: go-unit-tests
-      - id: go-build
+      - id: go-build-mod
       - id: go-mod-tidy
-  - repo: https://github.com/sirosen/texthooks
-    rev: 0.5.0
+      - id: golangci-lint-mod
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.32.0
     hooks:
-      - id: fix-smartquotes
+      - id: yamllint
+        args: [--format, colored]
+  - repo: https://github.com/zricethezav/gitleaks
+    rev: v8.17.0
+    hooks:
+      - id: gitleaks
   - repo: https://github.com/compilerla/conventional-pre-commit
     rev: v2.3.0
     hooks:

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
@@ -71,4 +71,4 @@ This Code of Conduct is adapted from the [Contributor Covenant][homepage], versi
 available at [http://contributor-covenant.org/version/1/4][version]
 
 [homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
+[version]: http://contributor-covenant.org/version/1/4/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -32,4 +32,3 @@ If your change is adding your newly released Comcast Open Source project to http
 Before Comcast merges your code into the project you must sign the [Comcast Contributor License Agreement (CLA)](https://gist.github.com/ComcastOSS/a7b8933dd8e368535378cda25c92d19a).
 
 If you haven't previously signed a Comcast CLA, you'll automatically be asked to when you open a pull request. Alternatively, we can send you a PDF that you can sign and scan back to us. Please create a new GitHub issue to request a PDF version of the CLA.
-
diff --git a/NOTICE.md b/NOTICE.md
@@ -1,4 +1,4 @@
-SCTE-35 
+SCTE-35
 
 Copyright 2021 Comcast Cable Communications Management, LLC
 

diff --git a/examples/README.md b/examples/README.md
@@ -8,4 +8,3 @@ To execute an example from the command line:
 ```shell
 $ go run ./examples/simple_decode
 ```
-
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/bamiaux/iobit v0.0.0-20170418073505-498159a04883
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
+	golang.org/x/text v0.10.0
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -19,6 +19,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

diff --git a/pkg/scte35/scte35_test.go b/pkg/scte35/scte35_test.go
@@ -20,7 +20,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/xml"
-	"fmt"
 	"io"
 	"os"
 	"testing"
@@ -30,6 +29,11 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestMain(m *testing.M) {
+	scte35.Logger.SetOutput(os.Stderr)
+	os.Exit(m.Run())
+}
+
 func TestDecodeBase64(t *testing.T) {
 	scte35.Logger.SetOutput(os.Stderr)
 	defer scte35.Logger.SetOutput(io.Discard)
@@ -63,7 +67,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationTypeID:   scte35.SegmentationTypeProviderPOStart,
 						SegmentationDuration: uint64ptr(0x0001a599b0),
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca0a18a)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca0a18a)),
 						},
 						SegmentNum: 2,
 					},
@@ -118,7 +122,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x4800008e,
 						SegmentationTypeID:  scte35.SegmentationTypeProviderPOEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca0a18a)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca0a18a)),
 						},
 						SegmentNum: 2,
 					},
@@ -147,7 +151,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000018,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ccbc344)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ccbc344)),
 						},
 					},
 					&scte35.SegmentationDescriptor{
@@ -160,7 +164,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000019,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramStart,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca4dba0)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca4dba0)),
 						},
 					},
 				},
@@ -188,7 +192,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000008,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramOverlapStart,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca56cf5)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca56cf5)),
 						},
 					},
 				},
@@ -216,7 +220,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x4800000a,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramBlackoutOverride,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca0a1e3)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca0a1e3)),
 						},
 					},
 					&scte35.SegmentationDescriptor{
@@ -229,7 +233,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000009,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca0a18a)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca0a18a)),
 						},
 					},
 				},
@@ -257,7 +261,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000007,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca56c97)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca56c97)),
 						},
 					},
 				},
@@ -285,7 +289,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x480000ad,
 						SegmentationTypeID:  scte35.SegmentationTypeProviderPOEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002cb2d79d)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002cb2d79d)),
 						},
 						SegmentNum: 2,
 					},
@@ -299,7 +303,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000026,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramEnd,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002cb2d79d)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002cb2d79d)),
 						},
 					},
 					&scte35.SegmentationDescriptor{
@@ -312,7 +316,7 @@ func TestDecodeBase64(t *testing.T) {
 						SegmentationEventID: 0x48000027,
 						SegmentationTypeID:  scte35.SegmentationTypeProgramStart,
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002cb2d7b3)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002cb2d7b3)),
 						},
 					},
 				},
@@ -388,7 +392,7 @@ func TestDecodeBase64(t *testing.T) {
 		},
 		"Empty String": {
 			binary: "",
-			err:    fmt.Errorf("splice_info_section: %w", scte35.ErrBufferOverflow),
+			err:    scte35.ErrBufferOverflow,
 		},
 		"Invalid Base64 Encoding": {
 			binary: "/DBaf%^",
@@ -468,7 +472,7 @@ func TestDecodeBase64(t *testing.T) {
 		},
 		"SpliceInsert Time Specified With Invalid Component Count": {
 			binary: "FkC1lwP3uTQD0VvxHwVBEH89G6B7VjzaZ9eNuyUF9q8pYAIXsRM9ZpDCczBeDbytQhXkssQstGJVGcvjZ3tiIMULiA4BpRHlzLGFa0q6aVMtzk8ZRUeLcxtKibgVOKBBnkCbOQyhSflFiDkrAAIp+Fk+VRsByTSkPN3RvyK+lWcjHElhwa9hNFcAy4dm3DdeRXnrD3I2mISNc7DkgS0ReotPyp94FV77xMHT4D7SYL48XU20UM4bgg==",
-			err:    fmt.Errorf("splice_insert: %w", scte35.ErrBufferOverflow),
+			err:    scte35.ErrBufferOverflow,
 		},
 		"Signal with non-CUEI descriptor": {
 			binary: "/DBPAAAAAAAAAP/wBQb/Gq9LggA5AAVTQVBTCwIwQ1VFSf////9//wAAFI4PDxx1cm46bmJjdW5pLmNvbTpicmM6NDk5ODY2NDM0MQoBbM98zw==",
@@ -507,7 +511,7 @@ func TestDecodeBase64(t *testing.T) {
 		},
 		"Invalid CRC_32": {
 			binary: "/DA4AAAAAAAAAP/wFAUABDEAf+//mWEhzP4Azf5gAQAAAAATAhFDVUVJAAAAAX+/AQIwNAEAAKeYO3Q=",
-			err:    fmt.Errorf("splice_info_section: %w", scte35.ErrCRC32Invalid),
+			err:    scte35.ErrCRC32Invalid,
 		},
 		"Alignment Stuffing without Encryption": {
 			binary: "/DAeAAAAAAAAAP///wViAA/nf18ACQAAAAAskJv+YPtE",
@@ -523,13 +527,103 @@ func TestDecodeBase64(t *testing.T) {
 			},
 			legacy: true, // binary wont match because of stuffing
 		},
+		"UPID with Valid ASCII Invalid UTF8": {
+			binary: "/DDHAAAAABc0AP/wBQb/tVo+agCxAhdDVUVJQA4hwH+fCAgAAAAAPj6IcCMAAAIXQ1VFSUAOI1x/nwgIAAAAAD4+iHARAAACF0NVRUlADiHgf58ICAAAAAA+Poi2EAAAAhxDVUVJQA4hyn/fAABSlKwICAAAAAA+Poi2IgAAAkZDVUVJQA4h1n/PAABSlKwNMgoMFHf5uXs0AAAAAAAADhh0eXBlPUxBJmR1cj02MDAwMCZ0aWVy/DDHAAAAAAAAAP8ABQb/HPCt2w==",
+			expected: scte35.SpliceInfoSection{
+				SpliceCommand: &scte35.TimeSignal{
+					SpliceTime: scte35.SpliceTime{
+						PTSTime: uint64ptr(7337557610),
+					},
+				},
+				SpliceDescriptors: scte35.SpliceDescriptors{
+					&scte35.SegmentationDescriptor{
+						SegmentationEventID: uint32(1074667968),
+						SegmentationTypeID:  scte35.SegmentationTypeBreakEnd,
+						DeliveryRestrictions: &scte35.DeliveryRestrictions{
+							ArchiveAllowedFlag:     true,
+							WebDeliveryAllowedFlag: true,
+							NoRegionalBlackoutFlag: true,
+							DeviceRestrictions:     3,
+						},
+						SegmentationUPIDs: []scte35.SegmentationUPID{
+							{Type: scte35.SegmentationUPIDTypeTI, Value: "1044285552"},
+						},
+					},
+					&scte35.SegmentationDescriptor{
+						SegmentationEventID: uint32(1074668380),
+						SegmentationTypeID:  scte35.SegmentationTypeProgramEnd,
+						DeliveryRestrictions: &scte35.DeliveryRestrictions{
+							ArchiveAllowedFlag:     true,
+							WebDeliveryAllowedFlag: true,
+							NoRegionalBlackoutFlag: true,
+							DeviceRestrictions:     3,
+						},
+						SegmentationUPIDs: []scte35.SegmentationUPID{
+							{Type: scte35.SegmentationUPIDTypeTI, Value: "1044285552"},
+						},
+					},
+					&scte35.SegmentationDescriptor{
+						SegmentationEventID: uint32(1074668000),
+						SegmentationTypeID:  scte35.SegmentationTypeProgramStart,
+						DeliveryRestrictions: &scte35.DeliveryRestrictions{
+							ArchiveAllowedFlag:     true,
+							WebDeliveryAllowedFlag: true,
+							NoRegionalBlackoutFlag: true,
+							DeviceRestrictions:     3,
+						},
+						SegmentationUPIDs: []scte35.SegmentationUPID{
+							{Type: scte35.SegmentationUPIDTypeTI, Value: "1044285622"},
+						},
+					},
+					&scte35.SegmentationDescriptor{
+						SegmentationEventID:  uint32(1074667978),
+						SegmentationDuration: uint64ptr(5412012),
+						SegmentationTypeID:   scte35.SegmentationTypeBreakStart,
+						DeliveryRestrictions: &scte35.DeliveryRestrictions{
+							ArchiveAllowedFlag:     true,
+							WebDeliveryAllowedFlag: true,
+							NoRegionalBlackoutFlag: true,
+							DeviceRestrictions:     3,
+						},
+						SegmentationUPIDs: []scte35.SegmentationUPID{
+							{Type: scte35.SegmentationUPIDTypeTI, Value: "1044285622"},
+						},
+					},
+					&scte35.SegmentationDescriptor{
+						SegmentationEventID:  uint32(1074667990),
+						SegmentationTypeID:   0x05,
+						SegmentationDuration: uint64ptr(5412012),
+						SegmentNum:           6,
+						SegmentsExpected:     255,
+						DeliveryRestrictions: &scte35.DeliveryRestrictions{
+							ArchiveAllowedFlag:     true,
+							WebDeliveryAllowedFlag: false,
+							NoRegionalBlackoutFlag: true,
+							DeviceRestrictions:     3,
+						},
+						SegmentationUPIDs: []scte35.SegmentationUPID{
+							{Type: scte35.SegmentationUPIDTypeEIDR, Value: "10.5239/F9B9-7B34-0000-0000-0000"},
+							{Type: scte35.SegmentationUPIDTypeADS, Value: "type=LA&dur=60000&tierü0"},
+							{Type: uint32(199)},
+							{Type: uint32(0)},
+							{Type: uint32(0)},
+							{Type: uint32(0)},
+							{Type: uint32(255)},
+						},
+					},
+				},
+				PTSAdjustment: uint64(5940),
+				Tier:          4095,
+				SAPType:       3,
+			},
+		},
 	}
 
 	for k, c := range cases {
 		t.Run(k, func(t *testing.T) {
 			// decode the binary
 			sis, err := scte35.DecodeBase64(c.binary)
-			require.Equal(t, c.err, err)
+			require.ErrorIs(t, err, c.err)
 			if err != nil {
 				return
 			}
@@ -591,7 +685,7 @@ func TestDecodeHex(t *testing.T) {
 						SegmentationTypeID:   scte35.SegmentationTypeProviderPOStart,
 						SegmentationDuration: uint64ptr(0x0001a599b0),
 						SegmentationUPIDs: []scte35.SegmentationUPID{
-							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, bytes(0x000000002ca0a18a)),
+							scte35.NewSegmentationUPID(scte35.SegmentationUPIDTypeTI, toBytes(0x000000002ca0a18a)),
 						},
 						SegmentNum: 2,
 					},
@@ -686,7 +780,7 @@ func TestTicksToDuration(t *testing.T) {
 
 // helper func to make test life a bit easier
 
-func bytes(i uint64) []byte {
+func toBytes(i uint64) []byte {
 	b := make([]byte, 8)
 	binary.BigEndian.PutUint64(b, i)
 	return b

diff --git a/pkg/scte35/segmentation_descriptor.go b/pkg/scte35/segmentation_descriptor.go
@@ -326,6 +326,8 @@ func (sd *SegmentationDescriptor) SegmentationUpidLength() int {
 
 // decode updates this splice_descriptor from binary.
 func (sd *SegmentationDescriptor) decode(b []byte) error {
+	var err error
+
 	r := iobit.NewReader(b)
 	r.Skip(8)  // splice_descriptor_tag
 	r.Skip(8)  // descriptor_length
@@ -378,6 +380,9 @@ func (sd *SegmentationDescriptor) decode(b []byte) error {
 					upidType := upidr.Uint32(8)
 					upidLength := int(upidr.Uint32(8))
 					upidValue := upidr.Bytes(upidLength)
+					if len(upidValue) < upidLength {
+						Logger.Printf("Cannot read value for segmentation_upid_type %d; %d of %d bytes remaining.", upidType, len(upidValue), upidLength)
+					}
 					sd.SegmentationUPIDs = append(
 						sd.SegmentationUPIDs,
 						NewSegmentationUPID(upidType, upidValue),
@@ -406,6 +411,9 @@ func (sd *SegmentationDescriptor) decode(b []byte) error {
 		}
 	}
 
+	if err != nil {
+		return err
+	}
 	if err := readerError(r); err != nil {
 		return fmt.Errorf("segmentation_descriptor: %w", err)
 	}