Update sig/token and stream options requests to use newer API endpoints

The original endpoint used to get the sig/token values appears to be completely dead now so switching to the newer GQL API endpoint to query for access details. Additionally, use osrtss/rtss to parse the m3u8 files rather than regex.
dbarbuzzi · Jan 15, 2021 · f8db4e1 · f8db4e1
1 parent 301c6fa
commit f8db4e1
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 49 deletions.
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.3.0
 	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
 	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
+	github.com/osrtss/rtss v0.0.0-20170322072109-b1617c76f6da
 	github.com/pkg/errors v0.8.0
 	github.com/schollz/progressbar/v3 v3.7.3
 	github.com/stretchr/testify v1.6.1 // indirect

diff --git a/go.sum b/go.sum
@@ -15,6 +15,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/osrtss/rtss v0.0.0-20170322072109-b1617c76f6da h1:/Z/TuENqR/F8BVs0WUpAIegWgfKY8EoP93i3kRMapRQ=
+github.com/osrtss/rtss v0.0.0-20170322072109-b1617c76f6da/go.mod h1:HP7OZkbc9jh3kEc5zfljE3gG++ejTIbq3HZRui5n+mo=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

diff --git a/tvd.go b/tvd.go
@@ -3,6 +3,7 @@ package main
 // Based on https://github.com/ArneVogel/concat
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,11 +13,11 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	"github.com/osrtss/rtss/m3u8"
 	"github.com/schollz/progressbar/v3"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
@@ -166,13 +167,13 @@ func createDefaultConfigFile() error {
 // DownloadVOD downloads a VOD based on the various info passed in the config
 func DownloadVOD(cfg Config) error {
 	fmt.Println("Fetching access token")
-	atr, err := getAuthToken(cfg.VodID, cfg.AuthToken)
+	ar, err := getAuthToken(cfg.VodID, cfg.ClientID)
 	if err != nil {
 		return err
 	}
 
 	fmt.Println("Fetching VOD stream options")
-	ql, err := getStreamOptions(cfg.VodID, atr)
+	ql, err := getStreamOptions(cfg.VodID, ar)
 	if err != nil {
 		return err
 	}
@@ -229,54 +230,95 @@ func DownloadVOD(cfg Config) error {
 	return nil
 }
 
-func getAuthToken(vodID int, authToken string) (AuthTokenResponse, error) {
+func getAuthToken(vodID int, clientID string) (AuthGQLResponse, error) {
 	log.Printf("[getAuthToken] vodID=%d\n", vodID)
-	var atr AuthTokenResponse
-	url := fmt.Sprintf("https://api.twitch.tv/api/vods/%d/access_token?oauth_token=%s", vodID, authToken)
-	respData, err := readURL(url)
+	var ar AuthGQLResponse
+
+	ap, err := generateAuthPayload(strconv.Itoa(vodID))
 	if err != nil {
-		return atr, err
+		return ar, err
 	}
 
-	err = json.Unmarshal(respData, &atr)
+	url := "https://gql.twitch.tv/gql"
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(ap))
 	if err != nil {
-		return atr, err
+		return ar, err
 	}
-	if len(atr.Sig) == 0 || len(atr.Token) == 0 {
-		return atr, fmt.Errorf("error: sig and/or token were empty: %+v", atr)
+	req.Header.Set("Client-ID", clientID)
+	req.Header.Set("Content-Type", "text/plain; charset=UTF-8")
+
+	rsp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return ar, err
 	}
+	defer func() {
+		err = rsp.Body.Close()
+		if err != nil {
+			fmt.Printf("error closing URL body for <%s>: %s", url, err.Error())
+			log.Println(err)
+		}
+	}()
 
-	log.Printf("access token: %+v\n", atr)
+	rspData, err := ioutil.ReadAll(rsp.Body)
+	if err != nil {
+		return ar, err
+	}
+
+	err = json.Unmarshal(rspData, &ar)
+	if err != nil {
+		return ar, err
+	}
+	if len(ar.Data.VideoPlaybackAccessToken.Signature) == 0 || len(ar.Data.VideoPlaybackAccessToken.Value) == 0 {
+		log.Printf("response: %s\n", rspData)
+		return ar, fmt.Errorf("error: sig and/or token were empty: %+v", ar)
+	}
 
-	return atr, nil
+	log.Printf("access token: %+v\n", ar)
+
+	return ar, nil
 }
 
-func getStreamOptions(vodID int, atr AuthTokenResponse) (map[string]string, error) {
-	log.Printf("[getAuthToken] vodID=%d, atr=%+v\n", vodID, atr)
+func getStreamOptions(vodID int, ar AuthGQLResponse) (map[string]string, error) {
+	log.Printf("[getStreamOptions] vodID=%d, ar=%+v\n", vodID, ar)
 	var ql = make(map[string]string)
 
-	url := fmt.Sprintf("http://usher.twitch.tv/vod/%d?nauthsig=%s&nauth=%s&allow_source=true", vodID, atr.Sig, atr.Token)
-	respData, err := readURL(url)
+	url := fmt.Sprintf(
+		"https://usher.ttvnw.net/vod/%d.m3u8?allow_source=true&sig=%s&token=%s",
+		vodID,
+		ar.Data.VideoPlaybackAccessToken.Signature,
+		ar.Data.VideoPlaybackAccessToken.Value,
+	)
+	rsp, err := http.Get(url)
 	if err != nil {
 		return nil, err
 	}
+	defer func() {
+		err = rsp.Body.Close()
+		if err != nil {
+			fmt.Printf("error closing URL body for <%s>: %s", url, err.Error())
+			log.Println(err)
+		}
+	}()
 
-	re := regexp.MustCompile(`BANDWIDTH=(\d+),.*?VIDEO="(.*?)"\n(.*?)\n`)
-	matches := re.FindAllStringSubmatch(string(respData), -1)
-	if len(matches) == 0 {
-		log.Printf("response for m3u:\n%s\n", respData)
-		return nil, fmt.Errorf("error: no matches found")
+	p, listType, err := m3u8.DecodeFrom(rsp.Body, true)
+	if err != nil {
+		return nil, err
 	}
 
-	bestBandwidth := 0
-	for _, match := range matches {
-		ql[match[2]] = match[3]
-		// "safe" to ignore error as regex only matches digits for this capture grouop
-		bandwidth, _ := strconv.Atoi(match[1])
-		if bandwidth > bestBandwidth {
-			bestBandwidth = bandwidth
-			ql["best"] = match[3]
+	switch listType {
+	case m3u8.MASTER:
+		masterPl := p.(*m3u8.MasterPlaylist)
+		var bestBandwidth uint32
+		for _, v := range masterPl.Variants {
+			ql[v.Resolution] = v.URI
+			if v.Bandwidth > bestBandwidth {
+				bestBandwidth = v.Bandwidth
+				ql["best"] = v.URI
+			}
+
 		}
+	default:
+		return nil, fmt.Errorf("m3u8 playlist was not the expected 'master' format")
 	}
 
 	log.Printf("qualities options found: %+v\n", ql)
@@ -286,31 +328,44 @@ func getStreamOptions(vodID int, atr AuthTokenResponse) (map[string]string, erro
 
 func getChunks(streamURL string) ([]Chunk, int, error) {
 	var chunks []Chunk
+	var chunkDur int
 
-	respData, err := readURL(streamURL)
+	rsp, err := http.Get(streamURL)
 	if err != nil {
 		return nil, 0, err
 	}
+	defer func() {
+		err = rsp.Body.Close()
+		if err != nil {
+			fmt.Printf("error closing URL body for <%s>: %s", streamURL, err.Error())
+			log.Println(err)
+		}
+	}()
 
-	re := regexp.MustCompile(`#EXTINF:(\d+\.\d+),\n(.*?)\n`)
-	matches := re.FindAllStringSubmatch(string(respData), -1)
-
-	// "safe" to ignore because we already fetched it
-	baseURL, _ := url.Parse(streamURL)
-	for _, match := range matches {
-		// "safe" to ignore error due to regex capture group
-		length, _ := strconv.ParseFloat(match[1], 64)
-		// "safe" to ignore error ... due to capture group?
-		chunkURL, _ := url.Parse(match[2])
-		chunkURL = baseURL.ResolveReference(chunkURL)
-		chunks = append(chunks, Chunk{Name: match[2], Length: length, URL: chunkURL})
+	p, listType, err := m3u8.DecodeFrom(rsp.Body, true)
+	if err != nil {
+		return nil, 0, err
 	}
-	log.Printf("found %d chunks", len(chunks))
 
-	re = regexp.MustCompile(`#EXT-X-TARGETDURATION:(\d+)\n`)
-	match := re.FindStringSubmatch(string(respData))
-	chunkDur, _ := strconv.Atoi(match[1])
-	log.Printf("target chunk duration: %d", chunkDur)
+	switch listType {
+	case m3u8.MEDIA:
+		mediaPl := p.(*m3u8.MediaPlaylist)
+
+		chunkDur = int(mediaPl.TargetDuration)
+		log.Printf("target chunk duration: %d", chunkDur)
+
+		// "safe" to ignore - previously fetched
+		baseURL, _ := url.Parse(streamURL)
+		for i := 0; i < int(mediaPl.Count()); i++ {
+			// "safe" to ignore - per format spec
+			s := mediaPl.Segments[i]
+			chunkPath, _ := url.Parse(s.URI)
+			chunkURL := baseURL.ResolveReference(chunkPath)
+			chunks = append(chunks, Chunk{Name: s.URI, Length: s.Duration, URL: chunkURL})
+		}
+	default:
+		return nil, 0, fmt.Errorf("m3u8 playlist was not the expected 'media' format")
+	}
 
 	return chunks, chunkDur, nil
 }