Merge pull request #32 from Davo36/audio-file-size-checks

Implemented file size checks

audiofilelibrary.go

@@ -69,6 +69,10 @@ func OpenLibrary(soundsDirectory string) (*AudioFileLibrary, error) {
 	// Get IDs from the filenames.
 	for _, file := range files {
 		fileID, err := extractIDFromFileName(file.Name())
+		if file.Name() == scheduleFilename {
+			// This is the schedule file, just ignore it here.
+			continue
+		}
 		if err == nil {
 			library.FilesByID[fileID] = file.Name()
 		} else {

downloader.go

@@ -33,10 +33,12 @@ import (
 )
 
 const (
-	scheduleFilename  = "schedule.json"
-	connTimeout       = time.Minute * 2
-	connRetryInterval = time.Minute * 10
-	maxConnRetries    = 3
+	scheduleFilename      = "schedule.json"
+	connTimeout           = time.Minute * 2
+	connRetryInterval     = time.Minute * 10
+	maxConnRetries        = 3
+	maxDownloadRetries    = 4
+	retryDownloadInterval = 30 * time.Second
 )
 
 type Downloader struct {
@@ -141,7 +143,7 @@ func (dl *Downloader) GetTodaysSchedule() playlist.Schedule {
 
 // GetFilesForSchedule will get all files from the IDs in the schedule and save to disk.
 func (dl *Downloader) GetFilesForSchedule(schedule playlist.Schedule) (map[int]string, error) {
-	
+
 	referencedFiles := schedule.GetReferencedSounds()
 
 	audioLibrary, err := OpenLibrary(dl.audioDir)
@@ -174,42 +176,86 @@ func (dl *Downloader) listAvailableFiles(audioLibrary *AudioFileLibrary, referen
 	return availableFiles
 }
 
-// Check that the sound file is valid.  This may mean checking it's size on disk compared
-// to the info the API server sends us, or even it's file hash.
-func (dl *Downloader) validateSoundFile(audioLibrary *AudioFileLibrary, fileID int) bool {
-	// Just return true for now.
+// Check that the sound file is valid.
+func (dl *Downloader) validateSoundFile(fileNameOnDisk string, fileSize int) bool {
+
+	// Check size on disk is the same as the size the api-server tells us this file should be.
+	fileInfo, err := os.Stat(fileNameOnDisk)
+	if err != nil {
+		return false
+	}
+	if fileInfo.Size() != int64(fileSize) {
+		return false
+	}
+
 	return true
+
 }
 
-func (dl *Downloader) downloadAllNewFiles(audioLibrary *AudioFileLibrary, referencedFiles []int) {
-	log.Println("Starting downloading audio files.")
-	for _, fileID := range referencedFiles {
-		if _, exists := audioLibrary.GetFileNameOnDisk(fileID); exists {
-			valid := dl.validateSoundFile(audioLibrary, fileID)
-			if exists && valid {
-				continue
-			}
-		} else {
-			log.Printf("Attempting to download file with id %d", fileID)
+// Removes a file from the disk of the device.  Also takes it out of the audioLibrary so it won't be accessed later.
+func (dl *Downloader) removeAudioFile(audioLibrary *AudioFileLibrary, fileID int, fileNameOnDisk string) {
 
-			fileInfo, err := dl.api.GetFileDetails(fileID)
-			if err != nil {
-				log.Printf("Could not download file with id %d. Downloading next file", fileID)
-				continue
-			}
+	delete(audioLibrary.FilesByID, fileID)
+
+	err := os.Remove(fileNameOnDisk)
+	if err != nil {
+		log.Printf("Could not remove file with ID %d and name %s from disk. Error is: %s", fileID, fileNameOnDisk, err)
+	}
 
-			fileNameOnDisk := MakeFileName(fileInfo.File.Details.OriginalName, fileInfo.File.Details.Name, fileID)
+}
 
-			if err = dl.api.DownloadFile(fileInfo, filepath.Join(dl.audioDir, fileNameOnDisk)); err != nil {
-				log.Printf("Could not download file with id %d.  Error is %s. Downloading next file", fileID, err)
-			} else {
-				// Add this file to our audio library.
+// Try and download a single audio file from the API server.
+func (dl *Downloader) downloadAudioFile(audioLibrary *AudioFileLibrary, fileID int, fileResp *api.FileResponse) {
+
+	fileNameOnDisk := MakeFileName(fileResp.File.Details.OriginalName, fileResp.File.Details.Name, fileID)
+	log.Printf("Processing file with id %d and name %s.", fileID, fileNameOnDisk)
+
+	for i := 1; i <= maxDownloadRetries; i++ {
+		if err := dl.api.DownloadFile(fileResp, filepath.Join(dl.audioDir, fileNameOnDisk)); err != nil {
+			log.Printf("Error dowloading sound file id %d and name %s.  Error is %s.", fileID, fileNameOnDisk, err)
+		} else {
+			if dl.validateSoundFile(filepath.Join(dl.audioDir, fileNameOnDisk), fileResp.FileSize) {
+				// File is valid, add it to our audio library.
 				audioLibrary.FilesByID[fileID] = fileNameOnDisk
+				return
 			}
+			log.Printf("File with ID %d and name %s is not valid. Removing from disk.", fileID, fileNameOnDisk)
+			dl.removeAudioFile(audioLibrary, fileID, filepath.Join(dl.audioDir, fileNameOnDisk))
+		}
+		if i < maxDownloadRetries {
+			log.Println("Trying again in", retryDownloadInterval)
+			time.Sleep(retryDownloadInterval)
+		}
+	}
+
+	log.Printf("Could not download and validate file with ID %d and name %s.", fileID, fileNameOnDisk)
 
+}
+
+func (dl *Downloader) downloadAllNewFiles(audioLibrary *AudioFileLibrary, referencedFiles []int) {
+
+	log.Println("Starting downloading audio files.")
+
+	for _, fileID := range referencedFiles {
+
+		// Get file details then download the file.  Try more than once if necessary.
+		for i := 1; i <= maxDownloadRetries; i++ {
+			if fileResp, err := dl.api.GetFileDetails(fileID); err != nil {
+				log.Printf("Error getting file details for file with ID %d. Error is %s", fileID, err)
+			} else {
+				dl.downloadAudioFile(audioLibrary, fileID, fileResp)
+				break
+			}
+			if i < maxDownloadRetries {
+				log.Println("Trying again in", retryDownloadInterval)
+				time.Sleep(retryDownloadInterval)
+			}
 		}
+
 	}
+
 	log.Println("Downloading audio files complete.")
+
 }
 
 // GetSchedule will get the audio schedule

go.mod

@@ -3,11 +3,18 @@ module github.com/TheCacophonyProject/audiobait
 go 1.12
 
 require (
-	github.com/TheCacophonyProject/go-api v0.0.0-20190722123419-412d616d6d5e
+	github.com/TheCacophonyProject/go-api v0.0.0-20190913044515-638d1f88861e
 	github.com/TheCacophonyProject/modemd v0.0.0-20190708011609-1940f5b6677b
 	github.com/TheCacophonyProject/window v0.0.0-20180531024057-3a2d74f7fcf5
 	github.com/alexflint/go-arg v1.0.0
 	github.com/godbus/dbus v4.1.0+incompatible
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect
+	golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
+	golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 // indirect
+	golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 )

go.sum

@@ -1,5 +1,7 @@
 github.com/TheCacophonyProject/go-api v0.0.0-20190722123419-412d616d6d5e h1:jREi18eZzKUPs8jBXmnMarz+IpNzgd42dSOklQtfrjY=
 github.com/TheCacophonyProject/go-api v0.0.0-20190722123419-412d616d6d5e/go.mod h1:7OQjzdX6PbCJDxVRgTqNH9y12Ew+Q1sx9nwcRG7fj9s=
+github.com/TheCacophonyProject/go-api v0.0.0-20190913044515-638d1f88861e h1:bHee94pos9Cqtx7QFyagc1iiYeF7pQo+jmA1t98AF9Y=
+github.com/TheCacophonyProject/go-api v0.0.0-20190913044515-638d1f88861e/go.mod h1:7OQjzdX6PbCJDxVRgTqNH9y12Ew+Q1sx9nwcRG7fj9s=
 github.com/TheCacophonyProject/modemd v0.0.0-20190708011609-1940f5b6677b h1:iJVZlC3nzjJ2EoWLBr7zKhL7t9hUleghmUIC+3ObB88=
 github.com/TheCacophonyProject/modemd v0.0.0-20190708011609-1940f5b6677b/go.mod h1:txGgnRinv6H0/jB4n71MpuS+2qtu8tAlntIjY5udXDU=
 github.com/TheCacophonyProject/window v0.0.0-20180531024057-3a2d74f7fcf5 h1:ihqo17eciOVXn6tZDw0G8L/xJOJ2iaNR9selfF+0uo4=
@@ -24,16 +26,32 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=