Skip to content

Commit

Permalink
Merge pull request #76 from isZumpo/thumbnails
Browse files Browse the repository at this point in the history
Add thumbnail support
  • Loading branch information
tphakala authored Jun 17, 2024
2 parents bd65f01 + f68715d commit 7ce09d5
Show file tree
Hide file tree
Showing 11 changed files with 602 additions and 28 deletions.
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ require (
)

require (
cgt.name/pkg/go-mwclient v1.3.0 // indirect
github.com/antonholmquist/jason v1.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
Expand All @@ -36,13 +38,15 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/k3a/html2text v1.2.1 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.5.0 // indirect
Expand Down
17 changes: 17 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
cgt.name/pkg/go-mwclient v1.3.0 h1:8PtOxq+aL4wtyLZXHPImpNtUioFq/DxBwae2C44j2gE=
cgt.name/pkg/go-mwclient v1.3.0/go.mod h1:X1auRhzIA0Bz5Yx7Yei29vUqr/Ju+r8IWudnJmmAG30=
github.com/antonholmquist/jason v1.0.0 h1:Ytg94Bcf1Bfi965K2q0s22mig/n4eGqEij/atENBhA0=
github.com/antonholmquist/jason v1.0.0/go.mod h1:+GxMEKI0Va2U8h3os6oiUAetHAlGMvxjdpAH/9uvUMA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
Expand Down Expand Up @@ -30,6 +34,7 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
Expand All @@ -40,6 +45,9 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY=
github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand All @@ -63,6 +71,8 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 h1:j2kD3MT1z4PXCiUllUJF9mWUESr9TWKS7iEKsQ/IipM=
github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down Expand Up @@ -93,6 +103,8 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/smallnest/ringbuffer v0.0.0-20230728150354-35801fa39d0e h1:KHiRfgBfn0d3lv2kXs4iayASb6TdInNNIHe75zX0sqg=
github.com/smallnest/ringbuffer v0.0.0-20230728150354-35801fa39d0e/go.mod h1:mXcZNMJHswhQDDJZIjdtJoG97JIwIa/HdcHNM3w15T0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
Expand Down Expand Up @@ -133,14 +145,17 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -149,10 +164,12 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
Expand Down
15 changes: 14 additions & 1 deletion internal/datastore/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type Interface interface {
GetHourlyOccurrences(date, commonName string, minConfidenceNormalized float64) ([24]int, error)
SpeciesDetections(species, date, hour string, sortAscending bool, limit int, offset int) ([]Note, error)
GetLastDetections(numDetections int) ([]Note, error)
GetAllDetectedSpecies() ([]Note, error)
SearchNotes(query string, sortAscending bool, limit int, offset int) ([]Note, error)
GetNoteClipPath(noteID string) (string, error)
DeleteNoteClipPath(noteID string) error
Expand Down Expand Up @@ -185,7 +186,7 @@ func (ds *DataStore) GetTopBirdsData(selectedDate string, minConfidenceNormalize
const reportCount = 30 // Consider making this a configurable parameter

err := ds.DB.Table("notes").
Select("common_name, COUNT(*) as count").
Select("common_name", "scientific_name", "COUNT(*) as count").
Where("date = ? AND confidence >= ?", selectedDate, minConfidenceNormalized).
Group("common_name").
//Having("COUNT(*) > ?", 1).
Expand Down Expand Up @@ -318,6 +319,18 @@ func (ds *DataStore) GetLastDetections(numDetections int) ([]Note, error) {
return notes, nil
}

// GetLastDetections retrieves all detected species.
func (ds *DataStore) GetAllDetectedSpecies() ([]Note, error) {
var results []Note

err := ds.DB.Table("notes").
Select("scientific_name").
Group("scientific_name").
Scan(&results).Error

return results, err
}

// SearchNotes performs a search on notes with optional sorting, pagination, and limits.
func (ds *DataStore) SearchNotes(query string, sortAscending bool, limit int, offset int) ([]Note, error) {
var notes []Note
Expand Down
109 changes: 109 additions & 0 deletions internal/httpcontroller/imageprovider/imageprovider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package imageprovider

import (
"slices"
"sync"
)

type ImageProvider interface {
fetch(scientificName string) (BirdImage, error)
}

type BirdImage struct {
Url string
LicenseName string
LicenseUrl string
AuthorName string
AuthorUrl string
}

type BirdImageCache struct {
dataMap sync.Map
dataMutexMap sync.Map
birdImageProvider ImageProvider
nonBirdImageProvider ImageProvider
}

type emptyImageProvider struct {
}

func (l *emptyImageProvider) fetch(scientificName string) (BirdImage, error) {
return BirdImage{}, nil
}

func initCache(e ImageProvider) *BirdImageCache {
return &BirdImageCache{
birdImageProvider: e,
nonBirdImageProvider: &emptyImageProvider{}, // TODO: Use a real image provider for non-birds
}
}

func CreateDefaultCache() (*BirdImageCache, error) {
provider, err := NewWikiMediaProvider()
if err != nil {
return nil, err
}
return initCache(provider), nil
}

func (c *BirdImageCache) Get(scientificName string) (info BirdImage, err error) {
// Check if the bird image is already in the cache
birdImage, ok := c.dataMap.Load(scientificName)
if ok {
return birdImage.(BirdImage), nil
}

// Use a per-item mutex to ensure only one query is performed per item
mu, _ := c.dataMutexMap.LoadOrStore(scientificName, &sync.Mutex{})
mutex := mu.(*sync.Mutex)

mutex.Lock()
defer mutex.Unlock()

// Check again if bird image is cached after acquiring the lock
birdImage, ok = c.dataMap.Load(scientificName)
if ok {
return birdImage.(BirdImage), nil
}

// Fetch the bird image from the image provider
fetchedBirdImage, err := c.fetch(scientificName)
if err != nil {
// TODO for now store a empty result in the cache to avoid future queries that would fail.
// In the future, look at the error and decide if it was caused by networking and is recoverable.
// And if it was, do not store the empty result in the cache.
c.dataMap.Store(scientificName, BirdImage{})
return BirdImage{}, err
}

// Store the fetched image information in the cache
c.dataMap.Store(scientificName, fetchedBirdImage)

return fetchedBirdImage, nil
}

func (c *BirdImageCache) fetch(scientificName string) (info BirdImage, err error) {
var imageProviderToUse ImageProvider

// Determine the image provider based on the scientific name
if slices.Contains([]string{
"Dog",
"Engine",
"Environmental",
"Fireworks",
"Gun",
"Human non-vocal",
"Human vocal",
"Human whistle",
"Noise",
"Power tools",
"Siren",
}, scientificName) {
imageProviderToUse = c.nonBirdImageProvider
} else {
imageProviderToUse = c.birdImageProvider
}

// Fetch the image from the image provider
return imageProviderToUse.fetch(scientificName)
}
54 changes: 54 additions & 0 deletions internal/httpcontroller/imageprovider/imageprovider_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package imageprovider

import (
"testing"
)

type mockProvider struct {
fetchCounter int
}

func (l *mockProvider) fetch(scientificName string) (BirdImage, error) {
l.fetchCounter++
return BirdImage{}, nil
}

// TestBirdImageCache ensures the bird image cache behaves as expected.
// It tests whether the cache correctly handles duplicate requests.
func TestBirdImageCache(t *testing.T) {
// Create a mock image provider and initialize the cache.
mockBirdProvider := &mockProvider{}
mockNonBirdProvider := &mockProvider{}
cache := &BirdImageCache{
birdImageProvider: mockBirdProvider,
nonBirdImageProvider: mockNonBirdProvider,
}

entriesToTest := []string{
"a",
"b",
"a", // Duplicate request
"Human non-vocal", // Non-bird request
}

for _, entry := range entriesToTest {
_, err := cache.Get(entry)
if err != nil {
t.Errorf("Unexpected error for entry %s: %v", entry, err)
}
}

// Verify that the bird provider's fetch method was called exactly twice.
expectedBirdFetchCalls := 2
if mockBirdProvider.fetchCounter != expectedBirdFetchCalls {
t.Errorf("Expected %d calls to bird provider, got %d",
expectedBirdFetchCalls, mockBirdProvider.fetchCounter)
}

// Verify that the non-bird provider's fetch method was called exactly once.
expectedNonBirdFetchCalls := 1
if mockNonBirdProvider.fetchCounter != expectedNonBirdFetchCalls {
t.Errorf("Expected %d calls to non-bird provider, got %d",
expectedNonBirdFetchCalls, mockNonBirdProvider.fetchCounter)
}
}
Loading

0 comments on commit 7ce09d5

Please sign in to comment.