From 0e2b0585ff698fe4e8cb362af432c8e4a117f3e6 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 15:59:40 -0800 Subject: [PATCH 1/7] mark Sidecar and Srcs as v0 meta --- meta/encoding.go | 10 +++++----- meta/encoding_test.go | 15 +++++++++++---- meta/meta.go | 6 +++--- meta/meta_test.go | 6 +++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/meta/encoding.go b/meta/encoding.go index 86bb4cb..510de8f 100644 --- a/meta/encoding.go +++ b/meta/encoding.go @@ -23,13 +23,13 @@ func (m Meta) MarshalJSON() ([]byte, error) { Version: m.Version, Type: m.Type, Size: m.Size, - SrcSpecific: m.Srcs, + SrcSpecific: m.V0Srcs, } if !m.Inherent.isZero() { j.Inherent = &m.Inherent } - if !m.Sidecar.isZero() { - j.Sidecar = &m.Sidecar + if !m.V0Sidecar.isZero() { + j.Sidecar = &m.V0Sidecar } return json.Marshal(j) } @@ -50,9 +50,9 @@ func (m *Meta) UnmarshalJSON(data []byte) error { m.Inherent = *j.Inherent } if j.Sidecar != nil { - m.Sidecar = *j.Sidecar + m.V0Sidecar = *j.Sidecar } - m.Srcs = j.SrcSpecific + m.V0Srcs = j.SrcSpecific return nil } diff --git a/meta/encoding_test.go b/meta/encoding_test.go index 962be2a..7bc8f27 100644 --- a/meta/encoding_test.go +++ b/meta/encoding_test.go @@ -41,17 +41,24 @@ func TestMetaJSON(t *testing.T) { }, }, }, - Sidecar: Content{ + }, + json: `{"version":"v1","type":"jpg","size":100,"inherent":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}`, + }, + { + desc: "v0 sidecar", + meta: Meta{ + Version: "v1", + V0Sidecar: Content{ Created: time.Date(2, 2, 3, 4, 5, 6, 7, time.UTC), }, }, - json: `{"version":"v1","type":"jpg","size":100,"inherent":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}},"sidecar":{"created":"0002-02-03T04:05:06.000000007Z"}}`, + json: `{"version":"v1","type":"","size":0,"sidecar":{"created":"0002-02-03T04:05:06.000000007Z"}}`, }, { - desc: "src-specific fields", + desc: "v0 src-specific", meta: Meta{ Version: "v1", - Srcs: SrcSpecific{ + V0Srcs: SrcSpecific{ //Flickr: }, }, diff --git a/meta/meta.go b/meta/meta.go index d798cba..76f87ed 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -36,10 +36,10 @@ type Meta struct { // Metadata that came from nearby, such as an XMP sidecar file or other // source of metadata. - Sidecar Content + V0Sidecar Content // Metadata that came from the source of the data. - Srcs SrcSpecific + V0Srcs SrcSpecific } // New initializes a new Meta at the current version. @@ -70,7 +70,7 @@ func ParseJSON(r io.Reader) (*Meta, error) { // time available it returns time.Time's zero value. func (m *Meta) DateCreated() time.Time { times := []time.Time{ - m.Sidecar.Created, + m.V0Sidecar.Created, m.Inherent.Created, } for _, t := range times { diff --git a/meta/meta_test.go b/meta/meta_test.go index c4ffd84..07366f1 100644 --- a/meta/meta_test.go +++ b/meta/meta_test.go @@ -26,15 +26,15 @@ func TestMetaDateCreated(t *testing.T) { { desc: "sidecar with date", m: &Meta{ - Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, + V0Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, }, want: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC), }, { desc: "prefers sidecar to inherent", m: &Meta{ - Inherent: Content{Created: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC)}, - Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, + Inherent: Content{Created: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC)}, + V0Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, }, want: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC), }, From 19950e010ef3b7ce12e6a3be0521181b73ace458 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 16:20:00 -0800 Subject: [PATCH 2/7] create Meta.Src map to cleanly store meta for each source --- meta/encoding.go | 40 ++++++++++++++++++++++++++++------------ meta/encoding_test.go | 36 ++++++++++++++++++++++++++++++++++-- meta/meta.go | 14 +++++++++++++- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/meta/encoding.go b/meta/encoding.go index 510de8f..7cc39ea 100644 --- a/meta/encoding.go +++ b/meta/encoding.go @@ -5,25 +5,29 @@ import ( "time" "github.com/recentralized/structure/data" + "github.com/recentralized/structure/index" ) type metaJSON struct { - Version string `json:"version"` - Type data.Type `json:"type"` - ContentType data.Type `json:"content_type,omitempty"` - Size int64 `json:"size"` - Inherent *Content `json:"inherent,omitempty"` - Sidecar *Content `json:"sidecar,omitempty"` - SrcSpecific // Embedded fields. + Version string `json:"version"` + Type data.Type `json:"type"` + ContentType data.Type `json:"content_type,omitempty"` + Size int64 `json:"size"` + Src map[index.SrcID]SrcSpecific `json:"src,omitempty"` + + // V0 Fields + Inherent *Content `json:"inherent,omitempty"` + Sidecar *Content `json:"sidecar,omitempty"` + V0SrcSpecific // Embedded fields. } // MarshalJSON converts Meta to JSON. func (m Meta) MarshalJSON() ([]byte, error) { j := metaJSON{ - Version: m.Version, - Type: m.Type, - Size: m.Size, - SrcSpecific: m.V0Srcs, + Version: m.Version, + Type: m.Type, + Size: m.Size, + V0SrcSpecific: m.V0Srcs, } if !m.Inherent.isZero() { j.Inherent = &m.Inherent @@ -31,6 +35,12 @@ func (m Meta) MarshalJSON() ([]byte, error) { if !m.V0Sidecar.isZero() { j.Sidecar = &m.V0Sidecar } + if m.Src != nil { + j.Src = make(map[index.SrcID]SrcSpecific) + for k, v := range m.Src { + j.Src[k] = v + } + } return json.Marshal(j) } @@ -46,13 +56,19 @@ func (m *Meta) UnmarshalJSON(data []byte) error { m.Type = j.ContentType } m.Size = j.Size + if j.Src != nil { + m.Src = make(map[index.SrcID]SrcSpecific) + for k, v := range j.Src { + m.Src[k] = v + } + } if j.Inherent != nil { m.Inherent = *j.Inherent } if j.Sidecar != nil { m.V0Sidecar = *j.Sidecar } - m.V0Srcs = j.SrcSpecific + m.V0Srcs = j.V0SrcSpecific return nil } diff --git a/meta/encoding_test.go b/meta/encoding_test.go index 7bc8f27..b22259d 100644 --- a/meta/encoding_test.go +++ b/meta/encoding_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/recentralized/structure/data" + "github.com/recentralized/structure/index" ) func TestMetaJSON(t *testing.T) { @@ -28,6 +29,13 @@ func TestMetaJSON(t *testing.T) { Version: "v1", Type: data.JPG, Size: 100, + }, + json: `{"version":"v1","type":"jpg","size":100}`, + }, + { + desc: "inherent", + meta: Meta{ + Version: "v1", Inherent: Content{ Created: time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC), Image: Image{ @@ -42,7 +50,31 @@ func TestMetaJSON(t *testing.T) { }, }, }, - json: `{"version":"v1","type":"jpg","size":100,"inherent":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}`, + json: `{"version":"v1","type":"","size":0,"inherent":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}`, + }, + { + desc: "src-specific", + meta: Meta{ + Version: "v1", + Src: map[index.SrcID]SrcSpecific{ + index.SrcID("s1"): SrcSpecific{ + Content: Content{ + Created: time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC), + Image: Image{ + Width: 100, + Height: 60, + }, + Exif: Exif{ + "CreateData": ExifValue{ + ID: "0x9004", + Val: "2013:07:17 19:59:58", + }, + }, + }, + }, + }, + }, + json: `{"version":"v1","type":"","size":0,"src":{"s1":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}}`, }, { desc: "v0 sidecar", @@ -58,7 +90,7 @@ func TestMetaJSON(t *testing.T) { desc: "v0 src-specific", meta: Meta{ Version: "v1", - V0Srcs: SrcSpecific{ + V0Srcs: V0SrcSpecific{ //Flickr: }, }, diff --git a/meta/meta.go b/meta/meta.go index 76f87ed..0b11b64 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -7,6 +7,7 @@ import ( "time" "github.com/recentralized/structure/data" + "github.com/recentralized/structure/index" ) const ( @@ -34,12 +35,17 @@ type Meta struct { // Metadata that came from the content itself. Inherent Content + // Metadata that came from elsewhere on the source. + Src map[index.SrcID]SrcSpecific + // Metadata that came from nearby, such as an XMP sidecar file or other // source of metadata. + // Deprecated: Read from V0 only. Use Src[srcID] instead. V0Sidecar Content // Metadata that came from the source of the data. - V0Srcs SrcSpecific + // Deprecated: Read from V0 only. Use Src[srcID] instead. + V0Srcs V0SrcSpecific } // New initializes a new Meta at the current version. @@ -102,9 +108,15 @@ type Image struct { // SrcSpecific contains source-specific metadata. type SrcSpecific struct { + Content //Flickr *flickr.FlickrActivity `json:"flickr,omitempty"` } +// V0SrcSpecific contains source-specific metadata. +type V0SrcSpecific struct { + // TODO: port v0 sources +} + func (m Content) isZero() bool { return m.Created.IsZero() && m.Image.isZero() && From d7a0133a21d597a399a2b6631d09dacb6923acb7 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 17:03:58 -0800 Subject: [PATCH 3/7] retain the idea of Sidecar --- meta/encoding_test.go | 4 ++-- meta/meta.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/encoding_test.go b/meta/encoding_test.go index b22259d..88ff6e2 100644 --- a/meta/encoding_test.go +++ b/meta/encoding_test.go @@ -58,7 +58,7 @@ func TestMetaJSON(t *testing.T) { Version: "v1", Src: map[index.SrcID]SrcSpecific{ index.SrcID("s1"): SrcSpecific{ - Content: Content{ + Sidecar: &Content{ Created: time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC), Image: Image{ Width: 100, @@ -74,7 +74,7 @@ func TestMetaJSON(t *testing.T) { }, }, }, - json: `{"version":"v1","type":"","size":0,"src":{"s1":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}}`, + json: `{"version":"v1","type":"","size":0,"src":{"s1":{"sidecar":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}}}`, }, { desc: "v0 sidecar", diff --git a/meta/meta.go b/meta/meta.go index 0b11b64..eb8ba6a 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -108,7 +108,7 @@ type Image struct { // SrcSpecific contains source-specific metadata. type SrcSpecific struct { - Content + Sidecar *Content `json:"sidecar,omitempty"` //Flickr *flickr.FlickrActivity `json:"flickr,omitempty"` } From aa460afdc4fa7e36dc1f2a6fd2f296bb19e1ed66 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 17:06:24 -0800 Subject: [PATCH 4/7] sidecar test --- meta/encoding_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/encoding_test.go b/meta/encoding_test.go index 88ff6e2..d37c3b4 100644 --- a/meta/encoding_test.go +++ b/meta/encoding_test.go @@ -53,7 +53,7 @@ func TestMetaJSON(t *testing.T) { json: `{"version":"v1","type":"","size":0,"inherent":{"created":"0001-02-03T04:05:06.000000007Z","image":{"width":100,"height":60},"exif":{"CreateData":{"id":"0x9004","val":"2013:07:17 19:59:58"}}}}`, }, { - desc: "src-specific", + desc: "src-specific: sidecar", meta: Meta{ Version: "v1", Src: map[index.SrcID]SrcSpecific{ From 0e0a929c60b9829fadb07301022dfe24cd9362b1 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 22:04:25 -0800 Subject: [PATCH 5/7] update meta example --- examples/meta/main.go | 13 ++++++++++--- examples/meta/output.json | 14 +++++++++----- meta/encoding.go | 13 +++++++------ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/examples/meta/main.go b/examples/meta/main.go index 4781da4..c6738b4 100644 --- a/examples/meta/main.go +++ b/examples/meta/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/recentralized/structure/data" + "github.com/recentralized/structure/index" "github.com/recentralized/structure/meta" ) @@ -28,6 +29,8 @@ func main() { func buildMeta() (*meta.Meta, error) { + srcID := index.SrcID("e8400c72-f7d0-53f9-98ca-ee23238231fe") + doc := meta.New() doc.Type = data.JPG doc.Size = 1024 @@ -42,9 +45,13 @@ func buildMeta() (*meta.Meta, error) { "ExposureTime": meta.ExifValue{ID: "ShutterSpeed", Val: "1/60"}, }, } - doc.Sidecar = meta.Content{ - Exif: meta.Exif{ - "FNumber": meta.ExifValue{ID: "0x829d", Val: 1.8}, + doc.Src = map[index.SrcID]meta.SrcSpecific{ + srcID: meta.SrcSpecific{ + Sidecar: &meta.Content{ + Exif: meta.Exif{ + "FNumber": meta.ExifValue{ID: "0x829d", Val: 1.8}, + }, + }, }, } diff --git a/examples/meta/output.json b/examples/meta/output.json index 04ff190..425ff87 100644 --- a/examples/meta/output.json +++ b/examples/meta/output.json @@ -15,11 +15,15 @@ } } }, - "sidecar": { - "exif": { - "FNumber": { - "id": "0x829d", - "val": 1.8 + "src": { + "e8400c72-f7d0-53f9-98ca-ee23238231fe": { + "sidecar": { + "exif": { + "FNumber": { + "id": "0x829d", + "val": 1.8 + } + } } } } diff --git a/meta/encoding.go b/meta/encoding.go index 7cc39ea..98c8b7b 100644 --- a/meta/encoding.go +++ b/meta/encoding.go @@ -9,14 +9,15 @@ import ( ) type metaJSON struct { - Version string `json:"version"` - Type data.Type `json:"type"` - ContentType data.Type `json:"content_type,omitempty"` - Size int64 `json:"size"` - Src map[index.SrcID]SrcSpecific `json:"src,omitempty"` + Version string `json:"version"` + Type data.Type `json:"type"` + ContentType data.Type `json:"content_type,omitempty"` + Size int64 `json:"size"` + + Inherent *Content `json:"inherent,omitempty"` + Src map[index.SrcID]SrcSpecific `json:"src,omitempty"` // V0 Fields - Inherent *Content `json:"inherent,omitempty"` Sidecar *Content `json:"sidecar,omitempty"` V0SrcSpecific // Embedded fields. } From b2cb31f44a9ef7d893bcad9868a886f329052593 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Sat, 29 Dec 2018 22:06:49 -0800 Subject: [PATCH 6/7] fix gofmt --- examples/meta/main.go | 2 +- meta/encoding_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/meta/main.go b/examples/meta/main.go index c6738b4..0512742 100644 --- a/examples/meta/main.go +++ b/examples/meta/main.go @@ -46,7 +46,7 @@ func buildMeta() (*meta.Meta, error) { }, } doc.Src = map[index.SrcID]meta.SrcSpecific{ - srcID: meta.SrcSpecific{ + srcID: { Sidecar: &meta.Content{ Exif: meta.Exif{ "FNumber": meta.ExifValue{ID: "0x829d", Val: 1.8}, diff --git a/meta/encoding_test.go b/meta/encoding_test.go index f234329..2eaaf76 100644 --- a/meta/encoding_test.go +++ b/meta/encoding_test.go @@ -57,7 +57,7 @@ func TestMetaJSON(t *testing.T) { meta: Meta{ Version: "v1", Src: map[index.SrcID]SrcSpecific{ - index.SrcID("s1"): SrcSpecific{ + index.SrcID("s1"): { Sidecar: &Content{ Created: time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC), Image: Image{ @@ -81,7 +81,7 @@ func TestMetaJSON(t *testing.T) { meta: Meta{ Version: "v1", Src: map[index.SrcID]SrcSpecific{ - index.SrcID("s1"): SrcSpecific{ + index.SrcID("s1"): { Flickr: &FlickrMedia{ ID: "123", }, From 9d30f5a79b36b8ab2fe6da4f99d56b031a90eeb3 Mon Sep 17 00:00:00 2001 From: Ryan Carver Date: Mon, 31 Dec 2018 22:18:21 -0800 Subject: [PATCH 7/7] meta.DateCreated looks at all Sidecar data --- meta/meta.go | 9 +++++++++ meta/meta_test.go | 30 +++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/meta/meta.go b/meta/meta.go index 063a6e6..b5f74b9 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "io" + "sort" "time" "github.com/recentralized/structure/data" @@ -79,6 +80,14 @@ func (m *Meta) DateCreated() time.Time { m.V0Sidecar.Created, m.Inherent.Created, } + if len(m.Src) > 0 { + for _, v := range m.Src { + if v.Sidecar != nil { + times = append(times, v.Sidecar.Created) + } + } + sort.Slice(times, func(i, j int) bool { return times[i].Before(times[j]) }) + } for _, t := range times { if !t.IsZero() { return t diff --git a/meta/meta_test.go b/meta/meta_test.go index 07366f1..9876949 100644 --- a/meta/meta_test.go +++ b/meta/meta_test.go @@ -4,6 +4,8 @@ import ( "reflect" "testing" "time" + + "github.com/recentralized/structure/index" ) func TestMetaDateCreated(t *testing.T) { @@ -24,25 +26,43 @@ func TestMetaDateCreated(t *testing.T) { want: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC), }, { - desc: "sidecar with date", + desc: "oldest of inherent or sidecars", + m: &Meta{ + Inherent: Content{Created: time.Date(2009, 1, 1, 1, 1, 1, 1, time.UTC)}, + Src: map[index.SrcID]SrcSpecific{ + index.SrcID("a"): { + Sidecar: &Content{Created: time.Date(2004, 1, 1, 1, 1, 1, 1, time.UTC)}, + }, + index.SrcID("b"): { + Sidecar: &Content{Created: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC)}, + }, + index.SrcID("c"): { + Sidecar: &Content{Created: time.Date(2003, 1, 1, 1, 1, 1, 1, time.UTC)}, + }, + }, + }, + want: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC), + }, + { + desc: "v0: sidecar with date", m: &Meta{ V0Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, }, want: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC), }, { - desc: "prefers sidecar to inherent", + desc: "v0: prefers sidecar to inherent", m: &Meta{ Inherent: Content{Created: time.Date(2001, 1, 1, 1, 1, 1, 1, time.UTC)}, - V0Sidecar: Content{Created: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC)}, + V0Sidecar: Content{Created: time.Date(2002, 1, 1, 1, 1, 1, 1, time.UTC)}, }, - want: time.Date(2001, 2, 1, 1, 1, 1, 1, time.UTC), + want: time.Date(2002, 1, 1, 1, 1, 1, 1, time.UTC), }, } for _, tt := range tests { got := tt.m.DateCreated() if got, want := got, tt.want; !reflect.DeepEqual(got, want) { - t.Errorf("%q Meta.DateCreated()\ngot %#v\nwant %#v", tt.desc, got, want) + t.Errorf("%q Meta.DateCreated()\ngot %s\nwant %s", tt.desc, got, want) } } }