Skip to content

Commit e4306b0

Browse files
jacobsajacobsa
committed
Guess GCS object content types based on file extension.
Fixes #155.
2 parents 63a85e3 + 5ce445a commit e4306b0

12 files changed

+556
-77
lines changed

‎docs/semantics.md

+11
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,17 @@ unlinks it. Process A continues to have a consistent view of the file's
311311
contents until it closes the file handle, at which point the contents are lost.
312312

313313

314+
### GCS object metadata
315+
316+
gcsfuse sets the following pieces of GCS object metadata for file objects:
317+
318+
* `contentType` is set to GCS's best guess as to the MIME type of the file,
319+
based on its file extension.
320+
321+
* The custom metadata key `gcsfuse_mtime` is set to track mtime, as discussed
322+
above.
323+
324+
314325
<a name="dir-inodes"></a>
315326
# Directory inodes
316327

‎internal/fs/fs.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
121121
return
122122
}
123123

124+
// Set up a bucket that infers content types when creating files.
125+
bucket := gcsx.NewContentTypeBucket(cfg.Bucket)
126+
124127
// Create the object syncer.
125128
if cfg.TmpObjectPrefix == "" {
126129
err = errors.New("You must set TmpObjectPrefix.")
@@ -130,13 +133,13 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
130133
syncer := gcsx.NewSyncer(
131134
cfg.AppendThreshold,
132135
cfg.TmpObjectPrefix,
133-
cfg.Bucket)
136+
bucket)
134137

135138
// Set up the basic struct.
136139
fs := &fileSystem{
137140
mtimeClock: timeutil.RealClock(),
138141
cacheClock: cfg.CacheClock,
139-
bucket: cfg.Bucket,
142+
bucket: bucket,
140143
syncer: syncer,
141144
tempDir: cfg.TempDir,
142145
implicitDirs: cfg.ImplicitDirectories,
@@ -169,7 +172,7 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
169172
},
170173
fs.implicitDirs,
171174
fs.dirTypeCacheTTL,
172-
cfg.Bucket,
175+
fs.bucket,
173176
fs.mtimeClock,
174177
fs.cacheClock)
175178

‎internal/fs/local_modifications_test.go

+59
Original file line numberDiff line numberDiff line change
@@ -1322,6 +1322,27 @@ func (t *DirectoryTest) RootAtimeCtimeAndMtime() {
13221322
ExpectThat(mtime, timeutil.TimeNear(mountTime, delta))
13231323
}
13241324

1325+
func (t *DirectoryTest) ContentTypes() {
1326+
testCases := []string{
1327+
"foo/",
1328+
"foo.jpg/",
1329+
"foo.txt/",
1330+
}
1331+
1332+
for _, name := range testCases {
1333+
p := path.Join(t.mfs.Dir(), name)
1334+
1335+
// Create the directory.
1336+
err := os.Mkdir(p, 0700)
1337+
AssertEq(nil, err)
1338+
1339+
// There should be no content type set in GCS.
1340+
o, err := t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name})
1341+
AssertEq(nil, err)
1342+
ExpectEq("", o.ContentType, "name: %q", name)
1343+
}
1344+
}
1345+
13251346
////////////////////////////////////////////////////////////////////////
13261347
// File interaction
13271348
////////////////////////////////////////////////////////////////////////
@@ -2135,6 +2156,44 @@ func (t *FileTest) AtimeAndCtime() {
21352156
ExpectThat(ctime, timeutil.TimeNear(createTime, delta))
21362157
}
21372158

2159+
func (t *FileTest) ContentTypes() {
2160+
testCases := map[string]string{
2161+
"foo.jpg": "image/jpeg",
2162+
"bar.txt": "text/plain; charset=utf-8",
2163+
"baz": "",
2164+
}
2165+
2166+
runOne := func(name string, expected string) {
2167+
p := path.Join(t.mfs.Dir(), name)
2168+
2169+
// Create a file.
2170+
f, err := os.Create(p)
2171+
AssertEq(nil, err)
2172+
defer f.Close()
2173+
2174+
// Check the GCS content type.
2175+
o, err := t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name})
2176+
AssertEq(nil, err)
2177+
ExpectEq(expected, o.ContentType, "name: %q", name)
2178+
2179+
// Modify the file and cause a new generation to be written out.
2180+
_, err = f.Write([]byte("taco"))
2181+
AssertEq(nil, err)
2182+
2183+
err = f.Sync()
2184+
AssertEq(nil, err)
2185+
2186+
// The GCS content type should still be correct.
2187+
o, err = t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name})
2188+
AssertEq(nil, err)
2189+
ExpectEq(expected, o.ContentType, "name: %q", name)
2190+
}
2191+
2192+
for name, expected := range testCases {
2193+
runOne(name, expected)
2194+
}
2195+
}
2196+
21382197
////////////////////////////////////////////////////////////////////////
21392198
// Symlinks
21402199
////////////////////////////////////////////////////////////////////////

‎internal/gcsx/content_type_bucket.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright 2016 Google Inc. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package gcsx
16+
17+
import (
18+
"mime"
19+
"path"
20+
21+
"github.com/jacobsa/gcloud/gcs"
22+
"golang.org/x/net/context"
23+
)
24+
25+
// NewContentTypeBucket creates a wrapper bucket that guesses MIME types for
26+
// newly created or composed objects when an explicit type is not already set.
27+
func NewContentTypeBucket(b gcs.Bucket) gcs.Bucket {
28+
return contentTypeBucket{b}
29+
}
30+
31+
type contentTypeBucket struct {
32+
gcs.Bucket
33+
}
34+
35+
func (b contentTypeBucket) CreateObject(
36+
ctx context.Context,
37+
req *gcs.CreateObjectRequest) (o *gcs.Object, err error) {
38+
// Guess a content type if necessary.
39+
if req.ContentType == "" {
40+
req.ContentType = mime.TypeByExtension(path.Ext(req.Name))
41+
}
42+
43+
// Pass on the request.
44+
o, err = b.Bucket.CreateObject(ctx, req)
45+
return
46+
}
47+
48+
func (b contentTypeBucket) ComposeObjects(
49+
ctx context.Context,
50+
req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) {
51+
// Guess a content type if necessary.
52+
if req.ContentType == "" {
53+
req.ContentType = mime.TypeByExtension(path.Ext(req.DstName))
54+
}
55+
56+
// Pass on the request.
57+
o, err = b.Bucket.ComposeObjects(ctx, req)
58+
return
59+
}

‎internal/gcsx/content_type_bucket_test.go

+144
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright 2016 Google Inc. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package gcsx_test
16+
17+
import (
18+
"strings"
19+
"testing"
20+
21+
"github.com/googlecloudplatform/gcsfuse/internal/gcsx"
22+
"github.com/jacobsa/gcloud/gcs"
23+
"github.com/jacobsa/gcloud/gcs/gcsfake"
24+
"github.com/jacobsa/timeutil"
25+
"golang.org/x/net/context"
26+
)
27+
28+
var contentTypeBucketTestCases = []struct {
29+
name string
30+
request string // ContentType in request
31+
expected string // Expected final type
32+
}{
33+
/////////////////
34+
// No extension
35+
/////////////////
36+
37+
0: {
38+
name: "foo/bar",
39+
request: "",
40+
expected: "",
41+
},
42+
43+
1: {
44+
name: "foo/bar",
45+
request: "image/jpeg",
46+
expected: "image/jpeg",
47+
},
48+
49+
//////////////////////
50+
// Unknown extension
51+
//////////////////////
52+
53+
2: {
54+
name: "foo/bar.asdf",
55+
request: "",
56+
expected: "",
57+
},
58+
59+
3: {
60+
name: "foo/bar.asdf",
61+
request: "image/jpeg",
62+
expected: "image/jpeg",
63+
},
64+
65+
//////////////////////
66+
// Known extension
67+
//////////////////////
68+
69+
4: {
70+
name: "foo/bar.jpg",
71+
request: "",
72+
expected: "image/jpeg",
73+
},
74+
75+
5: {
76+
name: "foo/bar.jpg",
77+
request: "text/plain",
78+
expected: "text/plain",
79+
},
80+
}
81+
82+
func TestContentTypeBucket_CreateObject(t *testing.T) {
83+
for i, tc := range contentTypeBucketTestCases {
84+
// Set up a bucket.
85+
bucket := gcsx.NewContentTypeBucket(
86+
gcsfake.NewFakeBucket(timeutil.RealClock(), ""))
87+
88+
// Create the object.
89+
req := &gcs.CreateObjectRequest{
90+
Name: tc.name,
91+
ContentType: tc.request,
92+
Contents: strings.NewReader(""),
93+
}
94+
95+
o, err := bucket.CreateObject(context.Background(), req)
96+
if err != nil {
97+
t.Fatalf("Test case %d: CreateObject: %v", i, err)
98+
}
99+
100+
// Check the content type.
101+
if got, want := o.ContentType, tc.expected; got != want {
102+
t.Errorf("Test case %d: o.ContentType is %q, want %q", i, got, want)
103+
}
104+
}
105+
}
106+
107+
func TestContentTypeBucket_ComposeObjects(t *testing.T) {
108+
var err error
109+
ctx := context.Background()
110+
111+
for i, tc := range contentTypeBucketTestCases {
112+
// Set up a bucket.
113+
bucket := gcsx.NewContentTypeBucket(
114+
gcsfake.NewFakeBucket(timeutil.RealClock(), ""))
115+
116+
// Create a source object.
117+
const srcName = "some_src"
118+
_, err = bucket.CreateObject(ctx, &gcs.CreateObjectRequest{
119+
Name: srcName,
120+
Contents: strings.NewReader(""),
121+
})
122+
123+
if err != nil {
124+
t.Fatalf("Test case %d: CreateObject: %v", err)
125+
}
126+
127+
// Compose.
128+
req := &gcs.ComposeObjectsRequest{
129+
DstName: tc.name,
130+
ContentType: tc.request,
131+
Sources: []gcs.ComposeSource{{Name: srcName}},
132+
}
133+
134+
o, err := bucket.ComposeObjects(ctx, req)
135+
if err != nil {
136+
t.Fatalf("Test case %d: ComposeObject: %v", i, err)
137+
}
138+
139+
// Check the content type.
140+
if got, want := o.ContentType, tc.expected; got != want {
141+
t.Errorf("Test case %d: o.ContentType is %q, want %q", i, got, want)
142+
}
143+
}
144+
}

0 commit comments

Comments
 (0)