Add objectSize argument to ExternalStore.Create

Part of foxcpp/maddy#395 fix

delivery.go

@@ -4,12 +4,11 @@ import (
  "bufio"
  "bytes"
  "database/sql"
+ "errors"
  "io"
  "io/ioutil"
  "time"
 
- "errors"
-
  "github.com/emersion/go-imap"
  "github.com/emersion/go-imap/backend"
  "github.com/emersion/go-message/textproto"
@@ -259,7 +258,7 @@ func (d *Delivery) BodyParsed(header textproto.Header, bodyLen int, body Buffer)
  return wrapErr(err, "Body")
  }
 
- err = d.mboxDelivery(header, mbox, bodyLen, body, date, flagsStmt)
+ err = d.mboxDelivery(header, mbox, int64(bodyLen), body, date, flagsStmt)
  if err != nil {
  return err
  }
@@ -268,7 +267,7 @@ func (d *Delivery) BodyParsed(header textproto.Header, bodyLen int, body Buffer)
  return nil
 }
 
-func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen int, body Buffer, date time.Time, flagsStmt *sql.Stmt) (err error) {
+func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen int64, body Buffer, date time.Time, flagsStmt *sql.Stmt) (err error) {
  header = header.Copy()
  userHeader := d.perRcptHeader[mbox.user.username]
  for fields := userHeader.Fields(); fields.Next(); {
@@ -280,13 +279,13 @@ func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen i
  return wrapErr(err, "Body (WriteHeader)")
  }
 
- length := headerBlob.Len() + bodyLen
+ length := int64(headerBlob.Len()) + bodyLen
  bodyReader, err := body.Open()
  if err != nil {
  return err
  }
 
- bodyStruct, cachedHeader, extBodyKey, err := d.b.processParsedBody(headerBlob.Bytes(), header, bodyReader)
+ bodyStruct, cachedHeader, extBodyKey, err := d.b.processParsedBody(headerBlob.Bytes(), header, bodyReader, bodyLen)
  if err != nil {
  return err
  }
@@ -381,12 +380,18 @@ func (d *Delivery) Commit() error {
  return nil
 }
 
-func (b *Backend) processParsedBody(headerInput []byte, header textproto.Header, bodyLiteral io.Reader) (bodyStruct, cachedHeader []byte, extBodyKey string, err error) {
+func (b *Backend) processParsedBody(headerInput []byte, header textproto.Header, bodyLiteral io.Reader, bodyLen int64) (bodyStruct, cachedHeader []byte, extBodyKey string, err error) {
  extBodyKey, err = randomKey()
  if err != nil {
  return nil, nil, "", err
  }
- extWriter, err := b.extStore.Create(extBodyKey)
+
+ objSize := bodyLen
+ if b.Opts.CompressAlgo != "" {
+ objSize = -1
+ }
+
+ extWriter, err := b.extStore.Create(extBodyKey, objSize)
  if err != nil {
  return nil, nil, "", err
  }

external_store.go

@@ -44,7 +44,7 @@ ExternalStore is an interface used by go-imap-sql to store message bodies
 outside of main database.
 */
 type ExternalStore interface {
- Create(key string) (ExtStoreObj, error)
+ Create(key string, objectSize int64) (ExtStoreObj, error)
 
  // Open returns the ExtStoreObj that reads the message body specified by
  // passed key.

fsstore.go

@@ -25,7 +25,7 @@ func (s *FSStore) Open(key string) (ExtStoreObj, error) {
  return f, nil
 }
 
-func (s *FSStore) Create(key string) (ExtStoreObj, error) {
+func (s *FSStore) Create(key string, blobSize int64) (ExtStoreObj, error) {
  f, err := os.Create(filepath.Join(s.Root, key))
  if err != nil {
  return nil, ExternalError{
@@ -34,6 +34,14 @@ func (s *FSStore) Create(key string) (ExtStoreObj, error) {
  NonExistent: false,
  }
  }
+ if blobSize != -1 {
+ if err := f.Truncate(blobSize); err != nil {
+ return nil, ExternalError{
+ Key: key,
+ Err: err,
+ }
+ }
+ }
  return f, nil
 }
 

go.sum

@@ -1,8 +1,6 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147 h1:cdHOk66P3hpTDhXodyrt+LwFscLHo5DJ/Iy8Rs64pOU=
 github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147/go.mod h1:mOPegfAgLVXbhRm1bh2JTX08z2Y3HYmKYpbrKDeAzsQ=
 github.com/emersion/go-imap v1.0.0/go.mod h1:MEiDDwwQFcZ+L45Pa68jNGv0qU9kbW+SJzwDpvSfX1s=
 github.com/emersion/go-imap v1.0.4 h1:uiCAIHM6Z5Jwkma1zdNDWWXxSCqb+/xHBkHflD7XBro=
@@ -11,22 +9,16 @@ github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a h1:bM
 github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ=
 github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf h1:TmRfuPmhrwAhWKu2XaBaY9N+anRRDBO+E8VRVO9g3fY=
 github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w=
-github.com/emersion/go-imap-sortthread v1.1.0 h1:uRbmnQkeRny5ihKfLWBPJ/1jJdTZnCdh1zYpOagbubw=
-github.com/emersion/go-imap-sortthread v1.1.0/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc=
 github.com/emersion/go-imap-sortthread v1.1.1-0.20200727121200-18e5fb409fed h1:O1GZQnAy76K/DHEp2+S8ZI5hRmkTVNkCLc4Xb0c/RL8=
 github.com/emersion/go-imap-sortthread v1.1.1-0.20200727121200-18e5fb409fed/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc=
 github.com/emersion/go-imap-specialuse v0.0.0-20161227184202-ba031ced6a62 h1:4ZAfwfc8aDlj26kkEap1UDSwwDnJp9Ie8Uj1MSXAkPk=
 github.com/emersion/go-imap-specialuse v0.0.0-20161227184202-ba031ced6a62/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4=
-github.com/emersion/go-message v0.9.1 h1:s6HoJ6t+1wHWEs0G/QVR1r5bb6nvx2/b6DuQfknit14=
 github.com/emersion/go-message v0.9.1/go.mod h1:m3cK90skCWxm5sIMs1sXxly4Tn9Plvcf6eayHZJ1NzM=
-github.com/emersion/go-message v0.10.3 h1:4pajGb3Rq+gHLfRcWysgcwtGRNgLpB8LC6X/vRZ89d0=
 github.com/emersion/go-message v0.10.3/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c=
 github.com/emersion/go-message v0.10.4-0.20190609165112-592ace5bc1ca/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c=
-github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac=
 github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
 github.com/emersion/go-message v0.11.2 h1:oxO9SQ+3wgBAQRdk07eqfkCJ26Tl8ZHF7CcpGVoE00o=
 github.com/emersion/go-message v0.11.2/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
-github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197 h1:rDJPbyliyym8ZL/Wt71kdolp6yaD4fLIQz638E6JEt0=
 github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
 github.com/emersion/go-sasl v0.0.0-20190520160400-47d427600317/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
 github.com/emersion/go-sasl v0.0.0-20190817083125-240c8404624e/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
@@ -38,18 +30,11 @@ github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771 h1:xe
 github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771/go.mod h1:yUISYv/uXLQ6tQZcds/p/hdcZ5JzrEUifyED2VffWpc=
 github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1 h1:B4zNQ2r4qC7FLn8J8+LWt09fFW0tXddypBPS0+HI50s=
 github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1/go.mod h1:WJYkFIdxyljR/byiqcYMKUF4iFDej4CaIKe2JJrQxu8=
-github.com/foxcpp/go-imap-sortthread v1.1.0 h1:+fIk8LQ5jGn1F2oUh0sEFVEbEYNUMaMJu3FVRr4GRcs=
-github.com/foxcpp/go-imap-sortthread v1.1.0/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc=
-github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200720160710-8fb7dccfecfc h1:KBHo7sLBqAYplqZc2dgA6vsa9esTFp3yPcwrOcP/V+w=
-github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200720160710-8fb7dccfecfc/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc=
-github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200721100531-a34cd4e99342 h1:49wfUZk/6IR03WmTZSPEfqsNBL2Nnz166o/F/qfqNLs=
-github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200721100531-a34cd4e99342/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc=
 github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY=
 github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -64,7 +49,6 @@ github.com/lib/pq v1.4.0 h1:TmtCFbH+Aw0AixwyttznSMQDgbR5Yed/Gg6S8Funrhc=
 github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/martinlindhe/base36 v0.0.0-20190418230009-7c6542dfbb41 h1:CVsnY46BCLkX9XOhALJ/S7yb9ayc4eqjXSXO3tyB66A=
 github.com/martinlindhe/base36 v0.0.0-20190418230009-7c6542dfbb41/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
 github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A=
 github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
@@ -82,17 +66,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

mailbox.go

@@ -4,13 +4,12 @@ import (
  "bufio"
  "bytes"
  "database/sql"
+ "errors"
  "io"
  "io/ioutil"
  nettextproto "net/textproto"
  "time"
 
- "errors"
-
  "github.com/emersion/go-imap"
  appendlimit "github.com/emersion/go-imap-appendlimit"
  "github.com/emersion/go-imap/backend"
@@ -267,7 +266,13 @@ func (b *Backend) processBody(literal imap.Literal) (bodyStruct, cachedHeader []
  if err != nil {
  return nil, nil, "", err
  }
- extWriter, err := b.extStore.Create(extBodyKey)
+
+ objSize := literal.Len()
+ if b.Opts.CompressAlgo != "" {
+ objSize = 0
+ }
+
+ extWriter, err := b.extStore.Create(extBodyKey, int64(objSize))
  if err != nil {
  return nil, nil, "", err
  }