From f74ead8f06cdd7925cad691fd7b380a6d692a6bc Mon Sep 17 00:00:00 2001 From: "fox.cpp" Date: Sat, 28 Aug 2021 15:18:10 +0300 Subject: [PATCH] Add objectSize argument to ExternalStore.Create Part of https://github.com/foxcpp/maddy/issues/395 fix --- delivery.go | 21 +++++++++++++-------- external_store.go | 2 +- fsstore.go | 10 +++++++++- go.sum | 19 ------------------- mailbox.go | 11 ++++++++--- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/delivery.go b/delivery.go index dcd45a8..56094c5 100644 --- a/delivery.go +++ b/delivery.go @@ -4,12 +4,11 @@ import ( "bufio" "bytes" "database/sql" + "errors" "io" "io/ioutil" "time" - "errors" - "github.com/emersion/go-imap" "github.com/emersion/go-imap/backend" "github.com/emersion/go-message/textproto" @@ -259,7 +258,7 @@ func (d *Delivery) BodyParsed(header textproto.Header, bodyLen int, body Buffer) return wrapErr(err, "Body") } - err = d.mboxDelivery(header, mbox, bodyLen, body, date, flagsStmt) + err = d.mboxDelivery(header, mbox, int64(bodyLen), body, date, flagsStmt) if err != nil { return err } @@ -268,7 +267,7 @@ func (d *Delivery) BodyParsed(header textproto.Header, bodyLen int, body Buffer) return nil } -func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen int, body Buffer, date time.Time, flagsStmt *sql.Stmt) (err error) { +func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen int64, body Buffer, date time.Time, flagsStmt *sql.Stmt) (err error) { header = header.Copy() userHeader := d.perRcptHeader[mbox.user.username] for fields := userHeader.Fields(); fields.Next(); { @@ -280,13 +279,13 @@ func (d *Delivery) mboxDelivery(header textproto.Header, mbox Mailbox, bodyLen i return wrapErr(err, "Body (WriteHeader)") } - length := headerBlob.Len() + bodyLen + length := int64(headerBlob.Len()) + bodyLen bodyReader, err := body.Open() if err != nil { return err } - bodyStruct, cachedHeader, extBodyKey, err := d.b.processParsedBody(headerBlob.Bytes(), header, bodyReader) + bodyStruct, cachedHeader, extBodyKey, err := d.b.processParsedBody(headerBlob.Bytes(), header, bodyReader, bodyLen) if err != nil { return err } @@ -381,12 +380,18 @@ func (d *Delivery) Commit() error { return nil } -func (b *Backend) processParsedBody(headerInput []byte, header textproto.Header, bodyLiteral io.Reader) (bodyStruct, cachedHeader []byte, extBodyKey string, err error) { +func (b *Backend) processParsedBody(headerInput []byte, header textproto.Header, bodyLiteral io.Reader, bodyLen int64) (bodyStruct, cachedHeader []byte, extBodyKey string, err error) { extBodyKey, err = randomKey() if err != nil { return nil, nil, "", err } - extWriter, err := b.extStore.Create(extBodyKey) + + objSize := int64(len(headerInput)) + bodyLen + if b.Opts.CompressAlgo != "" { + objSize = -1 + } + + extWriter, err := b.extStore.Create(extBodyKey, objSize) if err != nil { return nil, nil, "", err } diff --git a/external_store.go b/external_store.go index ab4c2e9..d8451c2 100644 --- a/external_store.go +++ b/external_store.go @@ -44,7 +44,7 @@ ExternalStore is an interface used by go-imap-sql to store message bodies outside of main database. */ type ExternalStore interface { - Create(key string) (ExtStoreObj, error) + Create(key string, objectSize int64) (ExtStoreObj, error) // Open returns the ExtStoreObj that reads the message body specified by // passed key. diff --git a/fsstore.go b/fsstore.go index 4875626..e6328c9 100644 --- a/fsstore.go +++ b/fsstore.go @@ -25,7 +25,7 @@ func (s *FSStore) Open(key string) (ExtStoreObj, error) { return f, nil } -func (s *FSStore) Create(key string) (ExtStoreObj, error) { +func (s *FSStore) Create(key string, blobSize int64) (ExtStoreObj, error) { f, err := os.Create(filepath.Join(s.Root, key)) if err != nil { return nil, ExternalError{ @@ -34,6 +34,14 @@ func (s *FSStore) Create(key string) (ExtStoreObj, error) { NonExistent: false, } } + if blobSize != -1 { + if err := f.Truncate(blobSize); err != nil { + return nil, ExternalError{ + Key: key, + Err: err, + } + } + } return f, nil } diff --git a/go.sum b/go.sum index ba9d90d..5c1afbe 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147 h1:cdHOk66P3hpTDhXodyrt+LwFscLHo5DJ/Iy8Rs64pOU= github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147/go.mod h1:mOPegfAgLVXbhRm1bh2JTX08z2Y3HYmKYpbrKDeAzsQ= github.com/emersion/go-imap v1.0.0/go.mod h1:MEiDDwwQFcZ+L45Pa68jNGv0qU9kbW+SJzwDpvSfX1s= github.com/emersion/go-imap v1.0.4 h1:uiCAIHM6Z5Jwkma1zdNDWWXxSCqb+/xHBkHflD7XBro= @@ -11,22 +9,16 @@ github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a h1:bM github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ= github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf h1:TmRfuPmhrwAhWKu2XaBaY9N+anRRDBO+E8VRVO9g3fY= github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= -github.com/emersion/go-imap-sortthread v1.1.0 h1:uRbmnQkeRny5ihKfLWBPJ/1jJdTZnCdh1zYpOagbubw= -github.com/emersion/go-imap-sortthread v1.1.0/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc= github.com/emersion/go-imap-sortthread v1.1.1-0.20200727121200-18e5fb409fed h1:O1GZQnAy76K/DHEp2+S8ZI5hRmkTVNkCLc4Xb0c/RL8= github.com/emersion/go-imap-sortthread v1.1.1-0.20200727121200-18e5fb409fed/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc= github.com/emersion/go-imap-specialuse v0.0.0-20161227184202-ba031ced6a62 h1:4ZAfwfc8aDlj26kkEap1UDSwwDnJp9Ie8Uj1MSXAkPk= github.com/emersion/go-imap-specialuse v0.0.0-20161227184202-ba031ced6a62/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4= -github.com/emersion/go-message v0.9.1 h1:s6HoJ6t+1wHWEs0G/QVR1r5bb6nvx2/b6DuQfknit14= github.com/emersion/go-message v0.9.1/go.mod h1:m3cK90skCWxm5sIMs1sXxly4Tn9Plvcf6eayHZJ1NzM= -github.com/emersion/go-message v0.10.3 h1:4pajGb3Rq+gHLfRcWysgcwtGRNgLpB8LC6X/vRZ89d0= github.com/emersion/go-message v0.10.3/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c= github.com/emersion/go-message v0.10.4-0.20190609165112-592ace5bc1ca/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c= -github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac= github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= github.com/emersion/go-message v0.11.2 h1:oxO9SQ+3wgBAQRdk07eqfkCJ26Tl8ZHF7CcpGVoE00o= github.com/emersion/go-message v0.11.2/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= -github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197 h1:rDJPbyliyym8ZL/Wt71kdolp6yaD4fLIQz638E6JEt0= github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20190520160400-47d427600317/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20190817083125-240c8404624e/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= @@ -38,18 +30,11 @@ github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771 h1:xe github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771/go.mod h1:yUISYv/uXLQ6tQZcds/p/hdcZ5JzrEUifyED2VffWpc= github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1 h1:B4zNQ2r4qC7FLn8J8+LWt09fFW0tXddypBPS0+HI50s= github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1/go.mod h1:WJYkFIdxyljR/byiqcYMKUF4iFDej4CaIKe2JJrQxu8= -github.com/foxcpp/go-imap-sortthread v1.1.0 h1:+fIk8LQ5jGn1F2oUh0sEFVEbEYNUMaMJu3FVRr4GRcs= -github.com/foxcpp/go-imap-sortthread v1.1.0/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc= -github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200720160710-8fb7dccfecfc h1:KBHo7sLBqAYplqZc2dgA6vsa9esTFp3yPcwrOcP/V+w= -github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200720160710-8fb7dccfecfc/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc= -github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200721100531-a34cd4e99342 h1:49wfUZk/6IR03WmTZSPEfqsNBL2Nnz166o/F/qfqNLs= -github.com/foxcpp/go-imap-sortthread v1.1.1-0.20200721100531-a34cd4e99342/go.mod h1:opHOzblOHZKQM1JEy+GPk1217giNLa7kleyWTN06qnc= github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY= github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -64,7 +49,6 @@ github.com/lib/pq v1.4.0 h1:TmtCFbH+Aw0AixwyttznSMQDgbR5Yed/Gg6S8Funrhc= github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/martinlindhe/base36 v0.0.0-20190418230009-7c6542dfbb41 h1:CVsnY46BCLkX9XOhALJ/S7yb9ayc4eqjXSXO3tyB66A= github.com/martinlindhe/base36 v0.0.0-20190418230009-7c6542dfbb41/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A= github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= @@ -82,17 +66,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/mailbox.go b/mailbox.go index 543892d..2f9d5fc 100644 --- a/mailbox.go +++ b/mailbox.go @@ -4,13 +4,12 @@ import ( "bufio" "bytes" "database/sql" + "errors" "io" "io/ioutil" nettextproto "net/textproto" "time" - "errors" - "github.com/emersion/go-imap" appendlimit "github.com/emersion/go-imap-appendlimit" "github.com/emersion/go-imap/backend" @@ -267,7 +266,13 @@ func (b *Backend) processBody(literal imap.Literal) (bodyStruct, cachedHeader [] if err != nil { return nil, nil, "", err } - extWriter, err := b.extStore.Create(extBodyKey) + + objSize := literal.Len() + if b.Opts.CompressAlgo != "" { + objSize = 0 + } + + extWriter, err := b.extStore.Create(extBodyKey, int64(objSize)) if err != nil { return nil, nil, "", err }