From 5713613b5f7faa2006c559054148ed429faa28d9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Dec 2022 21:13:00 +0100 Subject: [PATCH 01/12] Add support for incoming emails. --- .drone.yml | 4 + .../doc/advanced/config-cheat-sheet.en-us.md | 14 + docs/content/doc/features/comparison.en-us.md | 2 +- docs/content/doc/features/comparison.zh-cn.md | 2 +- docs/content/doc/features/comparison.zh-tw.md | 2 +- .../content/doc/usage/incoming-email.en-us.md | 47 ++ go.mod | 5 + go.sum | 12 + models/unittest/testdb.go | 2 + modules/setting/incoming_email.go | 72 +++ modules/setting/setting.go | 1 + modules/util/pack.go | 33 ++ modules/util/pack_test.go | 28 ++ routers/init.go | 2 + services/mailer/incoming/incoming.go | 414 ++++++++++++++++++ services/mailer/incoming/incoming_handler.go | 160 +++++++ services/mailer/incoming/incoming_test.go | 129 ++++++ services/mailer/incoming/payload/payload.go | 71 +++ services/mailer/mail.go | 55 ++- services/mailer/mail_test.go | 57 +-- services/mailer/mailer.go | 4 + services/mailer/token/token.go | 116 +++++ tests/integration/incoming_email_test.go | 251 +++++++++++ tests/mysql.ini.tmpl | 10 + 24 files changed, 1457 insertions(+), 36 deletions(-) create mode 100644 docs/content/doc/usage/incoming-email.en-us.md create mode 100644 modules/setting/incoming_email.go create mode 100644 modules/util/pack.go create mode 100644 modules/util/pack_test.go create mode 100644 services/mailer/incoming/incoming.go create mode 100644 services/mailer/incoming/incoming_handler.go create mode 100644 services/mailer/incoming/incoming_test.go create mode 100644 services/mailer/incoming/payload/payload.go create mode 100644 services/mailer/token/token.go create mode 100644 tests/integration/incoming_email_test.go diff --git a/.drone.yml b/.drone.yml index d349a5f2fc1d8..7ba37ee8b86b0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -230,6 +230,10 @@ services: MINIO_ACCESS_KEY: 123456 MINIO_SECRET_KEY: 12345678 + - name: smtpimap + image: tabascoterrier/docker-imap-devel:latest + pull: always + steps: - name: fetch-tags image: docker:git diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 853f0c67f254c..8827c66bdf42e 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -743,6 +743,20 @@ and - `SEND_BUFFER_LEN`: **100**: Buffer length of mailing queue. **DEPRECATED** use `LENGTH` in `[queue.mailer]` - `SEND_AS_PLAIN_TEXT`: **false**: Send mails only in plain text, without HTML alternative. +## Incoming Email (`incoming_email`) + +- `ENABLED`: **false**: Enable handling of incoming emails. +- `REPLY_TO_ADDRESS`: **\**: # The email address including the %{token} placeholder that will be replaced per user/action. Example: `incoming+%{token}@example.com`. The placeholder must appear in the user part of the address (before the `@`). +- `HOST`: **\**: IMAP server host. +- `PORT`: **\**: IMAP server port. +- `USERNAME`: **\**: Username of the receiving account. +- `PASSWORD`: **\**: Password of the receiving account. +- `USE_TLS`: **false**: Whether the IMAP server uses TLS. +- `SKIP_TLS_VERIFY`: **false**: If set to `true`, completely ignores server certificate validation errors. This option is unsafe. +- `MAILBOX`: **INBOX**: The mailbox name where incoming mail will end up. +- `DELETE_HANDLED_MESSAGE`: **true**: Whether handled messages should be deleted from the mailbox. +- `MAXIMUM_MESSAGE_SIZE`: **0**: Maximum size of a message to handle. Bigger messages are ignored. + ## Cache (`cache`) - `ENABLED`: **true**: Enable the cache. diff --git a/docs/content/doc/features/comparison.en-us.md b/docs/content/doc/features/comparison.en-us.md index 9baa6d5123729..87b2f43de7d8f 100644 --- a/docs/content/doc/features/comparison.en-us.md +++ b/docs/content/doc/features/comparison.en-us.md @@ -106,7 +106,7 @@ _Symbols used in table:_ | Issue search | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | Global issue search | [/](https://github.com/go-gitea/gitea/issues/2434) | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | Issue dependency | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| Create issue via email | [✘](https://github.com/go-gitea/gitea/issues/6226) | ✘ | ✘ | ✘ | ✓ | ✓ | ✘ | +| Create issue via email | [✘](https://github.com/go-gitea/gitea/issues/6226) | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | | Service Desk | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | ✓ | ✓ | ✘ | ✘ | ## Pull/Merge requests diff --git a/docs/content/doc/features/comparison.zh-cn.md b/docs/content/doc/features/comparison.zh-cn.md index aaf8eb4abc516..c5bec58cb85b2 100644 --- a/docs/content/doc/features/comparison.zh-cn.md +++ b/docs/content/doc/features/comparison.zh-cn.md @@ -92,7 +92,7 @@ _表格中的符号含义:_ | 工单搜索 | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | 工单全局搜索 | [✘](https://github.com/go-gitea/gitea/issues/2434) | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | 工单依赖关系 | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| 通过 Email 创建工单 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✘ | ✓ | ✓ | ✘ | +| 通过 Email 创建工单 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✓ | ✓ | ✓ | ✘ | | 服务台 | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | [✓](https://gitlab.com/groups/gitlab-org/-/epics/3103) | ✓ | ✘ | ✘ | #### Pull/Merge requests diff --git a/docs/content/doc/features/comparison.zh-tw.md b/docs/content/doc/features/comparison.zh-tw.md index 015955f0a852b..4da9c74ec8cc8 100644 --- a/docs/content/doc/features/comparison.zh-tw.md +++ b/docs/content/doc/features/comparison.zh-tw.md @@ -93,7 +93,7 @@ menu: | 問題搜尋 | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | 全域問題搜尋 | [✘](https://github.com/go-gitea/gitea/issues/2434) | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | 問題相依 | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| 從電子郵件建立問題 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✘ | ✓ | ✓ | ✘ | +| 從電子郵件建立問題 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✓ | ✓ | ✓ | ✘ | | 服務台 | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | [✓](https://gitlab.com/groups/gitlab-org/-/epics/3103) | ✓ | ✘ | ✘ | ## 拉取/合併請求 diff --git a/docs/content/doc/usage/incoming-email.en-us.md b/docs/content/doc/usage/incoming-email.en-us.md new file mode 100644 index 0000000000000..f7b8ff5a7df13 --- /dev/null +++ b/docs/content/doc/usage/incoming-email.en-us.md @@ -0,0 +1,47 @@ +--- +date: "2022-12-01T00:00:00+00:00" +title: "Incoming Email" +slug: "incoming-email" +draft: false +toc: false +menu: + sidebar: + parent: "usage" + name: "Incoming Email" + weight: 13 + identifier: "incoming-email" +--- + +# Incoming Email + +Gitea supports the execution of several actions through incoming mails. This page describes how to set this up. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +Handling incoming email messages requires an IMAP-enabled email account. +The recommended strategy is to use [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing) but a catch-all mailbox does work too. +The receiving email address contains a user/action specific token which tells Gitea which action should be performed. +This token is expected in the `To` and `Delivered-To` header fields. + +Gitea tries to detect automatic responses to skip and the email server should be configured to reduce the incoming noise too (spam, newsletter). + +## Configuration + +To activate the handling of incoming email messages you have to configure the `incoming_email` section in the configuration file. + +The `REPLY_TO_ADDRESS` contains the address an email client will respond to. +This address needs to contain the `%{token}` placeholder which will be replaced with a token describing the user/action. +This placeholder must only appear once in the address and must be in the user part of the address (before the `@`). + +An example using email sub-addressing may look like this: `incoming+%{token}@example.com` + +If a catch-all mailbox is used, the placeholder may be used anywhere in the user part of the address: `incoming+%{token}@example.com`, `incoming_%{token}@example.com`, `%{token}@example.com` + +## Security + +Be careful when choosing the domain used for receiving incoming email. +It's recommended receiving incoming email on a subdomain, such as `incoming.example.com` to prevent potential security problems with other services running on `example.com`. diff --git a/go.mod b/go.mod index ca8c79c689003..2006e7693d4ee 100644 --- a/go.mod +++ b/go.mod @@ -159,8 +159,13 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/emersion/go-imap v1.2.1 // indirect + github.com/emersion/go-message v0.16.0 // indirect + github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/go.sum b/go.sum index d6748eae34de6..0a2b93652d6b1 100644 --- a/go.sum +++ b/go.sum @@ -388,11 +388,14 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 h1:PdsjTl0Cg+ZJgOx/CFV5NNgO1ThTreqdgKYiDCMHJwA= +github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21/go.mod h1:xJvkyD6Y2rZapGvPJLYo9dyx1s5dxBEDPa8T3YTuOk0= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -414,6 +417,15 @@ github.com/editorconfig/editorconfig-core-go/v2 v2.4.5 h1:kTcVMyCvFGQmTk0S5+R7GF github.com/editorconfig/editorconfig-core-go/v2 v2.4.5/go.mod h1:rDB5UUleQsOI1HLbojaBmDNR8oUUe31InmNDTVzcDHY= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA= +github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= +github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= +github.com/emersion/go-message v0.16.0 h1:uZLz8ClLv3V5fSFF/fFdW9jXjrZkXIpE1Fn8fKx7pO4= +github.com/emersion/go-message v0.16.0/go.mod h1:pDJDgf/xeUIF+eicT6B/hPX/ZbEorKkUMPOxrPVG2eQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 2e6c25ae48f5e..2fb43fafd3a71 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -107,6 +107,8 @@ func MainTest(m *testing.M, testOpts *TestOptions) { setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home") + setting.IncomingEmail.ReplyToAddress = "incoming+%{token}@localhost" + if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go new file mode 100644 index 0000000000000..cb5a29942d84e --- /dev/null +++ b/modules/setting/incoming_email.go @@ -0,0 +1,72 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "fmt" + "net/mail" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +var IncomingEmail = struct { + Enabled bool + ReplyToAddress string + TokenPlaceholder string `ini:"-"` + Host string + Port int + UseTLS bool `ini:"USE_TLS"` + SkipTLSVerify bool `ini:"SKIP_TLS_VERIFY"` + Username string + Password string + Mailbox string + DeleteHandledMessage bool + MaximumMessageSize uint32 +}{ + Mailbox: "INBOX", + DeleteHandledMessage: true, + TokenPlaceholder: "%{token}", +} + +func newIncomingEmail() { + if err := Cfg.Section("incoming_email").MapTo(&IncomingEmail); err != nil { + log.Fatal("Unable to map [incoming_email] section on to IncomingEmail. Error: %v", err) + } + + if !IncomingEmail.Enabled { + return + } + + if err := checkReplyToAddress(IncomingEmail.ReplyToAddress); err != nil { + log.Fatal("Invalid incoming_mail.REPLY_TO_ADDRESS (%s): %v", IncomingEmail.ReplyToAddress, err) + } +} + +func checkReplyToAddress(address string) error { + parsed, err := mail.ParseAddress(IncomingEmail.ReplyToAddress) + if err != nil { + return err + } + + if parsed.Name != "" { + return fmt.Errorf("name must not be set") + } + + c := strings.Count(IncomingEmail.ReplyToAddress, IncomingEmail.TokenPlaceholder) + switch c { + case 0: + return fmt.Errorf("%s must appear in the user part of the address (before the @)", IncomingEmail.TokenPlaceholder) + case 1: + default: + return fmt.Errorf("%s must appear only once", IncomingEmail.TokenPlaceholder) + } + + parts := strings.Split(IncomingEmail.ReplyToAddress, "@") + if !strings.Contains(parts[0], IncomingEmail.TokenPlaceholder) { + return fmt.Errorf("%s must appear in the user part of the address (before the @)", IncomingEmail.TokenPlaceholder) + } + + return nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index df7310b09b467..746186a70f62e 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1329,6 +1329,7 @@ func NewServices() { newSessionService() newCORSService() newMailService() + newIncomingEmail() newRegisterMailService() newNotifyMailService() newProxyService() diff --git a/modules/util/pack.go b/modules/util/pack.go new file mode 100644 index 0000000000000..ea27e6cce0296 --- /dev/null +++ b/modules/util/pack.go @@ -0,0 +1,33 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "encoding/gob" +) + +// PackData uses gob to encode the given data in sequence +func PackData(data ...interface{}) ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + for _, datum := range data { + if err := enc.Encode(datum); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// UnpackData uses gob to decode the given data in sequence +func UnpackData(buf []byte, data ...interface{}) error { + r := bytes.NewReader(buf) + enc := gob.NewDecoder(r) + for _, datum := range data { + if err := enc.Decode(datum); err != nil { + return err + } + } + return nil +} diff --git a/modules/util/pack_test.go b/modules/util/pack_test.go new file mode 100644 index 0000000000000..234d7abaca64c --- /dev/null +++ b/modules/util/pack_test.go @@ -0,0 +1,28 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPackAndUnpackData(t *testing.T) { + s := "string" + i := int64(4) + f := float32(4.1) + + var s2 string + var i2 int64 + var f2 float32 + + data, err := PackData(s, i, f) + assert.NoError(t, err) + + assert.NoError(t, UnpackData(data, &s2, &i2, &f2)) + assert.NoError(t, UnpackData(data, &s2)) + assert.Error(t, UnpackData(data, &i2)) + assert.Error(t, UnpackData(data, &s2, &f2)) +} diff --git a/routers/init.go b/routers/init.go index fecc5c439c6de..5cea7a78ca9df 100644 --- a/routers/init.go +++ b/routers/init.go @@ -41,6 +41,7 @@ import ( "code.gitea.io/gitea/services/automerge" "code.gitea.io/gitea/services/cron" "code.gitea.io/gitea/services/mailer" + mailer_incoming "code.gitea.io/gitea/services/mailer/incoming" markup_service "code.gitea.io/gitea/services/markup" repo_migrations "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" @@ -162,6 +163,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(task.Init) mustInit(repo_migrations.Init) eventsource.GetManager().Init() + mustInitCtx(ctx, mailer_incoming.Init) mustInitCtx(ctx, syncAppConfForGit) diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go new file mode 100644 index 0000000000000..ebe6259876965 --- /dev/null +++ b/services/mailer/incoming/incoming.go @@ -0,0 +1,414 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + net_mail "net/mail" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer/token" + + "github.com/dimiro1/reply" + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/client" + "github.com/emersion/go-message/mail" + "github.com/jaytaylor/html2text" +) + +var ( + addressTokenRegex *regexp.Regexp + referenceTokenRegex *regexp.Regexp +) + +func Init(ctx context.Context) error { + if !setting.IncomingEmail.Enabled { + return nil + } + + addressTokenRegex = regexp.MustCompile( + fmt.Sprintf( + `\A%s\z`, + strings.Replace(regexp.QuoteMeta(setting.IncomingEmail.ReplyToAddress), regexp.QuoteMeta(setting.IncomingEmail.TokenPlaceholder), "(.+)", 1), + ), + ) + referenceTokenRegex = regexp.MustCompile(fmt.Sprintf(`\Areply-(.+)@%s\z`, regexp.QuoteMeta(setting.Domain))) + + go func() { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Incoming Email", process.SystemProcessType, true) + defer finished() + + for { + select { + case <-ctx.Done(): + return + default: + if err := processIncomingEmail(ctx); err != nil { + log.Error("Error while processing incoming emails: %v", err) + } + select { + case <-ctx.Done(): + return + case <-time.NewTimer(10 * time.Second).C: + } + } + } + }() + + return nil +} + +func processIncomingEmail(ctx context.Context) error { + server := fmt.Sprintf("%s:%d", setting.IncomingEmail.Host, setting.IncomingEmail.Port) + + var c *client.Client + var err error + if setting.IncomingEmail.UseTLS { + c, err = client.DialTLS(server, &tls.Config{InsecureSkipVerify: setting.IncomingEmail.SkipTLSVerify}) + } else { + c, err = client.Dial(server) + } + if err != nil { + return fmt.Errorf("connected failed: %w", err) + } + + if err := c.Login(setting.IncomingEmail.Username, setting.IncomingEmail.Password); err != nil { + return fmt.Errorf("login failed: %w", err) + } + defer func() { + if err := c.Logout(); err != nil { + log.Error("Logout failed: %v", err) + } + }() + + if _, err := c.Select(setting.IncomingEmail.Mailbox, false); err != nil { + return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err) + } + + for { + select { + case <-ctx.Done(): + return nil + default: + if err := processMessages(ctx, c); err != nil { + return fmt.Errorf("do it failed: %w", err) + } + if err := waitForUpdates(ctx, c); err != nil { + return fmt.Errorf("wait for updates failed: %w", err) + } + select { + case <-ctx.Done(): + return nil + case <-time.NewTimer(time.Second).C: + } + } + } +} + +func waitForUpdates(ctx context.Context, c *client.Client) error { + updates := make(chan client.Update, 1) + + c.Updates = updates + defer func() { + c.Updates = nil + }() + + errs := make(chan error, 1) + stop := make(chan struct{}) + go func() { + errs <- c.Idle(stop, nil) + }() + + stopped := false + for { + select { + case update := <-updates: + switch update.(type) { + case *client.MailboxUpdate: + if !stopped { + close(stop) + stopped = true + } + default: + } + case err := <-errs: + if err != nil { + return fmt.Errorf("imap idle failed: %w", err) + } + return nil + case <-ctx.Done(): + return nil + } + } +} + +func processMessages(ctx context.Context, c *client.Client) error { + mbox, err := c.Select(setting.IncomingEmail.Mailbox, false) + if err != nil { + return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err) + } + + if mbox.Messages == 0 { + return nil + } + + criteria := imap.NewSearchCriteria() + criteria.WithoutFlags = []string{imap.SeenFlag} + criteria.Smaller = setting.IncomingEmail.MaximumMessageSize + ids, err := c.Search(criteria) + if err != nil { + return fmt.Errorf("search failed: %w", err) + } + + if len(ids) == 0 { + return nil + } + + seqset := new(imap.SeqSet) + seqset.AddNum(ids...) + messages := make(chan *imap.Message, 10) + + section := &imap.BodySectionName{} + + errs := make(chan error, 1) + go func() { + errs <- c.Fetch( + seqset, + []imap.FetchItem{section.FetchItem()}, + messages, + ) + }() + + handledSet := new(imap.SeqSet) +loop: + for { + select { + case <-ctx.Done(): + break loop + case msg, ok := <-messages: + if !ok { + if setting.IncomingEmail.DeleteHandledMessage && !handledSet.Empty() { + if err := c.Store( + handledSet, + imap.FormatFlagsOp(imap.AddFlags, true), + []interface{}{imap.DeletedFlag}, + nil, + ); err != nil { + return fmt.Errorf("store failed: %w", err) + } + + if err := c.Expunge(nil); err != nil { + return fmt.Errorf("expunge failed: %w", err) + } + } + return nil + } + + err := func() error { + r := msg.GetBody(section) + if r == nil { + return fmt.Errorf("get body failed: %w", err) + } + + mr, err := mail.CreateReader(r) + if err != nil { + return fmt.Errorf("create reader failed: %w", err) + } + defer mr.Close() + + if isAutomaticReply(mr.Header) { + log.Debug("Skipping automatic reply") + return nil + } + + t := searchTokenInHeaders(mr.Header) + if t == "" { + log.Debug("Token not found") + return nil + } + + handlerType, user, payload, err := token.ExtractToken(ctx, t) + if err != nil { + if _, ok := err.(*token.ErrToken); ok { + log.Info("Invalid email token: %v", err) + return nil + } + return err + } + + handler, ok := handlers[handlerType] + if !ok { + return fmt.Errorf("unexpected handler type: %v", handlerType) + } + + content, err := getContentFromMailReader(mr) + if err != nil { + return fmt.Errorf("getContentFromMailReader failed: %w", err) + } + + if err := handler.Handle(ctx, content, user, payload); err != nil { + return fmt.Errorf("Handle failed: %w", err) + } + + handledSet.AddNum(msg.SeqNum) + + return nil + }() + if err != nil { + log.Error("Error while processing message[]: %v", err) + } + } + } + + if err := <-errs; err != nil { + return fmt.Errorf("fetch failed: %w", err) + } + + return nil +} + +func isAutomaticReply(h mail.Header) bool { + autoSubmitted := h.Get("Auto-Submitted") + if autoSubmitted != "" && autoSubmitted != "no" { + return true + } + autoReply := h.Get("X-Autoreply") + if autoReply == "yes" { + return true + } + autoRespond := h.Get("X-Autorespond") + return autoRespond != "" +} + +func searchTokenInHeaders(h mail.Header) string { + if addressTokenRegex != nil { + to, _ := h.AddressList("To") + deliveredTo, _ := h.AddressList("Delivered-To") + for _, list := range [][]*net_mail.Address{ + to, + deliveredTo, + } { + for _, address := range list { + match := addressTokenRegex.FindStringSubmatch(address.Address) + if len(match) != 2 { + continue + } + + return match[1] + } + } + } + + references := h.Get("References") + for { + begin := strings.IndexByte(references, '<') + if begin == -1 { + break + } + begin++ + + end := strings.IndexByte(references, '>') + if end == -1 || begin > end { + break + } + + match := referenceTokenRegex.FindStringSubmatch(references[begin:end]) + if len(match) == 2 { + return match[1] + } + + references = references[end+1:] + } + + return "" +} + +type MailContent struct { + Content string + Attachments []*Attachment +} + +type Attachment struct { + Name string + Content bytes.Buffer +} + +func getContentFromMailReader(mr *mail.Reader) (*MailContent, error) { + contentText := "" + contentHTML := "" + attachments := make([]*Attachment, 0, 1) + + for { + p, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, fmt.Errorf("next part failed: %w", err) + } + + switch h := p.Header.(type) { + case *mail.InlineHeader: + contentType, _, err := h.ContentType() + if err != nil { + return nil, fmt.Errorf("ContentType failed: %w", err) + } + + if contentType == "text/plain" { + if contentText != "" { + continue + } + } else if contentType == "text/html" { + if contentHTML != "" { + continue + } + } else { + continue + } + + data, err := io.ReadAll(p.Body) + if err != nil { + return nil, fmt.Errorf("read body failed: %w", err) + } + + switch contentType { + case "text/plain": + contentText = string(data) + case "text/html": + contentHTML = string(data) + } + case *mail.AttachmentHeader: + attachment := &Attachment{} + attachment.Name, _ = h.Filename() + _, err := io.Copy(&attachment.Content, p.Body) + if err != nil { + return nil, fmt.Errorf("read attachment failed: %w", err) + } + + attachments = append(attachments, attachment) + } + } + + if contentText == "" && contentHTML != "" { + var err error + contentText, err = html2text.FromString(contentHTML) + if err != nil { + return nil, err + } + } + + return &MailContent{ + Content: reply.FromText(contentText), + Attachments: attachments, + }, nil +} diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go new file mode 100644 index 0000000000000..11897f6fcde79 --- /dev/null +++ b/services/mailer/incoming/incoming_handler.go @@ -0,0 +1,160 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "context" + "fmt" + + issues_model "code.gitea.io/gitea/models/issues" + access_model "code.gitea.io/gitea/models/perm/access" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/upload" + attachment_service "code.gitea.io/gitea/services/attachment" + comment_service "code.gitea.io/gitea/services/comments" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + "code.gitea.io/gitea/services/mailer/token" + pull_service "code.gitea.io/gitea/services/pull" +) + +type MailHandler interface { + Handle(ctx context.Context, content *MailContent, user *user_model.User, payload []byte) error +} + +var handlers = map[token.HandlerType]MailHandler{ + token.ReplyHandlerType: &ReplyHandler{}, + token.UnsubscribeHandlerType: &UnsubscribeHandler{}, +} + +// ReplyHandler handles incoming emails to create a reply from them +type ReplyHandler struct{} + +func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *user_model.User, payload []byte) error { + if user == nil { + return fmt.Errorf("user needed") + } + + if content.Content == "" && len(content.Attachments) == 0 { + return nil + } + + ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) + if err != nil { + return err + } + + var issue *issues_model.Issue + + switch r := ref.(type) { + case *issues_model.Issue: + issue = r + case *issues_model.Comment: + comment := r + + if err := comment.LoadIssue(ctx); err != nil { + return err + } + + issue = comment.Issue + default: + return fmt.Errorf("unsupported reply reference: %v", ref) + } + + if err := issue.LoadRepo(ctx); err != nil { + return err + } + + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, user) + if err != nil { + return err + } + + if !perm.CanWriteIssuesOrPulls(issue.IsPull) || issue.IsLocked && !user.IsAdmin { + log.Debug("can't write issue or pull") + return nil + } + + switch r := ref.(type) { + case *issues_model.Issue: + attachmentIDs := make([]string, 0, len(content.Attachments)) + if setting.Attachment.Enabled { + for _, attachment := range content.Attachments { + a, err := attachment_service.UploadAttachment(&attachment.Content, user.ID, issue.Repo.ID, 0, attachment.Name, setting.Attachment.AllowedTypes) + if err != nil { + if upload.IsErrFileTypeForbidden(err) { + log.Debug("Skipping disallowed attachment type") + continue + } + return err + } + attachmentIDs = append(attachmentIDs, a.UUID) + } + } + + _, err = comment_service.CreateIssueComment(ctx, user, issue.Repo, issue, content.Content, attachmentIDs) + if err != nil { + return fmt.Errorf("CreateIssueComment failed: %w", err) + } + case *issues_model.Comment: + comment := r + + if comment.Type == issues_model.CommentTypeCode { + _, err := pull_service.CreateCodeComment( + ctx, + user, + nil, + issue, + comment.Line, + content.Content, + comment.TreePath, + false, + comment.ReviewID, + "", + ) + if err != nil { + return fmt.Errorf("CreateCodeComment failed: %w", err) + } + } + } + return nil +} + +// UnsubscribeHandler handles unwatching issues/pulls +type UnsubscribeHandler struct{} + +func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, user *user_model.User, payload []byte) error { + if user == nil { + return fmt.Errorf("user needed") + } + + ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) + if err != nil { + return err + } + + switch r := ref.(type) { + case *issues_model.Issue: + issue := r + + if err := issue.LoadRepo(ctx); err != nil { + return err + } + + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, user) + if err != nil { + return err + } + + if !perm.CanReadIssuesOrPulls(issue.IsPull) { + log.Debug("can't read issue or pull") + return nil + } + + return issues_model.CreateOrUpdateIssueWatch(user.ID, issue.ID, false) + } + + return fmt.Errorf("unsupported unsubscribe reference: %v", ref) +} diff --git a/services/mailer/incoming/incoming_test.go b/services/mailer/incoming/incoming_test.go new file mode 100644 index 0000000000000..e470411690c47 --- /dev/null +++ b/services/mailer/incoming/incoming_test.go @@ -0,0 +1,129 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "strings" + "testing" + + "github.com/emersion/go-message/mail" + "github.com/stretchr/testify/assert" +) + +func TestIsAutomaticReply(t *testing.T) { + cases := []struct { + Headers map[string][]string + Expected bool + }{ + { + Headers: map[string][]string{}, + Expected: false, + }, + { + Headers: map[string][]string{ + "Auto-Submitted": {"no"}, + }, + Expected: false, + }, + { + Headers: map[string][]string{ + "Auto-Submitted": {"yes"}, + }, + Expected: true, + }, + { + Headers: map[string][]string{ + "X-Autoreply": {"no"}, + }, + Expected: false, + }, + { + Headers: map[string][]string{ + "X-Autoreply": {"yes"}, + }, + Expected: true, + }, + { + Headers: map[string][]string{ + "X-Autorespond": {"yes"}, + }, + Expected: true, + }, + } + + for _, c := range cases { + assert.Equal(t, c.Expected, isAutomaticReply(mail.HeaderFromMap(c.Headers))) + } +} + +func TestGetContentFromMailReader(t *testing.T) { + mailString := "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "mail content\r\n" + + "--text-boundary--\r\n" + + "--message-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: attachment; filename=attachment.txt\r\n" + + "\r\n" + + "attachment content\r\n" + + "--message-boundary--\r\n" + + mr, err := mail.CreateReader(strings.NewReader(mailString)) + assert.NoError(t, err) + content, err := getContentFromMailReader(mr) + assert.NoError(t, err) + assert.Equal(t, "mail content", content.Content) + assert.Len(t, content.Attachments, 1) + assert.Equal(t, "attachment.txt", content.Attachments[0].Name) + assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content.Bytes()) + + mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/html\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "

mail content

\r\n" + + "--text-boundary--\r\n" + + "--message-boundary--\r\n" + + mr, err = mail.CreateReader(strings.NewReader(mailString)) + assert.NoError(t, err) + content, err = getContentFromMailReader(mr) + assert.NoError(t, err) + assert.Equal(t, "mail content", content.Content) + assert.Empty(t, content.Attachments) + + mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "mail content without signature\r\n" + + "--\r\n" + + "signature\r\n" + + "--text-boundary--\r\n" + + "--message-boundary--\r\n" + + mr, err = mail.CreateReader(strings.NewReader(mailString)) + assert.NoError(t, err) + content, err = getContentFromMailReader(mr) + assert.NoError(t, err) + assert.Equal(t, "mail content without signature", content.Content) + assert.Empty(t, content.Attachments) +} diff --git a/services/mailer/incoming/payload/payload.go b/services/mailer/incoming/payload/payload.go new file mode 100644 index 0000000000000..e5097a688c130 --- /dev/null +++ b/services/mailer/incoming/payload/payload.go @@ -0,0 +1,71 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package payload + +import ( + "context" + "fmt" + + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/util" +) + +const replyPayloadVersion1 byte = 1 + +type payloadReferenceType byte + +const ( + payloadReferenceIssue payloadReferenceType = iota + payloadReferenceComment +) + +// CreateReferencePayload creates data which GetReferenceFromPayload resolves to the reference again. +func CreateReferencePayload(reference interface{}) ([]byte, error) { + var refType payloadReferenceType + var refID int64 + + switch r := reference.(type) { + case *issues_model.Issue: + refType = payloadReferenceIssue + refID = r.ID + case *issues_model.Comment: + refType = payloadReferenceComment + refID = r.ID + default: + return nil, fmt.Errorf("unsupported reference type") + } + + payload, err := util.PackData(refType, refID) + if err != nil { + return nil, err + } + + return append([]byte{replyPayloadVersion1}, payload...), nil +} + +// GetReferenceFromPayload resolves the reference from the payload +func GetReferenceFromPayload(ctx context.Context, payload []byte) (interface{}, error) { + if len(payload) < 1 { + return nil, fmt.Errorf("payload to small") + } + + if payload[0] != replyPayloadVersion1 { + return nil, fmt.Errorf("unsupported payload version") + } + + var ref payloadReferenceType + var id int64 + if err := util.UnpackData(payload[1:], &ref, &id); err != nil { + return nil, err + } + + switch ref { + case payloadReferenceIssue: + return issues_model.GetIssueByID(ctx, id) + case payloadReferenceComment: + return issues_model.GetCommentByID(ctx, id) + } + + return nil, fmt.Errorf("unsupported reference type") +} diff --git a/services/mailer/mail.go b/services/mailer/mail.go index a9e36e10f89b8..d9103f36da92d 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -30,6 +30,8 @@ import ( "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/translation" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + "code.gitea.io/gitea/services/mailer/token" "gopkg.in/gomail.v2" ) @@ -303,14 +305,57 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient msgID := createReference(ctx.Issue, ctx.Comment, ctx.ActionType) reference := createReference(ctx.Issue, nil, activities_model.ActionType(0)) + var replyPayload []byte + if ctx.Comment != nil && ctx.Comment.Type == issues_model.CommentTypeCode { + replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Comment) + } else { + replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Issue) + } + if err != nil { + return nil, err + } + + unsubscribePayload, err := incoming_payload.CreateReferencePayload(ctx.Issue) + if err != nil { + return nil, err + } + msgs := make([]*Message, 0, len(recipients)) for _, recipient := range recipients { msg := NewMessageFrom([]string{recipient.Email}, ctx.Doer.DisplayName(), setting.MailService.FromEmail, subject, mailBody.String()) msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info) - msg.SetHeader("Message-ID", "<"+msgID+">") - msg.SetHeader("In-Reply-To", "<"+reference+">") - msg.SetHeader("References", "<"+reference+">") + msg.SetHeader("Message-ID", msgID) + msg.SetHeader("In-Reply-To", reference) + + references := []string{reference} + listUnsubscribe := []string{"<" + ctx.Issue.HTMLURL() + ">"} + + if setting.IncomingEmail.Enabled { + if ctx.Comment != nil { + token, err := token.CreateToken(token.ReplyHandlerType, recipient, replyPayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + replyAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + msg.ReplyTo = replyAddress + msg.SetHeader("List-Post", fmt.Sprintf("", replyAddress)) + + references = append(references, fmt.Sprintf("", token, setting.Domain)) + } + } + + token, err := token.CreateToken(token.UnsubscribeHandlerType, recipient, unsubscribePayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + unsubAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + listUnsubscribe = append(listUnsubscribe, "") + } + } + + msg.SetHeader("References", references...) + msg.SetHeader("List-Unsubscribe", listUnsubscribe...) for key, value := range generateAdditionalHeaders(ctx, actType, recipient) { msg.SetHeader(key, value) @@ -346,7 +391,7 @@ func createReference(issue *issues_model.Issue, comment *issues_model.Comment, a } } - return fmt.Sprintf("%s/%s/%d%s@%s", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) + return fmt.Sprintf("<%s/%s/%d%s@%s>", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) } func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string { @@ -358,8 +403,6 @@ func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient // https://datatracker.ietf.org/doc/html/rfc2369 "List-Archive": fmt.Sprintf("<%s>", repo.HTMLURL()), - //"List-Post": https://github.com/go-gitea/gitea/pull/13585 - "List-Unsubscribe": ctx.Issue.HTMLURL(), "X-Mailer": "Gitea", "X-Gitea-Reason": reason, diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index acb1f69961194..238a93e8dbecd 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "html/template" + "regexp" "strings" "testing" texttmpl "text/template" @@ -67,6 +68,9 @@ func prepareMailerTest(t *testing.T) (doer *user_model.User, repo *repo_model.Re func TestComposeIssueCommentMessage(t *testing.T) { doer, _, issue, comment := prepareMailerTest(t) + setting.IncomingEmail.Enabled = true + defer func() { setting.IncomingEmail.Enabled = false }() + subjectTemplates = texttmpl.Must(texttmpl.New("issue/comment").Parse(subjectTpl)) bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl)) @@ -79,18 +83,20 @@ func TestComposeIssueCommentMessage(t *testing.T) { assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() - mailto := gomailMsg.GetHeader("To") - subject := gomailMsg.GetHeader("Subject") - messageID := gomailMsg.GetHeader("Message-ID") - inReplyTo := gomailMsg.GetHeader("In-Reply-To") - references := gomailMsg.GetHeader("References") - - assert.Len(t, mailto, 1, "exactly one recipient is expected in the To field") - assert.Equal(t, "Re: ", subject[0][:4], "Comment reply subject should contain Re:") - assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject[0]) - assert.Equal(t, "", inReplyTo[0], "In-Reply-To header doesn't match") - assert.Equal(t, "", references[0], "References header doesn't match") - assert.Equal(t, "", messageID[0], "Message-ID header doesn't match") + replyTo := gomailMsg.GetHeader("Reply-To")[0] + subject := gomailMsg.GetHeader("Subject")[0] + + assert.Len(t, gomailMsg.GetHeader("To"), 1, "exactly one recipient is expected in the To field") + tokenRegex := regexp.MustCompile(`\Aincoming\+(.+)@localhost\z`) + assert.Regexp(t, tokenRegex, replyTo) + token := tokenRegex.FindAllStringSubmatch(replyTo, 1)[0][1] + assert.Equal(t, "Re: ", subject[:4], "Comment reply subject should contain Re:") + assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject) + assert.Equal(t, "", gomailMsg.GetHeader("In-Reply-To")[0], "In-Reply-To header doesn't match") + assert.ElementsMatch(t, []string{"", ""}, gomailMsg.GetHeader("References"), "References header doesn't match") + assert.Equal(t, "", gomailMsg.GetHeader("Message-ID")[0], "Message-ID header doesn't match") + assert.Equal(t, "", gomailMsg.GetHeader("List-Post")[0]) + assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 2) // url + mailto } func TestComposeIssueMessage(t *testing.T) { @@ -120,6 +126,8 @@ func TestComposeIssueMessage(t *testing.T) { assert.Equal(t, "", inReplyTo[0], "In-Reply-To header doesn't match") assert.Equal(t, "", references[0], "References header doesn't match") assert.Equal(t, "", messageID[0], "Message-ID header doesn't match") + assert.Empty(t, gomailMsg.GetHeader("List-Post")) // incoming mail feature disabled + assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 1) // url without mailto } func TestTemplateSelection(t *testing.T) { @@ -239,7 +247,6 @@ func TestGenerateAdditionalHeaders(t *testing.T) { expected := map[string]string{ "List-ID": "user2/repo1 ", "List-Archive": "", - "List-Unsubscribe": "https://try.gitea.io/user2/repo1/issues/1", "X-Gitea-Reason": "dummy-reason", "X-Gitea-Sender": "< Ur Tw ><", "X-Gitea-Recipient": "Test", @@ -272,7 +279,6 @@ func Test_createReference(t *testing.T) { name string args args prefix string - suffix string }{ { name: "Open Issue", @@ -280,7 +286,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionCreateIssue, }, - prefix: fmt.Sprintf("%s/issues/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain), + prefix: fmt.Sprintf("<%s/issues/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain), }, { name: "Open Pull", @@ -288,7 +294,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionCreatePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain), + prefix: fmt.Sprintf("<%s/pulls/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain), }, { name: "Comment Issue", @@ -297,7 +303,7 @@ func Test_createReference(t *testing.T) { comment: comment, actionType: activities_model.ActionCommentIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), + prefix: fmt.Sprintf("<%s/issues/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), }, { name: "Comment Pull", @@ -306,7 +312,7 @@ func Test_createReference(t *testing.T) { comment: comment, actionType: activities_model.ActionCommentPull, }, - prefix: fmt.Sprintf("%s/pulls/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), + prefix: fmt.Sprintf("<%s/pulls/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), }, { name: "Close Issue", @@ -314,7 +320,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionCloseIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/close/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/issues/%d/close/", issue.Repo.FullName(), issue.Index), }, { name: "Close Pull", @@ -322,7 +328,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionClosePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index), }, { name: "Reopen Issue", @@ -330,7 +336,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionReopenIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index), }, { name: "Reopen Pull", @@ -338,7 +344,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionReopenPullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index), }, { name: "Merge Pull", @@ -346,7 +352,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionMergePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index), }, { name: "Ready Pull", @@ -354,7 +360,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionPullRequestReadyForReview, }, - prefix: fmt.Sprintf("%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index), }, } for _, tt := range tests { @@ -363,9 +369,6 @@ func Test_createReference(t *testing.T) { if !strings.HasPrefix(got, tt.prefix) { t.Errorf("createReference() = %v, want %v", got, tt.prefix) } - if !strings.HasSuffix(got, tt.suffix) { - t.Errorf("createReference() = %v, want %v", got, tt.prefix) - } }) } } diff --git a/services/mailer/mailer.go b/services/mailer/mailer.go index 2663b6b2bab11..ed7d720eed794 100644 --- a/services/mailer/mailer.go +++ b/services/mailer/mailer.go @@ -37,6 +37,7 @@ type Message struct { FromAddress string FromDisplayName string To []string + ReplyTo string Subject string Date time.Time Body string @@ -48,6 +49,9 @@ func (m *Message) ToMessage() *gomail.Message { msg := gomail.NewMessage() msg.SetAddressHeader("From", m.FromAddress, m.FromDisplayName) msg.SetHeader("To", m.To...) + if m.ReplyTo != "" { + msg.SetHeader("Reply-To", m.ReplyTo) + } for header := range m.Headers { msg.SetHeader(header, m.Headers[header]...) } diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go new file mode 100644 index 0000000000000..c7446551a5b8c --- /dev/null +++ b/services/mailer/token/token.go @@ -0,0 +1,116 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package token + +import ( + "context" + crypto_hmac "crypto/hmac" + "crypto/sha256" + "encoding/base32" + "fmt" + "time" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" +) + +const ( + tokenVersion1 byte = 1 + tokenLifetimeInYears = 1 +) + +type HandlerType byte + +const ( + UnknownHandlerType HandlerType = iota + ReplyHandlerType + UnsubscribeHandlerType +) + +var encodingWithoutPadding = base32.StdEncoding.WithPadding(base32.NoPadding) + +type ErrToken struct { + context string +} + +func (p *ErrToken) Error() string { + return "invalid email token: " + p.context +} + +// CreateToken creates a token for the action/user tuple +func CreateToken(ht HandlerType, user *user_model.User, data []byte) (string, error) { + payload, err := util.PackData( + time.Now().AddDate(tokenLifetimeInYears, 0, 0).Unix(), + ht, + data, + ) + if err != nil { + return "", err + } + + packagedData, err := util.PackData( + user.ID, + generateHmac([]byte(user.Rands), payload), + payload, + ) + if err != nil { + return "", err + } + + return encodingWithoutPadding.EncodeToString(append([]byte{tokenVersion1}, packagedData...)), nil +} + +// ExtractToken extracts the action/user tuple from the token and verifies the content +func ExtractToken(ctx context.Context, token string) (HandlerType, *user_model.User, []byte, error) { + data, err := encodingWithoutPadding.DecodeString(token) + if err != nil { + return UnknownHandlerType, nil, nil, err + } + + if len(data) < 1 { + return UnknownHandlerType, nil, nil, &ErrToken{"no data"} + } + + if data[0] != tokenVersion1 { + return UnknownHandlerType, nil, nil, &ErrToken{fmt.Sprintf("unsupported token version: %v", data[0])} + } + + var userID int64 + var hmac []byte + var payload []byte + if err := util.UnpackData(data[1:], &userID, &hmac, &payload); err != nil { + return UnknownHandlerType, nil, nil, err + } + + user, err := user_model.GetUserByIDCtx(ctx, userID) + if err != nil { + return UnknownHandlerType, nil, nil, err + } + + if !crypto_hmac.Equal(hmac, generateHmac([]byte(user.Rands), payload)) { + return UnknownHandlerType, nil, nil, &ErrToken{"verification failed"} + } + + var expiresUnix int64 + var handlerType HandlerType + var innerPayload []byte + if err := util.UnpackData(payload, &expiresUnix, &handlerType, &innerPayload); err != nil { + return UnknownHandlerType, nil, nil, err + } + + if time.Unix(expiresUnix, 0).Before(time.Now()) { + return UnknownHandlerType, nil, nil, &ErrToken{"token expired"} + } + + return handlerType, user, innerPayload, nil +} + +// generateHmac creates a trunkated HMAC for the given payload +func generateHmac(secret, payload []byte) []byte { + mac := crypto_hmac.New(sha256.New, secret) + mac.Write(payload) + hmac := mac.Sum(nil) + + return hmac[:10] // RFC2104 recommends not using less then 80 bits +} diff --git a/tests/integration/incoming_email_test.go b/tests/integration/incoming_email_test.go new file mode 100644 index 0000000000000..533b055df91c6 --- /dev/null +++ b/tests/integration/incoming_email_test.go @@ -0,0 +1,251 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "bytes" + "io" + "net" + "net/smtp" + "strings" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer/incoming" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + token_service "code.gitea.io/gitea/services/mailer/token" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" + "gopkg.in/gomail.v2" +) + +func TestIncomingEmail(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1}) + + t.Run("Payload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 1}) + + _, err := incoming_payload.CreateReferencePayload(user) + assert.Error(t, err) + + issuePayload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + commentPayload, err := incoming_payload.CreateReferencePayload(comment) + assert.NoError(t, err) + + _, err = incoming_payload.GetReferenceFromPayload(db.DefaultContext, []byte{1, 2, 3}) + assert.Error(t, err) + + ref, err := incoming_payload.GetReferenceFromPayload(db.DefaultContext, issuePayload) + assert.NoError(t, err) + assert.IsType(t, ref, new(issues_model.Issue)) + assert.EqualValues(t, issue.ID, ref.(*issues_model.Issue).ID) + + ref, err = incoming_payload.GetReferenceFromPayload(db.DefaultContext, commentPayload) + assert.NoError(t, err) + assert.IsType(t, ref, new(issues_model.Comment)) + assert.EqualValues(t, comment.ID, ref.(*issues_model.Comment).ID) + }) + + t.Run("Token", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + payload := []byte{1, 2, 3, 4, 5} + + token, err := token_service.CreateToken(token_service.ReplyHandlerType, user, payload) + assert.NoError(t, err) + assert.NotEmpty(t, token) + + ht, u, p, err := token_service.ExtractToken(db.DefaultContext, token) + assert.NoError(t, err) + assert.Equal(t, token_service.ReplyHandlerType, ht) + assert.Equal(t, user.ID, u.ID) + assert.Equal(t, payload, p) + }) + + t.Run("Handler", func(t *testing.T) { + t.Run("Reply", func(t *testing.T) { + t.Run("Comment", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + handler := &incoming.ReplyHandler{} + content := &incoming.MailContent{ + Content: "reply by mail", + Attachments: []*incoming.Attachment{ + { + Name: "attachment.txt", + Content: *bytes.NewBuffer([]byte("test")), + }, + }, + } + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + + assert.Error(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, nil, payload)) + assert.NoError(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, user, payload)) + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeComment, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + comment := comments[len(comments)-1] + assert.Equal(t, user.ID, comment.PosterID) + assert.Equal(t, content.Content, comment.Content) + assert.NoError(t, comment.LoadAttachments(db.DefaultContext)) + assert.Len(t, comment.Attachments, 1) + attachment := comment.Attachments[0] + assert.Equal(t, content.Attachments[0].Name, attachment.Name) + assert.EqualValues(t, 4, attachment.Size) + }) + + t.Run("CodeComment", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 6}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}) + + handler := &incoming.ReplyHandler{} + content := &incoming.MailContent{ + Content: "code reply by mail", + Attachments: []*incoming.Attachment{ + { + Name: "attachment.txt", + Content: *bytes.NewBuffer([]byte("test")), + }, + }, + } + + payload, err := incoming_payload.CreateReferencePayload(comment) + assert.NoError(t, err) + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeCode, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + comment = comments[len(comments)-1] + assert.Equal(t, user.ID, comment.PosterID) + assert.Equal(t, content.Content, comment.Content) + assert.NoError(t, comment.LoadAttachments(db.DefaultContext)) + assert.Empty(t, comment.Attachments) + }) + }) + + t.Run("Unsubscribe", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + watching, err := issues_model.CheckIssueWatch(user, issue) + assert.NoError(t, err) + assert.True(t, watching) + + handler := &incoming.UnsubscribeHandler{} + + content := &incoming.MailContent{ + Content: "unsub me", + } + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + watching, err = issues_model.CheckIssueWatch(user, issue) + assert.NoError(t, err) + assert.False(t, watching) + }) + }) + + if setting.IncomingEmail.Enabled { + t.Run("IMAP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + token, err := token_service.CreateToken(token_service.ReplyHandlerType, user, payload) + assert.NoError(t, err) + + msg := gomail.NewMessage() + msg.SetHeader("To", strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1)) + msg.SetHeader("From", user.Email) + msg.SetBody("text/plain", token) + err = gomail.Send(&smtpTestSender{}, msg) + assert.NoError(t, err) + + for i := 0; i < 5; i++ { + time.Sleep(1 * time.Second) + + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeComment, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + comment := comments[len(comments)-1] + if comment.Content == token { + assert.Equal(t, user.ID, comment.PosterID) + break + } else if i == 4 { + t.SkipNow() + } + } + }) + } +} + +type smtpTestSender struct{} + +func (s *smtpTestSender) Send(from string, to []string, msg io.WriterTo) error { + conn, err := net.Dial("tcp", net.JoinHostPort(setting.IncomingEmail.Host, "25")) + if err != nil { + return err + } + defer conn.Close() + + client, err := smtp.NewClient(conn, setting.IncomingEmail.Host) + if err != nil { + return err + } + + if err = client.Mail(from); err != nil { + return err + } + + for _, rec := range to { + if err = client.Rcpt(rec); err != nil { + return err + } + } + + w, err := client.Data() + if err != nil { + return err + } + if _, err := msg.WriteTo(w); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + + return client.Quit() +} diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 24a9a02dc46ce..e4e47aa4dfc4e 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -124,3 +124,13 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [packages] ENABLED = true + +[incoming_email] +ENABLED = true +HOST = smtpimap +PORT = 993 +USERNAME = debug@localdomain.test +PASSWORD = debug +USE_TLS = true +SKIP_TLS_VERIFY = true +REPLY_TO_ADDRESS = incoming+%{token}@localhost From e159bb82dae5e773dba93f7e1d29fd917a9b7f25 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Dec 2022 21:14:00 +0100 Subject: [PATCH 02/12] Add comments. --- services/mailer/incoming/incoming.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go index ebe6259876965..7adcdb117d009 100644 --- a/services/mailer/incoming/incoming.go +++ b/services/mailer/incoming/incoming.go @@ -54,7 +54,7 @@ func Init(ctx context.Context) error { case <-ctx.Done(): return default: - if err := processIncomingEmail(ctx); err != nil { + if err := processIncomingEmails(ctx); err != nil { log.Error("Error while processing incoming emails: %v", err) } select { @@ -69,7 +69,8 @@ func Init(ctx context.Context) error { return nil } -func processIncomingEmail(ctx context.Context) error { +// processIncomingEmails is the "main" method with the wait/process loop +func processIncomingEmails(ctx context.Context) error { server := fmt.Sprintf("%s:%d", setting.IncomingEmail.Host, setting.IncomingEmail.Port) var c *client.Client @@ -116,6 +117,7 @@ func processIncomingEmail(ctx context.Context) error { } } +// waitForUpdates uses IMAP IDLE to wait for new emails func waitForUpdates(ctx context.Context, c *client.Client) error { updates := make(chan client.Update, 1) @@ -153,6 +155,7 @@ func waitForUpdates(ctx context.Context, c *client.Client) error { } } +// processMessages searches unread mails and processes them. func processMessages(ctx context.Context, c *client.Client) error { mbox, err := c.Select(setting.IncomingEmail.Mailbox, false) if err != nil { @@ -278,6 +281,7 @@ loop: return nil } +// isAutomaticReply tests if the headers indicate an automatic reply func isAutomaticReply(h mail.Header) bool { autoSubmitted := h.Get("Auto-Submitted") if autoSubmitted != "" && autoSubmitted != "no" { @@ -291,6 +295,7 @@ func isAutomaticReply(h mail.Header) bool { return autoRespond != "" } +// searchTokenInHeaders looks for the token in To, Delivered-To and References func searchTokenInHeaders(h mail.Header) string { if addressTokenRegex != nil { to, _ := h.AddressList("To") @@ -344,6 +349,9 @@ type Attachment struct { Content bytes.Buffer } +// getContentFromMailReader reads the plain content and the attachments from the mail. +// If there is only HTML content, it gets converted to plain text. +// A potential reply/signature gets stripped from the content. func getContentFromMailReader(mr *mail.Reader) (*MailContent, error) { contentText := "" contentHTML := "" From 7c818d529880d114baac6a5b9a97a424e6385559 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 7 Dec 2022 12:20:56 +0000 Subject: [PATCH 03/12] Fix merge error. --- services/mailer/token/token.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go index c7446551a5b8c..41168175f0876 100644 --- a/services/mailer/token/token.go +++ b/services/mailer/token/token.go @@ -16,8 +16,8 @@ import ( ) const ( - tokenVersion1 byte = 1 - tokenLifetimeInYears = 1 + tokenVersion1 byte = 1 + tokenLifetimeInYears = 1 ) type HandlerType byte @@ -83,7 +83,7 @@ func ExtractToken(ctx context.Context, token string) (HandlerType, *user_model.U return UnknownHandlerType, nil, nil, err } - user, err := user_model.GetUserByIDCtx(ctx, userID) + user, err := user_model.GetUserByID(ctx, userID) if err != nil { return UnknownHandlerType, nil, nil, err } From ecb606d4303bb955cba20550b31f4b2989362753 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 7 Dec 2022 13:18:43 +0000 Subject: [PATCH 04/12] Use enmime library. --- assets/go-licenses.json | 25 +++++ go.mod | 8 +- go.sum | 12 ++- services/mailer/incoming/incoming.go | 108 +++++-------------- services/mailer/incoming/incoming_handler.go | 3 +- services/mailer/incoming/incoming_test.go | 53 +++++---- services/mailer/token/token.go | 2 +- tests/integration/incoming_email_test.go | 5 +- 8 files changed, 99 insertions(+), 117 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index c4b62475fd4bc..debaf746350b9 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -209,6 +209,11 @@ "path": "github.com/caddyserver/certmagic/LICENSE.txt", "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, + { + "name": "github.com/cention-sany/utf7", + "path": "github.com/cention-sany/utf7/LICENSE", + "licenseText": "Copyright (c) 2013 The Go-IMAP Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n * Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the\n distribution.\n\n * Neither the name of the go-imap project nor the names of its\n contributors may be used to endorse or promote products derived\n from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/cespare/xxhash/v2", "path": "github.com/cespare/xxhash/v2/LICENSE.txt", @@ -259,6 +264,11 @@ "path": "github.com/dgryski/go-rendezvous/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2017-2020 Damian Gryski \u003cdamian@gryski.com\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" }, + { + "name": "github.com/dimiro1/reply", + "path": "github.com/dimiro1/reply/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) Discourse\nCopyright (c) Claudemiro\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/djherbis/buffer", "path": "github.com/djherbis/buffer/LICENSE.txt", @@ -294,6 +304,16 @@ "path": "github.com/editorconfig/editorconfig-core-go/v2/LICENSE", "licenseText": "MIT License\nCopyright (c) 2016 The Editorconfig Team\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/emersion/go-imap", + "path": "github.com/emersion/go-imap/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2013 The Go-IMAP Authors\nCopyright (c) 2016 emersion\nCopyright (c) 2016 Proton Technologies AG\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, + { + "name": "github.com/emersion/go-sasl", + "path": "github.com/emersion/go-sasl/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2016 emersion\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/ethantkoenig/rupture", "path": "github.com/ethantkoenig/rupture/LICENSE", @@ -514,6 +534,11 @@ "path": "github.com/jaytaylor/html2text/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2015 Jay Taylor\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n" }, + { + "name": "github.com/jhillyerd/enmime", + "path": "github.com/jhillyerd/enmime/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2012-2016 James Hillyerd, All Rights Reserved\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/josharian/intern", "path": "github.com/josharian/intern/license.md", diff --git a/go.mod b/go.mod index 9e34eff2b8fa8..04674cc483cc7 100644 --- a/go.mod +++ b/go.mod @@ -21,11 +21,13 @@ require ( github.com/caddyserver/certmagic v0.17.2 github.com/chi-middleware/proxy v1.1.1 github.com/denisenkom/go-mssqldb v0.12.2 + github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 github.com/duo-labs/webauthn v0.0.0-20220815211337-00c9fb5711f5 github.com/dustin/go-humanize v1.0.0 github.com/editorconfig/editorconfig-core-go/v2 v2.4.5 + github.com/emersion/go-imap v1.2.1 github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.3 @@ -58,6 +60,7 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/huandu/xstrings v1.3.2 github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba + github.com/jhillyerd/enmime v0.10.1 github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4 @@ -146,6 +149,7 @@ require ( github.com/boombuler/barcode v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect + github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cloudflare/cfssl v1.6.1 // indirect github.com/cloudflare/circl v1.2.0 // indirect @@ -159,13 +163,9 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect - github.com/emersion/go-imap v1.2.1 // indirect - github.com/emersion/go-message v0.16.0 // indirect github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect - github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/go.sum b/go.sum index de1260bae0181..e006928000046 100644 --- a/go.sum +++ b/go.sum @@ -293,6 +293,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= +github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -421,11 +423,8 @@ github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA= github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= -github.com/emersion/go-message v0.16.0 h1:uZLz8ClLv3V5fSFF/fFdW9jXjrZkXIpE1Fn8fKx7pO4= -github.com/emersion/go-message v0.16.0/go.mod h1:pDJDgf/xeUIF+eicT6B/hPX/ZbEorKkUMPOxrPVG2eQ= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= -github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -580,6 +579,8 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/go-swagger/go-swagger v0.30.3 h1:HuzvdMRed/9Q8vmzVcfNBQByZVtT79DNZxZ18OprdoI= github.com/go-swagger/go-swagger v0.30.3/go.mod h1:neDPes8r8PCz2JPvHRDj8BTULLh4VJUt7n6MpQqxhHM= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013 h1:l9rI6sNaZgNC0LnF3MiE+qTmyBA/tZAg1rtyrGbUMK0= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU= github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -624,6 +625,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= @@ -925,6 +927,7 @@ github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg= github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -932,6 +935,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jhillyerd/enmime v0.10.1 h1:3VP8gFhK7R948YJBrna5bOgnTXEuPAoICo79kKkBKfA= +github.com/jhillyerd/enmime v0.10.1/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/z6mkcDHsA= github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4= github.com/jhump/protoreflect v1.8.2 h1:k2xE7wcUomeqwY0LDCYA16y4WWfyTcMx5mKhk0d4ua0= github.com/jhump/protoreflect v1.8.2/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= @@ -1719,6 +1724,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go index 7adcdb117d009..7e8b786a42381 100644 --- a/services/mailer/incoming/incoming.go +++ b/services/mailer/incoming/incoming.go @@ -4,12 +4,9 @@ package incoming import ( - "bytes" "context" "crypto/tls" - "errors" "fmt" - "io" net_mail "net/mail" "regexp" "strings" @@ -23,8 +20,7 @@ import ( "github.com/dimiro1/reply" "github.com/emersion/go-imap" "github.com/emersion/go-imap/client" - "github.com/emersion/go-message/mail" - "github.com/jaytaylor/html2text" + "github.com/jhillyerd/enmime" ) var ( @@ -224,18 +220,17 @@ loop: return fmt.Errorf("get body failed: %w", err) } - mr, err := mail.CreateReader(r) + env, err := enmime.ReadEnvelope(r) if err != nil { - return fmt.Errorf("create reader failed: %w", err) + return fmt.Errorf("read envelope failed: %w", err) } - defer mr.Close() - if isAutomaticReply(mr.Header) { + if isAutomaticReply(env) { log.Debug("Skipping automatic reply") return nil } - t := searchTokenInHeaders(mr.Header) + t := searchTokenInHeaders(env) if t == "" { log.Debug("Token not found") return nil @@ -255,7 +250,7 @@ loop: return fmt.Errorf("unexpected handler type: %v", handlerType) } - content, err := getContentFromMailReader(mr) + content, err := getContentFromMailReader(env) if err != nil { return fmt.Errorf("getContentFromMailReader failed: %w", err) } @@ -282,24 +277,24 @@ loop: } // isAutomaticReply tests if the headers indicate an automatic reply -func isAutomaticReply(h mail.Header) bool { - autoSubmitted := h.Get("Auto-Submitted") +func isAutomaticReply(env *enmime.Envelope) bool { + autoSubmitted := env.GetHeader("Auto-Submitted") if autoSubmitted != "" && autoSubmitted != "no" { return true } - autoReply := h.Get("X-Autoreply") + autoReply := env.GetHeader("X-Autoreply") if autoReply == "yes" { return true } - autoRespond := h.Get("X-Autorespond") + autoRespond := env.GetHeader("X-Autorespond") return autoRespond != "" } // searchTokenInHeaders looks for the token in To, Delivered-To and References -func searchTokenInHeaders(h mail.Header) string { +func searchTokenInHeaders(env *enmime.Envelope) string { if addressTokenRegex != nil { - to, _ := h.AddressList("To") - deliveredTo, _ := h.AddressList("Delivered-To") + to, _ := env.AddressList("To") + deliveredTo, _ := env.AddressList("Delivered-To") for _, list := range [][]*net_mail.Address{ to, deliveredTo, @@ -315,7 +310,7 @@ func searchTokenInHeaders(h mail.Header) string { } } - references := h.Get("References") + references := env.GetHeader("References") for { begin := strings.IndexByte(references, '<') if begin == -1 { @@ -346,77 +341,22 @@ type MailContent struct { type Attachment struct { Name string - Content bytes.Buffer + Content []byte } -// getContentFromMailReader reads the plain content and the attachments from the mail. -// If there is only HTML content, it gets converted to plain text. +// getContentFromMailReader grabs the plain content and the attachments from the mail. // A potential reply/signature gets stripped from the content. -func getContentFromMailReader(mr *mail.Reader) (*MailContent, error) { - contentText := "" - contentHTML := "" - attachments := make([]*Attachment, 0, 1) - - for { - p, err := mr.NextPart() - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, fmt.Errorf("next part failed: %w", err) - } - - switch h := p.Header.(type) { - case *mail.InlineHeader: - contentType, _, err := h.ContentType() - if err != nil { - return nil, fmt.Errorf("ContentType failed: %w", err) - } - - if contentType == "text/plain" { - if contentText != "" { - continue - } - } else if contentType == "text/html" { - if contentHTML != "" { - continue - } - } else { - continue - } - - data, err := io.ReadAll(p.Body) - if err != nil { - return nil, fmt.Errorf("read body failed: %w", err) - } - - switch contentType { - case "text/plain": - contentText = string(data) - case "text/html": - contentHTML = string(data) - } - case *mail.AttachmentHeader: - attachment := &Attachment{} - attachment.Name, _ = h.Filename() - _, err := io.Copy(&attachment.Content, p.Body) - if err != nil { - return nil, fmt.Errorf("read attachment failed: %w", err) - } - - attachments = append(attachments, attachment) - } - } - - if contentText == "" && contentHTML != "" { - var err error - contentText, err = html2text.FromString(contentHTML) - if err != nil { - return nil, err - } +func getContentFromMailReader(env *enmime.Envelope) (*MailContent, error) { + attachments := make([]*Attachment, 0, len(env.Attachments)) + for _, attachment := range env.Attachments { + attachments = append(attachments, &Attachment{ + Name: attachment.FileName, + Content: attachment.Content, + }) } return &MailContent{ - Content: reply.FromText(contentText), + Content: reply.FromText(env.Text), Attachments: attachments, }, nil } diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index 11897f6fcde79..5ef1fa4298d4f 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -4,6 +4,7 @@ package incoming import ( + "bytes" "context" "fmt" @@ -82,7 +83,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u attachmentIDs := make([]string, 0, len(content.Attachments)) if setting.Attachment.Enabled { for _, attachment := range content.Attachments { - a, err := attachment_service.UploadAttachment(&attachment.Content, user.ID, issue.Repo.ID, 0, attachment.Name, setting.Attachment.AllowedTypes) + a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), user.ID, issue.Repo.ID, 0, attachment.Name, setting.Attachment.AllowedTypes) if err != nil { if upload.IsErrFileTypeForbidden(err) { log.Debug("Skipping disallowed attachment type") diff --git a/services/mailer/incoming/incoming_test.go b/services/mailer/incoming/incoming_test.go index e470411690c47..39b7273cf0388 100644 --- a/services/mailer/incoming/incoming_test.go +++ b/services/mailer/incoming/incoming_test.go @@ -7,53 +7,64 @@ import ( "strings" "testing" - "github.com/emersion/go-message/mail" + "github.com/jhillyerd/enmime" "github.com/stretchr/testify/assert" ) func TestIsAutomaticReply(t *testing.T) { cases := []struct { - Headers map[string][]string + Headers map[string]string Expected bool }{ { - Headers: map[string][]string{}, + Headers: map[string]string{}, Expected: false, }, { - Headers: map[string][]string{ - "Auto-Submitted": {"no"}, + Headers: map[string]string{ + "Auto-Submitted": "no", }, Expected: false, }, { - Headers: map[string][]string{ - "Auto-Submitted": {"yes"}, + Headers: map[string]string{ + "Auto-Submitted": "yes", }, Expected: true, }, { - Headers: map[string][]string{ - "X-Autoreply": {"no"}, + Headers: map[string]string{ + "X-Autoreply": "no", }, Expected: false, }, { - Headers: map[string][]string{ - "X-Autoreply": {"yes"}, + Headers: map[string]string{ + "X-Autoreply": "yes", }, Expected: true, }, { - Headers: map[string][]string{ - "X-Autorespond": {"yes"}, + Headers: map[string]string{ + "X-Autorespond": "yes", }, Expected: true, }, } for _, c := range cases { - assert.Equal(t, c.Expected, isAutomaticReply(mail.HeaderFromMap(c.Headers))) + b := enmime.Builder(). + From("Dummy", "dummy@gitea.io"). + To("Dummy", "dummy@gitea.io") + for k, v := range c.Headers { + b = b.Header(k, v) + } + root, err := b.Build() + assert.NoError(t, err) + env, err := enmime.EnvelopeFromPart(root) + assert.NoError(t, err) + + assert.Equal(t, c.Expected, isAutomaticReply(env)) } } @@ -76,14 +87,14 @@ func TestGetContentFromMailReader(t *testing.T) { "attachment content\r\n" + "--message-boundary--\r\n" - mr, err := mail.CreateReader(strings.NewReader(mailString)) + env, err := enmime.ReadEnvelope(strings.NewReader(mailString)) assert.NoError(t, err) - content, err := getContentFromMailReader(mr) + content, err := getContentFromMailReader(env) assert.NoError(t, err) assert.Equal(t, "mail content", content.Content) assert.Len(t, content.Attachments, 1) assert.Equal(t, "attachment.txt", content.Attachments[0].Name) - assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content.Bytes()) + assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content) mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + "\r\n" + @@ -98,9 +109,9 @@ func TestGetContentFromMailReader(t *testing.T) { "--text-boundary--\r\n" + "--message-boundary--\r\n" - mr, err = mail.CreateReader(strings.NewReader(mailString)) + env, err = enmime.ReadEnvelope(strings.NewReader(mailString)) assert.NoError(t, err) - content, err = getContentFromMailReader(mr) + content, err = getContentFromMailReader(env) assert.NoError(t, err) assert.Equal(t, "mail content", content.Content) assert.Empty(t, content.Attachments) @@ -120,9 +131,9 @@ func TestGetContentFromMailReader(t *testing.T) { "--text-boundary--\r\n" + "--message-boundary--\r\n" - mr, err = mail.CreateReader(strings.NewReader(mailString)) + env, err = enmime.ReadEnvelope(strings.NewReader(mailString)) assert.NoError(t, err) - content, err = getContentFromMailReader(mr) + content, err = getContentFromMailReader(env) assert.NoError(t, err) assert.Equal(t, "mail content without signature", content.Content) assert.Empty(t, content.Attachments) diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go index 41168175f0876..5af61ca83b351 100644 --- a/services/mailer/token/token.go +++ b/services/mailer/token/token.go @@ -17,7 +17,7 @@ import ( const ( tokenVersion1 byte = 1 - tokenLifetimeInYears = 1 + tokenLifetimeInYears int = 1 ) type HandlerType byte diff --git a/tests/integration/incoming_email_test.go b/tests/integration/incoming_email_test.go index 533b055df91c6..b6496d2191407 100644 --- a/tests/integration/incoming_email_test.go +++ b/tests/integration/incoming_email_test.go @@ -4,7 +4,6 @@ package integration import ( - "bytes" "io" "net" "net/smtp" @@ -86,7 +85,7 @@ func TestIncomingEmail(t *testing.T) { Attachments: []*incoming.Attachment{ { Name: "attachment.txt", - Content: *bytes.NewBuffer([]byte("test")), + Content: []byte("test"), }, }, } @@ -127,7 +126,7 @@ func TestIncomingEmail(t *testing.T) { Attachments: []*incoming.Attachment{ { Name: "attachment.txt", - Content: *bytes.NewBuffer([]byte("test")), + Content: []byte("test"), }, }, } From f7e74bb38e28c68c6255ce81e692c76c46e72355 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 12 Dec 2022 14:55:49 +0000 Subject: [PATCH 05/12] Add example ini. --- custom/conf/app.example.ini | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 8824d31a4ae52..d2a4fa7ba0be9 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1658,6 +1658,47 @@ ROUTER = console ;; convert \r\n to \n for Sendmail ;SENDMAIL_CONVERT_CRLF = true +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[incoming_email] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Enable handling of incoming emails. +;ENABLED = false +;; +;; The email address including the %{token} placeholder that will be replaced per user/action. +;; Example: incoming+%{token}@example.com +;; The placeholder must appear in the user part of the address (before the @). +;REPLY_TO_ADDRESS = +;; +;; IMAP server host +;HOST = +;; +;; IMAP server port +;PORT = +;; +;; Username of the receiving account +;USERNAME = +;; +;; Password of the receiving account +;PASSWORD = +;; +;; Whether the IMAP server uses TLS. +;USE_TLS = false +;; +;; If set to true, completely ignores server certificate validation errors. This option is unsafe. +;SKIP_TLS_VERIFY = true +;; +;; The mailbox name where incoming mail will end up. +;MAILBOX = INBOX +;; +;; Whether handled messages should be deleted from the mailbox. +;DELETE_HANDLED_MESSAGE = true +;; +;; Maximum size of a message to handle. Bigger messages are ignored. +;MAXIMUM_MESSAGE_SIZE = 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[cache] From b82efefe3606fc019ae9cb097f7e18ba2ad8a9d8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 12 Dec 2022 15:43:09 +0000 Subject: [PATCH 06/12] Fix merge errors. --- services/mailer/incoming/incoming_handler.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index 5ef1fa4298d4f..2f39c9487db36 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/upload" attachment_service "code.gitea.io/gitea/services/attachment" - comment_service "code.gitea.io/gitea/services/comments" + issue_service "code.gitea.io/gitea/services/issue" incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" "code.gitea.io/gitea/services/mailer/token" pull_service "code.gitea.io/gitea/services/pull" @@ -83,7 +83,11 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u attachmentIDs := make([]string, 0, len(content.Attachments)) if setting.Attachment.Enabled { for _, attachment := range content.Attachments { - a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), user.ID, issue.Repo.ID, 0, attachment.Name, setting.Attachment.AllowedTypes) + a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{ + Name: attachment.Name, + UploaderID: user.ID, + RepoID: issue.Repo.ID, + }) if err != nil { if upload.IsErrFileTypeForbidden(err) { log.Debug("Skipping disallowed attachment type") @@ -95,7 +99,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u } } - _, err = comment_service.CreateIssueComment(ctx, user, issue.Repo, issue, content.Content, attachmentIDs) + _, err = issue_service.CreateIssueComment(ctx, user, issue.Repo, issue, content.Content, attachmentIDs) if err != nil { return fmt.Errorf("CreateIssueComment failed: %w", err) } From d85d4485022d9dc4de29b946ee4abcfcbbc73d7d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 14 Dec 2022 09:31:28 +0000 Subject: [PATCH 07/12] Add missing import. --- services/mailer/incoming/incoming_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index 2f39c9487db36..a140ec547c57d 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -10,6 +10,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" From 28a34fe6bda6c922f0b59330c3efaf07e609deec Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Dec 2022 20:51:06 +0100 Subject: [PATCH 08/12] Apply suggestions from code review Co-authored-by: silverwind --- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index f00bdab4e0dc6..010535433374f 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -747,7 +747,7 @@ and ## Incoming Email (`incoming_email`) - `ENABLED`: **false**: Enable handling of incoming emails. -- `REPLY_TO_ADDRESS`: **\**: # The email address including the %{token} placeholder that will be replaced per user/action. Example: `incoming+%{token}@example.com`. The placeholder must appear in the user part of the address (before the `@`). +- `REPLY_TO_ADDRESS`: **\**: The email address including the `%{token}` placeholder that will be replaced per user/action. Example: `incoming+%{token}@example.com`. The placeholder must appear in the user part of the address (before the `@`). - `HOST`: **\**: IMAP server host. - `PORT`: **\**: IMAP server port. - `USERNAME`: **\**: Username of the receiving account. From 80c1466fa59e8a35ba64719fc8886bc0488ee902 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Dec 2022 19:54:32 +0000 Subject: [PATCH 09/12] Log disallowed attachment name. --- services/mailer/incoming/incoming_handler.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index a140ec547c57d..14414e3d1c03e 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -39,10 +39,6 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u return fmt.Errorf("user needed") } - if content.Content == "" && len(content.Attachments) == 0 { - return nil - } - ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) if err != nil { return err @@ -91,7 +87,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u }) if err != nil { if upload.IsErrFileTypeForbidden(err) { - log.Debug("Skipping disallowed attachment type") + log.Info("Skipping disallowed attachment type: %s", attachment.Name) continue } return err @@ -100,6 +96,10 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u } } + if content.Content == "" && len(attachmentIDs) == 0 { + return nil + } + _, err = issue_service.CreateIssueComment(ctx, user, issue.Repo, issue, content.Content, attachmentIDs) if err != nil { return fmt.Errorf("CreateIssueComment failed: %w", err) @@ -107,6 +107,10 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u case *issues_model.Comment: comment := r + if content.Content == "" { + return nil + } + if comment.Type == issues_model.CommentTypeCode { _, err := pull_service.CreateCodeComment( ctx, From b0a2c8fb7300ed2e80217450babf2a8bc88ff5ca Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 13 Jan 2023 15:41:25 +0000 Subject: [PATCH 10/12] Add suggested changes. --- custom/conf/app.example.ini | 6 +- .../doc/advanced/config-cheat-sheet.en-us.md | 4 +- .../content/doc/usage/incoming-email.en-us.md | 2 +- modules/setting/incoming_email.go | 5 +- services/mailer/incoming/incoming.go | 88 ++++++++++--------- services/mailer/incoming/incoming_handler.go | 31 +++---- services/mailer/incoming/payload/payload.go | 11 ++- services/mailer/mail_test.go | 2 +- services/mailer/token/token.go | 16 +++- tests/integration/incoming_email_test.go | 31 ++++--- tests/mysql.ini.tmpl | 2 +- 11 files changed, 108 insertions(+), 90 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index efdadfe5b59b5..3233135e9da4d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1666,7 +1666,7 @@ ROUTER = console ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;[incoming_email] +;[email.incoming] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -1702,8 +1702,8 @@ ROUTER = console ;; Whether handled messages should be deleted from the mailbox. ;DELETE_HANDLED_MESSAGE = true ;; -;; Maximum size of a message to handle. Bigger messages are ignored. -;MAXIMUM_MESSAGE_SIZE = 0 +;; Maximum size of a message to handle. Bigger messages are ignored. Set to 0 to allow every size. +;MAXIMUM_MESSAGE_SIZE = 10485760 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 979b2d85d3d3b..7dbcad6b40f6d 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -750,7 +750,7 @@ and - `SEND_BUFFER_LEN`: **100**: Buffer length of mailing queue. **DEPRECATED** use `LENGTH` in `[queue.mailer]` - `SEND_AS_PLAIN_TEXT`: **false**: Send mails only in plain text, without HTML alternative. -## Incoming Email (`incoming_email`) +## Incoming Email (`email.incoming`) - `ENABLED`: **false**: Enable handling of incoming emails. - `REPLY_TO_ADDRESS`: **\**: The email address including the `%{token}` placeholder that will be replaced per user/action. Example: `incoming+%{token}@example.com`. The placeholder must appear in the user part of the address (before the `@`). @@ -762,7 +762,7 @@ and - `SKIP_TLS_VERIFY`: **false**: If set to `true`, completely ignores server certificate validation errors. This option is unsafe. - `MAILBOX`: **INBOX**: The mailbox name where incoming mail will end up. - `DELETE_HANDLED_MESSAGE`: **true**: Whether handled messages should be deleted from the mailbox. -- `MAXIMUM_MESSAGE_SIZE`: **0**: Maximum size of a message to handle. Bigger messages are ignored. +- `MAXIMUM_MESSAGE_SIZE`: **10485760**: Maximum size of a message to handle. Bigger messages are ignored. Set to 0 to allow every size. ## Cache (`cache`) diff --git a/docs/content/doc/usage/incoming-email.en-us.md b/docs/content/doc/usage/incoming-email.en-us.md index f7b8ff5a7df13..9f1f28e6681a3 100644 --- a/docs/content/doc/usage/incoming-email.en-us.md +++ b/docs/content/doc/usage/incoming-email.en-us.md @@ -31,7 +31,7 @@ Gitea tries to detect automatic responses to skip and the email server should be ## Configuration -To activate the handling of incoming email messages you have to configure the `incoming_email` section in the configuration file. +To activate the handling of incoming email messages you have to configure the `email.incoming` section in the configuration file. The `REPLY_TO_ADDRESS` contains the address an email client will respond to. This address needs to contain the `%{token}` placeholder which will be replaced with a token describing the user/action. diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go index b802c7cc5bf46..b6a637bccdc0a 100644 --- a/modules/setting/incoming_email.go +++ b/modules/setting/incoming_email.go @@ -28,11 +28,12 @@ var IncomingEmail = struct { Mailbox: "INBOX", DeleteHandledMessage: true, TokenPlaceholder: "%{token}", + MaximumMessageSize: 10485760, } func newIncomingEmail() { - if err := Cfg.Section("incoming_email").MapTo(&IncomingEmail); err != nil { - log.Fatal("Unable to map [incoming_email] section on to IncomingEmail. Error: %v", err) + if err := Cfg.Section("email.incoming").MapTo(&IncomingEmail); err != nil { + log.Fatal("Unable to map [email.incoming] section on to IncomingEmail. Error: %v", err) } if !IncomingEmail.Enabled { diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go index 9e46ff154128c..4e61dfef910d1 100644 --- a/services/mailer/incoming/incoming.go +++ b/services/mailer/incoming/incoming.go @@ -45,6 +45,9 @@ func Init(ctx context.Context) error { ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Incoming Email", process.SystemProcessType, true) defer finished() + // This background job processes incoming emails. It uses the IMAP IDLE command to get notified about incoming emails. + // The following loop restarts the processing logic after errors until ctx indicates to stop. + for { select { case <-ctx.Done(): @@ -77,15 +80,15 @@ func processIncomingEmails(ctx context.Context) error { c, err = client.Dial(server) } if err != nil { - return fmt.Errorf("connected failed: %w", err) + return fmt.Errorf("could not connect to server '%s': %w", server, err) } if err := c.Login(setting.IncomingEmail.Username, setting.IncomingEmail.Password); err != nil { - return fmt.Errorf("login failed: %w", err) + return fmt.Errorf("could not login: %w", err) } defer func() { if err := c.Logout(); err != nil { - log.Error("Logout failed: %v", err) + log.Error("Logout from incoming email server failed: %v", err) } }() @@ -93,13 +96,16 @@ func processIncomingEmails(ctx context.Context) error { return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err) } + // The following loop processes messages. If there are no messages available, IMAP IDLE is used to wait for new messages. + // This process is repeated until an IMAP error occurs or ctx indicates to stop. + for { select { case <-ctx.Done(): return nil default: if err := processMessages(ctx, c); err != nil { - return fmt.Errorf("do it failed: %w", err) + return fmt.Errorf("could not process messages: %w", err) } if err := waitForUpdates(ctx, c); err != nil { return fmt.Errorf("wait for updates failed: %w", err) @@ -153,21 +159,12 @@ func waitForUpdates(ctx context.Context, c *client.Client) error { // processMessages searches unread mails and processes them. func processMessages(ctx context.Context, c *client.Client) error { - mbox, err := c.Select(setting.IncomingEmail.Mailbox, false) - if err != nil { - return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err) - } - - if mbox.Messages == 0 { - return nil - } - criteria := imap.NewSearchCriteria() criteria.WithoutFlags = []string{imap.SeenFlag} criteria.Smaller = setting.IncomingEmail.MaximumMessageSize ids, err := c.Search(criteria) if err != nil { - return fmt.Errorf("search failed: %w", err) + return fmt.Errorf("imap search failed: %w", err) } if len(ids) == 0 { @@ -204,11 +201,11 @@ loop: []interface{}{imap.DeletedFlag}, nil, ); err != nil { - return fmt.Errorf("store failed: %w", err) + return fmt.Errorf("imap store failed: %w", err) } if err := c.Expunge(nil); err != nil { - return fmt.Errorf("expunge failed: %w", err) + return fmt.Errorf("imap expunge failed: %w", err) } } return nil @@ -217,29 +214,29 @@ loop: err := func() error { r := msg.GetBody(section) if r == nil { - return fmt.Errorf("get body failed: %w", err) + return fmt.Errorf("could not get body from message: %w", err) } env, err := enmime.ReadEnvelope(r) if err != nil { - return fmt.Errorf("read envelope failed: %w", err) + return fmt.Errorf("could not read envelope: %w", err) } if isAutomaticReply(env) { - log.Debug("Skipping automatic reply") + log.Debug("Skipping automatic email reply") return nil } t := searchTokenInHeaders(env) if t == "" { - log.Debug("Token not found") + log.Debug("Incoming email token not found in headers") return nil } handlerType, user, payload, err := token.ExtractToken(ctx, t) if err != nil { if _, ok := err.(*token.ErrToken); ok { - log.Info("Invalid email token: %v", err) + log.Info("Invalid incoming email token: %v", err) return nil } return err @@ -250,13 +247,10 @@ loop: return fmt.Errorf("unexpected handler type: %v", handlerType) } - content, err := getContentFromMailReader(env) - if err != nil { - return fmt.Errorf("getContentFromMailReader failed: %w", err) - } + content := getContentFromMailReader(env) if err := handler.Handle(ctx, content, user, payload); err != nil { - return fmt.Errorf("Handle failed: %w", err) + return fmt.Errorf("could not handle message: %w", err) } handledSet.AddNum(msg.SeqNum) @@ -264,13 +258,13 @@ loop: return nil }() if err != nil { - log.Error("Error while processing message[]: %v", err) + log.Error("Error while processing incoming email[%v]: %v", msg.Uid, err) } } } if err := <-errs; err != nil { - return fmt.Errorf("fetch failed: %w", err) + return fmt.Errorf("imap fetch failed: %w", err) } return nil @@ -294,19 +288,17 @@ func isAutomaticReply(env *enmime.Envelope) bool { func searchTokenInHeaders(env *enmime.Envelope) string { if addressTokenRegex != nil { to, _ := env.AddressList("To") + + token := searchTokenInAddresses(to) + if token != "" { + return token + } + deliveredTo, _ := env.AddressList("Delivered-To") - for _, list := range [][]*net_mail.Address{ - to, - deliveredTo, - } { - for _, address := range list { - match := addressTokenRegex.FindStringSubmatch(address.Address) - if len(match) != 2 { - continue - } - return match[1] - } + token = searchTokenInAddresses(deliveredTo) + if token != "" { + return token } } @@ -334,6 +326,20 @@ func searchTokenInHeaders(env *enmime.Envelope) string { return "" } +// searchTokenInAddresses looks for the token in an address +func searchTokenInAddresses(addresses []*net_mail.Address) string { + for _, address := range addresses { + match := addressTokenRegex.FindStringSubmatch(address.Address) + if len(match) != 2 { + continue + } + + return match[1] + } + + return "" +} + type MailContent struct { Content string Attachments []*Attachment @@ -346,7 +352,7 @@ type Attachment struct { // getContentFromMailReader grabs the plain content and the attachments from the mail. // A potential reply/signature gets stripped from the content. -func getContentFromMailReader(env *enmime.Envelope) (*MailContent, error) { +func getContentFromMailReader(env *enmime.Envelope) *MailContent { attachments := make([]*Attachment, 0, len(env.Attachments)) for _, attachment := range env.Attachments { attachments = append(attachments, &Attachment{ @@ -358,5 +364,5 @@ func getContentFromMailReader(env *enmime.Envelope) (*MailContent, error) { return &MailContent{ Content: reply.FromText(env.Text), Attachments: attachments, - }, nil + } } diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index af11a6d838ec9..173b362a5505f 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/upload" + "code.gitea.io/gitea/modules/util" attachment_service "code.gitea.io/gitea/services/attachment" issue_service "code.gitea.io/gitea/services/issue" incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" @@ -23,7 +24,7 @@ import ( ) type MailHandler interface { - Handle(ctx context.Context, content *MailContent, user *user_model.User, payload []byte) error + Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error } var handlers = map[token.HandlerType]MailHandler{ @@ -34,9 +35,9 @@ var handlers = map[token.HandlerType]MailHandler{ // ReplyHandler handles incoming emails to create a reply from them type ReplyHandler struct{} -func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *user_model.User, payload []byte) error { - if user == nil { - return fmt.Errorf("user needed") +func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error { + if doer == nil { + return util.NewInvalidArgumentErrorf("doer can't be nil") } ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) @@ -58,19 +59,19 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u issue = comment.Issue default: - return fmt.Errorf("unsupported reply reference: %v", ref) + return util.NewInvalidArgumentErrorf("unsupported reply reference: %v", ref) } if err := issue.LoadRepo(ctx); err != nil { return err } - perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, user) + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer) if err != nil { return err } - if !perm.CanWriteIssuesOrPulls(issue.IsPull) || issue.IsLocked && !user.IsAdmin { + if !perm.CanWriteIssuesOrPulls(issue.IsPull) || issue.IsLocked && !doer.IsAdmin { log.Debug("can't write issue or pull") return nil } @@ -82,7 +83,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u for _, attachment := range content.Attachments { a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{ Name: attachment.Name, - UploaderID: user.ID, + UploaderID: doer.ID, RepoID: issue.Repo.ID, }) if err != nil { @@ -100,7 +101,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u return nil } - _, err = issue_service.CreateIssueComment(ctx, user, issue.Repo, issue, content.Content, attachmentIDs) + _, err = issue_service.CreateIssueComment(ctx, doer, issue.Repo, issue, content.Content, attachmentIDs) if err != nil { return fmt.Errorf("CreateIssueComment failed: %w", err) } @@ -114,7 +115,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u if comment.Type == issues_model.CommentTypeCode { _, err := pull_service.CreateCodeComment( ctx, - user, + doer, nil, issue, comment.Line, @@ -135,9 +136,9 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, user *u // UnsubscribeHandler handles unwatching issues/pulls type UnsubscribeHandler struct{} -func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, user *user_model.User, payload []byte) error { - if user == nil { - return fmt.Errorf("user needed") +func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, doer *user_model.User, payload []byte) error { + if doer == nil { + return util.NewInvalidArgumentErrorf("doer can't be nil") } ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) @@ -153,7 +154,7 @@ func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, user *u return err } - perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, user) + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer) if err != nil { return err } @@ -163,7 +164,7 @@ func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, user *u return nil } - return issues_model.CreateOrUpdateIssueWatch(user.ID, issue.ID, false) + return issues_model.CreateOrUpdateIssueWatch(doer.ID, issue.ID, false) } return fmt.Errorf("unsupported unsubscribe reference: %v", ref) diff --git a/services/mailer/incoming/payload/payload.go b/services/mailer/incoming/payload/payload.go index 00a096ce806c6..eb82f5c3ed321 100644 --- a/services/mailer/incoming/payload/payload.go +++ b/services/mailer/incoming/payload/payload.go @@ -5,7 +5,6 @@ package payload import ( "context" - "fmt" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/modules/util" @@ -33,7 +32,7 @@ func CreateReferencePayload(reference interface{}) ([]byte, error) { refType = payloadReferenceComment refID = r.ID default: - return nil, fmt.Errorf("unsupported reference type") + return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", r) } payload, err := util.PackData(refType, refID) @@ -47,11 +46,11 @@ func CreateReferencePayload(reference interface{}) ([]byte, error) { // GetReferenceFromPayload resolves the reference from the payload func GetReferenceFromPayload(ctx context.Context, payload []byte) (interface{}, error) { if len(payload) < 1 { - return nil, fmt.Errorf("payload to small") + return nil, util.NewInvalidArgumentErrorf("payload to small") } if payload[0] != replyPayloadVersion1 { - return nil, fmt.Errorf("unsupported payload version") + return nil, util.NewInvalidArgumentErrorf("unsupported payload version") } var ref payloadReferenceType @@ -65,7 +64,7 @@ func GetReferenceFromPayload(ctx context.Context, payload []byte) (interface{}, return issues_model.GetIssueByID(ctx, id) case payloadReferenceComment: return issues_model.GetCommentByID(ctx, id) + default: + return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", ref) } - - return nil, fmt.Errorf("unsupported reference type") } diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index baf65ff861528..64f2f740ca2db 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -311,7 +311,7 @@ func Test_createReference(t *testing.T) { comment: comment, actionType: activities_model.ActionCommentPull, }, - prefix: fmt.Sprintf("<%s/pulls/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), + prefix: fmt.Sprintf("<%s/pulls/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), }, { name: "Close Issue", diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go index bb711256c20b0..c2e5620a99c56 100644 --- a/services/mailer/token/token.go +++ b/services/mailer/token/token.go @@ -15,6 +15,14 @@ import ( "code.gitea.io/gitea/modules/util" ) +// A token is a verifyable container describing an action. +// +// A token has a dynamic length depending on the contained data and has the following structure: +// | Token Version | User ID | HMAC | Payload | +// +// The payload is verifyable by the generated HMAC using the user secret. It contains: +// | Timestamp | Action/Handler Type | Action/Handler Data | + const ( tokenVersion1 byte = 1 tokenLifetimeInYears int = 1 @@ -34,8 +42,12 @@ type ErrToken struct { context string } -func (p *ErrToken) Error() string { - return "invalid email token: " + p.context +func (err *ErrToken) Error() string { + return "invalid email token: " + err.context +} + +func (err *ErrToken) Unwrap() error { + return util.ErrInvalidArgument } // CreateToken creates a token for the action/user tuple diff --git a/tests/integration/incoming_email_test.go b/tests/integration/incoming_email_test.go index fdb93c48419c4..b4478f57809fa 100644 --- a/tests/integration/incoming_email_test.go +++ b/tests/integration/incoming_email_test.go @@ -80,6 +80,13 @@ func TestIncomingEmail(t *testing.T) { defer tests.PrintCurrentTest(t)() handler := &incoming.ReplyHandler{} + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + + assert.Error(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, nil, payload)) + assert.NoError(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, user, payload)) + content := &incoming.MailContent{ Content: "reply by mail", Attachments: []*incoming.Attachment{ @@ -90,12 +97,6 @@ func TestIncomingEmail(t *testing.T) { }, } - payload, err := incoming_payload.CreateReferencePayload(issue) - assert.NoError(t, err) - - assert.Error(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, nil, payload)) - assert.NoError(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, user, payload)) - assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ @@ -175,6 +176,8 @@ func TestIncomingEmail(t *testing.T) { }) if setting.IncomingEmail.Enabled { + // This test connects to the configured email server and is currently only enabled for MySql integration tests. + // It sends a reply to create a comment. If the comment is not detected after 10 seconds the test fails. t.Run("IMAP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -190,27 +193,23 @@ func TestIncomingEmail(t *testing.T) { err = gomail.Send(&smtpTestSender{}, msg) assert.NoError(t, err) - for i := 0; i < 5; i++ { - time.Sleep(1 * time.Second) - + assert.Eventually(t, func() bool { comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ IssueID: issue.ID, Type: issues_model.CommentTypeComment, }) assert.NoError(t, err) assert.NotEmpty(t, comments) + comment := comments[len(comments)-1] - if comment.Content == token { - assert.Equal(t, user.ID, comment.PosterID) - break - } else if i == 4 { - t.SkipNow() - } - } + + return comment.PosterID == user.ID && comment.Content == token + }, 10*time.Second, 1*time.Second) }) } } +// A simple SMTP mail sender used for integration tests. type smtpTestSender struct{} func (s *smtpTestSender) Send(from string, to []string, msg io.WriterTo) error { diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index e4e47aa4dfc4e..44914d0879621 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -125,7 +125,7 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [packages] ENABLED = true -[incoming_email] +[email.incoming] ENABLED = true HOST = smtpimap PORT = 993 From 93b346949eac7fd2d66a3209c74b4928d0d3d733 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 14 Jan 2023 12:05:06 +0000 Subject: [PATCH 11/12] Fix typo. --- services/mailer/token/token.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go index c2e5620a99c56..8a5a762d6b5fd 100644 --- a/services/mailer/token/token.go +++ b/services/mailer/token/token.go @@ -15,12 +15,12 @@ import ( "code.gitea.io/gitea/modules/util" ) -// A token is a verifyable container describing an action. +// A token is a verifiable container describing an action. // // A token has a dynamic length depending on the contained data and has the following structure: // | Token Version | User ID | HMAC | Payload | // -// The payload is verifyable by the generated HMAC using the user secret. It contains: +// The payload is verifiable by the generated HMAC using the user secret. It contains: // | Timestamp | Action/Handler Type | Action/Handler Data | const ( From 661c367fc76e65a59d5579fb06ba71aae1050839 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 14 Jan 2023 13:35:01 +0000 Subject: [PATCH 12/12] return err --- services/mailer/incoming/incoming.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go index 4e61dfef910d1..2653e80586d71 100644 --- a/services/mailer/incoming/incoming.go +++ b/services/mailer/incoming/incoming.go @@ -33,13 +33,20 @@ func Init(ctx context.Context) error { return nil } - addressTokenRegex = regexp.MustCompile( + var err error + addressTokenRegex, err = regexp.Compile( fmt.Sprintf( `\A%s\z`, strings.Replace(regexp.QuoteMeta(setting.IncomingEmail.ReplyToAddress), regexp.QuoteMeta(setting.IncomingEmail.TokenPlaceholder), "(.+)", 1), ), ) - referenceTokenRegex = regexp.MustCompile(fmt.Sprintf(`\Areply-(.+)@%s\z`, regexp.QuoteMeta(setting.Domain))) + if err != nil { + return err + } + referenceTokenRegex, err = regexp.Compile(fmt.Sprintf(`\Areply-(.+)@%s\z`, regexp.QuoteMeta(setting.Domain))) + if err != nil { + return err + } go func() { ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Incoming Email", process.SystemProcessType, true)