From 301c79d0e8d1c211d2f3c740d41a4dd0a5c50653 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 9 Apr 2020 15:46:39 +0100 Subject: [PATCH 1/5] Set default room version to 4 --- roomserver/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/version/version.go b/roomserver/version/version.go index e60b5ef7a6..f2a67e74df 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -59,7 +59,7 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV2 + return gomatrixserverlib.RoomVersionV4 } // RoomVersions returns a map of all known room versions to this From a6d6fe7349cdbcd3d5be6f180bfe114e1b685af3 Mon Sep 17 00:00:00 2001 From: Cnly Date: Fri, 10 Apr 2020 18:17:29 +0800 Subject: [PATCH 2/5] Default to v1 when room_version key missing Signed-off-by: Alex Chen --- roomserver/storage/postgres/storage.go | 4 +--- roomserver/storage/sqlite3/storage.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index c91c59ebc6..297618c9dd 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,8 +20,6 @@ import ( "database/sql" "encoding/json" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" - // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/api" @@ -146,7 +144,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( if event.Type() != gomatrixserverlib.MRoomCreate { return gomatrixserverlib.RoomVersion(""), nil } - roomVersion = roomserverVersion.DefaultRoomVersion() + roomVersion = gomatrixserverlib.RoomVersionV1 var createContent gomatrixserverlib.CreateContent // The m.room.create event contains an optional "room_version" key in // the event content, so we need to unmarshal that first. diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index f6c692fd10..fdd1d00156 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -22,8 +22,6 @@ import ( "errors" "net/url" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" @@ -175,7 +173,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( if event.Type() != gomatrixserverlib.MRoomCreate { return gomatrixserverlib.RoomVersion(""), nil } - roomVersion = roomserverVersion.DefaultRoomVersion() + roomVersion = gomatrixserverlib.RoomVersionV1 var createContent gomatrixserverlib.CreateContent // The m.room.create event contains an optional "room_version" key in // the event content, so we need to unmarshal that first. From 7615857be072b6616ed6564d054e5d460122a099 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 16 Apr 2020 10:19:57 +0100 Subject: [PATCH 3/5] Squashed commit of the following: commit c1bca95adbe04a17d8feb91aa24dde9c4cffb1fb Author: Kegsay Date: Thu Apr 16 10:06:55 2020 +0100 Add SQL tracing via DENDRITE_TRACE_SQL (#968) * Add SQL tracing via DENDRITE_TRACE_SQL Add this to `internal/sqlutil` in preparation for #897 * Not entirely commit c2ea96190944d8e39b0c200e43b1eedfeeeb49a6 Author: Kegsay Date: Wed Apr 15 17:48:40 2020 +0100 Add HTTP trace logging (#965) * Dump all requests/response server-side * Wrap in DENDRITE_TRACE * DENDRITE_TRACE_HTTP is better * Bugfix for response body and linting * False is true and true is false * Linting * How did this get missed * More linting --- appservice/storage/postgres/storage.go | 3 +- appservice/storage/sqlite3/storage.go | 3 +- .../auth/storage/accounts/postgres/storage.go | 3 +- .../auth/storage/accounts/sqlite3/storage.go | 3 +- .../auth/storage/devices/postgres/storage.go | 3 +- .../auth/storage/devices/sqlite3/storage.go | 3 +- common/basecomponent/base.go | 5 +- common/httpapi.go | 55 +++++++++++- common/keydb/postgres/keydb.go | 4 +- common/keydb/sqlite3/keydb.go | 4 +- federationsender/storage/postgres/storage.go | 3 +- federationsender/storage/sqlite3/storage.go | 3 +- go.mod | 2 + go.sum | 5 ++ internal/sqlutil/trace.go | 89 +++++++++++++++++++ internal/sqlutil/trace_driver.go | 35 ++++++++ internal/sqlutil/trace_driver_wasm.go | 33 +++++++ mediaapi/storage/postgres/storage.go | 3 +- mediaapi/storage/sqlite3/storage.go | 3 +- publicroomsapi/storage/postgres/storage.go | 3 +- publicroomsapi/storage/sqlite3/storage.go | 3 +- roomserver/storage/postgres/storage.go | 4 +- roomserver/storage/sqlite3/storage.go | 4 +- syncapi/storage/postgres/syncserver.go | 3 +- syncapi/storage/sqlite3/syncserver.go | 3 +- 25 files changed, 259 insertions(+), 23 deletions(-) create mode 100644 internal/sqlutil/trace.go create mode 100644 internal/sqlutil/trace_driver.go create mode 100644 internal/sqlutil/trace_driver_wasm.go diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index ef92db8719..e145eeee23 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index d0538e263d..0cd1e4abc2 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( // Import SQLite database driver "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" ) @@ -36,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 4a0a2060b8..8ce367a3e9 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -46,7 +47,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index bfb7b4ea1f..e190ba6c28 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -49,7 +50,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 221c3998ea..3f613cf325 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +39,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 62c30322ea..85a8def2c3 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -40,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index dc27f54094..de3e04c216 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -24,6 +24,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -243,7 +244,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) + db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } @@ -253,7 +254,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sql.Open("postgres", string(cfg.Database.Naffka)) + db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/httpapi.go b/common/httpapi.go index 843336f5bd..e5324bd171 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -1,7 +1,12 @@ package common import ( + "io" "net/http" + "net/http/httptest" + "net/http/httputil" + "os" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/auth" @@ -46,12 +51,60 @@ func MakeAuthAPI( // MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler. // This is used for APIs that are called from the internet. func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler { + // TODO: We shouldn't be directly reading env vars here, inject it in instead. + // Refactor this when we split out config structs. + verbose := false + if os.Getenv("DENDRITE_TRACE_HTTP") == "1" { + verbose = true + } h := util.MakeJSONAPI(util.NewJSONRequestHandler(f)) withSpan := func(w http.ResponseWriter, req *http.Request) { + nextWriter := w + if verbose { + logger := logrus.NewEntry(logrus.StandardLogger()) + // Log outgoing response + rec := httptest.NewRecorder() + nextWriter = rec + defer func() { + resp := rec.Result() + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + logger.Debugf("Failed to dump outgoing response: %s", err) + } else { + strSlice := strings.Split(string(dump), "\n") + for _, s := range strSlice { + logger.Debug(s) + } + } + // copy the response to the client + for hdr, vals := range resp.Header { + for _, val := range vals { + w.Header().Add(hdr, val) + } + } + w.WriteHeader(resp.StatusCode) + // discard errors as this is for debugging + _, _ = io.Copy(w, resp.Body) + _ = resp.Body.Close() + }() + + // Log incoming request + dump, err := httputil.DumpRequest(req, true) + if err != nil { + logger.Debugf("Failed to dump incoming request: %s", err) + } else { + strSlice := strings.Split(string(dump), "\n") + for _, s := range strSlice { + logger.Debug(s) + } + } + } + span := opentracing.StartSpan(metricsName) defer span.Finish() req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span)) - h.ServeHTTP(w, req) + h.ServeHTTP(nextWriter, req) + } return http.HandlerFunc(withSpan) diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 2dd8c866f4..2879683e0f 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -17,11 +17,11 @@ package postgres import ( "context" - "database/sql" "math" "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -41,7 +41,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open("postgres", dataSourceName) + db, err := sqlutil.Open("postgres", dataSourceName) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 3c860d0c3b..82d2a491fc 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -17,12 +17,12 @@ package sqlite3 import ( "context" - "database/sql" "math" "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open(common.SQLiteDriverName(), dataSourceName) + db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName) if err != nil { return nil, err } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index d97b5d2965..b909a189bd 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // Database stores information needed by the federation sender @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 6ab9ae00a0..458d7d7e58 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // Database stores information needed by the federation sender @@ -37,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/go.mod b/go.mod index 20235ae349..953b49d324 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,12 @@ require ( github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 + github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.1.0 github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 + github.com/prometheus/common v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect diff --git a/go.sum b/go.sum index 5b4579bb07..f7295c38bf 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,10 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWso github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -452,6 +454,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww= +github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -688,6 +692,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go new file mode 100644 index 0000000000..3d5fa7dc7e --- /dev/null +++ b/internal/sqlutil/trace.go @@ -0,0 +1,89 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlutil + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/ngrok/sqlmw" + "github.com/sirupsen/logrus" +) + +var tracingEnabled = os.Getenv("DENDRITE_TRACE_SQL") == "1" + +type traceInterceptor struct { + sqlmw.NullInterceptor +} + +func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) { + startedAt := time.Now() + rows, err := stmt.QueryContext(ctx, args) + + logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) + + return rows, err +} + +func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) { + startedAt := time.Now() + result, err := stmt.ExecContext(ctx, args) + + logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) + + return result, err +} + +func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest []driver.Value) error { + err := rows.Next(dest) + if err == io.EOF { + // For all cases, we call Next() n+1 times, the first to populate the initial dest, then eventually + // it will io.EOF. If we log on each Next() call we log the last element twice, so don't. + return err + } + cols := rows.Columns() + logrus.Debug(strings.Join(cols, " | ")) + + b := strings.Builder{} + for i, val := range dest { + b.WriteString(fmt.Sprintf("%v", val)) + if i+1 <= len(dest)-1 { + b.WriteString(" | ") + } + } + logrus.Debug(b.String()) + return err +} + +// Open opens a database specified by its database driver name and a driver-specific data source name, +// usually consisting of at least a database name and connection information. Includes tracing driver +// if DENDRITE_TRACE_SQL=1 +func Open(driverName, dsn string) (*sql.DB, error) { + if tracingEnabled { + // install the wrapped driver + driverName += "-trace" + } + return sql.Open(driverName, dsn) +} + +func init() { + registerDrivers() +} diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go new file mode 100644 index 0000000000..f123b1e4d2 --- /dev/null +++ b/internal/sqlutil/trace_driver.go @@ -0,0 +1,35 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package sqlutil + +import ( + "database/sql" + + "github.com/lib/pq" + sqlite "github.com/mattn/go-sqlite3" + "github.com/ngrok/sqlmw" +) + +func registerDrivers() { + if !tracingEnabled { + return + } + // install the wrapped drivers + sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) + sql.Register("sqlite3-trace", sqlmw.Driver(&sqlite.SQLiteDriver{}, new(traceInterceptor))) + +} diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go new file mode 100644 index 0000000000..a3c163f505 --- /dev/null +++ b/internal/sqlutil/trace_driver_wasm.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package sqlutil + +import ( + "database/sql" + + sqlitejs "github.com/matrix-org/go-sqlite3-js" + "github.com/ngrok/sqlmw" +) + +func registerDrivers() { + if !tracingEnabled { + return + } + // install the wrapped drivers + sql.Register("sqlite3_js-trace", sqlmw.Driver(&sqlitejs.SqliteJsDriver{}, new(traceInterceptor))) + +} diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 6259f4a139..18126b151c 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,7 +36,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index cfa5bb9868..abafecf20a 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -36,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5a4bc8f9a7..8c4660ccac 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +39,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 80c04cab77..1216016286 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -40,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 297618c9dd..6f2b966108 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,6 +20,8 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal/sqlutil" + // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/api" @@ -37,7 +39,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index fdd1d00156..444a8fdd5b 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -22,6 +22,8 @@ import ( "errors" "net/url" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" @@ -50,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index ead1bf335d..7fd75f066f 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" // Import the postgres database driver. @@ -62,7 +63,7 @@ type SyncServerDatasource struct { func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 30f77e54db..29051cd068 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -27,6 +27,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" // Import the sqlite3 package @@ -78,7 +79,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } if err = d.prepare(); err != nil { From 2ba1159fe795dea6b242ed13cda707d206a4927e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 16 Apr 2020 12:30:39 +0100 Subject: [PATCH 4/5] Update gomatrixserverlib --- go.mod | 3 +-- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 953b49d324..ea45bf9260 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible @@ -27,7 +27,6 @@ require ( github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/prometheus/common v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect diff --git a/go.sum b/go.sum index f7295c38bf..f4b4e4ff8d 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 h1:YiXBJ/0ZeBzuh9Ym0iYaJgDBlFdz7nIVKArqkkEgPzM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea h1:aiBD966UX0l4rTPgAu+u8CaiMfy+N8qWlZ2TIBp+0eM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= From 8d753a24cc5097e1b569e3cfdcbea01243f82448 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 16 Apr 2020 12:35:41 +0100 Subject: [PATCH 5/5] Remove unneeded imports --- roomserver/storage/postgres/storage.go | 1 - roomserver/storage/sqlite3/storage.go | 1 - 2 files changed, 2 deletions(-) diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 207544572c..6f2b966108 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal/sqlutil" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. _ "github.com/lib/pq" diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 8ea26104f3..444a8fdd5b 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( "net/url" "github.com/matrix-org/dendrite/internal/sqlutil" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api"