From 048887b765d889ef2a80afcd33e667a4c418ef74 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 23 Jul 2021 00:18:53 -0500 Subject: [PATCH 01/13] Update MSC2716 federation test cases --- tests/msc2716_test.go | 55 +++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 7f3777b7..f5eae2a1 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -76,8 +76,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -190,8 +191,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -240,8 +242,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -290,8 +293,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) batchSendHistoricalMessages( @@ -315,8 +319,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -349,8 +354,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -407,8 +413,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -483,8 +490,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -521,9 +529,9 @@ func TestBackfillingHistory(t *testing.T) { historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] - // [1 insertion event + 2 historical events + 1 insertion event] - if len(historicalEventIDs) != 4 { - t.Fatalf("Expected eventID list should be length 15 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] + if len(historicalEventIDs) != 5 { + t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -571,12 +579,12 @@ func TestBackfillingHistory(t *testing.T) { return ev.Get("event_id").Str == markerEventID }) - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) - must.MatchResponse(t, messagesRes, match.HTTPResponse{ + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str @@ -590,8 +598,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) From d385044820e9bfc5c63da6238971f0d50e76fe84 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 26 Jul 2021 16:35:59 -0500 Subject: [PATCH 02/13] Uncomment some federated test cases --- tests/msc2716_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index f5eae2a1..731538cb 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -350,7 +350,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ @@ -409,7 +409,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ From faec5cac9db968525146b510d311b9ec650cd50e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 20:58:26 -0500 Subject: [PATCH 03/13] More proper waiting to avoid flakey test --- tests/msc2716_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b9b62b87..39676319 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -337,7 +337,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -392,7 +391,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -465,7 +463,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -484,7 +482,7 @@ func TestBackfillingHistory(t *testing.T) { // Mimic scrollback just through the latest messages remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, - // Limited so we can only see a few of the latest messages + // Limited so we can only see a portion of the latest messages "limit": []string{"5"}, })) @@ -554,6 +552,16 @@ func TestBackfillingHistory(t *testing.T) { return ev.Get("event_id").Str == markerEventID }) + // Make sure all of the base insertion event has been backfilled + // after the marker was received + fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { + if ev.Get("event_id").Str == baseInsertionEventID { + return true + } + + return false + }) + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, From edade9c932539b7485c26f177225afa275a27f37 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 21:19:17 -0500 Subject: [PATCH 04/13] Get both tests working and de-duplicate marker sending verifying --- tests/msc2716_test.go | 101 +++++++++++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 39676319..e5961e1e 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -534,39 +534,15 @@ func TestBackfillingHistory(t *testing.T) { }, }) - // Send a marker event to let all of the homeservers know about the - // insertion point where all of the historical messages are at - markerEvent := b.Event{ - Type: markerEventType, - Content: map[string]interface{}{ - markerInsertionContentField: baseInsertionEventID, - }, - } - // We can't use as.SendEventSynced(...) because application services can't use the /sync API - markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) - markerSendBody := client.ParseJSON(t, markerSendRes) - markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") - - // Make sure the marker event has reached the remote homeserver - remoteCharlie.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { - return ev.Get("event_id").Str == markerEventID - }) - - // Make sure all of the base insertion event has been backfilled - // after the marker was received - fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { - if ev.Get("event_id").Str == baseInsertionEventID { - return true - } - - return false - }) + // Send the marker event + sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) + // Make sure all of the historical messages are visible when we scrollback again must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { @@ -577,7 +553,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -615,15 +591,48 @@ func TestBackfillingHistory(t *testing.T) { ) batchSendResBody := client.ParseJSON(t, batchSendRes) historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] - // TODO: Send marker event + // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] + if len(historicalEventIDs) != 5 { + t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + } - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) + beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody) + // Since the original body can only be read once, create a new one from the body bytes we just read + beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) + // Make sure the history isn't visible before we expect it to be there. + // This is to avoid some bug in the homeserver using some unknown + // mechanism to distribute the historical messages to other homeservers. + must.MatchResponse(t, beforeMarkerMessagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONArrayEach("chunk", func(r gjson.Result) error { + // Throw if we find one of the historical events in the message response + for _, historicalEventID := range historicalEventIDs { + if r.Get("event_id").Str == historicalEventID { + return fmt.Errorf("Historical event (%s) found on remote homeserver before marker event was sent out\nmessage response (%d): %v\nhistoricalEventIDs (%d): %v", historicalEventID, len(eventDebugStringsFromBeforeMarkerResponse), eventDebugStringsFromBeforeMarkerResponse, len(historicalEventIDs), historicalEventIDs) + } + } + return nil + }), + }, + }) - must.MatchResponse(t, messagesRes, match.HTTPResponse{ + // Send the marker event + sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) + + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + // Make sure all of the historical messages are visible when we scrollback again + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str @@ -742,6 +751,36 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } } +func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { + // Send a marker event to let all of the homeservers know about the + // insertion point where all of the historical messages are at + markerEvent := b.Event{ + Type: markerEventType, + Content: map[string]interface{}{ + markerInsertionContentField: insertionEventID, + }, + } + // We can't use as.SendEventSynced(...) because application services can't use the /sync API + markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) + markerSendBody := client.ParseJSON(t, markerSendRes) + markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") + + // Make sure the marker event has reached the remote homeserver + c.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { + return ev.Get("event_id").Str == markerEventID + }) + + // Make sure all of the base insertion event has been backfilled + // after the marker was received + fetchUntilMessagesResponseHas(t, c, roomID, func(ev gjson.Result) bool { + if ev.Get("event_id").Str == insertionEventID { + return true + } + + return false + }) +} + func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count int) (eventIDs []string) { eventIDs = make([]string, count) for i := 0; i < len(eventIDs); i++ { From b72cc1feca921ccf40d1380b3ce0d382fca60830 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 22:56:45 -0500 Subject: [PATCH 05/13] Make error comment match helper function name --- tests/msc2716_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e5961e1e..e46bd3f0 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -666,7 +666,7 @@ func fetchUntilMessagesResponseHas(t *testing.T, c *client.CSAPI, roomID string, checkCounter := 0 for { if time.Since(start) > c.SyncUntilTimeout { - t.Fatalf("fetchMessagesUntilResponseHas timed out. Called check function %d times", checkCounter) + t.Fatalf("fetchUntilMessagesResponseHas timed out. Called check function %d times", checkCounter) } messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ From 6bc06f7aaff53d6dc4ba845de7f33184211aadf7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 4 Aug 2021 01:57:24 -0500 Subject: [PATCH 06/13] Add test for private room backfilling --- tests/msc2716_test.go | 151 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 13 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e46bd3f0..e281e072 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -46,12 +46,18 @@ var ( markerInsertionContentField = "org.matrix.msc2716.marker.insertion" ) -var createRoomOpts = map[string]interface{}{ +var createPublicRoomOpts = map[string]interface{}{ "preset": "public_chat", "name": "the hangout spot", "room_version": "org.matrix.msc2716", } +var createPrivateRoomOpts = map[string]interface{}{ + "preset": "private_chat", + "name": "the hangout spot", + "room_version": "org.matrix.msc2716", +} + func TestBackfillingHistory(t *testing.T) { deployment := Deploy(t, b.BlueprintHSWithApplicationService) defer deployment.Destroy(t) @@ -82,7 +88,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Backfilled historical events resolve with proper state in correct order", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create some normal messages in the timeline. We're creating them in @@ -193,7 +199,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Backfilled historical events from multiple users in the same chunk", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create the "live" event we are going to insert our backfilled events next to @@ -237,10 +243,10 @@ func TestBackfillingHistory(t *testing.T) { }) }) - t.Run("Backfilled historical events with m.historical do not come down in an incremental sync", func(t *testing.T) { + t.Run("Backfilled historical events do not come down in an incremental sync", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create the "live" event we are going to insert our backfilled events next to @@ -287,7 +293,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Unrecognised prev_event ID will throw an error", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) batchSendHistoricalMessages( t, @@ -309,7 +315,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Normal users aren't allowed to backfill messages", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -331,6 +337,48 @@ func TestBackfillingHistory(t *testing.T) { ) }) + t.Run("Should be able to backfill into private room", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPrivateRoomOpts) + as.InviteRoom(t, roomID, alice.UserID) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + []string{virtualUserID}, + roomID, + eventIdBefore, + timeAfterEventBefore, + "", + 3, + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + + messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) + }) + t.Run("TODO: Test if historical avatar/display name set back in time are picked up on historical messages", func(t *testing.T) { t.Skip("Skipping until implemented") // TODO: Try adding avatar and displayName and see if historical messages get this info @@ -339,7 +387,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -393,7 +441,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -466,7 +514,7 @@ func TestBackfillingHistory(t *testing.T) { //t.Skip("Skipping until federation is implemented") t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Join the room from a remote homeserver before any backfilled messages are sent @@ -556,7 +604,7 @@ func TestBackfillingHistory(t *testing.T) { //t.Skip("Skipping until federation is implemented") t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Join the room from a remote homeserver before any backfilled messages are sent @@ -798,6 +846,72 @@ func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count in return eventIDs } +func asdf( + virtualUserIDs []string, + insertTime time.Time, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + + state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) + for i, virtualUserID := range virtualUserIDs { + inviteEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": c.UserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "invite", + }, + "state_key": virtualUserID, + } + state_evs[2*i] = inviteEvent + + joinEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": virtualUserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "join", + }, + "state_key": virtualUserID, + } + + state_evs[2*i+1] = joinEvent + } + + return state_evs +} + +func createBatchSendMessages( + virtualUserIDs []string, + insertTime time.Time, + count int, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + timeBetweenMessagesMS := uint64(timeBetweenMessages / time.Millisecond) + + evs := make([]map[string]interface{}, count) + for i := 0; i < len(evs); i++ { + virtualUserID := virtualUserIDs[i%len(virtualUserIDs)] + + newEvent := map[string]interface{}{ + "type": "m.room.message", + "sender": virtualUserID, + "origin_server_ts": insertOriginServerTs + (timeBetweenMessagesMS * uint64(i)), + "content": map[string]interface{}{ + "msgtype": "m.text", + "body": fmt.Sprintf("Historical %d (chunk=%d)", i, chunkCount), + historicalContentField: true, + }, + } + + evs[i] = newEvent + } + + return evs +} + var chunkCount int64 = 0 func batchSendHistoricalMessages( @@ -834,8 +948,19 @@ func batchSendHistoricalMessages( evs[i] = newEvent } - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { + inviteEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": c.UserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "invite", + }, + "state_key": virtualUserID, + } + state_evs[2*i] = inviteEvent + joinEvent := map[string]interface{}{ "type": "m.room.member", "sender": virtualUserID, @@ -846,7 +971,7 @@ func batchSendHistoricalMessages( "state_key": virtualUserID, } - state_evs[i] = joinEvent + state_evs[2*i+1] = joinEvent } query := make(url.Values, 2) From 3b2358f99dabdbbc2c89b7c2e36ee86543193455 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 4 Aug 2021 02:26:07 -0500 Subject: [PATCH 07/13] Simplify batchSendHistoricalMessages function signature by using sorta composable functions --- tests/msc2716_test.go | 149 ++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 93 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e281e072..4189815c 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -112,15 +112,15 @@ func TestBackfillingHistory(t *testing.T) { eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) // Insert the most recent chunk of backfilled history + insertTime1 := timeAfterEventBefore.Add(timeBetweenMessages * 3) batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore.Add(timeBetweenMessages*3), "", - 3, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), // Status 200, ) @@ -130,15 +130,15 @@ func TestBackfillingHistory(t *testing.T) { // Insert another older chunk of backfilled history from the same user. // Make sure the meta data and joins still work on the subsequent chunk + insertTime2 := timeAfterEventBefore batchSendRes2 := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, nextChunkID, - 3, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime2), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime2, 3), // Status 200, ) @@ -213,16 +213,17 @@ func TestBackfillingHistory(t *testing.T) { virtualUserID3 := "@carol:hs1" ensureVirtualUserRegistered(t, as, "carol") + virtualUserList := []string{virtualUserID, virtualUserID2, virtualUserID3} + // Insert a backfilled event batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID, virtualUserID2, virtualUserID3}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 3, + createJoinStateEventsForBackfillRequest(virtualUserList, timeAfterEventBefore), + createMessageEventsForBackfillRequest(virtualUserList, timeAfterEventBefore, 3), // Status 200, ) @@ -261,12 +262,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), // Status 200, ) @@ -295,15 +295,15 @@ func TestBackfillingHistory(t *testing.T) { roomID := as.CreateRoom(t, createPublicRoomOpts) + insertTime := time.Now() batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, "$some-non-existant-event-id", - time.Now(), "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime, 1), // Status // TODO: Seems like this makes more sense as a 404 // But the current Synapse code around unknown prev events will throw -> @@ -325,12 +325,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendHistoricalMessages( t, alice, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), // Status // Normal user alice should not be able to backfill messages 403, @@ -349,16 +348,19 @@ func TestBackfillingHistory(t *testing.T) { eventIdBefore := eventIDsBefore[0] timeAfterEventBefore := time.Now() + var stateEvents []map[string]interface{} + stateEvents = append(stateEvents, createInviteStateEventsForBackfillRequest(as.UserID, []string{virtualUserID}, timeAfterEventBefore)...) + stateEvents = append(stateEvents, createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore)...) + // Insert a backfilled event batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 3, + stateEvents, + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 3), // Status 200, ) @@ -400,12 +402,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -472,12 +473,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, chunkId, - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -537,12 +537,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -628,12 +627,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -846,26 +844,41 @@ func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count in return eventIDs } -func asdf( +func createInviteStateEventsForBackfillRequest( + invitedByUserID string, virtualUserIDs []string, insertTime time.Time, ) []map[string]interface{} { // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) + state_evs := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { inviteEvent := map[string]interface{}{ "type": "m.room.member", - "sender": c.UserID, + "sender": invitedByUserID, "origin_server_ts": insertOriginServerTs, "content": map[string]interface{}{ "membership": "invite", }, "state_key": virtualUserID, } - state_evs[2*i] = inviteEvent + state_evs[i] = inviteEvent + } + + return state_evs +} + +func createJoinStateEventsForBackfillRequest( + virtualUserIDs []string, + insertTime time.Time, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + + state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + for i, virtualUserID := range virtualUserIDs { joinEvent := map[string]interface{}{ "type": "m.room.member", "sender": virtualUserID, @@ -876,13 +889,13 @@ func asdf( "state_key": virtualUserID, } - state_evs[2*i+1] = joinEvent + state_evs[i] = joinEvent } return state_evs } -func createBatchSendMessages( +func createMessageEventsForBackfillRequest( virtualUserIDs []string, insertTime time.Time, count int, @@ -917,63 +930,13 @@ var chunkCount int64 = 0 func batchSendHistoricalMessages( t *testing.T, c *client.CSAPI, - virtualUserIDs []string, roomID string, insertAfterEventId string, - insertTime time.Time, chunkID string, - count int, + state_events_at_start []map[string]interface{}, + events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { - // Timestamp in milliseconds - insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - - timeBetweenMessagesMS := uint64(timeBetweenMessages / time.Millisecond) - - evs := make([]map[string]interface{}, count) - for i := 0; i < len(evs); i++ { - virtualUserID := virtualUserIDs[i%len(virtualUserIDs)] - - newEvent := map[string]interface{}{ - "type": "m.room.message", - "sender": virtualUserID, - "origin_server_ts": insertOriginServerTs + (timeBetweenMessagesMS * uint64(i)), - "content": map[string]interface{}{ - "msgtype": "m.text", - "body": fmt.Sprintf("Historical %d (chunk=%d)", i, chunkCount), - historicalContentField: true, - }, - } - - evs[i] = newEvent - } - - state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) - for i, virtualUserID := range virtualUserIDs { - inviteEvent := map[string]interface{}{ - "type": "m.room.member", - "sender": c.UserID, - "origin_server_ts": insertOriginServerTs, - "content": map[string]interface{}{ - "membership": "invite", - }, - "state_key": virtualUserID, - } - state_evs[2*i] = inviteEvent - - joinEvent := map[string]interface{}{ - "type": "m.room.member", - "sender": virtualUserID, - "origin_server_ts": insertOriginServerTs, - "content": map[string]interface{}{ - "membership": "join", - }, - "state_key": virtualUserID, - } - - state_evs[2*i+1] = joinEvent - } - query := make(url.Values, 2) query.Add("prev_event", insertAfterEventId) // If provided, connect the chunk to the last insertion point @@ -986,8 +949,8 @@ func batchSendHistoricalMessages( "POST", []string{"_matrix", "client", "unstable", "org.matrix.msc2716", "rooms", roomID, "batch_send"}, client.WithJSONBody(t, map[string]interface{}{ - "events": evs, - "state_events_at_start": state_evs, + "events": events, + "state_events_at_start": state_events_at_start, }), client.WithContentType("application/json"), client.WithQueries(query), From 637ac8f9db653691840b44450c6af1c974a9d701 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 17:38:31 -0500 Subject: [PATCH 08/13] Remove test skips now that it's supported in Synapse Synapse support was added in https://github.com/matrix-org/synapse/pull/10498 --- tests/msc2716_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 4189815c..1c307330 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -511,7 +511,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) @@ -600,7 +599,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) From b06fc957becb54b31bb98e76b9e81d07a614400c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 18:48:11 -0500 Subject: [PATCH 09/13] Verify we only return state events that we pass in Synapse change added in https://github.com/matrix-org/synapse/pull/10552 --- tests/msc2716_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 1c307330..b1873809 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -290,6 +290,39 @@ func TestBackfillingHistory(t *testing.T) { }) }) + t.Run("Batch send endpoint only returns state events that we passed in via state_events_at_start", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPublicRoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + stateEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "state_events") + + // We only expect 1 state event to be returned because we only passed in 1 + // event into `?state_events_at_start` + if len(stateEventIDs) != 1 { + t.Fatalf("Expected only 1 state event to be returned but received %d: %s", len(stateEventIDs), stateEventIDs) + } + }) + t.Run("Unrecognised prev_event ID will throw an error", func(t *testing.T) { t.Parallel() @@ -336,6 +369,11 @@ func TestBackfillingHistory(t *testing.T) { ) }) + t.Run("TODO: Trying to send insertion event with same `next_chunk_id` will reject", func(t *testing.T) { + t.Skip("Skipping until implemented") + // (room_id, next_chunk_id) should be unique + }) + t.Run("Should be able to backfill into private room", func(t *testing.T) { t.Parallel() @@ -386,6 +424,10 @@ func TestBackfillingHistory(t *testing.T) { // TODO: Try adding avatar and displayName and see if historical messages get this info }) + t.Run("TODO: What happens when you point multiple chunks at the same insertion event?", func(t *testing.T) { + t.Skip("Skipping until implemented") + }) + t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { t.Parallel() From 0a215fe88ed045e1ff63635d9ba2c086027ac6bb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 21:04:49 -0500 Subject: [PATCH 10/13] Add tests for MSC2716 being backwards compatible with existing room versions Synapse changes added in https://github.com/matrix-org/synapse/pull/10566 --- tests/msc2716_test.go | 125 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 3 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b1873809..ee4e9ac1 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -726,9 +726,105 @@ func TestBackfillingHistory(t *testing.T) { }, }) }) + + t.Run("Existing room versions", func(t *testing.T) { + createUnsupportedMSC2716RoomOpts := map[string]interface{}{ + "preset": "public_chat", + "name": "the hangout spot", + // v6 is an existing room version that does not support MSC2716 + "room_version": "6", + } + + t.Run("Room creator can send MSC2716 events", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createUnsupportedMSC2716RoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + + messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) + }) + + t.Run("Not allowed to redact MSC2716 insertion, chunk, marker events", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createUnsupportedMSC2716RoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + insertionEventID := historicalEventIDs[0] + chunkEventID := historicalEventIDs[2] + baseInsertionEventID := historicalEventIDs[3] + + // Send the marker event + markerEventID := sendMarkerAndEnsureBackfilled(t, as, alice, roomID, baseInsertionEventID) + + redactEventID(t, alice, roomID, insertionEventID, 403) + redactEventID(t, alice, roomID, chunkEventID, 403) + redactEventID(t, alice, roomID, markerEventID, 403) + }) + }) }) } +var txnCounter int = 0 + +func getTxnID(prefix string) (txnID string) { + txnId := fmt.Sprintf("%s-%d", prefix, txnCounter) + + txnCounter++ + + return txnId +} + func makeInterfaceSlice(slice []string) []interface{} { interfaceSlice := make([]interface{}, len(slice)) for i := range slice { @@ -837,7 +933,7 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } } -func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { +func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) (markerEventID string) { // Send a marker event to let all of the homeservers know about the // insertion point where all of the historical messages are at markerEvent := b.Event{ @@ -846,10 +942,11 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA markerInsertionContentField: insertionEventID, }, } + txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") // We can't use as.SendEventSynced(...) because application services can't use the /sync API - markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) + markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, txnId}, client.WithJSONBody(t, markerEvent.Content)) markerSendBody := client.ParseJSON(t, markerSendRes) - markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") + markerEventID = client.GetJSONFieldStr(t, markerSendBody, "event_id") // Make sure the marker event has reached the remote homeserver c.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { @@ -865,6 +962,8 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA return false }) + + return markerEventID } func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count int) (eventIDs []string) { @@ -965,6 +1064,26 @@ func createMessageEventsForBackfillRequest( return evs } +func redactEventID(t *testing.T, c *client.CSAPI, roomID, eventID string, expectedStatus int) { + t.Helper() + + txnID := getTxnID("redactEventID-txn") + redactionRes := c.DoFunc( + t, + "PUT", + []string{"_matrix", "client", "r0", "rooms", roomID, "redact", eventID, txnID}, + client.WithJSONBody(t, map[string]interface{}{"reason": "chaos"}), + client.WithContentType("application/json"), + ) + redactionResBody := client.ParseJSON(t, redactionRes) + redactionResErrcode := client.GetJSONFieldStr(t, redactionResBody, "error") + redactionResError := client.GetJSONFieldStr(t, redactionResBody, "errcode") + + if redactionRes.StatusCode != expectedStatus { + t.Fatalf("msc2716.redactEventID: Expected redaction response to be %d but received %d -> %s: %s", expectedStatus, redactionRes.StatusCode, redactionResErrcode, redactionResError) + } +} + var chunkCount int64 = 0 func batchSendHistoricalMessages( From a958b34b99bafe74e27c8c152fe123dabff4ae67 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 18 Aug 2021 17:13:34 -0500 Subject: [PATCH 11/13] Add helper decorations --- tests/msc2716_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b1873809..069ae000 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -838,6 +838,8 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { + t.Helper() + // Send a marker event to let all of the homeservers know about the // insertion point where all of the historical messages are at markerEvent := b.Event{ @@ -977,6 +979,8 @@ func batchSendHistoricalMessages( events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { + t.Helper() + query := make(url.Values, 2) query.Add("prev_event", insertAfterEventId) // If provided, connect the chunk to the last insertion point From 858aef60f9123aeae58d03374988846b932e8219 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 Sep 2021 11:17:23 -0500 Subject: [PATCH 12/13] More idiomatic Go from review --- tests/msc2716_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 069ae000..4b2ddc77 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -319,7 +319,7 @@ func TestBackfillingHistory(t *testing.T) { // We only expect 1 state event to be returned because we only passed in 1 // event into `?state_events_at_start` if len(stateEventIDs) != 1 { - t.Fatalf("Expected only 1 state event to be returned but received %d: %s", len(stateEventIDs), stateEventIDs) + t.Fatalf("Expected only 1 state event to be returned but received %d: %v", len(stateEventIDs), stateEventIDs) } }) @@ -501,7 +501,7 @@ func TestBackfillingHistory(t *testing.T) { }, } // We can't use as.SendEventSynced(...) because application services can't use the /sync API - insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-m123"}, client.WithJSONBody(t, insertionEvent.Content)) + insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-i123"}, client.WithJSONBody(t, insertionEvent.Content)) insertionSendBody := client.ParseJSON(t, insertionSendRes) insertionEventID := client.GetJSONFieldStr(t, insertionSendBody, "event_id") // Make sure the insertion event has reached the homeserver @@ -592,7 +592,7 @@ func TestBackfillingHistory(t *testing.T) { // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] if len(historicalEventIDs) != 5 { - t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + t.Fatalf("Expected eventID list should be length 5 but saw %d: %v", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -894,7 +894,7 @@ func createInviteStateEventsForBackfillRequest( // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + stateEvents := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { inviteEvent := map[string]interface{}{ "type": "m.room.member", @@ -906,10 +906,10 @@ func createInviteStateEventsForBackfillRequest( "state_key": virtualUserID, } - state_evs[i] = inviteEvent + stateEvents[i] = inviteEvent } - return state_evs + return stateEvents } func createJoinStateEventsForBackfillRequest( @@ -919,7 +919,7 @@ func createJoinStateEventsForBackfillRequest( // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + stateEvents := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { joinEvent := map[string]interface{}{ "type": "m.room.member", @@ -931,10 +931,10 @@ func createJoinStateEventsForBackfillRequest( "state_key": virtualUserID, } - state_evs[i] = joinEvent + stateEvents[i] = joinEvent } - return state_evs + return stateEvents } func createMessageEventsForBackfillRequest( @@ -975,7 +975,7 @@ func batchSendHistoricalMessages( roomID string, insertAfterEventId string, chunkID string, - state_events_at_start []map[string]interface{}, + stateEventsAtStart []map[string]interface{}, events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { @@ -994,7 +994,7 @@ func batchSendHistoricalMessages( []string{"_matrix", "client", "unstable", "org.matrix.msc2716", "rooms", roomID, "batch_send"}, client.WithJSONBody(t, map[string]interface{}{ "events": events, - "state_events_at_start": state_events_at_start, + "state_events_at_start": stateEventsAtStart, }), client.WithContentType("application/json"), client.WithQueries(query), From 1de961835cf365089c9614dfff2042a185cd1416 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 Sep 2021 11:38:10 -0500 Subject: [PATCH 13/13] Use txn helper for other txns --- tests/msc2716_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index d68f6bfd..b0f881db 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -501,7 +501,8 @@ func TestBackfillingHistory(t *testing.T) { }, } // We can't use as.SendEventSynced(...) because application services can't use the /sync API - insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-i123"}, client.WithJSONBody(t, insertionEvent.Content)) + txnId := getTxnID("sendInsertionAndEnsureBackfilled-txn") + insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, txnId}, client.WithJSONBody(t, insertionEvent.Content)) insertionSendBody := client.ParseJSON(t, insertionSendRes) insertionEventID := client.GetJSONFieldStr(t, insertionSendBody, "event_id") // Make sure the insertion event has reached the homeserver @@ -944,8 +945,8 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA markerInsertionContentField: insertionEventID, }, } - txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") // We can't use as.SendEventSynced(...) because application services can't use the /sync API + txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, txnId}, client.WithJSONBody(t, markerEvent.Content)) markerSendBody := client.ParseJSON(t, markerSendRes) markerEventID = client.GetJSONFieldStr(t, markerSendBody, "event_id")