From d5272e38356e6a00758227412104a0084eb1549b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 10 Apr 2023 16:30:59 -0500 Subject: [PATCH 01/30] Draft: mostly test setup --- server/lib/matrix-utils/ensure-room-joined.js | 18 +- ...erver-name-from-matrix-room-id-or-alias.js | 21 + server/routes/room-routes.js | 49 +- shared/lib/reference-values.js | 6 + shared/lib/timestamp-utilities.js | 1 + test/e2e-tests.js | 714 ++++++++++-------- test/test-utils/client-utils.js | 162 ++-- test/test-utils/parse-archive-url-for-room.js | 6 +- 8 files changed, 599 insertions(+), 378 deletions(-) create mode 100644 server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js diff --git a/server/lib/matrix-utils/ensure-room-joined.js b/server/lib/matrix-utils/ensure-room-joined.js index e24f06d1..6a57c57a 100644 --- a/server/lib/matrix-utils/ensure-room-joined.js +++ b/server/lib/matrix-utils/ensure-room-joined.js @@ -4,15 +4,29 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson } = require('../fetch-endpoint'); +const getServerNameFromMatrixRoomIdOrAlias = require('./get-server-name-from-matrix-room-id-or-alias'); const config = require('../config'); const StatusError = require('../status-error'); const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); -async function ensureRoomJoined(accessToken, roomIdOrAlias, viaServers = []) { +async function ensureRoomJoined(accessToken, roomIdOrAlias, viaServers = new Set()) { + // We use a `Set` to ensure that we don't have duplicate servers in the list + assert(viaServers instanceof Set); + + // Let's do our best for the user to join the room. Since room ID's are + // unroutable on their own and won't be found if the server doesn't already + // know about the room, we'll try to join the room via the server name that + // we derived from the room ID or alias. + const viaServersWithAssumptions = new Set(viaServers); + const derivedServerName = getServerNameFromMatrixRoomIdOrAlias(roomIdOrAlias); + if (derivedServerName) { + viaServersWithAssumptions.add(derivedServerName); + } + let qs = new URLSearchParams(); - [].concat(viaServers).forEach((viaServer) => { + Array.from(viaServersWithAssumptions).forEach((viaServer) => { qs.append('server_name', viaServer); }); diff --git a/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js new file mode 100644 index 00000000..6c28a282 --- /dev/null +++ b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js @@ -0,0 +1,21 @@ +'use strict'; + +const assert = require('assert'); + +function getServerNameFromMatrixRoomIdOrAlias(roomIdOrAlias) { + assert(roomIdOrAlias); + + const pieces = roomIdOrAlias.split(':'); + // We can only derive the server name if there is a colon in the string. Since room + // IDs are supposed to be treated as opaque strings, there is a future possibility + // that they will not contain a colon. + if (pieces.length < 2) { + return null; + } + + const servername = pieces.slice(1).join(':'); + + return servername; +} + +module.exports = getServerNameFromMatrixRoomIdOrAlias; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 7dc457a9..551a0974 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -22,6 +22,7 @@ const { MS_LOOKUP, TIME_PRECISION_VALUES, DIRECTION, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, } = require('matrix-public-archive-shared/lib/reference-values'); const { ONE_DAY_IN_MS, ONE_HOUR_IN_MS, ONE_MINUTE_IN_MS, ONE_SECOND_IN_MS } = MS_LOOKUP; const { @@ -56,10 +57,6 @@ const router = express.Router({ mergeParams: true, }); -const VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP = { - r: '#', - roomid: '!', -}; const validSigilList = Object.values(VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP); const sigilRe = new RegExp(`^(${validSigilList.join('|')})`); @@ -159,6 +156,29 @@ function parseArchiveRangeFromReq(req) { }; } +function parseViaServersFromReq(req) { + const rawViaServers = [].concat(req.query.via || []); + if (rawViaServers.length === 0) { + return new Set(); + } + + const viaServerList = rawViaServers.map((viaServer) => { + // Sanity check to ensure that the via servers are strings (valid enough looking + // host names) + if (typeof viaServer !== 'string') { + throw new StatusError( + 400, + `?via server must be a string, got ${viaServer} (${typeof viaServer})` + ); + } + + return viaServer; + }); + + // We use a `Set` to ensure that we don't have duplicate servers in the list + return new Set(viaServerList); +} + router.use(redirectToCorrectArchiveUrlIfBadSigil); router.get( @@ -173,7 +193,11 @@ router.get( // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. - const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, req.query.via); + const roomId = await ensureRoomJoined( + matrixAccessToken, + roomIdOrAlias, + parseViaServersFromReq(req) + ); // Find the closest day to the current time with messages const { originServerTs } = await timestampToEvent({ @@ -192,7 +216,7 @@ router.get( // We can avoid passing along the `via` query parameter because we already // joined the room above (see `ensureRoomJoined`). // - //viaServers: req.query.via, + //viaServers: parseViaServersFromReq(req), }) ); }) @@ -245,7 +269,11 @@ router.get( // We have to wait for the room join to happen first before we can use the jump to // date endpoint - const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, req.query.via); + const roomId = await ensureRoomJoined( + matrixAccessToken, + roomIdOrAlias, + parseViaServersFromReq(req) + ); let eventIdForClosestEvent; let tsForClosestEvent; @@ -536,7 +564,11 @@ router.get( // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. - const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, req.query.via); + const roomId = await ensureRoomJoined( + matrixAccessToken, + roomIdOrAlias, + parseViaServersFromReq(req) + ); // Do these in parallel to avoid the extra time in sequential round-trips // (we want to display the archive page faster) @@ -587,6 +619,7 @@ router.get( const stylesUrl = urlJoin(basePath, '/css/styles.css'); const jsBundleUrl = urlJoin(basePath, '/js/entry-client-hydrogen.es.js'); + // XXX: The `renderHydrogenVmRenderScriptToPageHtml` API surface is pretty awkward const pageHtml = await renderHydrogenVmRenderScriptToPageHtml( path.resolve(__dirname, '../../shared/hydrogen-vm-render-script.js'), { diff --git a/shared/lib/reference-values.js b/shared/lib/reference-values.js index d9fec80f..caf08acb 100644 --- a/shared/lib/reference-values.js +++ b/shared/lib/reference-values.js @@ -28,9 +28,15 @@ const VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP = { '!': 'roomid', }; +const VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP = { + r: '#', + roomid: '!', +}; + module.exports = { MS_LOOKUP, TIME_PRECISION_VALUES, DIRECTION, VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, }; diff --git a/shared/lib/timestamp-utilities.js b/shared/lib/timestamp-utilities.js index a5ed40dc..1990add2 100644 --- a/shared/lib/timestamp-utilities.js +++ b/shared/lib/timestamp-utilities.js @@ -32,6 +32,7 @@ function roundUpTimestampToUtcSecond(ts) { return dateRountedUp.getTime(); } +// XXX: Should these just be renamed to `roundDownTimestampToUtcDay`? function getUtcStartOfDayTs(ts) { assert(typeof ts === 'number' || ts instanceof Date); const date = new Date(ts); diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 32ded282..9dd9a692 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -25,6 +25,7 @@ const { getTestClientForAs, getTestClientForHs, createTestRoom, + upgradeTestRoom, getCanonicalAlias, joinRoom, sendEvent, @@ -757,6 +758,318 @@ describe('matrix-public-archive', () => { }); describe('Jump forwards and backwards', () => { + function runJumpTestCase(testCase) { + // eslint-disable-next-line max-statements, complexity + it(testCase.testName, async () => { + // Setup + // -------------------------------------- + // -------------------------------------- + const eventMap = new Map(); + const fancyIdentifierToEventIdMap = new Map(); + const eventIdToFancyIdentifierMap = new Map(); + + function convertFancyIdentifierListToDebugEventIds(fancyEventIdentifiers) { + // eslint-disable-next-line max-nested-callbacks + return fancyEventIdentifiers.map((fancyId) => { + const eventId = fancyIdentifierToEventIdMap.get(fancyId); + if (!eventId) { + throw new Error( + `Unable to find ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( + Object.fromEntries(fancyIdentifierToEventIdMap.entries()), + null, + 2 + )}` + ); + } + const ts = eventMap.get(eventId)?.originServerTs; + const tsDebugString = ts && `${new Date(ts).toISOString()} (${ts})`; + return `${eventId} (${fancyId}) - ${tsDebugString}`; + }); + } + + function convertEventIdsToDebugEventIds(eventIds) { + // eslint-disable-next-line max-nested-callbacks + return eventIds.map((eventId) => { + const fancyEventId = eventIdToFancyIdentifierMap.get(eventId); + if (!fancyEventId) { + throw new Error( + `Unable to find ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( + Object.fromEntries(eventIdToFancyIdentifierMap.entries()), + null, + 2 + )}` + ); + } + const ts = eventMap.get(eventId)?.originServerTs; + const tsDebugString = ts && `${new Date(ts).toISOString()} (${ts})`; + return `${eventId} (${fancyEventId}) - ${tsDebugString}`; + }); + } + + const client = await getTestClientForHs(testMatrixServerUrl1); + + const { rooms, archiveMessageLimit, pages } = parseRoomDayMessageStructure( + testCase.roomDayMessageStructureString + ); + const fancyIdentifierToRoomIdMap = new Map(); + const roomIdToFancyIdentifierMap = new Map(); + let previousRoomId; + for (const [roomIndex, room] of rooms.entries()) { + let roomId; + if (previousRoomId) { + roomId = await upgradeTestRoom({ + client, + previousRoomId, + //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it + }); + } else { + roomId = await createTestRoom(client); + } + const fancyRoomId = `#room${roomIndex + 1}`; + fancyIdentifierToRoomIdMap.set(fancyRoomId, roomId); + roomIdToFancyIdentifierMap.set(roomId, fancyRoomId); + + // Join the archive user to the room before we create the test messages to + // avoid problems jumping to the latest activity since we can't control the + // timestamp of the membership event. + const archiveAppServiceUserClient = await getTestClientForAs(); + await joinRoom({ + client: archiveAppServiceUserClient, + roomId: roomId, + }); + + // Just spread things out a bit so the event times are more obvious + // and stand out from each other while debugging and so we just have + // to deal with hour time slicing + const eventSendTimeIncrement = ONE_HOUR_IN_MS; + + for (const eventMeta of room.events) { + const archiveDate = new Date(Date.UTC(2022, 0, eventMeta.dayNumber, 0, 0, 0, 1)); + const originServerTs = + archiveDate.getTime() + eventMeta.eventIndexInDay * eventSendTimeIncrement; + const content = { + msgtype: 'm.text', + body: `event${eventMeta.eventNumber} - day${eventMeta.dayNumber}.${eventMeta.eventIndexInDay}`, + }; + const eventId = await sendMessage({ + client, + roomId, + content, + // Technically, we don't have to set the timestamp to be unique or sequential but + // it still seems like a good idea to make the tests more clear. + timestamp: originServerTs, + }); + eventMap.set(eventId, { + roomId, + originServerTs, + content, + }); + const fancyEventId = `$event${eventMeta.eventNumber}`; + fancyIdentifierToEventIdMap.set(fancyEventId, eventId); + eventIdToFancyIdentifierMap.set(eventId, fancyEventId); + } + + previousRoomId = roomId; + } + + // Now Test + // -------------------------------------- + // -------------------------------------- + + // Make sure the archive is configured as the test expects + config.set('archiveMessageLimit', archiveMessageLimit); + + // eslint-disable-next-line max-nested-callbacks + const pagesKeyList = Object.keys(testCase).filter((key) => { + const isPageKey = key.startsWith('page'); + if (isPageKey) { + assert.match(key, /page\d+/); + return true; + } + + return false; + }); + assert( + pagesKeyList.length > 0, + 'You must have at least one `pageX` of expectations in your jump test case' + ); + // Make sure the page are in order + // eslint-disable-next-line max-nested-callbacks + pagesKeyList.reduce((prevPageCount, currentPageKey) => { + const pageNumber = parseInt(currentPageKey.match(/\d+$/)[0], 10); + assert( + prevPageCount + 1 === pageNumber, + `Page numbers must be sorted in each test case but found ` + + `${pageNumber} after ${prevPageCount} - pagesList=${pagesKeyList}` + ); + return pageNumber; + }, 0); + + // Get the URL for the first page to fetch + // + // Set the `archiveUrl` for debugging if the test fails here + const { roomIdOrAlias: startRoomFancyKey, urlDateTime: startUrlDateTime } = + parseArchiveUrlForRoom(urlJoin('https://example.com', testCase.startUrl)); + const startRoomIdOrAlias = fancyIdentifierToRoomIdMap.get(startRoomFancyKey); + assert( + startRoomIdOrAlias, + `Could not find room ID for ${startRoomFancyKey} in our list of known rooms ${JSON.stringify( + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + )}` + ); + archiveUrl = `${matrixPublicArchiveURLCreator.archiveUrlForRoom( + startRoomIdOrAlias + )}/date/${startUrlDateTime}`; + + // Loop through all of the pages of the test and ensure expectations + let alreadyEncounteredLastPage = false; + for (const pageKey of pagesKeyList) { + try { + if (alreadyEncounteredLastPage) { + assert.fail( + 'We should not see any more pages after we already saw a page without an action ' + + `which signals the end of expecations. Encountered ${pageKey} in ${pagesKeyList} ` + + 'after we already thought we were done' + ); + } + + const pageTestMeta = testCase[pageKey]; + const { + roomIdOrAlias: expectedRoomFancyId, + //urlDateTime: expectedUrlDateTime, + continueAtEvent: expectedContinueAtEvent, + } = parseArchiveUrlForRoom(urlJoin('https://example.com', pageTestMeta.url)); + const expectedRoomId = fancyIdentifierToRoomIdMap.get(expectedRoomFancyId); + assert( + expectedRoomId, + `Could not find room ID for ${expectedRoomFancyId} in our list of known rooms ${JSON.stringify( + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + )}` + ); + + // Fetch the given page. + const { data: archivePageHtml, res: pageRes } = await fetchEndpointAsText( + archiveUrl + ); + const pageDom = parseHTML(archivePageHtml); + const { + roomIdOrAliasUrlPart: actualRoomIdOrAliasUrlPart, + roomIdOrAlias: actualRoomId, + //urlDateTime: actualUrlDateTime, + continueAtEvent: actualContinueAtEventId, + } = parseArchiveUrlForRoom(pageRes.url); + const actualRoomFancyId = roomIdToFancyIdentifierMap.get(actualRoomId); + assert( + actualRoomFancyId, + `Could not find room ID for ${actualRoomId} in our list of known rooms ${JSON.stringify( + Object.fromEntries(roomIdToFancyIdentifierMap.entries()) + )}` + ); + let actualContinueAtEventFancyId; + if (actualContinueAtEventId) { + actualContinueAtEventFancyId = + eventIdToFancyIdentifierMap.get(actualContinueAtEventId); + assert( + actualContinueAtEventFancyId, + `Could not find event ID for ${actualContinueAtEventId} in our list of known events ${JSON.stringify( + Object.fromEntries(eventIdToFancyIdentifierMap.entries()) + )}` + ); + } + + // Replace messy room ID's and event ID's that change with every test + // run with their fancy ID's which correlate with the test meta so it's + // easier to reason about things when the assertion fails. + let actualUrlWithFancyIdentifies = pageRes.url + .replace( + `/roomid/${actualRoomIdOrAliasUrlPart}`, + // Slice to remove the sigil + `/r/${actualRoomFancyId.slice(1)}` + ) + .replace(actualContinueAtEventId, actualContinueAtEventFancyId); + // Assert the correct room and time precision in the URL + assert.match( + actualUrlWithFancyIdentifies, + new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`) + ); + + // If provided, assert that it's a smooth continuation to more messages. + // First by checking where the scroll is going to start from + if (expectedContinueAtEvent) { + const [expectedContinuationDebugEventId] = + convertFancyIdentifierListToDebugEventIds([expectedContinueAtEvent]); + const urlObj = new URL(pageRes.url, basePath); + const qs = new URLSearchParams(urlObj.search); + const continuationEventId = qs.get('at'); + if (!continuationEventId) { + throw new Error( + `Expected ?at=$xxx query parameter to be defined in the URL=${pageRes.url} but it was ${continuationEventId}. We expect it to match ${expectedContinuationDebugEventId}` + ); + } + const [continationDebugEventId] = convertEventIdsToDebugEventIds([ + continuationEventId, + ]); + assert.strictEqual(continationDebugEventId, expectedContinuationDebugEventId); + } + + const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] + // eslint-disable-next-line max-nested-callbacks + .map((eventEl) => { + return eventEl.getAttribute('data-event-id'); + }); + + const pageNumber = pageKey.replace('page', ''); + const page = pages[pageNumber - 1]; + const expectedEventsOnPage = page.events; + const expectedFancyIdsOnPage = expectedEventsOnPage.map( + // eslint-disable-next-line max-nested-callbacks + (event) => `$event${event.eventNumber}` + ); + + // Assert that the page contains all expected events + assert.deepEqual( + convertEventIdsToDebugEventIds(eventIdsOnPage), + convertFancyIdentifierListToDebugEventIds(expectedFancyIdsOnPage), + `Events on ${pageKey} should be as expected` + ); + + // Follow the next activity link. Aka, fetch messages for the 2nd page + let actionLinkSelector; + if (pageTestMeta.action === 'next') { + actionLinkSelector = '[data-testid="jump-to-next-activity-link"]'; + } else if (pageTestMeta.action === 'previous') { + actionLinkSelector = '[data-testid="jump-to-previous-activity-link"]'; + } else if (pageTestMeta.action === null) { + // No more pages to test ✅, move on + alreadyEncounteredLastPage = true; + continue; + } else { + throw new Error( + `Unexpected value for ${pageKey}.action=${pageTestMeta.action} that we don't know what to do with` + ); + } + const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); + const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); + // Move to the next iteration of the loop + // + // Set this for debugging if the test fails here + archiveUrl = jumpToActivityLinkHref; + } catch (err) { + const errorWithContext = new RethrownError( + `Encountered error while asserting ${pageKey}: ${err.message}`, + err + ); + // Copy these over so mocha generates a nice diff for us + if (err instanceof assert.AssertionError) { + errorWithContext.actual = err.actual; + errorWithContext.expected = err.expected; + } + throw errorWithContext; + } + } + }); + } + const jumpTestCases = [ { // In order to jump from the 1st page to the 2nd, we first jump forward 4 @@ -1225,7 +1538,6 @@ describe('matrix-public-archive', () => { // rangeStart), we look backwards for the closest event. Because we find // event7 as the closest, which is from a different day than event12 (page1 // rangeEnd), we can just display the day where event7 resides. - // testName: 'can jump backward from the start of one day with too many messages into the previous day with exactly the limit', roomDayMessageStructureString: ` @@ -1233,318 +1545,106 @@ describe('matrix-public-archive', () => { 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [day3 ] [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03T05:00', - page1: { - url: '/r/room1/date/2022/01/03T05:00', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/02?at=$event7', - action: null, - }, - }, - ]; - - jumpTestCases.forEach((testCase) => { - // eslint-disable-next-line max-statements, complexity - it(testCase.testName, async () => { - // Setup - // -------------------------------------- - // -------------------------------------- - const eventMap = new Map(); - const fancyIdentifierToEventIdMap = new Map(); - const eventIdToFancyIdentifierMap = new Map(); - - function convertFancyIdentifierListToDebugEventIds(fancyEventIdentifiers) { - // eslint-disable-next-line max-nested-callbacks - return fancyEventIdentifiers.map((fancyId) => { - const eventId = fancyIdentifierToEventIdMap.get(fancyId); - if (!eventId) { - throw new Error( - `Unable to find ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( - Object.fromEntries(fancyIdentifierToEventIdMap.entries()), - null, - 2 - )}` - ); - } - const ts = eventMap.get(eventId)?.originServerTs; - const tsDebugString = ts && `${new Date(ts).toISOString()} (${ts})`; - return `${eventId} (${fancyId}) - ${tsDebugString}`; - }); - } - - function convertEventIdsToDebugEventIds(eventIds) { - // eslint-disable-next-line max-nested-callbacks - return eventIds.map((eventId) => { - const fancyEventId = eventIdToFancyIdentifierMap.get(eventId); - if (!fancyEventId) { - throw new Error( - `Unable to find ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( - Object.fromEntries(eventIdToFancyIdentifierMap.entries()), - null, - 2 - )}` - ); - } - const ts = eventMap.get(eventId)?.originServerTs; - const tsDebugString = ts && `${new Date(ts).toISOString()} (${ts})`; - return `${eventId} (${fancyEventId}) - ${tsDebugString}`; - }); - } - - const client = await getTestClientForHs(testMatrixServerUrl1); - - const { rooms, archiveMessageLimit, pages } = parseRoomDayMessageStructure( - testCase.roomDayMessageStructureString - ); - const fancyIdentifierToRoomIdMap = new Map(); - const roomIdToFancyIdentifierMap = new Map(); - for (const [roomIndex, room] of rooms.entries()) { - // TODO: upgradeRoom to link one room to another - const roomId = await createTestRoom(client); - const fancyRoomId = `#room${roomIndex + 1}`; - fancyIdentifierToRoomIdMap.set(fancyRoomId, roomId); - roomIdToFancyIdentifierMap.set(roomId, fancyRoomId); - - // Join the archive user to the room before we create the test messages to - // avoid problems jumping to the latest activity since we can't control the - // timestamp of the membership event. - const archiveAppServiceUserClient = await getTestClientForAs(); - await joinRoom({ - client: archiveAppServiceUserClient, - roomId: roomId, - }); - - // Just spread things out a bit so the event times are more obvious - // and stand out from each other while debugging and so we just have - // to deal with hour time slicing - const eventSendTimeIncrement = ONE_HOUR_IN_MS; - - for (const eventMeta of room.events) { - const archiveDate = new Date(Date.UTC(2022, 0, eventMeta.dayNumber, 0, 0, 0, 1)); - const originServerTs = - archiveDate.getTime() + eventMeta.eventIndexInDay * eventSendTimeIncrement; - const content = { - msgtype: 'm.text', - body: `event${eventMeta.eventNumber} - day${eventMeta.dayNumber}.${eventMeta.eventIndexInDay}`, - }; - const eventId = await sendMessage({ - client, - roomId, - content, - // Technically, we don't have to set the timestamp to be unique or sequential but - // it still seems like a good idea to make the tests more clear. - timestamp: originServerTs, - }); - eventMap.set(eventId, { - roomId, - originServerTs, - content, - }); - const fancyEventId = `$event${eventMeta.eventNumber}`; - fancyIdentifierToEventIdMap.set(fancyEventId, eventId); - eventIdToFancyIdentifierMap.set(eventId, fancyEventId); - } - } + [page2 ] + `, + startUrl: '/r/room1/date/2022/01/03T05:00', + page1: { + url: '/r/room1/date/2022/01/03T05:00', + action: 'previous', + }, + page2: { + url: '/r/room1/date/2022/01/02?at=$event7', + action: null, + }, + }, + ]; - // Now Test - // -------------------------------------- - // -------------------------------------- + const jumpBackwardPredecessorTestCases = [ + { + testName: 'can jump backward from one room to the predecessor room', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2 ] + `, + startUrl: '/r/room2/date/2022/01/03T05:00', + page1: { + url: '/r/room2/date/2022/01/03T05:00', + action: 'previous', + }, + page2: { + url: '/r/room1/date/2022/01/02', + action: null, + }, + }, + { + testName: `will paginate to the oldest messages in the room (doesn't skip the last few) before jumping backward to the predecessor room`, + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2] + [page3 ] + `, + startUrl: '/r/room2/date/2022/01/03T05:00', + page1: { + url: '/r/room2/date/2022/01/03T05:00', + action: 'previous', + }, + page2: { + url: '/r/room2/date/2022/01/03', + action: 'previous', + }, + page3: { + url: '/r/room1/date/2022/01/02', + action: null, + }, + }, + ]; - // Make sure the archive is configured as the test expects - config.set('archiveMessageLimit', archiveMessageLimit); + const jumpForwardTombstoneTestCases = [ + { + testName: 'can jump forward from one room to the replacement room', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + |--jump-fwd-4-messages--->| + [page2 ] + `, + startUrl: '/r/room1/date/2022/01/02', + page1: { + url: '/r/room1/date/2022/01/02', + action: 'next', + }, + page2: { + url: '/r/room2/date/2022/01/03T04:00', + action: null, + }, + }, + ]; - // eslint-disable-next-line max-nested-callbacks - const pagesKeyList = Object.keys(testCase).filter((key) => { - const isPageKey = key.startsWith('page'); - if (isPageKey) { - assert.match(key, /page\d+/); - return true; - } + jumpTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); - return false; - }); - assert( - pagesKeyList.length > 0, - 'You must have at least one `pageX` of expectations in your jump test case' - ); - // Make sure the page are in order + describe('with room upgrades', () => { + describe('jump backward into predecessor rooms', () => { // eslint-disable-next-line max-nested-callbacks - pagesKeyList.reduce((prevPageCount, currentPageKey) => { - const pageNumber = parseInt(currentPageKey.match(/\d+$/)[0], 10); - assert( - prevPageCount + 1 === pageNumber, - `Page numbers must be sorted in each test case but found ` + - `${pageNumber} after ${prevPageCount} - pagesList=${pagesKeyList}` - ); - return pageNumber; - }, 0); - - // Get the URL for the first page to fetch - // - // Set the `archiveUrl` for debugging if the test fails here - const { roomIdOrAlias: startRoomFancyKey, urlDateTime: startUrlDateTime } = - parseArchiveUrlForRoom(urlJoin('https://example.com', testCase.startUrl)); - const startRoomIdOrAlias = fancyIdentifierToRoomIdMap.get(startRoomFancyKey); - assert( - startRoomIdOrAlias, - `Could not find room ID for ${startRoomFancyKey} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) - )}` - ); - archiveUrl = `${matrixPublicArchiveURLCreator.archiveUrlForRoom( - startRoomIdOrAlias - )}/date/${startUrlDateTime}`; - - // Loop through all of the pages of the test and ensure expectations - let alreadyEncounteredLastPage = false; - for (const pageKey of pagesKeyList) { - try { - if (alreadyEncounteredLastPage) { - assert.fail( - 'We should not see any more pages after we already saw a page without an action ' + - `which signals the end of expecations. Encountered ${pageKey} in ${pagesKeyList} ` + - 'after we already thought we were done' - ); - } - - const pageTestMeta = testCase[pageKey]; - const { - roomIdOrAlias: expectedRoomFancyId, - //urlDateTime: expectedUrlDateTime, - continueAtEvent: expectedContinueAtEvent, - } = parseArchiveUrlForRoom(urlJoin('https://example.com', pageTestMeta.url)); - const expectedRoomId = fancyIdentifierToRoomIdMap.get(expectedRoomFancyId); - assert( - expectedRoomId, - `Could not find room ID for ${expectedRoomFancyId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) - )}` - ); - - // Fetch the given page. - const { data: archivePageHtml, res: pageRes } = await fetchEndpointAsText( - archiveUrl - ); - const pageDom = parseHTML(archivePageHtml); - const { - roomIdOrAliasUrlPart: actualRoomIdOrAliasUrlPart, - roomIdOrAlias: actualRoomId, - //urlDateTime: actualUrlDateTime, - continueAtEvent: actualContinueAtEventId, - } = parseArchiveUrlForRoom(pageRes.url); - const actualRoomFancyId = roomIdToFancyIdentifierMap.get(actualRoomId); - assert( - actualRoomFancyId, - `Could not find room ID for ${actualRoomId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(roomIdToFancyIdentifierMap.entries()) - )}` - ); - let actualContinueAtEventFancyId; - if (actualContinueAtEventId) { - actualContinueAtEventFancyId = - eventIdToFancyIdentifierMap.get(actualContinueAtEventId); - assert( - actualContinueAtEventFancyId, - `Could not find event ID for ${actualContinueAtEventId} in our list of known events ${JSON.stringify( - Object.fromEntries(eventIdToFancyIdentifierMap.entries()) - )}` - ); - } - - // Replace messy room ID's and event ID's that change with every test - // run with their fancy ID's which correlate with the test meta so it's - // easier to reason about things when the assertion fails. - let actualUrlWithFancyIdentifies = pageRes.url - .replace( - `/roomid/${actualRoomIdOrAliasUrlPart}`, - // Slice to remove the sigil - `/r/${actualRoomFancyId.slice(1)}` - ) - .replace(actualContinueAtEventId, actualContinueAtEventFancyId); - // Assert the correct room and time precision in the URL - assert.match( - actualUrlWithFancyIdentifies, - new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`) - ); - - // If provided, assert that it's a smooth continuation to more messages. - // First by checking where the scroll is going to start from - if (expectedContinueAtEvent) { - const [expectedContinuationDebugEventId] = - convertFancyIdentifierListToDebugEventIds([expectedContinueAtEvent]); - const urlObj = new URL(pageRes.url, basePath); - const qs = new URLSearchParams(urlObj.search); - const continuationEventId = qs.get('at'); - if (!continuationEventId) { - throw new Error( - `Expected ?at=$xxx query parameter to be defined in the URL=${pageRes.url} but it was ${continuationEventId}. We expect it to match ${expectedContinuationDebugEventId}` - ); - } - const [continationDebugEventId] = convertEventIdsToDebugEventIds([ - continuationEventId, - ]); - assert.strictEqual(continationDebugEventId, expectedContinuationDebugEventId); - } - - const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] - // eslint-disable-next-line max-nested-callbacks - .map((eventEl) => { - return eventEl.getAttribute('data-event-id'); - }); - - const pageNumber = pageKey.replace('page', ''); - const page = pages[pageNumber - 1]; - const expectedEventsOnPage = page.events; - const expectedFancyIdsOnPage = expectedEventsOnPage.map( - // eslint-disable-next-line max-nested-callbacks - (event) => `$event${event.eventNumber}` - ); - - // Assert that the page contains all expected events - assert.deepEqual( - convertEventIdsToDebugEventIds(eventIdsOnPage), - convertFancyIdentifierListToDebugEventIds(expectedFancyIdsOnPage), - `Events on ${pageKey} should be as expected` - ); + jumpBackwardPredecessorTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); + }); - // Follow the next activity link. Aka, fetch messages for the 2nd page - let actionLinkSelector; - if (pageTestMeta.action === 'next') { - actionLinkSelector = '[data-testid="jump-to-next-activity-link"]'; - } else if (pageTestMeta.action === 'previous') { - actionLinkSelector = '[data-testid="jump-to-previous-activity-link"]'; - } else if (pageTestMeta.action === null) { - // No more pages to test ✅, move on - alreadyEncounteredLastPage = true; - continue; - } else { - throw new Error( - `Unexpected value for ${pageKey}.action=${pageTestMeta.action} that we don't know what to do with` - ); - } - const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); - const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); - // Move to the next iteration of the loop - // - // Set this for debugging if the test fails here - archiveUrl = jumpToActivityLinkHref; - } catch (err) { - const errorWithContext = new RethrownError( - `Encountered error while asserting ${pageKey}: ${err.message}`, - err - ); - // Copy these over so mocha generates a nice diff for us - if (err instanceof assert.AssertionError) { - errorWithContext.actual = err.actual; - errorWithContext.expected = err.expected; - } - throw errorWithContext; - } - } + describe('jump forward from tombstone to replacement rooms', () => { + // eslint-disable-next-line max-nested-callbacks + jumpForwardTombstoneTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); }); }); diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 4abc50ca..495c57ea 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -3,6 +3,7 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson, fetchEndpoint } = require('../../server/lib/fetch-endpoint'); +const getServerNameFromMatrixRoomIdOrAlias = require('../../server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias'); const config = require('../../server/lib/config'); const matrixAccessToken = config.get('matrixAccessToken'); @@ -85,6 +86,62 @@ async function getTestClientForHs(testMatrixServerUrl) { }; } +async function sendEvent({ client, roomId, eventType, stateKey, content, timestamp }) { + assert(client); + assert(roomId); + assert(content); + + let qs = new URLSearchParams(); + if (timestamp) { + assert( + timestamp && client.applicationServiceUserIdOverride, + 'We can only do `?ts` massaging from an application service access token. ' + + 'Expected `client.applicationServiceUserIdOverride` to be defined so we can act on behalf of that user' + ); + + qs.append('ts', timestamp); + } + + if (client.applicationServiceUserIdOverride) { + qs.append('user_id', client.applicationServiceUserIdOverride); + } + + let url; + if (stateKey) { + url = urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent( + roomId + )}/state/${eventType}/${stateKey}?${qs.toString()}` + ); + } else { + url = urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent( + roomId + )}/send/${eventType}/${getTxnId()}?${qs.toString()}` + ); + } + + const { data: sendResponse } = await fetchEndpointAsJson(url, { + method: 'PUT', + body: content, + accessToken: client.accessToken, + }); + + const eventId = sendResponse['event_id']; + assert(eventId); + return eventId; +} + +const WORLD_READABLE_STATE_EVENT = { + type: 'm.room.history_visibility', + state_key: '', + content: { + history_visibility: 'world_readable', + }, +}; + // Create a public room to test in async function createTestRoom(client, overrideCreateOptions = {}) { let qs = new URLSearchParams(); @@ -103,15 +160,7 @@ async function createTestRoom(client, overrideCreateOptions = {}) { preset: 'public_chat', name: roomName, room_alias_name: roomAlias, - initial_state: [ - { - type: 'm.room.history_visibility', - state_key: '', - content: { - history_visibility: 'world_readable', - }, - }, - ], + initial_state: [WORLD_READABLE_STATE_EVENT], visibility: 'public', ...overrideCreateOptions, }, @@ -124,6 +173,52 @@ async function createTestRoom(client, overrideCreateOptions = {}) { return roomId; } +async function upgradeTestRoom({ + client, + oldRoomId, + useMsc3946DynamicPredecessor = false, + overrideCreateOptions = {}, +}) { + assert(client); + assert(oldRoomId); + + const createOptions = { + ...overrideCreateOptions, + }; + // Setup the pointer from the new room to the old room + if (useMsc3946DynamicPredecessor) { + createOptions.initial_state = [ + WORLD_READABLE_STATE_EVENT, + { + type: 'org.matrix.msc3946.room_predecessor', + state_key: '', + content: { + predecessor_room_id: oldRoomId, + via_servers: [getServerNameFromMatrixRoomIdOrAlias(oldRoomId)], + }, + }, + ]; + } else { + createOptions.creation_content = { + predecessor: oldRoomId, + }; + } + + const newRoomid = await createTestRoom(client, createOptions); + + // Now send the tombstone event pointing from the old room to the new room + await sendEvent({ + client, + oldRoomId, + eventType: 'm.room.tombstone', + content: { + replacement_room: newRoomid, + }, + }); + + return newRoomid; +} + async function getCanonicalAlias({ client, roomId }) { const { data: stateCanonicalAliasRes } = await fetchEndpointAsJson( urlJoin( @@ -167,54 +262,6 @@ async function joinRoom({ client, roomId, viaServers }) { return joinedRoomId; } -async function sendEvent({ client, roomId, eventType, stateKey, content, timestamp }) { - assert(client); - assert(roomId); - assert(content); - - let qs = new URLSearchParams(); - if (timestamp) { - assert( - timestamp && client.applicationServiceUserIdOverride, - 'We can only do `?ts` massaging from an application service access token. ' + - 'Expected `client.applicationServiceUserIdOverride` to be defined so we can act on behalf of that user' - ); - - qs.append('ts', timestamp); - } - - if (client.applicationServiceUserIdOverride) { - qs.append('user_id', client.applicationServiceUserIdOverride); - } - - let url; - if (stateKey) { - url = urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/rooms/${encodeURIComponent( - roomId - )}/state/${eventType}/${stateKey}?${qs.toString()}` - ); - } else { - url = urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/rooms/${encodeURIComponent( - roomId - )}/send/${eventType}/${getTxnId()}?${qs.toString()}` - ); - } - - const { data: sendResponse } = await fetchEndpointAsJson(url, { - method: 'PUT', - body: content, - accessToken: client.accessToken, - }); - - const eventId = sendResponse['event_id']; - assert(eventId); - return eventId; -} - async function sendMessage({ client, roomId, content, timestamp }) { return sendEvent({ client, roomId, eventType: 'm.room.message', content, timestamp }); } @@ -345,6 +392,7 @@ module.exports = { getTestClientForAs, getTestClientForHs, createTestRoom, + upgradeTestRoom, getCanonicalAlias, joinRoom, sendEvent, diff --git a/test/test-utils/parse-archive-url-for-room.js b/test/test-utils/parse-archive-url-for-room.js index 090c6375..eada9b4a 100644 --- a/test/test-utils/parse-archive-url-for-room.js +++ b/test/test-utils/parse-archive-url-for-room.js @@ -1,7 +1,7 @@ 'use strict'; const { - VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, } = require('matrix-public-archive-shared/lib/reference-values'); // http://archive.matrix.org/r/some-room:matrix.org/date/2022/11/16T23:59:59?at=$xxx @@ -13,9 +13,7 @@ function parseArchiveUrlForRoom(archiveUrlForRoom) { /\/(r|roomid)\/(.*?)\/date\/(.*)/ ); - const [sigil] = Object.entries(VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP).find( - ([_sigil, entityDescriptor]) => roomIdOrAliasDescriptor === entityDescriptor - ); + const sigil = VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP[roomIdOrAliasDescriptor]; const roomIdOrAlias = `${sigil}${roomIdOrAliasUrlPart}`; const continueAtEvent = urlObj.searchParams.get('at'); From 359f95decf6ea1ee214ba2822c2f223fe263fb0b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Apr 2023 02:34:53 -0500 Subject: [PATCH 02/30] Draft: jumping to predecessor/successor --- server/lib/matrix-utils/fetch-room-data.js | 87 +++++++++++++-- .../lib/parse-via-servers-from-user-input.js | 29 +++++ server/routes/room-routes.js | 105 +++++++++++------- shared/lib/url-creator.js | 15 ++- test/e2e-tests.js | 20 +++- test/test-utils/client-utils.js | 5 +- 6 files changed, 202 insertions(+), 59 deletions(-) create mode 100644 server/lib/parse-via-servers-from-user-input.js diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index d6b73e6e..81aed601 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -4,44 +4,71 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson } = require('../fetch-endpoint'); +const parseViaServersFromUserInput = require('../parse-via-servers-from-user-input'); const { traceFunction } = require('../../tracing/trace-utilities'); const config = require('../config'); const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); +// eslint-disable-next-line max-statements async function fetchRoomData(accessToken, roomId) { assert(accessToken); assert(roomId); + const stateCreateEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.create?format=event` + ); const stateNameEndpoint = urlJoin( matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.name` + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.name?format=event` ); const canoncialAliasEndpoint = urlJoin( matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.canonical_alias` + `_matrix/client/r0/rooms/${encodeURIComponent( + roomId + )}/state/m.room.canonical_alias?format=event` ); const stateAvatarEndpoint = urlJoin( matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.avatar` + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.avatar?format=event` ); const stateHistoryVisibilityEndpoint = urlJoin( matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.history_visibility` + `_matrix/client/r0/rooms/${encodeURIComponent( + roomId + )}/state/m.room.history_visibility?format=event` ); const stateJoinRulesEndpoint = urlJoin( matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.join_rules` + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.join_rules?format=event` + ); + + const statePredecessorEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent( + roomId + )}/state/org.matrix.msc3946.room_predecessor?format=event` + ); + const stateTombstoneEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.tombstone?format=event` ); const [ + stateCreateResDataOutcome, stateNameResDataOutcome, stateCanonicalAliasResDataOutcome, stateAvatarResDataOutcome, stateHistoryVisibilityResDataOutcome, stateJoinRulesResDataOutcome, + statePredecessorResDataOutcome, + stateTombstoneResDataOutcome, ] = await Promise.allSettled([ + fetchEndpointAsJson(stateCreateEndpoint, { + accessToken, + }), fetchEndpointAsJson(stateNameEndpoint, { accessToken, }), @@ -57,36 +84,70 @@ async function fetchRoomData(accessToken, roomId) { fetchEndpointAsJson(stateJoinRulesEndpoint, { accessToken, }), + fetchEndpointAsJson(statePredecessorEndpoint, { + accessToken, + }), + fetchEndpointAsJson(stateTombstoneEndpoint, { + accessToken, + }), ]); let name; if (stateNameResDataOutcome.reason === undefined) { const { data } = stateNameResDataOutcome.value; - name = data.name; + name = data?.content?.name; } let canonicalAlias; if (stateCanonicalAliasResDataOutcome.reason === undefined) { const { data } = stateCanonicalAliasResDataOutcome.value; - canonicalAlias = data.alias; + canonicalAlias = data?.content?.alias; } let avatarUrl; if (stateAvatarResDataOutcome.reason === undefined) { const { data } = stateAvatarResDataOutcome.value; - avatarUrl = data.url; + avatarUrl = data?.content?.url; } let historyVisibility; if (stateHistoryVisibilityResDataOutcome.reason === undefined) { const { data } = stateHistoryVisibilityResDataOutcome.value; - historyVisibility = data.history_visibility; + historyVisibility = data?.content?.history_visibility; } let joinRule; if (stateJoinRulesResDataOutcome.reason === undefined) { const { data } = stateJoinRulesResDataOutcome.value; - joinRule = data.join_rule; + joinRule = data?.content?.join_rule; + } + + let roomCreationTs; + if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + roomCreationTs = data?.origin_server_ts; + } + + let predecessorRoomId; + let predecessorSetTs; + let predecessorViaServers; + if (statePredecessorResDataOutcome.reason === undefined) { + const { data } = statePredecessorResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor_room_id; + predecessorSetTs = data?.origin_server_ts; + predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); + } else if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor; + predecessorSetTs = data?.origin_server_ts; + } + + let successorRoomId; + let successorSetTs; + if (stateTombstoneResDataOutcome.reason === undefined) { + const { data } = stateTombstoneResDataOutcome.value; + successorRoomId = data?.content?.replacement_room; + successorSetTs = data?.origin_server_ts; } return { @@ -96,6 +157,12 @@ async function fetchRoomData(accessToken, roomId) { avatarUrl, historyVisibility, joinRule, + roomCreationTs, + predecessorRoomId, + predecessorViaServers, + predecessorSetTs, + successorRoomId, + successorSetTs, }; } diff --git a/server/lib/parse-via-servers-from-user-input.js b/server/lib/parse-via-servers-from-user-input.js new file mode 100644 index 00000000..ce6cc6e0 --- /dev/null +++ b/server/lib/parse-via-servers-from-user-input.js @@ -0,0 +1,29 @@ +'use strict'; + +const StatusError = require('../lib/status-error'); + +function parseViaServersFromUserInput(rawViaServers) { + // `rawViaServers` could be an array or a single string. Turn it into an array no matter what + const rawViaServerList = [].concat(rawViaServers || []); + if (rawViaServerList.length === 0) { + return new Set(); + } + + const viaServerList = rawViaServerList.map((viaServer) => { + // Sanity check to ensure that the via servers are strings (valid enough looking + // host names) + if (typeof viaServer !== 'string') { + throw new StatusError( + 400, + `?via server must be a string, got ${viaServer} (${typeof viaServer})` + ); + } + + return viaServer; + }); + + // We use a `Set` to ensure that we don't have duplicate servers in the list + return new Set(viaServerList); +} + +module.exports = parseViaServersFromUserInput; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 551a0974..837e480c 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -11,6 +11,7 @@ const timeoutMiddleware = require('./timeout-middleware'); const redirectToCorrectArchiveUrlIfBadSigil = require('./redirect-to-correct-archive-url-if-bad-sigil-middleware'); const { HTTPResponseError } = require('../lib/fetch-endpoint'); +const parseViaServersFromUserInput = require('../lib/parse-via-servers-from-user-input'); const fetchRoomData = require('../lib/matrix-utils/fetch-room-data'); const fetchEventsFromTimestampBackwards = require('../lib/matrix-utils/fetch-events-from-timestamp-backwards'); const ensureRoomJoined = require('../lib/matrix-utils/ensure-room-joined'); @@ -156,29 +157,6 @@ function parseArchiveRangeFromReq(req) { }; } -function parseViaServersFromReq(req) { - const rawViaServers = [].concat(req.query.via || []); - if (rawViaServers.length === 0) { - return new Set(); - } - - const viaServerList = rawViaServers.map((viaServer) => { - // Sanity check to ensure that the via servers are strings (valid enough looking - // host names) - if (typeof viaServer !== 'string') { - throw new StatusError( - 400, - `?via server must be a string, got ${viaServer} (${typeof viaServer})` - ); - } - - return viaServer; - }); - - // We use a `Set` to ensure that we don't have duplicate servers in the list - return new Set(viaServerList); -} - router.use(redirectToCorrectArchiveUrlIfBadSigil); router.get( @@ -196,7 +174,7 @@ router.get( const roomId = await ensureRoomJoined( matrixAccessToken, roomIdOrAlias, - parseViaServersFromReq(req) + parseViaServersFromUserInput(req.query.via) ); // Find the closest day to the current time with messages @@ -216,7 +194,7 @@ router.get( // We can avoid passing along the `via` query parameter because we already // joined the room above (see `ensureRoomJoined`). // - //viaServers: parseViaServersFromReq(req), + //viaServers: parseViaServersFromUserInput(req.query.via), }) ); }) @@ -272,7 +250,7 @@ router.get( const roomId = await ensureRoomJoined( matrixAccessToken, roomIdOrAlias, - parseViaServersFromReq(req) + parseViaServersFromUserInput(req.query.via) ); let eventIdForClosestEvent; @@ -551,23 +529,12 @@ router.get( precisionFromUrl = TIME_PRECISION_VALUES.minutes; } - // Just 404 if anyone is trying to view the future, no need to waste resources on that - const nowTs = Date.now(); - if (toTimestamp > roundUpTimestampToUtcDay(nowTs)) { - throw new StatusError( - 404, - `You can't view the history of a room on a future day (${new Date( - toTimestamp - ).toISOString()} > ${new Date(nowTs).toISOString()}). Go back` - ); - } - // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. const roomId = await ensureRoomJoined( matrixAccessToken, roomIdOrAlias, - parseViaServersFromReq(req) + parseViaServersFromUserInput(req.query.via) ); // Do these in parallel to avoid the extra time in sequential round-trips @@ -606,6 +573,68 @@ router.get( ); } + // Check if we need to navigate to the predecessor room + if ( + roomData?.predecessorRoomId && + // Since we're looking backwards from the given day, if we don't see any events, + // then we can assume that it's before the start of the room (it's the only way we + // would see no events). + events.length === 0 + ) { + const roomCreationTs = roomData?.roomCreationTs; + if (!roomCreationTs) { + throw new StatusError( + 500, + 'Unable to fetch room creation event to determine time when room was created' + ); + } + + // Jump to the predecessor room and continue at the last event of the room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData?.predecessorRoomId, { + viaServers: Array.from(roomData?.predecessorViaServers || []), + dir: DIRECTION.backward, + currentRangeStartTs: roomCreationTs, + currentRangeEndTs: toTimestamp, + }) + ); + } + + const nowTs = Date.now(); + + // We only care to navigate to the successor room if we're trying to view something + // past when the successor was set (it's an indicator that we need to go to the new + // room from this time forward). + const isNavigatedPastSuccessor = toTimestamp > roomData?.successorSetTs; + // But if we're viewing the day when the successor was set, we want to allow viewing + // the room up until the successor was set. + const newestEvent = events[events.length - 1]; + const isNewestEventFromSameDay = + newestEvent && + newestEvent?.origin_server_ts && + areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); + // Check if we need to navigate to the successor room + if (roomData?.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { + // Jump to the successor room and continue at the first event of the room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData?.successorRoomId, { + dir: DIRECTION.forward, + currentRangeStartTs: 0, + currentRangeEndTs: 0, + }) + ); + } + // If no successor, just 404 if anyone is trying to view the future, no need to + // waste resources on that + else if (toTimestamp > roundUpTimestampToUtcDay(nowTs)) { + throw new StatusError( + 404, + `You can't view the history of a room on a future day (${new Date( + toTimestamp + ).toISOString()} > ${new Date(nowTs).toISOString()}). Go back` + ); + } + // Default to no indexing (safe default) let shouldIndex = false; if (stopSearchEngineIndexing) { diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js index 02e5ca3f..1d3ebf68 100644 --- a/shared/lib/url-creator.js +++ b/shared/lib/url-creator.js @@ -58,6 +58,7 @@ class URLCreator { archiveUrlForRoom(roomIdOrAlias, { viaServers = [] } = {}) { assert(roomIdOrAlias); + assert(Array.isArray(viaServers)); let qs = new URLSearchParams(); [].concat(viaServers).forEach((viaServer) => { qs.append('via', viaServer); @@ -75,6 +76,7 @@ class URLCreator { ) { assert(roomIdOrAlias); assert(date); + assert(Array.isArray(viaServers)); // `preferredPrecision` is optional but if they gave a value, make sure it's something expected if (preferredPrecision) { assert( @@ -117,16 +119,23 @@ class URLCreator { )}${qsToUrlPiece(qs)}`; } - archiveJumpUrlForRoom(roomIdOrAlias, { dir, currentRangeStartTs, currentRangeEndTs }) { + archiveJumpUrlForRoom( + roomIdOrAlias, + { dir, currentRangeStartTs, currentRangeEndTs, viaServers = [] } + ) { assert(roomIdOrAlias); assert(dir); - assert(currentRangeStartTs); - assert(currentRangeEndTs); + assert(typeof currentRangeStartTs === 'number'); + assert(typeof currentRangeEndTs === 'number'); + assert(Array.isArray(viaServers)); let qs = new URLSearchParams(); qs.append('dir', dir); qs.append('currentRangeStartTs', currentRangeStartTs); qs.append('currentRangeEndTs', currentRangeEndTs); + [].concat(viaServers).forEach((viaServer) => { + qs.append('via', viaServer); + }); const urlPath = this._getArchiveUrlPathForRoomIdOrAlias(roomIdOrAlias); diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 9dd9a692..cf188af5 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -134,7 +134,7 @@ describe('matrix-public-archive', () => { true, `Expected ${event.event_id} (${event.type}: "${ event.content.body - }") to be in room on hs2=${JSON.stringify(room2EventIds)}` + }") to be in room on hs2=${JSON.stringify(room2EventIds, null, 2)}` ); }); }); @@ -819,7 +819,7 @@ describe('matrix-public-archive', () => { if (previousRoomId) { roomId = await upgradeTestRoom({ client, - previousRoomId, + oldRoomId: previousRoomId, //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it }); } else { @@ -914,7 +914,9 @@ describe('matrix-public-archive', () => { assert( startRoomIdOrAlias, `Could not find room ID for ${startRoomFancyKey} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()), + null, + 2 )}` ); archiveUrl = `${matrixPublicArchiveURLCreator.archiveUrlForRoom( @@ -943,7 +945,9 @@ describe('matrix-public-archive', () => { assert( expectedRoomId, `Could not find room ID for ${expectedRoomFancyId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()), + null, + 2 )}` ); @@ -962,7 +966,9 @@ describe('matrix-public-archive', () => { assert( actualRoomFancyId, `Could not find room ID for ${actualRoomId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(roomIdToFancyIdentifierMap.entries()) + Object.fromEntries(roomIdToFancyIdentifierMap.entries()), + null, + 2 )}` ); let actualContinueAtEventFancyId; @@ -972,7 +978,9 @@ describe('matrix-public-archive', () => { assert( actualContinueAtEventFancyId, `Could not find event ID for ${actualContinueAtEventId} in our list of known events ${JSON.stringify( - Object.fromEntries(eventIdToFancyIdentifierMap.entries()) + Object.fromEntries(eventIdToFancyIdentifierMap.entries()), + null, + 2 )}` ); } diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 495c57ea..e2a0af81 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -107,7 +107,7 @@ async function sendEvent({ client, roomId, eventType, stateKey, content, timesta } let url; - if (stateKey) { + if (typeof stateKey === 'string') { url = urlJoin( client.homeserverUrl, `/_matrix/client/v3/rooms/${encodeURIComponent( @@ -209,8 +209,9 @@ async function upgradeTestRoom({ // Now send the tombstone event pointing from the old room to the new room await sendEvent({ client, - oldRoomId, + roomId: oldRoomId, eventType: 'm.room.tombstone', + stateKey: '', content: { replacement_room: newRoomid, }, From 7f98efa35e0efaf90f862a17fedbd033e5b4ade0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Apr 2023 17:43:29 -0500 Subject: [PATCH 03/30] Better log when event is missing from expected --- package-lock.json | 11 +++++-- package.json | 1 + server/routes/room-routes.js | 1 + test/e2e-tests.js | 56 ++++++++++++++++++++++++++++++--- test/test-utils/client-utils.js | 23 ++++++++++++++ 5 files changed, 85 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index dc7c0584..38890352 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "url-join": "^4.0.1" }, "devDependencies": { + "chalk": "^4.1.2", "escape-string-regexp": "^4.0.0", "eslint": "^8.37.0", "eslint-config-prettier": "^8.8.0", @@ -2463,8 +2464,9 @@ }, "node_modules/chalk": { "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, - "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -5480,8 +5482,9 @@ }, "node_modules/supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, - "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -7494,6 +7497,8 @@ }, "chalk": { "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, "requires": { "ansi-styles": "^4.1.0", @@ -9371,6 +9376,8 @@ }, "supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "requires": { "has-flag": "^4.0.0" diff --git a/package.json b/package.json index b1612499..3f5b43e3 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "node": ">=16.0.0" }, "devDependencies": { + "chalk": "^4.1.2", "escape-string-regexp": "^4.0.0", "eslint": "^8.37.0", "eslint-config-prettier": "^8.8.0", diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 837e480c..d11dd65a 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -262,6 +262,7 @@ router.get( // updated value between each e2e test const archiveMessageLimit = config.get('archiveMessageLimit'); + console.log(`jumping from ${new Date(ts).toISOString()} (${ts}) in direction ${dir}`); // Find the closest event to the given timestamp ({ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent } = await timestampToEvent({ diff --git a/test/e2e-tests.js b/test/e2e-tests.js index cf188af5..7f1fba45 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -9,6 +9,7 @@ const urlJoin = require('url-join'); const escapeStringRegexp = require('escape-string-regexp'); const { parseHTML } = require('linkedom'); const { readFile } = require('fs').promises; +const chalk = require('chalk'); const RethrownError = require('../server/lib/rethrown-error'); const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); @@ -31,6 +32,7 @@ const { sendEvent, sendMessage, createMessagesInRoom, + getMessagesInRoom, updateProfile, uploadContent, } = require('./test-utils/client-utils'); @@ -774,7 +776,7 @@ describe('matrix-public-archive', () => { const eventId = fancyIdentifierToEventIdMap.get(fancyId); if (!eventId) { throw new Error( - `Unable to find ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( + `Could not find fancy ID ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( Object.fromEntries(fancyIdentifierToEventIdMap.entries()), null, 2 @@ -793,7 +795,7 @@ describe('matrix-public-archive', () => { const fancyEventId = eventIdToFancyIdentifierMap.get(eventId); if (!fancyEventId) { throw new Error( - `Unable to find ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( + `Could not find event ID for ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( Object.fromEntries(eventIdToFancyIdentifierMap.entries()), null, 2 @@ -872,6 +874,50 @@ describe('matrix-public-archive', () => { previousRoomId = roomId; } + // Assemble a list of events to to reference and assist with debugging when + // some assertion fails + const fancyRoomIdToDebugEventsInRoom = new Map(); + for (const [fancyRoomId, roomId] of fancyIdentifierToRoomIdMap.entries()) { + const archiveAppServiceUserClient = await getTestClientForAs(); + const eventsInRoom = await getMessagesInRoom({ + client: archiveAppServiceUserClient, + roomId: roomId, + // This is arbitrarily larger than any amount of messages we would ever + // send in the tests + limit: 1000, + }); + const eventDebugStrings = eventsInRoom.map((event) => { + let relevantContentString = ''; + if (event.type === 'm.room.message' && event.content.msgtype === 'm.text') { + relevantContentString = ` "${event.content.body}"`; + } + + return `${event.type}${event.state_key ? ` (${event.state_key})` : ''}: ${ + event.event_id + }${relevantContentString} - ${new Date(event.origin_server_ts).toISOString()}`; + }); + + fancyRoomIdToDebugEventsInRoom.set(fancyRoomId, eventDebugStrings); + } + + // String used to log out all possible events in the room + function getDebugStringForEventsInRoomsAndLookForEventId(eventIdToLookFor) { + return `For reference, here are all of the events in the room: ${JSON.stringify( + Object.fromEntries(fancyRoomIdToDebugEventsInRoom.entries()), + null, + 2 + ) + .split(/\r?\n/) + .map((line) => { + if (line.includes(eventIdToLookFor)) { + return chalk.yellow(line); + } + + return line; + }) + .join('\n')}`; + } + // Now Test // -------------------------------------- // -------------------------------------- @@ -977,11 +1023,11 @@ describe('matrix-public-archive', () => { eventIdToFancyIdentifierMap.get(actualContinueAtEventId); assert( actualContinueAtEventFancyId, - `Could not find event ID for ${actualContinueAtEventId} in our list of known events ${JSON.stringify( + `Could not find event ID for ${actualContinueAtEventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( Object.fromEntries(eventIdToFancyIdentifierMap.entries()), null, 2 - )}` + )}\n${getDebugStringForEventsInRoomsAndLookForEventId(actualContinueAtEventId)}` ); } @@ -1064,7 +1110,7 @@ describe('matrix-public-archive', () => { archiveUrl = jumpToActivityLinkHref; } catch (err) { const errorWithContext = new RethrownError( - `Encountered error while asserting ${pageKey}: ${err.message}`, + `Encountered error while asserting ${pageKey}: (see original error below)`, err ); // Copy these over so mocha generates a nice diff for us diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index e2a0af81..cf1bb3c4 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -307,6 +307,28 @@ async function createMessagesInRoom({ return { eventIds, eventMap }; } +async function getMessagesInRoom({ client, roomId, limit }) { + assert(client); + assert(roomId); + assert(limit); + + let qs = new URLSearchParams(); + qs.append('limit', limit); + + const { data } = await fetchEndpointAsJson( + urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/messages?${qs.toString()}` + ), + { + method: 'GET', + accessToken: client.accessToken, + } + ); + + return data.chunk; +} + async function updateProfile({ client, displayName, avatarUrl }) { let qs = new URLSearchParams(); if (client.applicationServiceUserIdOverride) { @@ -399,6 +421,7 @@ module.exports = { sendEvent, sendMessage, createMessagesInRoom, + getMessagesInRoom, updateProfile, uploadContent, }; From 63fac4880f118c6ed001e1fbcf87f5d214f16a6f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 04:52:01 -0500 Subject: [PATCH 04/30] Add note on alternative place to start from --- server/routes/room-routes.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index d11dd65a..9db859af 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -595,6 +595,9 @@ router.get( matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData?.predecessorRoomId, { viaServers: Array.from(roomData?.predecessorViaServers || []), dir: DIRECTION.backward, + // XXX: Should we start from the tombstone event in the predecessor room that + // points to this room if it exists? (this would require another lookup that + // we might want to avoid) currentRangeStartTs: roomCreationTs, currentRangeEndTs: toTimestamp, }) From d27b6c24a8faf01dec982a63696b283c3f16aabb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 16:13:50 -0500 Subject: [PATCH 05/30] Add comment explaining what we're parsing here --- .../get-server-name-from-matrix-room-id-or-alias.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js index 1ddcf5a0..1f515454 100644 --- a/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js +++ b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js @@ -4,6 +4,9 @@ const assert = require('assert'); // See https://spec.matrix.org/v1.5/appendices/#server-name function getServerNameFromMatrixRoomIdOrAlias(roomIdOrAlias) { + // `roomIdOrAlias` looks like `!foo:matrix.org` or `#foo:matrix.org` or even something + // as crazy as `!foo:[1234:5678::abcd]:1234` where `[1234:5678::abcd]:1234` is the + // server name part we're trying to parse out (see tests for more examples) assert(roomIdOrAlias); const pieces = roomIdOrAlias.split(':'); From 0e214495bf9828382af25f8360ff072a189967f8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 19:05:54 -0500 Subject: [PATCH 06/30] Crude working to jump backwards to predecessor in first test case --- package.json | 2 +- server/lib/matrix-utils/fetch-room-data.js | 210 ++++++++++-------- server/routes/room-routes.js | 87 +++++--- test/e2e-tests.js | 55 ++++- test/test-utils/client-utils.js | 4 + .../parse-room-day-message-structure.js | 10 - 6 files changed, 230 insertions(+), 138 deletions(-) diff --git a/package.json b/package.json index 3f5b43e3..3d29a6f4 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "start": "node server/server.js", "start-dev": "node server/start-dev.js", "test": "npm run mocha -- test/**/*-tests.js --timeout 15000", - "test-e2e-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --interactive", + "test-e2e-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --bail --interactive", "nodemon": "nodemon", "vite": "vite", "mocha": "mocha", diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index 81aed601..8441ff89 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -4,6 +4,7 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson } = require('../fetch-endpoint'); +const ensureRoomJoined = require('./ensure-room-joined'); const parseViaServersFromUserInput = require('../parse-via-servers-from-user-input'); const { traceFunction } = require('../../tracing/trace-utilities'); @@ -11,86 +12,137 @@ const config = require('../config'); const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); +function getStateEndpointForRoomIdAndEventType(roomId, eventType) { + return urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent( + eventType + )}?format=event` + ); +} + // eslint-disable-next-line max-statements -async function fetchRoomData(accessToken, roomId) { - assert(accessToken); +async function fetchRoomData(matrixAccessToken, roomId) { + assert(matrixAccessToken); assert(roomId); - const stateCreateEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.create?format=event` - ); - const stateNameEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.name?format=event` - ); - const canoncialAliasEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent( - roomId - )}/state/m.room.canonical_alias?format=event` - ); - const stateAvatarEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.avatar?format=event` - ); - const stateHistoryVisibilityEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent( - roomId - )}/state/m.room.history_visibility?format=event` - ); - const stateJoinRulesEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.join_rules?format=event` - ); + const mainFetchPromiseBundle = Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.name'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.canonical_alias'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.avatar'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'm.room.history_visibility'), + { + accessToken: matrixAccessToken, + } + ), + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.join_rules'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.tombstone'), { + accessToken: matrixAccessToken, + }), + ]); - const statePredecessorEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent( - roomId - )}/state/org.matrix.msc3946.room_predecessor?format=event` - ); - const stateTombstoneEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.tombstone?format=event` - ); + const predessorFetchPromiseBundle = Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), + { + accessToken: matrixAccessToken, + } + ), + ]); + + const predecessorInfoPromise = (async () => { + const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = + await predessorFetchPromiseBundle; + + let predecessorRoomId; + let predecessorViaServers; + if (statePredecessorResDataOutcome.reason === undefined) { + const { data } = statePredecessorResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor_room_id; + predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); + } else if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor; + } + + let roomCreationTs; + if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + roomCreationTs = data?.origin_server_ts; + } + + return { + roomCreationTs, + predecessorRoomId, + predecessorViaServers, + }; + })(); + + // TODO: This is pretty ugly/messy. Refactor this and maybe a different pattern for this type of thing. + let _predecessorRoomTombstoneSetTs; + const getPredecessorRoomTombstoneSetTs = async () => { + if (_predecessorRoomTombstoneSetTs) { + return _predecessorRoomTombstoneSetTs; + } + + const { predecessorRoomId, predecessorViaServers } = await predecessorInfoPromise; + + if (predecessorRoomId) { + await ensureRoomJoined(matrixAccessToken, predecessorRoomId, predecessorViaServers); + + // Fetch the tombstone from the predessor room + const [predecessorStateTombstoneResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(predecessorRoomId, 'm.room.tombstone'), + { + accessToken: matrixAccessToken, + } + ), + ]); + + let predecessorSuccessorRoomId; + let predecessorSuccessorSetTs; + if (predecessorStateTombstoneResDataOutcome.reason === undefined) { + const { data } = predecessorStateTombstoneResDataOutcome.value; + predecessorSuccessorRoomId = data?.content?.replacement_room; + predecessorSuccessorSetTs = data?.origin_server_ts; + } + + // Make sure the the room that the predecessor specifies as the replacement room + // is the same as what the current room is. This is a good signal that the rooms + // are a true continuation of each other and the room admins agree. + if (predecessorSuccessorRoomId !== roomId) { + return null; + } + + _predecessorRoomTombstoneSetTs = predecessorSuccessorSetTs; + + return _predecessorRoomTombstoneSetTs; + } + + return null; + }; const [ - stateCreateResDataOutcome, stateNameResDataOutcome, stateCanonicalAliasResDataOutcome, stateAvatarResDataOutcome, stateHistoryVisibilityResDataOutcome, stateJoinRulesResDataOutcome, - statePredecessorResDataOutcome, stateTombstoneResDataOutcome, - ] = await Promise.allSettled([ - fetchEndpointAsJson(stateCreateEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateNameEndpoint, { - accessToken, - }), - fetchEndpointAsJson(canoncialAliasEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateAvatarEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateHistoryVisibilityEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateJoinRulesEndpoint, { - accessToken, - }), - fetchEndpointAsJson(statePredecessorEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateTombstoneEndpoint, { - accessToken, - }), - ]); + ] = await mainFetchPromiseBundle; let name; if (stateNameResDataOutcome.reason === undefined) { @@ -122,25 +174,7 @@ async function fetchRoomData(accessToken, roomId) { joinRule = data?.content?.join_rule; } - let roomCreationTs; - if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - roomCreationTs = data?.origin_server_ts; - } - - let predecessorRoomId; - let predecessorSetTs; - let predecessorViaServers; - if (statePredecessorResDataOutcome.reason === undefined) { - const { data } = statePredecessorResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor_room_id; - predecessorSetTs = data?.origin_server_ts; - predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); - } else if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor; - predecessorSetTs = data?.origin_server_ts; - } + const { roomCreationTs, predecessorRoomId, predecessorViaServers } = await predecessorInfoPromise; let successorRoomId; let successorSetTs; @@ -160,7 +194,7 @@ async function fetchRoomData(accessToken, roomId) { roomCreationTs, predecessorRoomId, predecessorViaServers, - predecessorSetTs, + getPredecessorRoomTombstoneSetTs, successorRoomId, successorSetTs, }; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 9db859af..4f275689 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -564,44 +564,69 @@ router.get( // Only `world_readable` or `shared` rooms that are `public` are viewable in the archive const allowedToViewRoom = - roomData?.historyVisibility === 'world_readable' || - (roomData?.historyVisibility === 'shared' && roomData?.joinRule === 'public'); + roomData.historyVisibility === 'world_readable' || + (roomData.historyVisibility === 'shared' && roomData.joinRule === 'public'); if (!allowedToViewRoom) { throw new StatusError( 403, - `Only \`world_readable\` or \`shared\` rooms that are \`public\` can be viewed in the archive. ${roomData.id} has m.room.history_visiblity=${roomData?.historyVisibility} m.room.join_rules=${roomData?.joinRule}` + `Only \`world_readable\` or \`shared\` rooms that are \`public\` can be viewed in the archive. ${roomData.id} has m.room.history_visiblity=${roomData.historyVisibility} m.room.join_rules=${roomData.joinRule}` ); } - // Check if we need to navigate to the predecessor room - if ( - roomData?.predecessorRoomId && - // Since we're looking backwards from the given day, if we don't see any events, - // then we can assume that it's before the start of the room (it's the only way we - // would see no events). - events.length === 0 - ) { - const roomCreationTs = roomData?.roomCreationTs; - if (!roomCreationTs) { + // Since we're looking backwards from the given day, if we don't see any events, + // then we can assume that it's before the start of the room (it's the only way we + // would see no events). + const hasNavigatedBeforeStartOfRoom = events.length === 0; + // Check if we need to navigate backward to the predecessor room + if (hasNavigatedBeforeStartOfRoom && roomData.predecessorRoomId) { + const PREDECESSOR_ROOM_TOMBSTONE_FETCH_TIMEOUT_MS = 8 * ONE_SECOND_IN_MS; + const predecessorRoomTombstoneSetTs = await Promise.race([ + // Try to continue from the tombstone event in the predecessor room because that + // is the signal that the room admins gave to indicate the end of the room in + // favor of the other regardless of further activity that may have occured in + // the room. + roomData.getPredecessorRoomTombstoneSetTs(), + // If it takes too long to get the predecessor room tombstone, then abort and + // will fallback to something else + new Promise((resolve) => { + setTimeout(resolve, PREDECESSOR_ROOM_TOMBSTONE_FETCH_TIMEOUT_MS); + }), + ]); + + // Fallback to the room creation event time if we can't find the predecessor room + // tombstone which will work just fine and as expected for normal room upgrade + // scenarios. + const continueAtTsInPredecessorRoom = + predecessorRoomTombstoneSetTs ?? roomData.roomCreationTs; + + console.log( + 'continueAtTsInPredecessorRoom', + new Date(continueAtTsInPredecessorRoom).toISOString(), + continueAtTsInPredecessorRoom + ); + + if (continueAtTsInPredecessorRoom === null || continueAtTsInPredecessorRoom === undefined) { throw new StatusError( 500, - 'Unable to fetch room creation event to determine time when room was created' + `You navigated past the end of the room and it has a predecessor set ` + + `(${roomData.predecessorRoomId}) but we were unable to find a suitable place to jump to. ` + + `We could just redirect you to that predecessor room but we decided to throw an error ` + + `instead because we should be able to fallback to the room creation time in any case. ` + + `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + + `event unless the server is just broken. You can try refreshing to try again.` + ); + } else { + // Jump to the predecessor room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.predecessorRoomId, { + viaServers: Array.from(roomData.predecessorViaServers || []), + dir: DIRECTION.backward, + currentRangeStartTs: continueAtTsInPredecessorRoom, + currentRangeEndTs: toTimestamp, + }) ); } - - // Jump to the predecessor room and continue at the last event of the room - res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData?.predecessorRoomId, { - viaServers: Array.from(roomData?.predecessorViaServers || []), - dir: DIRECTION.backward, - // XXX: Should we start from the tombstone event in the predecessor room that - // points to this room if it exists? (this would require another lookup that - // we might want to avoid) - currentRangeStartTs: roomCreationTs, - currentRangeEndTs: toTimestamp, - }) - ); } const nowTs = Date.now(); @@ -609,7 +634,7 @@ router.get( // We only care to navigate to the successor room if we're trying to view something // past when the successor was set (it's an indicator that we need to go to the new // room from this time forward). - const isNavigatedPastSuccessor = toTimestamp > roomData?.successorSetTs; + const isNavigatedPastSuccessor = toTimestamp > roomData.successorSetTs; // But if we're viewing the day when the successor was set, we want to allow viewing // the room up until the successor was set. const newestEvent = events[events.length - 1]; @@ -617,11 +642,11 @@ router.get( newestEvent && newestEvent?.origin_server_ts && areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); - // Check if we need to navigate to the successor room - if (roomData?.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { + // Check if we need to navigate forward to the successor room + if (roomData.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { // Jump to the successor room and continue at the first event of the room res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData?.successorRoomId, { + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.successorRoomId, { dir: DIRECTION.forward, currentRangeStartTs: 0, currentRangeEndTs: 0, diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 7f1fba45..67525412 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -150,7 +150,7 @@ describe('matrix-public-archive', () => { const archiveDate = new Date(Date.UTC(2022, 0, 15)); let archiveUrl; let numMessagesSent = 0; - afterEach(() => { + afterEach(function () { if (interactive) { // eslint-disable-next-line no-console console.log('Interactive URL for test', archiveUrl); @@ -162,8 +162,16 @@ describe('matrix-public-archive', () => { // changes in the UI). numMessagesSent = 0; - // Reset any custom modifications made for a particular test - config.reset(); + // Reset any custom modifications made for a particular test. + // + // We don't reset when interactive and there is a failure because we want to be + // able to preview the interactive URL with the same config as seen in the test + // and because we also set `--bail` when running `npm run test-e2e-interactive`, + // the tests stop running after the first failure so it doesn't leak state between + // tests. + if (!interactive && this.currentTest.state !== 'passed') { + config.reset(); + } }); // Sends a message and makes sure that a timestamp was provided @@ -810,21 +818,27 @@ describe('matrix-public-archive', () => { const client = await getTestClientForHs(testMatrixServerUrl1); - const { rooms, archiveMessageLimit, pages } = parseRoomDayMessageStructure( + const { rooms, pages } = parseRoomDayMessageStructure( testCase.roomDayMessageStructureString ); const fancyIdentifierToRoomIdMap = new Map(); const roomIdToFancyIdentifierMap = new Map(); let previousRoomId; + let lastEventTsUsedInPreviousRoom; for (const [roomIndex, room] of rooms.entries()) { let roomId; if (previousRoomId) { roomId = await upgradeTestRoom({ client, oldRoomId: previousRoomId, - //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it + //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it. + // We `+ 1` just to space out the tombstone from the last event so + // things are sequential `/timestamp_to_event` doesn't get confused. + timestamp: lastEventTsUsedInPreviousRoom + 1, }); } else { + // TODO: Pass `timestamp` to `createTestRoom()` when it supports it, see + // https://github.com/matrix-org/synapse/issues/15346 roomId = await createTestRoom(client); } const fancyRoomId = `#room${roomIndex + 1}`; @@ -869,6 +883,7 @@ describe('matrix-public-archive', () => { const fancyEventId = `$event${eventMeta.eventNumber}`; fancyIdentifierToEventIdMap.set(fancyEventId, eventId); eventIdToFancyIdentifierMap.set(eventId, fancyEventId); + lastEventTsUsedInPreviousRoom = originServerTs; } previousRoomId = roomId; @@ -923,7 +938,8 @@ describe('matrix-public-archive', () => { // -------------------------------------- // Make sure the archive is configured as the test expects - config.set('archiveMessageLimit', archiveMessageLimit); + assert(testCase.archiveMessageLimit); + config.set('archiveMessageLimit', testCase.archiveMessageLimit); // eslint-disable-next-line max-nested-callbacks const pagesKeyList = Object.keys(testCase).filter((key) => { @@ -1146,6 +1162,7 @@ describe('matrix-public-archive', () => { |--jump-fwd-4-messages-->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', @@ -1177,6 +1194,7 @@ describe('matrix-public-archive', () => { |--jump-fwd-4-messages-->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', @@ -1202,6 +1220,7 @@ describe('matrix-public-archive', () => { |--jump-fwd-4-messages-->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/04T01:00', page1: { url: '/r/room1/date/2022/01/04T01:00', @@ -1227,6 +1246,7 @@ describe('matrix-public-archive', () => { |---jump-fwd-4-messages--->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/03', page1: { url: '/r/room1/date/2022/01/03', @@ -1258,6 +1278,7 @@ describe('matrix-public-archive', () => { // |---jump-fwd-4-messages--->| // [page2 ] // `, + // archiveMessageLimit: 4, // startUrl: '/r/room1/date/2022/01/04', // page1: { // url: '/r/room1/date/2022/01/04', @@ -1294,6 +1315,7 @@ describe('matrix-public-archive', () => { |-jump-fwd-3-msg->| [page2 ] `, + archiveMessageLimit: 3, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', @@ -1323,6 +1345,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/03', page1: { url: '/r/room1/date/2022/01/03', @@ -1350,6 +1373,7 @@ describe('matrix-public-archive', () => { |------------------jump-fwd-8-msg---------------------->| [page2 ] `, + archiveMessageLimit: 8, startUrl: '/r/room1/date/2022/01/04', page1: { url: '/r/room1/date/2022/01/04', @@ -1373,6 +1397,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 8, startUrl: '/r/room1/date/2022/01/09', page1: { url: '/r/room1/date/2022/01/09', @@ -1400,6 +1425,7 @@ describe('matrix-public-archive', () => { |--jump-fwd-4-messages-->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', @@ -1426,6 +1452,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/04', page1: { url: '/r/room1/date/2022/01/04', @@ -1451,6 +1478,7 @@ describe('matrix-public-archive', () => { |---jump-fwd-4-messages--->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', @@ -1476,6 +1504,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/03', page1: { url: '/r/room1/date/2022/01/03', @@ -1501,6 +1530,7 @@ describe('matrix-public-archive', () => { |---jump-fwd-4-messages--->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02T6:00', page1: { url: '/r/room1/date/2022/01/02T6:00', @@ -1526,6 +1556,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02T11:00', page1: { url: '/r/room1/date/2022/01/02T11:00', @@ -1551,6 +1582,7 @@ describe('matrix-public-archive', () => { |---jump-fwd-4-messages--->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02T06:00', page1: { url: '/r/room1/date/2022/01/02T06:00', @@ -1577,6 +1609,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/03T06:00', page1: { url: '/r/room1/date/2022/01/03T06:00', @@ -1601,6 +1634,7 @@ describe('matrix-public-archive', () => { [page1 ] [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/03T05:00', page1: { url: '/r/room1/date/2022/01/03T05:00', @@ -1615,21 +1649,24 @@ describe('matrix-public-archive', () => { const jumpBackwardPredecessorTestCases = [ { + // Page2 doesn't only shows 4 messages ($event4-7) because it also has the + // tombstone event which is hidden testName: 'can jump backward from one room to the predecessor room', roomDayMessageStructureString: ` [room1 ] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [day3 ] [page1 ] - [page2 ] + [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room2/date/2022/01/03T05:00', page1: { url: '/r/room2/date/2022/01/03T05:00', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/02', + url: '/r/room1/date/2022/01/02T05:00?at=$event7', action: null, }, }, @@ -1643,6 +1680,7 @@ describe('matrix-public-archive', () => { [page2] [page3 ] `, + archiveMessageLimit: 4, startUrl: '/r/room2/date/2022/01/03T05:00', page1: { url: '/r/room2/date/2022/01/03T05:00', @@ -1670,6 +1708,7 @@ describe('matrix-public-archive', () => { |--jump-fwd-4-messages--->| [page2 ] `, + archiveMessageLimit: 4, startUrl: '/r/room1/date/2022/01/02', page1: { url: '/r/room1/date/2022/01/02', diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index cf1bb3c4..31d336f8 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -178,6 +178,7 @@ async function upgradeTestRoom({ oldRoomId, useMsc3946DynamicPredecessor = false, overrideCreateOptions = {}, + timestamp, }) { assert(client); assert(oldRoomId); @@ -204,6 +205,8 @@ async function upgradeTestRoom({ }; } + // TODO: Pass `timestamp` to `createTestRoom()` when it supports it, see + // https://github.com/matrix-org/synapse/issues/15346 const newRoomid = await createTestRoom(client, createOptions); // Now send the tombstone event pointing from the old room to the new room @@ -215,6 +218,7 @@ async function upgradeTestRoom({ content: { replacement_room: newRoomid, }, + timestamp, }); return newRoomid; diff --git a/test/test-utils/parse-room-day-message-structure.js b/test/test-utils/parse-room-day-message-structure.js index b9c4ab85..25416688 100644 --- a/test/test-utils/parse-room-day-message-structure.js +++ b/test/test-utils/parse-room-day-message-structure.js @@ -191,19 +191,9 @@ function parseRoomDayMessageStructure(roomDayMessageStructureString) { events, }; }); - // Ensure that each page has the same number of events on it - const numEventsOnEachPage = pages.map((page) => page.events.length); - // The page limit is X but each page will display X + 1 messages because we fetch one - // extra to determine overflow. - const archiveMessageLimit = numEventsOnEachPage[0] - 1; - assert( - numEventsOnEachPage.every((numEvents) => numEvents === archiveMessageLimit + 1), - `Expected all pages to have the same number of events (archiveMessageLimit + 1) where archiveMessageLimit=${archiveMessageLimit} but found ${numEventsOnEachPage}` - ); return { rooms, - archiveMessageLimit, pages, }; } From 14f6e7269ba67254e3d906642c642a972ee07f60 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 19:20:41 -0500 Subject: [PATCH 07/30] Fix m.room.create predecessor format --- server/lib/matrix-utils/fetch-room-data.js | 2 +- server/routes/room-routes.js | 6 +++--- test/e2e-tests.js | 4 ++-- test/test-utils/client-utils.js | 10 +++++++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index 8441ff89..e594ad9a 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -74,7 +74,7 @@ async function fetchRoomData(matrixAccessToken, roomId) { predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); } else if (stateCreateResDataOutcome.reason === undefined) { const { data } = stateCreateResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor; + predecessorRoomId = data?.content?.predecessor?.room_id; } let roomCreationTs; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 4f275689..f0cb38ac 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -609,12 +609,12 @@ router.get( if (continueAtTsInPredecessorRoom === null || continueAtTsInPredecessorRoom === undefined) { throw new StatusError( 500, - `You navigated past the end of the room and it has a predecessor set ` + - `(${roomData.predecessorRoomId}) but we were unable to find a suitable place to jump to. ` + + `You navigated past the end of the room and it has a predecessor set (${roomData.predecessorRoomId}) ` + + `but we were unable to find a suitable place to jump to and continue from. ` + `We could just redirect you to that predecessor room but we decided to throw an error ` + `instead because we should be able to fallback to the room creation time in any case. ` + `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + - `event unless the server is just broken. You can try refreshing to try again.` + `event for this room unless the server is just broken. You can try refreshing to try again.` ); } else { // Jump to the predecessor room diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 67525412..6bf50c6c 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -837,8 +837,8 @@ describe('matrix-public-archive', () => { timestamp: lastEventTsUsedInPreviousRoom + 1, }); } else { - // TODO: Pass `timestamp` to `createTestRoom()` when it supports it, see - // https://github.com/matrix-org/synapse/issues/15346 + // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it + // supports it, see https://github.com/matrix-org/synapse/issues/15346 roomId = await createTestRoom(client); } const fancyRoomId = `#room${roomIndex + 1}`; diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 31d336f8..049680ff 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -201,12 +201,16 @@ async function upgradeTestRoom({ ]; } else { createOptions.creation_content = { - predecessor: oldRoomId, + predecessor: { + room_id: oldRoomId, + // The event ID of the last known event in the old room (supposedly required). + //event_id: TODO, + }, }; } - // TODO: Pass `timestamp` to `createTestRoom()` when it supports it, see - // https://github.com/matrix-org/synapse/issues/15346 + // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it supports it, + // see https://github.com/matrix-org/synapse/issues/15346 const newRoomid = await createTestRoom(client, createOptions); // Now send the tombstone event pointing from the old room to the new room From 4549cc92f91caf0c800e2d428deed3c5dcee5251 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 20:47:25 -0500 Subject: [PATCH 08/30] Slight refactor --- server/lib/matrix-utils/fetch-room-data.js | 53 +++++++++++----------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index e594ad9a..2399f214 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -50,21 +50,18 @@ async function fetchRoomData(matrixAccessToken, roomId) { }), ]); - const predessorFetchPromiseBundle = Promise.allSettled([ - fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { - accessToken: matrixAccessToken, - }), - fetchEndpointAsJson( - getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), - { - accessToken: matrixAccessToken, - } - ), - ]); - const predecessorInfoPromise = (async () => { - const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = - await predessorFetchPromiseBundle; + const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), + { + accessToken: matrixAccessToken, + } + ), + ]); let predecessorRoomId; let predecessorViaServers; @@ -90,13 +87,9 @@ async function fetchRoomData(matrixAccessToken, roomId) { }; })(); - // TODO: This is pretty ugly/messy. Refactor this and maybe a different pattern for this type of thing. - let _predecessorRoomTombstoneSetTs; - const getPredecessorRoomTombstoneSetTs = async () => { - if (_predecessorRoomTombstoneSetTs) { - return _predecessorRoomTombstoneSetTs; - } - + // TODO: This is pretty ugly/messy. Refactor this and maybe use a different pattern + // for this type of thing. + const _getPredecessorRoomTombstoneSetTs = async () => { const { predecessorRoomId, predecessorViaServers } = await predecessorInfoPromise; if (predecessorRoomId) { @@ -123,18 +116,26 @@ async function fetchRoomData(matrixAccessToken, roomId) { // Make sure the the room that the predecessor specifies as the replacement room // is the same as what the current room is. This is a good signal that the rooms // are a true continuation of each other and the room admins agree. - if (predecessorSuccessorRoomId !== roomId) { - return null; + if (predecessorSuccessorRoomId === roomId) { + return predecessorSuccessorSetTs; } - _predecessorRoomTombstoneSetTs = predecessorSuccessorSetTs; - - return _predecessorRoomTombstoneSetTs; + return null; } return null; }; + // Memoize this function so we only ever do the extra work once + let _predecessorRoomTombstoneSetTsResult; + const getPredecessorRoomTombstoneSetTs = async () => { + if (_predecessorRoomTombstoneSetTsResult === undefined) { + _predecessorRoomTombstoneSetTsResult = _getPredecessorRoomTombstoneSetTs(); + } + + return _predecessorRoomTombstoneSetTsResult; + }; + const [ stateNameResDataOutcome, stateCanonicalAliasResDataOutcome, From ed6666754eb21673d04c4032929c98dabfc6cc7e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Apr 2023 20:52:09 -0500 Subject: [PATCH 09/30] Add note about assuming we are already joined --- server/routes/room-routes.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index f0cb38ac..8ad43fdc 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -532,6 +532,10 @@ router.get( // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. + // + // XXX: It would be better if we just tried fetching first and assume that we are + // already joined and only join after we see a 403 Forbidden error (we should do + // this for all places we `ensureRoomJoined`) const roomId = await ensureRoomJoined( matrixAccessToken, roomIdOrAlias, From 60b11dd3c49e080c59ee004c8c1cc5597ed7c8a0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 02:49:14 -0500 Subject: [PATCH 10/30] Re-usable URL converting utilities --- test/e2e-tests.js | 357 +++++++++++++++++++++++++++------------------- 1 file changed, 214 insertions(+), 143 deletions(-) diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 6bf50c6c..baf34930 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -777,6 +777,27 @@ describe('matrix-public-archive', () => { const eventMap = new Map(); const fancyIdentifierToEventIdMap = new Map(); const eventIdToFancyIdentifierMap = new Map(); + const fancyIdentifierToRoomIdMap = new Map(); + const roomIdToFancyIdentifierMap = new Map(); + const fancyRoomIdToDebugEventsInRoom = new Map(); + + // String used to log out all possible events in the room + function getDebugStringForEventsInRoomsAndLookForEventId(eventIdToLookFor) { + return `For reference, here are all of the events in the rooms: ${JSON.stringify( + Object.fromEntries(fancyRoomIdToDebugEventsInRoom.entries()), + null, + 2 + ) + .split(/\r?\n/) + .map((line) => { + if (line.includes(eventIdToLookFor)) { + return chalk.yellow(line); + } + + return line; + }) + .join('\n')}`; + } function convertFancyIdentifierListToDebugEventIds(fancyEventIdentifiers) { // eslint-disable-next-line max-nested-callbacks @@ -807,7 +828,7 @@ describe('matrix-public-archive', () => { Object.fromEntries(eventIdToFancyIdentifierMap.entries()), null, 2 - )}` + )}\n${getDebugStringForEventsInRoomsAndLookForEventId(eventId)}` ); } const ts = eventMap.get(eventId)?.originServerTs; @@ -816,13 +837,67 @@ describe('matrix-public-archive', () => { }); } + function convertUrlBetween(inputUrl, roomMap, eventMap) { + const { + roomIdOrAliasUrlPart: inputRoomIdOrAliasUrlPart, + roomIdOrAlias: inputRoomIdOrAlias, + //urlDateTime: actualUrlDateTime, + continueAtEvent: inputContinueAtEventId, + } = parseArchiveUrlForRoom(inputUrl); + + let outputContinueAtEventId; + if (inputContinueAtEventId) { + outputContinueAtEventId = eventMap.get(inputContinueAtEventId); + assert( + outputContinueAtEventId, + `Could not find event ID for ${inputContinueAtEventId} in the map=${JSON.stringify( + Object.fromEntries(eventMap.entries()), + null, + 2 + )}\n${getDebugStringForEventsInRoomsAndLookForEventId(inputContinueAtEventId)}` + ); + } + + const outputRoomIdOrAlias = roomMap.get(inputRoomIdOrAlias); + assert( + outputRoomIdOrAlias, + `Could not find room ID for ${inputRoomIdOrAlias} in our map ${JSON.stringify( + Object.fromEntries(roomMap.entries()), + null, + 2 + )}` + ); + + return inputUrl + .replace( + `/roomid/${inputRoomIdOrAliasUrlPart}`, + // Slice to remove the sigil + `/roomid/${outputRoomIdOrAlias.slice(1)}` + ) + .replace(inputContinueAtEventId, outputContinueAtEventId); + } + + function convertUrlWithFancyIdsToActualUrl(urlWithFancyIds) { + return convertUrlBetween( + urlWithFancyIds, + fancyIdentifierToRoomIdMap, + fancyIdentifierToEventIdMap + ); + } + + function convertActualUrlToUrlWithFancyIds(urlWithFancyIds) { + return convertUrlBetween( + urlWithFancyIds, + roomIdToFancyIdentifierMap, + eventIdToFancyIdentifierMap + ); + } + const client = await getTestClientForHs(testMatrixServerUrl1); const { rooms, pages } = parseRoomDayMessageStructure( testCase.roomDayMessageStructureString ); - const fancyIdentifierToRoomIdMap = new Map(); - const roomIdToFancyIdentifierMap = new Map(); let previousRoomId; let lastEventTsUsedInPreviousRoom; for (const [roomIndex, room] of rooms.entries()) { @@ -841,7 +916,7 @@ describe('matrix-public-archive', () => { // supports it, see https://github.com/matrix-org/synapse/issues/15346 roomId = await createTestRoom(client); } - const fancyRoomId = `#room${roomIndex + 1}`; + const fancyRoomId = `!room${roomIndex + 1}`; fancyIdentifierToRoomIdMap.set(fancyRoomId, roomId); roomIdToFancyIdentifierMap.set(roomId, fancyRoomId); @@ -876,6 +951,7 @@ describe('matrix-public-archive', () => { timestamp: originServerTs, }); eventMap.set(eventId, { + type: 'm.room.message', roomId, originServerTs, content, @@ -891,7 +967,6 @@ describe('matrix-public-archive', () => { // Assemble a list of events to to reference and assist with debugging when // some assertion fails - const fancyRoomIdToDebugEventsInRoom = new Map(); for (const [fancyRoomId, roomId] of fancyIdentifierToRoomIdMap.entries()) { const archiveAppServiceUserClient = await getTestClientForAs(); const eventsInRoom = await getMessagesInRoom({ @@ -915,24 +990,6 @@ describe('matrix-public-archive', () => { fancyRoomIdToDebugEventsInRoom.set(fancyRoomId, eventDebugStrings); } - // String used to log out all possible events in the room - function getDebugStringForEventsInRoomsAndLookForEventId(eventIdToLookFor) { - return `For reference, here are all of the events in the room: ${JSON.stringify( - Object.fromEntries(fancyRoomIdToDebugEventsInRoom.entries()), - null, - 2 - ) - .split(/\r?\n/) - .map((line) => { - if (line.includes(eventIdToLookFor)) { - return chalk.yellow(line); - } - - return line; - }) - .join('\n')}`; - } - // Now Test // -------------------------------------- // -------------------------------------- @@ -1018,46 +1075,9 @@ describe('matrix-public-archive', () => { archiveUrl ); const pageDom = parseHTML(archivePageHtml); - const { - roomIdOrAliasUrlPart: actualRoomIdOrAliasUrlPart, - roomIdOrAlias: actualRoomId, - //urlDateTime: actualUrlDateTime, - continueAtEvent: actualContinueAtEventId, - } = parseArchiveUrlForRoom(pageRes.url); - const actualRoomFancyId = roomIdToFancyIdentifierMap.get(actualRoomId); - assert( - actualRoomFancyId, - `Could not find room ID for ${actualRoomId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(roomIdToFancyIdentifierMap.entries()), - null, - 2 - )}` - ); - let actualContinueAtEventFancyId; - if (actualContinueAtEventId) { - actualContinueAtEventFancyId = - eventIdToFancyIdentifierMap.get(actualContinueAtEventId); - assert( - actualContinueAtEventFancyId, - `Could not find event ID for ${actualContinueAtEventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( - Object.fromEntries(eventIdToFancyIdentifierMap.entries()), - null, - 2 - )}\n${getDebugStringForEventsInRoomsAndLookForEventId(actualContinueAtEventId)}` - ); - } - // Replace messy room ID's and event ID's that change with every test - // run with their fancy ID's which correlate with the test meta so it's - // easier to reason about things when the assertion fails. - let actualUrlWithFancyIdentifies = pageRes.url - .replace( - `/roomid/${actualRoomIdOrAliasUrlPart}`, - // Slice to remove the sigil - `/r/${actualRoomFancyId.slice(1)}` - ) - .replace(actualContinueAtEventId, actualContinueAtEventFancyId); // Assert the correct room and time precision in the URL + const actualUrlWithFancyIdentifies = convertActualUrlToUrlWithFancyIds(pageRes.url); assert.match( actualUrlWithFancyIdentifies, new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`) @@ -1088,6 +1108,21 @@ describe('matrix-public-archive', () => { return eventEl.getAttribute('data-event-id'); }); + // We only care about messages for now (no easy way to specify the + // primordial room creation or member events) + const eventIdsOnPageWeCareAboutToAssert = eventIdsOnPage.filter((eventId) => { + const event = eventMap.get(eventId); + if (!event) { + return false; + } + + assert( + event?.type, + `Event should have a type: ${JSON.stringify(event, null, 2)}}` + ); + return event?.type === 'm.room.message'; + }); + const pageNumber = pageKey.replace('page', ''); const page = pages[pageNumber - 1]; const expectedEventsOnPage = page.events; @@ -1098,17 +1133,22 @@ describe('matrix-public-archive', () => { // Assert that the page contains all expected events assert.deepEqual( - convertEventIdsToDebugEventIds(eventIdsOnPage), + convertEventIdsToDebugEventIds(eventIdsOnPageWeCareAboutToAssert), convertFancyIdentifierListToDebugEventIds(expectedFancyIdsOnPage), `Events on ${pageKey} should be as expected` ); // Follow the next activity link. Aka, fetch messages for the 2nd page let actionLinkSelector; + let nextPageLink; if (pageTestMeta.action === 'next') { actionLinkSelector = '[data-testid="jump-to-next-activity-link"]'; } else if (pageTestMeta.action === 'previous') { actionLinkSelector = '[data-testid="jump-to-previous-activity-link"]'; + } else if (pageTestMeta.action?.startsWith('navigate:')) { + const navigateUrlWithFancyIds = pageTestMeta.action.replace('navigate:', ''); + const fullNavigateUrlWithFancyIds = urlJoin(basePath, navigateUrlWithFancyIds); + nextPageLink = convertUrlWithFancyIdsToActualUrl(fullNavigateUrlWithFancyIds); } else if (pageTestMeta.action === null) { // No more pages to test ✅, move on alreadyEncounteredLastPage = true; @@ -1118,12 +1158,17 @@ describe('matrix-public-archive', () => { `Unexpected value for ${pageKey}.action=${pageTestMeta.action} that we don't know what to do with` ); } - const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); - const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); + + if (actionLinkSelector) { + const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); + const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); + nextPageLink = jumpToActivityLinkHref; + } + // Move to the next iteration of the loop // // Set this for debugging if the test fails here - archiveUrl = jumpToActivityLinkHref; + archiveUrl = nextPageLink; } catch (err) { const errorWithContext = new RethrownError( `Encountered error while asserting ${pageKey}: (see original error below)`, @@ -1163,13 +1208,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/03?at=$event7', + url: '/roomid/room1/date/2022/01/03?at=$event7', action: null, }, }, @@ -1195,13 +1240,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/03?at=$event7', + url: '/roomid/room1/date/2022/01/03?at=$event7', action: null, }, }, @@ -1221,13 +1266,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/04T01:00', + startUrl: '/roomid/room1/date/2022/01/04T01:00', page1: { - url: '/r/room1/date/2022/01/04T01:00', + url: '/roomid/room1/date/2022/01/04T01:00', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/04?at=$event11', + url: '/roomid/room1/date/2022/01/04?at=$event11', action: null, }, }, @@ -1247,13 +1292,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/03', + startUrl: '/roomid/room1/date/2022/01/03', page1: { - url: '/r/room1/date/2022/01/03', + url: '/roomid/room1/date/2022/01/03', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/04?at=$event10', + url: '/roomid/room1/date/2022/01/04?at=$event10', action: null, }, }, @@ -1279,9 +1324,9 @@ describe('matrix-public-archive', () => { // [page2 ] // `, // archiveMessageLimit: 4, - // startUrl: '/r/room1/date/2022/01/04', + // startUrl: '/roomid/room1/date/2022/01/04', // page1: { - // url: '/r/room1/date/2022/01/04', + // url: '/roomid/room1/date/2022/01/04', // action: 'next', // }, // page2: { @@ -1289,7 +1334,7 @@ describe('matrix-public-archive', () => { // // date forward by a day so we can display the empty view for that day. // // // // TODO: This page probably doesn't need a `?at=` continue event - // url: '/r/room1/date/2022/01/05?at=TODO', + // url: '/roomid/room1/date/2022/01/05?at=TODO', // action: null, // }, // }, @@ -1316,16 +1361,16 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 3, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { // We expect the URL to look like `T02:00` because we're rendering part way // through day3 and while we could get away with just hour precision, the // default precision has hours and minutes. - url: '/r/room1/date/2022/01/03T02:00?at=$event7', + url: '/roomid/room1/date/2022/01/03T02:00?at=$event7', action: null, }, }, @@ -1346,15 +1391,15 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/03', + startUrl: '/roomid/room1/date/2022/01/03', page1: { - url: '/r/room1/date/2022/01/03', + url: '/roomid/room1/date/2022/01/03', action: 'previous', }, page2: { // Continuing from the first event of day2 since we already saw the rest // of day2 in the first page - url: '/r/room1/date/2022/01/02?at=$event4', + url: '/roomid/room1/date/2022/01/02?at=$event4', action: null, }, }, @@ -1374,13 +1419,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 8, - startUrl: '/r/room1/date/2022/01/04', + startUrl: '/roomid/room1/date/2022/01/04', page1: { - url: '/r/room1/date/2022/01/04', + url: '/roomid/room1/date/2022/01/04', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/07?at=$event13', + url: '/roomid/room1/date/2022/01/07?at=$event13', action: null, }, }, @@ -1398,13 +1443,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 8, - startUrl: '/r/room1/date/2022/01/09', + startUrl: '/roomid/room1/date/2022/01/09', page1: { - url: '/r/room1/date/2022/01/09', + url: '/roomid/room1/date/2022/01/09', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/06?at=$event12', + url: '/roomid/room1/date/2022/01/06?at=$event12', action: null, }, }, @@ -1426,13 +1471,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/03T03:00?at=$event7', + url: '/roomid/room1/date/2022/01/03T03:00?at=$event7', action: null, }, }, @@ -1453,13 +1498,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/04', + startUrl: '/roomid/room1/date/2022/01/04', page1: { - url: '/r/room1/date/2022/01/04', + url: '/roomid/room1/date/2022/01/04', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/02?at=$event9', + url: '/roomid/room1/date/2022/01/02?at=$event9', action: null, }, }, @@ -1479,13 +1524,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/03T03:00?at=$event9', + url: '/roomid/room1/date/2022/01/03T03:00?at=$event9', action: null, }, }, @@ -1505,13 +1550,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/03', + startUrl: '/roomid/room1/date/2022/01/03', page1: { - url: '/r/room1/date/2022/01/03', + url: '/roomid/room1/date/2022/01/03', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/03T01:00?at=$event9', + url: '/roomid/room1/date/2022/01/03T01:00?at=$event9', action: null, }, }, @@ -1531,13 +1576,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02T6:00', + startUrl: '/roomid/room1/date/2022/01/02T6:00', page1: { - url: '/r/room1/date/2022/01/02T6:00', + url: '/roomid/room1/date/2022/01/02T6:00', action: 'next', }, page2: { - url: '/r/room1/date/2022/01/02T09:00?at=$event9', + url: '/roomid/room1/date/2022/01/02T09:00?at=$event9', action: null, }, }, @@ -1557,13 +1602,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02T11:00', + startUrl: '/roomid/room1/date/2022/01/02T11:00', page1: { - url: '/r/room1/date/2022/01/02T11:00', + url: '/roomid/room1/date/2022/01/02T11:00', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/02T06:00?at=$event8', + url: '/roomid/room1/date/2022/01/02T06:00?at=$event8', action: null, }, }, @@ -1583,14 +1628,14 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02T06:00', + startUrl: '/roomid/room1/date/2022/01/02T06:00', page1: { - url: '/r/room1/date/2022/01/02T06:00', + url: '/roomid/room1/date/2022/01/02T06:00', action: 'next', }, page2: { // Continuing from the unseen event in day2 - url: '/r/room1/date/2022/01/03T02:00?at=$event9', + url: '/roomid/room1/date/2022/01/03T02:00?at=$event9', action: null, }, }, @@ -1610,13 +1655,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/03T06:00', + startUrl: '/roomid/room1/date/2022/01/03T06:00', page1: { - url: '/r/room1/date/2022/01/03T06:00', + url: '/roomid/room1/date/2022/01/03T06:00', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/03T01:00?at=$event8', + url: '/roomid/room1/date/2022/01/03T01:00?at=$event8', action: null, }, }, @@ -1635,13 +1680,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/03T05:00', + startUrl: '/roomid/room1/date/2022/01/03T05:00', page1: { - url: '/r/room1/date/2022/01/03T05:00', + url: '/roomid/room1/date/2022/01/03T05:00', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/02?at=$event7', + url: '/roomid/room1/date/2022/01/02?at=$event7', action: null, }, }, @@ -1649,8 +1694,8 @@ describe('matrix-public-archive', () => { const jumpBackwardPredecessorTestCases = [ { - // Page2 doesn't only shows 4 messages ($event4-7) because it also has the - // tombstone event which is hidden + // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 because it + // also has the tombstone event which is hidden testName: 'can jump backward from one room to the predecessor room', roomDayMessageStructureString: ` [room1 ] [room2 ] @@ -1660,38 +1705,64 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room2/date/2022/01/03T05:00', + startUrl: '/roomid/room2/date/2022/01/03T05:00', page1: { - url: '/r/room2/date/2022/01/03T05:00', + url: '/roomid/room2/date/2022/01/03T05:00', action: 'previous', }, page2: { - url: '/r/room1/date/2022/01/02T05:00?at=$event7', + url: '/roomid/room1/date/2022/01/02T05:00?at=$event7', action: null, }, }, + // { + // // This doesn't work well because of the primordial create room events which + // // we can't control the timestamp of or assert properly in this diagram. If + // // we ever get timestamp massaging on the `/createRoom` endpoint (see + // // https://github.com/matrix-org/synapse/issues/15346), we could make this + // // work by increasing the `archiveMessageLimit` to something that would + // // encompass all of the primordial events along with the last few messages. + // testName: `will paginate to the oldest messages in the room (doesn't skip the last few) before jumping backward to the predecessor room`, + // roomDayMessageStructureString: ` + // [room1 ] [room2 ] + // 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + // [day1 ] [day2 ] [day3 ] + // [page1 ] + // [page2] + // [page3 ] + // `, + // archiveMessageLimit: 4, + // startUrl: '/roomid/room2/date/2022/01/03', + // page1: { + // url: '/roomid/room2/date/2022/01/03', + // action: 'previous', + // }, + // page2: { + // url: '/roomid/room2/date/2022/01/03T02:00?at=$event9', + // action: 'previous', + // }, + // page3: { + // url: '/roomid/room1/date/2022/01/02', + // action: null, + // }, + // }, { - testName: `will paginate to the oldest messages in the room (doesn't skip the last few) before jumping backward to the predecessor room`, + testName: 'jumping back before room was created will go down the predecessor chain', roomDayMessageStructureString: ` - [room1 ] [room2 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - [page2] - [page3 ] + [room1 ] [room2 ] [room3 ] [room4 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + [page2 ] `, - archiveMessageLimit: 4, - startUrl: '/r/room2/date/2022/01/03T05:00', + archiveMessageLimit: 3, + startUrl: '/roomid/room4/date/2022/01/08', page1: { - url: '/r/room2/date/2022/01/03T05:00', - action: 'previous', + url: '/roomid/room4/date/2022/01/08', + action: 'navigate:/roomid/room4/date/2022/01/02', }, page2: { - url: '/r/room2/date/2022/01/03', - action: 'previous', - }, - page3: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: null, }, }, @@ -1709,13 +1780,13 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 4, - startUrl: '/r/room1/date/2022/01/02', + startUrl: '/roomid/room1/date/2022/01/02', page1: { - url: '/r/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/02', action: 'next', }, page2: { - url: '/r/room2/date/2022/01/03T04:00', + url: '/roomid/room2/date/2022/01/03T04:00', action: null, }, }, From bd949019fd46fb2163049fc4659dc96d1939768e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 03:19:22 -0500 Subject: [PATCH 11/30] Add test for same day --- server/routes/room-routes.js | 22 ++++++++++++---------- test/e2e-tests.js | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 8ad43fdc..84afcae3 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -620,17 +620,18 @@ router.get( `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + `event for this room unless the server is just broken. You can try refreshing to try again.` ); - } else { - // Jump to the predecessor room - res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.predecessorRoomId, { - viaServers: Array.from(roomData.predecessorViaServers || []), - dir: DIRECTION.backward, - currentRangeStartTs: continueAtTsInPredecessorRoom, - currentRangeEndTs: toTimestamp, - }) - ); } + + // Jump to the predecessor room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.predecessorRoomId, { + viaServers: Array.from(roomData.predecessorViaServers || []), + dir: DIRECTION.backward, + currentRangeStartTs: continueAtTsInPredecessorRoom, + currentRangeEndTs: toTimestamp, + }) + ); + return; } const nowTs = Date.now(); @@ -656,6 +657,7 @@ router.get( currentRangeEndTs: 0, }) ); + return; } // If no successor, just 404 if anyone is trying to view the future, no need to // waste resources on that diff --git a/test/e2e-tests.js b/test/e2e-tests.js index baf34930..e1dfb6b5 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1715,6 +1715,28 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 because it + // also has the tombstone event which is hidden + testName: 'can jump backward from one room to the predecessor room (same day)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room2/date/2022/01/02T10:00', + page1: { + url: '/roomid/room2/date/2022/01/02T10:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02T05:00?at=$event7', + action: null, + }, + }, // { // // This doesn't work well because of the primordial create room events which // // we can't control the timestamp of or assert properly in this diagram. If From 078fa0a0e6c8899f028a2ffcaf07e3ce1d973a17 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 03:32:22 -0500 Subject: [PATCH 12/30] Change up tactic to display the given day of the predecessor room And will implement the continuation timestamp stuff as part of `/jump` instead. --- server/routes/room-routes.js | 78 ++++++++++-------------------------- test/e2e-tests.js | 15 ++++--- 2 files changed, 32 insertions(+), 61 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 84afcae3..256b05bc 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -536,11 +536,8 @@ router.get( // XXX: It would be better if we just tried fetching first and assume that we are // already joined and only join after we see a 403 Forbidden error (we should do // this for all places we `ensureRoomJoined`) - const roomId = await ensureRoomJoined( - matrixAccessToken, - roomIdOrAlias, - parseViaServersFromUserInput(req.query.via) - ); + const viaServers = parseViaServersFromUserInput(req.query.via); + const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, viaServers); // Do these in parallel to avoid the extra time in sequential round-trips // (we want to display the archive page faster) @@ -584,52 +581,17 @@ router.get( const hasNavigatedBeforeStartOfRoom = events.length === 0; // Check if we need to navigate backward to the predecessor room if (hasNavigatedBeforeStartOfRoom && roomData.predecessorRoomId) { - const PREDECESSOR_ROOM_TOMBSTONE_FETCH_TIMEOUT_MS = 8 * ONE_SECOND_IN_MS; - const predecessorRoomTombstoneSetTs = await Promise.race([ - // Try to continue from the tombstone event in the predecessor room because that - // is the signal that the room admins gave to indicate the end of the room in - // favor of the other regardless of further activity that may have occured in - // the room. - roomData.getPredecessorRoomTombstoneSetTs(), - // If it takes too long to get the predecessor room tombstone, then abort and - // will fallback to something else - new Promise((resolve) => { - setTimeout(resolve, PREDECESSOR_ROOM_TOMBSTONE_FETCH_TIMEOUT_MS); - }), - ]); - - // Fallback to the room creation event time if we can't find the predecessor room - // tombstone which will work just fine and as expected for normal room upgrade - // scenarios. - const continueAtTsInPredecessorRoom = - predecessorRoomTombstoneSetTs ?? roomData.roomCreationTs; - - console.log( - 'continueAtTsInPredecessorRoom', - new Date(continueAtTsInPredecessorRoom).toISOString(), - continueAtTsInPredecessorRoom - ); - - if (continueAtTsInPredecessorRoom === null || continueAtTsInPredecessorRoom === undefined) { - throw new StatusError( - 500, - `You navigated past the end of the room and it has a predecessor set (${roomData.predecessorRoomId}) ` + - `but we were unable to find a suitable place to jump to and continue from. ` + - `We could just redirect you to that predecessor room but we decided to throw an error ` + - `instead because we should be able to fallback to the room creation time in any case. ` + - `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + - `event for this room unless the server is just broken. You can try refreshing to try again.` - ); - } - - // Jump to the predecessor room + // Jump to the predecessor room at the date/time the user is trying to visit at res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.predecessorRoomId, { - viaServers: Array.from(roomData.predecessorViaServers || []), - dir: DIRECTION.backward, - currentRangeStartTs: continueAtTsInPredecessorRoom, - currentRangeEndTs: toTimestamp, - }) + matrixPublicArchiveURLCreator.archiveUrlForDate( + roomData.predecessorRoomId, + new Date(toTimestamp), + { + preferredPrecision: precisionFromUrl, + // XXX: Should we also try combining `viaServers` we used to get to this room? + viaServers: Array.from(roomData.predecessorViaServers || []), + } + ) ); return; } @@ -649,13 +611,17 @@ router.get( areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); // Check if we need to navigate forward to the successor room if (roomData.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { - // Jump to the successor room and continue at the first event of the room + // Jump to the successor room at the date/time the user is trying to visit at res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(roomData.successorRoomId, { - dir: DIRECTION.forward, - currentRangeStartTs: 0, - currentRangeEndTs: 0, - }) + matrixPublicArchiveURLCreator.archiveUrlForDate( + roomData.successorRoomId, + new Date(toTimestamp), + { + preferredPrecision: precisionFromUrl, + // Just try to pass on the `viaServers` the user was using to get to this room + viaServers: Array.from(viaServers || []), + } + ) ); return; } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index e1dfb6b5..f3daeb64 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1694,8 +1694,9 @@ describe('matrix-public-archive', () => { const jumpBackwardPredecessorTestCases = [ { - // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 because it - // also has the tombstone event which is hidden + // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden testName: 'can jump backward from one room to the predecessor room', roomDayMessageStructureString: ` [room1 ] [room2 ] @@ -1716,8 +1717,9 @@ describe('matrix-public-archive', () => { }, }, { - // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 because it - // also has the tombstone event which is hidden + // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden testName: 'can jump backward from one room to the predecessor room (same day)', roomDayMessageStructureString: ` [room1 ] [room2 ] @@ -1769,13 +1771,16 @@ describe('matrix-public-archive', () => { // }, // }, { + // Page2 doesn't only shows 3 messages ($event2-4) instead of 4 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden testName: 'jumping back before room was created will go down the predecessor chain', roomDayMessageStructureString: ` [room1 ] [room2 ] [room3 ] [room4 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] [page1 ] - [page2 ] + [page2 ] `, archiveMessageLimit: 3, startUrl: '/roomid/room4/date/2022/01/08', From 54d7feb1144640dfb73a8dc4e97467a724c77ec0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 03:36:27 -0500 Subject: [PATCH 13/30] Bail earlier --- server/routes/room-routes.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 256b05bc..3f6b0477 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -509,6 +509,7 @@ router.get( timeoutMiddleware, // eslint-disable-next-line max-statements, complexity asyncHandler(async function (req, res) { + const nowTs = Date.now(); const roomIdOrAlias = getRoomIdOrAliasFromReq(req); // We pull this fresh from the config for each request to ensure we have an @@ -530,6 +531,17 @@ router.get( precisionFromUrl = TIME_PRECISION_VALUES.minutes; } + // Just 404 if anyone is trying to view the future, no need to waste resources on + // that + if (toTimestamp > roundUpTimestampToUtcDay(nowTs)) { + throw new StatusError( + 404, + `You can't view the history of a room on a future day (${new Date( + toTimestamp + ).toISOString()} > ${new Date(nowTs).toISOString()}). Go back` + ); + } + // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. // @@ -596,8 +608,6 @@ router.get( return; } - const nowTs = Date.now(); - // We only care to navigate to the successor room if we're trying to view something // past when the successor was set (it's an indicator that we need to go to the new // room from this time forward). @@ -625,16 +635,6 @@ router.get( ); return; } - // If no successor, just 404 if anyone is trying to view the future, no need to - // waste resources on that - else if (toTimestamp > roundUpTimestampToUtcDay(nowTs)) { - throw new StatusError( - 404, - `You can't view the history of a room on a future day (${new Date( - toTimestamp - ).toISOString()} > ${new Date(nowTs).toISOString()}). Go back` - ); - } // Default to no indexing (safe default) let shouldIndex = false; From f0bbc2bad1c9b35c141bfda2d9b1f5cf8267a816 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 03:45:52 -0500 Subject: [PATCH 14/30] Cleaner fetching --- server/lib/matrix-utils/fetch-room-data.js | 180 ++++++++------------- 1 file changed, 69 insertions(+), 111 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index 2399f214..098aec33 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -4,7 +4,6 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson } = require('../fetch-endpoint'); -const ensureRoomJoined = require('./ensure-room-joined'); const parseViaServersFromUserInput = require('../parse-via-servers-from-user-input'); const { traceFunction } = require('../../tracing/trace-utilities'); @@ -21,12 +20,78 @@ function getStateEndpointForRoomIdAndEventType(roomId, eventType) { ); } +async function fetchPredecessorInfo(matrixAccessToken, roomId) { + const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { + accessToken: matrixAccessToken, + }), + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), + { + accessToken: matrixAccessToken, + } + ), + ]); + + let predecessorRoomId; + let predecessorViaServers; + if (statePredecessorResDataOutcome.reason === undefined) { + const { data } = statePredecessorResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor_room_id; + predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); + } else if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor?.room_id; + } + + let roomCreationTs; + if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + roomCreationTs = data?.origin_server_ts; + } + + return { + roomCreationTs, + predecessorRoomId, + predecessorViaServers, + }; +} + +async function fetchSuccessorInfo(matrixAccessToken, roomId) { + const [stateTombstoneResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.tombstone'), { + accessToken: matrixAccessToken, + }), + ]); + + let successorRoomId; + let successorSetTs; + if (stateTombstoneResDataOutcome.reason === undefined) { + const { data } = stateTombstoneResDataOutcome.value; + successorRoomId = data?.content?.replacement_room; + successorSetTs = data?.origin_server_ts; + } + + return { + successorRoomId, + successorSetTs, + }; +} + // eslint-disable-next-line max-statements async function fetchRoomData(matrixAccessToken, roomId) { assert(matrixAccessToken); assert(roomId); - const mainFetchPromiseBundle = Promise.allSettled([ + const [ + stateNameResDataOutcome, + stateCanonicalAliasResDataOutcome, + stateAvatarResDataOutcome, + stateHistoryVisibilityResDataOutcome, + stateJoinRulesResDataOutcome, + { roomCreationTs, predecessorRoomId, predecessorViaServers }, + { successorRoomId, successorSetTs }, + ] = await Promise.allSettled([ fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.name'), { accessToken: matrixAccessToken, }), @@ -45,106 +110,10 @@ async function fetchRoomData(matrixAccessToken, roomId) { fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.join_rules'), { accessToken: matrixAccessToken, }), - fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.tombstone'), { - accessToken: matrixAccessToken, - }), + fetchPredecessorInfo(matrixAccessToken, roomId), + fetchSuccessorInfo(matrixAccessToken, roomId), ]); - const predecessorInfoPromise = (async () => { - const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ - fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { - accessToken: matrixAccessToken, - }), - fetchEndpointAsJson( - getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), - { - accessToken: matrixAccessToken, - } - ), - ]); - - let predecessorRoomId; - let predecessorViaServers; - if (statePredecessorResDataOutcome.reason === undefined) { - const { data } = statePredecessorResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor_room_id; - predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); - } else if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor?.room_id; - } - - let roomCreationTs; - if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - roomCreationTs = data?.origin_server_ts; - } - - return { - roomCreationTs, - predecessorRoomId, - predecessorViaServers, - }; - })(); - - // TODO: This is pretty ugly/messy. Refactor this and maybe use a different pattern - // for this type of thing. - const _getPredecessorRoomTombstoneSetTs = async () => { - const { predecessorRoomId, predecessorViaServers } = await predecessorInfoPromise; - - if (predecessorRoomId) { - await ensureRoomJoined(matrixAccessToken, predecessorRoomId, predecessorViaServers); - - // Fetch the tombstone from the predessor room - const [predecessorStateTombstoneResDataOutcome] = await Promise.allSettled([ - fetchEndpointAsJson( - getStateEndpointForRoomIdAndEventType(predecessorRoomId, 'm.room.tombstone'), - { - accessToken: matrixAccessToken, - } - ), - ]); - - let predecessorSuccessorRoomId; - let predecessorSuccessorSetTs; - if (predecessorStateTombstoneResDataOutcome.reason === undefined) { - const { data } = predecessorStateTombstoneResDataOutcome.value; - predecessorSuccessorRoomId = data?.content?.replacement_room; - predecessorSuccessorSetTs = data?.origin_server_ts; - } - - // Make sure the the room that the predecessor specifies as the replacement room - // is the same as what the current room is. This is a good signal that the rooms - // are a true continuation of each other and the room admins agree. - if (predecessorSuccessorRoomId === roomId) { - return predecessorSuccessorSetTs; - } - - return null; - } - - return null; - }; - - // Memoize this function so we only ever do the extra work once - let _predecessorRoomTombstoneSetTsResult; - const getPredecessorRoomTombstoneSetTs = async () => { - if (_predecessorRoomTombstoneSetTsResult === undefined) { - _predecessorRoomTombstoneSetTsResult = _getPredecessorRoomTombstoneSetTs(); - } - - return _predecessorRoomTombstoneSetTsResult; - }; - - const [ - stateNameResDataOutcome, - stateCanonicalAliasResDataOutcome, - stateAvatarResDataOutcome, - stateHistoryVisibilityResDataOutcome, - stateJoinRulesResDataOutcome, - stateTombstoneResDataOutcome, - ] = await mainFetchPromiseBundle; - let name; if (stateNameResDataOutcome.reason === undefined) { const { data } = stateNameResDataOutcome.value; @@ -175,16 +144,6 @@ async function fetchRoomData(matrixAccessToken, roomId) { joinRule = data?.content?.join_rule; } - const { roomCreationTs, predecessorRoomId, predecessorViaServers } = await predecessorInfoPromise; - - let successorRoomId; - let successorSetTs; - if (stateTombstoneResDataOutcome.reason === undefined) { - const { data } = stateTombstoneResDataOutcome.value; - successorRoomId = data?.content?.replacement_room; - successorSetTs = data?.origin_server_ts; - } - return { id: roomId, name, @@ -195,7 +154,6 @@ async function fetchRoomData(matrixAccessToken, roomId) { roomCreationTs, predecessorRoomId, predecessorViaServers, - getPredecessorRoomTombstoneSetTs, successorRoomId, successorSetTs, }; From b8a80fdffb4a597e85b4a56e4ec2873e472ac0b5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Apr 2023 04:44:43 -0500 Subject: [PATCH 15/30] Working predecessor tests --- server/lib/matrix-utils/fetch-room-data.js | 22 ++++- server/routes/room-routes.js | 99 +++++++++++++++++++--- test/e2e-tests.js | 11 ++- 3 files changed, 113 insertions(+), 19 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index 098aec33..b27f03fb 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -89,8 +89,8 @@ async function fetchRoomData(matrixAccessToken, roomId) { stateAvatarResDataOutcome, stateHistoryVisibilityResDataOutcome, stateJoinRulesResDataOutcome, - { roomCreationTs, predecessorRoomId, predecessorViaServers }, - { successorRoomId, successorSetTs }, + predecessorInfoOutcome, + successorInfoOutcome, ] = await Promise.allSettled([ fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.name'), { accessToken: matrixAccessToken, @@ -144,6 +144,18 @@ async function fetchRoomData(matrixAccessToken, roomId) { joinRule = data?.content?.join_rule; } + let roomCreationTs; + let predecessorRoomId; + let predecessorViaServers; + if (predecessorInfoOutcome.reason === undefined) { + ({ roomCreationTs, predecessorRoomId, predecessorViaServers } = predecessorInfoOutcome.value); + } + let successorRoomId; + let successorSetTs; + if (successorInfoOutcome.reason === undefined) { + ({ successorRoomId, successorSetTs } = successorInfoOutcome.value); + } + return { id: roomId, name, @@ -159,4 +171,8 @@ async function fetchRoomData(matrixAccessToken, roomId) { }; } -module.exports = traceFunction(fetchRoomData); +module.exports = { + fetchRoomData: traceFunction(fetchRoomData), + fetchPredecessorInfo: traceFunction(fetchPredecessorInfo), + fetchSuccessorInfo: traceFunction(fetchSuccessorInfo), +}; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 3f6b0477..b588573d 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -12,7 +12,11 @@ const redirectToCorrectArchiveUrlIfBadSigil = require('./redirect-to-correct-arc const { HTTPResponseError } = require('../lib/fetch-endpoint'); const parseViaServersFromUserInput = require('../lib/parse-via-servers-from-user-input'); -const fetchRoomData = require('../lib/matrix-utils/fetch-room-data'); +const { + fetchRoomData, + fetchPredecessorInfo, + fetchSuccessorInfo, +} = require('../lib/matrix-utils/fetch-room-data'); const fetchEventsFromTimestampBackwards = require('../lib/matrix-utils/fetch-events-from-timestamp-backwards'); const ensureRoomJoined = require('../lib/matrix-utils/ensure-room-joined'); const timestampToEvent = require('../lib/matrix-utils/timestamp-to-event'); @@ -228,6 +232,11 @@ router.get( '?dir query parameter must be [f|b]' ); + // We have to wait for the room join to happen first before we can use the jump to + // date endpoint (or any other Matrix endpoint) + const viaServers = parseViaServersFromUserInput(req.query.via); + const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, viaServers); + let ts; if (dir === DIRECTION.backward) { // We `- 1` so we don't jump to the same event because the endpoint is inclusive. @@ -242,17 +251,9 @@ router.get( // the same timestamp ts = currentRangeEndTs + 1; } else { - throw new Error(`Unable to handle unknown dir=${dir} in /jump`); + throw new StatusError(400, `Unable to handle unknown dir=${dir} in /jump`); } - // We have to wait for the room join to happen first before we can use the jump to - // date endpoint - const roomId = await ensureRoomJoined( - matrixAccessToken, - roomIdOrAlias, - parseViaServersFromUserInput(req.query.via) - ); - let eventIdForClosestEvent; let tsForClosestEvent; let newOriginServerTs; @@ -356,7 +357,7 @@ router.get( // `/messages?dir=f` backfills, we won't have this problem anymore because any // messages backfilled in the forwards direction would be picked up the same going // backwards. - if (dir === DIRECTION.forward) { + else if (dir === DIRECTION.forward) { // Use `/messages?dir=f` and get the `end` pagination token to paginate from. And // then start the scroll from the top of the page so they can continue. // @@ -458,10 +459,84 @@ router.get( } } catch (err) { const is404Error = err instanceof HTTPResponseError && err.response.status === 404; + // A 404 error just means there is no more messages to paginate in that room and + // we should try to go to the predecessor/successor room appropriately. + if (is404Error) { + if (dir === DIRECTION.backward) { + const { roomCreationTs, predecessorRoomId, predecessorViaServers } = + await fetchPredecessorInfo(matrixAccessToken, roomId); + + // We have to join the predecessor room before we can fetch the successor info + // (this could be our first time seeing the room) + await ensureRoomJoined(matrixAccessToken, predecessorRoomId, viaServers); + const { + successorRoomId: successorRoomIdForPredecessor, + successorSetTs: successorSetTsForPredecessor, + } = await fetchSuccessorInfo(matrixAccessToken, predecessorRoomId); + + // Try to continue from the tombstone event in the predecessor room because + // that is the signal that the room admins gave to indicate the end of the + // room in favor of the other regardless of further activity that may have + // occured in the room. + // + // Make sure the the room that the predecessor specifies as the replacement + // room is the same as what the current room is. This is a good signal that + // the rooms are a true continuation of each other and the room admins agree. + let continueAtTsInPredecessorRoom; + if (successorRoomIdForPredecessor === roomId) { + continueAtTsInPredecessorRoom = successorSetTsForPredecessor; + } + // Fallback to the room creation event time if we can't find the predecessor + // room tombstone which will work just fine and as expected for normal room + // upgrade scenarios. + else { + continueAtTsInPredecessorRoom = roomCreationTs; + } + + if ( + continueAtTsInPredecessorRoom === null || + continueAtTsInPredecessorRoom === undefined + ) { + throw new StatusError( + 500, + `You navigated past the end of the room and it has a predecessor set (${predecessorRoomId}) ` + + `but we were unable to find a suitable place to jump to and continue from. ` + + `We could just redirect you to that predecessor room but we decided to throw an error ` + + `instead because we should be able to fallback to the room creation time in any case. ` + + `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + + `event for this room unless the server is just broken right now. You can try refreshing to try again.` + ); + } + + // Jump to the predecessor room at the appropriate timestamp to continue from + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(predecessorRoomId, { + viaServers: Array.from(predecessorViaServers || []), + dir: DIRECTION.backward, + currentRangeStartTs: continueAtTsInPredecessorRoom, + currentRangeEndTs: currentRangeEndTs, + }) + ); + return; + } else if (dir === DIRECTION.forward) { + const { successorRoomId } = await fetchSuccessorInfo(matrixAccessToken, roomId); + if (successorRoomId) { + // Jump to the successor room and continue at the first event of the room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(successorRoomId, { + dir: DIRECTION.forward, + currentRangeStartTs: 0, + currentRangeEndTs: 0, + }) + ); + return; + } + } + } // Only throw if it's something other than a 404 error. 404 errors are fine, they // just mean there is no more messages to paginate in that room and we were // already viewing the latest in the room. - if (!is404Error) { + else { throw err; } } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index f3daeb64..259749cb 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1694,7 +1694,7 @@ describe('matrix-public-archive', () => { const jumpBackwardPredecessorTestCases = [ { - // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 + // Page2 only shows 4 messages ($event4-7) instead of 5 // (`archiveMessageLimit` + 1) because it also has the tombstone event which // is hidden testName: 'can jump backward from one room to the predecessor room', @@ -1712,12 +1712,12 @@ describe('matrix-public-archive', () => { action: 'previous', }, page2: { - url: '/roomid/room1/date/2022/01/02T05:00?at=$event7', + url: '/roomid/room1/date/2022/01/02?at=$event7', action: null, }, }, { - // Page2 doesn't only shows 4 messages ($event4-7) instead of 5 + // Page2 only shows 4 messages ($event4-7) instead of 5 // (`archiveMessageLimit` + 1) because it also has the tombstone event which // is hidden testName: 'can jump backward from one room to the predecessor room (same day)', @@ -1797,12 +1797,15 @@ describe('matrix-public-archive', () => { const jumpForwardTombstoneTestCases = [ { + // Page1 only shows 4 messages ($event4-7) instead of 5 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden testName: 'can jump forward from one room to the replacement room', roomDayMessageStructureString: ` [room1 ] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [day3 ] - [page1 ] + [page1 ] |--jump-fwd-4-messages--->| [page2 ] `, From 17a045a8d20cfb9909f3540577b34afe10bcc6c2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 14:59:01 -0500 Subject: [PATCH 16/30] Fix jumping forward when there is a multiple day gap --- server/lib/matrix-utils/fetch-room-data.js | 69 +++++-- server/routes/room-routes.js | 195 ++++++++++++++---- shared/lib/url-creator.js | 18 +- shared/viewmodels/ArchiveRoomViewModel.js | 7 + .../JumpToNextActivitySummaryTileViewModel.js | 12 ++ ...pToPreviousActivitySummaryTileViewModel.js | 12 ++ test/e2e-tests.js | 157 +++++++++----- test/test-utils/client-utils.js | 2 +- 8 files changed, 360 insertions(+), 112 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index b27f03fb..65f11b1e 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -20,11 +20,44 @@ function getStateEndpointForRoomIdAndEventType(roomId, eventType) { ); } -async function fetchPredecessorInfo(matrixAccessToken, roomId) { - const [stateCreateResDataOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ +// TODO: Remove this when we have MSC3999 (see usage) +const removeMe_fetchRoomCreateEventId = traceFunction(async function (matrixAccessToken, roomId) { + const { data } = await fetchEndpointAsJson( + urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/messages?dir=f&limit1` + ), + { + accessToken: matrixAccessToken, + } + ); + + const roomCreateEventId = data?.chunk?.[0]?.event_id; + + return roomCreateEventId; +}); + +const fetchRoomCreationInfo = traceFunction(async function (matrixAccessToken, roomId) { + const [stateCreateResDataOutcome] = await Promise.allSettled([ fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { accessToken: matrixAccessToken, }), + ]); + + let roomCreationTs; + let predecessorRoomId; + if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + roomCreationTs = data?.origin_server_ts; + predecessorRoomId = data?.content?.predecessor?.room_id; + } + + return { eventId, roomCreationTs, predecessorRoomId }; +}); + +const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, roomId) { + const [roomCreationInfoOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ + fetchRoomCreationInfo(matrixAccessToken, roomId), fetchEndpointAsJson( getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), { @@ -35,29 +68,27 @@ async function fetchPredecessorInfo(matrixAccessToken, roomId) { let predecessorRoomId; let predecessorViaServers; + // Prefer the dynamic predecessor from the dedicated state event if (statePredecessorResDataOutcome.reason === undefined) { const { data } = statePredecessorResDataOutcome.value; predecessorRoomId = data?.content?.predecessor_room_id; predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); - } else if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - predecessorRoomId = data?.content?.predecessor?.room_id; } - - let roomCreationTs; - if (stateCreateResDataOutcome.reason === undefined) { - const { data } = stateCreateResDataOutcome.value; - roomCreationTs = data?.origin_server_ts; + // Then fallback to the predecessor defined by the room creation event + else if (roomCreationInfoOutcome.predecessorRoomId) { + predecessorRoomId = roomCreationInfoOutcome.predecessorRoomId; } + const { roomCreationTs } = roomCreationInfoOutcome; + return { roomCreationTs, predecessorRoomId, predecessorViaServers, }; -} +}); -async function fetchSuccessorInfo(matrixAccessToken, roomId) { +const fetchSuccessorInfo = traceFunction(async function (matrixAccessToken, roomId) { const [stateTombstoneResDataOutcome] = await Promise.allSettled([ fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.tombstone'), { accessToken: matrixAccessToken, @@ -76,10 +107,10 @@ async function fetchSuccessorInfo(matrixAccessToken, roomId) { successorRoomId, successorSetTs, }; -} +}); // eslint-disable-next-line max-statements -async function fetchRoomData(matrixAccessToken, roomId) { +const fetchRoomData = traceFunction(async function (matrixAccessToken, roomId) { assert(matrixAccessToken); assert(roomId); @@ -169,10 +200,12 @@ async function fetchRoomData(matrixAccessToken, roomId) { successorRoomId, successorSetTs, }; -} +}); module.exports = { - fetchRoomData: traceFunction(fetchRoomData), - fetchPredecessorInfo: traceFunction(fetchPredecessorInfo), - fetchSuccessorInfo: traceFunction(fetchSuccessorInfo), + fetchRoomData, + fetchRoomCreationInfo, + fetchPredecessorInfo, + fetchSuccessorInfo, + removeMe_fetchRoomCreateEventId, }; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index b588573d..6fb767d7 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -20,6 +20,7 @@ const { const fetchEventsFromTimestampBackwards = require('../lib/matrix-utils/fetch-events-from-timestamp-backwards'); const ensureRoomJoined = require('../lib/matrix-utils/ensure-room-joined'); const timestampToEvent = require('../lib/matrix-utils/timestamp-to-event'); +const { removeMe_fetchRoomCreateEventId } = require('../lib/matrix-utils/fetch-room-data'); const getMessagesResponseFromEventId = require('../lib/matrix-utils/get-messages-response-from-event-id'); const renderHydrogenVmRenderScriptToPageHtml = require('../hydrogen-render/render-hydrogen-vm-render-script-to-page-html'); const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); @@ -232,24 +233,38 @@ router.get( '?dir query parameter must be [f|b]' ); + const timelineStartEventId = req.query.timelineStartEventId; + assert( + ['string', 'undefined'].includes(typeof timelineStartEventId), + `?timelineStartEventId must be a string or undefined but saw ${typeof timelineStartEventId}` + ); + const timelineEndEventId = req.query.timelineEndEventId; + assert( + ['string', 'undefined'].includes(typeof timelineStartEventId), + `?timelineEndEventId must be a string or undefined but saw ${typeof timelineStartEventId}` + ); + // We have to wait for the room join to happen first before we can use the jump to // date endpoint (or any other Matrix endpoint) const viaServers = parseViaServersFromUserInput(req.query.via); const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, viaServers); let ts; + let fromCausalEventId; if (dir === DIRECTION.backward) { // We `- 1` so we don't jump to the same event because the endpoint is inclusive. // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp ts = currentRangeStartTs - 1; + fromCausalEventId = timelineStartEventId; } else if (dir === DIRECTION.forward) { // We `+ 1` so we don't jump to the same event because the endpoint is inclusive // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp ts = currentRangeEndTs + 1; + fromCausalEventId = timelineEndEventId; } else { throw new StatusError(400, `Unable to handle unknown dir=${dir} in /jump`); } @@ -263,15 +278,65 @@ router.get( // updated value between each e2e test const archiveMessageLimit = config.get('archiveMessageLimit'); - console.log(`jumping from ${new Date(ts).toISOString()} (${ts}) in direction ${dir}`); + console.log( + `jumping from ${new Date( + ts + ).toISOString()} (${ts}) (fromCausalEventId=${fromCausalEventId}) in direction ${dir}` + ); + let roomCreateEventId; // Find the closest event to the given timestamp - ({ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent } = - await timestampToEvent({ - accessToken: matrixAccessToken, - roomId, - ts: ts, - direction: dir, - })); + [{ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent }, roomCreateEventId] = + await Promise.all([ + timestampToEvent({ + accessToken: matrixAccessToken, + roomId, + ts: ts, + direction: dir, + // Since timestamps are untrusted and can be crafted to make loops in the + // timeline. We use this as a signal to keep progressing from this event + // regardless of what timestamp shenanigans are going on. See MSC3999 + // (https://github.com/matrix-org/matrix-spec-proposals/pull/3999) + // + // TODO: Add tests for timestamp loops once Synapse supports MSC3999. We + // currently just have this set in case some server has this implemented in + // the future but there currently is no implementation (as of 2023-04-17) and + // we can't have passing tests without a server implementation first. + 'org.matrix.msc3999.event_id': fromCausalEventId, + }), + removeMe_fetchRoomCreateEventId(matrixAccessToken, roomId), + ]); + console.log( + 'found eventIdForClosestEvent', + eventIdForClosestEvent, + new Date(tsForClosestEvent).toISOString(), + tsForClosestEvent + ); + + // Without MSC3999, we currently only detect one kind of loop where the + // `m.room.create` has a timestamp that comes after the timestamp massaged events + // in the room. This is a common pattern for historical Gitter rooms where we + // created the room and then imported a bunch of messages at a time before the + // room was created. + // + // By nature of having an `timelineEndEventId`, we know we are already paginated + // past the `m.room.create` event which is always the first event in the room. So + // we can use that to detect the end of the room before we loop back around to the + // start of the room. + // + // XXX: Once we have MSC3999, we can remove this check in favor of that mechanism + // (TODO: Create issue to track this) + if ( + dir === DIRECTION.forward && + timelineEndEventId && + eventIdForClosestEvent === roomCreateEventId + ) { + throw new StatusError( + 404, + `/jump?dir=${dir}: We detected a loop back to the beginning of the room so we can assume ` + + `we hit the end of the room instead of doing a loop. We throw a 404 error here we hit ` + + `the normal 404 no more /messages error handling below` + ); + } // Based on what we found was the closest, figure out the URL that will represent // the next chunk in the desired direction. @@ -279,7 +344,7 @@ router.get( // // When jumping backwards, since a given room archive URL represents the end of // the day/time-period looking backward (scroll is also anchored to the bottom), - // we just need to get the user to the previous time-period. + // we just need to get the user to the *next* previous time-period. // // We are trying to avoid sending the user to the same time period they were just // viewing. i.e, if they were visiting `/2020/01/02T16:00:00` (displays messages @@ -354,13 +419,11 @@ router.get( // XXX: This is flawed in the fact that when we go `/messages?dir=b` later, it // could backfill messages which will fill up the response before we perfectly // connect and continue from the position they were jumping from before. When - // `/messages?dir=f` backfills, we won't have this problem anymore because any - // messages backfilled in the forwards direction would be picked up the same going - // backwards. + // `/messages?dir=f` backfills (forwards fill), we won't have this problem anymore + // because any messages backfilled in the forwards direction would be picked up + // the same going backwards. See MSC4000 + // (https://github.com/matrix-org/matrix-spec-proposals/pull/4000). else if (dir === DIRECTION.forward) { - // Use `/messages?dir=f` and get the `end` pagination token to paginate from. And - // then start the scroll from the top of the page so they can continue. - // // XXX: It would be cool to somehow cache this response and re-use our work here // for the actual room display that we redirect to from this route. No need for // us go out 100 messages, only for us to go backwards 100 messages again in the @@ -376,13 +439,28 @@ router.get( if (!messageResData.chunk?.length) { throw new StatusError( 404, - `/jump?dir=${dir}: /messages response didn't contain any more messages to jump to` + `/jump?dir=${dir}: /messages response didn't contain any more messages to jump to so we can assume we reached the end of the room.` ); } - const timestampOfLastMessage = - messageResData.chunk[messageResData.chunk.length - 1].origin_server_ts; - const dateOfLastMessage = new Date(timestampOfLastMessage); + const firstMessage = messageResData.chunk[0]; + const tsOfFirstMessage = firstMessage.origin_server_ts; + console.log( + 'tsOfFirstMessage', + new Date(tsOfFirstMessage).toISOString(), + tsOfFirstMessage, + firstMessage.event_id + ); + + const lastMessage = messageResData.chunk[messageResData.chunk.length - 1]; + const tsOfLastMessage = lastMessage.origin_server_ts; + const dateOfLastMessage = new Date(tsOfLastMessage); + console.log( + 'tsOfLastMessage', + new Date(tsOfLastMessage).toISOString(), + tsOfLastMessage, + lastMessage.event_id + ); // Back-track from the last message timestamp to the nearest date boundary. // Because we're back-tracking a couple events here, when we paginate back out @@ -394,28 +472,53 @@ router.get( // back-tracking but then we get ugly URL's every time you jump instead of being // able to back-track and round down to the nearest hour in a lot of cases. The // other reason not to return the exact date is maybe there multiple messages at - // the same timestamp and we will lose messages in the gap it displays more than - // we thought. - const msGapFromJumpPointToLastMessage = timestampOfLastMessage - ts; - const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; - const moreThanHourGap = msGapFromJumpPointToLastMessage > ONE_HOUR_IN_MS; - const moreThanMinuteGap = msGapFromJumpPointToLastMessage > ONE_MINUTE_IN_MS; - const moreThanSecondGap = msGapFromJumpPointToLastMessage > ONE_SECOND_IN_MS; + // the same timestamp and we will lose messages in the gap because it displays + // more than we thought. + const fromDifferentDay = !areTimestampsFromSameUtcDay(currentRangeEndTs, tsOfLastMessage); + const fromDifferentHour = !areTimestampsFromSameUtcHour(currentRangeEndTs, tsOfLastMessage); + const fromDifferentMinute = !areTimestampsFromSameUtcMinute( + currentRangeEndTs, + tsOfLastMessage + ); + const fromDifferentSecond = !areTimestampsFromSameUtcSecond( + currentRangeEndTs, + tsOfLastMessage + ); + + // To handle sparsely populated days (quiet days) or conversely busy days with + // too many messages to today, we also check that TODO + const hasMessagesInBetweenFromDifferentDay = !areTimestampsFromSameUtcDay( + tsOfFirstMessage, + tsOfLastMessage + ); + const hasMessagesInBetweenFromDifferentHour = !areTimestampsFromSameUtcHour( + tsOfFirstMessage, + tsOfLastMessage + ); + const hasMessagesInBetweenFromDifferentMinute = !areTimestampsFromSameUtcMinute( + tsOfFirstMessage, + tsOfLastMessage + ); + const hasMessagesInBetweenFromDifferentSecond = !areTimestampsFromSameUtcSecond( + tsOfFirstMessage, + tsOfLastMessage + ); // If the `/messages` response returns less than the `archiveMessageLimit` // looking forwards, it means we're looking at the latest events in the room. We - // can simply just display the day that the latest event occured on or given + // can simply just display the day that the latest event occured on or the given // rangeEnd (whichever is later). const haveReachedLatestMessagesInRoom = messageResData.chunk?.length < archiveMessageLimit; if (haveReachedLatestMessagesInRoom) { - const latestDesiredTs = Math.max(currentRangeEndTs, timestampOfLastMessage); + const latestDesiredTs = Math.max(currentRangeEndTs, tsOfLastMessage); const latestDesiredDate = new Date(latestDesiredTs); const utcMidnightTs = getUtcStartOfDayTs(latestDesiredDate); newOriginServerTs = utcMidnightTs; preferredPrecision = TIME_PRECISION_VALUES.none; } - // More than a day gap here, so we can just back-track to the nearest day - else if (moreThanDayGap) { + // More than a day gap here, so we can just back-track to the nearest day as + // long as there are messages we haven't seen yet if we visit the nearest day. + else if (fromDifferentDay && hasMessagesInBetweenFromDifferentDay) { const utcMidnightOfDayBefore = getUtcStartOfDayTs(dateOfLastMessage); // We `- 1` from UTC midnight to get the timestamp that is a millisecond // before the next day but we choose a no time precision so we jump to just @@ -426,20 +529,25 @@ router.get( newOriginServerTs = endOfDayBeforeTs; preferredPrecision = TIME_PRECISION_VALUES.none; } - // More than a hour gap here, we will need to back-track to the nearest hour - else if (moreThanHourGap) { + // More than a hour gap here, we will need to back-track to the nearest hour as + // long as there are messages we haven't seen yet if we visit the nearest hour. + else if (fromDifferentHour && hasMessagesInBetweenFromDifferentHour) { const utcTopOfHourBefore = getUtcStartOfHourTs(dateOfLastMessage); newOriginServerTs = utcTopOfHourBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a minute gap here, we will need to back-track to the nearest minute - else if (moreThanMinuteGap) { + // as long as there are messages we haven't seen yet if we visit the nearest + // minute. + else if (fromDifferentMinute && hasMessagesInBetweenFromDifferentMinute) { const utcTopOfMinuteBefore = getUtcStartOfMinuteTs(dateOfLastMessage); newOriginServerTs = utcTopOfMinuteBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a second gap here, we will need to back-track to the nearest second - else if (moreThanSecondGap) { + // as long as there are messages we haven't seen yet if we visit the nearest + // second. + else if (fromDifferentSecond && hasMessagesInBetweenFromDifferentSecond) { const utcTopOfSecondBefore = getUtcStartOfSecondTs(dateOfLastMessage); newOriginServerTs = utcTopOfSecondBefore; preferredPrecision = TIME_PRECISION_VALUES.seconds; @@ -458,7 +566,10 @@ router.get( } } } catch (err) { - const is404Error = err instanceof HTTPResponseError && err.response.status === 404; + const is404HTTPResponseError = + err instanceof HTTPResponseError && err.response.status === 404; + const is404StatusError = err instanceof StatusError && err.status === 404; + const is404Error = is404HTTPResponseError || is404StatusError; // A 404 error just means there is no more messages to paginate in that room and // we should try to go to the predecessor/successor room appropriately. if (is404Error) { @@ -509,12 +620,19 @@ router.get( } // Jump to the predecessor room at the appropriate timestamp to continue from + console.log( + `/jump hit the beginning of the room, jumping to predecessorRoomId=${predecessorRoomId}` + ); res.redirect( matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(predecessorRoomId, { viaServers: Array.from(predecessorViaServers || []), dir: DIRECTION.backward, currentRangeStartTs: continueAtTsInPredecessorRoom, currentRangeEndTs: currentRangeEndTs, + // We don't need to define + // `currentRangeStartEventId`/`currentRangeEndEventId` here because we're + // jumping to a completely new room so the event IDs won't pertain to the + // new room and we don't have any to use anyway. }) ); return; @@ -522,11 +640,18 @@ router.get( const { successorRoomId } = await fetchSuccessorInfo(matrixAccessToken, roomId); if (successorRoomId) { // Jump to the successor room and continue at the first event of the room + console.log( + `/jump hit the end of the room, jumping to successorRoomId=${successorRoomId}` + ); res.redirect( matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(successorRoomId, { dir: DIRECTION.forward, currentRangeStartTs: 0, currentRangeEndTs: 0, + // We don't need to define + // `currentRangeStartEventId`/`currentRangeEndEventId` here because we're + // jumping to a completely new room so the event IDs won't pertain to the + // new room and we don't have any to use anyway. }) ); return; diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js index 1d3ebf68..4285d217 100644 --- a/shared/lib/url-creator.js +++ b/shared/lib/url-creator.js @@ -121,18 +121,34 @@ class URLCreator { archiveJumpUrlForRoom( roomIdOrAlias, - { dir, currentRangeStartTs, currentRangeEndTs, viaServers = [] } + { + dir, + currentRangeStartTs, + currentRangeEndTs, + timelineStartEventId, + timelineEndEventId, + viaServers = [], + } ) { assert(roomIdOrAlias); assert(dir); assert(typeof currentRangeStartTs === 'number'); assert(typeof currentRangeEndTs === 'number'); assert(Array.isArray(viaServers)); + // `timelineStartEventId` and `timelineEndEventId` are optional because the + // timeline could be showing 0 events or we could be jumping with no knowledge of + // what was shown before. let qs = new URLSearchParams(); qs.append('dir', dir); qs.append('currentRangeStartTs', currentRangeStartTs); qs.append('currentRangeEndTs', currentRangeEndTs); + if (timelineStartEventId) { + qs.append('timelineStartEventId', timelineStartEventId); + } + if (timelineEndEventId) { + qs.append('timelineEndEventId', timelineEndEventId); + } [].concat(viaServers).forEach((viaServer) => { qs.append('via', viaServer); }); diff --git a/shared/viewmodels/ArchiveRoomViewModel.js b/shared/viewmodels/ArchiveRoomViewModel.js index 84b0d78f..6ff0bd52 100644 --- a/shared/viewmodels/ArchiveRoomViewModel.js +++ b/shared/viewmodels/ArchiveRoomViewModel.js @@ -363,6 +363,9 @@ class ArchiveRoomViewModel extends ViewModel { // to paginate from const jumpRangeEndTimestamp = this._dayTimestampTo; + const timelineStartEventId = events[0]?.event_id; + const timelineEndEventId = events[events.length - 1]?.event_id; + // Check whether the given day represented in the URL has any events on the page // from that day. We only need to check the last event which would be closest to // `_dayTimestampTo` anyway. @@ -394,6 +397,8 @@ class ArchiveRoomViewModel extends ViewModel { canonicalAlias: this._room.canonicalAlias, jumpRangeStartTimestamp, jumpRangeEndTimestamp, + timelineStartEventId, + timelineEndEventId, // This is a bit cheating but I don't know how else to pass this kind of // info to the Tile viewmodel basePath: this._basePath, @@ -417,6 +422,8 @@ class ArchiveRoomViewModel extends ViewModel { dayTimestamp: this._dayTimestampTo, jumpRangeStartTimestamp, jumpRangeEndTimestamp, + timelineStartEventId, + timelineEndEventId, // This is a bit cheating but I don't know how else to pass this kind of // info to the Tile viewmodel basePath: this._basePath, diff --git a/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js b/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js index fd88a0be..7876985e 100644 --- a/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js +++ b/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js @@ -38,6 +38,16 @@ class JumpToNextActivitySummaryTileViewModel extends SimpleTile { return this._entry?.content?.['jumpRangeEndTimestamp']; } + // The first event shown in the timeline. + get timelineStartEventId() { + return this._entry?.content?.['timelineStartEventId']; + } + + // The last event shown in the timeline. + get timelineEndEventId() { + return this._entry?.content?.['timelineEndEventId']; + } + get jumpToNextActivityUrl() { return this._matrixPublicArchiveURLCreator.archiveJumpUrlForRoom( this._entry?.content?.['canonicalAlias'] || this._entry.roomId, @@ -45,6 +55,8 @@ class JumpToNextActivitySummaryTileViewModel extends SimpleTile { dir: DIRECTION.forward, currentRangeStartTs: this.jumpRangeStartTimestamp, currentRangeEndTs: this.jumpRangeEndTimestamp, + timelineStartEventId: this.timelineStartEventId, + timelineEndEventId: this.timelineEndEventId, } ); } diff --git a/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js b/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js index db72b8ee..6f2d6846 100644 --- a/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js +++ b/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js @@ -30,6 +30,16 @@ class JumpToPreviousActivitySummaryTileViewModel extends SimpleTile { return this._entry?.content?.['jumpRangeEndTimestamp']; } + // The first event shown in the timeline. + get timelineStartEventId() { + return this._entry?.content?.['timelineStartEventId']; + } + + // The last event shown in the timeline. + get timelineEndEventId() { + return this._entry?.content?.['timelineEndEventId']; + } + get jumpToPreviousActivityUrl() { return this._matrixPublicArchiveURLCreator.archiveJumpUrlForRoom( this._entry?.content?.['canonicalAlias'] || this._entry.roomId, @@ -37,6 +47,8 @@ class JumpToPreviousActivitySummaryTileViewModel extends SimpleTile { dir: DIRECTION.backward, currentRangeStartTs: this.jumpRangeStartTimestamp, currentRangeEndTs: this.jumpRangeEndTimestamp, + timelineStartEventId: this.timelineStartEventId, + timelineEndEventId: this.timelineEndEventId, } ); } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 259749cb..1a348e0c 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -790,7 +790,7 @@ describe('matrix-public-archive', () => { ) .split(/\r?\n/) .map((line) => { - if (line.includes(eventIdToLookFor)) { + if (eventIdToLookFor && line.includes(eventIdToLookFor)) { return chalk.yellow(line); } @@ -913,7 +913,7 @@ describe('matrix-public-archive', () => { }); } else { // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it - // supports it, see https://github.com/matrix-org/synapse/issues/15346 + // supports it, see https://github.com/matrix-org/matrix-public-archive/issues/169 roomId = await createTestRoom(client); } const fancyRoomId = `!room${roomIndex + 1}`; @@ -990,6 +990,11 @@ describe('matrix-public-archive', () => { fancyRoomIdToDebugEventsInRoom.set(fancyRoomId, eventDebugStrings); } + console.log( + 'getDebugStringForEventsInRoomsAndLookForEventId()\n', + getDebugStringForEventsInRoomsAndLookForEventId() + ); + // Now Test // -------------------------------------- // -------------------------------------- @@ -1076,11 +1081,24 @@ describe('matrix-public-archive', () => { ); const pageDom = parseHTML(archivePageHtml); + const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] + // eslint-disable-next-line max-nested-callbacks + .map((eventEl) => { + return eventEl.getAttribute('data-event-id'); + }); + // Assert the correct room and time precision in the URL const actualUrlWithFancyIdentifies = convertActualUrlToUrlWithFancyIds(pageRes.url); assert.match( actualUrlWithFancyIdentifies, - new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`) + new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`), + `The actual URL (${actualUrlWithFancyIdentifies}) for the page did not match the expected URL (${ + pageTestMeta.url + }).\nFor reference, here are the events on the page: ${JSON.stringify( + eventIdsOnPage, + null, + 2 + )}\n${getDebugStringForEventsInRoomsAndLookForEventId()}` ); // If provided, assert that it's a smooth continuation to more messages. @@ -1102,14 +1120,8 @@ describe('matrix-public-archive', () => { assert.strictEqual(continationDebugEventId, expectedContinuationDebugEventId); } - const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] - // eslint-disable-next-line max-nested-callbacks - .map((eventEl) => { - return eventEl.getAttribute('data-event-id'); - }); - // We only care about messages for now (no easy way to specify the - // primordial room creation or member events) + // primordial room creation or member events in the test expectations) const eventIdsOnPageWeCareAboutToAssert = eventIdsOnPage.filter((eventId) => { const event = eventMap.get(eventId); if (!event) { @@ -1306,7 +1318,7 @@ describe('matrix-public-archive', () => { // creation events which are created in now time vs the timestamp massaging we // do for the message fixtures. We can uncomment this once Synapse supports // timestamp massaging for `/createRoom`, see - // https://github.com/matrix-org/synapse/issues/15346 + // https://github.com/matrix-org/matrix-public-archive/issues/169 // // { // // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 @@ -1481,6 +1493,37 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // Test to make sure we can jump forwards from the 1st page to the 2nd page + // when there is a multiple-day gap between the end of the first page to the + // next messages. + // + // We jump forward 4 messages (`archiveMessageLimit`), then back-track to + // the nearest hour because even though there is more than a day gap in the + // jump, there aren't any mesages in between from another day. Because, we + // back-tracked to the nearest hour, this starts us from event9, and then we + // display 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: 'can jump forward to the next activity when there is a multiple day gap', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day5 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/05T03:00?at=$event7', + action: null, + }, + }, { // Test to make sure we can jump backwards from the 1st page to the 2nd page // with too many messages to display on a single day. @@ -1697,7 +1740,7 @@ describe('matrix-public-archive', () => { // Page2 only shows 4 messages ($event4-7) instead of 5 // (`archiveMessageLimit` + 1) because it also has the tombstone event which // is hidden - testName: 'can jump backward from one room to the predecessor room', + testName: 'can jump backward from one room to the predecessor room (different day)', roomDayMessageStructureString: ` [room1 ] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 @@ -1739,13 +1782,38 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // Page2 doesn't only shows 3 messages ($event2-4) instead of 4 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden + testName: 'jumping back before room was created will go down the predecessor chain', + roomDayMessageStructureString: ` + [room1 ] [room2 ] [room3 ] [room4 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 3, + startUrl: '/roomid/room4/date/2022/01/08', + page1: { + url: '/roomid/room4/date/2022/01/08', + action: 'navigate:/roomid/room4/date/2022/01/02', + }, + page2: { + url: '/roomid/room1/date/2022/01/02', + action: null, + }, + }, + // This doesn't work well because of the primordial create room events which + // we can't control the timestamp of or assert properly in this diagram. If we + // ever get timestamp massaging on the `/createRoom` endpoint (see + // https://github.com/matrix-org/matrix-public-archive/issues/169), we could + // make this work by increasing the `archiveMessageLimit` to something that + // would encompass all of the primordial events along with the last few + // messages. + // // { - // // This doesn't work well because of the primordial create room events which - // // we can't control the timestamp of or assert properly in this diagram. If - // // we ever get timestamp massaging on the `/createRoom` endpoint (see - // // https://github.com/matrix-org/synapse/issues/15346), we could make this - // // work by increasing the `archiveMessageLimit` to something that would - // // encompass all of the primordial events along with the last few messages. // testName: `will paginate to the oldest messages in the room (doesn't skip the last few) before jumping backward to the predecessor room`, // roomDayMessageStructureString: ` // [room1 ] [room2 ] @@ -1770,53 +1838,28 @@ describe('matrix-public-archive', () => { // action: null, // }, // }, - { - // Page2 doesn't only shows 3 messages ($event2-4) instead of 4 - // (`archiveMessageLimit` + 1) because it also has the tombstone event which - // is hidden - testName: 'jumping back before room was created will go down the predecessor chain', - roomDayMessageStructureString: ` - [room1 ] [room2 ] [room3 ] [room4 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 - [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] - [page1 ] - [page2 ] - `, - archiveMessageLimit: 3, - startUrl: '/roomid/room4/date/2022/01/08', - page1: { - url: '/roomid/room4/date/2022/01/08', - action: 'navigate:/roomid/room4/date/2022/01/02', - }, - page2: { - url: '/roomid/room1/date/2022/01/02', - action: null, - }, - }, ]; - const jumpForwardTombstoneTestCases = [ + const jumpForwardSuccessorTestCases = [ { - // Page1 only shows 4 messages ($event4-7) instead of 5 - // (`archiveMessageLimit` + 1) because it also has the tombstone event which - // is hidden - testName: 'can jump forward from one room to the replacement room', + // TODO: Explain intracies of the pages + testName: 'can jump forward from one room to the successor room (different day)', roomDayMessageStructureString: ` - [room1 ] [room2 ] + [room1] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - |--jump-fwd-4-messages--->| - [page2 ] + [day1 ] [day2 ] + [page1] + |--jump-fwd-10-messages~~> + [page2] `, - archiveMessageLimit: 4, - startUrl: '/roomid/room1/date/2022/01/02', + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01T02:00', page1: { - url: '/roomid/room1/date/2022/01/02', + url: '/roomid/room1/date/2022/01/01T02:00', action: 'next', }, page2: { - url: '/roomid/room2/date/2022/01/03T04:00', + url: '/roomid/room2/date/2022/01/02T04:00?at=$event3', action: null, }, }, @@ -1834,9 +1877,9 @@ describe('matrix-public-archive', () => { }); }); - describe('jump forward from tombstone to replacement rooms', () => { + describe('jump forward from tombstone to replacement/successor rooms', () => { // eslint-disable-next-line max-nested-callbacks - jumpForwardTombstoneTestCases.forEach((testCase) => { + jumpForwardSuccessorTestCases.forEach((testCase) => { runJumpTestCase(testCase); }); }); diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 049680ff..92d277bc 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -210,7 +210,7 @@ async function upgradeTestRoom({ } // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it supports it, - // see https://github.com/matrix-org/synapse/issues/15346 + // see https://github.com/matrix-org/matrix-public-archive/issues/169 const newRoomid = await createTestRoom(client, createOptions); // Now send the tombstone event pointing from the old room to the new room From 0834f3ff37e546e63808affd3015e10ad29d2c24 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 15:13:13 -0500 Subject: [PATCH 17/30] Fix assertions in jump forward successor test --- test/e2e-tests.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 1a348e0c..e95dde8f 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1783,7 +1783,7 @@ describe('matrix-public-archive', () => { }, }, { - // Page2 doesn't only shows 3 messages ($event2-4) instead of 4 + // Page2 only shows 3 messages ($event2-4) instead of 4 // (`archiveMessageLimit` + 1) because it also has the tombstone event which // is hidden testName: 'jumping back before room was created will go down the predecessor chain', @@ -1842,15 +1842,22 @@ describe('matrix-public-archive', () => { const jumpForwardSuccessorTestCases = [ { - // TODO: Explain intracies of the pages + // We jump from event3 which is found as the closest event looking forward + // because the timestamp massaged events come before `m.room.create` and + // other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, + // then back-track to the nearest hour which starts off at event11 and + // render the page with 5 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow. testName: 'can jump forward from one room to the successor room (different day)', roomDayMessageStructureString: ` [room1] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [page1] - |--jump-fwd-10-messages~~> - [page2] + |------------------jump-fwd-10-messages----------------------->| + [page2 ] `, archiveMessageLimit: 10, startUrl: '/roomid/room1/date/2022/01/01T02:00', @@ -1859,7 +1866,7 @@ describe('matrix-public-archive', () => { action: 'next', }, page2: { - url: '/roomid/room2/date/2022/01/02T04:00?at=$event3', + url: '/roomid/room2/date/2022/01/02T09:00?at=$event3', action: null, }, }, From c91a3265e8314f88f509077caa74d69a84f4ba12 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 16:00:39 -0500 Subject: [PATCH 18/30] Fix predecessor not being picked up --- server/lib/matrix-utils/fetch-room-data.js | 6 +++--- server/routes/room-routes.js | 9 ++++++++- test/e2e-tests.js | 20 +++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index 65f11b1e..d5115b1b 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -52,7 +52,7 @@ const fetchRoomCreationInfo = traceFunction(async function (matrixAccessToken, r predecessorRoomId = data?.content?.predecessor?.room_id; } - return { eventId, roomCreationTs, predecessorRoomId }; + return { roomCreationTs, predecessorRoomId }; }); const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, roomId) { @@ -75,8 +75,8 @@ const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, ro predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); } // Then fallback to the predecessor defined by the room creation event - else if (roomCreationInfoOutcome.predecessorRoomId) { - predecessorRoomId = roomCreationInfoOutcome.predecessorRoomId; + else if (roomCreationInfoOutcome.reason === undefined) { + ({ predecessorRoomId } = roomCreationInfoOutcome.value); } const { roomCreationTs } = roomCreationInfoOutcome; diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 6fb767d7..b9e0d136 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -281,7 +281,7 @@ router.get( console.log( `jumping from ${new Date( ts - ).toISOString()} (${ts}) (fromCausalEventId=${fromCausalEventId}) in direction ${dir}` + ).toISOString()} (${ts}) (fromCausalEventId=${fromCausalEventId}) in direction ${dir} (roomId=${roomId}))` ); let roomCreateEventId; // Find the closest event to the given timestamp @@ -577,6 +577,13 @@ router.get( const { roomCreationTs, predecessorRoomId, predecessorViaServers } = await fetchPredecessorInfo(matrixAccessToken, roomId); + if (!predecessorRoomId) { + throw new StatusError( + 404, + `No predecessor room found for ${roomId} so we can't jump backwards to anywhere (you already reached the end of the room)` + ); + } + // We have to join the predecessor room before we can fetch the successor info // (this could be our first time seeing the room) await ensureRoomJoined(matrixAccessToken, predecessorRoomId, viaServers); diff --git a/test/e2e-tests.js b/test/e2e-tests.js index e95dde8f..369b1eab 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -784,7 +784,15 @@ describe('matrix-public-archive', () => { // String used to log out all possible events in the room function getDebugStringForEventsInRoomsAndLookForEventId(eventIdToLookFor) { return `For reference, here are all of the events in the rooms: ${JSON.stringify( - Object.fromEntries(fancyRoomIdToDebugEventsInRoom.entries()), + Object.fromEntries( + Array.from(fancyRoomIdToDebugEventsInRoom.entries()).map((entry) => { + const fancyRoomId = entry[0]; + const newKey = `${fancyRoomId} - ${fancyIdentifierToRoomIdMap.get( + fancyRoomId + )}`; + return [newKey, entry[1]]; + }) + ), null, 2 ) @@ -980,6 +988,16 @@ describe('matrix-public-archive', () => { let relevantContentString = ''; if (event.type === 'm.room.message' && event.content.msgtype === 'm.text') { relevantContentString = ` "${event.content.body}"`; + } else if (event.type === 'm.room.create') { + const predecessorRoomId = event.content?.predecessor?.room_id; + if (predecessorRoomId) { + relevantContentString = ` "predecessor=${predecessorRoomId}"`; + } + } else if (event.type === 'm.room.tombstone') { + const replacementRoomId = event.content?.replacement_room; + if (replacementRoomId) { + relevantContentString = ` "successor=${replacementRoomId}"`; + } } return `${event.type}${event.state_key ? ` (${event.state_key})` : ''}: ${ From 4fd584dc1abb5cf817613292b7d63739bfee009c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 16:48:09 -0500 Subject: [PATCH 19/30] Add more unconfirmed tests --- test/e2e-tests.js | 112 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 369b1eab..d706fe59 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1861,8 +1861,8 @@ describe('matrix-public-archive', () => { const jumpForwardSuccessorTestCases = [ { // We jump from event3 which is found as the closest event looking forward - // because the timestamp massaged events come before `m.room.create` and - // other primordial events here (related to + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to // https://github.com/matrix-org/matrix-public-archive/issues/169). From // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, // then back-track to the nearest hour which starts off at event11 and @@ -1878,9 +1878,9 @@ describe('matrix-public-archive', () => { [page2 ] `, archiveMessageLimit: 10, - startUrl: '/roomid/room1/date/2022/01/01T02:00', + startUrl: '/roomid/room1/date/2022/01/01', page1: { - url: '/roomid/room1/date/2022/01/01T02:00', + url: '/roomid/room1/date/2022/01/01', action: 'next', }, page2: { @@ -1888,6 +1888,110 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // TODO + testName: 'can jump forward from one room to the successor room (same day)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + |------------------jump-fwd-10-messages----------------------->| + [page2 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/02T05:00', + page1: { + url: '/roomid/room1/date/2022/01/01T05:00', + action: 'next', + }, + page2: { + url: '/roomid/room2/date/2022/01/02?at=$event8', + action: null, + }, + }, + { + // We jump from event3 which is found as the closest event looking forward + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, + // then back-track to the nearest hour which starts off at event11 and + // render the page with 5 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow. + testName: 'can jump forward from one room to the successor room (multiple day gap)', + roomDayMessageStructureString: ` + [room1] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day5 ] + [page1] + |------------------jump-fwd-10-messages----------------------->| + [page2 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room2/date/2022/01/05T09:00?at=$event3', + action: null, + }, + }, + { + // TODO + testName: 'can jump forward from one room to the successor room (across multiple days)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] + [page1] + [page2 ] + [page3 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/02T01:00?at=$event3', + action: null, + }, + page3: { + url: '/roomid/room2/date/2022/01/03T08:00?at=$event5', + action: null, + }, + }, + { + // TODO + testName: + 'can jump forward from one room to the successor room (across multiple days and day gaps)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day4 ] [day6 ] + [page1] + [page2 ] + [page3 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/04T01:00?at=$event3', + action: null, + }, + page3: { + url: '/roomid/room2/date/2022/01/06T08:00?at=$event5', + action: null, + }, + }, ]; jumpTestCases.forEach((testCase) => { From e9704a6977adc637be36e9829f897de6004ce65f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 17:50:04 -0500 Subject: [PATCH 20/30] Revert back gap time so paginate out further as expected --- server/routes/room-routes.js | 89 +++++++++++++++++------------------ test/e2e-tests.js | 91 +++++++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 68 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index b9e0d136..31d6f916 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -454,7 +454,6 @@ router.get( const lastMessage = messageResData.chunk[messageResData.chunk.length - 1]; const tsOfLastMessage = lastMessage.origin_server_ts; - const dateOfLastMessage = new Date(tsOfLastMessage); console.log( 'tsOfLastMessage', new Date(tsOfLastMessage).toISOString(), @@ -462,6 +461,40 @@ router.get( lastMessage.event_id ); + let msGapFromJumpPointToLastMessage; + // If someone is jumping from `0`, let's assume this is their first time + // navigating in the room and are just trying to get to the first messages in + // the room. Instead of using `0` which give us `moreThanDayGap=true` every time + // (unless someone sent messages in 1970 :P), and round us down to the nearest + // day before any of the messages in the room start, let's just use the start of + // the timeline as the start which will show us a page of content on the first + // try. For the backwards direction, we could have a similar check but with + // `currentRangeStartTs === Infinity` check but it's not necessary since we + // don't have to do any back-tracking extra work. + if (currentRangeEndTs === 0) { + msGapFromJumpPointToLastMessage = tsOfLastMessage - tsOfFirstMessage; + } + // Otherwise do the normal calculation: where we jumped to - where we jumped from + else { + msGapFromJumpPointToLastMessage = tsOfLastMessage - currentRangeEndTs; + } + const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; + const moreThanHourGap = msGapFromJumpPointToLastMessage > ONE_HOUR_IN_MS; + const moreThanMinuteGap = msGapFromJumpPointToLastMessage > ONE_MINUTE_IN_MS; + const moreThanSecondGap = msGapFromJumpPointToLastMessage > ONE_SECOND_IN_MS; + + // If the first message is on different day than the last message, then we know + // there are messages on days before the last mesage and can safely round to the + // nearest day and see new content still. + // + // We use this information to handle situations where we jump over multiple day + // gaps with no messages in between. In those cases, we don't want to round down + // where there are no messages in the gap. + const hasMessagesOnDayBeforeDayOfLastMessage = !areTimestampsFromSameUtcDay( + tsOfFirstMessage, + tsOfLastMessage + ); + // Back-track from the last message timestamp to the nearest date boundary. // Because we're back-tracking a couple events here, when we paginate back out // by the `archiveMessageLimit` later in the room route, it will gurantee some @@ -474,36 +507,7 @@ router.get( // other reason not to return the exact date is maybe there multiple messages at // the same timestamp and we will lose messages in the gap because it displays // more than we thought. - const fromDifferentDay = !areTimestampsFromSameUtcDay(currentRangeEndTs, tsOfLastMessage); - const fromDifferentHour = !areTimestampsFromSameUtcHour(currentRangeEndTs, tsOfLastMessage); - const fromDifferentMinute = !areTimestampsFromSameUtcMinute( - currentRangeEndTs, - tsOfLastMessage - ); - const fromDifferentSecond = !areTimestampsFromSameUtcSecond( - currentRangeEndTs, - tsOfLastMessage - ); - - // To handle sparsely populated days (quiet days) or conversely busy days with - // too many messages to today, we also check that TODO - const hasMessagesInBetweenFromDifferentDay = !areTimestampsFromSameUtcDay( - tsOfFirstMessage, - tsOfLastMessage - ); - const hasMessagesInBetweenFromDifferentHour = !areTimestampsFromSameUtcHour( - tsOfFirstMessage, - tsOfLastMessage - ); - const hasMessagesInBetweenFromDifferentMinute = !areTimestampsFromSameUtcMinute( - tsOfFirstMessage, - tsOfLastMessage - ); - const hasMessagesInBetweenFromDifferentSecond = !areTimestampsFromSameUtcSecond( - tsOfFirstMessage, - tsOfLastMessage - ); - + // // If the `/messages` response returns less than the `archiveMessageLimit` // looking forwards, it means we're looking at the latest events in the room. We // can simply just display the day that the latest event occured on or the given @@ -518,8 +522,8 @@ router.get( } // More than a day gap here, so we can just back-track to the nearest day as // long as there are messages we haven't seen yet if we visit the nearest day. - else if (fromDifferentDay && hasMessagesInBetweenFromDifferentDay) { - const utcMidnightOfDayBefore = getUtcStartOfDayTs(dateOfLastMessage); + else if (moreThanDayGap && hasMessagesOnDayBeforeDayOfLastMessage) { + const utcMidnightOfDayBefore = getUtcStartOfDayTs(tsOfLastMessage); // We `- 1` from UTC midnight to get the timestamp that is a millisecond // before the next day but we choose a no time precision so we jump to just // the bare date without a time. A bare date in the `/date/2022/12/16` @@ -529,26 +533,21 @@ router.get( newOriginServerTs = endOfDayBeforeTs; preferredPrecision = TIME_PRECISION_VALUES.none; } - // More than a hour gap here, we will need to back-track to the nearest hour as - // long as there are messages we haven't seen yet if we visit the nearest hour. - else if (fromDifferentHour && hasMessagesInBetweenFromDifferentHour) { - const utcTopOfHourBefore = getUtcStartOfHourTs(dateOfLastMessage); + // More than a hour gap here, we will need to back-track to the nearest hour + else if (moreThanHourGap) { + const utcTopOfHourBefore = getUtcStartOfHourTs(tsOfLastMessage); newOriginServerTs = utcTopOfHourBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a minute gap here, we will need to back-track to the nearest minute - // as long as there are messages we haven't seen yet if we visit the nearest - // minute. - else if (fromDifferentMinute && hasMessagesInBetweenFromDifferentMinute) { - const utcTopOfMinuteBefore = getUtcStartOfMinuteTs(dateOfLastMessage); + else if (moreThanMinuteGap) { + const utcTopOfMinuteBefore = getUtcStartOfMinuteTs(tsOfLastMessage); newOriginServerTs = utcTopOfMinuteBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a second gap here, we will need to back-track to the nearest second - // as long as there are messages we haven't seen yet if we visit the nearest - // second. - else if (fromDifferentSecond && hasMessagesInBetweenFromDifferentSecond) { - const utcTopOfSecondBefore = getUtcStartOfSecondTs(dateOfLastMessage); + else if (moreThanSecondGap) { + const utcTopOfSecondBefore = getUtcStartOfSecondTs(tsOfLastMessage); newOriginServerTs = utcTopOfSecondBefore; preferredPrecision = TIME_PRECISION_VALUES.seconds; } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index d706fe59..fcb1b3af 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1865,16 +1865,17 @@ describe('matrix-public-archive', () => { // come before `m.room.create` and other primordial events here (related to // https://github.com/matrix-org/matrix-public-archive/issues/169). From // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, - // then back-track to the nearest hour which starts off at event11 and - // render the page with 5 messages because we fetch one more than - // `archiveMessageLimit` to determine overflow. + // then back-track to the nearest hour which starts off at event11 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow but there aren't enough + // messages. testName: 'can jump forward from one room to the successor room (different day)', roomDayMessageStructureString: ` [room1] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [page1] - |------------------jump-fwd-10-messages----------------------->| + |------------------jump-fwd-10-messages----------------->| [page2 ] `, archiveMessageLimit: 10, @@ -1889,20 +1890,27 @@ describe('matrix-public-archive', () => { }, }, { - // TODO + // We jump from event8 which is found as the closest event looking forward + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event8, we jump forward 10 messages (`archiveMessageLimit`) to the end of + // the room, then go to the day of the last message which will show us all + // messages in room2 because we fetch one more than `archiveMessageLimit` to + // determine overflow which is more messages than room2 has. testName: 'can jump forward from one room to the successor room (same day)', roomDayMessageStructureString: ` [room1 ] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day2 ] [page1 ] - |------------------jump-fwd-10-messages----------------------->| - [page2 ] + |------------------jump-fwd-10-messages----------------------->| + [page2 ] `, archiveMessageLimit: 10, startUrl: '/roomid/room1/date/2022/01/02T05:00', page1: { - url: '/roomid/room1/date/2022/01/01T05:00', + url: '/roomid/room1/date/2022/01/02T05:00', action: 'next', }, page2: { @@ -1915,17 +1923,18 @@ describe('matrix-public-archive', () => { // from the ts=0 in the successor room because the timestamp massaged events // come before `m.room.create` and other primordial events here (related to // https://github.com/matrix-org/matrix-public-archive/issues/169). From - // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, - // then back-track to the nearest hour which starts off at event11 and - // render the page with 5 messages because we fetch one more than - // `archiveMessageLimit` to determine overflow. + // event3, we jump forward 10 messages (`archiveMessageLimit`) to event13, + // then back-track to the nearest hour which starts off at event11 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow but there aren't enough + // messages. testName: 'can jump forward from one room to the successor room (multiple day gap)', roomDayMessageStructureString: ` [room1] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 [day1 ] [day5 ] [page1] - |------------------jump-fwd-10-messages----------------------->| + |----------------jump-fwd-10-messages------------------->| [page2 ] `, archiveMessageLimit: 10, @@ -1940,15 +1949,36 @@ describe('matrix-public-archive', () => { }, }, { - // TODO + // For the jump from page1 to page2, we jump forward 10 messages which gets + // us to the end of the room. + // + // For the jump from page2 to page3, since we see the end of room1, we jump + // to the successor room and find the closest event from ts=0 looking + // forward which is event5 because the timestamp massaged events come before + // `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event5, we jump forward 10 messages (`archiveMessageLimit`) to event14, + // then back-track to the *day* before the last message found which starts off + // at event6 and try to render the page with 11 messages because we fetch + // one more than `archiveMessageLimit` to determine overflow but there + // aren't enough messages. + // + // For the jump from page3 to page4, we jump forward 10 messages to event16, + // then back-track to the nearest hour which starts off at event15 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit`. testName: 'can jump forward from one room to the successor room (across multiple days)', roomDayMessageStructureString: ` [room1 ] [room2 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 [day1 ] [day2 ] [day3 ] [page1] + |--jump-10->| [page2 ] - [page3 ] + |------------------jump-fwd-10-messages------------------->| + [page3] + |----------------------jump-fwd-10-messages----------------------->| + [page4 ] `, archiveMessageLimit: 10, startUrl: '/roomid/room1/date/2022/01/01', @@ -1957,16 +1987,20 @@ describe('matrix-public-archive', () => { action: 'next', }, page2: { - url: '/roomid/room1/date/2022/01/02T01:00?at=$event3', - action: null, + url: '/roomid/room1/date/2022/01/02?at=$event3', + action: 'next', }, page3: { - url: '/roomid/room2/date/2022/01/03T08:00?at=$event5', + url: '/roomid/room2/date/2022/01/02?at=$event5', + action: 'next', + }, + page4: { + url: '/roomid/room2/date/2022/01/03T09:00?at=$event7', action: null, }, }, { - // TODO + // (same as the test above just with more day gaps) testName: 'can jump forward from one room to the successor room (across multiple days and day gaps)', roomDayMessageStructureString: ` @@ -1974,8 +2008,12 @@ describe('matrix-public-archive', () => { 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 [day1 ] [day4 ] [day6 ] [page1] + |--jump-10->| [page2 ] - [page3 ] + |------------------jump-fwd-10-messages------------------->| + [page3] + |----------------------jump-fwd-10-messages----------------------->| + [page4 ] `, archiveMessageLimit: 10, startUrl: '/roomid/room1/date/2022/01/01', @@ -1984,14 +2022,21 @@ describe('matrix-public-archive', () => { action: 'next', }, page2: { - url: '/roomid/room1/date/2022/01/04T01:00?at=$event3', - action: null, + url: '/roomid/room1/date/2022/01/04?at=$event3', + action: 'next', }, page3: { - url: '/roomid/room2/date/2022/01/06T08:00?at=$event5', + // You might expect `/date/2022/01/04?at=$event5` here but we just get the + // UTC day before the day of last message we jumped to (event14) + url: '/roomid/room2/date/2022/01/05?at=$event5', + action: 'next', + }, + page4: { + url: '/roomid/room2/date/2022/01/06T09:00?at=$event7', action: null, }, }, + // TODO: Test with navigation back to date in time spanning multiple room upgrades ]; jumpTestCases.forEach((testCase) => { From decb7777404871728f943fca2f132f62715c2f13 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 21:44:55 -0500 Subject: [PATCH 21/30] Fix jumping backwards when messages from the same hour --- server/routes/room-routes.js | 68 +++-- shared/lib/timestamp-utilities.js | 39 ++- shared/viewmodels/ArchiveRoomViewModel.js | 6 +- shared/views/CalendarView.js | 4 +- test/e2e-tests.js | 28 +- test/shared/lib/timestamp-utilties-tests.js | 286 ++++++++++++++++++-- 6 files changed, 382 insertions(+), 49 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 31d6f916..cdef0a1f 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -36,14 +36,20 @@ const { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, + getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - areTimestampsFromSameUtcDay, - areTimestampsFromSameUtcHour, - areTimestampsFromSameUtcMinute, - areTimestampsFromSameUtcSecond, + + doTimestampsShareRoundedUpUtcHour, + doTimestampsShareRoundedUpUtcMinute, + doTimestampsShareRoundedUpUtcSecond, + + doTimestampsStartFromSameUtcDay, + doTimestampsStartFromSameUtcHour, + doTimestampsStartFromSameUtcMinute, + doTimestampsStartFromSameUtcSecond, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); const config = require('../lib/config'); @@ -256,6 +262,8 @@ router.get( // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp + // + // TODO: Remove the `- 1` when we have the MSC3999 causal event ID support ts = currentRangeStartTs - 1; fromCausalEventId = timelineStartEventId; } else if (dir === DIRECTION.forward) { @@ -263,6 +271,8 @@ router.get( // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp + // + // TODO: Remove the `+ 1` when we have the MSC3999 causal event ID support ts = currentRangeEndTs + 1; fromCausalEventId = timelineEndEventId; } else { @@ -344,24 +354,45 @@ router.get( // // When jumping backwards, since a given room archive URL represents the end of // the day/time-period looking backward (scroll is also anchored to the bottom), - // we just need to get the user to the *next* previous time-period. + // we just need to move the user to the time-period just prior the current one. // // We are trying to avoid sending the user to the same time period they were just // viewing. i.e, if they were visiting `/2020/01/02T16:00:00` (displays messages // backwards from that time up to the limit), which had more messages than we // could display in that day, jumping backwards from the earliest displayed event - // in the displayed range, say `T12:00:05` would still give us the same day - // `/2020/01/02` and we want to redirect them to previous chunk from that same - // day, like `/2020/01/02T12:00:00` + // in the displayed range (say that occured on `T12:00:25`) would still give us + // the same day `/2020/01/02` and we want to redirect them to previous chunk from + // that same day that still encompasses the closest message looking backwards, + // like `/2020/01/02T13:00:00` if (dir === DIRECTION.backward) { + // `currentRangeEndTs` represents what is being displayed in the URL (we fetch + // from this time backwards to render a page): + // + // - When the URL is `/date/2020/01/02`, `currentRangeEndTs=1578009599999` + // (2020-01-02T23:59:59.999Z) + // - When the URL is `/date/2022/11/16T02:00`, `currentRangeEndTs=1577930400000` + // (2020-01-02T02:00:00.000Z) + // + // We choose `currentRangeEndTs` vs the `ts` (the jump point) because TODO: why? + // + // We use `doTimestampsStartFromSameUtcDay` for day precision because TODO: why? const fromSameDay = - tsForClosestEvent && areTimestampsFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + doTimestampsStartFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); + // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: why? const fromSameHour = - tsForClosestEvent && areTimestampsFromSameUtcHour(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); const fromSameMinute = - tsForClosestEvent && areTimestampsFromSameUtcMinute(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); const fromSameSecond = - tsForClosestEvent && areTimestampsFromSameUtcSecond(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); + console.log('fromSameDay', fromSameDay); + console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); + console.log('fromSameMinute', fromSameMinute); + console.log('fromSameSecond', fromSameSecond); // The closest event is from the same second we tried to jump from. Since we // can't represent something smaller than a second in the URL yet (we could do @@ -476,7 +507,8 @@ router.get( } // Otherwise do the normal calculation: where we jumped to - where we jumped from else { - msGapFromJumpPointToLastMessage = tsOfLastMessage - currentRangeEndTs; + // TODO: Should we use `ts` be `currentRangeStartTs`? + msGapFromJumpPointToLastMessage = tsOfLastMessage - ts; } const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; const moreThanHourGap = msGapFromJumpPointToLastMessage > ONE_HOUR_IN_MS; @@ -485,12 +517,12 @@ router.get( // If the first message is on different day than the last message, then we know // there are messages on days before the last mesage and can safely round to the - // nearest day and see new content still. + // nearest day and still see new content. // - // We use this information to handle situations where we jump over multiple day + // We use this information to handle situations where we jump over multiple-day // gaps with no messages in between. In those cases, we don't want to round down - // where there are no messages in the gap. - const hasMessagesOnDayBeforeDayOfLastMessage = !areTimestampsFromSameUtcDay( + // to a day where there are no messages in the gap. + const hasMessagesOnDayBeforeDayOfLastMessage = !doTimestampsStartFromSameUtcDay( tsOfFirstMessage, tsOfLastMessage ); @@ -824,7 +856,7 @@ router.get( const isNewestEventFromSameDay = newestEvent && newestEvent?.origin_server_ts && - areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); + doTimestampsStartFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); // Check if we need to navigate forward to the successor room if (roomData.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { // Jump to the successor room at the date/time the user is trying to visit at diff --git a/shared/lib/timestamp-utilities.js b/shared/lib/timestamp-utilities.js index 1990add2..e70fd04d 100644 --- a/shared/lib/timestamp-utilities.js +++ b/shared/lib/timestamp-utilities.js @@ -70,19 +70,35 @@ function getUtcStartOfSecondTs(ts) { ); } -function areTimestampsFromSameUtcDay(aTs, bTs) { +function doTimestampsShareRoundedUpUtcDay(aTs, bTs) { + return roundUpTimestampToUtcDay(aTs) === roundUpTimestampToUtcDay(bTs); +} + +function doTimestampsShareRoundedUpUtcHour(aTs, bTs) { + return roundUpTimestampToUtcHour(aTs) === roundUpTimestampToUtcHour(bTs); +} + +function doTimestampsShareRoundedUpUtcMinute(aTs, bTs) { + return roundUpTimestampToUtcMinute(aTs) === roundUpTimestampToUtcMinute(bTs); +} + +function doTimestampsShareRoundedUpUtcSecond(aTs, bTs) { + return roundUpTimestampToUtcSecond(aTs) === roundUpTimestampToUtcSecond(bTs); +} + +function doTimestampsStartFromSameUtcDay(aTs, bTs) { return getUtcStartOfDayTs(aTs) === getUtcStartOfDayTs(bTs); } -function areTimestampsFromSameUtcHour(aTs, bTs) { +function doTimestampsStartFromSameUtcHour(aTs, bTs) { return getUtcStartOfHourTs(aTs) === getUtcStartOfHourTs(bTs); } -function areTimestampsFromSameUtcMinute(aTs, bTs) { +function doTimestampsStartFromSameUtcMinute(aTs, bTs) { return getUtcStartOfMinuteTs(aTs) === getUtcStartOfMinuteTs(bTs); } -function areTimestampsFromSameUtcSecond(aTs, bTs) { +function doTimestampsStartFromSameUtcSecond(aTs, bTs) { return getUtcStartOfSecondTs(aTs) === getUtcStartOfSecondTs(bTs); } @@ -91,12 +107,19 @@ module.exports = { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, + getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - areTimestampsFromSameUtcDay, - areTimestampsFromSameUtcHour, - areTimestampsFromSameUtcMinute, - areTimestampsFromSameUtcSecond, + + doTimestampsShareRoundedUpUtcDay, + doTimestampsShareRoundedUpUtcHour, + doTimestampsShareRoundedUpUtcMinute, + doTimestampsShareRoundedUpUtcSecond, + + doTimestampsStartFromSameUtcDay, + doTimestampsStartFromSameUtcHour, + doTimestampsStartFromSameUtcMinute, + doTimestampsStartFromSameUtcSecond, }; diff --git a/shared/viewmodels/ArchiveRoomViewModel.js b/shared/viewmodels/ArchiveRoomViewModel.js index 6ff0bd52..3c3af6f7 100644 --- a/shared/viewmodels/ArchiveRoomViewModel.js +++ b/shared/viewmodels/ArchiveRoomViewModel.js @@ -26,7 +26,7 @@ const { const stubPowerLevelsObservable = require('matrix-public-archive-shared/lib/stub-powerlevels-observable'); const { TIME_PRECISION_VALUES } = require('matrix-public-archive-shared/lib/reference-values'); const { - areTimestampsFromSameUtcDay, + doTimestampsStartFromSameUtcDay, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); let txnCount = 0; @@ -148,7 +148,7 @@ class ArchiveRoomViewModel extends ViewModel { (precisionFromUrl !== TIME_PRECISION_VALUES.none && !events.length) || // Only show the time selector when we're showing events all from the same day. (events.length && - areTimestampsFromSameUtcDay(timelineRangeStartTimestamp, timelineRangeEndTimestamp)); + doTimestampsStartFromSameUtcDay(timelineRangeStartTimestamp, timelineRangeEndTimestamp)); this._timeSelectorViewModel = new TimeSelectorViewModel({ room, @@ -371,7 +371,7 @@ class ArchiveRoomViewModel extends ViewModel { // `_dayTimestampTo` anyway. const lastEventTs = events[events.length - 1]?.origin_server_ts; const hasEventsFromGivenDay = - lastEventTs && areTimestampsFromSameUtcDay(lastEventTs, this._dayTimestampTo); + lastEventTs && doTimestampsStartFromSameUtcDay(lastEventTs, this._dayTimestampTo); let daySummaryKind; if (events.length === 0) { daySummaryKind = 'no-events-at-all'; diff --git a/shared/views/CalendarView.js b/shared/views/CalendarView.js index 3b7709da..45c87c62 100644 --- a/shared/views/CalendarView.js +++ b/shared/views/CalendarView.js @@ -4,7 +4,7 @@ const { TemplateView } = require('hydrogen-view-sdk'); const { - areTimestampsFromSameUtcDay, + doTimestampsStartFromSameUtcDay, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); // Get the number of days in the given month where the `inputDate` lies. @@ -157,7 +157,7 @@ class CalendarView extends TemplateView { const isDayInFuture = dayNumberDate.getTime() - todayTs > 0; // The current day displayed in the archive - const isActive = areTimestampsFromSameUtcDay( + const isActive = doTimestampsStartFromSameUtcDay( dayNumberDate.getTime(), vm.activeDate.getTime() ); diff --git a/test/e2e-tests.js b/test/e2e-tests.js index fcb1b3af..5f3b4450 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -940,7 +940,8 @@ describe('matrix-public-archive', () => { // Just spread things out a bit so the event times are more obvious // and stand out from each other while debugging and so we just have // to deal with hour time slicing - const eventSendTimeIncrement = ONE_HOUR_IN_MS; + const eventSendTimeIncrement = + testCase.timeIncrementBetweenMessages || ONE_HOUR_IN_MS; for (const eventMeta of room.events) { const archiveDate = new Date(Date.UTC(2022, 0, eventMeta.dayNumber, 0, 0, 0, 1)); @@ -1751,6 +1752,31 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // TODO + testName: + 'can jump backward to the previous activity when less than an hour between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T01:00', + page1: { + url: '/roomid/room1/date/2022/01/01T01:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:17?at=$event9', + action: null, + }, + }, ]; const jumpBackwardPredecessorTestCases = [ diff --git a/test/shared/lib/timestamp-utilties-tests.js b/test/shared/lib/timestamp-utilties-tests.js index f456d5ba..7a45ad34 100644 --- a/test/shared/lib/timestamp-utilties-tests.js +++ b/test/shared/lib/timestamp-utilties-tests.js @@ -7,16 +7,32 @@ const { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, + getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - areTimestampsFromSameUtcDay, - areTimestampsFromSameUtcHour, - areTimestampsFromSameUtcMinute, - areTimestampsFromSameUtcSecond, + + doTimestampsShareRoundedUpUtcDay, + doTimestampsShareRoundedUpUtcHour, + doTimestampsShareRoundedUpUtcMinute, + doTimestampsShareRoundedUpUtcSecond, + + doTimestampsStartFromSameUtcDay, + doTimestampsStartFromSameUtcHour, + doTimestampsStartFromSameUtcMinute, + doTimestampsStartFromSameUtcSecond, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); +// Handles things like `Uncaught RangeError: Invalid time value` +function getStringifiedTimestampForTestTitle(inputTs) { + try { + new Date(inputTs).toISOString(); + } catch (err) { + return `Invalid time value (${inputTs})`; + } +} + describe('timestamp-utilities', () => { describe('roundUpTimestampToUtcX', () => { function testRoundUpFunction(roundUpFunctionToTest, testMeta) { @@ -272,18 +288,18 @@ describe('timestamp-utilities', () => { }); }); - describe('areTimestampsFromSameUtcX', () => { - function testAreTimestampsFromSameXFunction(areTimestampsFromSameXFunctionToTest, testMeta) { - it(`${testMeta.description} -- A=${new Date( + describe('doTimestampsStartFromSameUtcX', () => { + function testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcXToTest, testMeta) { + it(`${testMeta.description} -- A=${getStringifiedTimestampForTestTitle( testMeta.inputATs - ).toISOString()} and B=${new Date(testMeta.inputBTs).toISOString()} should${ + )} and B=${getStringifiedTimestampForTestTitle(testMeta.inputBTs)} should${ testMeta.expected ? '' : ' *NOT*' } be from the same day`, () => { assert(testMeta.inputATs); assert(testMeta.inputBTs); assert(testMeta.expected !== undefined); - const actualValue = areTimestampsFromSameXFunctionToTest( + const actualValue = doTimestampsStartFromSameUtcXToTest( testMeta.inputATs, testMeta.inputBTs ); @@ -291,7 +307,7 @@ describe('timestamp-utilities', () => { }); } - describe('areTimestampsFromSameUtcDay', () => { + describe('doTimestampsStartFromSameUtcDay', () => { [ { description: 'same timestamp is considered from the same day', @@ -333,11 +349,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcDay, testMeta); + testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcDay, testMeta); }); }); - describe('areTimestampsFromSameUtcHour', () => { + describe('doTimestampsStartFromSameUtcHour', () => { [ { description: 'same timestamp is considered from the same hour', @@ -386,11 +402,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcHour, testMeta); + testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcHour, testMeta); }); }); - describe('areTimestampsFromSameUtcMinute', () => { + describe('doTimestampsStartFromSameUtcMinute', () => { [ { description: 'same timestamp is considered from the same minute', @@ -439,11 +455,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcMinute, testMeta); + testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcMinute, testMeta); }); }); - describe('areTimestampsFromSameUtcSecond', () => { + describe('doTimestampsStartFromSameUtcSecond', () => { [ { description: 'same timestamp is considered from the same second', @@ -492,7 +508,243 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcSecond, testMeta); + testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcSecond, testMeta); + }); + }); + }); + + describe('doTimestampsShareRoundedUpUtcX', () => { + function testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcXToTest, testMeta) { + it(`${testMeta.description} -- A=${getStringifiedTimestampForTestTitle( + testMeta.inputATs + )} and B=${getStringifiedTimestampForTestTitle(testMeta.inputBTs)} should${ + testMeta.expected ? '' : ' *NOT*' + } share rounded up day`, () => { + assert(testMeta.inputATs); + assert(testMeta.inputBTs); + assert(testMeta.expected !== undefined); + + const actualValue = doTimestampsShareRoundedUpUtcXToTest( + testMeta.inputATs, + testMeta.inputBTs + ); + assert.strictEqual(actualValue, testMeta.expected); + }); + } + + describe('doTimestampsShareRoundedUpUtcDay', () => { + [ + { + description: 'same timestamp is considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: 'timestamp from the middle of the same day is considered sharing', + inputATs: new Date('2022-01-15T01:03:03.003Z').getTime(), + inputBTs: new Date('2022-01-15T05:05:05.005Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at extremes of the day for this function is considered sharing (.001 rounds up to the next day)', + inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), + inputBTs: new Date('2022-01-16T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at 00:00:00.000 extreme is a different day than anything after it', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), + expected: false, + }, + { + description: + 'timestamp from different days (exactly 24 hours apart) should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-16T00:00:00.000Z').getTime(), + expected: false, + }, + { + description: + 'timestamp that is only 1ms from the other at end of the should be considered sharing', + inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp that is less than a day apart but from different days should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T04:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-14T20:00:00.000Z').getTime(), + expected: false, + }, + ].forEach((testMeta) => { + testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcDay, testMeta); + }); + }); + + describe('doTimestampsShareRoundedUpUtcHour', () => { + [ + { + description: 'same timestamp is considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: 'timestamp from the middle of the same hour is considered sharing', + inputATs: new Date('2022-01-15T05:03:03.003Z').getTime(), + inputBTs: new Date('2022-01-15T05:35:05.005Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at extremes of the hour for this function is considered sharing (.001 rounds up to the next day)', + inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), + inputBTs: new Date('2022-01-15T01:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at 00:00:00.000 extreme is a different hour than anything after it', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), + expected: false, + }, + { + description: + 'timestamp from different hours (exactly 60 minutes apart) should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T01:00:00.000Z').getTime(), + expected: false, + }, + { + description: + 'timestamp that is only 1ms from the other at end of the should be considered sharing', + inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp that is less than a hour apart but from different hours should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T04:45:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T05:10:00.000Z').getTime(), + expected: false, + }, + ].forEach((testMeta) => { + testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcHour, testMeta); + }); + }); + + describe('doTimestampsShareRoundedUpUtcMinute', () => { + [ + { + description: 'same timestamp is considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: 'timestamp from the middle of the same minute is considered sharing', + inputATs: new Date('2022-01-15T05:30:03.003Z').getTime(), + inputBTs: new Date('2022-01-15T05:30:35.005Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at extremes of the minute for this function is considered sharing (.001 rounds up to the next day)', + inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), + inputBTs: new Date('2022-01-15T00:01:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at 00:00:00.000 extreme is a different minute than anything after it', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), + expected: false, + }, + { + description: + 'timestamp from different minutes (exactly 60 seconds apart) should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:01:00.000Z').getTime(), + expected: false, + }, + { + description: + 'timestamp that is only 1ms from the other at end of the should be considered sharing', + inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp that is less than a minute apart but from different minutes should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T05:45:45.000Z').getTime(), + inputBTs: new Date('2022-01-15T05:46:10.000Z').getTime(), + expected: false, + }, + ].forEach((testMeta) => { + testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcMinute, testMeta); + }); + }); + + describe('doTimestampsShareRoundedUpUtcSecond', () => { + [ + { + description: 'same timestamp is considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: 'timestamp from the middle of the same second is considered sharing', + inputATs: new Date('2022-01-15T05:30:35.003Z').getTime(), + inputBTs: new Date('2022-01-15T05:30:35.035Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at extremes of the second for this function is considered sharing (.001 rounds up to the next day)', + inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:01.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp at 00:00:00.000 extreme is a different second than anything after it', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), + expected: false, + }, + { + description: + 'timestamp from different seconds (exactly 1000ms apart) should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:01.000Z').getTime(), + expected: false, + }, + { + description: + 'timestamp that is only 1ms from the other at end of the should be considered sharing', + inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), + inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), + expected: true, + }, + { + description: + 'timestamp that is less than a second apart but from different seconds should *NOT* be considered sharing', + inputATs: new Date('2022-01-15T04:45:45.750Z').getTime(), + inputBTs: new Date('2022-01-15T05:45:46.110Z').getTime(), + expected: false, + }, + ].forEach((testMeta) => { + testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcSecond, testMeta); }); }); }); From 4d2a130a8f53abd790849967d03971f278adc475 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 21:57:01 -0500 Subject: [PATCH 22/30] Add some tests that fail starting from larger precision We shouldn't see the same messages after paginating --- server/routes/room-routes.js | 6 ++- test/e2e-tests.js | 75 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index cdef0a1f..f269e0f9 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -376,10 +376,14 @@ router.get( // We choose `currentRangeEndTs` vs the `ts` (the jump point) because TODO: why? // // We use `doTimestampsStartFromSameUtcDay` for day precision because TODO: why? + // ... because a day should be from T00:00:00.000 to T23:59:59.999. const fromSameDay = tsForClosestEvent && doTimestampsStartFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); - // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: why? + // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: + // why? ... so that when the URL is `T02:00`, a message from `T01:23` will still + // be considered from the same hour. But also when the URL is `T01:00`, a + // message from `T01:23` will be considered from a *different* hour. const fromSameHour = tsForClosestEvent && doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 5f3b4450..1fa6dfa0 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1777,6 +1777,81 @@ describe('matrix-public-archive', () => { action: null, }, }, + { + // TODO + testName: + 'can jump backward to the previous activity when less than an hour between all messages (starting from day precision)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:17?at=$event9', + action: null, + }, + }, + { + // TODO + testName: + 'can jump backward to the previous activity when less than an minute between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a second for each but less than an minute when you multiply + // this across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:01', + page1: { + url: '/roomid/room1/date/2022/01/01T00:01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:17?at=$event9', + action: null, + }, + }, + { + // TODO + testName: + 'can jump backward to the previous activity when less than an minute between all messages (starting from day precision)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a second for each but less than an minute when you multiply + // this across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:17?at=$event9', + action: null, + }, + }, ]; const jumpBackwardPredecessorTestCases = [ From 4a2bbcb4c3dac1937a7b99365f4691bbf3331a59 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 23:07:22 -0500 Subject: [PATCH 23/30] Try `ts` and doTimestampsStartFromSameUtcX(ts, tsForClosestEvent) --- server/lib/matrix-utils/fetch-room-data.js | 30 ++++++++-- server/routes/room-routes.js | 64 ++++++++++++++++------ test/e2e-tests.js | 11 +++- test/test-utils/client-utils.js | 7 ++- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index d5115b1b..055f50ce 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -20,7 +20,11 @@ function getStateEndpointForRoomIdAndEventType(roomId, eventType) { ); } -// TODO: Remove this when we have MSC3999 (see usage) +// Unfortunately, we can't just get the event ID from the `/state?format=event` +// endpoint, so we have to do this trick. Related to +// https://github.com/matrix-org/synapse/issues/15454 +// +// TODO: Remove this when we have MSC3999 (because it's the only usage) const removeMe_fetchRoomCreateEventId = traceFunction(async function (matrixAccessToken, roomId) { const { data } = await fetchEndpointAsJson( urlJoin( @@ -46,13 +50,15 @@ const fetchRoomCreationInfo = traceFunction(async function (matrixAccessToken, r let roomCreationTs; let predecessorRoomId; + let predecessorLastKnownEventId; if (stateCreateResDataOutcome.reason === undefined) { const { data } = stateCreateResDataOutcome.value; roomCreationTs = data?.origin_server_ts; + predecessorLastKnownEventId = data?.content?.event_id; predecessorRoomId = data?.content?.predecessor?.room_id; } - return { roomCreationTs, predecessorRoomId }; + return { roomCreationTs, predecessorRoomId, predecessorLastKnownEventId }; }); const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, roomId) { @@ -67,23 +73,28 @@ const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, ro ]); let predecessorRoomId; + let predecessorLastKnownEventId; let predecessorViaServers; // Prefer the dynamic predecessor from the dedicated state event if (statePredecessorResDataOutcome.reason === undefined) { const { data } = statePredecessorResDataOutcome.value; predecessorRoomId = data?.content?.predecessor_room_id; + predecessorLastKnownEventId = data?.content?.last_known_event_id; predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); } // Then fallback to the predecessor defined by the room creation event else if (roomCreationInfoOutcome.reason === undefined) { - ({ predecessorRoomId } = roomCreationInfoOutcome.value); + ({ predecessorRoomId, predecessorLastKnownEventId } = roomCreationInfoOutcome.value); } - const { roomCreationTs } = roomCreationInfoOutcome; + const { roomCreationTs: currentRoomCreationTs } = roomCreationInfoOutcome; return { - roomCreationTs, + // This is prefixed with "current" so we don't get this confused with the + // predecessor room creation timestamp. + currentRoomCreationTs, predecessorRoomId, + predecessorLastKnownEventId, predecessorViaServers, }; }); @@ -177,9 +188,15 @@ const fetchRoomData = traceFunction(async function (matrixAccessToken, roomId) { let roomCreationTs; let predecessorRoomId; + let predecessorLastKnownEventId; let predecessorViaServers; if (predecessorInfoOutcome.reason === undefined) { - ({ roomCreationTs, predecessorRoomId, predecessorViaServers } = predecessorInfoOutcome.value); + ({ + currentRoomCreationTs: roomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + } = predecessorInfoOutcome.value); } let successorRoomId; let successorSetTs; @@ -196,6 +213,7 @@ const fetchRoomData = traceFunction(async function (matrixAccessToken, roomId) { joinRule, roomCreationTs, predecessorRoomId, + predecessorLastKnownEventId, predecessorViaServers, successorRoomId, successorSetTs, diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index f269e0f9..96e818a8 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -378,21 +378,23 @@ router.get( // We use `doTimestampsStartFromSameUtcDay` for day precision because TODO: why? // ... because a day should be from T00:00:00.000 to T23:59:59.999. const fromSameDay = - tsForClosestEvent && - doTimestampsStartFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && doTimestampsStartFromSameUtcDay(ts, tsForClosestEvent); // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: // why? ... so that when the URL is `T02:00`, a message from `T01:23` will still // be considered from the same hour. But also when the URL is `T01:00`, a // message from `T01:23` will be considered from a *different* hour. const fromSameHour = tsForClosestEvent && - doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); + //doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); + doTimestampsStartFromSameUtcHour(ts, tsForClosestEvent); const fromSameMinute = tsForClosestEvent && - doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); + //doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); + doTimestampsStartFromSameUtcMinute(ts, tsForClosestEvent); const fromSameSecond = tsForClosestEvent && - doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); + //doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); + doTimestampsStartFromSameUtcSecond(ts, tsForClosestEvent); console.log('fromSameDay', fromSameDay); console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); console.log('fromSameMinute', fromSameMinute); @@ -609,8 +611,12 @@ router.get( // we should try to go to the predecessor/successor room appropriately. if (is404Error) { if (dir === DIRECTION.backward) { - const { roomCreationTs, predecessorRoomId, predecessorViaServers } = - await fetchPredecessorInfo(matrixAccessToken, roomId); + const { + currentRoomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + } = await fetchPredecessorInfo(matrixAccessToken, roomId); if (!predecessorRoomId) { throw new StatusError( @@ -627,6 +633,23 @@ router.get( successorSetTs: successorSetTsForPredecessor, } = await fetchSuccessorInfo(matrixAccessToken, predecessorRoomId); + let tombstoneEventId; + if (!predecessorLastKnownEventId) { + // This is a hack because we can't get the tombstone event ID directly from + // `fetchSuccessorInfo(...)` and the `/state?format=event` + // endpoint, so we have to do this trick. Related to + // https://github.com/matrix-org/synapse/issues/15454 + // + // We just assume this is the tombstone event ID but in any case it gets us to + // an event that happened at the same time. + ({ eventId: tombstoneEventId } = await timestampToEvent({ + accessToken: matrixAccessToken, + roomId: predecessorRoomId, + ts: successorSetTsForPredecessor, + direction: DIRECTION.backward, + })); + } + // Try to continue from the tombstone event in the predecessor room because // that is the signal that the room admins gave to indicate the end of the // room in favor of the other regardless of further activity that may have @@ -643,7 +666,7 @@ router.get( // room tombstone which will work just fine and as expected for normal room // upgrade scenarios. else { - continueAtTsInPredecessorRoom = roomCreationTs; + continueAtTsInPredecessorRoom = currentRoomCreationTs; } if ( @@ -665,17 +688,22 @@ router.get( console.log( `/jump hit the beginning of the room, jumping to predecessorRoomId=${predecessorRoomId}` ); + // Going backwards, we already know where to go so we can navigate straight there res.redirect( - matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(predecessorRoomId, { - viaServers: Array.from(predecessorViaServers || []), - dir: DIRECTION.backward, - currentRangeStartTs: continueAtTsInPredecessorRoom, - currentRangeEndTs: currentRangeEndTs, - // We don't need to define - // `currentRangeStartEventId`/`currentRangeEndEventId` here because we're - // jumping to a completely new room so the event IDs won't pertain to the - // new room and we don't have any to use anyway. - }) + matrixPublicArchiveURLCreator.archiveUrlForDate( + predecessorRoomId, + new Date(continueAtTsInPredecessorRoom), + { + viaServers: Array.from(predecessorViaServers || []), + scrollStartEventId: predecessorLastKnownEventId || tombstoneEventId, + // We can just visit a rough time where the tombstone is as we assume + // it's the last event in the room or at least the last event we care + // about. A given day should be good for most cases but it's possible + // that messages are sent after the tombstone and we end up missing the + // tombstone. + preferredPrecision: TIME_PRECISION_VALUES.none, + } + ) ); return; } else if (dir === DIRECTION.forward) { diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 1fa6dfa0..c46e2b0a 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -911,7 +911,7 @@ describe('matrix-public-archive', () => { for (const [roomIndex, room] of rooms.entries()) { let roomId; if (previousRoomId) { - roomId = await upgradeTestRoom({ + const { newRoomid, tombstoneEventId } = await upgradeTestRoom({ client, oldRoomId: previousRoomId, //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it. @@ -919,6 +919,11 @@ describe('matrix-public-archive', () => { // things are sequential `/timestamp_to_event` doesn't get confused. timestamp: lastEventTsUsedInPreviousRoom + 1, }); + roomId = newRoomid; + + const fancyEventId = `$tombstone`; + fancyIdentifierToEventIdMap.set(fancyEventId, tombstoneEventId); + eventIdToFancyIdentifierMap.set(tombstoneEventId, fancyEventId); } else { // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it // supports it, see https://github.com/matrix-org/matrix-public-archive/issues/169 @@ -1874,7 +1879,7 @@ describe('matrix-public-archive', () => { action: 'previous', }, page2: { - url: '/roomid/room1/date/2022/01/02?at=$event7', + url: '/roomid/room1/date/2022/01/02?at=$tombstone', action: null, }, }, @@ -1897,7 +1902,7 @@ describe('matrix-public-archive', () => { action: 'previous', }, page2: { - url: '/roomid/room1/date/2022/01/02T05:00?at=$event7', + url: '/roomid/room1/date/2022/01/02?at=$tombstone', action: null, }, }, diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 92d277bc..e101618d 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -214,7 +214,7 @@ async function upgradeTestRoom({ const newRoomid = await createTestRoom(client, createOptions); // Now send the tombstone event pointing from the old room to the new room - await sendEvent({ + const tombstoneEventId = await sendEvent({ client, roomId: oldRoomId, eventType: 'm.room.tombstone', @@ -225,7 +225,10 @@ async function upgradeTestRoom({ timestamp, }); - return newRoomid; + return { + newRoomid, + tombstoneEventId, + }; } async function getCanonicalAlias({ client, roomId }) { From 43bdc7d160eef879354159289778c620f224cc9d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Apr 2023 23:50:38 -0500 Subject: [PATCH 24/30] Passing tests and maybe good logic --- server/routes/room-routes.js | 51 ++++++++++++++++++++++++++++-------- test/e2e-tests.js | 10 +++---- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 96e818a8..f958e6c6 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -42,6 +42,7 @@ const { getUtcStartOfMinuteTs, getUtcStartOfSecondTs, + doTimestampsShareRoundedUpUtcDay, doTimestampsShareRoundedUpUtcHour, doTimestampsShareRoundedUpUtcMinute, doTimestampsShareRoundedUpUtcSecond, @@ -378,28 +379,56 @@ router.get( // We use `doTimestampsStartFromSameUtcDay` for day precision because TODO: why? // ... because a day should be from T00:00:00.000 to T23:59:59.999. const fromSameDay = - tsForClosestEvent && doTimestampsStartFromSameUtcDay(ts, tsForClosestEvent); + tsForClosestEvent && + doTimestampsStartFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: // why? ... so that when the URL is `T02:00`, a message from `T01:23` will still // be considered from the same hour. But also when the URL is `T01:00`, a // message from `T01:23` will be considered from a *different* hour. const fromSameHour = tsForClosestEvent && - //doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); - doTimestampsStartFromSameUtcHour(ts, tsForClosestEvent); + doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); + //doTimestampsStartFromSameUtcHour(ts, tsForClosestEvent); const fromSameMinute = tsForClosestEvent && - //doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); - doTimestampsStartFromSameUtcMinute(ts, tsForClosestEvent); + doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); + //doTimestampsStartFromSameUtcMinute(ts, tsForClosestEvent); const fromSameSecond = tsForClosestEvent && - //doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); - doTimestampsStartFromSameUtcSecond(ts, tsForClosestEvent); + doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); + //doTimestampsStartFromSameUtcSecond(ts, tsForClosestEvent); console.log('fromSameDay', fromSameDay); console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); console.log('fromSameMinute', fromSameMinute); console.log('fromSameSecond', fromSameSecond); + // TODO: Give some context for this logic + const currentRangeFromSameDay = doTimestampsStartFromSameUtcDay( + currentRangeStartTs, + tsForClosestEvent + ); + const currentRangeFromSameHour = doTimestampsShareRoundedUpUtcHour( + currentRangeStartTs, + tsForClosestEvent + ); + const currentRangeFromSameMinute = doTimestampsShareRoundedUpUtcMinute( + currentRangeStartTs, + tsForClosestEvent + ); + const currentRangeFromSameSecond = doTimestampsShareRoundedUpUtcSecond( + currentRangeStartTs, + tsForClosestEvent + ); + console.log('currentRangeFromSameDay', currentRangeFromSameDay); + console.log('currentRangeFromSameHour', currentRangeFromSameHour); + console.log('currentRangeFromSameMinute', currentRangeFromSameMinute); + console.log( + 'currentRangeFromSameSecond', + currentRangeFromSameSecond, + currentRangeStartTs, + tsForClosestEvent + ); + // The closest event is from the same second we tried to jump from. Since we // can't represent something smaller than a second in the URL yet (we could do // ms but it's a concious choice to make the URL cleaner, @@ -410,28 +439,28 @@ router.get( // XXX: If there is too many messages all within the same second, people will be // stuck visiting the same page over and over every time they try to jump // backwards from that range. - if (fromSameSecond) { + if (fromSameSecond || currentRangeFromSameSecond) { newOriginServerTs = tsForClosestEvent; preferredPrecision = TIME_PRECISION_VALUES.seconds; } // The closest event is from the same minute we tried to jump from, we will need // to round up to the nearest second so that the URL encompasses the closest // event looking backwards - else if (fromSameMinute) { + else if (fromSameMinute || currentRangeFromSameMinute) { newOriginServerTs = roundUpTimestampToUtcSecond(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.seconds; } // The closest event is from the same hour we tried to jump from, we will need // to round up to the nearest minute so that the URL encompasses the closest // event looking backwards - else if (fromSameHour) { + else if (fromSameHour || currentRangeFromSameHour) { newOriginServerTs = roundUpTimestampToUtcMinute(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.minutes; } // The closest event is from the same day we tried to jump from, we will need to // round up to the nearest hour so that the URL encompasses the closest event // looking backwards - else if (fromSameDay) { + else if (fromSameDay || currentRangeFromSameDay) { newOriginServerTs = roundUpTimestampToUtcHour(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.minutes; } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index c46e2b0a..ee85ba20 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1414,17 +1414,14 @@ describe('matrix-public-archive', () => { // From the first page with too many messages, starting at event5(page1 // rangeStart), we look backwards for the closest event. Because we find // event4 as the closest, which is from a different day from event9(page1 - // rangeEnd), we can just display the day where event5 resides. - // - // Even though there is overlap between - // the pages, our scroll continues from the event where the 1st page starts. + // rangeEnd), we can just display the time where event4 resides. testName: 'can jump backward to the previous activity', roomDayMessageStructureString: ` [room1 ] 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 [day1 ] [day2 ] [day3 ] [day4 ] [page1 ] - [page2 ] + [page2 ] `, archiveMessageLimit: 4, startUrl: '/roomid/room1/date/2022/01/03', @@ -1435,7 +1432,7 @@ describe('matrix-public-archive', () => { page2: { // Continuing from the first event of day2 since we already saw the rest // of day2 in the first page - url: '/roomid/room1/date/2022/01/02?at=$event4', + url: '/roomid/room1/date/2022/01/02T01:00?at=$event4', action: null, }, }, @@ -1857,6 +1854,7 @@ describe('matrix-public-archive', () => { action: null, }, }, + // TODO: Add tests for "less than X" for the forwards direction ]; const jumpBackwardPredecessorTestCases = [ From a969b4dcedf8da88500354c520bd10c92b665fbc Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 00:14:50 -0500 Subject: [PATCH 25/30] Simplify and tests still pass --- server/routes/room-routes.js | 70 ++++++++---------------------------- 1 file changed, 15 insertions(+), 55 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index f958e6c6..fefc4e77 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -366,69 +366,25 @@ router.get( // that same day that still encompasses the closest message looking backwards, // like `/2020/01/02T13:00:00` if (dir === DIRECTION.backward) { - // `currentRangeEndTs` represents what is being displayed in the URL (we fetch - // from this time backwards to render a page): - // - // - When the URL is `/date/2020/01/02`, `currentRangeEndTs=1578009599999` - // (2020-01-02T23:59:59.999Z) - // - When the URL is `/date/2022/11/16T02:00`, `currentRangeEndTs=1577930400000` - // (2020-01-02T02:00:00.000Z) - // - // We choose `currentRangeEndTs` vs the `ts` (the jump point) because TODO: why? - // - // We use `doTimestampsStartFromSameUtcDay` for day precision because TODO: why? - // ... because a day should be from T00:00:00.000 to T23:59:59.999. + // We choose `currentRangeStartTs` instead of `ts` (the jump point) because TODO: why? + // and we don't choose `currentRangeEndTs` because TODO: why? const fromSameDay = tsForClosestEvent && - doTimestampsStartFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); - // We use `doTimestampsShareRoundedUpUtcX` for any time precision because TODO: - // why? ... so that when the URL is `T02:00`, a message from `T01:23` will still - // be considered from the same hour. But also when the URL is `T01:00`, a - // message from `T01:23` will be considered from a *different* hour. + doTimestampsStartFromSameUtcDay(currentRangeStartTs, tsForClosestEvent); const fromSameHour = tsForClosestEvent && - doTimestampsShareRoundedUpUtcHour(currentRangeEndTs, tsForClosestEvent); - //doTimestampsStartFromSameUtcHour(ts, tsForClosestEvent); + doTimestampsStartFromSameUtcHour(currentRangeStartTs, tsForClosestEvent); const fromSameMinute = tsForClosestEvent && - doTimestampsShareRoundedUpUtcMinute(currentRangeEndTs, tsForClosestEvent); - //doTimestampsStartFromSameUtcMinute(ts, tsForClosestEvent); + doTimestampsStartFromSameUtcMinute(currentRangeStartTs, tsForClosestEvent); const fromSameSecond = tsForClosestEvent && - doTimestampsShareRoundedUpUtcSecond(currentRangeEndTs, tsForClosestEvent); - //doTimestampsStartFromSameUtcSecond(ts, tsForClosestEvent); + doTimestampsStartFromSameUtcSecond(currentRangeStartTs, tsForClosestEvent); console.log('fromSameDay', fromSameDay); console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); console.log('fromSameMinute', fromSameMinute); console.log('fromSameSecond', fromSameSecond); - // TODO: Give some context for this logic - const currentRangeFromSameDay = doTimestampsStartFromSameUtcDay( - currentRangeStartTs, - tsForClosestEvent - ); - const currentRangeFromSameHour = doTimestampsShareRoundedUpUtcHour( - currentRangeStartTs, - tsForClosestEvent - ); - const currentRangeFromSameMinute = doTimestampsShareRoundedUpUtcMinute( - currentRangeStartTs, - tsForClosestEvent - ); - const currentRangeFromSameSecond = doTimestampsShareRoundedUpUtcSecond( - currentRangeStartTs, - tsForClosestEvent - ); - console.log('currentRangeFromSameDay', currentRangeFromSameDay); - console.log('currentRangeFromSameHour', currentRangeFromSameHour); - console.log('currentRangeFromSameMinute', currentRangeFromSameMinute); - console.log( - 'currentRangeFromSameSecond', - currentRangeFromSameSecond, - currentRangeStartTs, - tsForClosestEvent - ); - // The closest event is from the same second we tried to jump from. Since we // can't represent something smaller than a second in the URL yet (we could do // ms but it's a concious choice to make the URL cleaner, @@ -439,28 +395,28 @@ router.get( // XXX: If there is too many messages all within the same second, people will be // stuck visiting the same page over and over every time they try to jump // backwards from that range. - if (fromSameSecond || currentRangeFromSameSecond) { + if (fromSameSecond) { newOriginServerTs = tsForClosestEvent; preferredPrecision = TIME_PRECISION_VALUES.seconds; } // The closest event is from the same minute we tried to jump from, we will need // to round up to the nearest second so that the URL encompasses the closest // event looking backwards - else if (fromSameMinute || currentRangeFromSameMinute) { + else if (fromSameMinute) { newOriginServerTs = roundUpTimestampToUtcSecond(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.seconds; } // The closest event is from the same hour we tried to jump from, we will need // to round up to the nearest minute so that the URL encompasses the closest // event looking backwards - else if (fromSameHour || currentRangeFromSameHour) { + else if (fromSameHour) { newOriginServerTs = roundUpTimestampToUtcMinute(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.minutes; } // The closest event is from the same day we tried to jump from, we will need to // round up to the nearest hour so that the URL encompasses the closest event // looking backwards - else if (fromSameDay || currentRangeFromSameDay) { + else if (fromSameDay) { newOriginServerTs = roundUpTimestampToUtcHour(tsForClosestEvent); preferredPrecision = TIME_PRECISION_VALUES.minutes; } @@ -542,7 +498,7 @@ router.get( } // Otherwise do the normal calculation: where we jumped to - where we jumped from else { - // TODO: Should we use `ts` be `currentRangeStartTs`? + // TODO: Should we use `ts` or `currentRangeStartTs` here? msGapFromJumpPointToLastMessage = tsOfLastMessage - ts; } const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; @@ -721,6 +677,10 @@ router.get( res.redirect( matrixPublicArchiveURLCreator.archiveUrlForDate( predecessorRoomId, + // XXX: We should probably go fetch and use the timestamp from + // `predecessorLastKnownEventId` here but that requires an extra + // `timestampToEvent(...)` lookup. We can assume it's close to the + // tombstone for now. new Date(continueAtTsInPredecessorRoom), { viaServers: Array.from(predecessorViaServers || []), From 198a4b4416c10518f47ac843aa5649b5eea7cff2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 00:50:21 -0500 Subject: [PATCH 26/30] Add missing tests (all tests passing) --- test/e2e-tests.js | 111 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 6 deletions(-) diff --git a/test/e2e-tests.js b/test/e2e-tests.js index ee85ba20..e0817ec6 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1754,8 +1754,14 @@ describe('matrix-public-archive', () => { action: null, }, }, + // Tests for "less than X" for the forwards direction + // -------------------------------------------------- { - // TODO + // From the first page with too many messages, starting at event14 with + // minute precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an hour away from event9, we have to + // round up to the nearest minute. testName: 'can jump backward to the previous activity when less than an hour between all messages', roomDayMessageStructureString: ` @@ -1780,7 +1786,11 @@ describe('matrix-public-archive', () => { }, }, { - // TODO + // From the first page with too many messages, starting at event14 with + // day precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an hour away from event9, we have to + // round up to the nearest minute. testName: 'can jump backward to the previous activity when less than an hour between all messages (starting from day precision)', roomDayMessageStructureString: ` @@ -1805,7 +1815,11 @@ describe('matrix-public-archive', () => { }, }, { - // TODO + // From the first page with too many messages, starting at event14 with + // minute precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an minute away from event9, we have to + // round up to the nearest second. testName: 'can jump backward to the previous activity when less than an minute between all messages', roomDayMessageStructureString: ` @@ -1830,7 +1844,11 @@ describe('matrix-public-archive', () => { }, }, { - // TODO + // From the first page with too many messages, starting at event14 with + // day precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an minute away from event9, we have to + // round up to the nearest second. testName: 'can jump backward to the previous activity when less than an minute between all messages (starting from day precision)', roomDayMessageStructureString: ` @@ -1854,7 +1872,69 @@ describe('matrix-public-archive', () => { action: null, }, }, - // TODO: Add tests for "less than X" for the forwards direction + // Tests for "less than X" for the forwards direction + // -------------------------------------------------- + // We can't do the `(start from day precision)` variants when jumping forwards + // because day precision starts off at `T23:59:59.999Z` and jumping forward + // will always land us in the next day. + { + // We jump forward 4 messages (`archiveMessageLimit`) to event10, then + // back-track to the nearest minute which starts off at event9 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward to the next activity when less than an hour between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:11', + page1: { + url: '/roomid/room1/date/2022/01/01T00:11', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:18?at=$event7', + action: null, + }, + }, + { + // We jump forward 4 messages (`archiveMessageLimit`) to event10, then + // back-track to the nearest second which starts off at event9 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward to the next activity when less than an minute between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + // More than a second for each but less than an minute when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:00:11', + page1: { + url: '/roomid/room1/date/2022/01/01T00:00:11', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:18?at=$event7', + action: null, + }, + }, ]; const jumpBackwardPredecessorTestCases = [ @@ -2140,7 +2220,26 @@ describe('matrix-public-archive', () => { action: null, }, }, - // TODO: Test with navigation back to date in time spanning multiple room upgrades + { + testName: 'jumping forward past the end of the room will go down the successor chain', + roomDayMessageStructureString: ` + [room1 ] [room2 ] [room3 ] [room4 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 3, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'navigate:/roomid/room1/date/2022/01/08', + }, + page2: { + url: '/roomid/room4/date/2022/01/08', + action: null, + }, + }, ]; jumpTestCases.forEach((testCase) => { From d00cbfea581980c1649418a0650c87caabd49611 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 00:56:07 -0500 Subject: [PATCH 27/30] Remove `doTimestampsShareRoundedUpUtcX` utilities and rename `doTimestampsStartFromSameUtcX` back to `areTimestampsFromSameUtcX` --- server/routes/room-routes.js | 29 +- shared/lib/timestamp-utilities.js | 39 +-- shared/viewmodels/ArchiveRoomViewModel.js | 6 +- shared/views/CalendarView.js | 4 +- test/shared/lib/timestamp-utilties-tests.js | 286 ++------------------ 5 files changed, 40 insertions(+), 324 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index fefc4e77..3e458882 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -36,21 +36,14 @@ const { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, - getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - - doTimestampsShareRoundedUpUtcDay, - doTimestampsShareRoundedUpUtcHour, - doTimestampsShareRoundedUpUtcMinute, - doTimestampsShareRoundedUpUtcSecond, - - doTimestampsStartFromSameUtcDay, - doTimestampsStartFromSameUtcHour, - doTimestampsStartFromSameUtcMinute, - doTimestampsStartFromSameUtcSecond, + areTimestampsFromSameUtcDay, + areTimestampsFromSameUtcHour, + areTimestampsFromSameUtcMinute, + areTimestampsFromSameUtcSecond, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); const config = require('../lib/config'); @@ -369,17 +362,15 @@ router.get( // We choose `currentRangeStartTs` instead of `ts` (the jump point) because TODO: why? // and we don't choose `currentRangeEndTs` because TODO: why? const fromSameDay = - tsForClosestEvent && - doTimestampsStartFromSameUtcDay(currentRangeStartTs, tsForClosestEvent); + tsForClosestEvent && areTimestampsFromSameUtcDay(currentRangeStartTs, tsForClosestEvent); const fromSameHour = - tsForClosestEvent && - doTimestampsStartFromSameUtcHour(currentRangeStartTs, tsForClosestEvent); + tsForClosestEvent && areTimestampsFromSameUtcHour(currentRangeStartTs, tsForClosestEvent); const fromSameMinute = tsForClosestEvent && - doTimestampsStartFromSameUtcMinute(currentRangeStartTs, tsForClosestEvent); + areTimestampsFromSameUtcMinute(currentRangeStartTs, tsForClosestEvent); const fromSameSecond = tsForClosestEvent && - doTimestampsStartFromSameUtcSecond(currentRangeStartTs, tsForClosestEvent); + areTimestampsFromSameUtcSecond(currentRangeStartTs, tsForClosestEvent); console.log('fromSameDay', fromSameDay); console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); console.log('fromSameMinute', fromSameMinute); @@ -513,7 +504,7 @@ router.get( // We use this information to handle situations where we jump over multiple-day // gaps with no messages in between. In those cases, we don't want to round down // to a day where there are no messages in the gap. - const hasMessagesOnDayBeforeDayOfLastMessage = !doTimestampsStartFromSameUtcDay( + const hasMessagesOnDayBeforeDayOfLastMessage = !areTimestampsFromSameUtcDay( tsOfFirstMessage, tsOfLastMessage ); @@ -877,7 +868,7 @@ router.get( const isNewestEventFromSameDay = newestEvent && newestEvent?.origin_server_ts && - doTimestampsStartFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); + areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); // Check if we need to navigate forward to the successor room if (roomData.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { // Jump to the successor room at the date/time the user is trying to visit at diff --git a/shared/lib/timestamp-utilities.js b/shared/lib/timestamp-utilities.js index e70fd04d..1990add2 100644 --- a/shared/lib/timestamp-utilities.js +++ b/shared/lib/timestamp-utilities.js @@ -70,35 +70,19 @@ function getUtcStartOfSecondTs(ts) { ); } -function doTimestampsShareRoundedUpUtcDay(aTs, bTs) { - return roundUpTimestampToUtcDay(aTs) === roundUpTimestampToUtcDay(bTs); -} - -function doTimestampsShareRoundedUpUtcHour(aTs, bTs) { - return roundUpTimestampToUtcHour(aTs) === roundUpTimestampToUtcHour(bTs); -} - -function doTimestampsShareRoundedUpUtcMinute(aTs, bTs) { - return roundUpTimestampToUtcMinute(aTs) === roundUpTimestampToUtcMinute(bTs); -} - -function doTimestampsShareRoundedUpUtcSecond(aTs, bTs) { - return roundUpTimestampToUtcSecond(aTs) === roundUpTimestampToUtcSecond(bTs); -} - -function doTimestampsStartFromSameUtcDay(aTs, bTs) { +function areTimestampsFromSameUtcDay(aTs, bTs) { return getUtcStartOfDayTs(aTs) === getUtcStartOfDayTs(bTs); } -function doTimestampsStartFromSameUtcHour(aTs, bTs) { +function areTimestampsFromSameUtcHour(aTs, bTs) { return getUtcStartOfHourTs(aTs) === getUtcStartOfHourTs(bTs); } -function doTimestampsStartFromSameUtcMinute(aTs, bTs) { +function areTimestampsFromSameUtcMinute(aTs, bTs) { return getUtcStartOfMinuteTs(aTs) === getUtcStartOfMinuteTs(bTs); } -function doTimestampsStartFromSameUtcSecond(aTs, bTs) { +function areTimestampsFromSameUtcSecond(aTs, bTs) { return getUtcStartOfSecondTs(aTs) === getUtcStartOfSecondTs(bTs); } @@ -107,19 +91,12 @@ module.exports = { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, - getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - - doTimestampsShareRoundedUpUtcDay, - doTimestampsShareRoundedUpUtcHour, - doTimestampsShareRoundedUpUtcMinute, - doTimestampsShareRoundedUpUtcSecond, - - doTimestampsStartFromSameUtcDay, - doTimestampsStartFromSameUtcHour, - doTimestampsStartFromSameUtcMinute, - doTimestampsStartFromSameUtcSecond, + areTimestampsFromSameUtcDay, + areTimestampsFromSameUtcHour, + areTimestampsFromSameUtcMinute, + areTimestampsFromSameUtcSecond, }; diff --git a/shared/viewmodels/ArchiveRoomViewModel.js b/shared/viewmodels/ArchiveRoomViewModel.js index 3c3af6f7..6ff0bd52 100644 --- a/shared/viewmodels/ArchiveRoomViewModel.js +++ b/shared/viewmodels/ArchiveRoomViewModel.js @@ -26,7 +26,7 @@ const { const stubPowerLevelsObservable = require('matrix-public-archive-shared/lib/stub-powerlevels-observable'); const { TIME_PRECISION_VALUES } = require('matrix-public-archive-shared/lib/reference-values'); const { - doTimestampsStartFromSameUtcDay, + areTimestampsFromSameUtcDay, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); let txnCount = 0; @@ -148,7 +148,7 @@ class ArchiveRoomViewModel extends ViewModel { (precisionFromUrl !== TIME_PRECISION_VALUES.none && !events.length) || // Only show the time selector when we're showing events all from the same day. (events.length && - doTimestampsStartFromSameUtcDay(timelineRangeStartTimestamp, timelineRangeEndTimestamp)); + areTimestampsFromSameUtcDay(timelineRangeStartTimestamp, timelineRangeEndTimestamp)); this._timeSelectorViewModel = new TimeSelectorViewModel({ room, @@ -371,7 +371,7 @@ class ArchiveRoomViewModel extends ViewModel { // `_dayTimestampTo` anyway. const lastEventTs = events[events.length - 1]?.origin_server_ts; const hasEventsFromGivenDay = - lastEventTs && doTimestampsStartFromSameUtcDay(lastEventTs, this._dayTimestampTo); + lastEventTs && areTimestampsFromSameUtcDay(lastEventTs, this._dayTimestampTo); let daySummaryKind; if (events.length === 0) { daySummaryKind = 'no-events-at-all'; diff --git a/shared/views/CalendarView.js b/shared/views/CalendarView.js index 45c87c62..3b7709da 100644 --- a/shared/views/CalendarView.js +++ b/shared/views/CalendarView.js @@ -4,7 +4,7 @@ const { TemplateView } = require('hydrogen-view-sdk'); const { - doTimestampsStartFromSameUtcDay, + areTimestampsFromSameUtcDay, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); // Get the number of days in the given month where the `inputDate` lies. @@ -157,7 +157,7 @@ class CalendarView extends TemplateView { const isDayInFuture = dayNumberDate.getTime() - todayTs > 0; // The current day displayed in the archive - const isActive = doTimestampsStartFromSameUtcDay( + const isActive = areTimestampsFromSameUtcDay( dayNumberDate.getTime(), vm.activeDate.getTime() ); diff --git a/test/shared/lib/timestamp-utilties-tests.js b/test/shared/lib/timestamp-utilties-tests.js index 7a45ad34..f456d5ba 100644 --- a/test/shared/lib/timestamp-utilties-tests.js +++ b/test/shared/lib/timestamp-utilties-tests.js @@ -7,32 +7,16 @@ const { roundUpTimestampToUtcHour, roundUpTimestampToUtcMinute, roundUpTimestampToUtcSecond, - getUtcStartOfDayTs, getUtcStartOfHourTs, getUtcStartOfMinuteTs, getUtcStartOfSecondTs, - - doTimestampsShareRoundedUpUtcDay, - doTimestampsShareRoundedUpUtcHour, - doTimestampsShareRoundedUpUtcMinute, - doTimestampsShareRoundedUpUtcSecond, - - doTimestampsStartFromSameUtcDay, - doTimestampsStartFromSameUtcHour, - doTimestampsStartFromSameUtcMinute, - doTimestampsStartFromSameUtcSecond, + areTimestampsFromSameUtcDay, + areTimestampsFromSameUtcHour, + areTimestampsFromSameUtcMinute, + areTimestampsFromSameUtcSecond, } = require('matrix-public-archive-shared/lib/timestamp-utilities'); -// Handles things like `Uncaught RangeError: Invalid time value` -function getStringifiedTimestampForTestTitle(inputTs) { - try { - new Date(inputTs).toISOString(); - } catch (err) { - return `Invalid time value (${inputTs})`; - } -} - describe('timestamp-utilities', () => { describe('roundUpTimestampToUtcX', () => { function testRoundUpFunction(roundUpFunctionToTest, testMeta) { @@ -288,18 +272,18 @@ describe('timestamp-utilities', () => { }); }); - describe('doTimestampsStartFromSameUtcX', () => { - function testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcXToTest, testMeta) { - it(`${testMeta.description} -- A=${getStringifiedTimestampForTestTitle( + describe('areTimestampsFromSameUtcX', () => { + function testAreTimestampsFromSameXFunction(areTimestampsFromSameXFunctionToTest, testMeta) { + it(`${testMeta.description} -- A=${new Date( testMeta.inputATs - )} and B=${getStringifiedTimestampForTestTitle(testMeta.inputBTs)} should${ + ).toISOString()} and B=${new Date(testMeta.inputBTs).toISOString()} should${ testMeta.expected ? '' : ' *NOT*' } be from the same day`, () => { assert(testMeta.inputATs); assert(testMeta.inputBTs); assert(testMeta.expected !== undefined); - const actualValue = doTimestampsStartFromSameUtcXToTest( + const actualValue = areTimestampsFromSameXFunctionToTest( testMeta.inputATs, testMeta.inputBTs ); @@ -307,7 +291,7 @@ describe('timestamp-utilities', () => { }); } - describe('doTimestampsStartFromSameUtcDay', () => { + describe('areTimestampsFromSameUtcDay', () => { [ { description: 'same timestamp is considered from the same day', @@ -349,11 +333,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcDay, testMeta); + testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcDay, testMeta); }); }); - describe('doTimestampsStartFromSameUtcHour', () => { + describe('areTimestampsFromSameUtcHour', () => { [ { description: 'same timestamp is considered from the same hour', @@ -402,11 +386,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcHour, testMeta); + testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcHour, testMeta); }); }); - describe('doTimestampsStartFromSameUtcMinute', () => { + describe('areTimestampsFromSameUtcMinute', () => { [ { description: 'same timestamp is considered from the same minute', @@ -455,11 +439,11 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcMinute, testMeta); + testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcMinute, testMeta); }); }); - describe('doTimestampsStartFromSameUtcSecond', () => { + describe('areTimestampsFromSameUtcSecond', () => { [ { description: 'same timestamp is considered from the same second', @@ -508,243 +492,7 @@ describe('timestamp-utilities', () => { expected: false, }, ].forEach((testMeta) => { - testDoTimestampsStartFromSameUtcX(doTimestampsStartFromSameUtcSecond, testMeta); - }); - }); - }); - - describe('doTimestampsShareRoundedUpUtcX', () => { - function testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcXToTest, testMeta) { - it(`${testMeta.description} -- A=${getStringifiedTimestampForTestTitle( - testMeta.inputATs - )} and B=${getStringifiedTimestampForTestTitle(testMeta.inputBTs)} should${ - testMeta.expected ? '' : ' *NOT*' - } share rounded up day`, () => { - assert(testMeta.inputATs); - assert(testMeta.inputBTs); - assert(testMeta.expected !== undefined); - - const actualValue = doTimestampsShareRoundedUpUtcXToTest( - testMeta.inputATs, - testMeta.inputBTs - ); - assert.strictEqual(actualValue, testMeta.expected); - }); - } - - describe('doTimestampsShareRoundedUpUtcDay', () => { - [ - { - description: 'same timestamp is considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: 'timestamp from the middle of the same day is considered sharing', - inputATs: new Date('2022-01-15T01:03:03.003Z').getTime(), - inputBTs: new Date('2022-01-15T05:05:05.005Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at extremes of the day for this function is considered sharing (.001 rounds up to the next day)', - inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), - inputBTs: new Date('2022-01-16T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at 00:00:00.000 extreme is a different day than anything after it', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), - expected: false, - }, - { - description: - 'timestamp from different days (exactly 24 hours apart) should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-16T00:00:00.000Z').getTime(), - expected: false, - }, - { - description: - 'timestamp that is only 1ms from the other at end of the should be considered sharing', - inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp that is less than a day apart but from different days should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T04:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-14T20:00:00.000Z').getTime(), - expected: false, - }, - ].forEach((testMeta) => { - testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcDay, testMeta); - }); - }); - - describe('doTimestampsShareRoundedUpUtcHour', () => { - [ - { - description: 'same timestamp is considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: 'timestamp from the middle of the same hour is considered sharing', - inputATs: new Date('2022-01-15T05:03:03.003Z').getTime(), - inputBTs: new Date('2022-01-15T05:35:05.005Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at extremes of the hour for this function is considered sharing (.001 rounds up to the next day)', - inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), - inputBTs: new Date('2022-01-15T01:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at 00:00:00.000 extreme is a different hour than anything after it', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), - expected: false, - }, - { - description: - 'timestamp from different hours (exactly 60 minutes apart) should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T01:00:00.000Z').getTime(), - expected: false, - }, - { - description: - 'timestamp that is only 1ms from the other at end of the should be considered sharing', - inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp that is less than a hour apart but from different hours should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T04:45:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T05:10:00.000Z').getTime(), - expected: false, - }, - ].forEach((testMeta) => { - testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcHour, testMeta); - }); - }); - - describe('doTimestampsShareRoundedUpUtcMinute', () => { - [ - { - description: 'same timestamp is considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: 'timestamp from the middle of the same minute is considered sharing', - inputATs: new Date('2022-01-15T05:30:03.003Z').getTime(), - inputBTs: new Date('2022-01-15T05:30:35.005Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at extremes of the minute for this function is considered sharing (.001 rounds up to the next day)', - inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), - inputBTs: new Date('2022-01-15T00:01:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at 00:00:00.000 extreme is a different minute than anything after it', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), - expected: false, - }, - { - description: - 'timestamp from different minutes (exactly 60 seconds apart) should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:01:00.000Z').getTime(), - expected: false, - }, - { - description: - 'timestamp that is only 1ms from the other at end of the should be considered sharing', - inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp that is less than a minute apart but from different minutes should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T05:45:45.000Z').getTime(), - inputBTs: new Date('2022-01-15T05:46:10.000Z').getTime(), - expected: false, - }, - ].forEach((testMeta) => { - testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcMinute, testMeta); - }); - }); - - describe('doTimestampsShareRoundedUpUtcSecond', () => { - [ - { - description: 'same timestamp is considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: 'timestamp from the middle of the same second is considered sharing', - inputATs: new Date('2022-01-15T05:30:35.003Z').getTime(), - inputBTs: new Date('2022-01-15T05:30:35.035Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at extremes of the second for this function is considered sharing (.001 rounds up to the next day)', - inputATs: new Date('2022-01-15T00:00:00.001Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:01.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp at 00:00:00.000 extreme is a different second than anything after it', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.001Z').getTime(), - expected: false, - }, - { - description: - 'timestamp from different seconds (exactly 1000ms apart) should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T00:00:00.000Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:01.000Z').getTime(), - expected: false, - }, - { - description: - 'timestamp that is only 1ms from the other at end of the should be considered sharing', - inputATs: new Date('2022-01-14T23:59:59.999Z').getTime(), - inputBTs: new Date('2022-01-15T00:00:00.000Z').getTime(), - expected: true, - }, - { - description: - 'timestamp that is less than a second apart but from different seconds should *NOT* be considered sharing', - inputATs: new Date('2022-01-15T04:45:45.750Z').getTime(), - inputBTs: new Date('2022-01-15T05:45:46.110Z').getTime(), - expected: false, - }, - ].forEach((testMeta) => { - testDoTimestampsShareRoundedUpUtcX(doTimestampsShareRoundedUpUtcSecond, testMeta); + testAreTimestampsFromSameXFunction(areTimestampsFromSameUtcSecond, testMeta); }); }); }); From 1d894877b29de1b560f0a2e2d4dd0646fb6444ff Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 00:58:38 -0500 Subject: [PATCH 28/30] Remove debug logging --- server/routes/room-routes.js | 38 +++--------------------------------- test/e2e-tests.js | 5 ----- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 3e458882..576ea94a 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -282,11 +282,6 @@ router.get( // updated value between each e2e test const archiveMessageLimit = config.get('archiveMessageLimit'); - console.log( - `jumping from ${new Date( - ts - ).toISOString()} (${ts}) (fromCausalEventId=${fromCausalEventId}) in direction ${dir} (roomId=${roomId}))` - ); let roomCreateEventId; // Find the closest event to the given timestamp [{ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent }, roomCreateEventId] = @@ -309,12 +304,6 @@ router.get( }), removeMe_fetchRoomCreateEventId(matrixAccessToken, roomId), ]); - console.log( - 'found eventIdForClosestEvent', - eventIdForClosestEvent, - new Date(tsForClosestEvent).toISOString(), - tsForClosestEvent - ); // Without MSC3999, we currently only detect one kind of loop where the // `m.room.create` has a timestamp that comes after the timestamp massaged events @@ -371,10 +360,6 @@ router.get( const fromSameSecond = tsForClosestEvent && areTimestampsFromSameUtcSecond(currentRangeStartTs, tsForClosestEvent); - console.log('fromSameDay', fromSameDay); - console.log('fromSameHour', fromSameHour, ts, tsForClosestEvent, currentRangeEndTs); - console.log('fromSameMinute', fromSameMinute); - console.log('fromSameSecond', fromSameSecond); // The closest event is from the same second we tried to jump from. Since we // can't represent something smaller than a second in the URL yet (we could do @@ -458,21 +443,9 @@ router.get( const firstMessage = messageResData.chunk[0]; const tsOfFirstMessage = firstMessage.origin_server_ts; - console.log( - 'tsOfFirstMessage', - new Date(tsOfFirstMessage).toISOString(), - tsOfFirstMessage, - firstMessage.event_id - ); const lastMessage = messageResData.chunk[messageResData.chunk.length - 1]; const tsOfLastMessage = lastMessage.origin_server_ts; - console.log( - 'tsOfLastMessage', - new Date(tsOfLastMessage).toISOString(), - tsOfLastMessage, - lastMessage.event_id - ); let msGapFromJumpPointToLastMessage; // If someone is jumping from `0`, let's assume this is their first time @@ -660,11 +633,9 @@ router.get( ); } - // Jump to the predecessor room at the appropriate timestamp to continue from - console.log( - `/jump hit the beginning of the room, jumping to predecessorRoomId=${predecessorRoomId}` - ); - // Going backwards, we already know where to go so we can navigate straight there + // Jump to the predecessor room at the appropriate timestamp to continue from. + // Since we're going backwards, we already know where to go so we can navigate + // straight there. res.redirect( matrixPublicArchiveURLCreator.archiveUrlForDate( predecessorRoomId, @@ -690,9 +661,6 @@ router.get( const { successorRoomId } = await fetchSuccessorInfo(matrixAccessToken, roomId); if (successorRoomId) { // Jump to the successor room and continue at the first event of the room - console.log( - `/jump hit the end of the room, jumping to successorRoomId=${successorRoomId}` - ); res.redirect( matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(successorRoomId, { dir: DIRECTION.forward, diff --git a/test/e2e-tests.js b/test/e2e-tests.js index e0817ec6..ebd63df9 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1014,11 +1014,6 @@ describe('matrix-public-archive', () => { fancyRoomIdToDebugEventsInRoom.set(fancyRoomId, eventDebugStrings); } - console.log( - 'getDebugStringForEventsInRoomsAndLookForEventId()\n', - getDebugStringForEventsInRoomsAndLookForEventId() - ); - // Now Test // -------------------------------------- // -------------------------------------- From 81bc77a9565a2eac1857da16c894a1dc0a0b44b6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 01:15:03 -0500 Subject: [PATCH 29/30] Prefer tracking just by searching MSC3999 everywhere --- server/routes/room-routes.js | 1 - 1 file changed, 1 deletion(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 576ea94a..57669db3 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -317,7 +317,6 @@ router.get( // start of the room. // // XXX: Once we have MSC3999, we can remove this check in favor of that mechanism - // (TODO: Create issue to track this) if ( dir === DIRECTION.forward && timelineEndEventId && From 9389bb24dcbddd5fdeeee54ec5f5cae6d364c566 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 01:20:27 -0500 Subject: [PATCH 30/30] Add comment thread reference for future reading --- server/routes/room-routes.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 57669db3..16a4416a 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -347,8 +347,10 @@ router.get( // that same day that still encompasses the closest message looking backwards, // like `/2020/01/02T13:00:00` if (dir === DIRECTION.backward) { - // We choose `currentRangeStartTs` instead of `ts` (the jump point) because TODO: why? - // and we don't choose `currentRangeEndTs` because TODO: why? + // We choose `currentRangeStartTs` instead of `ts` (the jump point) because + // TODO: why? and we don't choose `currentRangeEndTs` because TODO: why? - I + // feel like I can't justify this, see + // https://github.com/matrix-org/matrix-public-archive/pull/167#discussion_r1170850432 const fromSameDay = tsForClosestEvent && areTimestampsFromSameUtcDay(currentRangeStartTs, tsForClosestEvent); const fromSameHour =