From 07ddd3ec8c415d253bbe805eefab844e193399e2 Mon Sep 17 00:00:00 2001 From: Guilherme Dellagustin Date: Sat, 25 Feb 2023 23:32:02 +0100 Subject: [PATCH 1/4] Comments: Load optimization With this commit, the comments are pushed to the UI via a chunked HTTP request, as soon as their replies are loaded. There is further room for optimization by sending the comments and replies in different steps. --- server/index.js | 39 +++++++++++++- ui/src/components/Comments/index.tsx | 78 +++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/server/index.js b/server/index.js index bc6f5f1f..ce69e1e1 100644 --- a/server/index.js +++ b/server/index.js @@ -137,13 +137,48 @@ app.use('/api/comments/byepisodeid', async (req, res) => { const cache = new InMemoryCache(); const fetcher = makeRateLimitedFetcher(fetch); + const sentCommenters = {}; + const threadcap = await makeThreadcap(socialInteract[0].uri, { userAgent, cache, fetcher }); - await updateThreadcap(threadcap, { updateTime: new Date().toISOString(), userAgent, cache, fetcher }); + const callbacks = { + onEvent: e => { + console.log(e); + if (e.kind === 'node-processed' && e.part === 'replies') { + console.log(threadcap); + writeThreadcapChunk(e.nodeId, threadcap, sentCommenters, res); + } + } + } + + await updateThreadcap(threadcap, { updateTime: new Date().toISOString(), userAgent, cache, fetcher, callbacks }); - res.send(threadcap) + res.end() }) +function writeThreadcapChunk(processedNodeId, threadcap, sentCommenters, res) { + const threadcapChunk = {}; + + threadcapChunk.roots = threadcap.roots.filter((root) => root === processedNodeId); + threadcapChunk.nodes = {}; + threadcapChunk.nodes[processedNodeId] = threadcap.nodes[processedNodeId]; + + const comment = threadcapChunk.nodes[processedNodeId].comment; + + // nodes are always new, but commenters can be repeated, we only include + // them in th chunk if they have not been sent before, as there is no purpose + // on sending them again. + // this could be determines by inspecting previous nodes, but this way + // is easier. + if(comment && !sentCommenters[comment.attributedTo]) { + sentCommenters[comment.attributedTo] = true; + threadcapChunk.commenters = {}; + threadcapChunk.commenters[comment.attributedTo] = threadcap.commenters[comment.attributedTo]; + } + + res.write(JSON.stringify(threadcapChunk)) +} + // ------------------------------------------------ // ---------- Static files for API data ----------- // ------------------------------------------------ diff --git a/ui/src/components/Comments/index.tsx b/ui/src/components/Comments/index.tsx index 98115f52..8f06d88e 100644 --- a/ui/src/components/Comments/index.tsx +++ b/ui/src/components/Comments/index.tsx @@ -21,7 +21,8 @@ interface StateComment { attributedTo?: Commenter, replies?: StateComment[], commentError?: string, - repliesError?: string + repliesError?: string, + loaded: boolean } interface Commenter { @@ -43,7 +44,7 @@ class Comment extends React.PureComponent { render(): React.ReactNode { return (
- {!this.props.comment.commentError && + {!this.props.comment.commentError && this.props.comment.loaded && @@ -73,6 +74,14 @@ class Comment extends React.PureComponent { !this.props.comment.commentError && this.props.comment.content &&
} + {!this.props.comment.loaded && + + + + Loading... + + + } {this.props.comment.commentError && @@ -103,9 +112,10 @@ export default class Comments extends React.PureComponent { } async onClickShowComments() { - const stateToSet: any = { - showComments: true, - loadingComments: false + const responseBody = { + roots: [], + nodes: {}, + commenters: {} }; if(!this.state.comments.length) { @@ -115,12 +125,49 @@ export default class Comments extends React.PureComponent { const response = await fetch('/api/comments/byepisodeid?' + new URLSearchParams({id: String(this.props.id) })); - const responseBody = await response.json(); + const reader = response.body.getReader(); - stateToSet.comments = responseBody.roots.map((root) => Comments.buildStateComment(root, responseBody)); - } + const thisComponent = this; - this.setState(stateToSet); + await reader.read().then(function processChunk({done, value}) { + if(done) { + console.log('done'); + thisComponent.setState({ + loadingComments: false + }); + return; + } + const parsedChunk = JSON.parse(new TextDecoder().decode(value)); + + updateResponseBody(responseBody, parsedChunk); + + console.log(responseBody); + + const stateToSet: any = { + showComments: true, + }; + + stateToSet.comments = responseBody.roots.map((root) => Comments.buildStateComment(root, responseBody)); + + thisComponent.setState(stateToSet); + return reader.read().then(processChunk); + }); + + function updateResponseBody(responseBody, parsedChunk) { + responseBody.roots = responseBody.roots.concat(parsedChunk.roots); + for(let key in parsedChunk.nodes) { + responseBody.nodes[key] = parsedChunk.nodes[key]; + } + for(let key in parsedChunk.commenters) { + responseBody.commenters[key] = parsedChunk.commenters[key]; + } + } + } + else { + this.setState({ + showComments: true + }); + } } async onClickHideComments() { @@ -128,17 +175,20 @@ export default class Comments extends React.PureComponent { } private static buildStateComment(commentUrl: string, commentsApiResponseBody): StateComment | null { + let stateComment: StateComment = { + url: commentUrl, + loaded: false + } + const node = commentsApiResponseBody.nodes[commentUrl]; if(!node) { - return null; + return stateComment; } - const commenter = node.comment && commentsApiResponseBody.commenters[node.comment.attributedTo]; + stateComment.loaded = true; - let stateComment: StateComment = { - url: commentUrl - } + const commenter = node.comment && commentsApiResponseBody.commenters[node.comment.attributedTo]; if(node.comment) { const summary = node.comment.summary && DOMPurify.sanitize(Comments.resolveLanguageTaggedValues(node.comment.summary)); From 39fa9eafe86fad4916cdac86f36ae38cf3bbc1c6 Mon Sep 17 00:00:00 2001 From: Guilherme Dellagustin Date: Sun, 26 Feb 2023 18:17:58 +0100 Subject: [PATCH 2/4] Comments: PR #236 clean up excessive log --- server/index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/index.js b/server/index.js index ce69e1e1..fd595fde 100644 --- a/server/index.js +++ b/server/index.js @@ -143,9 +143,7 @@ app.use('/api/comments/byepisodeid', async (req, res) => { const callbacks = { onEvent: e => { - console.log(e); if (e.kind === 'node-processed' && e.part === 'replies') { - console.log(threadcap); writeThreadcapChunk(e.nodeId, threadcap, sentCommenters, res); } } From 00d5293bb7bdbb2052ed372df326fbe20db9a16c Mon Sep 17 00:00:00 2001 From: Guilherme Dellagustin Date: Mon, 27 Feb 2023 22:11:57 +0100 Subject: [PATCH 3/4] Update ui/src/components/Comments/index.tsx Removed leftover debug logs. Co-authored-by: Eric P --- ui/src/components/Comments/index.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/src/components/Comments/index.tsx b/ui/src/components/Comments/index.tsx index 8f06d88e..c486dbef 100644 --- a/ui/src/components/Comments/index.tsx +++ b/ui/src/components/Comments/index.tsx @@ -131,7 +131,6 @@ export default class Comments extends React.PureComponent { await reader.read().then(function processChunk({done, value}) { if(done) { - console.log('done'); thisComponent.setState({ loadingComments: false }); From 2a0cfa3afc6209f82a908c8e41aecc298c046cc2 Mon Sep 17 00:00:00 2001 From: Guilherme Dellagustin Date: Mon, 27 Feb 2023 22:12:05 +0100 Subject: [PATCH 4/4] Update ui/src/components/Comments/index.tsx Removed leftover debug logs. Co-authored-by: Eric P --- ui/src/components/Comments/index.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/src/components/Comments/index.tsx b/ui/src/components/Comments/index.tsx index c486dbef..e175d91b 100644 --- a/ui/src/components/Comments/index.tsx +++ b/ui/src/components/Comments/index.tsx @@ -140,7 +140,6 @@ export default class Comments extends React.PureComponent { updateResponseBody(responseBody, parsedChunk); - console.log(responseBody); const stateToSet: any = { showComments: true,