From f1ff8bb6bb73a392808e44b6ff15221cbaf52826 Mon Sep 17 00:00:00 2001 From: Barry Deen Date: Wed, 11 Sep 2024 12:55:40 -0400 Subject: [PATCH 1/3] prevent archiving from getting stuck --- main.go | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index e46445d..f17f521 100644 --- a/main.go +++ b/main.go @@ -147,7 +147,7 @@ func LoadConfig() Config { godotenv.Load(".env") if os.Getenv("REFRESH_INTERVAL_HOURS") == "" { - os.Setenv("REFRESH_INTERVAL_HOURS", "24") + os.Setenv("REFRESH_INTERVAL_HOURS", "3") } refreshInterval, _ := strconv.Atoi(os.Getenv("REFRESH_INTERVAL_HOURS")) @@ -193,7 +193,7 @@ func updateTrustNetworkFilter() { func refreshTrustNetwork(relay *khatru.Relay, ctx context.Context) { runTrustNetworkRefresh := func() { - timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Second) + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() filters := []nostr.Filter{{ @@ -210,7 +210,7 @@ func refreshTrustNetwork(relay *khatru.Relay, ctx context.Context) { log.Println("🌐 building web of trust graph") for i := 0; i < len(oneHopNetwork); i += 100 { - timeout, cancel := context.WithTimeout(ctx, 3*time.Second) + timeout, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() end := i + 100 @@ -321,19 +321,27 @@ func archiveTrustedNotes(relay *khatru.Relay, ctx context.Context) { var trustedNotes uint64 var untrustedNotes uint64 trustNetworkFilterMu.Lock() - for ev := range pool.SubMany(timeout, seedRelays, filters) { - if trustNetworkFilter.Has(xxhash.Sum64([]byte(ev.Event.PubKey))) { - if len(ev.Event.Tags) > 2000 { - continue - } + defer trustNetworkFilterMu.Unlock() - relay.AddEvent(ctx, ev.Event) - log.Println("📦 archived note from", ev.Event.PubKey) - trustedNotes++ - } else { - untrustedNotes++ + for ev := range pool.SubManyEose(timeout, seedRelays, filters) { + select { + case <-ctx.Done(): + log.Println("⏰ Archive process terminated due to timeout") + return + default: + if trustNetworkFilter.Has(xxhash.Sum64([]byte(ev.Event.PubKey))) { + if len(ev.Event.Tags) > 3000 { + continue + } + + relay.AddEvent(ctx, ev.Event) + log.Println("📦 archived note: ", ev.Event.ID) + trustedNotes++ + } else { + untrustedNotes++ + } } } - trustNetworkFilterMu.Unlock() + log.Println("📦 archived", trustedNotes, "trusted notes and discarded", untrustedNotes, "untrusted notes") } From 5d9325793f7f5075947b84ff160d78211a53e0db Mon Sep 17 00:00:00 2001 From: Barry Deen Date: Wed, 11 Sep 2024 13:02:07 -0400 Subject: [PATCH 2/3] return when done, remose eose --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index f17f521..2a2ecec 100644 --- a/main.go +++ b/main.go @@ -323,7 +323,7 @@ func archiveTrustedNotes(relay *khatru.Relay, ctx context.Context) { trustNetworkFilterMu.Lock() defer trustNetworkFilterMu.Unlock() - for ev := range pool.SubManyEose(timeout, seedRelays, filters) { + for ev := range pool.SubMany(timeout, seedRelays, filters) { select { case <-ctx.Done(): log.Println("⏰ Archive process terminated due to timeout") @@ -344,4 +344,5 @@ func archiveTrustedNotes(relay *khatru.Relay, ctx context.Context) { } log.Println("📦 archived", trustedNotes, "trusted notes and discarded", untrustedNotes, "untrusted notes") + return } From 58953dacc070dda906edad7b4a3ea8c6bfde0ef5 Mon Sep 17 00:00:00 2001 From: Barry Deen Date: Wed, 11 Sep 2024 20:13:56 -0400 Subject: [PATCH 3/3] update go-nostr and eventstore --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3b1c424..b5dfb12 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fasthttp/websocket v1.5.7 // indirect - github.com/fiatjaf/eventstore v0.8.1 // indirect + github.com/fiatjaf/eventstore v0.8.2 // indirect github.com/fiatjaf/khatru v0.8.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -33,7 +33,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/nbd-wtf/go-nostr v0.34.15 // indirect + github.com/nbd-wtf/go-nostr v0.35.1-0.20240912000534-64c6057c08ec // indirect github.com/pkg/errors v0.9.1 // indirect github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect github.com/rs/cors v1.7.0 // indirect diff --git a/go.sum b/go.sum index 9efbe0c..5137c66 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/fiatjaf/eventstore v0.5.1 h1:tTh+JYP0RME51VY2QB2Gvtzj6QTaZAnSVhgZtrYq github.com/fiatjaf/eventstore v0.5.1/go.mod h1:r5yCFmrVNT2b1xUOuMnDVS3xPGh97y8IgTcLyY2rYP8= github.com/fiatjaf/eventstore v0.8.1 h1:51LchQNy0Hpb0YQHwqYR5pKBpfDs/KjySlWCbbz2pkc= github.com/fiatjaf/eventstore v0.8.1/go.mod h1:bsp0Ibv0CIcVuFcoM2AEerMWmXRhF8uWXMf+dClhuow= +github.com/fiatjaf/eventstore v0.8.2 h1:nCa3UuJNV5Y5t+SDoPQe7PBmKJ6dhm9TQ/WyR4SCbIM= +github.com/fiatjaf/eventstore v0.8.2/go.mod h1:ck3RxufitHUBjID1RLcRxfX+NMywQzMsdfNpSt6m+9U= github.com/fiatjaf/khatru v0.8.0 h1:hofUi4qbSqkJiKD4rC9EyNdi9obzBvp3ykJOBxuu/h8= github.com/fiatjaf/khatru v0.8.0/go.mod h1:jRmqbbIbEH+y0unt3wMUBwqY/btVussqx5SmBoGhXtg= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -110,6 +112,8 @@ github.com/nbd-wtf/go-nostr v0.34.14 h1:o4n2LkuAtdIjNYJ23sFbcx68UXLnji4j8hYR1Sd2 github.com/nbd-wtf/go-nostr v0.34.14/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs= github.com/nbd-wtf/go-nostr v0.34.15 h1:InUaCM3Mw20U3hyoZBpY39LELHBmMa4bK81iGCY5+W4= github.com/nbd-wtf/go-nostr v0.34.15/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs= +github.com/nbd-wtf/go-nostr v0.35.1-0.20240912000534-64c6057c08ec h1:83Ce39kWnvq6VgfMKSMtcbwBcSmnab2VDX+tlKWZuqo= +github.com/nbd-wtf/go-nostr v0.35.1-0.20240912000534-64c6057c08ec/go.mod h1:RNovrTNC8aQNqkwfdX6y4thxHN3zueewqyZoSiDBaQY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=