bottomless: emit restored snapshot for waiters (tursodatabase#1252)

This fixes an issue where a db gets restored from bottomless and doesn't
get any writes until shutdown. At this point, the current generation is
the same as the restored one but the graceful shutdown process expects
to wait for that generation to be uploaded which never happens because
there are no writes. This change adds a snapshot generation emit call at
restore time to allow graceful shutdown to happen when there are no
writes without having to checkpoint and upload a new snapshot.

bottomless/src/replicator.rs

@@ -487,6 +487,7 @@ impl Replicator {
                     Err(_) => true,
                 })
                 .await?;
+            tracing::debug!("done waiting");
             match res.deref() {
                 Ok(_) => Ok(true),
                 Err(e) => Err(anyhow!("Failed snapshot generation {}: {}", generation, e)),
@@ -923,6 +924,7 @@ impl Replicator {
             let _ = self.snapshot_notifier.send(Ok(self.generation().ok()));
             return Ok(None);
         }
+        tracing::debug!("snapshotting db file");
         if !self.main_db_exists_and_not_empty().await {
             let generation = self.generation()?;
             tracing::debug!(
@@ -1561,6 +1563,9 @@ impl Replicator {
         };
 
         let (action, recovered) = self.restore_from(generation, timestamp).await?;
+
+        let _ = self.snapshot_notifier.send(Ok(Some(generation)));
+
         tracing::info!(
             "Restoring from generation {generation}: action={action:?}, recovered={recovered}"
         );

libsql-server/local-test-envs

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export LIBSQL_BOTTOMLESS_AWS_ACCESS_KEY_ID=minioadmin
+export LIBSQL_BOTTOMLESS_AWS_DEFAULT_REGION=us-east-1
+export LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY=minioadmin
+export LIBSQL_BOTTOMLESS_BUCKET=turso-dev
+export LIBSQL_BOTTOMLESS_DATABASE_ID=5d64e223-21a3-4835-9815-9613216d9859
+export LIBSQL_BOTTOMLESS_ENDPOINT=http://localhost:9000
+export LIBSQL_BOTTOMLESS_VERIFY_CRC=false
+export SQLD_BACKUP_META_STORE=true
+export SQLD_ENABLE_BOTTOMLESS_REPLICATION=true
+export SQLD_META_STORE_ACCESS_KEY_ID=minioadmin
+export SQLD_META_STORE_BACKUP_ID=metastore-dev
+export SQLD_META_STORE_BACKUP_INTERVAL_S=1
+export SQLD_META_STORE_BUCKET_ENDPOINT=http://localhost:9000
+export SQLD_META_STORE_BUCKET_NAME=turso-dev
+export SQLD_META_STORE_REGION=us-east-1
+export SQLD_META_STORE_SECRET_ACCESS=minioadmin