Merge pull request #2666 from cloudflare/kenton/enable-sql

Allow SQL interface to be enabled without "experimental" compat flag.

src/workerd/api/actor-state.c++

@@ -16,6 +16,7 @@
 #include "sql.h"
 #include <workerd/api/web-socket.h>
 #include <workerd/io/hibernation-manager.h>
+#include <workerd/io/features.h>
 
 namespace workerd::api {
 
@@ -700,14 +701,51 @@ jsg::Promise<void> DurableObjectStorage::sync(jsg::Lock& js) {
  }
 }
 
-jsg::Ref<SqlStorage> DurableObjectStorage::getSql(jsg::Lock& js) {
+SqliteDatabase& DurableObjectStorage::getSqliteDb(jsg::Lock& js) {
  KJ_IF_SOME(db, cache->getSqliteDatabase()) {
- return jsg::alloc<SqlStorage>(db, JSG_THIS);
+ // Actor is SQLite-backed but let's make sure SQL is configured to be enabled.
+ if (enableSql) {
+ return db;
+ } else if (FeatureFlags::get(js).getWorkerdExperimental()) {
+ // For backwards-compatibility, if the `experimental` compat flag is on, enable SQL. This is
+ // deprecated, though, so warn in this case.
+
+ // TODO(soon): Uncomment this warning after the D1 simulator has been updated to use
+ // `enableSql`. Otherwise, people doing local dev against D1 may see the warning
+ // spuriously.
+
+ // IoContext::current().logWarningOnce(
+ // "Enabling SQL API based on the 'experimental' flag, but this will stop working soon. "
+ // "Instead, please set `enableSql = true` in your workerd config for the DO namespace. "
+ // "If using wrangler, under `[[migrations]]` in wrangler.toml, change `new_classes` to "
+ // "`new_sqlite_classes`.");
+
+ return db;
+ } else {
+ // We're presumably running local workerd, which always uses SQLite for DO storage, but we're
+ // trying to simulate a non-SQLite DO namespace for testing purposes.
+ JSG_FAIL_REQUIRE(Error,
+ "SQL is not enabled for this Durable Object class. To enable it, set "
+ "`enableSql = true` in your workerd config for the class. If using wrangler, "
+ "under `[[migrations]]` in wrangler.toml, change `new_classes` to "
+ "`new_sqlite_classes`. Note that this change cannot be made after the class is "
+ "already deployed to production.");
+ }
  } else {
- JSG_FAIL_REQUIRE(Error, "Durable Object is not backed by SQL.");
+ // We're in production (not local workerd) and this DO namespace is not backed by SQLite.
+ JSG_FAIL_REQUIRE(Error,
+ "This Durable Object is not backed by SQLite storage, so the SQL API is not available. "
+ "SQL can be enabled on a new Durable Object class by using the `new_sqlite_classes` "
+ "instead of `new_classes` under `[[migrations]]` in your wrangler.toml, but an "
+ "already-deployed class cannot be converted to SQLite (except by deleting the existing "
+ "data).");
  }
 }
 
+jsg::Ref<SqlStorage> DurableObjectStorage::getSql(jsg::Lock& js) {
+ return jsg::alloc<SqlStorage>(JSG_THIS);
+}
+
 kj::Promise<kj::String> DurableObjectStorage::getCurrentBookmark() {
  return cache->getCurrentBookmark();
 }

src/workerd/api/actor-state.h

@@ -172,12 +172,17 @@ class DurableObjectTransaction;
 
 class DurableObjectStorage: public jsg::Object, public DurableObjectStorageOperations {
 public:
- DurableObjectStorage(IoPtr<ActorCacheInterface> cache): cache(kj::mv(cache)) {}
+ DurableObjectStorage(IoPtr<ActorCacheInterface> cache, bool enableSql)
+ : cache(kj::mv(cache)),
+ enableSql(enableSql) {}
 
  ActorCacheInterface& getActorCacheInterface() {
  return *cache;
  }
 
+ // Throws if not SQLite-backed.
+ SqliteDatabase& getSqliteDb(jsg::Lock& js);
+
  struct TransactionOptions {
  jsg::Optional<kj::Date> asOfTime;
  jsg::Optional<bool> lowPriority;
@@ -233,14 +238,12 @@ class DurableObjectStorage: public jsg::Object, public DurableObjectStorageOpera
  JSG_METHOD(deleteAlarm);
  JSG_METHOD(sync);
 
- if (flags.getWorkerdExperimental()) {
- JSG_LAZY_INSTANCE_PROPERTY(sql, getSql);
- JSG_METHOD(transactionSync);
+ JSG_LAZY_INSTANCE_PROPERTY(sql, getSql);
+ JSG_METHOD(transactionSync);
 
- JSG_METHOD(getCurrentBookmark);
- JSG_METHOD(getBookmarkForTime);
- JSG_METHOD(onNextSessionRestoreBookmark);
- }
+ JSG_METHOD(getCurrentBookmark);
+ JSG_METHOD(getBookmarkForTime);
+ JSG_METHOD(onNextSessionRestoreBookmark);
 
  JSG_TS_OVERRIDE({
  get<T = unknown>(key: string, options?: DurableObjectGetOptions): Promise<T | undefined>;
@@ -268,6 +271,7 @@ class DurableObjectStorage: public jsg::Object, public DurableObjectStorageOpera
 
 private:
  IoPtr<ActorCacheInterface> cache;
+ bool enableSql;
  uint transactionSyncDepth = 0;
 };
 
@@ -501,12 +505,7 @@ class DurableObjectState: public jsg::Object {
  JSG_METHOD(getHibernatableWebSocketEventTimeout);
  JSG_METHOD(getTags);
 
- if (flags.getWorkerdExperimental()) {
- // TODO(someday): This currently exists for testing purposes only but maybe it could be
- // useful to apps in actual production? It's a convenient way to bail out when you discover
- // your state is inconsistent.
- JSG_METHOD(abort);
- }
+ JSG_METHOD(abort);
 
  JSG_TS_ROOT();
  JSG_TS_OVERRIDE({

src/workerd/api/sql-test.wd-test

@@ -11,14 +11,18 @@ const config :Workerd.Config = (
 
 const mainWorker :Workerd.Worker = (
  compatibilityDate = "2024-03-04",
+
+ # "experimental" flag is needed to test `sql.ingest()`
  compatibilityFlags = ["experimental", "nodejs_compat"],
 
  modules = [
  (name = "worker", esModule = embed "sql-test.js"),
  ],
 
  durableObjectNamespaces = [
- (className = "DurableObjectExample", uniqueKey = "210bd0cbd803ef7883a1ee9d86cce06e"),
+ ( className = "DurableObjectExample",
+ uniqueKey = "210bd0cbd803ef7883a1ee9d86cce06e",
+ enableSql = true ),
  ],
 
  durableObjectStorage = (localDisk = "TEST_TMPDIR"),

src/workerd/api/sql.c++

@@ -8,34 +8,33 @@
 
 namespace workerd::api {
 
-SqlStorage::SqlStorage(SqliteDatabase& sqlite, jsg::Ref<DurableObjectStorage> storage)
- : sqlite(IoContext::current().addObject(sqlite)),
- storage(kj::mv(storage)) {}
+SqlStorage::SqlStorage(jsg::Ref<DurableObjectStorage> storage): storage(kj::mv(storage)) {}
 
 SqlStorage::~SqlStorage() {}
 
 jsg::Ref<SqlStorage::Cursor> SqlStorage::exec(
  jsg::Lock& js, kj::String querySql, jsg::Arguments<BindingValue> bindings) {
  SqliteDatabase::Regulator& regulator = *this;
- return jsg::alloc<Cursor>(*sqlite, regulator, querySql, kj::mv(bindings));
+ return jsg::alloc<Cursor>(getDb(js), regulator, querySql, kj::mv(bindings));
 }
 
 SqlStorage::IngestResult SqlStorage::ingest(jsg::Lock& js, kj::String querySql) {
  SqliteDatabase::Regulator& regulator = *this;
- auto result = sqlite->ingestSql(regulator, querySql);
+ auto result = getDb(js).ingestSql(regulator, querySql);
  return IngestResult(
  kj::str(result.remainder), result.rowsRead, result.rowsWritten, result.statementCount);
 }
 
 jsg::Ref<SqlStorage::Statement> SqlStorage::prepare(jsg::Lock& js, kj::String query) {
- return jsg::alloc<Statement>(sqlite->prepare(*this, query));
+ return jsg::alloc<Statement>(getDb(js).prepare(*this, query));
 }
 
-double SqlStorage::getDatabaseSize() {
- int64_t pages = execMemoized(pragmaPageCount,
+double SqlStorage::getDatabaseSize(jsg::Lock& js) {
+ auto& db = getDb(js);
+ int64_t pages = execMemoized(db, pragmaPageCount,
  "select (select * from pragma_page_count) - (select * from pragma_freelist_count);")
  .getInt64(0);
- return pages * getPageSize();
+ return pages * getPageSize(db);
 }
 
 bool SqlStorage::isAllowedName(kj::StringPtr name) const {

src/workerd/api/sql.h

@@ -14,7 +14,7 @@ namespace workerd::api {
 
 class SqlStorage final: public jsg::Object, private SqliteDatabase::Regulator {
 public:
- SqlStorage(SqliteDatabase& sqlite, jsg::Ref<DurableObjectStorage> storage);
+ SqlStorage(jsg::Ref<DurableObjectStorage> storage);
  ~SqlStorage();
 
  using BindingValue = kj::Maybe<kj::OneOf<kj::Array<const byte>, kj::String, double>>;
@@ -28,7 +28,7 @@ class SqlStorage final: public jsg::Object, private SqliteDatabase::Regulator {
 
  jsg::Ref<Statement> prepare(jsg::Lock& js, kj::String query);
 
- double getDatabaseSize();
+ double getDatabaseSize(jsg::Lock& js);
 
  JSG_RESOURCE_TYPE(SqlStorage, CompatibilityFlags::Reader flags) {
  JSG_METHOD(exec);
@@ -58,15 +58,19 @@ class SqlStorage final: public jsg::Object, private SqliteDatabase::Regulator {
  void onError(kj::StringPtr message) const override;
  bool allowTransactions() const override;
 
- IoPtr<SqliteDatabase> sqlite;
+ SqliteDatabase& getDb(jsg::Lock& js) {
+ return storage->getSqliteDb(js);
+ }
+
  jsg::Ref<DurableObjectStorage> storage;
 
  kj::Maybe<uint> pageSize;
  kj::Maybe<IoOwn<SqliteDatabase::Statement>> pragmaPageCount;
  kj::Maybe<IoOwn<SqliteDatabase::Statement>> pragmaGetMaxPageCount;
 
  template <size_t size, typename... Params>
- SqliteDatabase::Query execMemoized(kj::Maybe<IoOwn<SqliteDatabase::Statement>>& slot,
+ SqliteDatabase::Query execMemoized(SqliteDatabase& db,
+ kj::Maybe<IoOwn<SqliteDatabase::Statement>>& slot,
  const char (&sqlCode)[size],
  Params&&... params) {
  // Run a (trusted) statement, preparing it on the first call and reusing the prepared version
@@ -76,16 +80,16 @@ class SqlStorage final: public jsg::Object, private SqliteDatabase::Regulator {
  KJ_IF_SOME(s, slot) {
  stmt = &*s;
  } else {
- stmt = &*slot.emplace(IoContext::current().addObject(kj::heap(sqlite->prepare(sqlCode))));
+ stmt = &*slot.emplace(IoContext::current().addObject(kj::heap(db.prepare(sqlCode))));
  }
  return stmt->run(kj::fwd<Params>(params)...);
  }
 
- uint64_t getPageSize() {
+ uint64_t getPageSize(SqliteDatabase& db) {
  KJ_IF_SOME(p, pageSize) {
  return p;
  } else {
- return pageSize.emplace(sqlite->run("PRAGMA page_size;").getInt64(0));
+ return pageSize.emplace(db.run("PRAGMA page_size;").getInt64(0));
  }
  }
 };

src/workerd/server/server.c++

@@ -1956,11 +1956,21 @@ public:
  });
  };
 
+ bool enableSql = true;
+ KJ_SWITCH_ONEOF(config) {
+ KJ_CASE_ONEOF(c, Durable) {
+ enableSql = c.enableSql;
+ }
+ KJ_CASE_ONEOF(c, Ephemeral) {
+ enableSql = c.enableSql;
+ }
+ }
+
  auto makeStorage =
- [](jsg::Lock& js, const Worker::Api& api,
+ [enableSql = enableSql](jsg::Lock& js, const Worker::Api& api,
  ActorCacheInterface& actorCache) -> jsg::Ref<api::DurableObjectStorage> {
  return jsg::alloc<api::DurableObjectStorage>(
- IoContext::current().addObject(actorCache));
+ IoContext::current().addObject(actorCache), enableSql);
  };
 
  TimerChannel& timerChannel = service;
@@ -3560,7 +3570,8 @@ void Server::startServices(jsg::V8System& v8System,
  hadDurable = true;
  serviceActorConfigs.insert(kj::str(ns.getClassName()),
  Durable{.uniqueKey = kj::str(ns.getUniqueKey()),
- .isEvictable = !ns.getPreventEviction()});
+ .isEvictable = !ns.getPreventEviction(),
+ .enableSql = ns.getEnableSql()});
  continue;
  case config::Worker::DurableObjectNamespace::EPHEMERAL_LOCAL:
  if (!experimental) {
@@ -3569,8 +3580,8 @@ void Server::startServices(jsg::V8System& v8System,
  "experimental feature which may change or go away in the future. You must run "
  "workerd with `--experimental` to use this feature."));
  }
- serviceActorConfigs.insert(
- kj::str(ns.getClassName()), Ephemeral{.isEvictable = !ns.getPreventEviction()});
+ serviceActorConfigs.insert(kj::str(ns.getClassName()),
+ Ephemeral{.isEvictable = !ns.getPreventEviction(), .enableSql = ns.getEnableSql()});
  continue;
  }
  reportConfigError(kj::str("Encountered unknown DurableObjectNamespace type in service \"",

src/workerd/server/server.h

@@ -97,9 +97,11 @@ class Server final: private kj::TaskSet::ErrorHandler {
  struct Durable {
  kj::String uniqueKey;
  bool isEvictable;
+ bool enableSql;
  };
  struct Ephemeral {
  bool isEvictable;
+ bool enableSql;
  };
  using ActorConfig = kj::OneOf<Durable, Ephemeral>;
 

src/workerd/server/workerd.capnp

@@ -579,6 +579,14 @@ struct Worker {
  # pinned to memory forever, so we provide this flag to change the default behavior.
  #
  # Note that this is only supported in Workerd; production Durable Objects cannot toggle eviction.
+
+ enableSql @4 :Bool;
+ # Whether or not Durable Objects in this namespace can use the `storage.sql` API to execute SQL
+ # queries.
+ #
+ # workerd uses SQLite to back all Durable Objects, but the SQL API is hidden by default to
+ # emulate behavior of traditional DO namespaces on Cloudflare that aren't SQLite-backed. This
+ # flag should be enabled when testing code that will run on a SQLite-backed namespace.
  }
 
  durableObjectUniqueKeyModifier @8 :Text;

src/workerd/tests/test-fixture.c++

@@ -364,7 +364,8 @@ TestFixture::TestFixture(SetupParams&& params)
  auto makeStorage =
  [](jsg::Lock& js, const Worker::Api& api,
  ActorCacheInterface& actorCache) -> jsg::Ref<api::DurableObjectStorage> {
- return jsg::alloc<api::DurableObjectStorage>(IoContext::current().addObject(actorCache));
+ return jsg::alloc<api::DurableObjectStorage>(
+ IoContext::current().addObject(actorCache), /*enableSql=*/false);
  };
  actor = kj::refcounted<Worker::Actor>(*worker, /*tracker=*/kj::none, kj::mv(id),
  /*hasTransient=*/false, makeActorCache,