[wip] add bundle analyzers and start to cut untentional dependency links

packages/host/app/components/card-prerender.gts

@@ -5,7 +5,7 @@ import Component from '@glimmer/component';
 
 import { didCancel, enqueueTask, restartableTask } from 'ember-concurrency';
 
-import { type IndexWriter } from '@cardstack/runtime-common';
+import { type IndexWriter } from '@cardstack/runtime-common/index-writer';
 import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream';
 import {
   getReader,

packages/host/app/lib/current-run.ts

@@ -19,20 +19,15 @@ import {
   identifyCard,
   moduleFrom,
   isCardDef,
-  IndexWriter,
   unixTime,
-  type Batch,
   type LooseCardResource,
-  type InstanceEntry,
-  type ErrorEntry,
   type CodeRef,
   type RealmInfo,
   type IndexResults,
   type SingleCardDocument,
   type CardResource,
   type Relationship,
   type TextFileRef,
-  type LastModifiedTimes,
 } from '@cardstack/runtime-common';
 import { Deferred } from '@cardstack/runtime-common/deferred';
 import {
@@ -41,8 +36,18 @@ import {
   serializableError,
   type SerializedError,
 } from '@cardstack/runtime-common/error';
-import { RealmPaths, LocalPath } from '@cardstack/runtime-common/paths';
-import { isIgnored } from '@cardstack/runtime-common/realm-index-updater';
+import {
+  IndexWriter,
+  type Batch,
+  type InstanceEntry,
+  type ErrorEntry,
+  type LastModifiedTimes,
+} from '@cardstack/runtime-common/index-writer';
+import {
+  isIgnored,
+  RealmPaths,
+  LocalPath,
+} from '@cardstack/runtime-common/paths';
 import { type Reader, type Stats } from '@cardstack/runtime-common/worker';
 
 import {

packages/host/app/routes/card.gts

@@ -14,7 +14,7 @@ import MatrixService from '@cardstack/host/services/matrix-service';
 import OperatorModeStateService, {
   SerializedState as OperatorModeSerializedState,
 } from '@cardstack/host/services/operator-mode-state-service';
-import Realm from '@cardstack/host/services/realm';
+import type RealmService from '@cardstack/host/services/realm';
 import RealmInfoService from '@cardstack/host/services/realm-info-service';
 
 import { CardDef } from 'https://cardstack.com/base/card-api';
@@ -44,12 +44,12 @@ export default class RenderCard extends Route<Model | null> {
     clientSecret: { refreshModel: true },
   };
 
-  @service declare cardService: CardService;
-  @service declare router: RouterService;
-  @service declare operatorModeStateService: OperatorModeStateService;
-  @service declare matrixService: MatrixService;
-  @service declare realm: Realm;
-  @service declare realmInfoService: RealmInfoService;
+  @service private declare cardService: CardService;
+  @service private declare router: RouterService;
+  @service private declare operatorModeStateService: OperatorModeStateService;
+  @service private declare matrixService: MatrixService;
+  @service private declare realmService: RealmService;
+  @service private declare realmInfoService: RealmInfoService;
 
   hasLoadMatrixBeenExecuted = false;
 
@@ -86,7 +86,7 @@ export default class RenderCard extends Route<Model | null> {
           model = resource.card;
         }
       } else {
-        let indexCardURL = new URL(this.realm.defaultReadableRealm.path);
+        let indexCardURL = new URL(this.realmService.defaultReadableRealm.path);
         let indexCardResource = getCard(this, () => indexCardURL.href);
         await indexCardResource.loaded;
         model = indexCardResource.card;
@@ -123,16 +123,16 @@ export default class RenderCard extends Route<Model | null> {
     // so users will be redirected to operator mode.
     // We can update the codes below after we have a clear idea on how to implement authentication in guest mode.
     let isPublicReadableRealm = await this.realmInfoService.isPublicReadable(
-      new URL(this.realm.defaultReadableRealm.path),
+      new URL(this.realmService.defaultReadableRealm.path),
     );
     if (
       !isPublicReadableRealm &&
       !transition.to?.queryParams['operatorModeEnabled']
     ) {
       let path = transition.to?.params?.path ?? '';
       let url = path
-        ? new URL(`/${path}`, this.realm.defaultReadableRealm.path)
-        : new URL('./', this.realm.defaultReadableRealm.path);
+        ? new URL(`/${path}`, this.realmService.defaultReadableRealm.path)
+        : new URL('./', this.realmService.defaultReadableRealm.path);
       await this.router.replaceWith(`card`, {
         queryParams: {
           operatorModeEnabled: 'true',

packages/host/app/services/local-indexer.ts

@@ -1,6 +1,7 @@
 import Service from '@ember/service';
 
-import { type IndexResults, type IndexWriter } from '@cardstack/runtime-common';
+import type { IndexResults } from '@cardstack/runtime-common';
+import type { IndexWriter } from '@cardstack/runtime-common/index-writer';
 
 import { type TestRealmAdapter } from '@cardstack/host/tests/helpers/adapter';
 

packages/host/ember-cli-build.js

@@ -8,6 +8,9 @@ const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin');
 const MomentLocalesPlugin = require('moment-locales-webpack-plugin');
 const { GlimmerScopedCSSWebpackPlugin } = require('glimmer-scoped-css/webpack');
 const withSideWatch = require('./lib/with-side-watch');
+const StatoscopeWebpackPlugin = require('@statoscope/webpack-plugin').default;
+const BundleAnalyzerPlugin =
+  require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 
 module.exports = function (defaults) {
   const app = new EmberApp(defaults, {
@@ -72,6 +75,10 @@ module.exports = function (defaults) {
             ],
           },
           plugins: [
+            new StatoscopeWebpackPlugin({
+              saveReportTo: '/tmp/report-[name]-[hash].html',
+            }),
+            new BundleAnalyzerPlugin(),
             new GlimmerScopedCSSWebpackPlugin(),
             new MonacoWebpackPlugin(),
             new webpack.ProvidePlugin({

packages/host/package.json

@@ -60,6 +60,7 @@
     "@simple-dom/serializer": "^1.4.0",
     "@simple-dom/void-map": "^1.4.0",
     "@sqlite.org/sqlite-wasm": "3.45.1-build1",
+    "@statoscope/webpack-plugin": "^5.28.2",
     "@types/flat": "^5.0.5",
     "@types/htmlbars-inline-precompile": "^3.0.3",
     "@types/indefinite": "^2.3.4",
@@ -169,7 +170,8 @@
     "typescript": "~5.1.6",
     "uuid": "^9.0.1",
     "wait-for-localhost-cli": "^3.2.0",
-    "webpack": "^5.89.0"
+    "webpack": "^5.89.0",
+    "webpack-bundle-analyzer": "^4.10.2"
   },
   "engines": {
     "node": ">= 18"

packages/host/tests/helpers/index.gts

@@ -20,7 +20,6 @@ import {
   RunnerOptionsManager,
   type RealmInfo,
   type TokenClaims,
-  IndexWriter,
   type RunnerRegistration,
   type IndexRunner,
   type IndexResults,
@@ -32,6 +31,7 @@ import {
   testRealmInfo,
   testRealmURL,
 } from '@cardstack/runtime-common/helpers/const';
+import { IndexWriter } from '@cardstack/runtime-common/index-writer';
 import { Loader } from '@cardstack/runtime-common/loader';
 
 import { Realm } from '@cardstack/runtime-common/realm';

packages/host/tests/integration/components/ai-module-creation-test.gts

@@ -3,7 +3,8 @@ import GlimmerComponent from '@glimmer/component';
 
 import { module, test } from 'qunit';
 
-import { baseRealm, Loader, type Realm } from '@cardstack/runtime-common';
+import { baseRealm, Loader } from '@cardstack/runtime-common';
+import type { Realm } from '@cardstack/runtime-common/realm';
 
 import CardPrerender from '@cardstack/host/components/card-prerender';
 import OperatorMode from '@cardstack/host/components/operator-mode/container';

packages/host/tests/integration/components/prerendered-card-search-test.gts

@@ -10,12 +10,8 @@ import { waitUntil } from '@ember/test-helpers';
 
 import { module, test } from 'qunit';
 
-import {
-  Loader,
-  Query,
-  type Realm,
-  baseRealm,
-} from '@cardstack/runtime-common';
+import { Loader, Query, baseRealm } from '@cardstack/runtime-common';
+import type { Realm } from '@cardstack/runtime-common/realm';
 
 import CardPrerender from '@cardstack/host/components/card-prerender';
 import PrerenderedCardSearch from '@cardstack/host/components/prerendered-card-search';

packages/host/tests/integration/realm-indexing-and-querying-test.gts

@@ -10,11 +10,12 @@ import {
   type CodeRef,
   type LooseSingleCardDocument,
   type IndexedInstance,
-  type Realm,
 } from '@cardstack/runtime-common';
+
 import stripScopedCSSAttributes from '@cardstack/runtime-common/helpers/strip-scoped-css-attributes';
 import { Loader } from '@cardstack/runtime-common/loader';
 import { RealmPaths } from '@cardstack/runtime-common/paths';
+import type { Realm } from '@cardstack/runtime-common/realm';
 
 import { RealmIndexQueryEngine } from '@cardstack/runtime-common/realm-index-query-engine';
 

packages/host/tests/integration/realm-test.ts

@@ -5,7 +5,8 @@ import { module, test } from 'qunit';
 
 import { validate as uuidValidate } from 'uuid';
 
-import { baseRealm, Realm } from '@cardstack/runtime-common';
+import { baseRealm } from '@cardstack/runtime-common';
+
 import { isSingleCardDocument } from '@cardstack/runtime-common/card-document';
 import {
   cardSrc,
@@ -14,6 +15,7 @@ import {
 
 import stripScopedCSSGlimmerAttributes from '@cardstack/runtime-common/helpers/strip-scoped-css-glimmer-attributes';
 import { Loader } from '@cardstack/runtime-common/loader';
+import { Realm } from '@cardstack/runtime-common/realm';
 
 import type * as CardAPI from 'https://cardstack.com/base/card-api';
 import type * as StringFieldMod from 'https://cardstack.com/base/string';

packages/host/tests/unit/index-writer-test.ts

@@ -1,7 +1,8 @@
 import { module, test } from 'qunit';
 
-import { IndexWriter, IndexQueryEngine } from '@cardstack/runtime-common';
+import { IndexQueryEngine } from '@cardstack/runtime-common';
 import { runSharedTest } from '@cardstack/runtime-common/helpers';
+import { IndexWriter } from '@cardstack/runtime-common/index-writer';
 // eslint-disable-next-line ember/no-test-import-export
 import indexWriterTests from '@cardstack/runtime-common/tests/index-writer-test';
 

packages/realm-server/main.ts

@@ -1,12 +1,12 @@
 import './setup-logger'; // This should be first
 import {
-  Realm,
   VirtualNetwork,
   logger,
   RunnerOptionsManager,
   permissionsExist,
   insertPermissions,
 } from '@cardstack/runtime-common';
+import { Realm } from '@cardstack/runtime-common/realm';
 import { NodeAdapter } from './node-realm';
 import yargs from 'yargs';
 import { RealmServer } from './server';

packages/realm-server/pg-queue.ts

@@ -14,8 +14,10 @@ import {
   Deferred,
   upsert,
   Job,
-  setErrorReporter,
 } from '@cardstack/runtime-common';
+
+import { setErrorReporter } from '@cardstack/runtime-common/realm';
+
 import PgAdapter from './pg-adapter';
 import * as Sentry from '@sentry/node';
 

packages/realm-server/server.ts

@@ -3,7 +3,6 @@ import cors from '@koa/cors';
 import Router from '@koa/router';
 import { Memoize } from 'typescript-memoize';
 import {
-  Realm,
   logger,
   SupportedMimeType,
   insertPermissions,
@@ -13,6 +12,8 @@ import {
   type Queue,
   type RealmPermissions,
 } from '@cardstack/runtime-common';
+import { Realm } from '@cardstack/runtime-common/realm';
+
 import {
   ensureDirSync,
   writeJSONSync,

packages/realm-server/tests/helpers/index.ts

@@ -2,7 +2,6 @@ import { writeFileSync, writeJSONSync } from 'fs-extra';
 import { NodeAdapter } from '../../node-realm';
 import { resolve, join } from 'path';
 import {
-  Realm,
   LooseSingleCardDocument,
   baseRealm,
   RealmPermissions,
@@ -14,10 +13,12 @@ import {
   maybeHandleScopedCSSRequest,
   insertPermissions,
   IndexWriter,
-  type MatrixConfig,
   type Queue,
   type IndexRunner,
 } from '@cardstack/runtime-common';
+
+import { Realm, type MatrixConfig } from '@cardstack/runtime-common/realm';
+
 import { dirSync } from 'tmp';
 import { getLocalConfig as getSynapseConfig } from '../../synapse';
 import { makeFastBootIndexRunner } from '../../fastboot';

packages/realm-server/tests/indexing-test.ts

@@ -3,9 +3,10 @@ import { dirSync, setGracefulCleanup } from 'tmp';
 import {
   baseRealm,
   LooseSingleCardDocument,
-  Realm,
   RealmPermissions,
 } from '@cardstack/runtime-common';
+import { Realm } from '@cardstack/runtime-common/realm';
+
 import {
   createRealm,
   testRealm,

packages/realm-server/tests/loader-test.ts

@@ -2,10 +2,11 @@ import { module, test } from 'qunit';
 import {
   Loader,
   VirtualNetwork,
-  type Realm,
   fetcher,
   maybeHandleScopedCSSRequest,
 } from '@cardstack/runtime-common';
+import type { Realm } from '@cardstack/runtime-common/realm';
+
 import { dirSync, setGracefulCleanup, DirResult } from 'tmp';
 import {
   createRealm,

packages/realm-server/tests/realm-server-test.ts

@@ -23,14 +23,15 @@ import {
   loadCard,
   Deferred,
   RealmPaths,
-  Realm,
   RealmPermissions,
   fetchUserPermissions,
   baseCardRef,
   type Queue,
   type LooseSingleCardDocument,
   type SingleCardDocument,
 } from '@cardstack/runtime-common';
+import { Realm } from '@cardstack/runtime-common/realm';
+
 import { stringify } from 'qs';
 import { v4 as uuidv4 } from 'uuid';
 import { Query } from '@cardstack/runtime-common/query';

packages/runtime-common/index.ts

@@ -65,12 +65,10 @@ export * from './constants';
 export * from './queue';
 export * from './expression';
 export * from './index-query-engine';
-export * from './index-writer';
 export * from './index-structure';
 export * from './db';
 export * from './worker';
 export * from './stream';
-export * from './realm';
 export * from './fetcher';
 export * from './scoped-css';
 export * from './authorization-middleware';

packages/runtime-common/paths.ts

@@ -1,3 +1,5 @@
+import type { Ignore } from 'ignore';
+
 interface LocalOptions {
   preserveQuerystring?: boolean;
 }
@@ -70,3 +72,33 @@ function ensureTrailingSlash(realmUrlString: string) {
 //    http://example.com/my-realm/hello/world/ maps to local path "hello/world"
 //
 export type LocalPath = string;
+
+export function isIgnored(
+  realmURL: URL,
+  ignoreMap: Map<string, Ignore>,
+  url: URL,
+): boolean {
+  if (url.href === realmURL.href) {
+    return false; // you can't ignore the entire realm
+  }
+  if (url.href === realmURL.href + '.realm.json') {
+    return true;
+  }
+  if (ignoreMap.size === 0) {
+    return false;
+  }
+  // Test URL against closest ignore. (Should the ignores cascade? so that the
+  // child ignore extends the parent ignore?)
+  let ignoreURLs = [...ignoreMap.keys()];
+  let matchingIgnores = ignoreURLs.filter((u) => url.href.includes(u));
+  let ignoreURL = matchingIgnores.sort((a, b) => b.length - a.length)[0] as
+    | string
+    | undefined;
+  if (!ignoreURL) {
+    return false;
+  }
+  let ignore = ignoreMap.get(ignoreURL)!;
+  let realmPath = new RealmPaths(realmURL);
+  let pathname = realmPath.local(url);
+  return ignore.test(pathname).ignored;
+}