diff --git a/Changelog.md b/Changelog.md index ca395911c..96a80f9c5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,8 @@ - **Fixed** context menu styles ([#600](https://github.com/aws/graph-explorer/pull/600)) +- **Improved** logging on server and around schema sync + ([#604](https://github.com/aws/graph-explorer/pull/604)) - **Improved** support for databases with thousands of node & edge types ([#599](https://github.com/aws/graph-explorer/pull/599)) - **Fixed** alignment of close button in search panel diff --git a/packages/graph-explorer-proxy-server/src/error-handler.ts b/packages/graph-explorer-proxy-server/src/error-handler.ts index 2583345db..5d73bfa40 100644 --- a/packages/graph-explorer-proxy-server/src/error-handler.ts +++ b/packages/graph-explorer-proxy-server/src/error-handler.ts @@ -1,5 +1,5 @@ import { NextFunction, Request, Response } from "express"; -import { logger, logRequestAndResponse } from "./logging.js"; +import { logger } from "./logging.js"; /** * Global error handler @@ -14,7 +14,7 @@ export function handleError(error: unknown) { export function errorHandlingMiddleware() { return ( error: unknown, - request: Request, + _request: Request, response: Response, _next: NextFunction ) => { @@ -22,8 +22,6 @@ export function errorHandlingMiddleware() { response.status(errorInfo.status); - logRequestAndResponse(request, response); - response.send({ error: errorInfo, }); diff --git a/packages/graph-explorer-proxy-server/src/logging.ts b/packages/graph-explorer-proxy-server/src/logging.ts index fc0b47fda..ad3c4b4d6 100644 --- a/packages/graph-explorer-proxy-server/src/logging.ts +++ b/packages/graph-explorer-proxy-server/src/logging.ts @@ -40,7 +40,7 @@ function logLevelFromStatusCode(statusCode: number): LogLevel { } else if (statusCode >= 500) { return "error"; } else if (statusCode >= 300 && statusCode < 400) { - return "silent"; + return "debug"; } return "debug"; } @@ -49,7 +49,7 @@ function logLevelFromStatusCode(statusCode: number): LogLevel { export function logRequestAndResponse(req: Request, res: Response) { const logLevel = logLevelFromStatusCode(res.statusCode); - const requestMessage = `${res.statusCode} - ${req.method} ${req.path}`; + const requestMessage = `[${req.method} ${req.path}] Response ${res.statusCode} ${res.statusMessage}`; switch (logLevel) { case "debug": @@ -79,8 +79,10 @@ export function requestLoggingMiddleware() { return; } - // Wait for the request to complete. - req.on("end", () => { + logger.trace(`[${req.method} ${req.path}] Request received`); + + // Wait for the response to finish + res.on("finish", () => { logRequestAndResponse(req, res); }); diff --git a/packages/graph-explorer-proxy-server/src/node-server.ts b/packages/graph-explorer-proxy-server/src/node-server.ts index 1faef2b23..5a72c7c01 100644 --- a/packages/graph-explorer-proxy-server/src/node-server.ts +++ b/packages/graph-explorer-proxy-server/src/node-server.ts @@ -282,7 +282,7 @@ app.post("/gremlin", (req, res, next) => { if (!queryString) { return res .status(400) - .send({ error: "[Proxy]Gremlin: query not provided" }); + .send({ error: "[Proxy] Gremlin: query not provided" }); } if (shouldLogDbQuery) { @@ -484,7 +484,7 @@ app.post("/logger", (req, res, next) => { if (headers["message"] === undefined) { throw new Error("No log message passed."); } else { - message = headers["message"].replaceAll("\\", ""); + message = JSON.parse(headers["message"]).replaceAll("\\", ""); } if (level.toLowerCase() === "error") { proxyLogger.error(message); diff --git a/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts b/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts index 90ebe0dd8..036b5dd49 100644 --- a/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts +++ b/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts @@ -78,6 +78,7 @@ export function createGremlinExplorer( const summary = await fetchSummary(connection, featureFlags, options); return fetchSchema( _gremlinFetch(connection, featureFlags, options), + remoteLogger, summary ); }, diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.test.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.test.ts index 72ea66440..2e656250c 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.test.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.test.ts @@ -1,12 +1,16 @@ import globalMockFetch from "@/connector/testUtils/globalMockFetch"; import mockGremlinFetch from "@/connector/testUtils/mockGremlinFetch"; import fetchSchema from "./fetchSchema"; +import { ClientLoggerConnector } from "@/connector/LoggerConnector"; describe("Gremlin > fetchSchema", () => { beforeEach(globalMockFetch); it("Should return an inferred schema", async () => { - const schemaResponse = await fetchSchema(mockGremlinFetch()); + const schemaResponse = await fetchSchema( + mockGremlinFetch(), + new ClientLoggerConnector() + ); expect(schemaResponse).toMatchObject({ vertices: [ diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts index f012a1aba..d59c93530 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts @@ -1,4 +1,4 @@ -import { logger, sanitizeText } from "@/utils"; +import { sanitizeText } from "@/utils"; import type { SchemaResponse } from "@/connector/useGEFetchTypes"; import edgeLabelsTemplate from "../templates/edgeLabelsTemplate"; import edgesSchemaTemplate from "../templates/edgesSchemaTemplate"; @@ -7,6 +7,7 @@ import verticesSchemaTemplate from "../templates/verticesSchemaTemplate"; import type { GEdge, GInt64, GVertex } from "../types"; import { GraphSummary, GremlinFetch } from "../types"; import { chunk } from "lodash"; +import { LoggerConnector } from "@/connector/LoggerConnector"; const BATCH_SIZE = 100; @@ -87,10 +88,11 @@ type RawEdgesSchemaResponse = { }; const fetchVertexLabels = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector ): Promise> => { const labelsTemplate = vertexLabelsTemplate(); - logger.log("[Gremlin Explorer] Fetching vertex labels with counts..."); + remoteLogger.info("[Gremlin Explorer] Fetching vertex labels with counts..."); const data = await gremlinFetch(labelsTemplate); const values = data.result.data["@value"][0]["@value"]; @@ -99,6 +101,10 @@ const fetchVertexLabels = async ( labelsWithCounts[values[i] as string] = (values[i + 1] as GInt64)["@value"]; } + remoteLogger.info( + `[Gremlin Explorer] Found ${Object.keys(labelsWithCounts).length} vertex labels.` + ); + return labelsWithCounts; }; @@ -112,6 +118,7 @@ const TYPE_MAP = { const fetchVerticesAttributes = async ( gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector, labels: Array, countsByLabel: Record ): Promise => { @@ -124,12 +131,12 @@ const fetchVerticesAttributes = async ( // Batch in to sets of 100 const batches = chunk(labels, BATCH_SIZE); + remoteLogger.info("[Gremlin Explorer] Fetching vertices attributes..."); for (const batch of batches) { const verticesTemplate = verticesSchemaTemplate({ types: batch, }); - logger.log("[Gremlin Explorer] Fetching vertices attributes..."); const response = await gremlinFetch(verticesTemplate); const verticesSchemas = response.result.data["@value"][0]["@value"]; @@ -157,23 +164,34 @@ const fetchVerticesAttributes = async ( } } + remoteLogger.info( + `[Gremlin Explorer] Found ${vertices.flatMap(v => v.attributes).length} vertex attributes across ${vertices.length} vertex types.` + ); + return vertices; }; const fetchVerticesSchema = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector ): Promise => { - const countsByLabel = await fetchVertexLabels(gremlinFetch); + const countsByLabel = await fetchVertexLabels(gremlinFetch, remoteLogger); const labels = Object.keys(countsByLabel); - return fetchVerticesAttributes(gremlinFetch, labels, countsByLabel); + return fetchVerticesAttributes( + gremlinFetch, + remoteLogger, + labels, + countsByLabel + ); }; const fetchEdgeLabels = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector ): Promise> => { const labelsTemplate = edgeLabelsTemplate(); - logger.log("[Gremlin Explorer] Fetching edge labels with counts..."); + remoteLogger.info("[Gremlin Explorer] Fetching edge labels with counts..."); const data = await gremlinFetch(labelsTemplate); const values = data.result.data["@value"][0]["@value"]; @@ -182,11 +200,16 @@ const fetchEdgeLabels = async ( labelsWithCounts[values[i] as string] = (values[i + 1] as GInt64)["@value"]; } + remoteLogger.info( + `[Gremlin Explorer] Found ${Object.keys(labelsWithCounts).length} edge labels.` + ); + return labelsWithCounts; }; const fetchEdgesAttributes = async ( gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector, labels: Array, countsByLabel: Record ): Promise => { @@ -198,11 +221,11 @@ const fetchEdgesAttributes = async ( // Batch in to sets of 100 const batches = chunk(labels, BATCH_SIZE); + remoteLogger.info("[Gremlin Explorer] Fetching edges attributes..."); for (const batch of batches) { const edgesTemplate = edgesSchemaTemplate({ types: batch, }); - logger.log("[Gremlin Explorer] Fetching edges attributes..."); const data = await gremlinFetch(edgesTemplate); const edgesSchemas = data.result.data["@value"][0]["@value"]; @@ -227,16 +250,26 @@ const fetchEdgesAttributes = async ( } } + remoteLogger.info( + `[Gremlin Explorer] Found ${edges.flatMap(e => e.attributes).length} edge attributes across ${edges.length} edge types.` + ); + return edges; }; const fetchEdgesSchema = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector ): Promise => { - const countsByLabel = await fetchEdgeLabels(gremlinFetch); + const countsByLabel = await fetchEdgeLabels(gremlinFetch, remoteLogger); const labels = Object.keys(countsByLabel); - return fetchEdgesAttributes(gremlinFetch, labels, countsByLabel); + return fetchEdgesAttributes( + gremlinFetch, + remoteLogger, + labels, + countsByLabel + ); }; /** @@ -252,21 +285,26 @@ const fetchEdgesSchema = async ( */ const fetchSchema = async ( gremlinFetch: GremlinFetch, + remoteLogger: LoggerConnector, summary?: GraphSummary ): Promise => { if (!summary) { - logger.log("[Gremlin Explorer] No summary statistics"); + remoteLogger.info("[Gremlin Explorer] No summary statistics"); - const vertices = await fetchVerticesSchema(gremlinFetch); + const vertices = await fetchVerticesSchema(gremlinFetch, remoteLogger); const totalVertices = vertices.reduce((total, vertex) => { return total + (vertex.total ?? 0); }, 0); - const edges = await fetchEdgesSchema(gremlinFetch); + const edges = await fetchEdgesSchema(gremlinFetch, remoteLogger); const totalEdges = edges.reduce((total, edge) => { return total + (edge.total ?? 0); }, 0); + remoteLogger.info( + `[Gremlin Explorer] Schema sync successful (${totalVertices} vertices; ${totalEdges} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); + return { totalVertices, vertices, @@ -275,19 +313,25 @@ const fetchSchema = async ( }; } - logger.log("[Gremlin Explorer] Using summary statistics"); + remoteLogger.info("[Gremlin Explorer] Using summary statistics"); const vertices = await fetchVerticesAttributes( gremlinFetch, + remoteLogger, summary.nodeLabels, {} ); const edges = await fetchEdgesAttributes( gremlinFetch, + remoteLogger, summary.edgeLabels, {} ); + remoteLogger.info( + `[Gremlin Explorer] Schema sync successful (${summary.numNodes} vertices; ${summary.numEdges} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); + return { totalVertices: summary.numNodes, vertices, diff --git a/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts b/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts index 2123e64ec..da355854e 100644 --- a/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts +++ b/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts @@ -53,6 +53,7 @@ export function createOpenCypherExplorer( ); return fetchSchema( _openCypherFetch(connection, featureFlags, options), + remoteLogger, summary ); }, diff --git a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.test.ts b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.test.ts index bbcbf39d0..f0923fc69 100644 --- a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.test.ts +++ b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.test.ts @@ -1,6 +1,7 @@ import { vi } from "vitest"; import { SchemaResponse } from "@/connector/useGEFetchTypes"; import fetchSchema from "./fetchSchema"; +import { ClientLoggerConnector } from "@/connector/LoggerConnector"; describe("OpenCypher > fetchSchema", () => { it("Should return a schema", async () => { @@ -15,7 +16,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); const expected: SchemaResponse = { edges: [ @@ -141,7 +145,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.vertices.length).toBe(0); }); @@ -167,7 +174,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.vertices.length).toBe(0); }); @@ -193,7 +203,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.vertices.length).toBe(0); }); @@ -211,7 +224,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema).toBeDefined(); }); @@ -226,7 +242,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.edges.length).toBe(0); }); @@ -252,7 +271,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.edges.length).toBe(0); }); @@ -278,7 +300,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.edges.length).toBe(0); }); @@ -304,7 +329,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema.edges.length).toBe(0); }); @@ -337,7 +365,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema).toBeDefined(); }); @@ -354,7 +385,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema).toBeDefined(); }); @@ -379,7 +413,10 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - const schema = await fetchSchema(openCypherFetchFn); + const schema = await fetchSchema( + openCypherFetchFn, + new ClientLoggerConnector() + ); expect(schema).toBeDefined(); const routeEdge = schema.edges[0]; expect(routeEdge.total).toBeUndefined(); @@ -398,7 +435,7 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - await fetchSchema(openCypherFetchFn); + await fetchSchema(openCypherFetchFn, new ClientLoggerConnector()); expect(openCypherFetchFn.mock.calls[3][0]).toStrictEqual( "MATCH () -[e:`route`]- () RETURN e AS object LIMIT 1" @@ -420,7 +457,7 @@ describe("OpenCypher > fetchSchema", () => { throw new Error(query); }); - await fetchSchema(openCypherFetchFn); + await fetchSchema(openCypherFetchFn, new ClientLoggerConnector()); expect(openCypherFetchFn.mock.calls[3][0]).toStrictEqual( "MATCH () -[e:`route`]- () RETURN e AS object LIMIT 1" diff --git a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts index 21e6a86e9..da36837f1 100644 --- a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts @@ -11,6 +11,7 @@ import vertexLabelsTemplate from "../templates/vertexLabelsTemplate"; import verticesSchemaTemplate from "../templates/verticesSchemaTemplate"; import type { OCEdge, OCVertex } from "../types"; import { GraphSummary, OpenCypherFetch } from "../types"; +import { LoggerConnector } from "@/connector/LoggerConnector"; // Response types for raw data returned by OpenCypher queries type RawVertexLabelsResponse = { @@ -55,9 +56,13 @@ type RawEdgesSchemaResponse = { // Fetches all vertex labels and their counts const fetchVertexLabels = async ( - openCypherFetch: OpenCypherFetch + openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector ): Promise> => { const labelsTemplate = vertexLabelsTemplate(); + remoteLogger.info( + "[openCypher Explorer] Fetching vertex labels with counts..." + ); const data = await openCypherFetch(labelsTemplate); const values = data.results || []; @@ -77,12 +82,17 @@ const fetchVertexLabels = async ( labelsWithCounts[label] = vertex.count; } + remoteLogger.info( + `[openCypher Explorer] Found ${Object.keys(labelsWithCounts).length} vertex labels.` + ); + return labelsWithCounts; }; // Fetches attributes for all vertices with the given labels const fetchVerticesAttributes = async ( openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector, labels: Array, countsByLabel: Record ): Promise => { @@ -90,6 +100,7 @@ const fetchVerticesAttributes = async ( return []; } + remoteLogger.info("[openCypher Explorer] Fetching vertices attributes..."); const responses = await batchPromisesSerially( labels, DEFAULT_CONCURRENT_REQUESTS_LIMIT, @@ -135,24 +146,38 @@ const fetchVerticesAttributes = async ( }) .filter(vertexSchema => vertexSchema != null); + remoteLogger.info( + `[openCypher Explorer] Found ${vertices.flatMap(v => v.attributes).length} vertex attributes across ${vertices.length} vertex types.` + ); + return vertices; }; // Fetches schema for all vertices const fetchVerticesSchema = async ( - openCypherFetch: OpenCypherFetch + openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector ): Promise => { - const countsByLabel = await fetchVertexLabels(openCypherFetch); + const countsByLabel = await fetchVertexLabels(openCypherFetch, remoteLogger); const labels = Object.keys(countsByLabel); - return fetchVerticesAttributes(openCypherFetch, labels, countsByLabel); + return fetchVerticesAttributes( + openCypherFetch, + remoteLogger, + labels, + countsByLabel + ); }; // Fetches all edge labels and their counts const fetchEdgeLabels = async ( - openCypherFetch: OpenCypherFetch + openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector ): Promise> => { const labelsTemplate = edgeLabelsTemplate(); + remoteLogger.info( + "[openCypher Explorer] Fetching edge labels with counts..." + ); const data = await openCypherFetch(labelsTemplate); const values = data.results; @@ -177,12 +202,17 @@ const fetchEdgeLabels = async ( labelsWithCounts[label] = edge.count; } + remoteLogger.info( + `[openCypher Explorer] Found ${Object.keys(labelsWithCounts).length} edge labels.` + ); + return labelsWithCounts; }; // Fetches attributes for all edges with the given labels const fetchEdgesAttributes = async ( openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector, labels: Array, countsByLabel: Record ): Promise => { @@ -190,6 +220,7 @@ const fetchEdgesAttributes = async ( return []; } + remoteLogger.info("[openCypher Explorer] Fetching edges attributes..."); const responses = await batchPromisesSerially( labels, DEFAULT_CONCURRENT_REQUESTS_LIMIT, @@ -234,17 +265,27 @@ const fetchEdgesAttributes = async ( }) .filter(edgeSchema => edgeSchema != null); + remoteLogger.info( + `[openCypher Explorer] Found ${edges.flatMap(e => e.attributes).length} edge attributes across ${edges.length} edge types.` + ); + return edges; }; // Fetches schema for all edges const fetchEdgesSchema = async ( - openCypherFetch: OpenCypherFetch + openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector ): Promise => { - const countsByLabel = await fetchEdgeLabels(openCypherFetch); + const countsByLabel = await fetchEdgeLabels(openCypherFetch, remoteLogger); const labels = Object.keys(countsByLabel); - return fetchEdgesAttributes(openCypherFetch, labels, countsByLabel); + return fetchEdgesAttributes( + openCypherFetch, + remoteLogger, + labels, + countsByLabel + ); }; /** @@ -264,19 +305,27 @@ const fetchEdgesSchema = async ( */ const fetchSchema = async ( openCypherFetch: OpenCypherFetch, + remoteLogger: LoggerConnector, summary?: GraphSummary ): Promise => { if (!summary) { - const vertices = (await fetchVerticesSchema(openCypherFetch)) || []; + remoteLogger.info("[openCypher Explorer] No summary statistics"); + + const vertices = + (await fetchVerticesSchema(openCypherFetch, remoteLogger)) || []; const totalVertices = vertices.reduce((total, vertex) => { return total + (vertex.total ?? 0); }, 0); - const edges = (await fetchEdgesSchema(openCypherFetch)) || []; + const edges = (await fetchEdgesSchema(openCypherFetch, remoteLogger)) || []; const totalEdges = edges.reduce((total, edge) => { return total + (edge.total ?? 0); }, 0); + remoteLogger.info( + `[openCypher Explorer] Schema sync successful (${totalVertices} vertices; ${totalEdges} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); + return { totalVertices, vertices, @@ -285,11 +334,26 @@ const fetchSchema = async ( }; } + remoteLogger.info("[openCypher Explorer] Using summary statistics"); + const vertices = - (await fetchVerticesAttributes(openCypherFetch, summary.nodeLabels, {})) || - []; + (await fetchVerticesAttributes( + openCypherFetch, + remoteLogger, + summary.nodeLabels, + {} + )) || []; const edges = - (await fetchEdgesAttributes(openCypherFetch, summary.edgeLabels, {})) || []; + (await fetchEdgesAttributes( + openCypherFetch, + remoteLogger, + summary.edgeLabels, + {} + )) || []; + + remoteLogger.info( + `[openCypher Explorer] Schema sync successful (${summary.numNodes} vertices; ${summary.numEdges} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); return { totalVertices: summary.numNodes, diff --git a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts index cc03c2424..becf35c15 100644 --- a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts @@ -1,10 +1,11 @@ -import { batchPromisesSerially, logger } from "@/utils"; +import { batchPromisesSerially } from "@/utils"; import { DEFAULT_CONCURRENT_REQUESTS_LIMIT } from "@/utils/constants"; import type { SchemaResponse } from "@/connector/useGEFetchTypes"; import classesWithCountsTemplates from "../templates/classesWithCountsTemplates"; import predicatesByClassTemplate from "../templates/predicatesByClassTemplate"; import predicatesWithCountsTemplate from "../templates/predicatesWithCountsTemplate"; import { GraphSummary, RawValue, SparqlFetch } from "../types"; +import { LoggerConnector } from "@/connector/LoggerConnector"; type RawClassesWCountsResponse = { results: { @@ -61,6 +62,7 @@ const displayDescCandidates = [rdfsComment, skosNote, skosDefinition]; const fetchPredicatesByClass = async ( sparqlFetch: SparqlFetch, + remoteLogger: LoggerConnector, classes: Array, countsByClass: Record ) => { @@ -71,10 +73,9 @@ const fetchPredicatesByClass = async ( const classPredicatesTemplate = predicatesByClassTemplate({ class: resourceClass, }); - logger.log("[SPARQL Explorer] Fetching predicates by class...", { - resourceClass, - countsByClass, - }); + remoteLogger.info( + `[SPARQL Explorer] Fetching predicates by class ${resourceClass}...` + ); const predicatesResponse = await sparqlFetch( classPredicatesTemplate @@ -109,9 +110,12 @@ const fetchPredicatesByClass = async ( })); }; -const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { +const fetchClassesSchema = async ( + sparqlFetch: SparqlFetch, + remoteLogger: LoggerConnector +) => { const classesTemplate = classesWithCountsTemplates(); - logger.log("[SPARQL Explorer] Fetching classes schema..."); + remoteLogger.info("[SPARQL Explorer] Fetching classes schema..."); const classesCounts = await sparqlFetch(classesTemplate); @@ -124,14 +128,20 @@ const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { ); }); - return fetchPredicatesByClass(sparqlFetch, classes, countsByClass); + return fetchPredicatesByClass( + sparqlFetch, + remoteLogger, + classes, + countsByClass + ); }; const fetchPredicatesWithCounts = async ( - sparqlFetch: SparqlFetch + sparqlFetch: SparqlFetch, + remoteLogger: LoggerConnector ): Promise> => { const template = predicatesWithCountsTemplate(); - logger.log("[SPARQL Explorer] Fetching predicates with counts..."); + remoteLogger.info("[SPARQL Explorer] Fetching predicates with counts..."); const data = await sparqlFetch(template); const values = data.results.bindings; @@ -143,8 +153,11 @@ const fetchPredicatesWithCounts = async ( return labelsWithCounts; }; -const fetchPredicatesSchema = async (sparqlFetch: SparqlFetch) => { - const allLabels = await fetchPredicatesWithCounts(sparqlFetch); +const fetchPredicatesSchema = async ( + sparqlFetch: SparqlFetch, + remoteLogger: LoggerConnector +) => { + const allLabels = await fetchPredicatesWithCounts(sparqlFetch, remoteLogger); return Object.entries(allLabels).map(([label, count]) => { return { @@ -168,19 +181,24 @@ const fetchPredicatesSchema = async (sparqlFetch: SparqlFetch) => { */ const fetchSchema = async ( sparqlFetch: SparqlFetch, + remoteLogger: LoggerConnector, summary?: GraphSummary ): Promise => { if (!summary) { - const vertices = await fetchClassesSchema(sparqlFetch); + const vertices = await fetchClassesSchema(sparqlFetch, remoteLogger); const totalVertices = vertices.reduce((total, vertex) => { return total + (vertex.total ?? 0); }, 0); - const edges = await fetchPredicatesSchema(sparqlFetch); + const edges = await fetchPredicatesSchema(sparqlFetch, remoteLogger); const totalEdges = edges.reduce((total, edge) => { return total + (edge.total ?? 0); }, 0); + remoteLogger.info( + `[SPARQL Explorer] Schema sync successful (${totalVertices} vertices; ${totalEdges} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); + return { totalVertices, vertices, @@ -191,6 +209,7 @@ const fetchSchema = async ( const vertices = await fetchPredicatesByClass( sparqlFetch, + remoteLogger, summary.classes, {} ); @@ -205,6 +224,10 @@ const fetchSchema = async ( }); }); + remoteLogger.info( + `[SPARQL Explorer] Schema sync successful (${summary.numDistinctSubjects} vertices; ${summary.numQuads} edges; ${vertices.length} vertex types; ${edges.length} edge types)` + ); + return { totalVertices: summary.numDistinctSubjects, vertices, diff --git a/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts b/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts index 7f5d38ad4..d43e25770 100644 --- a/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts +++ b/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts @@ -206,6 +206,7 @@ export function createSparqlExplorer( const summary = await fetchSummary(connection, featureFlags, options); return fetchSchema( _sparqlFetch(connection, featureFlags, options), + remoteLogger, summary ); },