UBERF-6523: Allow to use zstd (#5333)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
hcengineering · Apr 12, 2024 · f8cc27b · f8cc27b
1 parent ce7fc92
commit f8cc27b
Show file tree

Hide file tree

Showing 10 changed files with 113 additions and 42 deletions.
diff --git a/plugins/workbench-resources/src/components/ServerManager.svelte b/plugins/workbench-resources/src/components/ServerManager.svelte
@@ -30,27 +30,34 @@
     endpoint = endpoint.substring(0, endpoint.length - 1)
   }
 
+  async function fetchStats (): Promise<void> {
+    await fetch(endpoint + `/api/v1/statistics?token=${token}`, {})
+      .then(async (json) => {
+        data = await json.json()
+        admin = data?.admin ?? false
+      })
+      .catch((err) => {
+        console.error(err)
+      })
+  }
+  async function fetchUIStats (): Promise<void> {
+    await fetch(`/api/v1/statistics?token=${token}`, {})
+      .then(async (json) => {
+        dataFront = await json.json()
+      })
+      .catch((err) => {
+        console.error(err)
+      })
+  }
+
   let data: any
   let dataFront: any
   let admin = false
   onDestroy(
     ticker.subscribe(() => {
-      void fetch(endpoint + `/api/v1/statistics?token=${token}`, {})
-        .then(async (json) => {
-          data = await json.json()
-          admin = data?.admin ?? false
-        })
-        .catch((err) => {
-          console.error(err)
-        })
+      void fetchStats()
 
-      void fetch(`/api/v1/statistics?token=${token}`, {})
-        .then(async (json) => {
-          dataFront = await json.json()
-        })
-        .catch((err) => {
-          console.error(err)
-        })
+      void fetchUIStats()
     })
   )
   const tabs: TabItem[] = [
@@ -264,6 +271,24 @@
         {/each}
       </div>
     {:else if selectedTab === 'statistics'}
+      {#if admin}
+        <div class="flex flex-col">
+          <div class="flex-row-center p-1">
+            <div class="p-3">1.</div>
+            <Button
+              icon={IconArrowRight}
+              label={getEmbeddedLabel('Wipe statistics')}
+              on:click={() => {
+                void fetch(endpoint + `/api/v1/manage?token=${token}&operation=wipe-statistics`, {
+                  method: 'PUT'
+                }).then(async () => {
+                  await fetchStats()
+                })
+              }}
+            />
+          </div>
+        </div>
+      {/if}
       <div class="flex-column p-3 h-full" style:overflow="auto">
         {#if metricsData !== undefined}
           <MetricsInfo metrics={metricsData} />

diff --git a/pods/account/Dockerfile b/pods/account/Dockerfile
@@ -1,8 +1,9 @@
-FROM node:20-alpine
+FROM node:20
 
 WORKDIR /usr/src/app
 
 COPY bundle/bundle.js ./
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm
 
 EXPOSE 3000
 CMD [ "node", "bundle.js" ]
diff --git a/pods/backup/Dockerfile b/pods/backup/Dockerfile
@@ -1,8 +1,9 @@
 
-FROM node:20-alpine
+FROM node:20
 
 WORKDIR /usr/src/app
 
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm
 COPY bundle/bundle.js ./
 
 EXPOSE 3000

diff --git a/pods/collaborator/Dockerfile b/pods/collaborator/Dockerfile
@@ -2,7 +2,7 @@
 FROM node:20
 
 WORKDIR /usr/src/app
-RUN npm install --ignore-scripts=false --verbose bufferutil --unsafe-perm
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm
 
 COPY bundle/bundle.js ./
 

diff --git a/pods/front/Dockerfile b/pods/front/Dockerfile
@@ -1,13 +1,11 @@
-FROM node:20-alpine
+FROM node:20
 
-RUN apk add dumb-init
 ENV NODE_ENV production
 
 WORKDIR /app
-RUN npm install --ignore-scripts=false --verbose sharp@v0.32.6 --unsafe-perm
-
+RUN npm install --ignore-scripts=false --verbose sharp@v0.32.6 bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm
 COPY bundle/bundle.js ./
 COPY dist/ ./dist/
 
 EXPOSE 8080
-CMD [ "dumb-init", "node", "./bundle.js" ]
+CMD [ "node", "./bundle.js" ]
diff --git a/pods/server/Dockerfile b/pods/server/Dockerfile
@@ -3,10 +3,9 @@ FROM node:20
 ENV NODE_ENV production
 
 WORKDIR /app
-RUN npm install --ignore-scripts=false --verbose bufferutil --unsafe-perm
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm
 
 COPY bundle/bundle.js ./
-# COPY ./dist/*.node ./
 
 EXPOSE 8080
 CMD [ "node", "./bundle.js" ]
diff --git a/server/mongo/src/rawAdapter.ts b/server/mongo/src/rawAdapter.ts
@@ -45,8 +45,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
       const db = getWorkspaceDB(await client.getClient(), workspace)
       const coll = db.collection(domain)
       let cursor = coll.find<T>(query as Filter<Document>, {
-        checkKeys: false,
-        enableUtf8Validation: false
+        checkKeys: false
       })
 
       let total: number = -1

diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts
@@ -486,8 +486,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     if (options?.total === true) {
       totalPipeline.push({ $count: 'total' })
       const totalCursor = this.db.collection(domain).aggregate(totalPipeline, {
-        checkKeys: false,
-        enableUtf8Validation: false
+        checkKeys: false
       })
       const arr = await toArray(totalCursor)
       total = arr?.[0]?.total ?? 0
@@ -592,8 +591,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     const mongoQuery = this.translateQuery(_class, query)
 
     let cursor = coll.find<T>(mongoQuery, {
-      checkKeys: false,
-      enableUtf8Validation: false
+      checkKeys: false
     })
 
     if (options?.projection !== undefined) {
@@ -1252,15 +1250,19 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
   }
 
   async getModel (ctx: MeasureContext): Promise<Tx[]> {
-    const modelProjection = {
-      '%hash%': 0
-    }
     const cursor = await ctx.with('find', {}, async () =>
-      this.db
-        .collection<Tx>(DOMAIN_TX)
-        .find({ objectSpace: core.space.Model })
-        .sort({ _id: 1, modifiedOn: 1 })
-        .project<Tx>(modelProjection)
+      this.db.collection<Tx>(DOMAIN_TX).find(
+        { objectSpace: core.space.Model },
+        {
+          sort: {
+            _id: 1,
+            modifiedOn: 1
+          },
+          projection: {
+            '%hash%': 0
+          }
+        }
+      )
     )
     const model = await ctx.with('to-array', {}, async () => await toArray<Tx>(cursor))
     // We need to put all core.account.System transactions first

diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts
@@ -22,7 +22,7 @@ import cors from 'cors'
 import express from 'express'
 import http, { type IncomingMessage } from 'http'
 import { WebSocketServer, type RawData, type WebSocket } from 'ws'
-import { getStatistics } from './stats'
+import { getStatistics, wipeStatistics } from './stats'
 import {
   LOGGING_ENABLED,
   type ConnectionSocket,
@@ -118,7 +118,14 @@ export function startHttpServer (
 
           res.writeHead(200)
           res.end()
-          break
+          return
+        }
+        case 'wipe-statistics': {
+          wipeStatistics(ctx)
+
+          res.writeHead(200)
+          res.end()
+          return
         }
         case 'reboot': {
           process.exit(0)

diff --git a/server/ws/src/stats.ts b/server/ws/src/stats.ts
@@ -1,4 +1,10 @@
-import { type MeasureContext, metricsAggregate } from '@hcengineering/core'
+import {
+  type MeasureContext,
+  MeasureMetricsContext,
+  type Metrics,
+  metricsAggregate,
+  type MetricsData
+} from '@hcengineering/core'
 import os from 'os'
 import { type SessionManager } from './types'
 
@@ -33,3 +39,36 @@ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, ad
 
   return data
 }
+
+/**
+ * @public
+ */
+export function wipeStatistics (ctx: MeasureContext): void {
+  const toClean: (Metrics | MetricsData)[] = []
+  function cleanMetrics (m: Metrics | MetricsData): void {
+    m.operations = 0
+    m.value = 0
+    m.topResult = undefined
+    if ('measurements' in m) {
+      for (const v of Object.values(m.measurements)) {
+        toClean.push(v)
+      }
+      for (const v of Object.values(m.params)) {
+        for (const vv of Object.values(v)) {
+          toClean.push(vv)
+        }
+      }
+    }
+  }
+
+  if (ctx instanceof MeasureMetricsContext) {
+    toClean.push(ctx.metrics)
+    while (toClean.length > 0) {
+      const v = toClean.shift()
+      if (v === undefined) {
+        break
+      }
+      cleanMetrics(v)
+    }
+  }
+}