fix: remove logs from embedder

packages/embedder/EmbeddingsJobQueueStream.ts

@@ -3,7 +3,6 @@ import ms from 'ms'
 import sleep from 'parabol-client/utils/sleep'
 import 'parabol-server/initSentry'
 import getKysely from 'parabol-server/postgres/getKysely'
-import {Logger} from '../server/utils/Logger'
 import {WorkflowOrchestrator} from './WorkflowOrchestrator'
 import {DBJob} from './custom'
 
@@ -47,7 +46,6 @@ export class EmbeddingsJobQueueStream implements AsyncIterableIterator<DBJob> {
  try {
  const job = (await getJob(false)) || (await getJob(true))
  if (!job) {
- Logger.log('JobQueueStream: no jobs found')
  // queue is empty, so sleep for a while
  await sleep(ms('10s'))
  return this.next()

packages/embedder/JobQueueError.ts

@@ -5,15 +5,19 @@ export class JobQueueError extends Error {
  jobData?: Record<string, any>
 
  constructor(
- message: string,
+ message: string | Error,
  retryDelay?: number,
  maxRetries?: number,
  jobData?: Record<string, any>
  ) {
- super(message)
- this.message = message
+ const strMessage = message instanceof Error ? message.message : message
+ super(strMessage)
+ this.message = strMessage
  this.retryDelay = retryDelay
  this.maxRetries = maxRetries
  this.jobData = jobData
+ if (message instanceof Error) {
+ this.stack = message.stack
+ }
  }
 }

packages/embedder/WorkflowOrchestrator.ts

@@ -1,7 +1,6 @@
 import {sql} from 'kysely'
 import RootDataLoader from 'parabol-server/dataloader/RootDataLoader'
 import getKysely from 'parabol-server/postgres/getKysely'
-import {Logger} from '../server/utils/Logger'
 import {EmbedderJobType} from './EmbedderJobType'
 import {JobQueueError} from './JobQueueError'
 import {DBJob, JobType, Workflow} from './custom'
@@ -37,7 +36,6 @@ export class WorkflowOrchestrator {
  }
 
  private failJob = async (jobId: number, retryCount: number, error: JobQueueError) => {
- Logger.error(error)
  const pg = getKysely()
  const {message, retryDelay, jobData} = error
  const maxRetries = error.maxRetries ?? 10

packages/embedder/ai_models/TextEmbeddingsInference.ts

@@ -57,6 +57,7 @@ export class TextEmbeddingsInference extends AbstractEmbeddingsModel {
  this.client = client
  }
  async getTokens(content: string) {
+ if (!content) return []
  const {data, error} = await this.client.POST('/tokenize', {
  body: {add_special_tokens: true, inputs: content}
  })

packages/embedder/embedder.ts

@@ -61,7 +61,6 @@ const run = async () => {
  const counter = logPerformance(10, 6)
 
  for await (const _ of streams) {
- // Logger.log(`Worker ${idx} finished job ${message.id}`)
  counter.i++
  }
 

packages/embedder/indexing/retrospectiveDiscussionTopic.ts

@@ -46,7 +46,8 @@ async function formatThread(
  ? 'Anonymous'
  : await getPreferredNameByUserId(comment.createdBy, dataLoader)
  const how = depth === 0 ? 'wrote' : 'replied'
- const content = comment.plaintextContent.slice(0, MAX_TEXT_LENGTH)
+ const content = comment.plaintextContent?.slice(0, MAX_TEXT_LENGTH)
+ if (!content) return ''
  const formattedPost = `${indent}- ${author} ${how}, "${content}"\n`
 
  // Recursively format child threads
@@ -98,14 +99,14 @@ export const createTextFromRetrospectiveDiscussionTopic = async (
  if (prompt.description) markdown += `: ${prompt.description}`
  markdown += `".\n`
  }
- for (const reflection of reflections.filter((r) => r.promptId === prompt.id)) {
+ const matchingReflections = reflections.filter((r) => r.promptId === prompt.id)
+ for (const reflection of matchingReflections) {
+ const content = reflection.plaintextContent?.slice(0, MAX_TEXT_LENGTH)
+ if (!content) continue
  const author = newMeeting.disableAnonymity
  ? await getPreferredNameByUserId(reflection.creatorId, dataLoader)
  : 'Anonymous'
- markdown += ` - ${author} wrote, "${reflection.plaintextContent.slice(
- 0,
- MAX_TEXT_LENGTH
- )}"\n`
+ markdown += ` - ${author} wrote, "${content}"\n`
  }
  markdown += `\n`
  }

packages/embedder/logPerformance.ts

@@ -6,9 +6,14 @@ export const logPerformance = (logEvery: number, resetEvery: number) => {
  const counter = {i: 0}
  let start = performance.now()
  let logs = 0
+ let lastJobs = 0
  setInterval(() => {
  const duration = performance.now() - start
- Logger.log(`Jobs per second: ${Math.round((counter.i / duration) * 1000)}`)
+ const jobs = Math.round((counter.i / duration) * 1000)
+ if (jobs !== lastJobs) {
+ Logger.log(`Jobs per second: ${jobs}`)
+ }
+ lastJobs = jobs
  if (++logs >= resetEvery) {
  counter.i = 0
  logs = 0

packages/embedder/workflows/embedMetadata.ts

@@ -36,9 +36,7 @@ export const embedMetadata: JobQueueStepRun<
  .where('id', '=', embeddingsMetadataId)
  .execute()
  } catch (e) {
- // get the trace since the error message may be unobvious
- console.trace(e)
- return new JobQueueError(`unable to create embedding text: ${e}`, undefined, 0, {
+ return new JobQueueError(e as Error, undefined, 0, {
  forceBuildText: true
  })
  }

packages/embedder/workflows/rerankRetroTopics.ts

@@ -24,9 +24,7 @@ export const rerankRetroTopics: JobQueueStepRun<{
  const {body} = await createEmbeddingTextFrom(metadata, dataLoader, true)
  rerankText = body
  } catch (e) {
- // get the trace since the error message may be unobvious
- console.trace(e)
- return new JobQueueError(`unable to create embedding text: ${e}`)
+ return new JobQueueError(e as Error)
  }
 
  const modelManager = getModelManager()