Merge branch 'chore/npmless-builds' into chore/pipeline_npm_version_u…

…pdates

README.md

@@ -216,35 +216,9 @@ If you are having issues between updates of the builder, please use the guide [h
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
-<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
-<!-- prettier-ignore-start -->
-<!-- markdownlint-disable -->
-<table>
-  <tr>
-    <td align="center"><a href="http://martinmck.com"><img src="https://avatars1.githubusercontent.com/u/11256663?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Martin McKeaveney</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=shogunpurple" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=shogunpurple" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=shogunpurple" title="Tests">⚠️</a> <a href="#infra-shogunpurple" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
-    <td align="center"><a href="http://www.michaeldrury.co.uk/"><img src="https://avatars2.githubusercontent.com/u/4407001?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Michael Drury</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=mike12345567" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=mike12345567" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=mike12345567" title="Tests">⚠️</a> <a href="#infra-mike12345567" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
-    <td align="center"><a href="https://github.com/aptkingston"><img src="https://avatars3.githubusercontent.com/u/9075550?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Andrew Kingston</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=aptkingston" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=aptkingston" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=aptkingston" title="Tests">⚠️</a> <a href="#design-aptkingston" title="Design">🎨</a></td>
-    <td align="center"><a href="https://budibase.com/"><img src="https://avatars3.githubusercontent.com/u/3524181?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Michael Shanks</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/kevmodrome"><img src="https://avatars3.githubusercontent.com/u/534488?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Kevin Åberg Kultalahti</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://www.budibase.com/"><img src="https://avatars2.githubusercontent.com/u/49767913?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Joe</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Code">💻</a> <a href="#content-joebudi" title="Content">🖋</a> <a href="#design-joebudi" title="Design">🎨</a></td>
-    <td align="center"><a href="https://github.com/Rory-Powell"><img src="https://avatars.githubusercontent.com/u/8755148?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Rory Powell</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Tests">⚠️</a></td>
-  </tr>
-  <tr>
-    <td align="center"><a href="https://github.com/PClmnt"><img src="https://avatars.githubusercontent.com/u/5665926?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Peter Clement</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/Conor-Mack"><img src="https://avatars1.githubusercontent.com/u/36074859?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Conor_Mack</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/pngwn"><img src="https://avatars1.githubusercontent.com/u/12937446?v=4?s=100" width="100px;" alt=""/><br /><sub><b>pngwn</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/HugoLd"><img src="https://avatars0.githubusercontent.com/u/26521848?v=4?s=100" width="100px;" alt=""/><br /><sub><b>HugoLd</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=HugoLd" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/victoriasloan"><img src="https://avatars.githubusercontent.com/u/9913651?v=4?s=100" width="100px;" alt=""/><br /><sub><b>victoriasloan</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=victoriasloan" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/yashank09"><img src="https://avatars.githubusercontent.com/u/37672190?v=4?s=100" width="100px;" alt=""/><br /><sub><b>yashank09</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=yashank09" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/SOVLOOKUP"><img src="https://avatars.githubusercontent.com/u/53158137?v=4?s=100" width="100px;" alt=""/><br /><sub><b>SOVLOOKUP</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=SOVLOOKUP" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/seoulaja"><img src="https://avatars.githubusercontent.com/u/15101654?v=4?s=100" width="100px;" alt=""/><br /><sub><b>seoulaja</b></sub></a><br /><a href="#translation-seoulaja" title="Translation">🌍</a></td>
-    <td align="center"><a href="https://github.com/mslourens"><img src="https://avatars.githubusercontent.com/u/1907152?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Maurits Lourens</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=mslourens" title="Tests">⚠️</a> <a href="https://github.com/Budibase/budibase/commits?author=mslourens" title="Code">💻</a></td>
-  </tr>
-</table>
-
-<!-- markdownlint-restore -->
-<!-- prettier-ignore-end -->
-
-<!-- ALL-CONTRIBUTORS-LIST:END -->
-
-This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+
+<a href="https://github.com/Budibase/budibase/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=Budibase/budibase" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.999-alpha.40",
+  "version": "2.5.6-alpha.19",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",
@@ -31,4 +31,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

packages/backend-core/src/constants/db.ts

@@ -14,6 +14,7 @@ export enum ViewName {
   USER_BY_APP = "by_app",
   USER_BY_EMAIL = "by_email2",
   BY_API_KEY = "by_api_key",
+  /** @deprecated - could be deleted */
   USER_BY_BUILDERS = "by_builders",
   LINK = "by_link",
   ROUTING = "screen_routes",

packages/backend-core/src/context/mainContext.ts

@@ -115,10 +115,10 @@ export async function doInContext(appId: string, task: any): Promise<any> {
   )
 }
 
-export async function doInTenant(
+export async function doInTenant<T>(
   tenantId: string | null,
-  task: any
-): Promise<any> {
+  task: () => T
+): Promise<T> {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {
     tenantId = tenantId || DEFAULT_TENANT_ID

packages/backend-core/src/db/lucene.ts

@@ -243,7 +243,7 @@ export class QueryBuilder<T> {
     }
     // Escape characters
     if (!this.#noEscaping && escape && originalType === "string") {
-      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
+      value = `${value}`.replace(/[ \/#+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
     }
 
     // Wrap in quotes
@@ -320,6 +320,18 @@ export class QueryBuilder<T> {
       return `${key}:(${statement})`
     }
 
+    const fuzzy = (key: string, value: any) => {
+      if (!value) {
+        return null
+      }
+      value = builder.preprocess(value, {
+        escape: true,
+        lowercase: true,
+        type: "fuzzy",
+      })
+      return `${key}:/.*${value}.*/`
+    }
+
     const notContains = (key: string, value: any) => {
       const allPrefix = allOr ? "*:* AND " : ""
       const mode = allOr ? "AND" : undefined
@@ -408,17 +420,7 @@ export class QueryBuilder<T> {
       })
     }
     if (this.#query.fuzzy) {
-      build(this.#query.fuzzy, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        value = builder.preprocess(value, {
-          escape: true,
-          lowercase: true,
-          type: "fuzzy",
-        })
-        return `${key}:${value}~`
-      })
+      build(this.#query.fuzzy, fuzzy)
     }
     if (this.#query.equal) {
       build(this.#query.equal, equal)

packages/backend-core/src/db/views.ts

@@ -7,7 +7,7 @@ import {
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
-import { Database, DatabaseQueryOpts } from "@budibase/types"
+import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types"
 import env from "../environment"
 
 const DESIGN_DB = "_design/database"
@@ -119,44 +119,56 @@ export interface QueryViewOptions {
   arrayResponse?: boolean
 }
 
-export const queryView = async <T>(
+export async function queryViewRaw<T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db: Database,
   createFunc: any,
   opts?: QueryViewOptions
-): Promise<T[] | T | undefined> => {
+): Promise<AllDocsResponse<T>> {
   try {
-    let response = await db.query<T>(`database/${viewName}`, params)
-    const rows = response.rows
-    const docs = rows.map((row: any) =>
-      params.include_docs ? row.doc : row.value
-    )
-
-    // if arrayResponse has been requested, always return array regardless of length
-    if (opts?.arrayResponse) {
-      return docs as T[]
-    } else {
-      // return the single document if there is only one
-      return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
-    }
+    const response = await db.query<T>(`database/${viewName}`, params)
+    // await to catch error
+    return response
   } catch (err: any) {
     const pouchNotFound = err && err.name === "not_found"
     const couchNotFound = err && err.status === 404
     if (pouchNotFound || couchNotFound) {
       await removeDeprecated(db, viewName)
       await createFunc()
-      return queryView(viewName, params, db, createFunc, opts)
+      return queryViewRaw(viewName, params, db, createFunc, opts)
     } else if (err.status === 409) {
       // can happen when multiple queries occur at once, view couldn't be created
       // other design docs being updated, re-run
-      return queryView(viewName, params, db, createFunc, opts)
+      return queryViewRaw(viewName, params, db, createFunc, opts)
     } else {
       throw err
     }
   }
 }
 
+export const queryView = async <T>(
+  viewName: ViewName,
+  params: DatabaseQueryOpts,
+  db: Database,
+  createFunc: any,
+  opts?: QueryViewOptions
+): Promise<T[] | T | undefined> => {
+  const response = await queryViewRaw<T>(viewName, params, db, createFunc, opts)
+  const rows = response.rows
+  const docs = rows.map((row: any) =>
+    params.include_docs ? row.doc : row.value
+  )
+
+  // if arrayResponse has been requested, always return array regardless of length
+  if (opts?.arrayResponse) {
+    return docs as T[]
+  } else {
+    // return the single document if there is only one
+    return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
+  }
+}
+
 // PLATFORM
 
 async function createPlatformView(viewJs: string, viewName: ViewName) {
@@ -208,22 +220,33 @@ export const queryPlatformView = async <T>(
   })
 }
 
+const CreateFuncByName: any = {
+  [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
+  [ViewName.BY_API_KEY]: createApiKeyView,
+  [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
+  [ViewName.USER_BY_APP]: createUserAppView,
+}
+
 export const queryGlobalView = async <T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db?: Database,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
-  const CreateFuncByName: any = {
-    [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
-    [ViewName.BY_API_KEY]: createApiKeyView,
-    [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
-    [ViewName.USER_BY_APP]: createUserAppView,
-  }
   // can pass DB in if working with something specific
   if (!db) {
     db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
   return queryView(viewName, params, db!, createFn, opts)
 }
+
+export async function queryGlobalViewRaw<T>(
+  viewName: ViewName,
+  params: DatabaseQueryOpts,
+  opts?: QueryViewOptions
+) {
+  const db = getGlobalDB()
+  const createFn = CreateFuncByName[viewName]
+  return queryViewRaw<T>(viewName, params, db, createFn, opts)
+}

packages/backend-core/src/events/identification.ts

@@ -306,4 +306,5 @@ export default {
   identify,
   identifyGroup,
   getInstallationId,
+  getUniqueTenantId,
 }

packages/backend-core/src/installation.ts

@@ -33,7 +33,7 @@ async function createInstallDoc(platformDb: Database) {
   }
 }
 
-const getInstallFromDB = async (): Promise<Installation> => {
+export const getInstallFromDB = async (): Promise<Installation> => {
   return doWithDB(
     StaticDatabases.PLATFORM_INFO.name,
     async (platformDb: any) => {

packages/backend-core/src/middleware/authenticated.ts

@@ -44,7 +44,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
   // check both the primary and the fallback internal api keys
   // this allows for rotation
   if (isValidInternalAPIKey(apiKey)) {
-    return { valid: true }
+    return { valid: true, user: undefined }
   }
   const decrypted = decrypt(apiKey)
   const tenantId = decrypted.split(SEPARATOR)[0]

packages/backend-core/src/redis/redlockImpl.ts

@@ -1,10 +1,16 @@
-import Redlock, { Options } from "redlock"
+import Redlock from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
 import env from "../environment"
 
-const getClient = async (type: LockType): Promise<Redlock> => {
+const getClient = async (
+  type: LockType,
+  opts?: Redlock.Options
+): Promise<Redlock> => {
+  if (type === LockType.CUSTOM) {
+    return newRedlock(opts)
+  }
   if (env.isTest() && type !== LockType.TRY_ONCE) {
     return newRedlock(OPTIONS.TEST)
   }
@@ -56,7 +62,7 @@ const OPTIONS = {
   },
 }
 
-const newRedlock = async (opts: Options = {}) => {
+const newRedlock = async (opts: Redlock.Options = {}) => {
   let options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
   const client = redisWrapper.getClient()

packages/backend-core/src/users.ts

@@ -1,15 +1,16 @@
 import {
-  ViewName,
-  getUsersByAppParams,
-  getProdAppID,
-  generateAppUserID,
-  queryGlobalView,
-  UNICODE_MAX,
-  DocumentType,
-  SEPARATOR,
   directCouchFind,
+  DocumentType,
+  generateAppUserID,
   getGlobalUserParams,
+  getProdAppID,
+  getUsersByAppParams,
   pagination,
+  queryGlobalView,
+  queryGlobalViewRaw,
+  SEPARATOR,
+  UNICODE_MAX,
+  ViewName,
 } from "./db"
 import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
@@ -239,3 +240,11 @@ export const paginatedUsers = async ({
     getKey,
   })
 }
+
+export async function getUserCount() {
+  const response = await queryGlobalViewRaw(ViewName.USER_BY_EMAIL, {
+    limit: 0, // to be as fast as possible - we just want the total rows count
+    include_docs: false,
+  })
+  return response.total_rows
+}

packages/backend-core/src/utils/utils.ts

@@ -46,8 +46,9 @@ export async function resolveAppUrl(ctx: Ctx) {
   }
 
   // search prod apps for a url that matches
-  const apps: App[] = await context.doInTenant(tenantId, () =>
-    getAllApps({ dev: false })
+  const apps: App[] = await context.doInTenant(
+    tenantId,
+    () => getAllApps({ dev: false }) as Promise<App[]>
   )
   const app = apps.filter(
     a => a.url && a.url.toLowerCase() === possibleAppUrl
@@ -221,27 +222,6 @@ export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"
 }
 
-async function getBuilders() {
-  const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, {
-    include_docs: false,
-  })
-
-  if (!builders) {
-    return []
-  }
-
-  if (Array.isArray(builders)) {
-    return builders
-  } else {
-    return [builders]
-  }
-}
-
-export async function getBuildersCount() {
-  const builders = await getBuilders()
-  return builders.length
-}
-
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tests/core/utilities/index.ts

@@ -2,5 +2,5 @@ export * as mocks from "./mocks"
 export * as structures from "./structures"
 export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
-
+export * as utils from "./utils"
 export * from "./jestUtils"