From 12b1ebbb92dac6345d4c9f52258e4e2806d102fb Mon Sep 17 00:00:00 2001 From: dblythy Date: Tue, 31 Jan 2023 12:22:42 +1100 Subject: [PATCH 1/4] perf: increase parallel save speed --- spec/SchemaPerformance.spec.js | 33 ++++++++++++++++++ src/Config.js | 4 --- src/Controllers/DatabaseController.js | 8 ++--- src/Controllers/SchemaController.js | 48 ++++++++++++++++++--------- src/Controllers/index.js | 1 + 5 files changed, 70 insertions(+), 24 deletions(-) diff --git a/spec/SchemaPerformance.spec.js b/spec/SchemaPerformance.spec.js index 0471871c54..0a5765dede 100644 --- a/spec/SchemaPerformance.spec.js +++ b/spec/SchemaPerformance.spec.js @@ -204,4 +204,37 @@ describe('Schema Performance', function () { ); expect(getAllSpy.calls.count()).toBe(2); }); + + // beforeEach(async () => { + // if (SIMULATE_TTL) { + // jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; + // } + // }); + + // afterAll(() => { + // jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000; + // }); + + fit('can save objects', async () => { + // const user = await Parse.User.signUp('username', 'password'); + + // const start = Date.now(); + // for (let i = 0; i < 2000; i++) { + // const obj = new Parse.Object('TestObj'); + // await obj.save(); + // } + // const end = Date.now() - start; + // console.log(end); + + const start_parellel = Date.now(); + const objects = []; + for (let i = 0; i < 2000; i++) { + const obj = new Parse.Object('TestObj'); + obj.set('field1', 'uuid()'); + objects.push(obj); + } + await Promise.all(objects.map(o => o.save())); + const end_parellel = Date.now() - start_parellel; + console.log(end_parellel); + }); }); diff --git a/src/Config.js b/src/Config.js index bd7c6f21af..606dd39b72 100644 --- a/src/Config.js +++ b/src/Config.js @@ -37,11 +37,7 @@ export class Config { const config = new Config(); config.applicationId = applicationId; Object.keys(cacheInfo).forEach(key => { - if (key == 'databaseController') { - config.database = new DatabaseController(cacheInfo.databaseController.adapter, config); - } else { config[key] = cacheInfo[key]; - } }); config.mount = removeTrailingSlash(mount); config.generateSessionExpiresAt = config.generateSessionExpiresAt.bind(config); diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index e3ac5723ab..e5f5ba0b50 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -407,15 +407,13 @@ class DatabaseController { loadSchema( options: LoadSchemaOptions = { clearCache: false } ): Promise { - if (this.schemaPromise != null) { + if (this.schemaPromise) { return this.schemaPromise; } - this.schemaPromise = SchemaController.load(this.adapter, options); - this.schemaPromise.then( - () => delete this.schemaPromise, + this.schemaPromise = SchemaController.load(this.adapter, options).catch( () => delete this.schemaPromise ); - return this.loadSchema(options); + return this.schemaPromise; } loadSchemaIfNeeded( diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 62757d251d..b28a17c631 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -694,6 +694,7 @@ export default class SchemaController { constructor(databaseAdapter: StorageAdapter) { this._dbAdapter = databaseAdapter; + this._addingClasses = {}; this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields); this.protectedFields = Config.get(Parse.applicationId).protectedFields; @@ -741,13 +742,23 @@ export default class SchemaController { } setAllClasses(): Promise> { - return this._dbAdapter - .getAllClasses() - .then(allSchemas => allSchemas.map(injectDefaultSchema)) + if (this._setAllPromise) { + return this._setAllPromise; + } + let id = "set all classes " + Math.random().toString(16).slice(2); + console.time('all end ' + id); + this._setAllPromise = this._dbAdapter.getAllClasses().then(allSchemas => { + console.timeEnd('all end ' + id); + return allSchemas.map(injectDefaultSchema) + }) .then(allSchemas => { SchemaCache.put(allSchemas); return allSchemas; }); + setTimeout(() => { + delete this._setAllPromise; + }, 200); + return this._setAllPromise; } getOneSchema( @@ -803,19 +814,26 @@ export default class SchemaController { return Promise.reject(validationError); } try { - const adapterSchema = await this._dbAdapter.createClass( - className, - convertSchemaToAdapterSchema({ - fields, - classLevelPermissions, - indexes, + if (this._addingClasses[className]) { + return this._addingClasses[className]; + } + this._addingClasses[className] = this._dbAdapter + .createClass( className, - }) - ); - // TODO: Remove by updating schema cache directly - await this.reloadData({ clearCache: true }); - const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); - return parseSchema; + convertSchemaToAdapterSchema({ + fields, + classLevelPermissions, + indexes, + className, + }) + ) + .then(async adapterSchema => { + const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); + delete this._addingClasses[className]; + this.schemaData[className] = parseSchema; + return parseSchema; + }); + return this._addingClasses[className]; } catch (error) { if (error && error.code === Parse.Error.DUPLICATE_VALUE) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`); diff --git a/src/Controllers/index.js b/src/Controllers/index.js index 0a9b3db57d..2bca7ef2f8 100644 --- a/src/Controllers/index.js +++ b/src/Controllers/index.js @@ -61,6 +61,7 @@ export function getControllers(options: ParseServerOptions) { parseGraphQLController, liveQueryController, databaseController, + database: databaseController, hooksController, authDataManager, schemaCache: SchemaCache, From b13efd49ac958daa1f1ced6ec5c0233f5fafa18e Mon Sep 17 00:00:00 2001 From: dblythy Date: Tue, 31 Jan 2023 13:32:13 +1100 Subject: [PATCH 2/4] Update SchemaController.js --- src/Controllers/SchemaController.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index b28a17c631..349d353816 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -814,6 +814,9 @@ export default class SchemaController { return Promise.reject(validationError); } try { + if (this.schemaData[className]) { + return this.schemaData[className]; + } if (this._addingClasses[className]) { return this._addingClasses[className]; } From 113e2a808d0fa6718c3107ca760fa6ac4aaba678 Mon Sep 17 00:00:00 2001 From: dblythy Date: Tue, 31 Jan 2023 14:32:56 +1100 Subject: [PATCH 3/4] revert --- src/Controllers/SchemaController.js | 51 +++++++++-------------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 349d353816..62757d251d 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -694,7 +694,6 @@ export default class SchemaController { constructor(databaseAdapter: StorageAdapter) { this._dbAdapter = databaseAdapter; - this._addingClasses = {}; this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields); this.protectedFields = Config.get(Parse.applicationId).protectedFields; @@ -742,23 +741,13 @@ export default class SchemaController { } setAllClasses(): Promise> { - if (this._setAllPromise) { - return this._setAllPromise; - } - let id = "set all classes " + Math.random().toString(16).slice(2); - console.time('all end ' + id); - this._setAllPromise = this._dbAdapter.getAllClasses().then(allSchemas => { - console.timeEnd('all end ' + id); - return allSchemas.map(injectDefaultSchema) - }) + return this._dbAdapter + .getAllClasses() + .then(allSchemas => allSchemas.map(injectDefaultSchema)) .then(allSchemas => { SchemaCache.put(allSchemas); return allSchemas; }); - setTimeout(() => { - delete this._setAllPromise; - }, 200); - return this._setAllPromise; } getOneSchema( @@ -814,29 +803,19 @@ export default class SchemaController { return Promise.reject(validationError); } try { - if (this.schemaData[className]) { - return this.schemaData[className]; - } - if (this._addingClasses[className]) { - return this._addingClasses[className]; - } - this._addingClasses[className] = this._dbAdapter - .createClass( + const adapterSchema = await this._dbAdapter.createClass( + className, + convertSchemaToAdapterSchema({ + fields, + classLevelPermissions, + indexes, className, - convertSchemaToAdapterSchema({ - fields, - classLevelPermissions, - indexes, - className, - }) - ) - .then(async adapterSchema => { - const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); - delete this._addingClasses[className]; - this.schemaData[className] = parseSchema; - return parseSchema; - }); - return this._addingClasses[className]; + }) + ); + // TODO: Remove by updating schema cache directly + await this.reloadData({ clearCache: true }); + const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); + return parseSchema; } catch (error) { if (error && error.code === Parse.Error.DUPLICATE_VALUE) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`); From a775fa618be9a565059dd38706b14124243fb0b8 Mon Sep 17 00:00:00 2001 From: dblythy Date: Tue, 31 Jan 2023 16:33:26 +1100 Subject: [PATCH 4/4] perf: increase promise speed --- spec/SchemaPerformance.spec.js | 32 +++----- src/Auth.js | 29 +++++--- src/Config.js | 3 +- src/Controllers/SchemaController.js | 109 +++++++++++++++++++++------- 4 files changed, 111 insertions(+), 62 deletions(-) diff --git a/spec/SchemaPerformance.spec.js b/spec/SchemaPerformance.spec.js index 0a5765dede..f53aa0af51 100644 --- a/spec/SchemaPerformance.spec.js +++ b/spec/SchemaPerformance.spec.js @@ -205,36 +205,22 @@ describe('Schema Performance', function () { expect(getAllSpy.calls.count()).toBe(2); }); - // beforeEach(async () => { - // if (SIMULATE_TTL) { - // jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; - // } - // }); - - // afterAll(() => { - // jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000; - // }); - - fit('can save objects', async () => { - // const user = await Parse.User.signUp('username', 'password'); - - // const start = Date.now(); - // for (let i = 0; i < 2000; i++) { - // const obj = new Parse.Object('TestObj'); - // await obj.save(); - // } - // const end = Date.now() - start; - // console.log(end); - + fit('can save objects', async done => { const start_parellel = Date.now(); + await Parse.User.signUp('username', 'password'); + const objects = []; - for (let i = 0; i < 2000; i++) { + for (let i = 0; i < 1000; i++) { const obj = new Parse.Object('TestObj'); obj.set('field1', 'uuid()'); objects.push(obj); - } + // if ( i == 0) { + // await obj.save(); + // } + } await Promise.all(objects.map(o => o.save())); const end_parellel = Date.now() - start_parellel; console.log(end_parellel); + expect(end_parellel).toBeLessThan(6000); }); }); diff --git a/src/Auth.js b/src/Auth.js index abd14391db..f70776f131 100644 --- a/src/Auth.js +++ b/src/Auth.js @@ -3,6 +3,7 @@ import { isDeepStrictEqual } from 'util'; import { getRequestObject, resolveError } from './triggers'; import Deprecator from './Deprecator/Deprecator'; import { logger } from './logger'; +const authPromisesByUser = {}; // An Auth object tells you who is requesting something and whether // the master key was used. @@ -174,7 +175,9 @@ Auth.prototype.getUserRoles = function () { Auth.prototype.getRolesForUser = async function () { //Stack all Parse.Role - const results = []; + if (authPromisesByUser[this.user.id]) { + return authPromisesByUser[this.user.id] + } if (this.config) { const restWhere = { users: { @@ -184,15 +187,23 @@ Auth.prototype.getRolesForUser = async function () { }, }; const RestQuery = require('./RestQuery'); - await new RestQuery(this.config, master(this.config), '_Role', restWhere, {}).each(result => - results.push(result) - ); - } else { - await new Parse.Query(Parse.Role) - .equalTo('users', this.user) - .each(result => results.push(result.toJSON()), { useMasterKey: true }); + const findObjects = async () => { + const results = []; + await new RestQuery(this.config, master(this.config), '_Role', restWhere, {}).each(result => + results.push(result) + ); + return results; + } + authPromisesByUser[this.user.id] = findObjects(); + return authPromisesByUser[this.user.id]; } - return results; + authPromisesByUser[this.user.id] = new Parse.Query(Parse.Role) + .equalTo('users', this.user) + .findAll({ useMasterKey: true }).then(response => { + delete authPromisesByUser[this.user.id]; + return response; + }); + return authPromisesByUser[this.user.id]; }; // Iterates through the role tree and compiles a user's roles diff --git a/src/Config.js b/src/Config.js index 606dd39b72..e34fba3461 100644 --- a/src/Config.js +++ b/src/Config.js @@ -5,7 +5,6 @@ import { isBoolean, isString } from 'lodash'; import net from 'net'; import AppCache from './cache'; -import DatabaseController from './Controllers/DatabaseController'; import { logLevels as validLogLevels } from './Controllers/LoggerController'; import { AccountLockoutOptions, @@ -37,7 +36,7 @@ export class Config { const config = new Config(); config.applicationId = applicationId; Object.keys(cacheInfo).forEach(key => { - config[key] = cacheInfo[key]; + config[key] = cacheInfo[key]; }); config.mount = removeTrailingSlash(mount); config.generateSessionExpiresAt = config.generateSessionExpiresAt.bind(config); diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 62757d251d..c5b3d28e12 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -687,8 +687,10 @@ const typeToString = (type: SchemaField | string): string => { // the mongo format and the Parse format. Soon, this will all be Parse format. export default class SchemaController { _dbAdapter: StorageAdapter; + _reloadingData: { [string]: any }; + _addFieldPromises: { [string]: any }; + _addClassPromises: { [string]: any }; schemaData: { [string]: Schema }; - reloadDataPromise: ?Promise; protectedFields: any; userIdRegEx: RegExp; @@ -696,6 +698,10 @@ export default class SchemaController { this._dbAdapter = databaseAdapter; this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields); this.protectedFields = Config.get(Parse.applicationId).protectedFields; + this._reloadingData = { + promise: null, + className: '', + }; const customIds = Config.get(Parse.applicationId).allowCustomObjectId; @@ -709,24 +715,39 @@ export default class SchemaController { }); } - reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise { - if (this.reloadDataPromise && !options.clearCache) { - return this.reloadDataPromise; + reloadData(options: LoadSchemaOptions = { clearCache: false, className: '' }): Promise { + if (options.className) { + if (this._reloadingData.className === options.className) { + options.clearCache = false; + } else { + this._reloadingData.className = options.className; + } + } else { + this._reloadingData.className = ''; } - this.reloadDataPromise = this.getAllClasses(options) + if (this._reloadingData.promise && !options.clearCache) { + return this._reloadingData.promise; + } + this._reloadingData.promise = this.getAllClasses(options) .then( allSchemas => { this.schemaData = new SchemaData(allSchemas, this.protectedFields); - delete this.reloadDataPromise; + this._reloadingData = { + promise: null, + className: '', + }; }, err => { this.schemaData = new SchemaData(); - delete this.reloadDataPromise; + this._reloadingData = { + promise: null, + className: '', + }; throw err; } ) .then(() => {}); - return this.reloadDataPromise; + return this._reloadingData.promise; } getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise> { @@ -741,9 +762,14 @@ export default class SchemaController { } setAllClasses(): Promise> { + const r = 'classes ' + (Math.random() + 1).toString(36).substring(7); + console.time(r); return this._dbAdapter .getAllClasses() - .then(allSchemas => allSchemas.map(injectDefaultSchema)) + .then(allSchemas => { + // console.timeEnd(r); + return allSchemas.map(injectDefaultSchema); + }) .then(allSchemas => { SchemaCache.put(allSchemas); return allSchemas; @@ -787,7 +813,7 @@ export default class SchemaController { // on success, and rejects with an error on fail. Ensure you // have authorization (master key, or client class creation // enabled) before calling this function. - async addClassIfNotExists( + addClassIfNotExists( className: string, fields: SchemaFields = {}, classLevelPermissions: any, @@ -802,8 +828,14 @@ export default class SchemaController { } return Promise.reject(validationError); } - try { - const adapterSchema = await this._dbAdapter.createClass( + if (!this._addClassPromises) { + this._addClassPromises = {}; + } + if (this._addClassPromises[className]) { + return this._addClassPromises[className]; + } + this._addClassPromises[className] = this._dbAdapter + .createClass( className, convertSchemaToAdapterSchema({ fields, @@ -811,18 +843,24 @@ export default class SchemaController { indexes, className, }) - ); - // TODO: Remove by updating schema cache directly - await this.reloadData({ clearCache: true }); - const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); - return parseSchema; - } catch (error) { - if (error && error.code === Parse.Error.DUPLICATE_VALUE) { - throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`); - } else { - throw error; - } - } + ) + .then(async adapterSchema => { + await this.reloadData({ clearCache: true }); + const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema); + delete this._addClassPromises[className]; + return parseSchema; + }) + .catch(error => { + if (error && error.code === Parse.Error.DUPLICATE_VALUE) { + throw new Parse.Error( + Parse.Error.INVALID_CLASS_NAME, + `Class ${className} already exists.` + ); + } else { + throw error; + } + }); + return this._addClassPromises[className]; } updateClass( @@ -947,7 +985,7 @@ export default class SchemaController { // have failed because there's a race condition and a different // client is making the exact same schema update that we want. // So just reload the schema. - return this.reloadData({ clearCache: true }); + return this.reloadData({ clearCache: true, className }); }) .then(() => { // Ensure that the schema now validates @@ -957,7 +995,7 @@ export default class SchemaController { throw new Parse.Error(Parse.Error.INVALID_JSON, `Failed to add ${className}`); } }) - .catch(() => { + .catch(e => { // The schema still doesn't validate. Give up throw new Parse.Error(Parse.Error.INVALID_JSON, 'schema class name does not revalidate'); }) @@ -1131,9 +1169,22 @@ export default class SchemaController { return this._dbAdapter.updateFieldOptions(className, fieldName, type); } - return this._dbAdapter + if (!this._addFieldPromises) { + this._addFieldPromises = {}; + } + + if (!this._addFieldPromises[className]) { + this._addFieldPromises[className] = {}; + } + + if (this._addFieldPromises[className][`${fieldName}-${type}`]) { + return this._addFieldPromises[className][`${fieldName}-${type}`]; + } + + this._addFieldPromises[className][`${fieldName}-${type}`] = this._dbAdapter .addFieldIfNotExists(className, fieldName, type) .catch(error => { + delete this._addFieldPromises[className][`${fieldName}-${type}`]; if (error.code == Parse.Error.INCORRECT_TYPE) { // Make sure that we throw errors when it is appropriate to do so. throw error; @@ -1144,12 +1195,14 @@ export default class SchemaController { return Promise.resolve(); }) .then(() => { + delete this._addFieldPromises[className][`${fieldName}-${type}`]; return { className, fieldName, type, }; }); + return this._addFieldPromises[className][`${fieldName}-${type}`]; } ensureFields(fields: any) { @@ -1270,7 +1323,7 @@ export default class SchemaController { if (enforceFields.length !== 0) { // TODO: Remove by updating schema cache directly - await this.reloadData({ clearCache: true }); + await this.reloadData({ clearCache: true, className }); } this.ensureFields(enforceFields);