diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index ee0f7d4954..7bb8ca1deb 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1693,7 +1693,7 @@ class DatabaseController { // TODO: create indexes on first creation of a _User object. Otherwise it's impossible to // have a Parse app without it having a _User collection. - performInitialization() { + async performInitialization(options) { const requiredUserFields = { fields: { ...SchemaController.defaultColumns._Default, @@ -1750,13 +1750,17 @@ class DatabaseController { const adapterInit = this.adapter.performInitialization({ VolatileClassesSchemas: SchemaController.VolatileClassesSchemas, }); - return Promise.all([ + const promises = await Promise.all([ usernameUniqueness, emailUniqueness, roleUniqueness, adapterInit, indexPromise, ]); + if (options.schemas) { + await (await this.loadSchema()).loadSchemas(options.schemas); + } + return promises; } static _validateQuery: (any, boolean) => void; diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 8c799d0533..51c14b2ef2 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -28,6 +28,20 @@ import type { SchemaField, LoadSchemaOptions, } from './types'; +import logger from '../logger'; + +const defaultIndexes = Object.freeze({ + _Default: { + objectId: true, + }, + _User: { + username: true, + email: true, + }, + _Role: { + name: true, + }, +}); const defaultColumns: { [string]: SchemaFields } = Object.freeze({ // Contain the default columns for every parse object type (except _Join collection) @@ -592,34 +606,26 @@ const _GraphQLConfigSchema = { className: '_GraphQLConfig', fields: defaultColumns._GraphQLConfig, }; -const _PushStatusSchema = convertSchemaToAdapterSchema( - injectDefaultSchema({ - className: '_PushStatus', - fields: {}, - classLevelPermissions: {}, - }) -); -const _JobStatusSchema = convertSchemaToAdapterSchema( - injectDefaultSchema({ - className: '_JobStatus', - fields: {}, - classLevelPermissions: {}, - }) -); -const _JobScheduleSchema = convertSchemaToAdapterSchema( - injectDefaultSchema({ - className: '_JobSchedule', - fields: {}, - classLevelPermissions: {}, - }) -); -const _AudienceSchema = convertSchemaToAdapterSchema( - injectDefaultSchema({ - className: '_Audience', - fields: defaultColumns._Audience, - classLevelPermissions: {}, - }) -); +const _PushStatusSchema = convertSchemaToAdapterSchema({ + className: '_PushStatus', + fields: {}, + classLevelPermissions: {}, +}); +const _JobStatusSchema = convertSchemaToAdapterSchema({ + className: '_JobStatus', + fields: {}, + classLevelPermissions: {}, +}); +const _JobScheduleSchema = convertSchemaToAdapterSchema({ + className: '_JobSchedule', + fields: {}, + classLevelPermissions: {}, +}); +const _AudienceSchema = convertSchemaToAdapterSchema({ + className: '_Audience', + fields: defaultColumns._Audience, + classLevelPermissions: {}, +}); const VolatileClassesSchemas = [ _HooksSchema, _JobStatusSchema, @@ -651,6 +657,13 @@ const typeToString = (type: SchemaField | string): string => { return `${type.type}`; }; +const paramsAreEquals = (indexA, indexB) => { + const keysIndexA = Object.keys(indexA); + const keysIndexB = Object.keys(indexB); + if (keysIndexA.length !== keysIndexB.length) return false; + return keysIndexA.every(k => indexA[k] === indexB[k]); +}; + // Stores the entire schema of the app in a weird hybrid format somewhere between // the mongo format and the Parse format. Soon, this will all be Parse format. export default class SchemaController { @@ -813,6 +826,146 @@ export default class SchemaController { }); } + async loadSchemas(localSchemas, database: DatabaseController) { + const cloudSchemas = await this.getAllClasses({ clearCache: true }); + // We do not check classes to delete, developer need to delete manually classes + // to avoid data loss during auto migration + await Promise.all( + localSchemas.map(async localSchema => { + const cloudSchema = cloudSchemas.find( + ({ className }) => className === localSchema.className + ); + if (cloudSchema) { + await this.migrateClass(localSchema, cloudSchema, database); + } else { + if (!localSchema.className) + throw new Parse.Error( + 500, + `className is undefined on a schema. Check your schemas objects.` + ); + await this.addClassIfNotExists( + localSchema.className, + localSchema.fields, + localSchema.classLevelPermissions, + localSchema.indexes + ); + } + }) + ); + } + + async migrateClass(localSchema, cloudSchema, database: DatabaseController) { + localSchema.fields = localSchema.fields || {}; + localSchema.indexes = localSchema.indexes || {}; + localSchema.classLevelPermissions = localSchema.classLevelPermissions || {}; + + const fieldsToAdd = {}; + const fieldsToDelete = {}; + const indexesToAdd = {}; + const indexesToDelete = {}; + + const isNotDefaultField = fieldName => + !defaultColumns['_Default'][fieldName] || + (defaultColumns[localSchema.className] && + !defaultColumns[localSchema.className][fieldName]); + + Object.keys(localSchema.fields) + .filter(fieldName => isNotDefaultField(fieldName)) + .forEach(fieldName => { + if (!cloudSchema.fields[fieldName]) + fieldsToAdd[fieldName] = localSchema.fields[fieldName]; + }); + + Object.keys(cloudSchema.fields) + .filter(fieldName => isNotDefaultField(fieldName)) + .map(async fieldName => { + if (!localSchema.fields[fieldName]) { + fieldsToDelete[fieldName] = { __op: 'Delete' }; + return; + } + if ( + !paramsAreEquals( + cloudSchema.fields[fieldName], + localSchema.fields[fieldName] + ) + ) { + fieldsToDelete[fieldName] = { __op: 'Delete' }; + fieldsToAdd[fieldName] = localSchema.fields[fieldName]; + } + }); + + const isNotDefaultIndex = indexName => + !defaultIndexes['_Default'][indexName] || + (defaultIndexes[localSchema.className] && + !defaultIndexes[localSchema.className][indexName]); + + Object.keys(localSchema.indexes) + .filter(indexName => isNotDefaultIndex(indexName)) + .forEach(indexName => { + if (!cloudSchema.indexes[indexName]) + indexesToAdd[indexName] = localSchema.indexes[indexName]; + }); + + Object.keys(cloudSchema.indexes) + .filter(indexName => isNotDefaultIndex(indexName)) + .map(async indexName => { + if (!localSchema.indexes[indexName]) { + indexesToDelete[indexName] = { __op: 'Delete' }; + return; + } + if ( + !paramsAreEquals( + cloudSchema.indexes[indexName], + localSchema.indexes[indexName] + ) + ) { + indexesToDelete[indexName] = { __op: 'Delete' }; + indexesToAdd[indexName] = localSchema.indexes[indexName]; + } + }); + const CLPs = { + ...localSchema.classLevelPermissions, + // Block add field feature when developers use parse with schemas + // to avoid auto migration issues across SDKs + addField: {}, + }; + // Concurrent migrations can lead to errors + try { + await this.updateClass( + localSchema.className, + fieldsToDelete, + CLPs, + indexesToDelete, + database + ); + } catch (e) { + setTimeout(async () => { + try { + await this.migrateClass(localSchema, cloudSchema, database); + } catch (e) { + logger.error('Error occured during migration deletion time.'); + } + }, 15000); + } + try { + await this.updateClass( + localSchema.className, + fieldsToAdd, + CLPs, + indexesToAdd, + database + ); + } catch (e) { + setTimeout(async () => { + try { + await this.migrateClass(localSchema, cloudSchema, database); + } catch (e) { + logger.error('Error occured during migration addition.'); + } + }, 15000); + } + } + updateClass( className: string, submittedFields: SchemaFields, diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 4bded75ab9..f943c0fb56 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -528,4 +528,10 @@ module.exports.LiveQueryServerOptions = { help: 'Adapter module for the WebSocketServer', action: parsers.moduleOrObjectParser, }, + schemas: { + env: 'PARSE_SERVER_SCHEMAS', + help: + 'Schemas options allow to lock and pre define schemas of the parse server.', + action: parsers.objectParser, + }, }; diff --git a/src/Options/docs.js b/src/Options/docs.js index 098fefb563..701a0c1399 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -73,6 +73,7 @@ * @property {Boolean} verbose Set the logging to verbose * @property {Boolean} verifyUserEmails Enable (or disable) user email validation, defaults to false * @property {String} webhookKey Key sent with outgoing webhook calls + * @property {Parse.Schema[]} schemas lock and pre define schemas of the parse server. */ /** diff --git a/src/ParseServer.js b/src/ParseServer.js index 5611c50491..2ea19b6920 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -75,7 +75,7 @@ class ParseServer { this.config = Config.put(Object.assign({}, options, allControllers)); logging.setLogger(loggerController); - const dbInitPromise = databaseController.performInitialization(); + const dbInitPromise = databaseController.performInitialization(options); const hooksLoadPromise = hooksController.load(); // Note: Tests will start to fail if any validation happens after this is called. diff --git a/src/ParseServerSchemas.js b/src/ParseServerSchemas.js new file mode 100644 index 0000000000..1e2794c145 --- /dev/null +++ b/src/ParseServerSchemas.js @@ -0,0 +1,197 @@ +import Parse from '../utils/Parse'; + +// add class await schema.addClassIfNotExists(name,schema) +// update class await schema.updateClass(name,transformToParse(schemaFields, existingParseClass.fields),undefined,undefined,config.database); + +export const buildSchemas = async (localSchemas, config) => { + const schema = await config.database.loadSchema({ clearCache: true }); + const allCloudSchema = await schema + .getAllClasses(true) + .filter(s => !lib.isDefaultSchema(s.className)); + await Promise.all( + localSchemas.map(async localSchema => + lib.saveOrUpdate(allCloudSchema, localSchema) + ) + ); +}; + +export const lib = { + saveOrUpdate: async (allCloudSchema, localSchema) => { + const cloudSchema = allCloudSchema.find( + sc => sc.className === localSchema.className + ); + if (cloudSchema) { + await lib.updateSchema(localSchema, cloudSchema); + } else { + await lib.saveSchema(localSchema); + } + }, + saveSchema: async localSchema => { + const newLocalSchema = new Parse.Schema(localSchema.className); + // Handle fields + Object.keys(localSchema.fields) + .filter( + fieldName => !lib.isDefaultFields(localSchema.className, fieldName) + ) + .forEach(fieldName => { + const { type, ...others } = localSchema.fields[fieldName]; + lib.handleFields(newLocalSchema, fieldName, type, others); + }); + // Handle indexes + if (localSchema.indexes) { + Object.keys(localSchema.indexes).forEach(indexName => + newLocalSchema.addIndex(indexName, localSchema.indexes[indexName]) + ); + } + + newLocalSchema.setCLP(localSchema.classLevelPermissions); + return newLocalSchema.save(); + }, + updateSchema: async (localSchema, cloudSchema) => { + const newLocalSchema = new Parse.Schema(localSchema.className); + + // Handle fields + // Check addition + Object.keys(localSchema.fields) + .filter( + fieldName => !lib.isDefaultFields(localSchema.className, fieldName) + ) + .forEach(fieldName => { + const { type, ...others } = localSchema.fields[fieldName]; + if (!cloudSchema.fields[fieldName]) + lib.handleFields(newLocalSchema, fieldName, type, others); + }); + + // Check deletion + await Promise.all( + Object.keys(cloudSchema.fields) + .filter( + fieldName => !lib.isDefaultFields(localSchema.className, fieldName) + ) + .map(async fieldName => { + const field = cloudSchema.fields[fieldName]; + if (!localSchema.fields[fieldName]) { + newLocalSchema.deleteField(fieldName); + await newLocalSchema.update(); + return; + } + const localField = localSchema.fields[fieldName]; + if (!lib.paramsAreEquals(field, localField)) { + newLocalSchema.deleteField(fieldName); + await newLocalSchema.update(); + const { type, ...others } = localField; + lib.handleFields(newLocalSchema, fieldName, type, others); + } + }) + ); + + // Handle Indexes + // Check addition + const cloudIndexes = lib.fixCloudIndexes(cloudSchema.indexes); + + if (localSchema.indexes) { + Object.keys(localSchema.indexes).forEach(indexName => { + if ( + !cloudIndexes[indexName] && + !lib.isNativeIndex(localSchema.className, indexName) + ) + newLocalSchema.addIndex(indexName, localSchema.indexes[indexName]); + }); + } + + const indexesToAdd = []; + + // Check deletion + Object.keys(cloudIndexes).forEach(async indexName => { + if (!lib.isNativeIndex(localSchema.className, indexName)) { + if (!localSchema.indexes[indexName]) { + newLocalSchema.deleteIndex(indexName); + } else if ( + !lib.paramsAreEquals( + localSchema.indexes[indexName], + cloudIndexes[indexName] + ) + ) { + newLocalSchema.deleteIndex(indexName); + indexesToAdd.push({ + indexName, + index: localSchema.indexes[indexName], + }); + } + } + }); + newLocalSchema.setCLP(localSchema.classLevelPermissions); + await newLocalSchema.update(); + indexesToAdd.forEach(o => newLocalSchema.addIndex(o.indexName, o.index)); + return newLocalSchema.update(); + }, + + isDefaultSchema: className => + ['_Session', '_Role', '_PushStatus', '_Installation'].indexOf(className) !== + -1, + + isDefaultFields: (className, fieldName) => + [ + 'objectId', + 'createdAt', + 'updatedAt', + 'ACL', + 'emailVerified', + 'authData', + 'username', + 'password', + 'email', + ] + .filter( + value => + (className !== '_User' && value !== 'email') || className === '_User' + ) + .indexOf(fieldName) !== -1, + + fixCloudIndexes: cloudSchemaIndexes => { + if (!cloudSchemaIndexes) return {}; + // eslint-disable-next-line + const { _id_, ...others } = cloudSchemaIndexes; + + return { + objectId: { objectId: 1 }, + ...others, + }; + }, + + isNativeIndex: (className, indexName) => { + if (className === '_User') { + switch (indexName) { + case 'username_1': + return true; + case 'objectId': + return true; + case 'email_1': + return true; + default: + break; + } + } + return false; + }, + + paramsAreEquals: (indexA, indexB) => { + const keysIndexA = Object.keys(indexA); + const keysIndexB = Object.keys(indexB); + + // Check key name + if (keysIndexA.length !== keysIndexB.length) return false; + return keysIndexA.every(k => indexA[k] === indexB[k]); + }, + + handleFields: (newLocalSchema, fieldName, type, others) => { + if (type === 'Relation') { + newLocalSchema.addRelation(fieldName, others.targetClass); + } else if (type === 'Pointer') { + const { targetClass, ...others2 } = others; + newLocalSchema.addPointer(fieldName, targetClass, others2); + } else { + newLocalSchema.addField(fieldName, type, others); + } + }, +}; diff --git a/src/Routers/SchemasRouter.js b/src/Routers/SchemasRouter.js index efc831dc04..4365dd975c 100644 --- a/src/Routers/SchemasRouter.js +++ b/src/Routers/SchemasRouter.js @@ -6,6 +6,15 @@ var Parse = require('parse/node').Parse, import PromiseRouter from '../PromiseRouter'; import * as middleware from '../middlewares'; +const checkSchemasOption = req => { + if (req.config.schemas) { + throw new Parse.Error( + Parse.Error.OPERATION_FORBIDDEN, + 'This server use schemas option, schemas creation/modification operations are blocked. You can still delete a class.' + ); + } +}; + function classNameMismatchResponse(bodyClass, pathClass) { throw new Parse.Error( Parse.Error.INVALID_CLASS_NAME, @@ -42,6 +51,7 @@ function getOneSchema(req) { } function createSchema(req) { + checkSchemasOption(req); if (req.auth.isReadOnly) { throw new Parse.Error( Parse.Error.OPERATION_FORBIDDEN, @@ -76,6 +86,7 @@ function createSchema(req) { } function modifySchema(req) { + checkSchemasOption(req); if (req.auth.isReadOnly) { throw new Parse.Error( Parse.Error.OPERATION_FORBIDDEN, @@ -104,6 +115,8 @@ function modifySchema(req) { } const deleteSchema = req => { + // Here we do not check schemas options, developer need to delete manually classes + // to avoid data loss during auto migration if (req.auth.isReadOnly) { throw new Parse.Error( Parse.Error.OPERATION_FORBIDDEN,