feat: support for static loading logic of plugins (#5466)

* feat: extend database dialect

* chore: error message

* fix: pg version

* chore: error message

* feat: load plugins static import

* chore: static import

* fix: test

* chore: find packages

* fix: findAllPlugins

* feat: appendToBuiltInPlugins

* fix: runPluginStaticImports

* fix: create app

---------

Co-authored-by: chenos <chenlinxh@gmail.com>

packages/core/app/src/index.ts

@@ -7,15 +7,18 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
-import { Gateway } from '@nocobase/server';
+import { Gateway, runPluginStaticImports } from '@nocobase/server';
 import { getConfig } from './config';
 
-getConfig()
-  .then((config) => {
-    return Gateway.getInstance().run({
-      mainAppOptions: config,
-    });
-  })
-  .catch((e) => {
-    // console.error(e);
+async function initializeGateway() {
+  await runPluginStaticImports();
+  const config = await getConfig();
+  await Gateway.getInstance().run({
+    mainAppOptions: config,
   });
+}
+
+initializeGateway().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

packages/core/create-nocobase-app/src/generator.js

@@ -112,6 +112,7 @@ class AppGenerator extends Generator {
         envs.push(`DB_USER=${env.DB_USER || ''}`);
         envs.push(`DB_PASSWORD=${env.DB_PASSWORD || ''}`);
         break;
+      case 'kingbase':
       case 'postgres':
         if (!allDbDialect) {
           dependencies.push(`"pg": "^8.7.3"`);
@@ -125,7 +126,7 @@ class AppGenerator extends Generator {
         break;
     }
 
-    const keys = ['PLUGIN_PACKAGE_PREFIX', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USER', 'DB_PASSWORD', 'DB_STORAGE'];
+    const keys = ['DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USER', 'DB_PASSWORD', 'DB_STORAGE'];
 
     for (const key in env) {
       if (keys.includes(key)) {

packages/core/database/src/__tests__/dialect-extend/dialect-extend.test.ts

@@ -0,0 +1,38 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { mockDatabase } from '../';
+import { Database } from '../../database';
+import { BaseDialect } from '../../dialects/base-dialect';
+
+describe('dialect extend', () => {
+  let db: Database;
+
+  beforeEach(async () => {
+    db = mockDatabase();
+    await db.clean({ drop: true });
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  it('should register dialect', async () => {
+    class SubDialect extends BaseDialect {
+      static dialectName = 'test';
+
+      async checkDatabaseVersion(db: Database): Promise<boolean> {
+        return true;
+      }
+    }
+
+    Database.registerDialect(SubDialect);
+    expect(Database.getDialect('test')).toBe(SubDialect);
+  });
+});

packages/core/database/src/database.ts

@@ -18,7 +18,6 @@ import lodash from 'lodash';
 import { nanoid } from 'nanoid';
 import { basename, isAbsolute, resolve } from 'path';
 import safeJsonStringify from 'safe-json-stringify';
-import semver from 'semver';
 import {
   DataTypes,
   ModelStatic,
@@ -41,7 +40,7 @@ import { referentialIntegrityCheck } from './features/referential-integrity-chec
 import { ArrayFieldRepository } from './field-repository/array-field-repository';
 import * as FieldTypes from './fields';
 import { Field, FieldContext, RelationField } from './fields';
-import { checkDatabaseVersion } from './helpers';
+import { checkDatabaseVersion, registerDialects } from './helpers';
 import { InheritedCollection } from './inherited-collection';
 import InheritanceMap from './inherited-map';
 import { InterfaceManager } from './interface-manager';
@@ -85,6 +84,7 @@ import {
 import { patchSequelizeQueryInterface, snakeCase } from './utils';
 import { BaseValueParser, registerFieldValueParsers } from './value-parsers';
 import { ViewCollection } from './view-collection';
+import { BaseDialect } from './dialects/base-dialect';
 
 export type MergeOptions = merge.Options;
 
@@ -129,35 +129,9 @@ export type AddMigrationsOptions = {
 
 type OperatorFunc = (value: any, ctx?: RegisterOperatorsContext) => any;
 
-export const DialectVersionAccessors = {
-  sqlite: {
-    sql: 'select sqlite_version() as version',
-    get: (v: string) => v,
-  },
-  mysql: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return m[0];
-    },
-  },
-  mariadb: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return m[0];
-    },
-  },
-  postgres: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return semver.minVersion(m[0]).version;
-    },
-  },
-};
-
 export class Database extends EventEmitter implements AsyncEmitter {
+  static dialects = new Map<string, typeof BaseDialect>();
+
   sequelize: Sequelize;
   migrator: Umzug;
   migrations: Migrations;
@@ -181,13 +155,31 @@ export class Database extends EventEmitter implements AsyncEmitter {
   delayCollectionExtend = new Map<string, { collectionOptions: CollectionOptions; mergeOptions?: any }[]>();
   logger: Logger;
   interfaceManager = new InterfaceManager(this);
-
   collectionFactory: CollectionFactory = new CollectionFactory(this);
+  dialect: BaseDialect;
+
   declare emitAsync: (event: string | symbol, ...args: any[]) => Promise<boolean>;
 
+  static registerDialect(dialect: typeof BaseDialect) {
+    this.dialects.set(dialect.dialectName, dialect);
+  }
+
+  static getDialect(name: string) {
+    return this.dialects.get(name);
+  }
+
   constructor(options: DatabaseOptions) {
     super();
 
+    const dialectClass = Database.getDialect(options.dialect);
+
+    if (!dialectClass) {
+      throw new Error(`unsupported dialect ${options.dialect}`);
+    }
+
+    // @ts-ignore
+    this.dialect = new dialectClass();
+
     const opts = {
       sync: {
         alter: {
@@ -373,21 +365,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
    * @internal
    */
   sequelizeOptions(options) {
-    if (options.dialect === 'postgres') {
-      if (!options.hooks) {
-        options.hooks = {};
-      }
-
-      if (!options.hooks['afterConnect']) {
-        options.hooks['afterConnect'] = [];
-      }
-
-      options.hooks['afterConnect'].push(async (connection) => {
-        await connection.query('SET search_path TO public;');
-      });
-    }
-
-    return options;
+    return this.dialect.getSequelizeOptions(options);
   }
 
   /**
@@ -527,6 +505,10 @@ export class Database extends EventEmitter implements AsyncEmitter {
     return this.inDialect('mysql', 'mariadb');
   }
 
+  isPostgresCompatibleDialect() {
+    return this.inDialect('postgres');
+  }
+
   /**
    * Add collection to database
    * @param options
@@ -1036,5 +1018,6 @@ export const defineCollection = (collectionOptions: CollectionOptions) => {
 };
 
 applyMixins(Database, [AsyncEmitter]);
+registerDialects();
 
 export default Database;

packages/core/database/src/dialects/base-dialect.ts

@@ -0,0 +1,52 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Database, DatabaseOptions } from '../database';
+import semver from 'semver';
+
+export interface DialectVersionGuard {
+  sql: string;
+  get: (v: string) => string;
+  version: string;
+}
+
+export abstract class BaseDialect {
+  static dialectName: string;
+
+  getSequelizeOptions(options: DatabaseOptions) {
+    return options;
+  }
+
+  async checkDatabaseVersion(db: Database): Promise<boolean> {
+    const versionGuard = this.getVersionGuard();
+
+    const result = await db.sequelize.query(versionGuard.sql, {
+      type: 'SELECT',
+    });
+
+    // @ts-ignore
+    const version = versionGuard.get(result?.[0]?.version);
+
+    const versionResult = semver.satisfies(version, versionGuard.version);
+
+    if (!versionResult) {
+      throw new Error(
+        `to use ${(this.constructor as typeof BaseDialect).dialectName}, please ensure the version is ${
+          versionGuard.version
+        }, current version is ${version}`,
+      );
+    }
+
+    return true;
+  }
+
+  getVersionGuard(): DialectVersionGuard {
+    throw new Error('not implemented');
+  }
+}

packages/core/database/src/dialects/index.ts

@@ -0,0 +1,10 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export * from './base-dialect';

packages/core/database/src/dialects/mariadb-dialect.ts

@@ -0,0 +1,25 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { BaseDialect } from './base-dialect';
+
+export class MariadbDialect extends BaseDialect {
+  static dialectName = 'mariadb';
+
+  getVersionGuard() {
+    return {
+      sql: 'select version() as version',
+      get: (v: string) => {
+        const m = /([\d+.]+)/.exec(v);
+        return m[0];
+      },
+      version: '>=10.9',
+    };
+  }
+}

packages/core/database/src/dialects/mysql-dialect.ts

@@ -0,0 +1,25 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { BaseDialect } from './base-dialect';
+
+export class MysqlDialect extends BaseDialect {
+  static dialectName = 'mysql';
+
+  getVersionGuard() {
+    return {
+      sql: 'select version() as version',
+      get: (v: string) => {
+        const m = /([\d+.]+)/.exec(v);
+        return m[0];
+      },
+      version: '>=8.0.17',
+    };
+  }
+}

packages/core/database/src/dialects/postgres-dialect.ts

@@ -0,0 +1,42 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import semver from 'semver';
+import { BaseDialect } from './base-dialect';
+
+export class PostgresDialect extends BaseDialect {
+  static dialectName = 'postgres';
+
+  getSequelizeOptions(options: any) {
+    if (!options.hooks) {
+      options.hooks = {};
+    }
+
+    if (!options.hooks['afterConnect']) {
+      options.hooks['afterConnect'] = [];
+    }
+
+    options.hooks['afterConnect'].push(async (connection) => {
+      await connection.query('SET search_path TO public;');
+    });
+
+    return options;
+  }
+
+  getVersionGuard() {
+    return {
+      sql: 'select version() as version',
+      get: (v: string) => {
+        const m = /([\d+.]+)/.exec(v);
+        return semver.minVersion(m[0]).version;
+      },
+      version: '>=10',
+    };
+  }
+}

packages/core/database/src/dialects/sqlite-dialect.ts

@@ -0,0 +1,22 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { BaseDialect } from './base-dialect';
+
+export class SqliteDialect extends BaseDialect {
+  static dialectName = 'sqlite';
+
+  getVersionGuard() {
+    return {
+      sql: 'select sqlite_version() as version',
+      get: (v: string) => v,
+      version: '3.x',
+    };
+  }
+}

packages/core/database/src/helpers.ts

@@ -11,7 +11,10 @@
 
 import { Database, IDatabaseOptions } from './database';
 import fs from 'fs';
-import semver from 'semver';
+import { MysqlDialect } from './dialects/mysql-dialect';
+import { SqliteDialect } from './dialects/sqlite-dialect';
+import { MariadbDialect } from './dialects/mariadb-dialect';
+import { PostgresDialect } from './dialects/postgres-dialect';
 
 function getEnvValue(key, defaultValue?) {
   return process.env[key] || defaultValue;
@@ -103,55 +106,12 @@ function customLogger(queryString, queryObject) {
   }
 }
 
-const dialectVersionAccessors = {
-  sqlite: {
-    sql: 'select sqlite_version() as version',
-    get: (v: string) => v,
-    version: '3.x',
-  },
-  mysql: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return m[0];
-    },
-    version: '>=8.0.17',
-  },
-  mariadb: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return m[0];
-    },
-    version: '>=10.9',
-  },
-  postgres: {
-    sql: 'select version() as version',
-    get: (v: string) => {
-      const m = /([\d+.]+)/.exec(v);
-      return semver.minVersion(m[0]).version;
-    },
-    version: '>=10',
-  },
-};
-
 export async function checkDatabaseVersion(db: Database) {
-  const dialect = db.sequelize.getDialect();
-  const accessor = dialectVersionAccessors[dialect];
-  if (!accessor) {
-    throw new Error(`unsupported dialect ${dialect}`);
-  }
-
-  const result = await db.sequelize.query(accessor.sql, {
-    type: 'SELECT',
-  });
-
-  // @ts-ignore
-  const version = accessor.get(result?.[0]?.version);
-  const versionResult = semver.satisfies(version, accessor.version);
-  if (!versionResult) {
-    throw new Error(`to use ${dialect}, please ensure the version is ${accessor.version}`);
-  }
-
-  return true;
+  await db.dialect.checkDatabaseVersion(db);
+}
+
+export function registerDialects() {
+  [SqliteDialect, MysqlDialect, MariadbDialect, PostgresDialect].forEach((dialect) => {
+    Database.registerDialect(dialect);
+  });
 }

packages/core/database/src/index.ts

@@ -55,3 +55,4 @@ export * from './helpers';
 export { default as sqlParser, SQLParserTypes } from './sql-parser';
 export * from './interfaces';
 export { default as fieldTypeMap } from './view/field-type-map';
+export * from './dialects';

packages/core/database/src/query-interface/query-interface-builder.ts

@@ -20,7 +20,12 @@ export default function buildQueryInterface(db: Database) {
     sqlite: SqliteQueryInterface,
   };
 
+  if (db.isPostgresCompatibleDialect()) {
+    return new PostgresQueryInterface(db);
+  }
+
   const dialect = db.options.dialect;
+
   if (!map[dialect]) {
     return null;
   }

packages/core/server/src/index.ts

@@ -7,13 +7,24 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
+export * from './app-supervisor';
 export * from './application';
 export { Application as default } from './application';
+export * from './gateway';
 export * as middlewares from './middlewares';
 export * from './migration';
 export * from './plugin';
 export * from './plugin-manager';
-export * from './gateway';
-export * from './app-supervisor';
+
 export * from './sync-manager';
 export const OFFICIAL_PLUGIN_PREFIX = '@nocobase/plugin-';
+
+export {
+  appendToBuiltInPlugins,
+  findAllPlugins,
+  findBuiltInPlugins,
+  findLocalPlugins,
+  packageNameTrim,
+} from './plugin-manager/findPackageNames';
+
+export { runPluginStaticImports } from './run-plugin-static-imports';

packages/core/server/src/plugin-manager/findPackageNames.ts

@@ -7,17 +7,17 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
-import { PluginManager } from '@nocobase/server';
 import fg from 'fast-glob';
 import fs from 'fs-extra';
 import _ from 'lodash';
 import path from 'path';
+import { PluginManager } from './';
 
 function splitNames(name: string) {
   return (name || '').split(',').filter(Boolean);
 }
 
-export async function trim(packageNames: string[]) {
+async function trim(packageNames: string[]) {
   const nameOrPkgs = _.uniq(packageNames).filter(Boolean);
   const names = [];
   for (const nameOrPkg of nameOrPkgs) {
@@ -78,9 +78,16 @@ export async function findPackageNames() {
   }
 }
 
+async function getPackageJson() {
+  const packageJson = await fs.readJson(
+    path.resolve(process.env.NODE_MODULES_PATH, '@nocobase/preset-nocobase/package.json'),
+  );
+  return packageJson;
+}
+
 async function findNocobasePlugins() {
   try {
-    const packageJson = await fs.readJson(path.resolve(__dirname, '../../package.json'));
+    const packageJson = await getPackageJson();
     const pluginNames = Object.keys(packageJson.dependencies).filter((name) => name.startsWith('@nocobase/plugin-'));
     return trim(pluginNames);
   } catch (error) {
@@ -91,7 +98,7 @@ async function findNocobasePlugins() {
 export async function findBuiltInPlugins() {
   const { APPEND_PRESET_BUILT_IN_PLUGINS = '' } = process.env;
   try {
-    const packageJson = await fs.readJson(path.resolve(__dirname, '../../package.json'));
+    const packageJson = await getPackageJson();
     return trim(packageJson.builtIn.concat(splitNames(APPEND_PRESET_BUILT_IN_PLUGINS)));
   } catch (error) {
     return [];
@@ -103,7 +110,7 @@ export async function findLocalPlugins() {
   const plugins1 = await findNocobasePlugins();
   const plugins2 = await findPackageNames();
   const builtInPlugins = await findBuiltInPlugins();
-  const packageJson = await fs.readJson(path.resolve(__dirname, '../../package.json'));
+  const packageJson = await getPackageJson();
   const items = await trim(
     _.difference(
       plugins1.concat(plugins2).concat(splitNames(APPEND_PRESET_LOCAL_PLUGINS)),
@@ -112,3 +119,24 @@ export async function findLocalPlugins() {
   );
   return items;
 }
+
+export async function findAllPlugins() {
+  const builtInPlugins = await findBuiltInPlugins();
+  const localPlugins = await findLocalPlugins();
+  return _.uniq(builtInPlugins.concat(localPlugins));
+}
+
+export const packageNameTrim = trim;
+
+export async function appendToBuiltInPlugins(nameOrPkg: string) {
+  const APPEND_PRESET_BUILT_IN_PLUGINS = process.env.APPEND_PRESET_BUILT_IN_PLUGINS || '';
+  const keys = APPEND_PRESET_BUILT_IN_PLUGINS.split(',');
+  const { name, packageName } = await PluginManager.parseName(nameOrPkg);
+  if (keys.includes(packageName)) {
+    return;
+  }
+  if (keys.includes(name)) {
+    return;
+  }
+  process.env.APPEND_PRESET_BUILT_IN_PLUGINS += ',' + nameOrPkg;
+}

packages/core/server/src/run-plugin-static-imports.ts

@@ -0,0 +1,24 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { findAllPlugins, PluginManager } from '@nocobase/server';
+
+export async function runPluginStaticImports() {
+  const packages = await findAllPlugins();
+  for (const name of packages) {
+    const { packageName } = await PluginManager.parseName(name);
+    try {
+      const plugin = require(packageName);
+      if (plugin && plugin.staticImport) {
+        await plugin.staticImport();
+      }
+    } catch (error) {
+      continue;
+    }
+  }
+}

packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts

@@ -113,7 +113,7 @@ const defaultAppOptionsFactory = (appName: string, mainApp: Application) => {
       const mainStorageDir = path.dirname(mainAppStorage);
       rawDatabaseOptions.storage = path.join(mainStorageDir, `${appName}.sqlite`);
     }
-  } else if (process.env.USE_DB_SCHEMA_IN_SUBAPP === 'true' && rawDatabaseOptions.dialect === 'postgres') {
+  } else if (process.env.USE_DB_SCHEMA_IN_SUBAPP === 'true' && mainApp.db.isPostgresCompatibleDialect()) {
     rawDatabaseOptions.schema = appName;
   } else {
     rawDatabaseOptions.database = appName;

packages/presets/nocobase/src/server/index.ts

@@ -7,9 +7,8 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
-import { Plugin, PluginManager } from '@nocobase/server';
+import { findBuiltInPlugins, findLocalPlugins, packageNameTrim, Plugin, PluginManager } from '@nocobase/server';
 import _ from 'lodash';
-import { findBuiltInPlugins, findLocalPlugins, trim } from './findPackageNames';
 
 export class PresetNocoBase extends Plugin {
   splitNames(name: string) {
@@ -43,7 +42,7 @@ export class PresetNocoBase extends Plugin {
     });
     const plugins1 = await findBuiltInPlugins();
     const plugins2 = await findLocalPlugins();
-    return trim(_.uniq([...plugins1, ...plugins2, ...items.map((item) => item.name)]));
+    return packageNameTrim(_.uniq([...plugins1, ...plugins2, ...items.map((item) => item.name)]));
   }
 
   async getAllPlugins(locale = 'en-US') {