feat: db migrator (#432)

* feat: db migrator

* feat: modify the test description

packages/core/database/package.json

@@ -17,7 +17,8 @@
     "deepmerge": "^4.2.2",
     "flat": "^5.0.2",
     "glob": "^7.1.6",
-    "sequelize": "^6.9.0"
+    "sequelize": "^6.9.0",
+    "umzug": "^3.1.1"
   },
   "repository": {
     "type": "git",

packages/core/database/src/__tests__/migrator.test.ts

@@ -0,0 +1,52 @@
+import { Database, Migration, mockDatabase } from '@nocobase/database';
+
+const names = (migrations: Array<{ name: string }>) => migrations.map(m => m.name);
+
+describe('migrator', () => {
+  let db: Database;
+
+  beforeEach(async () => {
+
+    db = mockDatabase({
+      tablePrefix: 'test_',
+    });
+
+    await db.clean({ drop: true });
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  test('up and down', async () => {
+    const spy = jest.fn();
+    db.addMigration({
+      name: 'migration1',
+      migration: class extends Migration {
+        async up() {
+          spy('migration1-up');
+        }
+        async down() {
+          spy('migration1-down');
+        }
+      },
+    });
+    db.addMigration({
+      name: 'migration2',
+      migration: class extends Migration {
+        async up() {
+          spy('migration2-up');
+        }
+        async down() {
+          spy('migration2-down');
+        }
+      },
+    });
+    await db.migrator.up();
+    expect(names(await db.migrator.executed())).toEqual(['migration1', 'migration2']);
+    await db.migrator.down();
+    expect(names(await db.migrator.executed())).toEqual(['migration1']);
+    expect(spy).toHaveBeenCalledTimes(3);
+    expect(spy).toHaveBeenNthCalledWith(1, 'migration1-up');
+  });
+});

packages/core/database/src/database.ts

@@ -13,10 +13,12 @@ import {
   SyncOptions,
   Utils
 } from 'sequelize';
+import { SequelizeStorage, Umzug } from 'umzug';
 import { Collection, CollectionOptions, RepositoryType } from './collection';
 import { ImporterReader, ImportFileExtension } from './collection-importer';
 import * as FieldTypes from './fields';
 import { Field, FieldContext, RelationField } from './fields';
+import { Migrations } from './migration';
 import { Model } from './model';
 import { ModelHook } from './model-hook';
 import extendOperators from './operators';
@@ -36,9 +38,10 @@ interface MapOf<T> {
 
 export interface IDatabaseOptions extends Options {
   tablePrefix?: string;
+  migrator?: any;
 }
 
-export type DatabaseOptions = IDatabaseOptions | Sequelize;
+export type DatabaseOptions = IDatabaseOptions;
 
 interface RegisterOperatorsContext {
   db?: Database;
@@ -55,6 +58,8 @@ type OperatorFunc = (value: any, ctx?: RegisterOperatorsContext) => any;
 
 export class Database extends EventEmitter implements AsyncEmitter {
   sequelize: Sequelize;
+  migrator: Umzug;
+  migrations: Migrations;
   fieldTypes = new Map();
   options: IDatabaseOptions;
   models = new Map<string, ModelCtor<Model>>();
@@ -71,27 +76,24 @@ export class Database extends EventEmitter implements AsyncEmitter {
   constructor(options: DatabaseOptions) {
     super();
 
-    if (options instanceof Sequelize) {
-      this.sequelize = options;
-    } else {
-      const opts = {
-        sync: {
-          alter: {
-            drop: false,
-          },
-          force: false,
+    const opts = {
+      sync: {
+        alter: {
+          drop: false,
         },
-        ...options,
-      };
-      if (options.storage && options.storage !== ':memory:') {
-        if (!isAbsolute(options.storage)) {
-          opts.storage = resolve(process.cwd(), options.storage);
-        }
+        force: false,
+      },
+      ...options,
+    };
+
+    if (options.storage && options.storage !== ':memory:') {
+      if (!isAbsolute(options.storage)) {
+        opts.storage = resolve(process.cwd(), options.storage);
       }
-      this.sequelize = new Sequelize(opts);
-      this.options = opts;
     }
 
+    this.sequelize = new Sequelize(opts);
+    this.options = opts;
     this.collections = new Map();
     this.modelHook = new ModelHook(this);
 
@@ -116,6 +118,32 @@ export class Database extends EventEmitter implements AsyncEmitter {
     }
 
     this.initOperators();
+
+    const migratorOptions: any = this.options.migrator || {};
+
+    const context = {
+      db: this,
+      sequelize: this.sequelize,
+      queryInterface: this.sequelize.getQueryInterface(),
+      ...migratorOptions.context,
+    };
+
+    this.migrations = new Migrations(context);
+
+    this.migrator = new Umzug({
+      logger: migratorOptions.logger || console,
+      migrations: this.migrations.callback(),
+      context,
+      storage: new SequelizeStorage({
+        modelName: `${this.options.tablePrefix||''}migrations`,
+        ...migratorOptions.storage,
+        sequelize: this.sequelize,
+      }),
+    });
+  }
+
+  addMigration(item) {
+    return this.migrations.add(item);
   }
 
   /**

packages/core/database/src/index.ts

@@ -4,6 +4,7 @@ export * from './database';
 export { Database as default } from './database';
 export * from './fields';
 export * from './magic-attribute-model';
+export * from './migration';
 export * from './mock-database';
 export * from './model';
 export * from './relation-repository/belongs-to-many-repository';

packages/core/database/src/migration.ts

@@ -0,0 +1,75 @@
+import { QueryInterface, Sequelize } from 'sequelize';
+import Database from './database';
+
+export interface MigrationContext {
+  db: Database;
+  queryInterface: QueryInterface;
+  sequelize: Sequelize;
+}
+
+export class Migration {
+  public name: string;
+
+  public context: { db: Database };
+
+  constructor(context: MigrationContext) {
+    this.context = context;
+  }
+
+  get db() {
+    return this.context.db;
+  }
+
+  get sequelize() {
+    return this.context.db.sequelize;
+  }
+
+  get queryInterface() {
+    return this.context.db.sequelize.getQueryInterface();
+  }
+
+  async up() {
+    // todo
+  }
+
+  async down() {
+    // todo
+  }
+}
+
+export interface MigrationItem {
+  name: string;
+  migration?: typeof Migration;
+  up?: any;
+  down?: any;
+}
+
+export class Migrations {
+  items = [];
+  context: any;
+
+  constructor(context: any) {
+    this.context = context;
+  }
+
+  clear() {
+    this.items = [];
+  }
+
+  add(item: MigrationItem) {
+    const Migration = item.migration;
+    if (Migration) {
+      const migration = new Migration(this.context);
+      migration.name = item.name;
+      this.items.push(migration);
+    } else {
+      this.items.push(item);
+    }
+  }
+
+  callback() {
+    return (ctx) => {
+      return this.items;
+    };
+  }
+}

yarn.lock

@@ -4422,6 +4422,16 @@
     estree-walker "^1.0.1"
     picomatch "^2.2.2"
 
+"@rushstack/ts-command-line@^4.7.7":
+  version "4.11.0"
+  resolved "https://registry.npmjs.org/@rushstack%2fts-command-line/-/ts-command-line-4.11.0.tgz#4cd3b9f59b41aed600042936260fdaa55ca0184d"
+  integrity sha512-ptG9L0mjvJ5QtK11GsAFY+jGfsnqHDS6CY6Yw1xT7a9bhjfNYnf6UPwjV+pF6UgiucfNcMDNW9lkDLxvZKKxMg==
+  dependencies:
+    "@types/argparse" "1.0.38"
+    argparse "~1.0.9"
+    colors "~1.2.1"
+    string-argv "~0.3.1"
+
 "@sindresorhus/is@^0.14.0":
   version "0.14.0"
   resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
@@ -4642,6 +4652,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/argparse@1.0.38":
+  version "1.0.38"
+  resolved "https://registry.npmjs.org/@types%2fargparse/-/argparse-1.0.38.tgz#a81fd8606d481f873a3800c6ebae4f1d768a56a9"
+  integrity sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==
+
 "@types/aria-query@^4.2.0":
   version "4.2.2"
   resolved "https://registry.npmjs.org/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc"
@@ -6052,7 +6067,7 @@ arg@^5.0.0:
   resolved "https://registry.npmjs.org/arg/-/arg-5.0.1.tgz#eb0c9a8f77786cad2af8ff2b862899842d7b6adb"
   integrity sha512-e0hDa9H2Z9AwFkk2qDlwhoMYE4eToKarchkQHovNdLTCYMHZHeRjI71crOh+dio4K6u1IcwubQqo79Ga4CyAQA==
 
-argparse@^1.0.7:
+argparse@^1.0.7, argparse@~1.0.9:
   version "1.0.10"
   resolved "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
   integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
@@ -7695,6 +7710,11 @@ colors@~0.6.0-1:
   resolved "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz#2423fe6678ac0c5dae8852e5d0e5be08c997abcc"
   integrity sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=
 
+colors@~1.2.1:
+  version "1.2.5"
+  resolved "https://registry.npmjs.org/colors/-/colors-1.2.5.tgz#89c7ad9a374bc030df8013241f68136ed8835afc"
+  integrity sha512-erNRLao/Y3Fv54qUa0LBB+//Uf3YwMUmdJinN20yMXm9zdKKqH9wt7R9IIVZ+K7ShzfpLV/Zg8+VyrBJYB4lpg==
+
 columnify@^1.5.4:
   version "1.5.4"
   resolved "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz#4737ddf1c7b69a8a7c340570782e947eec8e78bb"
@@ -9203,6 +9223,11 @@ emitter-listener@^1.1.1:
   dependencies:
     shimmer "^1.2.0"
 
+emittery@^0.10.2:
+  version "0.10.2"
+  resolved "https://registry.npmjs.org/emittery/-/emittery-0.10.2.tgz#902eec8aedb8c41938c46e9385e9db7e03182933"
+  integrity sha512-aITqOwnLanpHLNXZJENbOgjUBeHocD+xsSJmNrjovKBW5HbSpW3d1pEls7GFQPUWXiwG9+0P4GtHfEqC/4M0Iw==
+
 emittery@^0.7.1:
   version "0.7.2"
   resolved "https://registry.npmjs.org/emittery/-/emittery-0.7.2.tgz#25595908e13af0f5674ab419396e2fb394cdfa82"
@@ -10392,6 +10417,14 @@ fs-extra@^9.1.0:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
+fs-jetpack@^4.1.0:
+  version "4.3.1"
+  resolved "https://registry.npmjs.org/fs-jetpack/-/fs-jetpack-4.3.1.tgz#cdfd4b64e6bfdec7c7dc55c76b39efaa7853bb20"
+  integrity sha512-dbeOK84F6BiQzk2yqqCVwCPWTxAvVGJ3fMQc6E2wuEohS28mR6yHngbrKuVCK1KHRx/ccByDylqu4H5PCP2urQ==
+  dependencies:
+    minimatch "^3.0.2"
+    rimraf "^2.6.3"
+
 fs-minipass@^1.2.7:
   version "1.2.7"
   resolved "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7"
@@ -16551,6 +16584,11 @@ pn@^1.1.0:
   resolved "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb"
   integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==
 
+pony-cause@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.npmjs.org/pony-cause/-/pony-cause-1.1.1.tgz#f795524f83bebbf1878bd3587b45f69143cbf3f9"
+  integrity sha512-PxkIc/2ZpLiEzQXu5YRDOUgBlfGYBY8156HY5ZcRAwwonMk5W/MrJP2LLkG/hF7GEQzaHo2aS7ho6ZLCOvf+6g==
+
 portfinder@^1.0.28:
   version "1.0.28"
   resolved "https://registry.npmmirror.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778"
@@ -20244,6 +20282,11 @@ strict-uri-encode@^2.0.0:
   resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"
   integrity sha1-ucczDHBChi9rFC3CdLvMWGbONUY=
 
+string-argv@~0.3.1:
+  version "0.3.1"
+  resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
+  integrity sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==
+
 string-convert@^0.2.0:
   version "0.2.1"
   resolved "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz#6982cc3049fbb4cd85f8b24568b9d9bf39eeff97"
@@ -21272,6 +21315,11 @@ type-fest@^0.8.1:
   resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
   integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
 
+type-fest@^2.0.0:
+  version "2.13.0"
+  resolved "https://registry.npmjs.org/type-fest/-/type-fest-2.13.0.tgz#d1ecee38af29eb2e863b22299a3d68ef30d2abfb"
+  integrity sha512-lPfAm42MxE4/456+QyIaaVBAwgpJb6xZ8PRu09utnhPdWwcyj9vgy6Sq0Z5yNbJ21EdxB5dRU/Qg8bsyAMtlcw==
+
 type-is@^1.6.16, type-is@^1.6.4:
   version "1.6.18"
   resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
@@ -21420,6 +21468,18 @@ umi@^3.0.0, umi@^3.5.20:
     react-dom "16.x"
     v8-compile-cache "2.3.0"
 
+umzug@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.npmjs.org/umzug/-/umzug-3.1.1.tgz#dfbe52308bf2908984380bdffd0c75c07831fd1f"
+  integrity sha512-sgMDzUK6ZKS3pjzRJpAHqSkvAQ+64Dourq6JfQv11i0nMu0/QqE3V3AUpj2pWYxFBaSvnUxKrzZQmPr6NZhvdQ==
+  dependencies:
+    "@rushstack/ts-command-line" "^4.7.7"
+    emittery "^0.10.2"
+    fs-jetpack "^4.1.0"
+    glob "^7.1.6"
+    pony-cause "^1.1.1"
+    type-fest "^2.0.0"
+
 unbox-primitive@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"