feat: encryption field (#4975)

* feat: add @nocobase/plugin-field-encryption

* fix: bug

* fix: hook

* fix: add operators

* feat: add hidden

* fix: i18n

* fix: bug

* feat: env add ENCRYPTION_FIELD_KEY

* fix: exception handling

* fix: error message i18n

* fix: add `addFieldInterfaces()` alias

* fix: bug

* fix: bug

* fix: bug

* fix: bug

* fix: workflow env

* fix: bug

* fix: e2e

* fix: e2e bug

* fix: move `checkKey()` to field

* fix: move EncryptionField to database package

* fix: move encryption plugin to pro

* chore: encryption field in field type map

* fix: unit test

* fix: remove console

* fix: add more value check

* fix: bug

* fix: bug

* fix: bug

---------

Co-authored-by: chenos <chenlinxh@gmail.com>
Co-authored-by: Chareice <chareice@live.com>

.env.e2e.example

@@ -98,3 +98,7 @@ DEFAULT_SMS_VERIFY_CODE_PROVIDER=
 
 # in nodejs 17+ that SSL v3 causes some ecosystem libraries to become incompatible. Configuring this option can prevent upgrading SSL V3
 # NODE_OPTIONS=--openssl-legacy-provider
+
+################# ENCRYPTION FIELD #################
+
+ENCRYPTION_FIELD_KEY=

.env.example

@@ -70,3 +70,7 @@ INIT_ROOT_EMAIL=admin@nocobase.com
 INIT_ROOT_PASSWORD=admin123
 INIT_ROOT_NICKNAME=Super Admin
 INIT_ROOT_USERNAME=nocobase
+
+################# ENCRYPTION FIELD #################
+
+ENCRYPTION_FIELD_KEY=

.env.test.example

@@ -68,3 +68,7 @@ INIT_ALI_SMS_VERIFY_CODE_SIGN=
 
 # use any string name (no space)
 DEFAULT_SMS_VERIFY_CODE_PROVIDER=
+
+################# ENCRYPTION FIELD #################
+
+ENCRYPTION_FIELD_KEY=

.github/workflows/e2e.yml

@@ -160,6 +160,7 @@ jobs:
           DB_PASSWORD: password
           DB_DATABASE: nocobase
           APPEND_PRESET_LOCAL_PLUGINS: ${{ steps.vars.outputs.var2 }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
 
       - name: Upload e2e-report
         if: ${{ !cancelled() }}
@@ -257,6 +258,7 @@ jobs:
           DB_PASSWORD: password
           DB_DATABASE: nocobase
           APPEND_PRESET_LOCAL_PLUGINS: ${{ steps.vars.outputs.var2 }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
 
       - name: Upload e2e-report
         if: ${{ !cancelled() }}
@@ -354,6 +356,7 @@ jobs:
           DB_PASSWORD: password
           DB_DATABASE: nocobase
           APPEND_PRESET_LOCAL_PLUGINS: ${{ steps.vars.outputs.var2 }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
 
       - name: Upload e2e-report
         if: ${{ !cancelled() }}
@@ -451,6 +454,7 @@ jobs:
           DB_PASSWORD: password
           DB_DATABASE: nocobase
           APPEND_PRESET_LOCAL_PLUGINS: ${{ steps.vars.outputs.var2 }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
 
       - name: Upload e2e-report
         if: ${{ !cancelled() }}

.github/workflows/manual-e2e.yml

@@ -1,6 +1,6 @@
 name: manual-e2e
 
-on: 
+on:
   workflow_dispatch:
     inputs:
       branch:
@@ -83,4 +83,5 @@ jobs:
           DB_PASSWORD: password
           DB_DATABASE: nocobase
           APPEND_PRESET_LOCAL_PLUGINS: ${{ steps.vars.outputs.var2 }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
     timeout-minutes: 120

.github/workflows/nocobase-test-backend.yml

@@ -65,6 +65,7 @@ jobs:
           DB_STORAGE: /tmp/db.sqlite
           DB_TEST_PREFIX: test_
           DB_UNDERSCORED: ${{ matrix.underscored }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
 
   postgres-test:
     strategy:
@@ -122,6 +123,7 @@ jobs:
           COLLECTION_MANAGER_SCHEMA: ${{ matrix.collection_schema }}
           DB_TEST_DISTRIBUTOR_PORT: 23450
           DB_TEST_PREFIX: test
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
     timeout-minutes: 60
 
   mysql-test:
@@ -168,6 +170,7 @@ jobs:
           DB_UNDERSCORED: ${{ matrix.underscored }}
           DB_TEST_DISTRIBUTOR_PORT: 23450
           DB_TEST_PREFIX: test_
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
     timeout-minutes: 60
   mariadb-test:
     strategy:
@@ -215,4 +218,5 @@ jobs:
           DB_UNDERSCORED: ${{ matrix.underscored }}
           DB_TEST_DISTRIBUTOR_PORT: 23450
           DB_TEST_PREFIX: test_
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]
     timeout-minutes: 60

.github/workflows/nocobase-test-windows.yml

@@ -72,3 +72,4 @@ jobs:
           DB_STORAGE: /tmp/db.sqlite
           DB_TEST_PREFIX: test_
           DB_UNDERSCORED: ${{ matrix.underscored }}
+          ENCRYPTION_FIELD_KEY: 1%&glK;<UA}aIxJVc53-4G(rTi0vg@J]

docker/app-mariadb/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - mariadb
     environment:
       - APP_KEY=your-secret-key # Replace it with your own app key
+      - ENCRYPTION_FIELD_KEY=your-secret-key # Replace it with your own app key
       - DB_DIALECT=mariadb
       - DB_HOST=mariadb
       - DB_DATABASE=nocobase
@@ -34,4 +35,4 @@ services:
     volumes:
       - ./storage/db/mariadb:/var/lib/mysql
     networks:
-      - nocobase
+      - nocobase

docker/app-mysql/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - mysql
     environment:
       - APP_KEY=your-secret-key # Replace it with your own app key
+      - ENCRYPTION_FIELD_KEY=your-secret-key # Replace it with your own app key
       - DB_DIALECT=mysql
       - DB_HOST=mysql
       - DB_DATABASE=nocobase

docker/app-postgres/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - nocobase
     environment:
       - APP_KEY=your-secret-key # Replace it with your own app key
+      - ENCRYPTION_FIELD_KEY=your-secret-key # Replace it with your own app key
       - DB_DIALECT=postgres
       - DB_HOST=postgres
       - DB_DATABASE=nocobase
@@ -32,4 +33,4 @@ services:
     volumes:
       - ./storage/db/postgres:/var/lib/postgresql/data
     networks:
-      - nocobase
+      - nocobase

packages/core/client/src/application/Application.tsx

@@ -44,6 +44,7 @@ import { OpenModeProvider } from '../modules/popup/OpenModeProvider';
 import { AppSchemaComponentProvider } from './AppSchemaComponentProvider';
 import type { Plugin } from './Plugin';
 import type { RequireJS } from './utils/requirejs';
+import type { CollectionFieldInterfaceFactory } from '../data-source';
 
 declare global {
   interface Window {
@@ -357,6 +358,10 @@ export class Application {
     return root;
   }
 
+  addFieldInterfaces(fieldInterfaceClasses: CollectionFieldInterfaceFactory[] = []) {
+    return this.dataSourceManager.collectionFieldInterfaceManager.addFieldInterfaces(fieldInterfaceClasses);
+  }
+
   addFieldInterfaceComponentOption(fieldName: string, componentOption: CollectionFieldInterfaceComponentOption) {
     return this.dataSourceManager.collectionFieldInterfaceManager.addFieldInterfaceComponentOption(
       fieldName,

packages/core/database/src/__tests__/fields/encryption-field.test.ts

@@ -0,0 +1,194 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { EncryptionField } from '../../fields/encryption-field';
+import { mockDatabase, MockDatabase } from '../../mock-database';
+
+describe('encryption field', () => {
+  let db: MockDatabase;
+
+  beforeEach(async () => {
+    db = mockDatabase();
+    await db.clean({ drop: true });
+    db.registerFieldTypes({
+      encryption: EncryptionField,
+    });
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  it('basic', async () => {
+    db.collection({
+      name: 'tests',
+      fields: [
+        {
+          type: 'encryption',
+          name: 'name1',
+          iv: '1234567890123456',
+        },
+      ],
+    });
+    await db.sync();
+    const r = db.getRepository('tests');
+    const model = await r.create({
+      values: {
+        name1: 'aaa',
+      },
+    });
+    expect(model.get('name1')).not.toBe('aaa');
+    const model2 = await r.findOne();
+    expect(model2.get('name1')).toBe('aaa');
+  });
+
+  it('should throw error when value is object', async () => {
+    db.collection({
+      name: 'tests',
+      fields: [
+        {
+          type: 'encryption',
+          name: 'name1',
+          iv: '1234567890123456',
+        },
+      ],
+    });
+    await db.sync();
+    const r = db.getRepository('tests');
+    let err: Error;
+    try {
+      await r.create({
+        values: {
+          name1: { obj: 'aaa' },
+        },
+      });
+    } catch (error) {
+      err = error;
+    }
+    expect(err?.message).toBe('string violation: name1 cannot be an array or an object');
+  });
+
+  it('should throw error when value is number', async () => {
+    db.collection({
+      name: 'tests',
+      fields: [
+        {
+          type: 'encryption',
+          name: 'name1',
+          iv: '1234567890123456',
+        },
+      ],
+    });
+    await db.sync();
+    const r = db.getRepository('tests');
+    let err: Error;
+    try {
+      await r.create({
+        values: {
+          name1: 123,
+        },
+      });
+    } catch (error) {
+      err = error;
+    }
+    expect(err?.message).toBe('Encrypt Failed: The value must be a string, but got number');
+  });
+
+  it('should throw error when `iv` incorrect', async () => {
+    db.collection({
+      name: 'tests',
+      fields: [
+        {
+          type: 'encryption',
+          name: 'name1',
+          iv: '1',
+        },
+      ],
+    });
+    await db.sync();
+    const r = db.getRepository('tests');
+    let err: Error;
+    try {
+      await r.create({
+        values: {
+          name1: 'aaa',
+        },
+      });
+    } catch (error) {
+      err = error;
+    }
+    expect(err.message).toBe('Encrypt Failed: The `iv` must be a 16-character string');
+  });
+
+  it('should not throw error when value is `null` or `undefined` or empty string', async () => {
+    db.collection({
+      name: 'tests',
+      fields: [
+        {
+          type: 'encryption',
+          name: 'name1',
+          iv: '1234567890123456',
+        },
+      ],
+    });
+    await db.sync();
+    const r = db.getRepository('tests');
+    const fn = vitest.fn();
+    try {
+      await r.create({
+        values: [
+          {
+            name1: null,
+          },
+          {
+            name1: undefined,
+          },
+          {
+            name1: '',
+          },
+        ],
+      });
+    } catch {
+      fn();
+    }
+    expect(fn).toBeCalledTimes(0);
+  });
+
+  // 无法测，因为 keyStr 项目启动时读取的环境变量，所以测试用例中修改没用
+  // it('should throw error when `ENCRYPTION_FIELD_KEY` not exists', async () => {
+  //   const key = process.env.ENCRYPTION_FIELD_KEY;
+  //   process.env.ENCRYPTION_FIELD_KEY = '';
+
+  //   db.collection({
+  //     name: 'tests',
+  //     fields: [
+  //       {
+  //         type: 'encryption',
+  //         name: 'name1',
+  //         iv: '1234567890123456',
+  //       },
+  //     ],
+  //   });
+  //   await db.sync();
+  //   const r = db.getRepository('tests');
+  //   let err: Error;
+  //   try {
+  //     await r.create({
+  //       values: {
+  //         name1: { obj: 'aaa' },
+  //       },
+  //     });
+  //   } catch (error) {
+  //     err = error;
+  //   }
+  // expect(err).toBeTruthy();
+
+  //   process.env.ENCRYPTION_FIELD_KEY = key;
+  // });
+});

packages/core/database/src/fields/encryption-field/encryption-field.ts

@@ -0,0 +1,89 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { DataTypes, Model } from 'sequelize';
+import { EncryptionError } from './errors/EncryptionError';
+import { aesCheckKey, aesDecrypt, aesEncrypt } from './utils';
+import { BaseColumnFieldOptions, Field } from '../field';
+
+export interface EncryptionFieldOptions extends BaseColumnFieldOptions {
+  type: 'encryption';
+  hidden?: boolean;
+}
+
+export class EncryptionField extends Field {
+  get dataType() {
+    return DataTypes.STRING;
+  }
+
+  init() {
+    aesCheckKey();
+    const { name, iv } = this.options;
+    this.writeListener = async (model: Model) => {
+      aesCheckKey();
+      if (!model.changed(name as any)) {
+        return;
+      }
+      const value = model.get(name) as string;
+      if (value !== undefined && value !== null) {
+        try {
+          const encrypted = await aesEncrypt(value, iv);
+          model.set(name, encrypted);
+        } catch (error) {
+          console.error(error);
+          if (error instanceof EncryptionError) {
+            throw error;
+          } else {
+            throw new EncryptionError('Encryption failed');
+          }
+        }
+      } else {
+        model.set(name, null);
+      }
+    };
+
+    this.findListener = async (instances, options) => {
+      aesCheckKey();
+      instances = Array.isArray(instances) ? instances : [instances];
+      await Promise.all(
+        instances.map(async (instance) => {
+          const value = instance.get?.(name);
+          if (value !== undefined && value !== null) {
+            try {
+              instance.set(name, await aesDecrypt(value, iv));
+            } catch (error) {
+              console.error(error);
+              if (error instanceof EncryptionError) {
+                throw error;
+              } else {
+                throw new EncryptionError(
+                  'Decryption failed, the environment variable `ENCRYPTION_FIELD_KEY` may be incorrect',
+                );
+              }
+            }
+          }
+          return instance;
+        }),
+      );
+    };
+  }
+
+  bind() {
+    super.bind();
+    // @ts-ignore
+    this.on('afterFind', this.findListener);
+    this.on('beforeSave', this.writeListener);
+  }
+
+  unbind() {
+    super.unbind();
+    this.off('afterFind', this.findListener);
+    this.off('beforeSave', this.writeListener);
+  }
+}

packages/core/database/src/fields/encryption-field/errors/EncryptionError.ts

@@ -0,0 +1,15 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export class EncryptionError extends Error {
+  constructor(message?: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = 'EncryptionError';
+  }
+}

packages/core/database/src/fields/encryption-field/index.ts

@@ -0,0 +1,12 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export * from './encryption-field';
+export * from './utils';
+export * from './errors/EncryptionError';

packages/core/database/src/fields/encryption-field/utils.ts

@@ -0,0 +1,126 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import crypto from 'crypto';
+import { EncryptionError } from './errors/EncryptionError';
+const algorithm = 'aes-256-cbc';
+
+const keyString = process.env.ENCRYPTION_FIELD_KEY || '';
+const defaultIvString = process.env.ENCRYPTION_FIELD_IV || 'Vc53-4G(rTi0vg@a'; // 如果没有设置 IV，使用默认值
+
+// 将字符串转换为 Buffer 对象
+const key = Buffer.from(keyString, 'utf8');
+
+export function aesEncrypt(text: string, ivString: string = defaultIvString) {
+  checkValueAndIv('Encrypt', text, ivString);
+
+  return new Promise((resolve, reject) => {
+    const iv = Buffer.from(ivString, 'utf8');
+    const cipher = crypto.createCipheriv(algorithm, key, iv);
+
+    let encrypted = '';
+
+    cipher.setEncoding('hex');
+
+    cipher.on('data', (chunk) => {
+      encrypted += chunk;
+    });
+
+    cipher.on('end', () => {
+      resolve(encrypted);
+    });
+
+    cipher.on('error', (err) => {
+      reject(err);
+    });
+
+    cipher.write(text);
+    cipher.end();
+  });
+}
+
+export function aesDecrypt(encrypted: string, ivString: string = defaultIvString) {
+  checkValueAndIv('Decrypt', encrypted, ivString);
+
+  return new Promise((resolve, reject) => {
+    const iv = Buffer.from(ivString, 'utf8');
+    const decipher = crypto.createDecipheriv(algorithm, key, iv);
+
+    let decrypted = '';
+
+    decipher.setEncoding('utf8');
+
+    decipher.on('data', (chunk) => {
+      decrypted += chunk;
+    });
+
+    decipher.on('end', () => {
+      resolve(decrypted);
+    });
+
+    decipher.on('error', (err) => {
+      reject(err);
+    });
+
+    decipher.write(encrypted, 'hex');
+    decipher.end();
+  });
+}
+
+export function aesEncryptSync(text: string, ivString: string = defaultIvString) {
+  checkValueAndIv('Encrypt', text, ivString);
+
+  const iv = Buffer.from(ivString, 'utf8');
+  const cipher = crypto.createCipheriv(algorithm, key, iv);
+  let encrypted = cipher.update(text, 'utf8', 'hex');
+  encrypted += cipher.final('hex');
+  return encrypted;
+}
+
+export function aseDecryptSync(encrypted: string, ivString: string = defaultIvString) {
+  checkValueAndIv('Decrypt', encrypted, ivString);
+
+  const iv = Buffer.from(ivString, 'utf8');
+  const decipher = crypto.createDecipheriv(algorithm, key, iv);
+  let decrypted = decipher.update(encrypted, 'hex', 'utf8');
+  decrypted += decipher.final('utf8');
+  return decrypted;
+}
+
+export function aesCheckKey() {
+  if (!keyString) {
+    throw new EncryptionError('The environment variable `ENCRYPTION_FIELD_KEY` is required, please set it');
+  }
+  if (typeof keyString !== 'string') {
+    throw new EncryptionError('The environment variable `ENCRYPTION_FIELD_KEY` must be a string');
+  }
+  if (keyString.length !== 32) {
+    throw new EncryptionError('The environment variable `ENCRYPTION_FIELD_KEY` must be a 32-character string');
+  }
+}
+
+export function checkValueAndIv(type: 'Decrypt' | 'Encrypt', value: string, iv: string) {
+  const msg = `${type} Failed: `;
+  if (typeof value !== 'string') {
+    throw new EncryptionError(msg + 'The value must be a string, but got ' + typeof value);
+  }
+
+  if (type === 'Decrypt') {
+    if (value.length % 2 !== 0) {
+      throw new EncryptionError(msg + `The encrypted value is invalid, not a hex string. The value is "${value}"`);
+    }
+  }
+
+  if (typeof iv !== 'string') {
+    throw new EncryptionError(msg + 'The `iv` must be a string, but got ' + typeof iv);
+  }
+
+  if (iv.length !== 16) {
+    throw new EncryptionError(msg + 'The `iv` must be a 16-character string');
+  }
+}

packages/core/database/src/fields/index.ts

@@ -35,6 +35,7 @@ import { UidFieldOptions } from './uid-field';
 import { UUIDFieldOptions } from './uuid-field';
 import { VirtualFieldOptions } from './virtual-field';
 import { NanoidFieldOptions } from './nanoid-field';
+import { EncryptionField } from './encryption-field';
 
 export * from './array-field';
 export * from './belongs-to-field';
@@ -59,6 +60,7 @@ export * from './uid-field';
 export * from './uuid-field';
 export * from './virtual-field';
 export * from './nanoid-field';
+export * from './encryption-field';
 
 export type FieldOptions =
   | BaseFieldOptions
@@ -87,4 +89,5 @@ export type FieldOptions =
   | BelongsToFieldOptions
   | HasOneFieldOptions
   | HasManyFieldOptions
-  | BelongsToManyFieldOptions;
+  | BelongsToManyFieldOptions
+  | EncryptionField;

packages/core/database/src/model-hook.ts

@@ -36,7 +36,10 @@ export class ModelHook {
   }
 
   findModelName(hookArgs) {
-    for (const arg of hookArgs) {
+    for (let arg of hookArgs) {
+      if (Array.isArray(arg)) {
+        arg = arg[0];
+      }
       if (arg?._previousDataValues) {
         return (<Model>arg).constructor.name;
       }

packages/core/database/src/view/field-type-map.ts

@@ -8,9 +8,9 @@
  */
 
 const postgres = {
-  'character varying': ['string', 'uuid', 'nanoid'],
-  varchar: ['string', 'uuid', 'nanoid'],
-  char: ['string', 'uuid', 'nanoid'],
+  'character varying': ['string', 'uuid', 'nanoid', 'encryption'],
+  varchar: ['string', 'uuid', 'nanoid', 'encryption'],
+  char: ['string', 'uuid', 'nanoid', 'encryption'],
 
   character: 'string',
   text: 'text',
@@ -53,8 +53,8 @@ const mysql = {
   'tinyint unsigned': ['integer', 'boolean', 'sort'],
   'mediumint unsigned': ['integer', 'boolean', 'sort'],
 
-  char: ['string', 'uuid', 'nanoid'],
-  varchar: ['string', 'uuid', 'nanoid'],
+  char: ['string', 'uuid', 'nanoid', 'encryption'],
+  varchar: ['string', 'uuid', 'nanoid', 'encryption'],
   date: 'date',
   time: 'time',
   tinytext: 'text',
@@ -79,7 +79,7 @@ const mysql = {
 
 const sqlite = {
   text: 'text',
-  varchar: ['string', 'uuid', 'nanoid'],
+  varchar: ['string', 'uuid', 'nanoid', 'encryption'],
 
   integer: 'integer',
   real: 'real',