From 5521d768ffe40fdb5a42fc0e4c77d21305409966 Mon Sep 17 00:00:00 2001 From: Luoyy <10894778+zishang520@users.noreply.github.com> Date: Wed, 27 Nov 2024 21:09:42 +0800 Subject: [PATCH] feat(cmd/gf): add `typeMapping` and `fieldMapping` feature support for command `gf gen genpbentity` (#3970) --- .../cmd/cmd_z_unit_gen_pbentity_test.go | 77 ++++++ .../internal/cmd/genpbentity/genpbentity.go | 224 ++++++++++++++---- .../genpbentity/generated/table_user.proto | 14 +- .../table_user_snake_screaming.proto | 14 +- .../cmd/testdata/issue/3545/table_user.proto | 14 +- .../cmd/testdata/issue/3685/table_user.proto | 23 ++ .../cmd/testdata/issue/3685/user.tpl.sql | 11 + 7 files changed, 307 insertions(+), 70 deletions(-) create mode 100644 cmd/gf/internal/cmd/testdata/issue/3685/table_user.proto create mode 100644 cmd/gf/internal/cmd/testdata/issue/3685/user.tpl.sql diff --git a/cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go b/cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go index 4a105697d..851dc2a7e 100644 --- a/cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go +++ b/cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go @@ -52,6 +52,8 @@ func Test_Gen_Pbentity_Default(t *testing.T) { NameCase: "", JsonCase: "", Option: "", + TypeMapping: nil, + FieldMapping: nil, } ) err = gutil.FillStructWithDefault(&in) @@ -115,6 +117,8 @@ func Test_Gen_Pbentity_NameCase_SnakeScreaming(t *testing.T) { NameCase: "SnakeScreaming", JsonCase: "", Option: "", + TypeMapping: nil, + FieldMapping: nil, } ) err = gutil.FillStructWithDefault(&in) @@ -179,6 +183,8 @@ func Test_Issue_3545(t *testing.T) { NameCase: "", JsonCase: "", Option: "", + TypeMapping: nil, + FieldMapping: nil, } ) err = gutil.FillStructWithDefault(&in) @@ -208,3 +214,74 @@ func Test_Issue_3545(t *testing.T) { } }) } + +// https://github.com/gogf/gf/issues/3685 +func Test_Issue_3685(t *testing.T) { + gtest.C(t, func(t *gtest.T) { + var ( + err error + db = testDB + table = "table_user" + sqlContent = fmt.Sprintf( + gtest.DataContent(`issue`, `3685`, `user.tpl.sql`), + table, + ) + ) + dropTableWithDb(db, table) + array := gstr.SplitAndTrim(sqlContent, ";") + for _, v := range array { + if _, err = db.Exec(ctx, v); err != nil { + t.AssertNil(err) + } + } + defer dropTableWithDb(db, table) + + var ( + path = gfile.Temp(guid.S()) + in = genpbentity.CGenPbEntityInput{ + Path: path, + Package: "", + Link: link, + Tables: "", + Prefix: "", + RemovePrefix: "", + RemoveFieldPrefix: "", + NameCase: "", + JsonCase: "", + Option: "", + TypeMapping: map[genpbentity.DBFieldTypeName]genpbentity.CustomAttributeType{ + "json": { + Type: "google.protobuf.Value", + Import: "google/protobuf/struct.proto", + }, + }, + FieldMapping: nil, + } + ) + err = gutil.FillStructWithDefault(&in) + t.AssertNil(err) + + err = gfile.Mkdir(path) + t.AssertNil(err) + defer gfile.Remove(path) + + _, err = genpbentity.CGenPbEntity{}.PbEntity(ctx, in) + t.AssertNil(err) + + // files + files, err := gfile.ScanDir(path, "*.proto", false) + t.AssertNil(err) + t.Assert(files, []string{ + path + filepath.FromSlash("/table_user.proto"), + }) + + // contents + testPath := gtest.DataPath("issue", "3685") + expectFiles := []string{ + testPath + filepath.FromSlash("/table_user.proto"), + } + for i := range files { + t.Assert(gfile.GetContents(files[i]), gfile.GetContents(expectFiles[i])) + } + }) +} diff --git a/cmd/gf/internal/cmd/genpbentity/genpbentity.go b/cmd/gf/internal/cmd/genpbentity/genpbentity.go index 12084671b..29a660b08 100644 --- a/cmd/gf/internal/cmd/genpbentity/genpbentity.go +++ b/cmd/gf/internal/cmd/genpbentity/genpbentity.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "path/filepath" + "regexp" "strings" "github.com/gogf/gf/cmd/gf/v2/internal/utility/utils" @@ -18,6 +19,7 @@ import ( "github.com/gogf/gf/cmd/gf/v2/internal/consts" "github.com/gogf/gf/cmd/gf/v2/internal/utility/mlog" + "github.com/gogf/gf/v2/container/garray" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" @@ -43,6 +45,9 @@ type ( NameCase string `name:"nameCase" short:"n" brief:"{CGenPbEntityBriefNameCase}" d:"Camel"` JsonCase string `name:"jsonCase" short:"j" brief:"{CGenPbEntityBriefJsonCase}" d:"none"` Option string `name:"option" short:"o" brief:"{CGenPbEntityBriefOption}"` + + TypeMapping map[DBFieldTypeName]CustomAttributeType `name:"typeMapping" short:"y" brief:"{CGenPbEntityBriefTypeMapping}" orphan:"true"` + FieldMapping map[DBTableFieldName]CustomAttributeType `name:"fieldMapping" short:"fm" brief:"{CGenPbEntityBriefFieldMapping}" orphan:"true"` } CGenPbEntityOutput struct{} @@ -52,6 +57,13 @@ type ( TableName string // TableName specifies the table name of the table. NewTableName string // NewTableName specifies the prefix-stripped name of the table. } + + DBTableFieldName = string + DBFieldTypeName = string + CustomAttributeType struct { + Type string `brief:"custom attribute type name"` + Import string `brief:"custom import for this type"` + } ) const ( @@ -69,7 +81,7 @@ gf gen pbentity -r user_ CGenPbEntityAd = ` CONFIGURATION SUPPORT Options are also supported by configuration file. - It's suggested using configuration file instead of command line arguments making producing. + It's suggested using configuration file instead of command line arguments making producing. The configuration node name is "gf.gen.pbentity", which also supports multiple databases, for example(config.yaml): gfcli: gen: @@ -88,6 +100,13 @@ CONFIGURATION SUPPORT option go_package = "protobuf/demos"; option java_package = "protobuf/demos"; option php_namespace = "protobuf/demos"; + typeMapping: + json: + type: google.protobuf.Value + import: google/protobuf/struct.proto + jsonb: + type: google.protobuf.Value + import: google/protobuf/struct.proto ` CGenPbEntityBriefPath = `directory path for generated files storing` CGenPbEntityBriefPackage = `package path for all entity proto files` @@ -106,7 +125,7 @@ it's not necessary and the default value is "default" case for message attribute names, default is "Camel": | Case | Example | |---------------- |--------------------| -| Camel | AnyKindOfString | +| Camel | AnyKindOfString | | CamelLower | anyKindOfString | default | Snake | any_kind_of_string | | SnakeScreaming | ANY_KIND_OF_STRING | @@ -119,8 +138,95 @@ case for message attribute names, default is "Camel": case for message json tag, cases are the same as "nameCase", default "CamelLower". set it to "none" to ignore json tag generating. ` + + CGenPbEntityBriefTypeMapping = `custom local type mapping for generated struct attributes relevant to fields of table` + CGenPbEntityBriefFieldMapping = `custom local type mapping for generated struct attributes relevant to specific fields of table` ) +var defaultTypeMapping = map[DBFieldTypeName]CustomAttributeType{ + // gdb.LocalTypeString + "string": { + Type: "string", + }, + // gdb.LocalTypeTime + // "time": { + // Type: "google.protobuf.Duration", + // Import: "google/protobuf/duration.proto", + // }, + // gdb.LocalTypeDate + "date": { + Type: "google.protobuf.Timestamp", + Import: "google/protobuf/timestamp.proto", + }, + // gdb.LocalTypeDatetime + "datetime": { + Type: "google.protobuf.Timestamp", + Import: "google/protobuf/timestamp.proto", + }, + // gdb.LocalTypeInt + "int": { + Type: "int32", + }, + // gdb.LocalTypeUint + "uint": { + Type: "uint32", + }, + // gdb.LocalTypeInt64 + "int64": { + Type: "int64", + }, + // gdb.LocalTypeUint64 + "uint64": { + Type: "uint64", + }, + // gdb.LocalTypeIntSlice + "[]int": { + Type: "repeated int32", + }, + // gdb.LocalTypeInt64Slice + "[]int64": { + Type: "repeated int64", + }, + // gdb.LocalTypeUint64Slice + "[]uint64": { + Type: "repeated uint64", + }, + // gdb.LocalTypeInt64Bytes + "int64-bytes": { + Type: "repeated int64", + }, + // gdb.LocalTypeUint64Bytes + "uint64-bytes": { + Type: "repeated uint64", + }, + // gdb.LocalTypeFloat32 + "float32": { + Type: "float", + }, + // gdb.LocalTypeFloat64 + "float64": { + Type: "double", + }, + // gdb.LocalTypeBytes + "[]byte": { + Type: "bytes", + }, + // gdb.LocalTypeBool + "bool": { + Type: "bool", + }, + // gdb.LocalTypeJson + // "json": { + // Type: "google.protobuf.Value", + // Import: "google/protobuf/struct.proto", + // }, + // gdb.LocalTypeJsonb + // "jsonb": { + // Type: "google.protobuf.Value", + // Import: "google/protobuf/struct.proto", + // }, +} + func init() { gtag.Sets(g.MapStrStr{ `CGenPbEntityConfig`: CGenPbEntityConfig, @@ -138,6 +244,8 @@ func init() { `CGenPbEntityBriefNameCase`: CGenPbEntityBriefNameCase, `CGenPbEntityBriefJsonCase`: CGenPbEntityBriefJsonCase, `CGenPbEntityBriefOption`: CGenPbEntityBriefOption, + `CGenPbEntityBriefTypeMapping`: CGenPbEntityBriefTypeMapping, + `CGenPbEntityBriefFieldMapping`: CGenPbEntityBriefFieldMapping, }) } @@ -210,6 +318,16 @@ func doGenPbEntityForArray(ctx context.Context, index int, in CGenPbEntityInput) mlog.Fatalf("fetching tables failed: \n %v", err) } } + // merge default typeMapping to input typeMapping. + if in.TypeMapping == nil { + in.TypeMapping = defaultTypeMapping + } else { + for key, typeMapping := range defaultTypeMapping { + if _, ok := in.TypeMapping[key]; !ok { + in.TypeMapping[key] = typeMapping + } + } + } for _, tableName := range tableNames { newTableName := tableName @@ -234,18 +352,24 @@ func generatePbEntityContentFile(ctx context.Context, in CGenPbEntityInternalInp // Change the `newTableName` if `Prefix` is given. newTableName := in.Prefix + in.NewTableName var ( - imports string - tableNameCamelCase = gstr.CaseCamel(newTableName) - tableNameSnakeCase = gstr.CaseSnake(newTableName) - entityMessageDefine = generateEntityMessageDefinition(tableNameCamelCase, fieldMap, in) - fileName = gstr.Trim(tableNameSnakeCase, "-_.") - path = filepath.FromSlash(gfile.Join(in.Path, fileName+".proto")) + tableNameCamelCase = gstr.CaseCamel(newTableName) + tableNameSnakeCase = gstr.CaseSnake(newTableName) + entityMessageDefine, appendImports = generateEntityMessageDefinition(tableNameCamelCase, fieldMap, in) + fileName = gstr.Trim(tableNameSnakeCase, "-_.") + path = filepath.FromSlash(gfile.Join(in.Path, fileName+".proto")) ) - if gstr.Contains(entityMessageDefine, "google.protobuf.Timestamp") { - imports = `import "google/protobuf/timestamp.proto";` + packageImportStr := "" + var packageImportsArray = garray.NewStrArray() + if len(appendImports) > 0 { + for _, appendImport := range appendImports { + packageImportStr = fmt.Sprintf(`import "%s";`, appendImport) + if packageImportsArray.Search(packageImportStr) == -1 { + packageImportsArray.Append(packageImportStr) + } + } } entityContent := gstr.ReplaceByMap(getTplPbEntityContent(""), g.MapStrStr{ - "{Imports}": imports, + "{Imports}": packageImportsArray.Join("\n"), "{PackageName}": gfile.Basename(in.Package), "{GoPackage}": in.Package, "{OptionContent}": in.Option, @@ -259,14 +383,19 @@ func generatePbEntityContentFile(ctx context.Context, in CGenPbEntityInternalInp } // generateEntityMessageDefinition generates and returns the message definition for specified table. -func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb.TableField, in CGenPbEntityInternalInput) string { +func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb.TableField, in CGenPbEntityInternalInput) (string, []string) { var ( - buffer = bytes.NewBuffer(nil) - array = make([][]string, len(fieldMap)) - names = sortFieldKeyForPbEntity(fieldMap) + appendImports []string + buffer = bytes.NewBuffer(nil) + array = make([][]string, len(fieldMap)) + names = sortFieldKeyForPbEntity(fieldMap) ) for index, name := range names { - array[index] = generateMessageFieldForPbEntity(index+1, fieldMap[name], in) + var imports string + array[index], imports = generateMessageFieldForPbEntity(index+1, fieldMap[name], in) + if imports != "" { + appendImports = append(appendImports, imports) + } } tw := tablewriter.NewWriter(buffer) tw.SetBorder(false) @@ -277,48 +406,38 @@ func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb tw.Render() stContent := buffer.String() // Let's do this hack of table writer for indent! - stContent = gstr.Replace(stContent, " #", "") + stContent = regexp.MustCompile(`\s+\n`).ReplaceAllString(gstr.Replace(stContent, " #", ""), "\n") buffer.Reset() buffer.WriteString(fmt.Sprintf("message %s {\n", entityName)) buffer.WriteString(stContent) buffer.WriteString("}") - return buffer.String() + return buffer.String(), appendImports } // generateMessageFieldForPbEntity generates and returns the message definition for specified field. -func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPbEntityInternalInput) []string { +func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPbEntityInternalInput) (attrLines []string, appendImport string) { var ( - localTypeName gdb.LocalType - comment string - jsonTagStr string - err error - ctx = gctx.GetInitCtx() + localTypeNameStr string + localTypeName gdb.LocalType + comment string + jsonTagStr string + err error + ctx = gctx.GetInitCtx() ) - localTypeName, err = in.DB.CheckLocalTypeForField(ctx, field.Type, nil) - if err != nil { - panic(err) + + if in.TypeMapping != nil && len(in.TypeMapping) > 0 { + localTypeName, err = in.DB.CheckLocalTypeForField(ctx, field.Type, nil) + if err != nil { + panic(err) + } + if localTypeName != "" { + if typeMapping, ok := in.TypeMapping[strings.ToLower(string(localTypeName))]; ok { + localTypeNameStr = typeMapping.Type + appendImport = typeMapping.Import + } + } } - var typeMapping = map[gdb.LocalType]string{ - gdb.LocalTypeString: "string", - gdb.LocalTypeDate: "google.protobuf.Timestamp", - gdb.LocalTypeDatetime: "google.protobuf.Timestamp", - gdb.LocalTypeInt: "int32", - gdb.LocalTypeUint: "uint32", - gdb.LocalTypeInt64: "int64", - gdb.LocalTypeUint64: "uint64", - gdb.LocalTypeIntSlice: "repeated int32", - gdb.LocalTypeInt64Slice: "repeated int64", - gdb.LocalTypeUint64Slice: "repeated uint64", - gdb.LocalTypeInt64Bytes: "repeated int64", - gdb.LocalTypeUint64Bytes: "repeated uint64", - gdb.LocalTypeFloat32: "float", - gdb.LocalTypeFloat64: "double", - gdb.LocalTypeBytes: "bytes", - gdb.LocalTypeBool: "bool", - gdb.LocalTypeJson: "string", - gdb.LocalTypeJsonb: "string", - } - localTypeNameStr := typeMapping[localTypeName] + if localTypeNameStr == "" { localTypeNameStr = "string" } @@ -351,12 +470,19 @@ func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPb newFiledName = gstr.TrimLeftStr(newFiledName, v, 1) } + if in.FieldMapping != nil && len(in.FieldMapping) > 0 { + if typeMapping, ok := in.FieldMapping[fmt.Sprintf("%s.%s", in.TableName, newFiledName)]; ok { + localTypeNameStr = typeMapping.Type + appendImport = typeMapping.Import + } + } + return []string{ " #" + localTypeNameStr, " #" + formatCase(newFiledName, in.NameCase), " #= " + gconv.String(index) + jsonTagStr + ";", " #" + fmt.Sprintf(`// %s`, comment), - } + }, appendImport } func getTplPbEntityContent(tplEntityPath string) string { diff --git a/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user.proto b/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user.proto index e5074b491..874a55a05 100644 --- a/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user.proto +++ b/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user.proto @@ -11,11 +11,11 @@ option go_package = "unittest"; import "google/protobuf/timestamp.proto"; message TableUser { - uint32 Id = 1; // User ID - string Passport = 2; // User Passport - string Password = 3; // User Password - string Nickname = 4; // User Nickname - string Score = 5; // Total score amount. - google.protobuf.Timestamp CreateAt = 6; // Created Time - google.protobuf.Timestamp UpdateAt = 7; // Updated Time + uint32 Id = 1; // User ID + string Passport = 2; // User Passport + string Password = 3; // User Password + string Nickname = 4; // User Nickname + string Score = 5; // Total score amount. + google.protobuf.Timestamp CreateAt = 6; // Created Time + google.protobuf.Timestamp UpdateAt = 7; // Updated Time } \ No newline at end of file diff --git a/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user_snake_screaming.proto b/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user_snake_screaming.proto index 365999ece..9613131da 100644 --- a/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user_snake_screaming.proto +++ b/cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user_snake_screaming.proto @@ -11,11 +11,11 @@ option go_package = "unittest"; import "google/protobuf/timestamp.proto"; message TableUser { - uint32 ID = 1; // User ID - string PASSPORT = 2; // User Passport - string PASSWORD = 3; // User Password - string NICKNAME = 4; // User Nickname - string SCORE = 5; // Total score amount. - google.protobuf.Timestamp CREATE_AT = 6; // Created Time - google.protobuf.Timestamp UPDATE_AT = 7; // Updated Time + uint32 ID = 1; // User ID + string PASSPORT = 2; // User Passport + string PASSWORD = 3; // User Password + string NICKNAME = 4; // User Nickname + string SCORE = 5; // Total score amount. + google.protobuf.Timestamp CREATE_AT = 6; // Created Time + google.protobuf.Timestamp UPDATE_AT = 7; // Updated Time } \ No newline at end of file diff --git a/cmd/gf/internal/cmd/testdata/issue/3545/table_user.proto b/cmd/gf/internal/cmd/testdata/issue/3545/table_user.proto index 6282f3a61..2b95b4056 100644 --- a/cmd/gf/internal/cmd/testdata/issue/3545/table_user.proto +++ b/cmd/gf/internal/cmd/testdata/issue/3545/table_user.proto @@ -11,11 +11,11 @@ option go_package = "github.com/gogf/gf/cmd/gf/v2/internal/cmd/api/pbentity"; import "google/protobuf/timestamp.proto"; message TableUser { - uint32 Id = 1; // User ID - string Passport = 2; // User Passport - string Password = 3; // User Password - string Nickname = 4; // User Nickname - string Score = 5; // Total score amount. - google.protobuf.Timestamp CreateAt = 6; // Created Time - google.protobuf.Timestamp UpdateAt = 7; // Updated Time + uint32 Id = 1; // User ID + string Passport = 2; // User Passport + string Password = 3; // User Password + string Nickname = 4; // User Nickname + string Score = 5; // Total score amount. + google.protobuf.Timestamp CreateAt = 6; // Created Time + google.protobuf.Timestamp UpdateAt = 7; // Updated Time } \ No newline at end of file diff --git a/cmd/gf/internal/cmd/testdata/issue/3685/table_user.proto b/cmd/gf/internal/cmd/testdata/issue/3685/table_user.proto new file mode 100644 index 000000000..803cfa81f --- /dev/null +++ b/cmd/gf/internal/cmd/testdata/issue/3685/table_user.proto @@ -0,0 +1,23 @@ +// ========================================================================== +// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. +// ========================================================================== + +syntax = "proto3"; + +package pbentity; + +option go_package = "github.com/gogf/gf/cmd/gf/v2/internal/cmd/api/pbentity"; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +message TableUser { + uint32 Id = 1; // User ID + string Passport = 2; // User Passport + string Password = 3; // User Password + string Nickname = 4; // User Nickname + string Score = 5; // Total score amount. + google.protobuf.Value Data = 6; // User Data + google.protobuf.Timestamp CreateAt = 7; // Created Time + google.protobuf.Timestamp UpdateAt = 8; // Updated Time +} \ No newline at end of file diff --git a/cmd/gf/internal/cmd/testdata/issue/3685/user.tpl.sql b/cmd/gf/internal/cmd/testdata/issue/3685/user.tpl.sql new file mode 100644 index 000000000..87493a6f8 --- /dev/null +++ b/cmd/gf/internal/cmd/testdata/issue/3685/user.tpl.sql @@ -0,0 +1,11 @@ +CREATE TABLE `%s` ( + `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'User ID', + `passport` varchar(45) NOT NULL COMMENT 'User Passport', + `password` varchar(45) NOT NULL COMMENT 'User Password', + `nickname` varchar(45) NOT NULL COMMENT 'User Nickname', + `score` decimal(10,2) unsigned DEFAULT NULL COMMENT 'Total score amount.', + `data` json DEFAULT NULL COMMENT 'User Data', + `create_at` datetime DEFAULT NULL COMMENT 'Created Time', + `update_at` datetime DEFAULT NULL COMMENT 'Updated Time', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; \ No newline at end of file