feat(cmd/gf): add typeMapping and fieldMapping feature support for command gf gen genpbentity (#3970)

cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go

@@ -52,6 +52,8 @@ func Test_Gen_Pbentity_Default(t *testing.T) {
 				NameCase:          "",
 				JsonCase:          "",
 				Option:            "",
+				TypeMapping:       nil,
+				FieldMapping:      nil,
 			}
 		)
 		err = gutil.FillStructWithDefault(&in)
@@ -115,6 +117,8 @@ func Test_Gen_Pbentity_NameCase_SnakeScreaming(t *testing.T) {
 				NameCase:          "SnakeScreaming",
 				JsonCase:          "",
 				Option:            "",
+				TypeMapping:       nil,
+				FieldMapping:      nil,
 			}
 		)
 		err = gutil.FillStructWithDefault(&in)
@@ -179,6 +183,8 @@ func Test_Issue_3545(t *testing.T) {
 				NameCase:          "",
 				JsonCase:          "",
 				Option:            "",
+				TypeMapping:       nil,
+				FieldMapping:      nil,
 			}
 		)
 		err = gutil.FillStructWithDefault(&in)
@@ -208,3 +214,74 @@ func Test_Issue_3545(t *testing.T) {
 		}
 	})
 }
+
+// https://github.com/gogf/gf/issues/3685
+func Test_Issue_3685(t *testing.T) {
+	gtest.C(t, func(t *gtest.T) {
+		var (
+			err        error
+			db         = testDB
+			table      = "table_user"
+			sqlContent = fmt.Sprintf(
+				gtest.DataContent(`issue`, `3685`, `user.tpl.sql`),
+				table,
+			)
+		)
+		dropTableWithDb(db, table)
+		array := gstr.SplitAndTrim(sqlContent, ";")
+		for _, v := range array {
+			if _, err = db.Exec(ctx, v); err != nil {
+				t.AssertNil(err)
+			}
+		}
+		defer dropTableWithDb(db, table)
+
+		var (
+			path = gfile.Temp(guid.S())
+			in   = genpbentity.CGenPbEntityInput{
+				Path:              path,
+				Package:           "",
+				Link:              link,
+				Tables:            "",
+				Prefix:            "",
+				RemovePrefix:      "",
+				RemoveFieldPrefix: "",
+				NameCase:          "",
+				JsonCase:          "",
+				Option:            "",
+				TypeMapping: map[genpbentity.DBFieldTypeName]genpbentity.CustomAttributeType{
+					"json": {
+						Type:   "google.protobuf.Value",
+						Import: "google/protobuf/struct.proto",
+					},
+				},
+				FieldMapping: nil,
+			}
+		)
+		err = gutil.FillStructWithDefault(&in)
+		t.AssertNil(err)
+
+		err = gfile.Mkdir(path)
+		t.AssertNil(err)
+		defer gfile.Remove(path)
+
+		_, err = genpbentity.CGenPbEntity{}.PbEntity(ctx, in)
+		t.AssertNil(err)
+
+		// files
+		files, err := gfile.ScanDir(path, "*.proto", false)
+		t.AssertNil(err)
+		t.Assert(files, []string{
+			path + filepath.FromSlash("/table_user.proto"),
+		})
+
+		// contents
+		testPath := gtest.DataPath("issue", "3685")
+		expectFiles := []string{
+			testPath + filepath.FromSlash("/table_user.proto"),
+		}
+		for i := range files {
+			t.Assert(gfile.GetContents(files[i]), gfile.GetContents(expectFiles[i]))
+		}
+	})
+}

cmd/gf/internal/cmd/genpbentity/genpbentity.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/gogf/gf/cmd/gf/v2/internal/utility/utils"
@@ -18,6 +19,7 @@ import (
 
 	"github.com/gogf/gf/cmd/gf/v2/internal/consts"
 	"github.com/gogf/gf/cmd/gf/v2/internal/utility/mlog"
+	"github.com/gogf/gf/v2/container/garray"
 	"github.com/gogf/gf/v2/database/gdb"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gctx"
@@ -43,6 +45,9 @@ type (
 		NameCase          string `name:"nameCase"          short:"n"  brief:"{CGenPbEntityBriefNameCase}" d:"Camel"`
 		JsonCase          string `name:"jsonCase"          short:"j"  brief:"{CGenPbEntityBriefJsonCase}" d:"none"`
 		Option            string `name:"option"            short:"o"  brief:"{CGenPbEntityBriefOption}"`
+
+		TypeMapping  map[DBFieldTypeName]CustomAttributeType  `name:"typeMapping"  short:"y"  brief:"{CGenPbEntityBriefTypeMapping}"  orphan:"true"`
+		FieldMapping map[DBTableFieldName]CustomAttributeType `name:"fieldMapping" short:"fm" brief:"{CGenPbEntityBriefFieldMapping}" orphan:"true"`
 	}
 	CGenPbEntityOutput struct{}
 
@@ -52,6 +57,13 @@ type (
 		TableName    string // TableName specifies the table name of the table.
 		NewTableName string // NewTableName specifies the prefix-stripped name of the table.
 	}
+
+	DBTableFieldName    = string
+	DBFieldTypeName     = string
+	CustomAttributeType struct {
+		Type   string `brief:"custom attribute type name"`
+		Import string `brief:"custom import for this type"`
+	}
 )
 
 const (
@@ -69,7 +81,7 @@ gf gen pbentity -r user_
 	CGenPbEntityAd = `
 CONFIGURATION SUPPORT
     Options are also supported by configuration file.
-    It's suggested using configuration file instead of command line arguments making producing. 
+    It's suggested using configuration file instead of command line arguments making producing.
     The configuration node name is "gf.gen.pbentity", which also supports multiple databases, for example(config.yaml):
     gfcli:
       gen:
@@ -88,6 +100,13 @@ CONFIGURATION SUPPORT
 			  option go_package    = "protobuf/demos";
 			  option java_package  = "protobuf/demos";
 			  option php_namespace = "protobuf/demos";
+            typeMapping:
+              json:
+                type: google.protobuf.Value
+                import: google/protobuf/struct.proto
+              jsonb:
+                type: google.protobuf.Value
+                import: google/protobuf/struct.proto
 `
 	CGenPbEntityBriefPath              = `directory path for generated files storing`
 	CGenPbEntityBriefPackage           = `package path for all entity proto files`
@@ -106,7 +125,7 @@ it's not necessary and the default value is "default"
 case for message attribute names, default is "Camel":
 | Case            | Example            |
 |---------------- |--------------------|
-| Camel           | AnyKindOfString    | 
+| Camel           | AnyKindOfString    |
 | CamelLower      | anyKindOfString    | default
 | Snake           | any_kind_of_string |
 | SnakeScreaming  | ANY_KIND_OF_STRING |
@@ -119,8 +138,95 @@ case for message attribute names, default is "Camel":
 case for message json tag, cases are the same as "nameCase", default "CamelLower".
 set it to "none" to ignore json tag generating.
 `
+
+	CGenPbEntityBriefTypeMapping  = `custom local type mapping for generated struct attributes relevant to fields of table`
+	CGenPbEntityBriefFieldMapping = `custom local type mapping for generated struct attributes relevant to specific fields of table`
 )
 
+var defaultTypeMapping = map[DBFieldTypeName]CustomAttributeType{
+	// gdb.LocalTypeString
+	"string": {
+		Type: "string",
+	},
+	// gdb.LocalTypeTime
+	// "time": {
+	// 	Type:   "google.protobuf.Duration",
+	// 	Import: "google/protobuf/duration.proto",
+	// },
+	// gdb.LocalTypeDate
+	"date": {
+		Type:   "google.protobuf.Timestamp",
+		Import: "google/protobuf/timestamp.proto",
+	},
+	// gdb.LocalTypeDatetime
+	"datetime": {
+		Type:   "google.protobuf.Timestamp",
+		Import: "google/protobuf/timestamp.proto",
+	},
+	// gdb.LocalTypeInt
+	"int": {
+		Type: "int32",
+	},
+	// gdb.LocalTypeUint
+	"uint": {
+		Type: "uint32",
+	},
+	// gdb.LocalTypeInt64
+	"int64": {
+		Type: "int64",
+	},
+	// gdb.LocalTypeUint64
+	"uint64": {
+		Type: "uint64",
+	},
+	// gdb.LocalTypeIntSlice
+	"[]int": {
+		Type: "repeated int32",
+	},
+	// gdb.LocalTypeInt64Slice
+	"[]int64": {
+		Type: "repeated int64",
+	},
+	// gdb.LocalTypeUint64Slice
+	"[]uint64": {
+		Type: "repeated uint64",
+	},
+	// gdb.LocalTypeInt64Bytes
+	"int64-bytes": {
+		Type: "repeated int64",
+	},
+	// gdb.LocalTypeUint64Bytes
+	"uint64-bytes": {
+		Type: "repeated uint64",
+	},
+	// gdb.LocalTypeFloat32
+	"float32": {
+		Type: "float",
+	},
+	// gdb.LocalTypeFloat64
+	"float64": {
+		Type: "double",
+	},
+	// gdb.LocalTypeBytes
+	"[]byte": {
+		Type: "bytes",
+	},
+	// gdb.LocalTypeBool
+	"bool": {
+		Type: "bool",
+	},
+	// gdb.LocalTypeJson
+	// "json": {
+	// 	Type:   "google.protobuf.Value",
+	// 	Import: "google/protobuf/struct.proto",
+	// },
+	// gdb.LocalTypeJsonb
+	// "jsonb": {
+	// 	Type:   "google.protobuf.Value",
+	// 	Import: "google/protobuf/struct.proto",
+	// },
+}
+
 func init() {
 	gtag.Sets(g.MapStrStr{
 		`CGenPbEntityConfig`:                 CGenPbEntityConfig,
@@ -138,6 +244,8 @@ func init() {
 		`CGenPbEntityBriefNameCase`:          CGenPbEntityBriefNameCase,
 		`CGenPbEntityBriefJsonCase`:          CGenPbEntityBriefJsonCase,
 		`CGenPbEntityBriefOption`:            CGenPbEntityBriefOption,
+		`CGenPbEntityBriefTypeMapping`:       CGenPbEntityBriefTypeMapping,
+		`CGenPbEntityBriefFieldMapping`:      CGenPbEntityBriefFieldMapping,
 	})
 }
 
@@ -210,6 +318,16 @@ func doGenPbEntityForArray(ctx context.Context, index int, in CGenPbEntityInput)
 			mlog.Fatalf("fetching tables failed: \n %v", err)
 		}
 	}
+	// merge default typeMapping to input typeMapping.
+	if in.TypeMapping == nil {
+		in.TypeMapping = defaultTypeMapping
+	} else {
+		for key, typeMapping := range defaultTypeMapping {
+			if _, ok := in.TypeMapping[key]; !ok {
+				in.TypeMapping[key] = typeMapping
+			}
+		}
+	}
 
 	for _, tableName := range tableNames {
 		newTableName := tableName
@@ -234,18 +352,24 @@ func generatePbEntityContentFile(ctx context.Context, in CGenPbEntityInternalInp
 	// Change the `newTableName` if `Prefix` is given.
 	newTableName := in.Prefix + in.NewTableName
 	var (
-		imports             string
+		tableNameCamelCase                 = gstr.CaseCamel(newTableName)
-		tableNameCamelCase  = gstr.CaseCamel(newTableName)
+		tableNameSnakeCase                 = gstr.CaseSnake(newTableName)
-		tableNameSnakeCase  = gstr.CaseSnake(newTableName)
+		entityMessageDefine, appendImports = generateEntityMessageDefinition(tableNameCamelCase, fieldMap, in)
-		entityMessageDefine = generateEntityMessageDefinition(tableNameCamelCase, fieldMap, in)
+		fileName                           = gstr.Trim(tableNameSnakeCase, "-_.")
-		fileName            = gstr.Trim(tableNameSnakeCase, "-_.")
+		path                               = filepath.FromSlash(gfile.Join(in.Path, fileName+".proto"))
-		path                = filepath.FromSlash(gfile.Join(in.Path, fileName+".proto"))
 	)
-	if gstr.Contains(entityMessageDefine, "google.protobuf.Timestamp") {
+	packageImportStr := ""
-		imports = `import "google/protobuf/timestamp.proto";`
+	var packageImportsArray = garray.NewStrArray()
+	if len(appendImports) > 0 {
+		for _, appendImport := range appendImports {
+			packageImportStr = fmt.Sprintf(`import "%s";`, appendImport)
+			if packageImportsArray.Search(packageImportStr) == -1 {
+				packageImportsArray.Append(packageImportStr)
+			}
+		}
 	}
 	entityContent := gstr.ReplaceByMap(getTplPbEntityContent(""), g.MapStrStr{
-		"{Imports}":       imports,
+		"{Imports}":       packageImportsArray.Join("\n"),
 		"{PackageName}":   gfile.Basename(in.Package),
 		"{GoPackage}":     in.Package,
 		"{OptionContent}": in.Option,
@@ -259,14 +383,19 @@ func generatePbEntityContentFile(ctx context.Context, in CGenPbEntityInternalInp
 }
 
 // generateEntityMessageDefinition generates and returns the message definition for specified table.
-func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb.TableField, in CGenPbEntityInternalInput) string {
+func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb.TableField, in CGenPbEntityInternalInput) (string, []string) {
 	var (
-		buffer = bytes.NewBuffer(nil)
+		appendImports []string
-		array  = make([][]string, len(fieldMap))
+		buffer        = bytes.NewBuffer(nil)
-		names  = sortFieldKeyForPbEntity(fieldMap)
+		array         = make([][]string, len(fieldMap))
+		names         = sortFieldKeyForPbEntity(fieldMap)
 	)
 	for index, name := range names {
-		array[index] = generateMessageFieldForPbEntity(index+1, fieldMap[name], in)
+		var imports string
+		array[index], imports = generateMessageFieldForPbEntity(index+1, fieldMap[name], in)
+		if imports != "" {
+			appendImports = append(appendImports, imports)
+		}
 	}
 	tw := tablewriter.NewWriter(buffer)
 	tw.SetBorder(false)
@@ -277,48 +406,38 @@ func generateEntityMessageDefinition(entityName string, fieldMap map[string]*gdb
 	tw.Render()
 	stContent := buffer.String()
 	// Let's do this hack of table writer for indent!
-	stContent = gstr.Replace(stContent, "  #", "")
+	stContent = regexp.MustCompile(`\s+\n`).ReplaceAllString(gstr.Replace(stContent, "  #", ""), "\n")
 	buffer.Reset()
 	buffer.WriteString(fmt.Sprintf("message %s {\n", entityName))
 	buffer.WriteString(stContent)
 	buffer.WriteString("}")
-	return buffer.String()
+	return buffer.String(), appendImports
 }
 
 // generateMessageFieldForPbEntity generates and returns the message definition for specified field.
-func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPbEntityInternalInput) []string {
+func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPbEntityInternalInput) (attrLines []string, appendImport string) {
 	var (
-		localTypeName gdb.LocalType
+		localTypeNameStr string
-		comment       string
+		localTypeName    gdb.LocalType
-		jsonTagStr    string
+		comment          string
-		err           error
+		jsonTagStr       string
-		ctx           = gctx.GetInitCtx()
+		err              error
+		ctx              = gctx.GetInitCtx()
 	)
-	localTypeName, err = in.DB.CheckLocalTypeForField(ctx, field.Type, nil)
+
-	if err != nil {
+	if in.TypeMapping != nil && len(in.TypeMapping) > 0 {
-		panic(err)
+		localTypeName, err = in.DB.CheckLocalTypeForField(ctx, field.Type, nil)
+		if err != nil {
+			panic(err)
+		}
+		if localTypeName != "" {
+			if typeMapping, ok := in.TypeMapping[strings.ToLower(string(localTypeName))]; ok {
+				localTypeNameStr = typeMapping.Type
+				appendImport = typeMapping.Import
+			}
+		}
 	}
-	var typeMapping = map[gdb.LocalType]string{
+
-		gdb.LocalTypeString:      "string",
-		gdb.LocalTypeDate:        "google.protobuf.Timestamp",
-		gdb.LocalTypeDatetime:    "google.protobuf.Timestamp",
-		gdb.LocalTypeInt:         "int32",
-		gdb.LocalTypeUint:        "uint32",
-		gdb.LocalTypeInt64:       "int64",
-		gdb.LocalTypeUint64:      "uint64",
-		gdb.LocalTypeIntSlice:    "repeated int32",
-		gdb.LocalTypeInt64Slice:  "repeated int64",
-		gdb.LocalTypeUint64Slice: "repeated uint64",
-		gdb.LocalTypeInt64Bytes:  "repeated int64",
-		gdb.LocalTypeUint64Bytes: "repeated uint64",
-		gdb.LocalTypeFloat32:     "float",
-		gdb.LocalTypeFloat64:     "double",
-		gdb.LocalTypeBytes:       "bytes",
-		gdb.LocalTypeBool:        "bool",
-		gdb.LocalTypeJson:        "string",
-		gdb.LocalTypeJsonb:       "string",
-	}
-	localTypeNameStr := typeMapping[localTypeName]
 	if localTypeNameStr == "" {
 		localTypeNameStr = "string"
 	}
@@ -351,12 +470,19 @@ func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPb
 		newFiledName = gstr.TrimLeftStr(newFiledName, v, 1)
 	}
 
+	if in.FieldMapping != nil && len(in.FieldMapping) > 0 {
+		if typeMapping, ok := in.FieldMapping[fmt.Sprintf("%s.%s", in.TableName, newFiledName)]; ok {
+			localTypeNameStr = typeMapping.Type
+			appendImport = typeMapping.Import
+		}
+	}
+
 	return []string{
 		"    #" + localTypeNameStr,
 		" #" + formatCase(newFiledName, in.NameCase),
 		" #= " + gconv.String(index) + jsonTagStr + ";",
 		" #" + fmt.Sprintf(`// %s`, comment),
-	}
+	}, appendImport
 }
 
 func getTplPbEntityContent(tplEntityPath string) string {

cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user.proto

@@ -11,11 +11,11 @@ option go_package = "unittest";
 import "google/protobuf/timestamp.proto";
 
 message TableUser {
-    uint32                    Id       = 1; // User ID              
+    uint32                    Id       = 1; // User ID
-    string                    Passport = 2; // User Passport        
+    string                    Passport = 2; // User Passport
-    string                    Password = 3; // User Password        
+    string                    Password = 3; // User Password
-    string                    Nickname = 4; // User Nickname        
+    string                    Nickname = 4; // User Nickname
-    string                    Score    = 5; // Total score amount.  
+    string                    Score    = 5; // Total score amount.
-    google.protobuf.Timestamp CreateAt = 6; // Created Time         
+    google.protobuf.Timestamp CreateAt = 6; // Created Time
-    google.protobuf.Timestamp UpdateAt = 7; // Updated Time         
+    google.protobuf.Timestamp UpdateAt = 7; // Updated Time
 }

cmd/gf/internal/cmd/testdata/genpbentity/generated/table_user_snake_screaming.proto

@@ -11,11 +11,11 @@ option go_package = "unittest";
 import "google/protobuf/timestamp.proto";
 
 message TableUser {
-    uint32                    ID        = 1; // User ID              
+    uint32                    ID        = 1; // User ID
-    string                    PASSPORT  = 2; // User Passport        
+    string                    PASSPORT  = 2; // User Passport
-    string                    PASSWORD  = 3; // User Password        
+    string                    PASSWORD  = 3; // User Password
-    string                    NICKNAME  = 4; // User Nickname        
+    string                    NICKNAME  = 4; // User Nickname
-    string                    SCORE     = 5; // Total score amount.  
+    string                    SCORE     = 5; // Total score amount.
-    google.protobuf.Timestamp CREATE_AT = 6; // Created Time         
+    google.protobuf.Timestamp CREATE_AT = 6; // Created Time
-    google.protobuf.Timestamp UPDATE_AT = 7; // Updated Time         
+    google.protobuf.Timestamp UPDATE_AT = 7; // Updated Time
 }

cmd/gf/internal/cmd/testdata/issue/3545/table_user.proto

@@ -11,11 +11,11 @@ option go_package = "github.com/gogf/gf/cmd/gf/v2/internal/cmd/api/pbentity";
 import "google/protobuf/timestamp.proto";
 
 message TableUser {
-    uint32                    Id       = 1; // User ID              
+    uint32                    Id       = 1; // User ID
-    string                    Passport = 2; // User Passport        
+    string                    Passport = 2; // User Passport
-    string                    Password = 3; // User Password        
+    string                    Password = 3; // User Password
-    string                    Nickname = 4; // User Nickname        
+    string                    Nickname = 4; // User Nickname
-    string                    Score    = 5; // Total score amount.  
+    string                    Score    = 5; // Total score amount.
-    google.protobuf.Timestamp CreateAt = 6; // Created Time         
+    google.protobuf.Timestamp CreateAt = 6; // Created Time
-    google.protobuf.Timestamp UpdateAt = 7; // Updated Time         
+    google.protobuf.Timestamp UpdateAt = 7; // Updated Time
 }

cmd/gf/internal/cmd/testdata/issue/3685/table_user.proto

@@ -0,0 +1,23 @@
+// ==========================================================================
+// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT.
+// ==========================================================================
+
+syntax = "proto3";
+
+package pbentity;
+
+option go_package = "github.com/gogf/gf/cmd/gf/v2/internal/cmd/api/pbentity";
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+message TableUser {
+    uint32                    Id       = 1; // User ID
+    string                    Passport = 2; // User Passport
+    string                    Password = 3; // User Password
+    string                    Nickname = 4; // User Nickname
+    string                    Score    = 5; // Total score amount.
+    google.protobuf.Value     Data     = 6; // User Data
+    google.protobuf.Timestamp CreateAt = 7; // Created Time
+    google.protobuf.Timestamp UpdateAt = 8; // Updated Time
+}

cmd/gf/internal/cmd/testdata/issue/3685/user.tpl.sql

@@ -0,0 +1,11 @@
+CREATE TABLE `%s` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'User ID',
+  `passport` varchar(45) NOT NULL COMMENT 'User Passport',
+  `password` varchar(45) NOT NULL COMMENT 'User Password',
+  `nickname` varchar(45) NOT NULL COMMENT 'User Nickname',
+  `score` decimal(10,2) unsigned DEFAULT NULL COMMENT 'Total score amount.',
+  `data` json DEFAULT NULL COMMENT 'User Data',
+  `create_at` datetime DEFAULT NULL COMMENT 'Created Time',
+  `update_at` datetime DEFAULT NULL COMMENT 'Updated Time',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3;