From 9589384b36aceb0555c8df95e0dad6188b46a095 Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 28 Sep 2024 18:10:00 +0800 Subject: [PATCH] refactor(encoding/gjson): change data parameter from type any to []byte (#3542) --- .golangci.yml | 3 + contrib/config/apollo/apollo.go | 1 + contrib/config/kubecm/kubecm.go | 2 +- contrib/config/nacos/nacos.go | 2 +- contrib/config/polaris/polaris.go | 2 +- .../drivers/mssql/mssql_z_unit_basic_test.go | 2 +- .../drivers/mysql/mysql_z_unit_core_test.go | 2 +- .../drivers/sqlite/sqlite_z_unit_core_test.go | 2 +- .../sqlitecgo/sqlitecgo_z_unit_core_test.go | 2 +- encoding/gjson/gjson_api_new_load.go | 303 ++---------------- encoding/gjson/gjson_api_new_load_content.go | 279 ++++++++++++++++ encoding/gjson/gjson_api_new_load_path.go | 34 ++ encoding/gjson/gjson_z_example_load_test.go | 70 ++-- .../gjson/gjson_z_example_pattern_test.go | 2 +- encoding/gjson/gjson_z_example_test.go | 91 +++--- .../gjson/gjson_z_unit_feature_json_test.go | 6 +- .../gjson/gjson_z_unit_feature_load_test.go | 22 +- .../gjson/gjson_z_unit_feature_struct_test.go | 25 +- encoding/gjson/gjson_z_unit_internal_test.go | 20 +- os/gcfg/gcfg_adapter_content.go | 2 +- os/gcfg/gcfg_adapter_file.go | 4 +- 21 files changed, 482 insertions(+), 394 deletions(-) create mode 100644 encoding/gjson/gjson_api_new_load_content.go create mode 100644 encoding/gjson/gjson_api_new_load_path.go diff --git a/.golangci.yml b/.golangci.yml index a587e3426..35b2998c8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -180,6 +180,9 @@ linters-settings: # https://golangci-lint.run/usage/linters/#govet govet: + # Report about shadowed variables. + # Default: false + # check-shadowing: true # Settings per analyzer. settings: # Analyzer name, run `go tool vet help` to see all analyzers. diff --git a/contrib/config/apollo/apollo.go b/contrib/config/apollo/apollo.go index 21b8cb800..3c967c775 100644 --- a/contrib/config/apollo/apollo.go +++ b/contrib/config/apollo/apollo.go @@ -13,6 +13,7 @@ import ( "github.com/apolloconfig/agollo/v4" apolloConfig "github.com/apolloconfig/agollo/v4/env/config" "github.com/apolloconfig/agollo/v4/storage" + "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" diff --git a/contrib/config/kubecm/kubecm.go b/contrib/config/kubecm/kubecm.go index a110193cf..2be721d61 100644 --- a/contrib/config/kubecm/kubecm.go +++ b/contrib/config/kubecm/kubecm.go @@ -138,7 +138,7 @@ func (c *Client) doUpdate(ctx context.Context, namespace string) (err error) { ) } var j *gjson.Json - if j, err = gjson.LoadContent(cm.Data[c.config.DataItem]); err != nil { + if j, err = gjson.LoadContent([]byte(cm.Data[c.config.DataItem])); err != nil { return gerror.Wrapf( err, `parse config map item from %s[%s] failed`, c.config.ConfigMap, c.config.DataItem, diff --git a/contrib/config/nacos/nacos.go b/contrib/config/nacos/nacos.go index 0314b3665..5327b9d6e 100644 --- a/contrib/config/nacos/nacos.go +++ b/contrib/config/nacos/nacos.go @@ -114,7 +114,7 @@ func (c *Client) updateLocalValue() (err error) { func (c *Client) doUpdate(content string) (err error) { var j *gjson.Json - if j, err = gjson.LoadContent(content); err != nil { + if j, err = gjson.LoadContent([]byte(content)); err != nil { return gerror.Wrap(err, `parse config map item from nacos failed`) } c.value.Set(j) diff --git a/contrib/config/polaris/polaris.go b/contrib/config/polaris/polaris.go index c14a8ff85..1f9d6ee05 100644 --- a/contrib/config/polaris/polaris.go +++ b/contrib/config/polaris/polaris.go @@ -147,7 +147,7 @@ func (c *Client) doUpdate(ctx context.Context) (err error) { return gerror.New("config file is empty") } var j *gjson.Json - if j, err = gjson.LoadContent(c.client.GetContent()); err != nil { + if j, err = gjson.LoadContent([]byte(c.client.GetContent())); err != nil { return gerror.Wrap(err, `parse config map item from polaris failed`) } c.value.Set(j) diff --git a/contrib/drivers/mssql/mssql_z_unit_basic_test.go b/contrib/drivers/mssql/mssql_z_unit_basic_test.go index 40d39f988..b283f3047 100644 --- a/contrib/drivers/mssql/mssql_z_unit_basic_test.go +++ b/contrib/drivers/mssql/mssql_z_unit_basic_test.go @@ -794,7 +794,7 @@ func Test_DB_ToJson(t *testing.T) { } // ToJson - resultJson, err := gjson.LoadContent(result.Json()) + resultJson, err := gjson.LoadContent([]byte(result.Json())) if err != nil { gtest.Fatal(err) } diff --git a/contrib/drivers/mysql/mysql_z_unit_core_test.go b/contrib/drivers/mysql/mysql_z_unit_core_test.go index b65678898..5dce861e1 100644 --- a/contrib/drivers/mysql/mysql_z_unit_core_test.go +++ b/contrib/drivers/mysql/mysql_z_unit_core_test.go @@ -894,7 +894,7 @@ func Test_DB_ToJson(t *testing.T) { } // ToJson - resultJson, err := gjson.LoadContent(result.Json()) + resultJson, err := gjson.LoadContent([]byte(result.Json())) if err != nil { gtest.Fatal(err) } diff --git a/contrib/drivers/sqlite/sqlite_z_unit_core_test.go b/contrib/drivers/sqlite/sqlite_z_unit_core_test.go index 95c5cd68d..1c151472f 100644 --- a/contrib/drivers/sqlite/sqlite_z_unit_core_test.go +++ b/contrib/drivers/sqlite/sqlite_z_unit_core_test.go @@ -831,7 +831,7 @@ func Test_DB_ToJson(t *testing.T) { } // ToJson - resultJson, err := gjson.LoadContent(result.Json()) + resultJson, err := gjson.LoadContent([]byte(result.Json())) if err != nil { gtest.Fatal(err) } diff --git a/contrib/drivers/sqlitecgo/sqlitecgo_z_unit_core_test.go b/contrib/drivers/sqlitecgo/sqlitecgo_z_unit_core_test.go index fee914add..fa2fe2c71 100644 --- a/contrib/drivers/sqlitecgo/sqlitecgo_z_unit_core_test.go +++ b/contrib/drivers/sqlitecgo/sqlitecgo_z_unit_core_test.go @@ -831,7 +831,7 @@ func Test_DB_ToJson(t *testing.T) { } // ToJson - resultJson, err := gjson.LoadContent(result.Json()) + resultJson, err := gjson.LoadContent([]byte(result.Json())) if err != nil { gtest.Fatal(err) } diff --git a/encoding/gjson/gjson_api_new_load.go b/encoding/gjson/gjson_api_new_load.go index ec45f55df..113e0dc96 100644 --- a/encoding/gjson/gjson_api_new_load.go +++ b/encoding/gjson/gjson_api_new_load.go @@ -7,22 +7,10 @@ package gjson import ( - "bytes" "reflect" - "github.com/gogf/gf/v2/encoding/gini" - "github.com/gogf/gf/v2/encoding/gproperties" - "github.com/gogf/gf/v2/encoding/gtoml" - "github.com/gogf/gf/v2/encoding/gxml" - "github.com/gogf/gf/v2/encoding/gyaml" - "github.com/gogf/gf/v2/errors/gcode" - "github.com/gogf/gf/v2/errors/gerror" - "github.com/gogf/gf/v2/internal/json" "github.com/gogf/gf/v2/internal/reflection" "github.com/gogf/gf/v2/internal/rwmutex" - "github.com/gogf/gf/v2/os/gfile" - "github.com/gogf/gf/v2/text/gregex" - "github.com/gogf/gf/v2/text/gstr" "github.com/gogf/gf/v2/util/gconv" ) @@ -57,16 +45,26 @@ func NewWithTag(data interface{}, tags string, safe ...bool) *Json { // or slice for data access reason, or it will make no sense. func NewWithOptions(data interface{}, options Options) *Json { var j *Json - switch data.(type) { - case string, []byte: - if r, err := loadContentWithOptions(data, options); err == nil { + switch result := data.(type) { + case []byte: + if r, err := loadContentWithOptions(result, options); err == nil { j = r - } else { - j = &Json{ - p: &data, - c: byte(defaultSplitChar), - vc: false, - } + break + } + j = &Json{ + p: &data, + c: byte(defaultSplitChar), + vc: false, + } + case string: + if r, err := loadContentWithOptions([]byte(result), options); err == nil { + j = r + break + } + j = &Json{ + p: &data, + c: byte(defaultSplitChar), + vc: false, } default: var ( @@ -98,266 +96,3 @@ func NewWithOptions(data interface{}, options Options) *Json { j.mu = rwmutex.Create(options.Safe) return j } - -// Load loads content from specified file `path`, and creates a Json object from its content. -func Load(path string, safe ...bool) (*Json, error) { - if p, err := gfile.Search(path); err != nil { - return nil, err - } else { - path = p - } - options := Options{ - Type: ContentType(gfile.Ext(path)), - } - if len(safe) > 0 && safe[0] { - options.Safe = true - } - return doLoadContentWithOptions(gfile.GetBytesWithCache(path), options) -} - -// LoadWithOptions creates a Json object from given JSON format content and options. -func LoadWithOptions(data interface{}, options Options) (*Json, error) { - return doLoadContentWithOptions(gconv.Bytes(data), options) -} - -// LoadJson creates a Json object from given JSON format content. -func LoadJson(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeJson, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadXml creates a Json object from given XML format content. -func LoadXml(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeXml, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadIni creates a Json object from given INI format content. -func LoadIni(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeIni, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadYaml creates a Json object from given YAML format content. -func LoadYaml(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeYaml, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadToml creates a Json object from given TOML format content. -func LoadToml(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeToml, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadProperties creates a Json object from given TOML format content. -func LoadProperties(data interface{}, safe ...bool) (*Json, error) { - option := Options{ - Type: ContentTypeProperties, - } - if len(safe) > 0 && safe[0] { - option.Safe = true - } - return doLoadContentWithOptions(gconv.Bytes(data), option) -} - -// LoadContent creates a Json object from given content, it checks the data type of `content` -// automatically, supporting data content type as follows: -// JSON, XML, INI, YAML and TOML. -func LoadContent(data interface{}, safe ...bool) (*Json, error) { - content := gconv.Bytes(data) - if len(content) == 0 { - return New(nil, safe...), nil - } - return LoadContentType(checkDataType(content), content, safe...) -} - -// LoadContentType creates a Json object from given type and content, -// supporting data content type as follows: -// JSON, XML, INI, YAML and TOML. -func LoadContentType(dataType ContentType, data interface{}, safe ...bool) (*Json, error) { - content := gconv.Bytes(data) - if len(content) == 0 { - return New(nil, safe...), nil - } - // ignore UTF8-BOM - if content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF { - content = content[3:] - } - options := Options{ - Type: dataType, - StrNumber: true, - } - if len(safe) > 0 && safe[0] { - options.Safe = true - } - return doLoadContentWithOptions(content, options) -} - -// IsValidDataType checks and returns whether given `dataType` a valid data type for loading. -func IsValidDataType(dataType ContentType) bool { - if dataType == "" { - return false - } - if dataType[0] == '.' { - dataType = dataType[1:] - } - switch dataType { - case - ContentTypeJson, - ContentTypeJs, - ContentTypeXml, - ContentTypeYaml, - ContentTypeYml, - ContentTypeToml, - ContentTypeIni, - ContentTypeProperties: - return true - } - return false -} - -func loadContentWithOptions(data interface{}, options Options) (*Json, error) { - content := gconv.Bytes(data) - if len(content) == 0 { - return NewWithOptions(nil, options), nil - } - if options.Type == "" { - options.Type = checkDataType(content) - } - return loadContentTypeWithOptions(content, options) -} - -func loadContentTypeWithOptions(data interface{}, options Options) (*Json, error) { - content := gconv.Bytes(data) - if len(content) == 0 { - return NewWithOptions(nil, options), nil - } - // ignore UTF8-BOM - if content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF { - content = content[3:] - } - return doLoadContentWithOptions(content, options) -} - -// doLoadContent creates a Json object from given content. -// It supports data content type as follows: -// JSON, XML, INI, YAML and TOML. -func doLoadContentWithOptions(data []byte, options Options) (*Json, error) { - var ( - err error - result interface{} - ) - if len(data) == 0 { - return NewWithOptions(nil, options), nil - } - if options.Type == "" { - options.Type = checkDataType(data) - } - options.Type = ContentType(gstr.TrimLeft( - string(options.Type), "."), - ) - switch options.Type { - case ContentTypeJson, ContentTypeJs: - - case ContentTypeXml: - if data, err = gxml.ToJson(data); err != nil { - return nil, err - } - - case ContentTypeYaml, ContentTypeYml: - if data, err = gyaml.ToJson(data); err != nil { - return nil, err - } - - case ContentTypeToml: - if data, err = gtoml.ToJson(data); err != nil { - return nil, err - } - - case ContentTypeIni: - if data, err = gini.ToJson(data); err != nil { - return nil, err - } - case ContentTypeProperties: - if data, err = gproperties.ToJson(data); err != nil { - return nil, err - } - - default: - err = gerror.NewCodef( - gcode.CodeInvalidParameter, - `unsupported type "%s" for loading`, - options.Type, - ) - } - if err != nil { - return nil, err - } - decoder := json.NewDecoder(bytes.NewReader(data)) - if options.StrNumber { - decoder.UseNumber() - } - if err = decoder.Decode(&result); err != nil { - return nil, err - } - switch result.(type) { - case string, []byte: - return nil, gerror.Newf(`json decoding failed for content: %s`, data) - } - return NewWithOptions(result, options), nil -} - -// checkDataType automatically checks and returns the data type for `content`. -// Note that it uses regular expression for loose checking, you can use LoadXXX/LoadContentType -// functions to load the content for certain content type. -func checkDataType(content []byte) ContentType { - if json.Valid(content) { - return ContentTypeJson - } else if gregex.IsMatch(`^<.+>[\S\s]+<.+>\s*$`, content) { - return ContentTypeXml - } else if !gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*"""[\s\S]+"""`, content) && - !gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*'''[\s\S]+'''`, content) && - ((gregex.IsMatch(`^[\n\r]*[\w\-\s\t]+\s*:\s*".+"`, content) || gregex.IsMatch(`^[\n\r]*[\w\-\s\t]+\s*:\s*\w+`, content)) || - (gregex.IsMatch(`[\n\r]+[\w\-\s\t]+\s*:\s*".+"`, content) || gregex.IsMatch(`[\n\r]+[\w\-\s\t]+\s*:\s*\w+`, content))) { - return ContentTypeYaml - } else if !gregex.IsMatch(`^[\s\t\n\r]*;.+`, content) && - !gregex.IsMatch(`[\s\t\n\r]+;.+`, content) && - !gregex.IsMatch(`[\n\r]+[\s\t\w\-]+\.[\s\t\w\-]+\s*=\s*.+`, content) && - (gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*".+"`, content) || gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, content)) { - return ContentTypeToml - } else if gregex.IsMatch(`\[[\w\.]+\]`, content) && - (gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*".+"`, content) || gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, content)) { - // Must contain "[xxx]" section. - return ContentTypeIni - } else if gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, content) { - return ContentTypeProperties - } else { - return "" - } -} diff --git a/encoding/gjson/gjson_api_new_load_content.go b/encoding/gjson/gjson_api_new_load_content.go new file mode 100644 index 000000000..7ddda24c1 --- /dev/null +++ b/encoding/gjson/gjson_api_new_load_content.go @@ -0,0 +1,279 @@ +// Copyright GoFrame Author(https://goframe.org). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gjson + +import ( + "bytes" + + "github.com/gogf/gf/v2/encoding/gini" + "github.com/gogf/gf/v2/encoding/gproperties" + "github.com/gogf/gf/v2/encoding/gtoml" + "github.com/gogf/gf/v2/encoding/gxml" + "github.com/gogf/gf/v2/encoding/gyaml" + "github.com/gogf/gf/v2/errors/gcode" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/internal/json" + "github.com/gogf/gf/v2/text/gregex" + "github.com/gogf/gf/v2/text/gstr" +) + +// LoadWithOptions creates a Json object from given JSON format content and options. +func LoadWithOptions(data []byte, options Options) (*Json, error) { + return loadContentWithOptions(data, options) +} + +// LoadJson creates a Json object from given JSON format content. +func LoadJson(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeJson, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadXml creates a Json object from given XML format content. +func LoadXml(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeXml, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadIni creates a Json object from given INI format content. +func LoadIni(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeIni, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadYaml creates a Json object from given YAML format content. +func LoadYaml(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeYaml, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadToml creates a Json object from given TOML format content. +func LoadToml(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeToml, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadProperties creates a Json object from given TOML format content. +func LoadProperties(data []byte, safe ...bool) (*Json, error) { + var option = Options{ + Type: ContentTypeProperties, + } + if len(safe) > 0 && safe[0] { + option.Safe = true + } + return loadContentWithOptions(data, option) +} + +// LoadContent creates a Json object from given content, it checks the data type of `content` +// automatically, supporting data content type as follows: +// JSON, XML, INI, YAML and TOML. +func LoadContent(data []byte, safe ...bool) (*Json, error) { + return LoadContentType("", data, safe...) +} + +// LoadContentType creates a Json object from given type and content, +// supporting data content type as follows: +// JSON, XML, INI, YAML and TOML. +func LoadContentType(dataType ContentType, data []byte, safe ...bool) (*Json, error) { + if len(data) == 0 { + return New(nil, safe...), nil + } + var options = Options{ + Type: dataType, + StrNumber: true, + } + if len(safe) > 0 && safe[0] { + options.Safe = true + } + return loadContentWithOptions(data, options) +} + +// IsValidDataType checks and returns whether given `dataType` a valid data type for loading. +func IsValidDataType(dataType ContentType) bool { + if dataType == "" { + return false + } + if dataType[0] == '.' { + dataType = dataType[1:] + } + switch dataType { + case + ContentTypeJson, + ContentTypeJs, + ContentTypeXml, + ContentTypeYaml, + ContentTypeYml, + ContentTypeToml, + ContentTypeIni, + ContentTypeProperties: + return true + } + return false +} + +func trimBOM(data []byte) []byte { + if len(data) < 3 { + return data + } + if data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF { + data = data[3:] + } + return data +} + +// loadContentWithOptions creates a Json object from given content. +// It supports data content type as follows: +// JSON, XML, INI, YAML and TOML. +func loadContentWithOptions(data []byte, options Options) (*Json, error) { + var ( + err error + result interface{} + ) + data = trimBOM(data) + if len(data) == 0 { + return NewWithOptions(nil, options), nil + } + if options.Type == "" { + options.Type, err = checkDataType(data) + if err != nil { + return nil, err + } + } + options.Type = ContentType(gstr.TrimLeft( + string(options.Type), "."), + ) + switch options.Type { + case ContentTypeJson, ContentTypeJs: + + case ContentTypeXml: + data, err = gxml.ToJson(data) + + case ContentTypeYaml, ContentTypeYml: + data, err = gyaml.ToJson(data) + + case ContentTypeToml: + data, err = gtoml.ToJson(data) + + case ContentTypeIni: + data, err = gini.ToJson(data) + + case ContentTypeProperties: + data, err = gproperties.ToJson(data) + + default: + err = gerror.NewCodef( + gcode.CodeInvalidParameter, + `unsupported type "%s" for loading`, + options.Type, + ) + } + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(data)) + if options.StrNumber { + decoder.UseNumber() + } + if err = decoder.Decode(&result); err != nil { + return nil, err + } + switch result.(type) { + case string, []byte: + return nil, gerror.Newf(`json decoding failed for content: %s`, data) + } + return NewWithOptions(result, options), nil +} + +// checkDataType automatically checks and returns the data type for `content`. +// Note that it uses regular expression for loose checking, you can use LoadXXX/LoadContentType +// functions to load the content for certain content type. +// TODO it is not graceful here automatic judging the data type. +// TODO it might be removed in the future, which lets the user explicitly specify the data type not automatic checking. +func checkDataType(data []byte) (ContentType, error) { + switch { + case json.Valid(data): + return ContentTypeJson, nil + + case isXmlContent(data): + return ContentTypeXml, nil + + case isYamlContent(data): + return ContentTypeYaml, nil + + case isTomlContent(data): + return ContentTypeToml, nil + + case isIniContent(data): + // Must contain "[xxx]" section. + return ContentTypeIni, nil + + case isPropertyContent(data): + return ContentTypeProperties, nil + + default: + return "", gerror.NewCode( + gcode.CodeOperationFailed, + `unable auto check the data format type`, + ) + } +} + +func isXmlContent(data []byte) bool { + return gregex.IsMatch(`^\s*<.+>[\S\s]+<.+>\s*$`, data) +} + +func isYamlContent(data []byte) bool { + return !gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*"""[\s\S]+"""`, data) && + !gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*'''[\s\S]+'''`, data) && + ((gregex.IsMatch(`^[\n\r]*[\w\-\s\t]+\s*:\s*".+"`, data) || + gregex.IsMatch(`^[\n\r]*[\w\-\s\t]+\s*:\s*\w+`, data)) || + (gregex.IsMatch(`[\n\r]+[\w\-\s\t]+\s*:\s*".+"`, data) || + gregex.IsMatch(`[\n\r]+[\w\-\s\t]+\s*:\s*\w+`, data))) +} + +func isTomlContent(data []byte) bool { + return !gregex.IsMatch(`^[\s\t\n\r]*;.+`, data) && + !gregex.IsMatch(`[\s\t\n\r]+;.+`, data) && + !gregex.IsMatch(`[\n\r]+[\s\t\w\-]+\.[\s\t\w\-]+\s*=\s*.+`, data) && + (gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*".+"`, data) || + gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, data)) +} + +func isIniContent(data []byte) bool { + return gregex.IsMatch(`\[[\w\.]+\]`, data) && + (gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*".+"`, data) || + gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, data)) +} + +func isPropertyContent(data []byte) bool { + return gregex.IsMatch(`[\n\r]*[\s\t\w\-\."]+\s*=\s*\w+`, data) +} diff --git a/encoding/gjson/gjson_api_new_load_path.go b/encoding/gjson/gjson_api_new_load_path.go new file mode 100644 index 000000000..7cdab5d86 --- /dev/null +++ b/encoding/gjson/gjson_api_new_load_path.go @@ -0,0 +1,34 @@ +// Copyright GoFrame Author(https://goframe.org). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gjson + +import "github.com/gogf/gf/v2/os/gfile" + +// Load loads content from specified file `path`, and creates a Json object from its content. +// Deprecated: use LoadPath instead. +func Load(path string, safe ...bool) (*Json, error) { + var isSafe bool + if len(safe) > 0 { + isSafe = safe[0] + } + return LoadPath(path, Options{ + Safe: isSafe, + }) +} + +// LoadPath loads content from specified file `path`, and creates a Json object from its content. +func LoadPath(path string, options Options) (*Json, error) { + if p, err := gfile.Search(path); err != nil { + return nil, err + } else { + path = p + } + if options.Type == "" { + options.Type = ContentType(gfile.Ext(path)) + } + return loadContentWithOptions(gfile.GetBytesWithCache(path), options) +} diff --git a/encoding/gjson/gjson_z_example_load_test.go b/encoding/gjson/gjson_z_example_load_test.go index 102029f6f..812cd2a70 100644 --- a/encoding/gjson/gjson_z_example_load_test.go +++ b/encoding/gjson/gjson_z_example_load_test.go @@ -29,7 +29,7 @@ func ExampleLoad() { } func ExampleLoadJson() { - jsonContent := `{"name":"john", "score":"100"}` + jsonContent := []byte(`{"name":"john", "score":"100"}`) j, _ := gjson.LoadJson(jsonContent) fmt.Println(j.Get("name")) fmt.Println(j.Get("score")) @@ -40,11 +40,13 @@ func ExampleLoadJson() { } func ExampleLoadXml() { - xmlContent := ` - - john - 100 - ` + xmlContent := []byte(` + + + john + 100 + +`) j, _ := gjson.LoadXml(xmlContent) fmt.Println(j.Get("base.name")) fmt.Println(j.Get("base.score")) @@ -55,11 +57,11 @@ func ExampleLoadXml() { } func ExampleLoadIni() { - iniContent := ` - [base] - name = john - score = 100 - ` + iniContent := []byte(` +[base] +name = john +score = 100 +`) j, _ := gjson.LoadIni(iniContent) fmt.Println(j.Get("base.name")) fmt.Println(j.Get("base.score")) @@ -70,10 +72,11 @@ func ExampleLoadIni() { } func ExampleLoadYaml() { - yamlContent := - `base: + yamlContent := []byte(` +base: name: john - score: 100` + score: 100 +`) j, _ := gjson.LoadYaml(yamlContent) fmt.Println(j.Get("base.name")) @@ -85,10 +88,11 @@ func ExampleLoadYaml() { } func ExampleLoadToml() { - tomlContent := - `[base] + tomlContent := []byte(` +[base] name = "john" - score = 100` + score = 100 +`) j, _ := gjson.LoadToml(tomlContent) fmt.Println(j.Get("base.name")) @@ -100,7 +104,7 @@ func ExampleLoadToml() { } func ExampleLoadContent() { - jsonContent := `{"name":"john", "score":"100"}` + jsonContent := []byte(`{"name":"john", "score":"100"}`) j, _ := gjson.LoadContent(jsonContent) @@ -132,11 +136,13 @@ func ExampleLoadContent_UTF8BOM() { } func ExampleLoadContent_Xml() { - xmlContent := ` - - john - 100 - ` + xmlContent := []byte(` + + + john + 100 + +`) x, _ := gjson.LoadContent(xmlContent) @@ -149,16 +155,20 @@ func ExampleLoadContent_Xml() { } func ExampleLoadContentType() { - jsonContent := `{"name":"john", "score":"100"}` - xmlContent := ` - - john - 100 - ` + var ( + jsonContent = []byte(`{"name":"john", "score":"100"}`) + xmlContent = []byte(` + + + john + 100 + +`) + ) j, _ := gjson.LoadContentType("json", jsonContent) x, _ := gjson.LoadContentType("xml", xmlContent) - j1, _ := gjson.LoadContentType("json", "") + j1, _ := gjson.LoadContentType("json", []byte("")) fmt.Println(j.Get("name")) fmt.Println(j.Get("score")) diff --git a/encoding/gjson/gjson_z_example_pattern_test.go b/encoding/gjson/gjson_z_example_pattern_test.go index db5f1308a..020d7bff7 100644 --- a/encoding/gjson/gjson_z_example_pattern_test.go +++ b/encoding/gjson/gjson_z_example_pattern_test.go @@ -51,7 +51,7 @@ func ExampleDecodeToJson_PatternViolenceCheck() { } func ExampleJson_Get_MapSliceChange() { - jsonContent := `{"map":{"key":"value"}, "slice":[59,90]}` + jsonContent := []byte(`{"map":{"key":"value"}, "slice":[59,90]}`) j, _ := gjson.LoadJson(jsonContent) m := j.Get("map").Map() fmt.Println(m) diff --git a/encoding/gjson/gjson_z_example_test.go b/encoding/gjson/gjson_z_example_test.go index d262be3ce..3e6c45f3f 100644 --- a/encoding/gjson/gjson_z_example_test.go +++ b/encoding/gjson/gjson_z_example_test.go @@ -871,13 +871,14 @@ func ExampleJson_IsNil() { } func ExampleJson_Get() { - data := - `{ - "users" : { - "count" : 1, - "array" : ["John", "Ming"] - } - }` + data := []byte(` +{ + "users" : { + "count" : 1, + "array" : ["John", "Ming"] + } +} +`) j, _ := gjson.LoadContent(data) fmt.Println(j.Get(".")) @@ -896,13 +897,14 @@ func ExampleJson_Get() { } func ExampleJson_GetJson() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "array" : ["John", "Ming"] } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -913,13 +915,14 @@ func ExampleJson_GetJson() { } func ExampleJson_GetJsons() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 3, "array" : [{"Age":18,"Name":"John"}, {"Age":20,"Name":"Tom"}] } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -934,16 +937,17 @@ func ExampleJson_GetJsons() { } func ExampleJson_GetJsonMap() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "array" : { "info" : {"Age":18,"Name":"John"}, "addr" : {"City":"Chengdu","Company":"Tencent"} } } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1056,9 +1060,9 @@ func ExampleJson_Contains() { } func ExampleJson_Len() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "nameArray" : ["Join", "Tom"], "infoMap" : { @@ -1067,7 +1071,8 @@ func ExampleJson_Len() { "addr" : "ChengDu" } } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1080,13 +1085,14 @@ func ExampleJson_Len() { } func ExampleJson_Append() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "array" : ["John", "Ming"] } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1099,13 +1105,14 @@ func ExampleJson_Append() { } func ExampleJson_MustAppend() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "array" : ["John", "Ming"] } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1118,9 +1125,9 @@ func ExampleJson_MustAppend() { } func ExampleJson_Map() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "info" : { "name" : "John", @@ -1128,7 +1135,8 @@ func ExampleJson_Map() { "addr" : "ChengDu" } } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1139,13 +1147,14 @@ func ExampleJson_Map() { } func ExampleJson_Array() { - data := - `{ - "users" : { + data := []byte(` +{ + "users" : { "count" : 1, "array" : ["John", "Ming"] } - }` + } +`) j, _ := gjson.LoadContent(data) @@ -1156,7 +1165,7 @@ func ExampleJson_Array() { } func ExampleJson_Scan() { - data := `{"name":"john","age":"18"}` + data := []byte(`{"name":"john","age":"18"}`) type BaseInfo struct { Name string @@ -1175,7 +1184,7 @@ func ExampleJson_Scan() { } func ExampleJson_Dump() { - data := `{"name":"john","age":"18"}` + data := []byte(`{"name":"john","age":"18"}`) j, _ := gjson.LoadContent(data) j.Dump() diff --git a/encoding/gjson/gjson_z_unit_feature_json_test.go b/encoding/gjson/gjson_z_unit_feature_json_test.go index 5e3cd3024..542b8e795 100644 --- a/encoding/gjson/gjson_z_unit_feature_json_test.go +++ b/encoding/gjson/gjson_z_unit_feature_json_test.go @@ -49,11 +49,11 @@ func Test_ToJson(t *testing.T) { } func Test_MapAttributeConvert(t *testing.T) { - var data = ` - { + var data = []byte(` +{ "title": {"l1":"标签1","l2":"标签2"} } -` +`) gtest.C(t, func(t *gtest.T) { j, err := gjson.LoadContent(data) gtest.AssertNil(err) diff --git a/encoding/gjson/gjson_z_unit_feature_load_test.go b/encoding/gjson/gjson_z_unit_feature_load_test.go index 51f9a2d8b..4a1f5e098 100644 --- a/encoding/gjson/gjson_z_unit_feature_load_test.go +++ b/encoding/gjson/gjson_z_unit_feature_load_test.go @@ -105,7 +105,8 @@ func Test_Load_XML(t *testing.T) { // XML gtest.C(t, func(t *gtest.T) { - xml := ` + xml := []byte(` + 0 @@ -114,7 +115,8 @@ func Test_Load_XML(t *testing.T) { GF框架 - ` + +`) j, err := gjson.LoadContent(xml) t.AssertNil(err) t.Assert(j.Get("Output.ipageIndex"), "2") @@ -250,10 +252,10 @@ func Test_Load_Basic(t *testing.T) { t.AssertNil(err) t.Assert(j.Interface(), nil) - j, err = gjson.LoadContent(`{"name": "gf"}`) + j, err = gjson.LoadContent([]byte(`{"name": "gf"}`)) t.AssertNil(err) - j, err = gjson.LoadContent(`{"name": "gf"""}`) + j, err = gjson.LoadContent([]byte(`{"name": "gf"""}`)) t.AssertNE(err, nil) j = gjson.New(&g.Map{"name": "gf"}) @@ -263,7 +265,7 @@ func Test_Load_Basic(t *testing.T) { } func Test_Load_Ini(t *testing.T) { - var data = ` + var data = []byte(` ;注释 @@ -277,7 +279,7 @@ enable=true user=root password=password -` +`) gtest.C(t, func(t *gtest.T) { j, err := gjson.LoadContent(data) @@ -320,7 +322,7 @@ enable=true } func Test_Load_YamlWithV3(t *testing.T) { - content := ` + content := []byte(` # CLI tool, only in development environment. # https://goframe.org/pages/viewpage.action?pageId=3673173 gfcli: @@ -355,7 +357,7 @@ gfcli: noModelComment : true overwriteDao : true modelFileForDao : "model_dao.go" -` +`) gtest.C(t, func(t *gtest.T) { _, err := gjson.LoadContent(content) t.AssertNil(err) @@ -363,7 +365,7 @@ gfcli: } func Test_Load_Properties(t *testing.T) { - var data = ` + var data = []byte(` #注释 @@ -375,7 +377,7 @@ DBINFO.type=mysql DBINFO.user=root DBINFO.password=password -` +`) gtest.C(t, func(t *gtest.T) { j, err := gjson.LoadContent(data) diff --git a/encoding/gjson/gjson_z_unit_feature_struct_test.go b/encoding/gjson/gjson_z_unit_feature_struct_test.go index 49f7c0a4a..df13c1b98 100644 --- a/encoding/gjson/gjson_z_unit_feature_struct_test.go +++ b/encoding/gjson/gjson_z_unit_feature_struct_test.go @@ -138,7 +138,7 @@ func Test_Struct1(t *testing.T) { type UserCollectionAddReq struct { BaseInfo []BaseInfoItem `db:"_" json:"baseInfo" field:"_"` } - jsonContent := `{ + jsonContent := []byte(`{ "baseInfo": [{ "idCardNumber": "520101199412141111", "isHouseholder": true, @@ -195,7 +195,8 @@ func Test_Struct1(t *testing.T) { "incomeInfo": [], "liabilityInfo": [] }] -}` +} +`) data := new(UserCollectionAddReq) j, err := gjson.LoadJson(jsonContent, true) t.AssertNil(err) @@ -218,12 +219,14 @@ func Test_Struct(t *testing.T) { Items []*Item `json:"items"` } - txt := `{ - "id":"88888", - "me":{"name":"mikey","day":"20009"}, - "txt":"hello", - "items":null - }` + txt := []byte(` +{ + "id":"88888", + "me":{"name":"mikey","day":"20009"}, + "txt":"hello", + "items":null +} +`) j, err := gjson.LoadContent(txt) t.AssertNil(err) @@ -281,13 +284,15 @@ func Test_Struct_Complicated(t *testing.T) { } gtest.C(t, func(t *gtest.T) { - jsonContent := `{ + jsonContent := []byte(` +{ "certList":[ {"certId":"2023313","certInfo":"{\"address\":\"xxxxxxx\",\"phoneNumber\":\"15084890\",\"companyName\":\"dddd\",\"communityCreditCode\":\"91110111MBE1G2B\",\"operateRange\":\"fff\",\"registerNo\":\"91110111MA00G2B\",\"legalPersonName\":\"rrr\"}","srcType":"1","statusCode":"2"}, {"certId":"2023314","certInfo":"{\"identNo\":\"342224196507051\",\"userRealname\":\"xxxx\",\"identType\":\"01\"}","srcType":"8","statusCode":"0"}, {"certId":"2023322","certInfo":"{\"businessLicense\":\"91110111MA00BE1G\",\"companyName\":\"sssss\",\"communityCreditCode\":\"91110111MA00BE1\"}","srcType":"2","statusCode":"0"} ] -}` +} +`) j, err := gjson.LoadContent(jsonContent) t.AssertNil(err) var response = new(Response) diff --git a/encoding/gjson/gjson_z_unit_internal_test.go b/encoding/gjson/gjson_z_unit_internal_test.go index 853b839c0..118096823 100644 --- a/encoding/gjson/gjson_z_unit_internal_test.go +++ b/encoding/gjson/gjson_z_unit_internal_test.go @@ -18,7 +18,9 @@ func Test_checkDataType(t *testing.T) { bb = """ dig := dig; END;""" `) - t.Assert(checkDataType(data), "toml") + dataType, err := checkDataType(data) + t.AssertNil(err) + t.Assert(dataType, "toml") }) gtest.C(t, func(t *gtest.T) { @@ -32,7 +34,9 @@ dd = 11 disk = "127.0.0.1:6379,0" cache = "127.0.0.1:6379,1" `) - t.Assert(checkDataType(data), "toml") + dataType, err := checkDataType(data) + t.AssertNil(err) + t.Assert(dataType, "toml") }) gtest.C(t, func(t *gtest.T) { @@ -86,7 +90,9 @@ dd = 11 // fmt.Println(gregex.IsMatch(`[\s\t\n\r]+[\w\-]+\s*:\s*".+"`, data)) // fmt.Println(gregex.IsMatch(`[\n\r]+[\w\-\s\t]+\s*:\s*\w+`, data)) // fmt.Println(gregex.MatchString(`[\n\r]+[\w\-\s\t]+\s*:\s*\w+`, string(data))) - t.Assert(checkDataType(data), "toml") + dataType, err := checkDataType(data) + t.AssertNil(err) + t.Assert(dataType, "toml") }) gtest.C(t, func(t *gtest.T) { @@ -111,7 +117,9 @@ k8s-inner-api = http://127.0.0.1:8081/kube/add conf_dir = ./config app_conf = ./config/app.ini `) - t.Assert(checkDataType(data), "ini") + dataType, err := checkDataType(data) + t.AssertNil(err) + t.Assert(dataType, "ini") }) gtest.C(t, func(t *gtest.T) { @@ -129,6 +137,8 @@ wget http://consul.infra:8500/v1/kv/app_{{.SwimlaneName}}/{{.RepoName}}/.env.qa? npm run build:qa """ `) - t.Assert(checkDataType(data), "toml") + dataType, err := checkDataType(data) + t.AssertNil(err) + t.Assert(dataType, "toml") }) } diff --git a/os/gcfg/gcfg_adapter_content.go b/os/gcfg/gcfg_adapter_content.go index 44daa7ef1..9a0b12a28 100644 --- a/os/gcfg/gcfg_adapter_content.go +++ b/os/gcfg/gcfg_adapter_content.go @@ -37,7 +37,7 @@ func NewAdapterContent(content ...string) (*AdapterContent, error) { // SetContent sets customized configuration content for specified `file`. // The `file` is unnecessary param, default is DefaultConfigFile. func (a *AdapterContent) SetContent(content string) error { - j, err := gjson.LoadContent(content, true) + j, err := gjson.LoadContent([]byte(content), true) if err != nil { return gerror.Wrap(err, `load configuration content failed`) } diff --git a/os/gcfg/gcfg_adapter_file.go b/os/gcfg/gcfg_adapter_file.go index f736d28c5..dc98a3787 100644 --- a/os/gcfg/gcfg_adapter_file.go +++ b/os/gcfg/gcfg_adapter_file.go @@ -262,9 +262,9 @@ func (a *AdapterFile) getJson(fileNameOrPath ...string) (configJson *gjson.Json, // Note that the underlying configuration json object operations are concurrent safe. dataType := gjson.ContentType(gfile.ExtName(filePath)) if gjson.IsValidDataType(dataType) && !isFromConfigContent { - configJson, err = gjson.LoadContentType(dataType, content, true) + configJson, err = gjson.LoadContentType(dataType, []byte(content), true) } else { - configJson, err = gjson.LoadContent(content, true) + configJson, err = gjson.LoadContent([]byte(content), true) } if err != nil { if filePath != "" {