From 945dd71251a7efb290dd78d831ada5a0b7fe16bd Mon Sep 17 00:00:00 2001 From: John Date: Wed, 29 May 2019 11:53:10 +0800 Subject: [PATCH] TODO updates --- TODO.MD | 9 ++- g/database/gdb/gdb_mssql.go | 110 ++++++++++++++++---------------- g/database/gdb/gdb_oracle.go | 110 ++++++++++++++++---------------- g/database/gdb/gdb_structure.go | 52 +++++++-------- 4 files changed, 143 insertions(+), 138 deletions(-) diff --git a/TODO.MD b/TODO.MD index 1a4d09dc1..f68a8f50e 100644 --- a/TODO.MD +++ b/TODO.MD @@ -6,7 +6,6 @@ 1. orm增加sqlite对Save方法的支持(去掉触发器语句); 1. ghttp.Server增加Ip访问控制功能(DenyIps&AllowIps)； 1. ghttp增加返回数据压缩机制； -1. gview中的template标签失效问题； 1. ghttp.Server增加proxy功能特性，本地proxy和远程proxy，本地即将路由规则映射；远程即反向代理； 1. gjson对大json数据的解析效率问题； 1. ghttp增加route name特性，并同时支持backend和template(提供内置函数)引用，可以通过RedirectRoute方法给定route name和路由参数跳转到指定的路由地址上； @@ -46,6 +45,11 @@ 1. gset.Add/Remove/Contains方法增加批量操作支持； 1. gmlock增加手动清理机制：当内存锁不再使用时，由调用端决定是否清理内存锁； 1. gtimer增加DelayAdd*方法返回Entry对象，以便DelayAdd*的定时任务也能进行状态控制；gcron同理需要改进； +1. 改进gdb对pgsql/mssql/oracle的支持，使用方法覆盖的方式改进操作，而不是完全依靠正则替换的方式； +1. gdb的Cache缓存功能增加可自定义缓存接口，以便支持外部缓存功能，缓存接口可以通过io.ReadWriter接口实现； + + + # DONE 1. gconv完善针对不同类型的判断，例如：尽量减少sprintf("%v", xxx)来执行string类型的转换； @@ -119,4 +123,5 @@ 1. gfile对于文件的读写强行使用了gfpool，在某些场景下不合适，需要考虑剥离开，并为开发者提供单独的指针池文件操作特性； 1. ghttp.Client自动Close机制； 1. ghttp路由功能增加分组路由特性； -1. 增加可选择性的orm tag特性，用以数据表记录与struct对象转换的键名属性映射； \ No newline at end of file +1. 增加可选择性的orm tag特性，用以数据表记录与struct对象转换的键名属性映射； +1. gview中的template标签失效问题； \ No newline at end of file diff --git a/g/database/gdb/gdb_mssql.go b/g/database/gdb/gdb_mssql.go index 7ad243643..bd3de9e08 100644 --- a/g/database/gdb/gdb_mssql.go +++ b/g/database/gdb/gdb_mssql.go @@ -67,13 +67,13 @@ func (db *dbMssql) parseSql(sql string) string { //下面的正则表达式匹配出SELECT和INSERT的关键字后分别做不同的处理，如有LIMIT则将LIMIT的关键字也匹配出 patten := `^\s*(?i)(SELECT)|(LIMIT\s*(\d+)\s*,\s*(\d+))` if gregex.IsMatchString(patten, sql) == false { - fmt.Println("not matched..") + //fmt.Println("not matched..") return sql } res, err := gregex.MatchAllString(patten, sql) if err != nil { - fmt.Println("MatchString error.", err) + //fmt.Println("MatchString error.", err) return "" } @@ -83,69 +83,69 @@ func (db *dbMssql) parseSql(sql string) string { index++ switch keyword { - case "SELECT": - //不含LIMIT关键字则不处理 - if len(res) < 2 || (strings.HasPrefix(res[index][0], "LIMIT") == false && strings.HasPrefix(res[index][0], "limit") == false) { - break - } - - //不含LIMIT则不处理 - if gregex.IsMatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) == false { - break - } - - //判断SQL中是否含有order by - selectStr := "" - orderbyStr := "" - haveOrderby := gregex.IsMatchString("((?i)SELECT)(.+)((?i)ORDER BY)", sql) - if haveOrderby { - //取order by 前面的字符串 - queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)ORDER BY)", sql) - - if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "ORDER BY") == false { + case "SELECT": + //不含LIMIT关键字则不处理 + if len(res) < 2 || (strings.HasPrefix(res[index][0], "LIMIT") == false && strings.HasPrefix(res[index][0], "limit") == false) { break } - selectStr = queryExpr[2] - //取order by表达式的值 - orderbyExpr, _ := gregex.MatchString("((?i)ORDER BY)(.+)((?i)LIMIT)", sql) - if len(orderbyExpr) != 4 || strings.EqualFold(orderbyExpr[1], "ORDER BY") == false || strings.EqualFold(orderbyExpr[3], "LIMIT") == false { + //不含LIMIT则不处理 + if gregex.IsMatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) == false { break } - orderbyStr = orderbyExpr[2] - } else { - queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) - if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "LIMIT") == false { - break - } - selectStr = queryExpr[2] - } - //取limit后面的取值范围 - first, limit := 0, 0 - for i := 1; i < len(res[index]); i++ { - if len(strings.TrimSpace(res[index][i])) == 0 { - continue - } + //判断SQL中是否含有order by + selectStr := "" + orderbyStr := "" + haveOrderby := gregex.IsMatchString("((?i)SELECT)(.+)((?i)ORDER BY)", sql) + if haveOrderby { + //取order by 前面的字符串 + queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)ORDER BY)", sql) - if strings.HasPrefix(res[index][i], "LIMIT") || strings.HasPrefix(res[index][i], "limit") { - first, _ = strconv.Atoi(res[index][i+1]) - limit, _ = strconv.Atoi(res[index][i+2]) - break - } - } + if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "ORDER BY") == false { + break + } + selectStr = queryExpr[2] - if haveOrderby { - sql = fmt.Sprintf("SELECT * FROM (SELECT ROW_NUMBER() OVER (ORDER BY %s) as ROWNUMBER_, %s ) as TMP_ WHERE TMP_.ROWNUMBER_ > %d AND TMP_.ROWNUMBER_ <= %d", orderbyStr, selectStr, first, limit) - } else { - if first == 0 { - first = limit + //取order by表达式的值 + orderbyExpr, _ := gregex.MatchString("((?i)ORDER BY)(.+)((?i)LIMIT)", sql) + if len(orderbyExpr) != 4 || strings.EqualFold(orderbyExpr[1], "ORDER BY") == false || strings.EqualFold(orderbyExpr[3], "LIMIT") == false { + break + } + orderbyStr = orderbyExpr[2] } else { - first = limit - first + queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) + if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "LIMIT") == false { + break + } + selectStr = queryExpr[2] } - sql = fmt.Sprintf("SELECT * FROM (SELECT TOP %d * FROM (SELECT TOP %d %s) as TMP1_ ) as TMP2_ ", first, limit, selectStr) - } - default: + + //取limit后面的取值范围 + first, limit := 0, 0 + for i := 1; i < len(res[index]); i++ { + if len(strings.TrimSpace(res[index][i])) == 0 { + continue + } + + if strings.HasPrefix(res[index][i], "LIMIT") || strings.HasPrefix(res[index][i], "limit") { + first, _ = strconv.Atoi(res[index][i+1]) + limit, _ = strconv.Atoi(res[index][i+2]) + break + } + } + + if haveOrderby { + sql = fmt.Sprintf("SELECT * FROM (SELECT ROW_NUMBER() OVER (ORDER BY %s) as ROWNUMBER_, %s ) as TMP_ WHERE TMP_.ROWNUMBER_ > %d AND TMP_.ROWNUMBER_ <= %d", orderbyStr, selectStr, first, limit) + } else { + if first == 0 { + first = limit + } else { + first = limit - first + } + sql = fmt.Sprintf("SELECT * FROM (SELECT TOP %d * FROM (SELECT TOP %d %s) as TMP1_ ) as TMP2_ ", first, limit, selectStr) + } + default: } return sql } diff --git a/g/database/gdb/gdb_oracle.go b/g/database/gdb/gdb_oracle.go index 5df9057c1..17c7d48fc 100644 --- a/g/database/gdb/gdb_oracle.go +++ b/g/database/gdb/gdb_oracle.go @@ -65,80 +65,80 @@ func (db *dbOracle) parseSql(sql string) string { //下面的正则表达式匹配出SELECT和INSERT的关键字后分别做不同的处理，如有LIMIT则将LIMIT的关键字也匹配出 patten := `^\s*(?i)(SELECT)|(INSERT)|(LIMIT\s*(\d+)\s*,\s*(\d+))` if gregex.IsMatchString(patten, sql) == false { - fmt.Println("not matched..") + //fmt.Println("not matched..") return sql } res, err := gregex.MatchAllString(patten, sql) if err != nil { - fmt.Println("MatchString error.", err) + //fmt.Println("MatchString error.", err) return "" } - index := 0 + index := 0 keyword := strings.TrimSpace(res[index][0]) - keyword = strings.ToUpper(keyword) + keyword = strings.ToUpper(keyword) index++ switch keyword { - case "SELECT": - //不含LIMIT关键字则不处理 - if len(res) < 2 || (strings.HasPrefix(res[index][0], "LIMIT") == false && strings.HasPrefix(res[index][0], "limit") == false) { - break - } - - //取limit前面的字符串 - if gregex.IsMatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) == false { - break - } - - queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) - if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "LIMIT") == false { - break - } - - //取limit后面的取值范围 - first, limit := 0, 0 - for i := 1; i < len(res[index]); i++ { - if len(strings.TrimSpace(res[index][i])) == 0 { - continue - } - - if strings.HasPrefix(res[index][i], "LIMIT") || strings.HasPrefix(res[index][i], "limit") { - first, _ = strconv.Atoi(res[index][i+1]) - limit, _ = strconv.Atoi(res[index][i+2]) + case "SELECT": + //不含LIMIT关键字则不处理 + if len(res) < 2 || (strings.HasPrefix(res[index][0], "LIMIT") == false && strings.HasPrefix(res[index][0], "limit") == false) { break } - } - //也可以使用between,据说这种写法的性能会比between好点,里层SQL中的ROWNUM_ >= limit可以缩小查询后的数据集规模 - sql = fmt.Sprintf("SELECT * FROM (SELECT GFORM.*, ROWNUM ROWNUM_ FROM (%s %s) GFORM WHERE ROWNUM <= %d) WHERE ROWNUM_ >= %d", queryExpr[1], queryExpr[2], limit, first) - case "INSERT": - //获取VALUE的值，匹配所有带括号的值,会将INSERT INTO后的值匹配到，所以下面的判断语句会判断数组长度是否小于3 - valueExpr, err := gregex.MatchAllString(`(\s*\(([^\(\)]*)\))`, sql) - if err != nil { - return sql - } + //取limit前面的字符串 + if gregex.IsMatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) == false { + break + } - //判断VALUE后的值是否有多个，只有在批量插入的时候才需要做转换，如只有1个VALUE则不需要做转换 - if len(valueExpr) < 3 { - break - } + queryExpr, _ := gregex.MatchString("((?i)SELECT)(.+)((?i)LIMIT)", sql) + if len(queryExpr) != 4 || strings.EqualFold(queryExpr[1], "SELECT") == false || strings.EqualFold(queryExpr[3], "LIMIT") == false { + break + } - //获取INTO后面的值 - tableExpr, err := gregex.MatchString(`(?i)\s*(INTO\s+\w+\(([^\(\)]*)\))`, sql) - if err != nil { - return sql - } - tableExpr[0] = strings.TrimSpace(tableExpr[0]) + //取limit后面的取值范围 + first, limit := 0, 0 + for i := 1; i < len(res[index]); i++ { + if len(strings.TrimSpace(res[index][i])) == 0 { + continue + } - sql = "INSERT ALL" - for i := 1; i < len(valueExpr); i++ { - sql += fmt.Sprintf(" %s VALUES%s", tableExpr[0], strings.TrimSpace(valueExpr[i][0])) - } - sql += " SELECT 1 FROM DUAL" + if strings.HasPrefix(res[index][i], "LIMIT") || strings.HasPrefix(res[index][i], "limit") { + first, _ = strconv.Atoi(res[index][i+1]) + limit, _ = strconv.Atoi(res[index][i+2]) + break + } + } - default: + //也可以使用between,据说这种写法的性能会比between好点,里层SQL中的ROWNUM_ >= limit可以缩小查询后的数据集规模 + sql = fmt.Sprintf("SELECT * FROM (SELECT GFORM.*, ROWNUM ROWNUM_ FROM (%s %s) GFORM WHERE ROWNUM <= %d) WHERE ROWNUM_ >= %d", queryExpr[1], queryExpr[2], limit, first) + case "INSERT": + //获取VALUE的值，匹配所有带括号的值,会将INSERT INTO后的值匹配到，所以下面的判断语句会判断数组长度是否小于3 + valueExpr, err := gregex.MatchAllString(`(\s*\(([^\(\)]*)\))`, sql) + if err != nil { + return sql + } + + //判断VALUE后的值是否有多个，只有在批量插入的时候才需要做转换，如只有1个VALUE则不需要做转换 + if len(valueExpr) < 3 { + break + } + + //获取INTO后面的值 + tableExpr, err := gregex.MatchString(`(?i)\s*(INTO\s+\w+\(([^\(\)]*)\))`, sql) + if err != nil { + return sql + } + tableExpr[0] = strings.TrimSpace(tableExpr[0]) + + sql = "INSERT ALL" + for i := 1; i < len(valueExpr); i++ { + sql += fmt.Sprintf(" %s VALUES%s", tableExpr[0], strings.TrimSpace(valueExpr[i][0])) + } + sql += " SELECT 1 FROM DUAL" + + default: } return sql } diff --git a/g/database/gdb/gdb_structure.go b/g/database/gdb/gdb_structure.go index 2530f5255..709b114cb 100644 --- a/g/database/gdb/gdb_structure.go +++ b/g/database/gdb/gdb_structure.go @@ -28,42 +28,42 @@ func (bs *dbBase) convertValue(fieldValue interface{}, fieldType string) interfa t, _ := gregex.ReplaceString(`\(.+\)`, "", fieldType) t = strings.ToLower(t) switch t { - case "binary", "varbinary", "blob", "tinyblob", "mediumblob", "longblob": - return gconv.Bytes(fieldValue) + case "binary", "varbinary", "blob", "tinyblob", "mediumblob", "longblob": + return gconv.Bytes(fieldValue) - case "bit", "int", "tinyint", "small_int", "medium_int": - return gconv.Int(fieldValue) + case "bit", "int", "tinyint", "small_int", "medium_int": + return gconv.Int(fieldValue) - case "big_int": - return gconv.Int64(fieldValue) + case "big_int": + return gconv.Int64(fieldValue) - case "float", "double", "decimal": - return gconv.Float64(fieldValue) + case "float", "double", "decimal": + return gconv.Float64(fieldValue) - case "bool": - return gconv.Bool(fieldValue) + case "bool": + return gconv.Bool(fieldValue) - default: - // 自动识别类型, 以便默认支持更多数据库类型 - switch { - case strings.Contains(t, "int"): - return gconv.Int(fieldValue) + default: + // 自动识别类型, 以便默认支持更多数据库类型 + switch { + case strings.Contains(t, "int"): + return gconv.Int(fieldValue) - case strings.Contains(t, "text") || strings.Contains(t, "char"): - return gconv.String(fieldValue) + case strings.Contains(t, "text") || strings.Contains(t, "char"): + return gconv.String(fieldValue) - case strings.Contains(t, "float") || strings.Contains(t, "double"): - return gconv.Float64(fieldValue) + case strings.Contains(t, "float") || strings.Contains(t, "double"): + return gconv.Float64(fieldValue) - case strings.Contains(t, "bool"): - return gconv.Bool(fieldValue) + case strings.Contains(t, "bool"): + return gconv.Bool(fieldValue) - case strings.Contains(t, "binary") || strings.Contains(t, "blob"): - return gconv.Bytes(fieldValue) + case strings.Contains(t, "binary") || strings.Contains(t, "blob"): + return gconv.Bytes(fieldValue) - default: - return gconv.String(fieldValue) - } + default: + return gconv.String(fieldValue) + } } }