From aa87d234e37606b6965c00722cbd9ff9c38984c6 Mon Sep 17 00:00:00 2001 From: daguang Date: Sat, 19 Feb 2022 15:09:44 +0800 Subject: [PATCH 01/15] improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 194 ++++++++++++++++++++++- contrib/drivers/clickhouse/go.mod | 9 +- contrib/drivers/clickhouse/go.sum | 187 ++++++++++++++++++++-- database/gdb/gdb_core_config.go | 3 + 4 files changed, 375 insertions(+), 18 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index 3b77030a2..6f33fe0eb 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -8,5 +8,197 @@ package clickhouse import ( - _ "github.com/ClickHouse/clickhouse-go" + "context" + "database/sql" + "errors" + "fmt" + "github.com/ClickHouse/clickhouse-go/v2" + + "github.com/gogf/gf/v2/container/gmap" + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/errors/gcode" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/text/gregex" + "github.com/gogf/gf/v2/text/gstr" + "github.com/gogf/gf/v2/util/gconv" ) + +// Driver is the driver for postgresql database. +type Driver struct { + *gdb.Core +} + +var ( + // tableFieldsMap caches the table information retrieved from database. + tableFieldsMap = gmap.New(true) +) + +func init() { + if err := gdb.Register(`clickhouse`, New()); err != nil { + panic(err) + } +} + +// New create and returns a driver that implements gdb.Driver, which supports operations for clickhouse. +func New() gdb.Driver { + return &Driver{} +} + +// New creates and returns a database object for clickhouse. +// It implements the interface of gdb.Driver for extra database driver installation. +func (d *Driver) New(core *gdb.Core, node *gdb.ConfigNode) (gdb.DB, error) { + return &Driver{ + Core: core, + }, nil +} + +// Open creates and returns an underlying sql.DB object for clickhouse. +func (d *Driver) Open(config *gdb.ConfigNode) (*sql.DB, error) { + var ( + source string + driver = "clickhouse" + ) + if config.Pass != "" { + source = fmt.Sprintf( + "clickhouse://%s:%s@%s:%s/%s", + config.User, config.Pass, config.Host, config.Port, config.Name) + } else { + source = fmt.Sprintf( + "clickhouse://%s@%s:%s/%s", + config.User, config.Host, config.Port, config.Name) + } + source += fmt.Sprintf( + "?charset=%s&debug=%s&compress=%s&secure=%s&skip_verify=%s", + config.Charset, gconv.String(config.Debug), gconv.String(config.Compress), gconv.String(config.Secure), gconv.String(config.SkipVerify), + ) + db, err := sql.Open(driver, source) + if err != nil { + return nil, err + } + return db, nil +} + +// Tables retrieves and returns the tables of current schema. +// It's mainly used in cli tool chain for automatically generating the models. +func (d *Driver) Tables(ctx context.Context, schema ...string) (tables []string, err error) { + var result gdb.Result + link, err := d.SlaveLink(schema...) + if err != nil { + return nil, err + } + query := fmt.Sprintf("select name from `system`.tables where database = '%s'", d.GetSchema()) + result, err = d.DoGetAll(ctx, link, query) + if err != nil { + return + } + for _, m := range result { + tables = append(tables, m["name"].String()) + } + return +} + +// TableFields retrieves and returns the fields' information of specified table of current schema. +// Also see DriverMysql.TableFields. +func (d *Driver) TableFields(ctx context.Context, table string, schema ...string) (fields map[string]*gdb.TableField, err error) { + charL, charR := d.GetChars() + table = gstr.Trim(table, charL+charR) + if gstr.Contains(table, " ") { + return nil, gerror.NewCode(gcode.CodeInvalidParameter, "function TableFields supports only single table operations") + } + useSchema := d.GetSchema() + if len(schema) > 0 && schema[0] != "" { + useSchema = schema[0] + } + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`clickhouse_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result gdb.Result + link gdb.Link + ) + if link, err = d.SlaveLink(useSchema); err != nil { + return nil + } + getColumnsSql := fmt.Sprintf("select name,position,default_expression,comment from `system`.columns c where database = '%s' and `table` = '%s'", d.GetSchema(), table) + result, err = d.DoGetAll(ctx, link, getColumnsSql) + if err != nil { + return nil + } + fields = make(map[string]*gdb.TableField) + for _, m := range result { + var ( + isNull = false + fieldType = m["type"].String() + ) + // in clickhouse , filed type like is Nullable(int) + fieldsResult, _ := gregex.MatchString(`^Nullable\((.*?)\)`, fieldType) + if len(fieldsResult) == 2 { + isNull = true + fieldType = fieldsResult[1] + } + fields[m["name"].String()] = &gdb.TableField{ + Index: m["position"].Int(), + Name: m["name"].String(), + Default: m["default_expression"].Val(), + Comment: m["comment"].String(), + //Key: m["Key"].String(), + Type: fieldType, + Null: isNull, + } + } + return fields + }, + ) + if v != nil { + fields = v.(map[string]*gdb.TableField) + } + return +} + +// FilteredLink retrieves and returns filtered `linkInfo` that can be using for +// logging or tracing purpose. +func (d *Driver) FilteredLink() string { + linkInfo := d.GetConfig().Link + if linkInfo == "" { + return "" + } + s, _ := gregex.ReplaceString( + `(.+?):(.+)@tcp(.+)`, + `$1:xxx@tcp$3`, + linkInfo, + ) + return s +} + +// PingMaster pings the master node to check authentication or keeps the connection alive. +func (d *Driver) PingMaster() error { + conn, err := d.Master() + if err != nil { + return err + } + return d.ping(conn) +} + +// PingSlave pings the slave node to check authentication or keeps the connection alive. +func (d *Driver) PingSlave() error { + conn, err := d.Slave() + if err != nil { + return err + } + return d.ping(conn) +} + +// ping Returns the Clickhouse specific error. +func (d *Driver) ping(conn *sql.DB) error { + err := conn.Ping() + if exception, ok := err.(*clickhouse.Exception); ok { + return errors.New(fmt.Sprintf("[%d]%s", exception.Code, exception.Message)) + } + return err +} + +// Transaction Clickhouse does not support transactions +// So when you call this method you get an error. +func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx *gdb.TX) error) error { + return errors.New("transaction operations are not supported") +} diff --git a/contrib/drivers/clickhouse/go.mod b/contrib/drivers/clickhouse/go.mod index 45192d7aa..2039e131c 100644 --- a/contrib/drivers/clickhouse/go.mod +++ b/contrib/drivers/clickhouse/go.mod @@ -3,11 +3,8 @@ module github.com/gogf/gf/contrib/drivers/clickhouse/v2 go 1.15 require ( - github.com/ClickHouse/clickhouse-go v1.5.2 - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/pretty v0.1.0 // indirect - github.com/stretchr/testify v1.7.0 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.0.11 + github.com/gogf/gf/v2 v2.0.0-00010101000000-000000000000 ) +replace github.com/gogf/gf/v2 => ../../../ diff --git a/contrib/drivers/clickhouse/go.sum b/contrib/drivers/clickhouse/go.sum index 4221d3424..466aecf9a 100644 --- a/contrib/drivers/clickhouse/go.sum +++ b/contrib/drivers/clickhouse/go.sum @@ -1,32 +1,197 @@ -github.com/ClickHouse/clickhouse-go v1.5.2 h1:yXgaOZ8WEHrd+okvZXjzulSt1zS33nM4ujfx9lVncl8= -github.com/ClickHouse/clickhouse-go v1.5.2/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/ClickHouse/clickhouse-go v1.5.3 h1:Vok8zUb/wlqc9u8oEqQzBMBRDoFd8NxPRqgYEqMnV88= +github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.0.11 h1:Oi0yzTeeX3ZSYi6DtELkpfz18UYlz+43ZEe1zMR2yk4= +github.com/ClickHouse/clickhouse-go/v2 v2.0.11/go.mod h1:IaAZKUDujV2l6q0QqQfr0hF1fUCeorC+am8/77QJe/g= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E= +github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= +github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= +github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= +github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +go.opentelemetry.io/otel v1.4.0 h1:7ESuKPq6zpjRaY5nvVDGiuwK7VAJ8MwkKnmNJ9whNZ4= +go.opentelemetry.io/otel v1.4.0/go.mod h1:jeAqMFKy2uLIxCtKxoFj0FAL5zAPKQagc3+GtBWakzk= +go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y= +go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM= +go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= +go.opentelemetry.io/otel/trace v1.4.0 h1:4OOUrPZdVFQkbzl/JSdvGCWIdw5ONXXxzHlaLlWppmo= +go.opentelemetry.io/otel/trace v1.4.0/go.mod h1:uc3eRsqDfWs9R7b92xbQbU42/eTNz4N+gLP8qJCi4aE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 h1:GLw7MR8AfAG2GmGcmVgObFOHXYypgGjnGno25RDwn3Y= +golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/database/gdb/gdb_core_config.go b/database/gdb/gdb_core_config.go index 451a14926..6f4bb41f7 100644 --- a/database/gdb/gdb_core_config.go +++ b/database/gdb/gdb_core_config.go @@ -48,6 +48,9 @@ type ConfigNode struct { UpdatedAt string `json:"updatedAt"` // (Optional) The filed name of table for automatic-filled updated datetime. DeletedAt string `json:"deletedAt"` // (Optional) The filed name of table for automatic-filled updated datetime. TimeMaintainDisabled bool `json:"timeMaintainDisabled"` // (Optional) Disable the automatic time maintaining feature. + Compress bool `json:"compress"` // (Optional,only Clickhouse) enable lz4 compression + Secure bool `json:"secure"` // (Optional,only Clickhouse) establish secure connection + SkipVerify bool `json:"skipVerify"` // (Optional,only Clickhouse) skip certificate verification } const ( From daf2b649efecef9be28193108e853b42797d4f61 Mon Sep 17 00:00:00 2001 From: daguang Date: Sat, 19 Feb 2022 16:59:17 +0800 Subject: [PATCH 02/15] improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 54 +++++- contrib/drivers/clickhouse/clickhouse_test.go | 162 ++++++++++++++++++ contrib/drivers/clickhouse/go.mod | 2 +- contrib/drivers/clickhouse/go.sum | 51 +----- database/gdb/gdb.go | 31 ++-- database/gdb/gdb_core.go | 12 ++ database/gdb/gdb_core_config.go | 2 - database/gdb/gdb_core_underlying.go | 7 +- 8 files changed, 252 insertions(+), 69 deletions(-) create mode 100644 contrib/drivers/clickhouse/clickhouse_test.go diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index 6f33fe0eb..4dea4a2ed 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -12,7 +12,7 @@ import ( "database/sql" "errors" "fmt" - "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go" "github.com/gogf/gf/v2/container/gmap" "github.com/gogf/gf/v2/database/gdb" @@ -30,7 +30,10 @@ type Driver struct { var ( // tableFieldsMap caches the table information retrieved from database. - tableFieldsMap = gmap.New(true) + tableFieldsMap = gmap.New(true) + ErrUnsupportedInsertIgnore = errors.New("unsupported method:InsertIgnore") + ErrUnsupportedInsertGetId = errors.New("unsupported method:InsertGetId") + ErrUnsupportedReplace = errors.New("unsupported method:Replace") ) func init() { @@ -68,13 +71,14 @@ func (d *Driver) Open(config *gdb.ConfigNode) (*sql.DB, error) { config.User, config.Host, config.Port, config.Name) } source += fmt.Sprintf( - "?charset=%s&debug=%s&compress=%s&secure=%s&skip_verify=%s", - config.Charset, gconv.String(config.Debug), gconv.String(config.Compress), gconv.String(config.Secure), gconv.String(config.SkipVerify), + "?charset=%s&debug=%s&compress=%s", + config.Charset, gconv.String(config.Debug), gconv.String(config.Compress), ) db, err := sql.Open(driver, source) if err != nil { return nil, err } + return db, nil } @@ -86,7 +90,7 @@ func (d *Driver) Tables(ctx context.Context, schema ...string) (tables []string, if err != nil { return nil, err } - query := fmt.Sprintf("select name from `system`.tables where database = '%s'", d.GetSchema()) + query := fmt.Sprintf("select name from `system`.tables where database = '%s'", d.GetConfig().Name) result, err = d.DoGetAll(ctx, link, query) if err != nil { return @@ -119,7 +123,7 @@ func (d *Driver) TableFields(ctx context.Context, table string, schema ...string if link, err = d.SlaveLink(useSchema); err != nil { return nil } - getColumnsSql := fmt.Sprintf("select name,position,default_expression,comment from `system`.columns c where database = '%s' and `table` = '%s'", d.GetSchema(), table) + getColumnsSql := fmt.Sprintf("select name,position,default_expression,comment from `system`.columns c where database = '%s' and `table` = '%s'", d.GetConfig().Name, table) result, err = d.DoGetAll(ctx, link, getColumnsSql) if err != nil { return nil @@ -202,3 +206,41 @@ func (d *Driver) ping(conn *sql.DB) error { func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx *gdb.TX) error) error { return errors.New("transaction operations are not supported") } + +// DoUpdateSQL in clickhouse ,use update must use alter +// eg. +// ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr +func (d *Driver) DoUpdateSQL(ctx context.Context, link gdb.Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) { + return d.Core.DoExec(ctx, link, fmt.Sprintf("ALTER TABLE %s UPDATE %s%s", table, updates, condition), args...) +} + +// DoDeleteSQL in clickhouse , delete must use alter +// eg. +// ALTER TABLE [db.]table DELETE WHERE filter_expr +func (d *Driver) DoDeleteSQL(ctx context.Context, link gdb.Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) { + return d.Core.DoExec(ctx, link, fmt.Sprintf("ALTER TABLE %s DELETE %s", table, condition), args...) +} + +func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, data gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { + return +} + +func (d *Driver) DoCommit(ctx context.Context, in gdb.DoCommitInput) (out gdb.DoCommitOutput, err error) { + in.IsIgnoreResult = true + return d.Core.DoCommit(ctx, in) +} + +// InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. +func (d *Driver) InsertIgnore(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { + return nil, ErrUnsupportedInsertIgnore +} + +// InsertAndGetId Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. +func (d *Driver) InsertAndGetId(ctx context.Context, table string, data interface{}, batch ...int) (int64, error) { + return 0, ErrUnsupportedInsertGetId +} + +// Replace Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. +func (d *Driver) Replace(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { + return nil, ErrUnsupportedReplace +} diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go new file mode 100644 index 000000000..76274d6a9 --- /dev/null +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -0,0 +1,162 @@ +package clickhouse + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/test/gtest" + "github.com/gogf/gf/v2/util/grand" +) + +// table DDL +// CREATE TABLE visits +// ( +// id UInt64, +// duration Float64, +// url String, +// created DateTime +//) +// ENGINE = MergeTree() +// PRIMARY KEY id +// ORDER BY id +func InitClickhouse() gdb.DB { + connect, err := gdb.New(gdb.ConfigNode{ + Host: "127.0.0.1", + Port: "9000", + User: "default", + Name: "default", + Type: "clickhouse", + Debug: true, + Compress: true, + }) + gtest.AssertNil(err) + gtest.AssertNE(connect, nil) + return connect +} + +func TestDriverClickhouse_Create(t *testing.T) { + gtest.AssertNil(createClickhouseTable(InitClickhouse())) +} + +func createClickhouseTable(connect gdb.DB) error { + sqlStr := "CREATE TABLE IF NOT EXISTS visits (id UInt64,duration Float64,url String,created DateTime) ENGINE = MergeTree() PRIMARY KEY id ORDER BY id" + _, err := connect.Exec(context.Background(), sqlStr) + return err +} + +func dropClickhouseTable(conn gdb.DB) { + sqlStr := fmt.Sprintf("DROP TABLE IF EXISTS `visits`") + _, _ = conn.Exec(context.Background(), sqlStr) +} + +func TestDriverClickhouse_New(t *testing.T) { + connect := InitClickhouse() + gtest.AssertNE(connect, nil) + gtest.AssertNil(connect.PingMaster()) + gtest.AssertNil(connect.PingSlave()) +} + +func TestDriverClickhouse_Tables(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + tables, err := connect.Tables(context.Background()) + gtest.AssertNil(err) + gtest.AssertNE(len(tables), 0) +} + +func TestDriverClickhouse_Transaction(t *testing.T) { + connect := InitClickhouse() + defer dropClickhouseTable(connect) + gtest.AssertNE(connect.Transaction(context.Background(), func(ctx context.Context, tx *gdb.TX) error { + return nil + }), nil) +} + +func TestDriverClickhouse_DoDelete(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + _, err := connect.Model("visits").Where("created >", "2021-01-01 00:00:00").Delete() + gtest.AssertNil(err) +} + +func TestDriverClickhouse_DoUpdate(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + _, err := connect.Model("visits").Where("created > ", "2021-01-01 15:15:15").Data(g.Map{ + "created": time.Now().Format("2006-01-02 15:04:05"), + }).Update() + gtest.AssertNil(err) +} + +func TestDriverClickhouse_Select(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + data, err := connect.Model("visits").All() + gtest.AssertNil(err) + gtest.AssertEQ(len(data), 0) +} + +func TestDriverClickhouse_DoInsert(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + type insertItem struct { + Id int `orm:"id"` + Duration float64 `orm:"duration"` + Url string `orm:"url"` + Created string `orm:"created"` + } + var ( + ctx = context.Background() + insertUrl = "https://goframe.org" + // insert one data + item = insertItem{ + Id: 0, + Duration: 1, + Url: insertUrl, + Created: time.Now().Format("2006-01-02 15:04:05"), + } + ) + _, err := connect.Model("visits").Ctx(ctx).Data(item).Insert() + gtest.AssertNil(err) + _, err = connect.Model("visits").Ctx(ctx).Data(item).InsertIgnore() + gtest.AssertNil(err) + + _, err = connect.Model("visits").Ctx(ctx).Data(item).InsertAndGetId() + _, err = connect.Model("visits").Ctx(ctx).Data(item).Save() + gtest.AssertNil(err) + // insert array data + list := []*insertItem{} + for i := 0; i < 999; i++ { + list = append(list, &insertItem{ + Id: grand.Intn(999), + Duration: float64(grand.Intn(999)), + Url: insertUrl, + Created: time.Now().Format("2006-01-02 15:04:05"), + }) + } + _, err = connect.Model("visits").Ctx(ctx).Data(list).Insert() + gtest.AssertNil(err) + _, err = connect.Model("visits").Ctx(ctx).Data(list).InsertIgnore() + gtest.AssertNil(err) + _, err = connect.Model("visits").Ctx(ctx).Data(list).InsertAndGetId() + _, err = connect.Model("visits").Ctx(ctx).Data(list).Save() + gtest.AssertNil(err) +} + +func TestDriverClickhouse_DoExec(t *testing.T) { + connect := InitClickhouse() + gtest.AssertNil(createClickhouseTable(connect)) + defer dropClickhouseTable(connect) + sqlStr := "OPTIMIZE table visits" + _, err := connect.Exec(context.Background(), sqlStr) + gtest.AssertNil(err) +} diff --git a/contrib/drivers/clickhouse/go.mod b/contrib/drivers/clickhouse/go.mod index 2039e131c..886a68985 100644 --- a/contrib/drivers/clickhouse/go.mod +++ b/contrib/drivers/clickhouse/go.mod @@ -3,7 +3,7 @@ module github.com/gogf/gf/contrib/drivers/clickhouse/v2 go 1.15 require ( - github.com/ClickHouse/clickhouse-go/v2 v2.0.11 + github.com/ClickHouse/clickhouse-go v1.5.2 github.com/gogf/gf/v2 v2.0.0-00010101000000-000000000000 ) diff --git a/contrib/drivers/clickhouse/go.sum b/contrib/drivers/clickhouse/go.sum index 466aecf9a..e59c58abc 100644 --- a/contrib/drivers/clickhouse/go.sum +++ b/contrib/drivers/clickhouse/go.sum @@ -1,15 +1,14 @@ github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/ClickHouse/clickhouse-go v1.5.3 h1:Vok8zUb/wlqc9u8oEqQzBMBRDoFd8NxPRqgYEqMnV88= -github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.0.11 h1:Oi0yzTeeX3ZSYi6DtELkpfz18UYlz+43ZEe1zMR2yk4= -github.com/ClickHouse/clickhouse-go/v2 v2.0.11/go.mod h1:IaAZKUDujV2l6q0QqQfr0hF1fUCeorC+am8/77QJe/g= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/ClickHouse/clickhouse-go v1.5.2 h1:yXgaOZ8WEHrd+okvZXjzulSt1zS33nM4ujfx9lVncl8= +github.com/ClickHouse/clickhouse-go v1.5.2/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E= github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -22,18 +21,12 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -48,21 +41,14 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -72,7 +58,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -86,46 +71,31 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= -github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= -github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI= go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= -go.opentelemetry.io/otel v1.4.0 h1:7ESuKPq6zpjRaY5nvVDGiuwK7VAJ8MwkKnmNJ9whNZ4= -go.opentelemetry.io/otel v1.4.0/go.mod h1:jeAqMFKy2uLIxCtKxoFj0FAL5zAPKQagc3+GtBWakzk= go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y= go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM= +go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4= go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= -go.opentelemetry.io/otel/trace v1.4.0 h1:4OOUrPZdVFQkbzl/JSdvGCWIdw5ONXXxzHlaLlWppmo= -go.opentelemetry.io/otel/trace v1.4.0/go.mod h1:uc3eRsqDfWs9R7b92xbQbU42/eTNz4N+gLP8qJCi4aE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -134,7 +104,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -143,7 +112,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -153,9 +121,8 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -164,9 +131,7 @@ golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 h1:GLw7MR8AfAG2GmGcmVgObF golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index 3db918764..85b8f0e7d 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -97,12 +97,14 @@ type DB interface { DoGetAll(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoGetAll. DoInsert(ctx context.Context, link Link, table string, data List, option DoInsertOption) (result sql.Result, err error) // See Core.DoInsert. DoUpdate(ctx context.Context, link Link, table string, data interface{}, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoUpdate. - DoDelete(ctx context.Context, link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete. - DoQuery(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoQuery. - DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec. - DoFilter(ctx context.Context, link Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) // See Core.DoFilter. - DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutput, err error) // See Core.DoCommit. - DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error) // See Core.DoPrepare. + DoUpdateSQL(ctx context.Context, link Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) + DoDelete(ctx context.Context, link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete. + DoDeleteSQL(ctx context.Context, link Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) + DoQuery(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoQuery. + DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec. + DoFilter(ctx context.Context, link Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) // See Core.DoFilter. + DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutput, err error) // See Core.DoCommit. + DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error) // See Core.DoPrepare. // =========================================================================== // Query APIs for convenience purpose. @@ -185,14 +187,15 @@ type Core struct { // DoCommitInput is the input parameters for function DoCommit. type DoCommitInput struct { - Db *sql.DB - Tx *sql.Tx - Stmt *sql.Stmt - Link Link - Sql string - Args []interface{} - Type string - IsTransaction bool + Db *sql.DB + Tx *sql.Tx + Stmt *sql.Stmt + Link Link + Sql string + Args []interface{} + Type string + IsTransaction bool + IsIgnoreResult bool } // DoCommitOutput is the output parameters for function DoCommit. diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 255b67c77..9f4f1ca59 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -598,6 +598,12 @@ func (c *Core) DoUpdate(ctx context.Context, link Link, table string, data inter return nil, err } } + return c.db.DoUpdateSQL(ctx, link, table, updates, condition, args...) +} + +// DoUpdateSQL Adapt to difference in different drives +// For example, Clickhouse's update +func (c *Core) DoUpdateSQL(ctx context.Context, link Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) { return c.db.DoExec(ctx, link, fmt.Sprintf("UPDATE %s SET %s%s", table, updates, condition), args...) } @@ -625,6 +631,12 @@ func (c *Core) DoDelete(ctx context.Context, link Link, table string, condition } } table = c.QuotePrefixTableName(table) + return c.db.DoDeleteSQL(ctx, link, table, condition, args...) +} + +// DoDeleteSQL Adapt to difference in different drives +// For example, Clickhouse's delete +func (c *Core) DoDeleteSQL(ctx context.Context, link Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) { return c.db.DoExec(ctx, link, fmt.Sprintf("DELETE FROM %s%s", table, condition), args...) } diff --git a/database/gdb/gdb_core_config.go b/database/gdb/gdb_core_config.go index 6f4bb41f7..85868abb1 100644 --- a/database/gdb/gdb_core_config.go +++ b/database/gdb/gdb_core_config.go @@ -49,8 +49,6 @@ type ConfigNode struct { DeletedAt string `json:"deletedAt"` // (Optional) The filed name of table for automatic-filled updated datetime. TimeMaintainDisabled bool `json:"timeMaintainDisabled"` // (Optional) Disable the automatic time maintaining feature. Compress bool `json:"compress"` // (Optional,only Clickhouse) enable lz4 compression - Secure bool `json:"secure"` // (Optional,only Clickhouse) establish secure connection - SkipVerify bool `json:"skipVerify"` // (Optional,only Clickhouse) skip certificate verification } const ( diff --git a/database/gdb/gdb_core_underlying.go b/database/gdb/gdb_core_underlying.go index 60a255c9a..4e0c0d068 100644 --- a/database/gdb/gdb_core_underlying.go +++ b/database/gdb/gdb_core_underlying.go @@ -11,6 +11,9 @@ import ( "context" "database/sql" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + "github.com/gogf/gf/v2" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/errors/gcode" @@ -18,8 +21,6 @@ import ( "github.com/gogf/gf/v2/internal/intlog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/guid" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" ) // Query commits one query SQL to underlying driver and returns the execution result. @@ -210,7 +211,7 @@ func (c *Core) DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutp } // Result handling. switch { - case sqlResult != nil: + case sqlResult != nil && !in.IsIgnoreResult: rowsAffected, err = sqlResult.RowsAffected() out.Result = sqlResult From ea60f7e05464e6b34eec0dc11eba5b9bedaada88 Mon Sep 17 00:00:00 2001 From: daguang Date: Sat, 19 Feb 2022 17:49:53 +0800 Subject: [PATCH 03/15] improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 10 ++-- contrib/drivers/clickhouse/clickhouse_test.go | 50 +++++++++++++++++++ database/gdb/gdb.go | 5 +- database/gdb/gdb_core.go | 14 +++--- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index 4dea4a2ed..a5fc94de8 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -221,15 +221,17 @@ func (d *Driver) DoDeleteSQL(ctx context.Context, link gdb.Link, table string, c return d.Core.DoExec(ctx, link, fmt.Sprintf("ALTER TABLE %s DELETE %s", table, condition), args...) } -func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, data gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { - return -} - +// DoCommit commits current sql and arguments to underlying sql driver. func (d *Driver) DoCommit(ctx context.Context, in gdb.DoCommitInput) (out gdb.DoCommitOutput, err error) { in.IsIgnoreResult = true return d.Core.DoCommit(ctx, in) } +func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { + option.IsIgnoreResult = true + return d.Core.DoInsert(ctx, link, table, list, option) +} + // InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. func (d *Driver) InsertIgnore(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { return nil, ErrUnsupportedInsertIgnore diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 76274d6a9..8f867be83 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -9,6 +9,7 @@ import ( "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/test/gtest" + "github.com/gogf/gf/v2/util/gconv" "github.com/gogf/gf/v2/util/grand" ) @@ -104,6 +105,55 @@ func TestDriverClickhouse_Select(t *testing.T) { gtest.AssertEQ(len(data), 0) } +func TestDriver_InsertIgnore(t *testing.T) { + connect := InitClickhouse() + _, err := connect.InsertIgnore(context.Background(), "", nil) + gtest.AssertEQ(err, ErrUnsupportedInsertIgnore) +} + +func TestDriver_InsertAndGetId(t *testing.T) { + connect := InitClickhouse() + _, err := connect.InsertAndGetId(context.Background(), "", nil) + gtest.AssertEQ(err, ErrUnsupportedInsertGetId) +} + +func TestDriver_Replace(t *testing.T) { + connect := InitClickhouse() + _, err := connect.Replace(context.Background(), "", nil) + gtest.AssertEQ(err, ErrUnsupportedReplace) +} + +func TestDriverClickhouse_DoInsertOne(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + _, err := connect.Model("visits").Data(g.Map{ + "id": grand.Intn(999), + "duration": grand.Intn(999), + "url": grand.Intn(999), + "created": grand.Intn(999), + }).Insert() + gtest.AssertNil(err) +} + +func TestDriver_DoInsertMany(t *testing.T) { + connect := InitClickhouse() + gtest.AssertEQ(createClickhouseTable(connect), nil) + defer dropClickhouseTable(connect) + tx, err := connect.Begin(context.Background()) + for i := 0; i < 10; i++ { + _, err = tx.Model("visits").Data(g.Map{ + "id": grand.Intn(999), + "duration": float64(grand.Intn(999)), + "url": gconv.String(grand.Intn(999)), + "created": time.Now().Format("2006-01-02 15:04:05"), + }). + Save() + gtest.AssertNil(err) + } + gtest.AssertNil(tx.Commit()) +} + func TestDriverClickhouse_DoInsert(t *testing.T) { connect := InitClickhouse() gtest.AssertEQ(createClickhouseTable(connect), nil) diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index 85b8f0e7d..ecbd8cfab 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -242,8 +242,9 @@ type Sql struct { type DoInsertOption struct { OnDuplicateStr string OnDuplicateMap map[string]interface{} - InsertOption int // Insert operation. - BatchCount int // Batch count for batch inserting. + InsertOption int // Insert operation. + BatchCount int // Batch count for batch inserting. + IsIgnoreResult bool // IgnoreResult } // TableField is the struct for table field. diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 9f4f1ca59..e92e1be94 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -450,12 +450,14 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, if err != nil { return stdSqlResult, err } - if affectedRows, err = stdSqlResult.RowsAffected(); err != nil { - err = gerror.WrapCode(gcode.CodeDbOperationError, err, `sql.Result.RowsAffected failed`) - return stdSqlResult, err - } else { - batchResult.Result = stdSqlResult - batchResult.Affected += affectedRows + if !option.IsIgnoreResult { + if affectedRows, err = stdSqlResult.RowsAffected(); err != nil { + err = gerror.WrapCode(gcode.CodeDbOperationError, err, `sql.Result.RowsAffected failed`) + return stdSqlResult, err + } else { + batchResult.Result = stdSqlResult + batchResult.Affected += affectedRows + } } params = params[:0] valueHolder = valueHolder[:0] From 455d724c014d236c7f2c2a156e8e5c37e5f30cc1 Mon Sep 17 00:00:00 2001 From: daguang Date: Sat, 19 Feb 2022 23:10:31 +0800 Subject: [PATCH 04/15] improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 63 ++++++++++++++++--- contrib/drivers/clickhouse/clickhouse_test.go | 46 +++++--------- database/gdb/gdb.go | 5 +- database/gdb/gdb_core.go | 18 +++--- 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index a5fc94de8..f6cb9cff8 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "github.com/ClickHouse/clickhouse-go" + "strings" "github.com/gogf/gf/v2/container/gmap" "github.com/gogf/gf/v2/database/gdb" @@ -34,6 +35,8 @@ var ( ErrUnsupportedInsertIgnore = errors.New("unsupported method:InsertIgnore") ErrUnsupportedInsertGetId = errors.New("unsupported method:InsertGetId") ErrUnsupportedReplace = errors.New("unsupported method:Replace") + ErrUnsupportedBegin = errors.New("unsupported method:Begin") + ErrUnsupportedTransaction = errors.New("unsupported method:Transaction") ) func init() { @@ -201,12 +204,6 @@ func (d *Driver) ping(conn *sql.DB) error { return err } -// Transaction Clickhouse does not support transactions -// So when you call this method you get an error. -func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx *gdb.TX) error) error { - return errors.New("transaction operations are not supported") -} - // DoUpdateSQL in clickhouse ,use update must use alter // eg. // ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr @@ -228,8 +225,50 @@ func (d *Driver) DoCommit(ctx context.Context, in gdb.DoCommitInput) (out gdb.Do } func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { - option.IsIgnoreResult = true - return d.Core.DoInsert(ctx, link, table, list, option) + var ( + keys []string // Field names. + valueHolder = make([]string, 0) + ) + // Handle the field names and placeholders. + for k := range list[0] { + keys = append(keys, k) + valueHolder = append(valueHolder, "?") + } + // Prepare the batch result pointer. + var ( + charL, charR = d.Core.GetChars() + batchResult = new(gdb.SqlResult) + keysStr = charL + strings.Join(keys, charR+","+charL) + charR + holderStr = strings.Join(valueHolder, ",") + listLength = len(list) + tx = &gdb.TX{} + stdSqlResult sql.Result + stmt *gdb.Stmt + ) + tx, err = d.Core.Begin(ctx) + if err != nil { + return + } + stmt, err = tx.Prepare(fmt.Sprintf( + "INSERT INTO %s(%s) VALUES (%s)", + d.QuotePrefixTableName(table), keysStr, + holderStr, + )) + if err != nil { + return + } + for i := 0; i < listLength; i++ { + params := []interface{}{} // Values that will be committed to underlying database driver. + for _, k := range keys { + params = append(params, list[i][k]) + } + // Prepare is allowed to execute only once in a transaction opened by clickhouse + stdSqlResult, err = stmt.ExecContext(ctx, params...) + if err != nil { + return stdSqlResult, err + } + } + return batchResult, tx.Commit() } // InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. @@ -246,3 +285,11 @@ func (d *Driver) InsertAndGetId(ctx context.Context, table string, data interfac func (d *Driver) Replace(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { return nil, ErrUnsupportedReplace } + +func (d *Driver) Begin(ctx context.Context) (tx *gdb.TX, err error) { + return nil, ErrUnsupportedBegin +} + +func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx *gdb.TX) error) error { + return ErrUnsupportedTransaction +} diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 8f867be83..821af332d 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -129,9 +129,9 @@ func TestDriverClickhouse_DoInsertOne(t *testing.T) { defer dropClickhouseTable(connect) _, err := connect.Model("visits").Data(g.Map{ "id": grand.Intn(999), - "duration": grand.Intn(999), - "url": grand.Intn(999), - "created": grand.Intn(999), + "duration": float64(grand.Intn(999)), + "url": gconv.String(grand.Intn(999)), + "created": time.Now().Format("2006-01-02 15:04:05"), }).Insert() gtest.AssertNil(err) } @@ -141,23 +141,13 @@ func TestDriver_DoInsertMany(t *testing.T) { gtest.AssertEQ(createClickhouseTable(connect), nil) defer dropClickhouseTable(connect) tx, err := connect.Begin(context.Background()) - for i := 0; i < 10; i++ { - _, err = tx.Model("visits").Data(g.Map{ - "id": grand.Intn(999), - "duration": float64(grand.Intn(999)), - "url": gconv.String(grand.Intn(999)), - "created": time.Now().Format("2006-01-02 15:04:05"), - }). - Save() - gtest.AssertNil(err) - } - gtest.AssertNil(tx.Commit()) + gtest.AssertEQ(err, ErrUnsupportedBegin) + gtest.AssertNil(tx) } func TestDriverClickhouse_DoInsert(t *testing.T) { connect := InitClickhouse() gtest.AssertEQ(createClickhouseTable(connect), nil) - defer dropClickhouseTable(connect) type insertItem struct { Id int `orm:"id"` Duration float64 `orm:"duration"` @@ -165,27 +155,24 @@ func TestDriverClickhouse_DoInsert(t *testing.T) { Created string `orm:"created"` } var ( - ctx = context.Background() insertUrl = "https://goframe.org" - // insert one data - item = insertItem{ + total = 0 + item = insertItem{ Id: 0, Duration: 1, Url: insertUrl, Created: time.Now().Format("2006-01-02 15:04:05"), } ) - _, err := connect.Model("visits").Ctx(ctx).Data(item).Insert() + _, err := connect.Model("visits").Data(item).Insert() gtest.AssertNil(err) - _, err = connect.Model("visits").Ctx(ctx).Data(item).InsertIgnore() + _, err = connect.Model("visits").Data(item).Save() gtest.AssertNil(err) - - _, err = connect.Model("visits").Ctx(ctx).Data(item).InsertAndGetId() - _, err = connect.Model("visits").Ctx(ctx).Data(item).Save() + total, err = connect.Model("visits").Count() gtest.AssertNil(err) - // insert array data + gtest.AssertEQ(total, 2) list := []*insertItem{} - for i := 0; i < 999; i++ { + for i := 0; i < 50; i++ { list = append(list, &insertItem{ Id: grand.Intn(999), Duration: float64(grand.Intn(999)), @@ -193,13 +180,14 @@ func TestDriverClickhouse_DoInsert(t *testing.T) { Created: time.Now().Format("2006-01-02 15:04:05"), }) } - _, err = connect.Model("visits").Ctx(ctx).Data(list).Insert() + _, err = connect.Model("visits").Data(list).Insert() gtest.AssertNil(err) - _, err = connect.Model("visits").Ctx(ctx).Data(list).InsertIgnore() + _, err = connect.Model("visits").Data(list).Save() gtest.AssertNil(err) - _, err = connect.Model("visits").Ctx(ctx).Data(list).InsertAndGetId() - _, err = connect.Model("visits").Ctx(ctx).Data(list).Save() + total, err = connect.Model("visits").Count() gtest.AssertNil(err) + gtest.AssertEQ(total, 102) + dropClickhouseTable(connect) } func TestDriverClickhouse_DoExec(t *testing.T) { diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index ecbd8cfab..85b8f0e7d 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -242,9 +242,8 @@ type Sql struct { type DoInsertOption struct { OnDuplicateStr string OnDuplicateMap map[string]interface{} - InsertOption int // Insert operation. - BatchCount int // Batch count for batch inserting. - IsIgnoreResult bool // IgnoreResult + InsertOption int // Insert operation. + BatchCount int // Batch count for batch inserting. } // TableField is the struct for table field. diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index e92e1be94..3290d2ec2 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -416,7 +416,7 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, operation = GetInsertOperationByOption(option.InsertOption) ) if option.InsertOption == InsertOptionSave { - onDuplicateStr = c.formatOnDuplicate(keys, option) + onDuplicateStr = c.FormatOnDuplicate(keys, option) } var ( listLength = len(list) @@ -450,14 +450,12 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, if err != nil { return stdSqlResult, err } - if !option.IsIgnoreResult { - if affectedRows, err = stdSqlResult.RowsAffected(); err != nil { - err = gerror.WrapCode(gcode.CodeDbOperationError, err, `sql.Result.RowsAffected failed`) - return stdSqlResult, err - } else { - batchResult.Result = stdSqlResult - batchResult.Affected += affectedRows - } + if affectedRows, err = stdSqlResult.RowsAffected(); err != nil { + err = gerror.WrapCode(gcode.CodeDbOperationError, err, `sql.Result.RowsAffected failed`) + return stdSqlResult, err + } else { + batchResult.Result = stdSqlResult + batchResult.Affected += affectedRows } params = params[:0] valueHolder = valueHolder[:0] @@ -466,7 +464,7 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, return batchResult, nil } -func (c *Core) formatOnDuplicate(columns []string, option DoInsertOption) string { +func (c *Core) FormatOnDuplicate(columns []string, option DoInsertOption) string { var onDuplicateStr string if option.OnDuplicateStr != "" { onDuplicateStr = option.OnDuplicateStr From 157e936f241c725e7d31943d41735f965797925c Mon Sep 17 00:00:00 2001 From: daguang Date: Wed, 23 Feb 2022 22:51:37 +0800 Subject: [PATCH 05/15] :sparkles: improve clickhouse driver --- .github/workflows/gf.yml | 7 +++ contrib/drivers/clickhouse/clickhouse.go | 47 +++++++++++------- contrib/drivers/clickhouse/clickhouse_test.go | 48 ++++++++++++++++--- database/gdb/gdb.go | 31 ++++++------ database/gdb/gdb_core.go | 12 ----- database/gdb/gdb_core_config.go | 1 - database/gdb/gdb_core_underlying.go | 3 +- 7 files changed, 95 insertions(+), 54 deletions(-) diff --git a/.github/workflows/gf.yml b/.github/workflows/gf.yml index 865216392..e3b33c659 100644 --- a/.github/workflows/gf.yml +++ b/.github/workflows/gf.yml @@ -76,6 +76,13 @@ jobs: --health-timeout 5s --health-retries 10 + clickhouse-server: + image: yandex/clickhouse-server + ports: + - 9000:9000 + - 8123:8123 + - 9001:9001 + # strategy set strategy: diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index f6cb9cff8..a10167819 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -37,6 +37,7 @@ var ( ErrUnsupportedReplace = errors.New("unsupported method:Replace") ErrUnsupportedBegin = errors.New("unsupported method:Begin") ErrUnsupportedTransaction = errors.New("unsupported method:Transaction") + ErrSQLNull = errors.New("SQL cannot be null") ) func init() { @@ -74,8 +75,8 @@ func (d *Driver) Open(config *gdb.ConfigNode) (*sql.DB, error) { config.User, config.Host, config.Port, config.Name) } source += fmt.Sprintf( - "?charset=%s&debug=%s&compress=%s", - config.Charset, gconv.String(config.Debug), gconv.String(config.Compress), + "?charset=%s&debug=%s", + config.Charset, gconv.String(config.Debug), ) db, err := sql.Open(driver, source) if err != nil { @@ -204,24 +205,38 @@ func (d *Driver) ping(conn *sql.DB) error { return err } -// DoUpdateSQL in clickhouse ,use update must use alter -// eg. -// ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr -func (d *Driver) DoUpdateSQL(ctx context.Context, link gdb.Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) { - return d.Core.DoExec(ctx, link, fmt.Sprintf("ALTER TABLE %s UPDATE %s%s", table, updates, condition), args...) -} - -// DoDeleteSQL in clickhouse , delete must use alter -// eg. -// ALTER TABLE [db.]table DELETE WHERE filter_expr -func (d *Driver) DoDeleteSQL(ctx context.Context, link gdb.Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) { - return d.Core.DoExec(ctx, link, fmt.Sprintf("ALTER TABLE %s DELETE %s", table, condition), args...) +// DoFilter handles the sql before posts it to database. +func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) { + // replace MySQL to Clickhouse SQL grammar + // MySQL eg: UPDATE visits SET xxx + // Clickhouse eg: ALTER TABLE visits UPDATE xxx + // MySQL eg: DELETE FROM VISIT + // Clickhouse eg: ALTER TABLE VISIT DELETE WHERE filter_expr + result, err := gregex.MatchString("(?i)^UPDATE|DELETE", sql) + if err != nil { + return "", nil, err + } + if len(result) != 0 { + sqlSlice := strings.Split(sql, " ") + if len(sqlSlice) < 3 { + return "", nil, ErrSQLNull + } + ck := []string{"ALTER", "TABLE"} + switch strings.ToUpper(result[0]) { + case "UPDATE": + sqlSlice = append(append(append(ck, sqlSlice[1]), result[0]), sqlSlice[3:]...) + return strings.Join(sqlSlice, " "), args, nil + case "DELETE": + sqlSlice = append(append(append(ck, sqlSlice[2]), result[0]), sqlSlice[3:]...) + return strings.Join(sqlSlice, " "), args, nil + } + } + return sql, args, nil } // DoCommit commits current sql and arguments to underlying sql driver. func (d *Driver) DoCommit(ctx context.Context, in gdb.DoCommitInput) (out gdb.DoCommitOutput, err error) { - in.IsIgnoreResult = true - return d.Core.DoCommit(ctx, in) + return d.Core.DoCommit(context.WithValue(ctx, "isIgnoreResult", true), in) } func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 821af332d..367560173 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -26,13 +26,12 @@ import ( // ORDER BY id func InitClickhouse() gdb.DB { connect, err := gdb.New(gdb.ConfigNode{ - Host: "127.0.0.1", - Port: "9000", - User: "default", - Name: "default", - Type: "clickhouse", - Debug: true, - Compress: true, + Host: "127.0.0.1", + Port: "9000", + User: "default", + Name: "default", + Type: "clickhouse", + Debug: true, }) gtest.AssertNil(err) gtest.AssertNE(connect, nil) @@ -94,6 +93,12 @@ func TestDriverClickhouse_DoUpdate(t *testing.T) { "created": time.Now().Format("2006-01-02 15:04:05"), }).Update() gtest.AssertNil(err) + _, err = connect.Model("visits").Data(g.Map{ + "created": time.Now().Format("2006-01-02 15:04:05"), + }).Update() + gtest.AssertNE(err, nil) + _, err = connect.Model("visits").Update() + gtest.AssertNE(err, nil) } func TestDriverClickhouse_Select(t *testing.T) { @@ -198,3 +203,32 @@ func TestDriverClickhouse_DoExec(t *testing.T) { _, err := connect.Exec(context.Background(), sqlStr) gtest.AssertNil(err) } + +func TestDriver_DoFilter(t *testing.T) { + rawSQL := "select * from visits where 1 = 1" + this := Driver{} + replaceSQL, _, err := this.DoFilter(nil, nil, rawSQL, nil) + gtest.AssertNil(err) + gtest.AssertEQ(rawSQL, replaceSQL) + rawSQL = "update visit set url = '1'" + replaceSQL, _, err = this.DoFilter(nil, nil, rawSQL, nil) + gtest.AssertNil(err) + // this SQL can't run ,clickhouse will report an error because there is no WHERE statement + gtest.AssertEQ(replaceSQL, "ALTER TABLE visit update url = '1'") + rawSQL = "delete from visit" + replaceSQL, _, err = this.DoFilter(nil, nil, rawSQL, nil) + gtest.AssertNil(err) + // this SQL can't run ,clickhouse will report an error because there is no WHERE statement + gtest.AssertEQ(replaceSQL, "ALTER TABLE visit delete") + + rawSQL = "update visit set url = '1' where url = '0'" + replaceSQL, _, err = this.DoFilter(nil, nil, rawSQL, nil) + gtest.AssertNil(err) + // this SQL can't run ,clickhouse will report an error because there is no WHERE statement + gtest.AssertEQ(replaceSQL, "ALTER TABLE visit update url = '1' where url = '0'") + rawSQL = "delete from visit where url='0'" + replaceSQL, _, err = this.DoFilter(nil, nil, rawSQL, nil) + gtest.AssertNil(err) + // this SQL can't run ,clickhouse will report an error because there is no WHERE statement + gtest.AssertEQ(replaceSQL, "ALTER TABLE visit delete where url='0'") +} diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index 85b8f0e7d..3db918764 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -97,14 +97,12 @@ type DB interface { DoGetAll(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoGetAll. DoInsert(ctx context.Context, link Link, table string, data List, option DoInsertOption) (result sql.Result, err error) // See Core.DoInsert. DoUpdate(ctx context.Context, link Link, table string, data interface{}, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoUpdate. - DoUpdateSQL(ctx context.Context, link Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) - DoDelete(ctx context.Context, link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete. - DoDeleteSQL(ctx context.Context, link Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) - DoQuery(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoQuery. - DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec. - DoFilter(ctx context.Context, link Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) // See Core.DoFilter. - DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutput, err error) // See Core.DoCommit. - DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error) // See Core.DoPrepare. + DoDelete(ctx context.Context, link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete. + DoQuery(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoQuery. + DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec. + DoFilter(ctx context.Context, link Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) // See Core.DoFilter. + DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutput, err error) // See Core.DoCommit. + DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error) // See Core.DoPrepare. // =========================================================================== // Query APIs for convenience purpose. @@ -187,15 +185,14 @@ type Core struct { // DoCommitInput is the input parameters for function DoCommit. type DoCommitInput struct { - Db *sql.DB - Tx *sql.Tx - Stmt *sql.Stmt - Link Link - Sql string - Args []interface{} - Type string - IsTransaction bool - IsIgnoreResult bool + Db *sql.DB + Tx *sql.Tx + Stmt *sql.Stmt + Link Link + Sql string + Args []interface{} + Type string + IsTransaction bool } // DoCommitOutput is the output parameters for function DoCommit. diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 3290d2ec2..3a40a3d25 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -598,12 +598,6 @@ func (c *Core) DoUpdate(ctx context.Context, link Link, table string, data inter return nil, err } } - return c.db.DoUpdateSQL(ctx, link, table, updates, condition, args...) -} - -// DoUpdateSQL Adapt to difference in different drives -// For example, Clickhouse's update -func (c *Core) DoUpdateSQL(ctx context.Context, link Link, table string, updates interface{}, condition string, args ...interface{}) (result sql.Result, err error) { return c.db.DoExec(ctx, link, fmt.Sprintf("UPDATE %s SET %s%s", table, updates, condition), args...) } @@ -631,12 +625,6 @@ func (c *Core) DoDelete(ctx context.Context, link Link, table string, condition } } table = c.QuotePrefixTableName(table) - return c.db.DoDeleteSQL(ctx, link, table, condition, args...) -} - -// DoDeleteSQL Adapt to difference in different drives -// For example, Clickhouse's delete -func (c *Core) DoDeleteSQL(ctx context.Context, link Link, table string, condition interface{}, args ...interface{}) (result sql.Result, err error) { return c.db.DoExec(ctx, link, fmt.Sprintf("DELETE FROM %s%s", table, condition), args...) } diff --git a/database/gdb/gdb_core_config.go b/database/gdb/gdb_core_config.go index 85868abb1..451a14926 100644 --- a/database/gdb/gdb_core_config.go +++ b/database/gdb/gdb_core_config.go @@ -48,7 +48,6 @@ type ConfigNode struct { UpdatedAt string `json:"updatedAt"` // (Optional) The filed name of table for automatic-filled updated datetime. DeletedAt string `json:"deletedAt"` // (Optional) The filed name of table for automatic-filled updated datetime. TimeMaintainDisabled bool `json:"timeMaintainDisabled"` // (Optional) Disable the automatic time maintaining feature. - Compress bool `json:"compress"` // (Optional,only Clickhouse) enable lz4 compression } const ( diff --git a/database/gdb/gdb_core_underlying.go b/database/gdb/gdb_core_underlying.go index 4e0c0d068..ae345c8b0 100644 --- a/database/gdb/gdb_core_underlying.go +++ b/database/gdb/gdb_core_underlying.go @@ -20,6 +20,7 @@ import ( "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/internal/intlog" "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" "github.com/gogf/gf/v2/util/guid" ) @@ -211,7 +212,7 @@ func (c *Core) DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutp } // Result handling. switch { - case sqlResult != nil && !in.IsIgnoreResult: + case sqlResult != nil && !gconv.Bool(ctx.Value("isIgnoreResult")): rowsAffected, err = sqlResult.RowsAffected() out.Result = sqlResult From 008e5ea19601fbf5578b967404bf43c735df51e1 Mon Sep 17 00:00:00 2001 From: daguang Date: Wed, 23 Feb 2022 22:53:28 +0800 Subject: [PATCH 06/15] :sparkles: improve clickhouse driver --- database/gdb/gdb_core.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 3a40a3d25..255b67c77 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -416,7 +416,7 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, operation = GetInsertOperationByOption(option.InsertOption) ) if option.InsertOption == InsertOptionSave { - onDuplicateStr = c.FormatOnDuplicate(keys, option) + onDuplicateStr = c.formatOnDuplicate(keys, option) } var ( listLength = len(list) @@ -464,7 +464,7 @@ func (c *Core) DoInsert(ctx context.Context, link Link, table string, list List, return batchResult, nil } -func (c *Core) FormatOnDuplicate(columns []string, option DoInsertOption) string { +func (c *Core) formatOnDuplicate(columns []string, option DoInsertOption) string { var onDuplicateStr string if option.OnDuplicateStr != "" { onDuplicateStr = option.OnDuplicateStr From 65341141fe5684930844688107094be35eb4f918 Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Thu, 24 Feb 2022 12:58:57 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/drivers/clickhouse/clickhouse.go | 6 ++---- contrib/drivers/clickhouse/clickhouse_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index a10167819..fa3c56120 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -252,10 +252,8 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list // Prepare the batch result pointer. var ( charL, charR = d.Core.GetChars() - batchResult = new(gdb.SqlResult) keysStr = charL + strings.Join(keys, charR+","+charL) + charR holderStr = strings.Join(valueHolder, ",") - listLength = len(list) tx = &gdb.TX{} stdSqlResult sql.Result stmt *gdb.Stmt @@ -272,7 +270,7 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list if err != nil { return } - for i := 0; i < listLength; i++ { + for i := 0; i < len(list); i++ { params := []interface{}{} // Values that will be committed to underlying database driver. for _, k := range keys { params = append(params, list[i][k]) @@ -283,7 +281,7 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list return stdSqlResult, err } } - return batchResult, tx.Commit() + return stdSqlResult, tx.Commit() } // InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 367560173..82cfae5a0 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -232,3 +232,12 @@ func TestDriver_DoFilter(t *testing.T) { // this SQL can't run ,clickhouse will report an error because there is no WHERE statement gtest.AssertEQ(replaceSQL, "ALTER TABLE visit delete where url='0'") } + +func TestDriver_TableFields(t *testing.T) { + connect := InitClickhouse() + gtest.AssertNil(createClickhouseTable(connect)) + defer dropClickhouseTable(connect) + field, err := connect.TableFields(context.Background(), "visits") + gtest.AssertNil(err) + gtest.AssertNQ(len(field), 4) +} From 00daeb318c22a160ce93e4849e7c187afe2ecbbe Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Thu, 24 Feb 2022 13:03:01 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/drivers/clickhouse/clickhouse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 82cfae5a0..67f532629 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -239,5 +239,5 @@ func TestDriver_TableFields(t *testing.T) { defer dropClickhouseTable(connect) field, err := connect.TableFields(context.Background(), "visits") gtest.AssertNil(err) - gtest.AssertNQ(len(field), 4) + gtest.AssertEQ(len(field), 4) } From a6bbb8424cbf52953fe4377c36ef38dbd81cf298 Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Thu, 24 Feb 2022 13:50:28 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/drivers/clickhouse/clickhouse_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 67f532629..064f5aa6c 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -240,4 +240,5 @@ func TestDriver_TableFields(t *testing.T) { field, err := connect.TableFields(context.Background(), "visits") gtest.AssertNil(err) gtest.AssertEQ(len(field), 4) + gtest.AssertNQ(field, nil) } From 5e3c0bd9aa887ae73f71b8b388292ea1b29b6ef8 Mon Sep 17 00:00:00 2001 From: daguang Date: Thu, 24 Feb 2022 21:06:26 +0800 Subject: [PATCH 10/15] :sparkles: improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 16 +++++++--------- contrib/drivers/clickhouse/clickhouse_test.go | 10 ++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index fa3c56120..f23fedfcf 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -65,19 +65,17 @@ func (d *Driver) Open(config *gdb.ConfigNode) (*sql.DB, error) { source string driver = "clickhouse" ) - if config.Pass != "" { + if config.Link != "" { + source = config.Link + } else if config.Pass != "" { source = fmt.Sprintf( - "clickhouse://%s:%s@%s:%s/%s", - config.User, config.Pass, config.Host, config.Port, config.Name) + "clickhouse://%s:%s@%s:%s/%s?charset=%s&debug=%s", + config.User, config.Pass, config.Host, config.Port, config.Name, config.Charset, gconv.String(config.Debug)) } else { source = fmt.Sprintf( - "clickhouse://%s@%s:%s/%s", - config.User, config.Host, config.Port, config.Name) + "clickhouse://%s@%s:%s/%s?charset=%s&debug=%s", + config.User, config.Host, config.Port, config.Name, config.Charset, gconv.String(config.Debug)) } - source += fmt.Sprintf( - "?charset=%s&debug=%s", - config.Charset, gconv.String(config.Debug), - ) db, err := sql.Open(driver, source) if err != nil { return nil, err diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 064f5aa6c..28b84ce4f 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -242,3 +242,13 @@ func TestDriver_TableFields(t *testing.T) { gtest.AssertEQ(len(field), 4) gtest.AssertNQ(field, nil) } + +func TestDriver_OpenLink(t *testing.T) { + connect, err := gdb.New(gdb.ConfigNode{ + Link: "clickhouse://default@127.0.0.1:9000/default?dial_timeout=200ms&max_execution_time=60&skip_verify=true&secure=false&compress=true", + Type: "clickhouse", + }) + gtest.AssertNil(err) + gtest.AssertNE(connect, nil) + gtest.AssertNil(connect.PingMaster()) +} From 5034f231a9d7bfc4d95bb13889479a3b9a0f96eb Mon Sep 17 00:00:00 2001 From: daguang Date: Mon, 4 Apr 2022 12:46:11 +0800 Subject: [PATCH 11/15] improve clickhouse driver --- contrib/drivers/clickhouse/clickhouse.go | 15 ++++++++------- contrib/drivers/clickhouse/go.sum | 8 ++++++-- database/gdb/gdb_core_ctx.go | 23 +++++++++++++++++++++++ database/gdb/gdb_core_underlying.go | 3 +-- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index f23fedfcf..3412cd993 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -93,7 +93,7 @@ func (d *Driver) Tables(ctx context.Context, schema ...string) (tables []string, return nil, err } query := fmt.Sprintf("select name from `system`.tables where database = '%s'", d.GetConfig().Name) - result, err = d.DoGetAll(ctx, link, query) + result, err = d.DoSelect(ctx, link, query) if err != nil { return } @@ -126,7 +126,7 @@ func (d *Driver) TableFields(ctx context.Context, table string, schema ...string return nil } getColumnsSql := fmt.Sprintf("select name,position,default_expression,comment from `system`.columns c where database = '%s' and `table` = '%s'", d.GetConfig().Name, table) - result, err = d.DoGetAll(ctx, link, getColumnsSql) + result, err = d.DoSelect(ctx, link, getColumnsSql) if err != nil { return nil } @@ -204,18 +204,18 @@ func (d *Driver) ping(conn *sql.DB) error { } // DoFilter handles the sql before posts it to database. -func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) { +func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, originSql string, args []interface{}) (newSql string, newArgs []interface{}, err error) { // replace MySQL to Clickhouse SQL grammar // MySQL eg: UPDATE visits SET xxx // Clickhouse eg: ALTER TABLE visits UPDATE xxx // MySQL eg: DELETE FROM VISIT // Clickhouse eg: ALTER TABLE VISIT DELETE WHERE filter_expr - result, err := gregex.MatchString("(?i)^UPDATE|DELETE", sql) + result, err := gregex.MatchString("(?i)^UPDATE|DELETE", originSql) if err != nil { return "", nil, err } if len(result) != 0 { - sqlSlice := strings.Split(sql, " ") + sqlSlice := strings.Split(originSql, " ") if len(sqlSlice) < 3 { return "", nil, ErrSQLNull } @@ -229,12 +229,13 @@ func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, sql string, args [ return strings.Join(sqlSlice, " "), args, nil } } - return sql, args, nil + return originSql, args, nil } // DoCommit commits current sql and arguments to underlying sql driver. func (d *Driver) DoCommit(ctx context.Context, in gdb.DoCommitInput) (out gdb.DoCommitOutput, err error) { - return d.Core.DoCommit(context.WithValue(ctx, "isIgnoreResult", true), in) + ctx = d.InjectIgnoreResult(ctx) + return d.Core.DoCommit(ctx, in) } func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { diff --git a/contrib/drivers/clickhouse/go.sum b/contrib/drivers/clickhouse/go.sum index e59c58abc..db27d0498 100644 --- a/contrib/drivers/clickhouse/go.sum +++ b/contrib/drivers/clickhouse/go.sum @@ -27,6 +27,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -43,13 +45,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/longbridgeapp/sqlparser v0.3.1 h1:iWOZWGIFgQrJRgobLXUNJdvqGRpbVXkyKUKUA5CNJBE= +github.com/longbridgeapp/sqlparser v0.3.1/go.mod h1:GIHaUq8zvYyHLCLMJJykx1CdM6LHtkUih/QaJXySSx4= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= diff --git a/database/gdb/gdb_core_ctx.go b/database/gdb/gdb_core_ctx.go index d41db8133..5b22137d6 100644 --- a/database/gdb/gdb_core_ctx.go +++ b/database/gdb/gdb_core_ctx.go @@ -26,6 +26,13 @@ type internalCtxData struct { const ( internalCtxDataKeyInCtx gctx.StrKey = "InternalCtxData" + + // IgnoreResultInCtx + // This option is only available in ClickHouse. + // Because ClickHouse does not support fetching insert/update results and returns errors when executed + // So need to ignore the results to avoid triggering errors + // Rather than ignoring errors after they are triggered + IgnoreResultInCtx gctx.StrKey = "IgnoreResult" ) func (c *Core) injectInternalCtxData(ctx context.Context) context.Context { @@ -38,6 +45,22 @@ func (c *Core) injectInternalCtxData(ctx context.Context) context.Context { }) } +func (c *Core) InjectIgnoreResult(ctx context.Context) context.Context { + if ctx.Value(IgnoreResultInCtx) != nil { + return ctx + } + return context.WithValue(ctx, IgnoreResultInCtx, &internalCtxData{ + DB: c.db, + }) +} + +func (c *Core) GetIgnoreResultFromCtx(ctx context.Context) *internalCtxData { + if v := ctx.Value(internalCtxDataKeyInCtx); v != nil { + return v.(*internalCtxData) + } + return nil +} + func (c *Core) getInternalCtxDataFromCtx(ctx context.Context) *internalCtxData { if v := ctx.Value(internalCtxDataKeyInCtx); v != nil { return v.(*internalCtxData) diff --git a/database/gdb/gdb_core_underlying.go b/database/gdb/gdb_core_underlying.go index 458cfea90..1c7e572d2 100644 --- a/database/gdb/gdb_core_underlying.go +++ b/database/gdb/gdb_core_underlying.go @@ -20,7 +20,6 @@ import ( "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/internal/intlog" "github.com/gogf/gf/v2/os/gtime" - "github.com/gogf/gf/v2/util/gconv" "github.com/gogf/gf/v2/util/guid" ) @@ -262,7 +261,7 @@ func (c *Core) DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutp } // Result handling. switch { - case sqlResult != nil && !gconv.Bool(ctx.Value("isIgnoreResult")): + case sqlResult != nil && c.GetIgnoreResultFromCtx(ctx) == nil: rowsAffected, err = sqlResult.RowsAffected() out.Result = sqlResult From c39a58f812bfe2e0655b00e63eca34639470b728 Mon Sep 17 00:00:00 2001 From: daguang Date: Mon, 4 Apr 2022 14:56:44 +0800 Subject: [PATCH 12/15] improve clickhouse driver --- database/gdb/gdb_core_ctx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/gdb/gdb_core_ctx.go b/database/gdb/gdb_core_ctx.go index 5b22137d6..52a264410 100644 --- a/database/gdb/gdb_core_ctx.go +++ b/database/gdb/gdb_core_ctx.go @@ -55,7 +55,7 @@ func (c *Core) InjectIgnoreResult(ctx context.Context) context.Context { } func (c *Core) GetIgnoreResultFromCtx(ctx context.Context) *internalCtxData { - if v := ctx.Value(internalCtxDataKeyInCtx); v != nil { + if v := ctx.Value(IgnoreResultInCtx); v != nil { return v.(*internalCtxData) } return nil From c90a9d45ee89142a0dd3cb768ff5c9972977173d Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Fri, 8 Apr 2022 09:44:42 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/drivers/clickhouse/clickhouse.go | 26 +++++++++---------- contrib/drivers/clickhouse/clickhouse_test.go | 8 +++--- database/gdb/gdb_core_ctx.go | 19 +++++++------- database/gdb/gdb_core_underlying.go | 2 +- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/contrib/drivers/clickhouse/clickhouse.go b/contrib/drivers/clickhouse/clickhouse.go index 3412cd993..35fd3c006 100644 --- a/contrib/drivers/clickhouse/clickhouse.go +++ b/contrib/drivers/clickhouse/clickhouse.go @@ -32,12 +32,12 @@ type Driver struct { var ( // tableFieldsMap caches the table information retrieved from database. tableFieldsMap = gmap.New(true) - ErrUnsupportedInsertIgnore = errors.New("unsupported method:InsertIgnore") - ErrUnsupportedInsertGetId = errors.New("unsupported method:InsertGetId") - ErrUnsupportedReplace = errors.New("unsupported method:Replace") - ErrUnsupportedBegin = errors.New("unsupported method:Begin") - ErrUnsupportedTransaction = errors.New("unsupported method:Transaction") - ErrSQLNull = errors.New("SQL cannot be null") + errUnsupportedInsertIgnore = errors.New("unsupported method:InsertIgnore") + errUnsupportedInsertGetId = errors.New("unsupported method:InsertGetId") + errUnsupportedReplace = errors.New("unsupported method:Replace") + errUnsupportedBegin = errors.New("unsupported method:Begin") + errUnsupportedTransaction = errors.New("unsupported method:Transaction") + errSQLNull = errors.New("SQL cannot be null") ) func init() { @@ -205,7 +205,7 @@ func (d *Driver) ping(conn *sql.DB) error { // DoFilter handles the sql before posts it to database. func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, originSql string, args []interface{}) (newSql string, newArgs []interface{}, err error) { - // replace MySQL to Clickhouse SQL grammar + // replace STD SQL to Clickhouse SQL grammar // MySQL eg: UPDATE visits SET xxx // Clickhouse eg: ALTER TABLE visits UPDATE xxx // MySQL eg: DELETE FROM VISIT @@ -217,7 +217,7 @@ func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, originSql string, if len(result) != 0 { sqlSlice := strings.Split(originSql, " ") if len(sqlSlice) < 3 { - return "", nil, ErrSQLNull + return "", nil, errSQLNull } ck := []string{"ALTER", "TABLE"} switch strings.ToUpper(result[0]) { @@ -285,23 +285,23 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list // InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. func (d *Driver) InsertIgnore(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { - return nil, ErrUnsupportedInsertIgnore + return nil, errUnsupportedInsertIgnore } // InsertAndGetId Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. func (d *Driver) InsertAndGetId(ctx context.Context, table string, data interface{}, batch ...int) (int64, error) { - return 0, ErrUnsupportedInsertGetId + return 0, errUnsupportedInsertGetId } // Replace Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE. func (d *Driver) Replace(ctx context.Context, table string, data interface{}, batch ...int) (sql.Result, error) { - return nil, ErrUnsupportedReplace + return nil, errUnsupportedReplace } func (d *Driver) Begin(ctx context.Context) (tx *gdb.TX, err error) { - return nil, ErrUnsupportedBegin + return nil, errUnsupportedBegin } func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx *gdb.TX) error) error { - return ErrUnsupportedTransaction + return errUnsupportedTransaction } diff --git a/contrib/drivers/clickhouse/clickhouse_test.go b/contrib/drivers/clickhouse/clickhouse_test.go index 28b84ce4f..3ea9f3cc2 100644 --- a/contrib/drivers/clickhouse/clickhouse_test.go +++ b/contrib/drivers/clickhouse/clickhouse_test.go @@ -113,19 +113,19 @@ func TestDriverClickhouse_Select(t *testing.T) { func TestDriver_InsertIgnore(t *testing.T) { connect := InitClickhouse() _, err := connect.InsertIgnore(context.Background(), "", nil) - gtest.AssertEQ(err, ErrUnsupportedInsertIgnore) + gtest.AssertEQ(err, errUnsupportedInsertIgnore) } func TestDriver_InsertAndGetId(t *testing.T) { connect := InitClickhouse() _, err := connect.InsertAndGetId(context.Background(), "", nil) - gtest.AssertEQ(err, ErrUnsupportedInsertGetId) + gtest.AssertEQ(err, errUnsupportedInsertGetId) } func TestDriver_Replace(t *testing.T) { connect := InitClickhouse() _, err := connect.Replace(context.Background(), "", nil) - gtest.AssertEQ(err, ErrUnsupportedReplace) + gtest.AssertEQ(err, errUnsupportedReplace) } func TestDriverClickhouse_DoInsertOne(t *testing.T) { @@ -146,7 +146,7 @@ func TestDriver_DoInsertMany(t *testing.T) { gtest.AssertEQ(createClickhouseTable(connect), nil) defer dropClickhouseTable(connect) tx, err := connect.Begin(context.Background()) - gtest.AssertEQ(err, ErrUnsupportedBegin) + gtest.AssertEQ(err, errUnsupportedBegin) gtest.AssertNil(tx) } diff --git a/database/gdb/gdb_core_ctx.go b/database/gdb/gdb_core_ctx.go index 52a264410..458d1527e 100644 --- a/database/gdb/gdb_core_ctx.go +++ b/database/gdb/gdb_core_ctx.go @@ -27,12 +27,11 @@ type internalCtxData struct { const ( internalCtxDataKeyInCtx gctx.StrKey = "InternalCtxData" - // IgnoreResultInCtx - // This option is only available in ClickHouse. - // Because ClickHouse does not support fetching insert/update results and returns errors when executed - // So need to ignore the results to avoid triggering errors - // Rather than ignoring errors after they are triggered - IgnoreResultInCtx gctx.StrKey = "IgnoreResult" + // `ignoreResultKeyInCtx` is a mark for some db drivers that do not support `RowsAffected` function, + // for example: `clickhouse`. The `clickhouse` does not support fetching insert/update results, + // but returns errors when execute `RowsAffected`. It here ignores the calling of `RowsAffected` + // to avoid triggering errors, rather than ignoring errors after they are triggered. + ignoreResultInCtx gctx.StrKey = "IgnoreResult" ) func (c *Core) injectInternalCtxData(ctx context.Context) context.Context { @@ -46,16 +45,16 @@ func (c *Core) injectInternalCtxData(ctx context.Context) context.Context { } func (c *Core) InjectIgnoreResult(ctx context.Context) context.Context { - if ctx.Value(IgnoreResultInCtx) != nil { + if ctx.Value(ignoreResultInCtx) != nil { return ctx } - return context.WithValue(ctx, IgnoreResultInCtx, &internalCtxData{ + return context.WithValue(ctx, ignoreResultInCtx, &internalCtxData{ DB: c.db, }) } -func (c *Core) GetIgnoreResultFromCtx(ctx context.Context) *internalCtxData { - if v := ctx.Value(IgnoreResultInCtx); v != nil { +func (c *Core) getIgnoreResultFromCtx(ctx context.Context) *internalCtxData { + if v := ctx.Value(ignoreResultInCtx); v != nil { return v.(*internalCtxData) } return nil diff --git a/database/gdb/gdb_core_underlying.go b/database/gdb/gdb_core_underlying.go index 1c7e572d2..c474dd51d 100644 --- a/database/gdb/gdb_core_underlying.go +++ b/database/gdb/gdb_core_underlying.go @@ -261,7 +261,7 @@ func (c *Core) DoCommit(ctx context.Context, in DoCommitInput) (out DoCommitOutp } // Result handling. switch { - case sqlResult != nil && c.GetIgnoreResultFromCtx(ctx) == nil: + case sqlResult != nil && c.getIgnoreResultFromCtx(ctx) == nil: rowsAffected, err = sqlResult.RowsAffected() out.Result = sqlResult From 1935412db927ca50ee5bceecb057a1fdc67c0235 Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Fri, 8 Apr 2022 10:07:14 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- database/gdb/gdb_core_ctx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/gdb/gdb_core_ctx.go b/database/gdb/gdb_core_ctx.go index 458d1527e..1bc0c0b59 100644 --- a/database/gdb/gdb_core_ctx.go +++ b/database/gdb/gdb_core_ctx.go @@ -31,7 +31,7 @@ const ( // for example: `clickhouse`. The `clickhouse` does not support fetching insert/update results, // but returns errors when execute `RowsAffected`. It here ignores the calling of `RowsAffected` // to avoid triggering errors, rather than ignoring errors after they are triggered. - ignoreResultInCtx gctx.StrKey = "IgnoreResult" + ignoreResultInCtx gctx.StrKey = "ignoreResult" ) func (c *Core) injectInternalCtxData(ctx context.Context) context.Context { From a1ec7cb896413daf01d9a8206ee881b1e8ce6bf1 Mon Sep 17 00:00:00 2001 From: daguang <28806852+DGuang21@users.noreply.github.com> Date: Fri, 8 Apr 2022 10:08:04 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E2=9C=A8=20improve=20clickhouse=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- database/gdb/gdb_core_ctx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/gdb/gdb_core_ctx.go b/database/gdb/gdb_core_ctx.go index 1bc0c0b59..458d1527e 100644 --- a/database/gdb/gdb_core_ctx.go +++ b/database/gdb/gdb_core_ctx.go @@ -31,7 +31,7 @@ const ( // for example: `clickhouse`. The `clickhouse` does not support fetching insert/update results, // but returns errors when execute `RowsAffected`. It here ignores the calling of `RowsAffected` // to avoid triggering errors, rather than ignoring errors after they are triggered. - ignoreResultInCtx gctx.StrKey = "ignoreResult" + ignoreResultInCtx gctx.StrKey = "IgnoreResult" ) func (c *Core) injectInternalCtxData(ctx context.Context) context.Context {