fix: 修改远程数据库依赖 (#1794)

This commit is contained in:
ssongliu 2023-08-01 11:20:16 +08:00 committed by GitHub
parent e0ca9507de
commit 09653f9c06
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 577 additions and 35 deletions

View File

@ -5,7 +5,7 @@ import (
"database/sql"
"fmt"
"os"
"path"
"os/exec"
"strings"
"time"
@ -14,8 +14,7 @@ import (
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/1Panel-dev/1Panel/backend/utils/common"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/jarvanstack/mysqldump"
"github.com/1Panel-dev/1Panel/backend/utils/mysql/helper"
)
type Remote struct {
@ -222,23 +221,26 @@ func (r *Remote) Backup(info BackupInfo) error {
f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755)
defer f.Close()
if err := mysqldump.Dump(dns, mysqldump.WithData(), mysqldump.WithDropTable(), mysqldump.WithWriter(f)); err != nil {
if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil {
return err
}
if err := fileOp.Compress([]string{fileNameItem}, info.TargetDir, info.FileName, files.Gz); err != nil {
return err
gzipCmd := exec.Command("gzip", fileNameItem)
stdout, err := gzipCmd.CombinedOutput()
if err != nil {
return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err)
}
return nil
}
func (r *Remote) Recover(info RecoverInfo) error {
fileOp := files.NewFileOp()
fileName := info.SourceFile
if strings.HasSuffix(info.SourceFile, ".sql.gz") {
fileName = strings.TrimSuffix(info.SourceFile, ".gz")
if err := fileOp.Decompress(info.SourceFile, path.Dir(fileName), files.Gz); err != nil {
return err
gzipCmd := exec.Command("gunzip", info.SourceFile)
stdout, err := gzipCmd.CombinedOutput()
if err != nil {
return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err)
}
}
dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F")
@ -247,7 +249,7 @@ func (r *Remote) Recover(info RecoverInfo) error {
return err
}
defer f.Close()
if err := mysqldump.Source(dns, f, mysqldump.WithMergeInsert(1000)); err != nil {
if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil {
return err
}
return nil

View File

@ -0,0 +1,327 @@
package helper
import (
"bufio"
"database/sql"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/1Panel-dev/1Panel/backend/global"
_ "github.com/go-sql-driver/mysql"
)
func init() {}
type dumpOption struct {
isData bool
tables []string
isAllTable bool
isDropTable bool
writer io.Writer
}
type DumpOption func(*dumpOption)
func WithDropTable() DumpOption {
return func(option *dumpOption) {
option.isDropTable = true
}
}
func WithData() DumpOption {
return func(option *dumpOption) {
option.isData = true
}
}
func WithTables(tables ...string) DumpOption {
return func(option *dumpOption) {
option.tables = tables
}
}
func WithAllTable() DumpOption {
return func(option *dumpOption) {
option.isAllTable = true
}
}
func WithWriter(writer io.Writer) DumpOption {
return func(option *dumpOption) {
option.writer = writer
}
}
func Dump(dns string, opts ...DumpOption) error {
start := time.Now()
global.LOG.Infof("dump start at %s\n", start.Format("2006-01-02 15:04:05"))
defer func() {
end := time.Now()
global.LOG.Infof("dump end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start))
}()
var err error
var o dumpOption
for _, opt := range opts {
opt(&o)
}
if len(o.tables) == 0 {
o.isAllTable = true
}
if o.writer == nil {
o.writer = os.Stdout
}
buf := bufio.NewWriter(o.writer)
defer buf.Flush()
_, _ = buf.WriteString("-- ----------------------------\n")
_, _ = buf.WriteString("-- MySQL Database Dump\n")
_, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n")
_, _ = buf.WriteString("-- ----------------------------\n")
_, _ = buf.WriteString("\n\n")
db, err := sql.Open("mysql", dns)
if err != nil {
global.LOG.Errorf("open mysql db failed, err: %v", err)
return err
}
defer db.Close()
dbName, err := getDBNameFromDNS(dns)
if err != nil {
global.LOG.Errorf("get db name from dns failed, err: %v", err)
return err
}
_, err = db.Exec(fmt.Sprintf("USE `%s`", dbName))
if err != nil {
global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err)
return err
}
var tables []string
if o.isAllTable {
tmp, err := getAllTables(db)
if err != nil {
global.LOG.Errorf("get all tables failed, err: %v", err)
return err
}
tables = tmp
} else {
tables = o.tables
}
for _, table := range tables {
if o.isDropTable {
_, _ = buf.WriteString(fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table))
}
err = writeTableStruct(db, table, buf)
if err != nil {
global.LOG.Errorf("write table struct failed, err: %v", err)
return err
}
if o.isData {
err = writeTableData(db, table, buf)
if err != nil {
global.LOG.Errorf("write table data failed, err: %v", err)
return err
}
}
}
_, _ = buf.WriteString("-- ----------------------------\n")
_, _ = buf.WriteString("-- Dumped by mysqldump\n")
_, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n")
_, _ = buf.WriteString("-- ----------------------------\n")
_ = buf.Flush()
return nil
}
func getCreateTableSQL(db *sql.DB, table string) (string, error) {
var createTableSQL string
err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL)
if err != nil {
return "", err
}
createTableSQL = strings.Replace(createTableSQL, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)
return createTableSQL, nil
}
func getAllTables(db *sql.DB) ([]string, error) {
var tables []string
rows, err := db.Query("SHOW TABLES")
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var table string
err = rows.Scan(&table)
if err != nil {
return nil, err
}
tables = append(tables, table)
}
return tables, nil
}
func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error {
_, _ = buf.WriteString("-- ----------------------------\n")
_, _ = buf.WriteString(fmt.Sprintf("-- Table structure for %s\n", table))
_, _ = buf.WriteString("-- ----------------------------\n")
createTableSQL, err := getCreateTableSQL(db, table)
if err != nil {
global.LOG.Errorf("get create table sql failed, err: %v", err)
return err
}
_, _ = buf.WriteString(createTableSQL)
_, _ = buf.WriteString(";")
_, _ = buf.WriteString("\n\n")
_, _ = buf.WriteString("\n\n")
return nil
}
func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error {
_, _ = buf.WriteString("-- ----------------------------\n")
_, _ = buf.WriteString(fmt.Sprintf("-- Records of %s\n", table))
_, _ = buf.WriteString("-- ----------------------------\n")
lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table))
if err != nil {
global.LOG.Errorf("exec `select * from %s` failed, err: %v", table, err)
return err
}
defer lineRows.Close()
var columns []string
columns, err = lineRows.Columns()
if err != nil {
global.LOG.Errorf("get columes falied, err: %v", err)
return err
}
columnTypes, err := lineRows.ColumnTypes()
if err != nil {
global.LOG.Errorf("get colume types failed, err: %v", err)
return err
}
var values [][]interface{}
for lineRows.Next() {
row := make([]interface{}, len(columns))
rowPointers := make([]interface{}, len(columns))
for i := range columns {
rowPointers[i] = &row[i]
}
err = lineRows.Scan(rowPointers...)
if err != nil {
global.LOG.Errorf("scan row data failed, err: %v", err)
return err
}
values = append(values, row)
}
for _, row := range values {
ssql := "INSERT INTO `" + table + "` VALUES ("
for i, col := range row {
if col == nil {
ssql += "NULL"
} else {
Type := columnTypes[i].DatabaseTypeName()
Type = strings.Replace(Type, "UNSIGNED", "", -1)
Type = strings.Replace(Type, " ", "", -1)
switch Type {
case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT":
if bs, ok := col.([]byte); ok {
ssql += string(bs)
} else {
ssql += fmt.Sprintf("%d", col)
}
case "FLOAT", "DOUBLE":
if bs, ok := col.([]byte); ok {
ssql += string(bs)
} else {
ssql += fmt.Sprintf("%f", col)
}
case "DECIMAL", "DEC":
ssql += fmt.Sprintf("%s", col)
case "DATE":
t, ok := col.(time.Time)
if !ok {
global.LOG.Errorf("the DATE type conversion failed., err: %v", err)
return err
}
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02"))
case "DATETIME":
t, ok := col.(time.Time)
if !ok {
global.LOG.Errorf("the DATETIME type conversion failed., err: %v", err)
return err
}
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05"))
case "TIMESTAMP":
t, ok := col.(time.Time)
if !ok {
global.LOG.Errorf("the TIMESTAMP type conversion failed., err: %v", err)
return err
}
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05"))
case "TIME":
t, ok := col.([]byte)
if !ok {
global.LOG.Errorf("the TIME type conversion failed., err: %v", err)
return err
}
ssql += fmt.Sprintf("'%s'", string(t))
case "YEAR":
t, ok := col.([]byte)
if !ok {
global.LOG.Errorf("the YEAR type conversion failed., err: %v", err)
return err
}
ssql += string(t)
case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT":
ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1))
case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB":
ssql += fmt.Sprintf("0x%X", col)
case "ENUM", "SET":
ssql += fmt.Sprintf("'%s'", col)
case "BOOL", "BOOLEAN":
if col.(bool) {
ssql += "true"
} else {
ssql += "false"
}
case "JSON":
ssql += fmt.Sprintf("'%s'", col)
default:
global.LOG.Errorf("unsupported colume type: %s", Type)
return fmt.Errorf("unsupported colume type: %s", Type)
}
}
if i < len(row)-1 {
ssql += ","
}
}
ssql += ");\n"
_, _ = buf.WriteString(ssql)
}
_, _ = buf.WriteString("\n\n")
return nil
}

View File

@ -0,0 +1,228 @@
package helper
import (
"bufio"
"database/sql"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/1Panel-dev/1Panel/backend/global"
)
type sourceOption struct {
dryRun bool
mergeInsert int
debug bool
}
type SourceOption func(*sourceOption)
func WithDryRun() SourceOption {
return func(o *sourceOption) {
o.dryRun = true
}
}
func WithMergeInsert(size int) SourceOption {
return func(o *sourceOption) {
o.mergeInsert = size
}
}
func WithDebug() SourceOption {
return func(o *sourceOption) {
o.debug = true
}
}
type dbWrapper struct {
DB *sql.DB
debug bool
dryRun bool
}
func newDBWrapper(db *sql.DB, dryRun, debug bool) *dbWrapper {
return &dbWrapper{
DB: db,
dryRun: dryRun,
debug: debug,
}
}
func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) {
if db.debug {
global.LOG.Debugf("query %s", query)
}
if db.dryRun {
return nil, nil
}
return db.DB.Exec(query, args...)
}
func Source(dns string, reader io.Reader, opts ...SourceOption) error {
start := time.Now()
global.LOG.Infof("source start at %s", start.Format("2006-01-02 15:04:05"))
defer func() {
end := time.Now()
global.LOG.Infof("source end at %s, cost %s", end.Format("2006-01-02 15:04:05"), end.Sub(start))
}()
var err error
var db *sql.DB
var o sourceOption
for _, opt := range opts {
opt(&o)
}
dbName, err := getDBNameFromDNS(dns)
if err != nil {
global.LOG.Errorf("get db name from dns failed, err: %v", err)
return err
}
db, err = sql.Open("mysql", dns)
if err != nil {
global.LOG.Errorf("open mysql db failed, err: %v", err)
return err
}
defer db.Close()
dbWrapper := newDBWrapper(db, o.dryRun, o.debug)
_, err = dbWrapper.Exec(fmt.Sprintf("USE `%s`;", dbName))
if err != nil {
global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err)
return err
}
db.SetConnMaxLifetime(3600)
r := bufio.NewReader(reader)
_, err = dbWrapper.Exec("SET autocommit=0;")
if err != nil {
global.LOG.Errorf("exec `set autocommit=0` failed, err: %v", err)
return err
}
for {
line, err := r.ReadString(';')
if err != nil {
if err == io.EOF {
break
}
global.LOG.Errorf("read sql failed, err: %v", err)
return err
}
ssql := string(line)
ssql, err = trim(ssql)
if err != nil {
global.LOG.Errorf("trim sql failed, err: %v", err)
return err
}
if o.mergeInsert > 1 && strings.HasPrefix(ssql, "INSERT INTO") {
var insertSQLs []string
insertSQLs = append(insertSQLs, ssql)
for i := 0; i < o.mergeInsert-1; i++ {
line, err := r.ReadString(';')
if err != nil {
if err == io.EOF {
break
}
global.LOG.Errorf("read merge insert sql failed, err: %v", err)
return err
}
ssql2 := string(line)
ssql2, err = trim(ssql2)
if err != nil {
global.LOG.Errorf("trim merge insert sql failed, err: %v", err)
return err
}
if strings.HasPrefix(ssql2, "INSERT INTO") {
insertSQLs = append(insertSQLs, ssql2)
continue
}
break
}
ssql, err = mergeInsert(insertSQLs)
if err != nil {
global.LOG.Errorf("do merge insert failed, err: %v", err)
return err
}
}
_, err = dbWrapper.Exec(ssql)
if err != nil {
global.LOG.Errorf("exec sql failed, err: %v", err)
return err
}
}
_, err = dbWrapper.Exec("COMMIT;")
if err != nil {
global.LOG.Errorf("exec `commit` failed, err: %v", err)
return err
}
_, err = dbWrapper.Exec("SET autocommit=1;")
if err != nil {
global.LOG.Errorf("exec `autocommit=1` failed, err: %v", err)
return err
}
return nil
}
func mergeInsert(insertSQLs []string) (string, error) {
if len(insertSQLs) == 0 {
return "", errors.New("no input provided")
}
builder := strings.Builder{}
sql1 := insertSQLs[0]
sql1 = strings.TrimSuffix(sql1, ";")
builder.WriteString(sql1)
for i, insertSQL := range insertSQLs[1:] {
if i < len(insertSQLs)-1 {
builder.WriteString(",")
}
valuesIdx := strings.Index(insertSQL, "VALUES")
if valuesIdx == -1 {
return "", errors.New("invalid SQL: missing VALUES keyword")
}
sqln := insertSQL[valuesIdx:]
sqln = strings.TrimPrefix(sqln, "VALUES")
sqln = strings.TrimSuffix(sqln, ";")
builder.WriteString(sqln)
}
builder.WriteString(";")
return builder.String(), nil
}
func trim(s string) (string, error) {
s = strings.TrimLeft(s, "\n")
s = strings.TrimSpace(s)
return s, nil
}
func getDBNameFromDNS(dns string) (string, error) {
ss1 := strings.Split(dns, "/")
if len(ss1) == 2 {
ss2 := strings.Split(ss1[1], "?")
if len(ss2) == 2 {
return ss2[0], nil
}
}
return "", fmt.Errorf("dns error: %s", dns)
}

View File

@ -159,6 +159,7 @@ const loadPassword = async () => {
const onSubmit = async () => {
let param = {
id: 0,
from: form.from,
value: form.password,
};
loading.value = true;

View File

@ -47,14 +47,7 @@
</el-form-item>
<el-form-item :label="$t('commons.table.type')" prop="from">
<el-select v-model="form.from">
<el-option
v-for="(item, index) in dbOptions"
:key="index"
:value="item.name"
:label="loadLabel(item)"
></el-option>
</el-select>
<el-tag>{{ loadLabel(form.from) }}</el-tag>
</el-form-item>
<el-form-item :label="$t('commons.table.description')" prop="description">
@ -83,13 +76,12 @@ import { reactive, ref } from 'vue';
import { Rules } from '@/global/form-rules';
import i18n from '@/lang';
import { ElForm } from 'element-plus';
import { addMysqlDB, listRemoteDBs } from '@/api/modules/database';
import { addMysqlDB } from '@/api/modules/database';
import DrawerHeader from '@/components/drawer-header/index.vue';
import { MsgSuccess } from '@/utils/message';
import { getRandomStr } from '@/utils/util';
const loading = ref();
const dbOptions = ref();
const createVisiable = ref(false);
const form = reactive({
name: '',
@ -108,16 +100,17 @@ const rules = reactive({
password: [Rules.requiredInput],
permission: [Rules.requiredSelect],
permissionIPs: [Rules.requiredInput],
from: [Rules.requiredSelect],
});
type FormInstance = InstanceType<typeof ElForm>;
const formRef = ref<FormInstance>();
interface DialogProps {
from: string;
mysqlName: string;
}
const acceptParams = (params: DialogProps): void => {
form.name = '';
form.from = params.from;
form.mysqlName = params.mysqlName;
form.format = 'utf8mb4';
form.username = '';
@ -125,20 +118,14 @@ const acceptParams = (params: DialogProps): void => {
form.permissionIPs = '';
form.description = '';
random();
loadDBOptions();
createVisiable.value = true;
};
const handleClose = () => {
createVisiable.value = false;
};
const loadDBOptions = async () => {
const res = await listRemoteDBs('mysql');
dbOptions.value = res.data || [];
};
function loadLabel(item: any) {
return (item.name === 'local' ? i18n.global.t('database.localDB') : item.name) + '(' + item.address + ')';
function loadLabel(from: any) {
return from === 'local' ? i18n.global.t('database.localDB') : from;
}
const random = async () => {

View File

@ -5,7 +5,7 @@
width="30%"
:close-on-click-modal="false"
>
<el-form ref="deleteForm" v-loading="loading">
<el-form ref="deleteForm" v-loading="loading" @submit.prevent>
<el-form-item>
<el-checkbox v-model="deleteReq.forceDelete" :label="$t('app.forceDelete')" />
<span class="input-help">

View File

@ -264,6 +264,7 @@ const mysqlVersion = ref();
const dialogRef = ref();
const onOpenDialog = async () => {
let params = {
from: paginationConfig.from,
mysqlName: mysqlName.value,
};
dialogRef.value!.acceptParams(params);

View File

@ -115,7 +115,6 @@ const acceptParams = (params: DialogProps): void => {
: i18n.global.t('database.permission');
changeForm.id = params.id;
changeForm.from = params.from;
console.log(changeForm.from);
changeForm.mysqlName = params.mysqlName;
changeForm.userName = params.username;
changeForm.password = params.password;

View File

@ -1,6 +1,6 @@
<template>
<div>
<el-form label-position="left" label-width="80px">
<el-form label-position="left" label-width="80px" @submit.prevent>
<el-form-item :label="$t('database.isOn')">
<el-switch
v-model="variables.slow_query_log"

1
go.mod
View File

@ -26,7 +26,6 @@ require (
github.com/golang-jwt/jwt/v4 v4.4.2
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.5.0
github.com/jarvanstack/mysqldump v0.7.0
github.com/jinzhu/copier v0.3.5
github.com/joho/godotenv v1.5.1
github.com/klauspost/compress v1.16.5

2
go.sum
View File

@ -507,8 +507,6 @@ github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs=
github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y=
github.com/jarvanstack/mysqldump v0.7.0 h1:lspwQQhLrpVpCCbNxs5GPNTAcrqYTLVLIO7txUbtYdc=
github.com/jarvanstack/mysqldump v0.7.0/go.mod h1:NBXaxyEQjiaLwLP9s3pGfal7aZL/5omKJrk+bC1E250=
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=