From ed9f0bb32f73c27ac8b1f9d595806a6d96398724 Mon Sep 17 00:00:00 2001 From: AlexAkulov Date: Sat, 15 Aug 2020 20:14:58 +0500 Subject: [PATCH] Replaced 'with' to ':' in error messages --- pkg/chbackup/backup.go | 42 ++++++++++++------------- pkg/chbackup/backup_destination.go | 12 ++++---- pkg/chbackup/clickhouse.go | 18 +++++------ pkg/chbackup/config.go | 4 +-- test/integration/integration_test.go | 46 ++++++++++------------------ 5 files changed, 55 insertions(+), 67 deletions(-) diff --git a/pkg/chbackup/backup.go b/pkg/chbackup/backup.go index 574ac3fe..8da8d744 100644 --- a/pkg/chbackup/backup.go +++ b/pkg/chbackup/backup.go @@ -150,13 +150,13 @@ func getTables(config Config) ([]Table, error) { } if err := ch.Connect(); err != nil { - return []Table{}, fmt.Errorf("can't connect to clickhouse with: %v", err) + return []Table{}, fmt.Errorf("can't connect to clickhouse: %v", err) } defer ch.Close() allTables, err := ch.GetTables() if err != nil { - return []Table{}, fmt.Errorf("can't get tables with: %v", err) + return []Table{}, fmt.Errorf("can't get tables: %v", err) } return allTables, nil } @@ -206,16 +206,16 @@ func restoreSchema(config Config, backupName string, tablePattern string) error Config: &config.ClickHouse, } if err := ch.Connect(); err != nil { - return fmt.Errorf("can't connect to clickhouse with %v", err) + return fmt.Errorf("can't connect to clickhouse: %v", err) } defer ch.Close() for _, schema := range tablesForRestore { if err := ch.CreateDatabase(schema.Database); err != nil { - return fmt.Errorf("can't create database `%s` %v", schema.Database, err) + return fmt.Errorf("can't create database '%s': %v", schema.Database, err) } if err := ch.CreateTable(schema); err != nil { - return fmt.Errorf("can't create table `%s`.`%s` %v", schema.Database, schema.Table, err) + return fmt.Errorf("can't create table '%s.%s': %v", schema.Database, schema.Table, err) } } return nil @@ -332,13 +332,13 @@ func Freeze(config Config, tablePattern string) error { Config: &config.ClickHouse, } if err := ch.Connect(); err != nil { - return fmt.Errorf("can't connect to clickhouse with: %v", err) + return fmt.Errorf("can't connect to clickhouse: %v", err) } defer ch.Close() dataPath, err := ch.GetDataPath() if err != nil || dataPath == "" { - return fmt.Errorf("can't get data path from clickhouse with: %v\nyou can set data_path in config file", err) + return fmt.Errorf("can't get data path from clickhouse: %v\nyou can set data_path in config file", err) } shadowPath := filepath.Join(dataPath, "shadow") @@ -353,15 +353,15 @@ func Freeze(config Config, tablePattern string) error { allTables, err := ch.GetTables() if err != nil { - return fmt.Errorf("can't get Clickhouse tables with: %v", err) + return fmt.Errorf("can't get tables from clickhouse: %v", err) } backupTables := parseTablePatternForFreeze(allTables, tablePattern) if len(backupTables) == 0 { - return fmt.Errorf("there are no tables in Clickhouse, create something to freeze") + return fmt.Errorf("there are no tables in clickhouse, create something to freeze") } for _, table := range backupTables { if table.Skip { - log.Printf("Skip `%s`.`%s`", table.Database, table.Name) + log.Printf("Skip '%s.%s'", table.Database, table.Name) continue } if err := ch.FreezeTable(table); err != nil { @@ -388,10 +388,10 @@ func CreateBackup(config Config, backupName, tablePattern string) error { } backupPath := path.Join(dataPath, "backup", backupName) if _, err := os.Stat(backupPath); err == nil || !os.IsNotExist(err) { - return fmt.Errorf("can't create backup with '%s' already exists", backupPath) + return fmt.Errorf("can't create backup '%s' already exists", backupPath) } if err := os.MkdirAll(backupPath, os.ModePerm); err != nil { - return fmt.Errorf("can't create backup with %v", err) + return fmt.Errorf("can't create backup: %v", err) } log.Printf("Create backup '%s'", backupName) if err := Freeze(config, tablePattern); err != nil { @@ -416,7 +416,7 @@ func CreateBackup(config Config, backupName, tablePattern string) error { relativePath := strings.Trim(strings.TrimPrefix(schema.Path, path.Join(dataPath, "metadata")), "/") newPath := path.Join(backupPath, "metadata", relativePath) if err := copyFile(schema.Path, newPath); err != nil { - return fmt.Errorf("can't backup metadata with %v", err) + return fmt.Errorf("can't backup metadata: %v", err) } } log.Println(" Done.") @@ -469,7 +469,7 @@ func RestoreData(config Config, backupName string, tablePattern string) error { Config: &config.ClickHouse, } if err := ch.Connect(); err != nil { - return fmt.Errorf("can't connect to clickhouse with: %v", err) + return fmt.Errorf("can't connect to clickhouse: %v", err) } defer ch.Close() @@ -503,10 +503,10 @@ func RestoreData(config Config, backupName string, tablePattern string) error { } for _, table := range restoreTables { if err := ch.CopyData(table); err != nil { - return fmt.Errorf("can't restore `%s`.`%s` with %v", table.Database, table.Name, err) + return fmt.Errorf("can't restore '%s.%s': %v", table.Database, table.Name, err) } if err := ch.AttachPatritions(table); err != nil { - return fmt.Errorf("can't attach partitions for table '%s.%s' with %v", table.Database, table.Name, err) + return fmt.Errorf("can't attach partitions for table '%s.%s': %v", table.Database, table.Name, err) } } return nil @@ -566,11 +566,11 @@ func Upload(config Config, backupName string, diffFrom string) error { err = bd.Connect() if err != nil { - return fmt.Errorf("can't connect to %s with : %v", bd.Kind(), err) + return fmt.Errorf("can't connect to %s: %v", bd.Kind(), err) } if err := GetLocalBackup(config, backupName); err != nil { - return fmt.Errorf("can't upload with %s", err) + return fmt.Errorf("can't upload: %v", err) } backupPath := path.Join(dataPath, "backup", backupName) log.Printf("Upload backup '%s'", backupName) @@ -579,7 +579,7 @@ func Upload(config Config, backupName string, diffFrom string) error { diffFromPath = path.Join(dataPath, "backup", diffFrom) } if err := bd.CompressedStreamUpload(backupPath, backupName, diffFromPath); err != nil { - return fmt.Errorf("can't upload with %v", err) + return fmt.Errorf("can't upload: %v", err) } if err := bd.RemoveOldBackups(bd.BackupsToKeep()); err != nil { return fmt.Errorf("can't remove old backups: %v", err) @@ -632,7 +632,7 @@ func Clean(config Config) error { } log.Printf("Clean %s", shadowDir) if err := cleanDir(shadowDir); err != nil { - return fmt.Errorf("can't remove contents from directory %v: %v", shadowDir, err) + return fmt.Errorf("can't clean '%s': %v", shadowDir, err) } return nil } @@ -691,7 +691,7 @@ func RemoveBackupRemote(config Config, backupName string) error { } err = bd.Connect() if err != nil { - return fmt.Errorf("can't connect to remote storage with: %v", err) + return fmt.Errorf("can't connect to remote storage: %v", err) } backupList, err := bd.BackupList() if err != nil { diff --git a/pkg/chbackup/backup_destination.go b/pkg/chbackup/backup_destination.go index ecbce512..2336a498 100644 --- a/pkg/chbackup/backup_destination.go +++ b/pkg/chbackup/backup_destination.go @@ -241,7 +241,7 @@ func (bd *BackupDestination) CompressedStreamDownload(remotePath string, localPa log.Printf("Backup '%s' required '%s'. Downloading.", remotePath, metafile.RequiredBackup) err := bd.CompressedStreamDownload(metafile.RequiredBackup, filepath.Join(filepath.Dir(localPath), metafile.RequiredBackup)) if err != nil && !os.IsExist(err) { - return fmt.Errorf("can't download '%s' with %v", metafile.RequiredBackup, err) + return fmt.Errorf("can't download '%s': %v", metafile.RequiredBackup, err) } } for _, hardlink := range metafile.Hardlinks { @@ -342,16 +342,16 @@ func (bd *BackupDestination) CompressedStreamUpload(localPath, remotePath, diffF } content, err := json.MarshalIndent(&metafile, "", "\t") if err != nil { - ferr = fmt.Errorf("can't marshal json with %v", err) + ferr = fmt.Errorf("can't marshal json: %v", err) return } tmpfile, err := ioutil.TempFile("", MetaFileName) if err != nil { - ferr = fmt.Errorf("can't create meta.info with %v", err) + ferr = fmt.Errorf("can't create meta.info: %v", err) return } if _, err := tmpfile.Write(content); err != nil { - ferr = fmt.Errorf("can't write to meta.info with %v", err) + ferr = fmt.Errorf("can't write to meta.info: %v", err) return } tmpfile.Close() @@ -359,7 +359,7 @@ func (bd *BackupDestination) CompressedStreamUpload(localPath, remotePath, diffF defer os.Remove(tmpFileName) info, err := os.Stat(tmpFileName) if err != nil { - ferr = fmt.Errorf("can't get stat with %v", err) + ferr = fmt.Errorf("can't get stat: %v", err) return } mf, err := os.Open(tmpFileName) @@ -375,7 +375,7 @@ func (bd *BackupDestination) CompressedStreamUpload(localPath, remotePath, diffF }, ReadCloser: mf, }); err != nil { - ferr = fmt.Errorf("can't add mata.json to archive with %v", err) + ferr = fmt.Errorf("can't add mata.json to archive: %v", err) return } } diff --git a/pkg/chbackup/clickhouse.go b/pkg/chbackup/clickhouse.go index 856ccdb7..206d78aa 100644 --- a/pkg/chbackup/clickhouse.go +++ b/pkg/chbackup/clickhouse.go @@ -141,7 +141,7 @@ func (ch *ClickHouse) GetVersion() (int, error) { var result []string q := "SELECT value FROM `system`.`build_options` where name='VERSION_INTEGER'" if err := ch.conn.Select(&result, q); err != nil { - return 0, fmt.Errorf("can't get ClickHouse version with %v", err) + return 0, fmt.Errorf("can't get сlickHouse version: %v", err) } if len(result) == 0 { return 0, nil @@ -157,7 +157,7 @@ func (ch *ClickHouse) FreezeTableOldWay(table Table) error { } q := fmt.Sprintf("SELECT DISTINCT partition_id FROM `system`.`parts` WHERE database='%s' AND table='%s'", table.Database, table.Name) if err := ch.conn.Select(&partitions, q); err != nil { - return fmt.Errorf("can't get partitions for \"%s.%s\" with %v", table.Database, table.Name, err) + return fmt.Errorf("can't get partitions for '%s.%s': %v", table.Database, table.Name, err) } log.Printf("Freeze '%v.%v'", table.Database, table.Name) for _, item := range partitions { @@ -174,7 +174,7 @@ func (ch *ClickHouse) FreezeTableOldWay(table Table) error { table.Name) } if _, err := ch.conn.Exec(query); err != nil { - return fmt.Errorf("can't freeze partition '%s' on '%s.%s' with: %v", item.PartitionID, table.Database, table.Name, err) + return fmt.Errorf("can't freeze partition '%s' on '%s.%s': %v", item.PartitionID, table.Database, table.Name, err) } } return nil @@ -190,10 +190,10 @@ func (ch *ClickHouse) FreezeTable(table Table) error { if version < 19001005 || ch.Config.FreezeByPart { return ch.FreezeTableOldWay(table) } - log.Printf("Freeze `%s`.`%s`", table.Database, table.Name) + log.Printf("Freeze '%s.%s'", table.Database, table.Name) query := fmt.Sprintf("ALTER TABLE `%v`.`%v` FREEZE;", table.Database, table.Name) if _, err := ch.conn.Exec(query); err != nil { - return fmt.Errorf("can't freeze `%s`.`%s` with: %v", table.Database, table.Name, err) + return fmt.Errorf("can't freeze '%s.%s': %v", table.Database, table.Name, err) } return nil } @@ -285,7 +285,7 @@ func (ch *ClickHouse) Chown(filename string) error { // CopyData - copy partitions for specific table to detached folder func (ch *ClickHouse) CopyData(table BackupTable) error { - log.Printf("Prepare data for restoring `%s`.`%s`", table.Database, table.Name) + log.Printf("Prepare data for restoring '%s.%s'", table.Database, table.Name) dataPath, err := ch.GetDataPath() if err != nil { return err @@ -325,11 +325,11 @@ func (ch *ClickHouse) CopyData(table BackupTable) error { return nil } if err := os.Link(filePath, dstFilePath); err != nil { - return fmt.Errorf("failed to crete hard link '%s' -> '%s' with %v", filePath, dstFilePath, err) + return fmt.Errorf("failed to crete hard link '%s' -> '%s': %v", filePath, dstFilePath, err) } return ch.Chown(dstFilePath) }); err != nil { - return fmt.Errorf("error during filepath.Walk for partition '%s' with %v", partition.Path, err) + return fmt.Errorf("error during filepath.Walk for partition '%s': %v", partition.Path, err) } } return nil @@ -359,7 +359,7 @@ func (ch *ClickHouse) CreateTable(table RestoreTable) error { if _, err := ch.conn.Exec(fmt.Sprintf("USE `%s`", table.Database)); err != nil { return err } - log.Printf("Create table `%s`.`%s`", table.Database, table.Table) + log.Printf("Create table '%s.%s'", table.Database, table.Table) if _, err := ch.conn.Exec(table.Query); err != nil { return err } diff --git a/pkg/chbackup/config.go b/pkg/chbackup/config.go index 39c149e7..75079fa7 100644 --- a/pkg/chbackup/config.go +++ b/pkg/chbackup/config.go @@ -112,10 +112,10 @@ func LoadConfig(configLocation string) (*Config, error) { return config, err } if err != nil { - return nil, fmt.Errorf("can't open with %v", err) + return nil, fmt.Errorf("can't open config file: %v", err) } if err := yaml.Unmarshal(configYaml, &config); err != nil { - return nil, fmt.Errorf("can't parse with %v", err) + return nil, fmt.Errorf("can't parse config file: %v", err) } if err := envconfig.Process("", config); err != nil { return nil, err diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 0e210dca..833e58c7 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -30,7 +30,7 @@ type TestDataStruct struct { } var testData = []TestDataStruct{ - TestDataStruct{ + { Database: dbName, Table: ".inner.table1", Schema: "(Date Date, TimeStamp DateTime, Log String) ENGINE = MergeTree(Date, (TimeStamp, Log), 8192)", @@ -44,8 +44,7 @@ var testData = []TestDataStruct{ }, Fields: []string{"Date", "TimeStamp", "Log"}, OrderBy: "TimeStamp", - }, - TestDataStruct{ + }, { Database: dbName, Table: "2. Таблица №2", Schema: "(id UInt64, User String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192", @@ -59,8 +58,7 @@ var testData = []TestDataStruct{ }, Fields: []string{"id", "User"}, OrderBy: "id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "table3", Schema: "(TimeStamp DateTime, Item String, Date Date MATERIALIZED toDate(TimeStamp)) ENGINE = MergeTree() PARTITION BY Date ORDER BY TimeStamp SETTINGS index_granularity = 8192", @@ -74,8 +72,7 @@ var testData = []TestDataStruct{ }, Fields: []string{"TimeStamp", "Item"}, OrderBy: "TimeStamp", - }, - TestDataStruct{ + }, { Database: dbName, Table: "table4", Schema: "(id UInt64, Col1 String, Col2 String, Col3 String, Col4 String, Col5 String) ENGINE = MergeTree PARTITION BY id ORDER BY (id, Col1, Col2, Col3, Col4, Col5) SETTINGS index_granularity = 8192", @@ -88,8 +85,7 @@ var testData = []TestDataStruct{ }(), Fields: []string{"id", "Col1", "Col2", "Col3", "Col4", "Col5"}, OrderBy: "id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table2", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() PARTITION BY toYYYYMM(order_time) ORDER BY (order_time, order_id)", @@ -99,8 +95,7 @@ var testData = []TestDataStruct{ }, Fields: []string{"order_id", "order_time", "amount"}, OrderBy: "order_id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table3", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(order_time) ORDER BY (order_time, order_id)", @@ -110,8 +105,7 @@ var testData = []TestDataStruct{ }, Fields: []string{"order_id", "order_time", "amount"}, OrderBy: "order_id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table4", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() ORDER BY (order_time, order_id)", @@ -125,7 +119,7 @@ var testData = []TestDataStruct{ } var incrementData = []TestDataStruct{ - TestDataStruct{ + { Database: dbName, Table: ".inner.table1", Schema: "(Date Date, TimeStamp DateTime, Log String) ENGINE = MergeTree(Date, (TimeStamp, Log), 8192)", @@ -134,8 +128,7 @@ var incrementData = []TestDataStruct{ }, Fields: []string{"Date", "TimeStamp", "Log"}, OrderBy: "TimeStamp", - }, - TestDataStruct{ + }, { Database: dbName, Table: "2. Таблица №2", Schema: "(id UInt64, User String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192", @@ -147,8 +140,7 @@ var incrementData = []TestDataStruct{ }, Fields: []string{"id", "User"}, OrderBy: "id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "table3", Schema: "(TimeStamp DateTime, Item String, Date Date MATERIALIZED toDate(TimeStamp)) ENGINE = MergeTree() PARTITION BY Date ORDER BY TimeStamp SETTINGS index_granularity = 8192", @@ -158,8 +150,7 @@ var incrementData = []TestDataStruct{ }, Fields: []string{"TimeStamp", "Item"}, OrderBy: "TimeStamp", - }, - TestDataStruct{ + }, { Database: dbName, Table: "table4", Schema: "(id UInt64, Col1 String, Col2 String, Col3 String, Col4 String, Col5 String) ENGINE = MergeTree PARTITION BY id ORDER BY (id, Col1, Col2, Col3, Col4, Col5) SETTINGS index_granularity = 8192", @@ -172,8 +163,7 @@ var incrementData = []TestDataStruct{ }(), Fields: []string{"id", "Col1", "Col2", "Col3", "Col4", "Col5"}, OrderBy: "id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table2", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() PARTITION BY toYYYYMM(order_time) ORDER BY (order_time, order_id)", @@ -183,8 +173,7 @@ var incrementData = []TestDataStruct{ }, Fields: []string{"order_id", "order_time", "amount"}, OrderBy: "order_id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table3", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(order_time) ORDER BY (order_time, order_id)", @@ -194,8 +183,7 @@ var incrementData = []TestDataStruct{ }, Fields: []string{"order_id", "order_time", "amount"}, OrderBy: "order_id", - }, - TestDataStruct{ + }, { Database: dbName, Table: "yuzhichang_table4", Schema: "(order_id String, order_time DateTime, amount Float64) ENGINE = MergeTree() ORDER BY (order_time, order_id)", @@ -378,7 +366,7 @@ func (ch *TestClickHouse) createTestData(data TestDataStruct) error { for _, row := range data.Rows { tx, err := ch.chbackup.GetConn().Beginx() if err != nil { - return fmt.Errorf("can't begin transaction with: %v", err) + return fmt.Errorf("can't begin transaction: %v", err) } if _, err := tx.NamedExec( fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES (:%s)", @@ -387,10 +375,10 @@ func (ch *TestClickHouse) createTestData(data TestDataStruct) error { strings.Join(data.Fields, ","), strings.Join(data.Fields, ",:"), ), row); err != nil { - return fmt.Errorf("can't add insert to transaction with: %v", err) + return fmt.Errorf("can't add insert to transaction: %v", err) } if err := tx.Commit(); err != nil { - return fmt.Errorf("can't commit with: %v", err) + return fmt.Errorf("can't commit: %v", err) } } return nil