Skip to content

Commit

Permalink
support pgsql type cast [T883] (#296)
Browse files Browse the repository at this point in the history 
* support quoted table\column names and returning clause for postgresql

* fix golint issues

* change order of value|column_name

* use column/table names without quotes, add tests

* support unescaped string

* return error on incorrectly escaped string
  • Loading branch information
@Lagovas @vixentael
Lagovas authored and vixentael committed Dec 31, 2018
1 parent 334ee25 commit 18651eb
Show file tree
Hide file tree
Showing 13 changed files with 2,602 additions and 2,511 deletions.
21 changes: 13 additions & 8 deletions encryptor/dbDataCoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ limitations under the License.
package encryptor

import (
"bytes"
"encoding/hex"
"errors"
"github.com/cossacklabs/acra/sqlparser"
"github.com/cossacklabs/acra/utils"
"github.com/sirupsen/logrus"
)

Expand Down Expand Up @@ -91,16 +91,21 @@ func (*PostgresqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
}
return binValue, err
case sqlparser.PgEscapeString, sqlparser.StrVal:
// 4 == "\x" + min 2 symbols for hex decoding
if len(val.Val) >= 4 && bytes.Equal(val.Val[:len(pgHexStringPrefix)], pgHexStringPrefix) {
hexValue := val.Val[len(pgHexStringPrefix):]
binValue := make([]byte, hex.DecodedLen(len(hexValue)))
_, err := hex.Decode(binValue, hexValue)
if err != nil {
// try to decode hex/octal encoding
binValue, err := utils.DecodeEscaped(val.Val)
if err != nil {
// return error on hex decode
if _, ok := err.(hex.InvalidByteError); err == hex.ErrLength || ok {
return nil, err
} else if err == utils.ErrDecodeEscapedString {
return nil, err
}
return binValue, err

logrus.WithError(err).Warningln("Can't decode value, process as unescaped string")
// return value as is because it may be string with printable characters that wasn't encoded on client
return val.Val, nil
}
return binValue, nil
}
}
return nil, errUnsupportedExpression
Expand Down
13 changes: 7 additions & 6 deletions encryptor/dbDataCoder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"encoding/hex"
"fmt"
"github.com/cossacklabs/acra/sqlparser"
"github.com/cossacklabs/acra/utils"
"testing"
)

Expand Down Expand Up @@ -120,12 +121,12 @@ func TestPostgresqlDBDataCoder_Decode(t *testing.T) {
Expr: sqlparser.NewHexVal([]byte(hex.EncodeToString(testData))[1:]),
},
{
Err: errUnsupportedExpression,
Err: utils.ErrDecodeEscapedString,
// short data
Expr: sqlparser.NewStrVal([]byte{1, 2, 3}),
},
{
Err: errUnsupportedExpression,
Err: nil,
// without prefix
Expr: sqlparser.NewStrVal([]byte(fmt.Sprintf("%s", hex.EncodeToString(testData)))),
},
Expand All @@ -136,12 +137,12 @@ func TestPostgresqlDBDataCoder_Decode(t *testing.T) {
},
// PgEscapeVal same as for StrVal
{
Err: errUnsupportedExpression,
Err: utils.ErrDecodeEscapedString,
// short data
Expr: sqlparser.NewPgEscapeString([]byte{1, 2, 3}),
},
{
Err: errUnsupportedExpression,
Err: nil,
// without prefix
Expr: sqlparser.NewPgEscapeString([]byte(fmt.Sprintf("%s", hex.EncodeToString(testData)))),
},
Expand All @@ -151,10 +152,10 @@ func TestPostgresqlDBDataCoder_Decode(t *testing.T) {
Expr: sqlparser.NewPgEscapeString([]byte(fmt.Sprintf("\\x%s", hex.EncodeToString(testData)[1:]))),
},
}
for _, testCase := range errTestCases {
for i, testCase := range errTestCases {
_, err := coder.Decode(testCase.Expr)
if err != testCase.Err {
t.Fatalf("Incorrect error. Took: %s; Expected: %s", err.Error(), testCase.Err.Error())
t.Fatalf("[%d] Incorrect error. Took: %s; Expected: %s", i, err, testCase.Err.Error())
}
}
}
Expand Down
7 changes: 4 additions & 3 deletions encryptor/queryDataEncryptor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ func NewPostgresqlQueryEncryptor(schema TableSchemaStore, clientID []byte, dataE

// encryptInsertQuery encrypt data in insert query in VALUES and ON DUPLICATE KEY UPDATE statements
func (encryptor *QueryDataEncryptor) encryptInsertQuery(insert *sqlparser.Insert) (bool, error) {
tableName := sqlparser.String(insert.Table.Name)
schema := encryptor.schemaStore.GetTableSchema(tableName)
tableName := insert.Table.Name
schema := encryptor.schemaStore.GetTableSchema(tableName.String())
if schema == nil {
// unsupported table, we have not schema and query hasn't columns description
logrus.Debugf("Hasn't schema for table %s", tableName)
Expand All @@ -55,7 +55,7 @@ func (encryptor *QueryDataEncryptor) encryptInsertQuery(insert *sqlparser.Insert
if len(insert.Columns) > 0 {
columnsName = make([]string, 0, len(insert.Columns))
for _, col := range insert.Columns {
columnsName = append(columnsName, sqlparser.String(col))
columnsName = append(columnsName, col.String())
}
} else if len(schema.Columns) > 0 {
columnsName = schema.Columns
Expand Down Expand Up @@ -127,6 +127,7 @@ func (encryptor *QueryDataEncryptor) encryptExpression(expr sqlparser.Expr, sche
}
}
}
logrus.WithField("column_name", columnName).Debugln("Skip encryption for column")
return false, nil
}

Expand Down
90 changes: 88 additions & 2 deletions encryptor/queryDataEncryptor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func normalizeQuery(query string, t *testing.T) string {
return sqlparser.String(parsed)
}

func TestMysqlQueryParser_Parse(t *testing.T) {
func TestGeneralQueryParser_Parse(t *testing.T) {
zoneID := zone.GenerateZoneID()
zoneIDStr := string(zoneID)
clientIDStr := "specified_client_id"
Expand Down Expand Up @@ -95,9 +95,16 @@ schemas:
if err != nil {
t.Fatalf("Can't parse config: %s", err.Error())
}
simpleStringData := []byte("string data")
encryptedValue := []byte("encrypted")
hexEncryptedValue := hex.EncodeToString(encryptedValue)
dataValue := "some data"
toPgHexValue := func(s string) string {
return fmt.Sprintf("\\x%s", s)
}
dataValue := make([]byte, 256)
for i := 0; i < 256; i++ {
dataValue[i] = byte(i)
}
dataHexValue := hex.EncodeToString([]byte(dataValue))
testData := []struct {
Query string
Expand All @@ -106,6 +113,7 @@ schemas:
ExpectedQueryData []interface{}
Changed bool
ExpectedIDS [][]byte
DataCoder DBDataCoder
}{
// 0. without list of columns and with schema, one value
{
Expand Down Expand Up @@ -296,6 +304,81 @@ schemas:
Changed: true,
ExpectedIDS: [][]byte{defaultClientID},
},
// 21. with double quoted table and column names
{
Query: `INSERT INTO "TableWithoutColumnSchema" ("zone_id", "specified_client_id", "other_column", "default_client_id") VALUES (X'%s', X'%s', 1, X'%s')`,
QueryData: []interface{}{dataHexValue, dataHexValue, dataHexValue},
ExpectedQueryData: []interface{}{hexEncryptedValue, hexEncryptedValue, hexEncryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{zoneID, specifiedClientID, defaultClientID},
},
// 22. with back quoted table and column names
{
Query: "INSERT INTO `TableWithoutColumnSchema` (`zone_id`, `specified_client_id`, `other_column`, `default_client_id`) VALUES (X'%s', X'%s', 1, X'%s')",
QueryData: []interface{}{dataHexValue, dataHexValue, dataHexValue},
ExpectedQueryData: []interface{}{hexEncryptedValue, hexEncryptedValue, hexEncryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{zoneID, specifiedClientID, defaultClientID},
},
// 23. update with double quoted identifiers
{
Query: `UPDATE "TableWithoutColumnSchema" as "t" set "other_column"=X'%s', "specified_client_id"=X'%s', "zone_id"=X'%s', "default_client_id"=X'%s'`,
QueryData: []interface{}{dataHexValue, dataHexValue, dataHexValue, dataHexValue},
ExpectedQueryData: []interface{}{dataHexValue, hexEncryptedValue, hexEncryptedValue, hexEncryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{specifiedClientID, zoneID, defaultClientID},
},
// 24. update with back quoted identifiers
{
Query: "UPDATE `TableWithoutColumnSchema` as `t` set `other_column`=X'%s', `specified_client_id`=X'%s', `zone_id`=X'%s', `default_client_id`=X'%s'",
QueryData: []interface{}{dataHexValue, dataHexValue, dataHexValue, dataHexValue},
ExpectedQueryData: []interface{}{dataHexValue, hexEncryptedValue, hexEncryptedValue, hexEncryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{specifiedClientID, zoneID, defaultClientID},
},
// 25. insert with data as simple string
{
Query: `INSERT INTO "TableWithoutColumnSchema" ("zone_id", "specified_client_id", "other_column", "default_client_id") VALUES ('%s', '%s', 1, '%s')`,
QueryData: []interface{}{simpleStringData, simpleStringData, simpleStringData},
ExpectedQueryData: []interface{}{encryptedValue, encryptedValue, encryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{zoneID, specifiedClientID, defaultClientID},
},
// 26. update with data as simple string
{
Query: `UPDATE "TableWithoutColumnSchema" as "t" set "other_column"='%s', "specified_client_id"='%s', "zone_id"='%s', "default_client_id"='%s'`,
QueryData: []interface{}{simpleStringData, simpleStringData, simpleStringData, simpleStringData},
ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue, encryptedValue},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{specifiedClientID, zoneID, defaultClientID},
},

// 27. insert with data as simple string for postgresql
{
Query: `INSERT INTO "TableWithoutColumnSchema" ("zone_id", "specified_client_id", "other_column", "default_client_id") VALUES ('%s', '%s', 1, '%s')`,
QueryData: []interface{}{simpleStringData, simpleStringData, simpleStringData},
ExpectedQueryData: []interface{}{toPgHexValue(hexEncryptedValue), toPgHexValue(hexEncryptedValue), toPgHexValue(hexEncryptedValue)},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{zoneID, specifiedClientID, defaultClientID},
DataCoder: &PostgresqlDBDataCoder{},
},
// 28. update with data as simple string for postgresql
{
Query: `UPDATE "TableWithoutColumnSchema" as "t" set "other_column"='%s', "specified_client_id"='%s', "zone_id"='%s', "default_client_id"='%s'`,
QueryData: []interface{}{simpleStringData, simpleStringData, simpleStringData, simpleStringData},
ExpectedQueryData: []interface{}{simpleStringData, toPgHexValue(hexEncryptedValue), toPgHexValue(hexEncryptedValue), toPgHexValue(hexEncryptedValue)},
Normalized: true,
Changed: true,
ExpectedIDS: [][]byte{specifiedClientID, zoneID, defaultClientID},
DataCoder: &PostgresqlDBDataCoder{},
},
}
encryptor := &testEncryptor{value: encryptedValue}
mysqlParser, err := NewMysqlQueryEncryptor(schemaStore, defaultClientID, encryptor)
Expand All @@ -305,6 +388,9 @@ schemas:

for i, testCase := range testData {
encryptor.reset()
if testCase.DataCoder != nil {
mysqlParser.dataCoder = testCase.DataCoder
}
query := fmt.Sprintf(testCase.Query, testCase.QueryData...)
expectedQuery := fmt.Sprintf(testCase.Query, testCase.ExpectedQueryData...)
if testCase.Normalized {
Expand Down
Loading

0 comments on commit 18651eb

Please sign in to comment.