From f54a1cfb1f189171ce8938b10918c98127423827 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 17:00:27 +0300 Subject: [PATCH 01/19] Move EncodingValue into the base module To reuse it later in mysql --- decryptor/base/encoder.go | 156 ++++++++++++++++++++++++++- decryptor/postgresql/data_encoder.go | 142 +++--------------------- 2 files changed, 170 insertions(+), 128 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index fdbb76750..a5ba64f54 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -1,6 +1,17 @@ package base -import "fmt" +import ( + "encoding/base64" + "encoding/binary" + "fmt" + "strconv" + + "github.com/cossacklabs/acra/encryptor/config" + "github.com/cossacklabs/acra/utils" + "github.com/sirupsen/logrus" + + "github.com/cossacklabs/acra/encryptor/config/common" +) // EncodingError is returned from encoding handlers when some failure occurs. // This error should be sent to the user directly, so it needs to be own type @@ -28,3 +39,146 @@ func (e *EncodingError) Is(err error) bool { func NewEncodingError(column string) error { return &EncodingError{column} } + +// EncodingValue represents a (possibly parsed and prepared) value that is +// ready to be encoded +type EncodingValue interface { + // AsPostgresBinary returns value encoded in postgres binary format + AsPostgresBinary() []byte + // AsPostgresText returns value encoded in postgres text format + AsPostgresText() []byte +} + +// ByteSequenceValue is an abstraction over all byte-sequence values -- strings +// and []byte (because they are encoded in the same way) +type ByteSequenceValue struct { + seq []byte +} + +// NewByteSequenceValue returns EncodingValue from a byte-string-like data +func NewByteSequenceValue(seq []byte) EncodingValue { + return &ByteSequenceValue{seq} +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For a byte sequence value (string or []byte) this is an identity operation +func (v *ByteSequenceValue) AsPostgresBinary() []byte { + return v.seq +} + +// AsPostgresText returns value encoded in postgres text format +// For a byte sequence value (string or []byte) this is a hex encoded string +func (v *ByteSequenceValue) AsPostgresText() []byte { + // all bytes should be encoded as valid bytea value + return utils.PgEncodeToHex(v.seq) +} + +// IntValue represents a {size*8}-bit integer ready for encoding +type IntValue struct { + size int + value int64 + strValue string +} + +// NewIntValue returns EncodingValue from integer with size*8 bits +func NewIntValue(size int, value int64, strValue string) EncodingValue { + return &IntValue{size, value, strValue} +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For an int value it is a big endian encoded integer +func (v *IntValue) AsPostgresBinary() []byte { + newData := make([]byte, v.size) + switch v.size { + case 4: + binary.BigEndian.PutUint32(newData, uint32(v.value)) + case 8: + binary.BigEndian.PutUint64(newData, uint64(v.value)) + } + return newData +} + +// AsPostgresText returns value encoded in postgres text format +// For an int this means returning textual representation of the integer +func (v *IntValue) AsPostgresText() []byte { + return []byte(v.strValue) +} + +// IdentityValue is an encodingValue that just returns data as is +type IdentityValue struct { + data []byte +} + +// NewIdentityValue returns EncodingValue as identity value +func NewIdentityValue(data []byte) EncodingValue { + return &IdentityValue{data} +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For identity value this means returning value as it is +func (v *IdentityValue) AsPostgresBinary() []byte { + return v.data +} + +// AsPostgresText returns value encoded in postgres text format +// For identity value this means returning value as it is +func (v *IdentityValue) AsPostgresText() []byte { + return v.data +} + +// EncodeDefault returns wrapped default value from settings ready for encoding +// returns nil if something went wrong, which in many cases indicates that the +// original value should be returned as it is +func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) EncodingValue { + strValue := setting.GetDefaultDataValue() + if strValue == nil { + logger.Errorln("Default value is not specified") + return nil + } + + dataType := setting.GetEncryptedDataType() + + switch dataType { + case common.EncryptedType_String: + return &IdentityValue{[]byte(*strValue)} + case common.EncryptedType_Bytes: + binValue, err := base64.StdEncoding.DecodeString(*strValue) + if err != nil { + logger.WithError(err).Errorln("Can't decode base64 default value") + return nil + } + return &ByteSequenceValue{seq: binValue} + case common.EncryptedType_Int32, common.EncryptedType_Int64: + size := 8 + if dataType == common.EncryptedType_Int32 { + size = 4 + } + value, err := strconv.ParseInt(*strValue, 10, 64) + if err != nil { + logger.WithError(err).Errorln("Can't parse default integer value") + return nil + } + + return &IntValue{size: size, value: value, strValue: *strValue} + } + return nil +} + +// EncodeOnFail returns either an error, which should be returned, or value, which +// should be encoded, because there is some problem with original, or `nil` +// which indicates that original value should be returned as is. +func EncodeOnFail(setting config.ColumnEncryptionSetting, logger *logrus.Entry) (EncodingValue, error) { + action := setting.GetResponseOnFail() + switch action { + case common.ResponseOnFailEmpty, common.ResponseOnFailCiphertext: + return nil, nil + + case common.ResponseOnFailDefault: + return EncodeDefault(setting, logger), nil + + case common.ResponseOnFailError: + return nil, NewEncodingError(setting.ColumnName()) + } + + return nil, fmt.Errorf("unknown action: %q", action) +} diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index 1d3141637..0ea6bf235 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -2,9 +2,7 @@ package postgresql import ( "context" - "encoding/base64" "encoding/binary" - "fmt" "strconv" "github.com/cossacklabs/acra/decryptor/base" @@ -29,15 +27,15 @@ func (p *PgSQLDataEncoderProcessor) ID() string { return "PgSQLDataEncoderProcessor" } -func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, encodingValue, error) { +func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, base.EncodingValue, error) { logger = logger.WithField("column", setting.ColumnName()).WithField("decrypted", base.IsDecryptedFromContext(ctx)) if len(data) == 0 { - return ctx, &identityValue{data}, nil + return ctx, base.NewIdentityValue(data), nil } switch setting.GetEncryptedDataType() { case common2.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { - value, err := encodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -45,17 +43,17 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } // decrypted values return as is, without any encoding - return ctx, &identityValue{data}, nil + return ctx, base.NewIdentityValue(data), nil case common2.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - value, err := encodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, logger) if err != nil { return ctx, nil, err } else if value != nil { return ctx, value, nil } } - return ctx, newByteSequence(data), nil + return ctx, base.NewByteSequenceValue(data), nil case common2.EncryptedType_Int32, common2.EncryptedType_Int64: size := 8 if setting.GetEncryptedDataType() == common2.EncryptedType_Int32 { @@ -66,12 +64,12 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by // if it's valid string literal and decrypted, return as is value, err := strconv.ParseInt(strValue, 10, 64) if err == nil { - val := intValue{size, value, strValue} - return ctx, &val, nil + val := base.NewIntValue(size, value, strValue) + return ctx, val, nil } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - value, err := encodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -79,20 +77,20 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } logger.Warningln("Can't decode int value and no default value") - return ctx, &identityValue{data}, nil + return ctx, base.NewIdentityValue(data), nil } // here we process AcraStruct/AcraBlock decryption without any encryptor config that defines data_type/token_type // values. If it was decrypted then we return it as valid bytea value if base.IsDecryptedFromContext(ctx) { - return ctx, &byteSequenceValue{seq: data}, nil + return ctx, base.NewByteSequenceValue(data), nil } // If it wasn't decrypted (due to inappropriate keys or not AcraStructs as payload) then we return it in same way // as it come to us. encodedValue, ok := getEncodedValueFromContext(ctx) if ok { - return ctx, &identityValue{encodedValue}, nil + return ctx, base.NewIdentityValue(encodedValue), nil } - return ctx, &identityValue{data}, nil + return ctx, base.NewIdentityValue(data), nil } // OnColumn encode binary value to text and back. Should be before and after tokenizer processor @@ -117,9 +115,9 @@ func (p *PgSQLDataEncoderProcessor) OnColumn(ctx context.Context, data []byte) ( } if columnInfo.IsBinaryFormat() { - return ctx, value.asBinary(), nil + return ctx, value.AsPostgresBinary(), nil } - return ctx, value.asText(), nil + return ctx, value.AsPostgresText(), nil } // PgSQLDataDecoderProcessor implements processor and decode binary/text values from DB @@ -233,113 +231,3 @@ func (p *PgSQLDataDecoderProcessor) OnColumn(ctx context.Context, data []byte) ( } return p.decodeText(ctx, data, columnSetting, columnInfo, logger) } - -// encodingValue represents a (possibly parsed and prepared) value that is -// ready to be encoded -type encodingValue interface { - asBinary() []byte - asText() []byte -} - -// byteSequenceValue is an abstraction over all byte-sequence values -- strings -// and []byte (because they are encoded in the same way) -type byteSequenceValue struct { - seq []byte -} - -func newByteSequence(seq []byte) encodingValue { - return &byteSequenceValue{seq} -} - -func (v byteSequenceValue) asBinary() []byte { return v.seq } -func (v byteSequenceValue) asText() []byte { - // all bytes should be encoded as valid bytea value - return utils.PgEncodeToHex(v.seq) -} - -// intValue represents a {size*8}-bit integer ready for encoding -type intValue struct { - size int - value int64 - strValue string -} - -func (v *intValue) asBinary() []byte { - newData := make([]byte, v.size) - switch v.size { - case 4: - binary.BigEndian.PutUint32(newData, uint32(v.value)) - case 8: - binary.BigEndian.PutUint64(newData, uint64(v.value)) - } - return newData -} - -func (v *intValue) asText() []byte { - return []byte(v.strValue) -} - -// identityValue is an encodingValue that just returns data as is -type identityValue struct { - data []byte -} - -func (v *identityValue) asBinary() []byte { return v.data } -func (v *identityValue) asText() []byte { return v.data } - -// encodeDefault returns wrapped default value from settings ready for encoding -// returns nil if something went wrong, which in many cases indicates that the -// original value should be returned as it is -func encodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) encodingValue { - strValue := setting.GetDefaultDataValue() - if strValue == nil { - logger.Errorln("Default value is not specified") - return nil - } - - dataType := setting.GetEncryptedDataType() - - switch dataType { - case common2.EncryptedType_String: - return &identityValue{[]byte(*strValue)} - case common2.EncryptedType_Bytes: - binValue, err := base64.StdEncoding.DecodeString(*strValue) - if err != nil { - logger.WithError(err).Errorln("Can't decode base64 default value") - return nil - } - return &byteSequenceValue{seq: binValue} - case common2.EncryptedType_Int32, common2.EncryptedType_Int64: - size := 8 - if dataType == common2.EncryptedType_Int32 { - size = 4 - } - value, err := strconv.ParseInt(*strValue, 10, 64) - if err != nil { - logger.WithError(err).Errorln("Can't parse default integer value") - return nil - } - - return &intValue{size: size, value: value, strValue: *strValue} - } - return nil -} - -// encodeOnFail returns either an error, which should be returned, or value, which -// should be encoded, because there is some problem with original, or `nil` -// which indicates that original value should be returned as is. -func encodeOnFail(setting config.ColumnEncryptionSetting, logger *logrus.Entry) (encodingValue, error) { - action := setting.GetResponseOnFail() - switch action { - case common2.ResponseOnFailEmpty, common2.ResponseOnFailCiphertext: - return nil, nil - - case common2.ResponseOnFailDefault: - return encodeDefault(setting, logger), nil - - case common2.ResponseOnFailError: - return nil, base.NewEncodingError(setting.ColumnName()) - } - - return nil, fmt.Errorf("unknown action: %q", action) -} From 1a913c17f02eca16f8f7025182be6656f5167a56 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 17:57:48 +0300 Subject: [PATCH 02/19] Move mysql.utils into base.utils To prevent import cycles --- decryptor/{mysql => base}/utils.go | 5 ++-- decryptor/mysql/column_field.go | 38 ++++++++++++++------------ decryptor/mysql/data_encoder.go | 23 ++++++++-------- decryptor/mysql/prepared_statements.go | 4 +-- decryptor/mysql/response_proxy.go | 17 ++++++------ 5 files changed, 46 insertions(+), 41 deletions(-) rename decryptor/{mysql => base}/utils.go (99%) diff --git a/decryptor/mysql/utils.go b/decryptor/base/utils.go similarity index 99% rename from decryptor/mysql/utils.go rename to decryptor/base/utils.go index 99307f5ce..70eba4005 100644 --- a/decryptor/mysql/utils.go +++ b/decryptor/base/utils.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mysql +package base import ( "errors" + "io" + "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" - "io" ) // ErrMalformPacket if packet parsing failed diff --git a/decryptor/mysql/column_field.go b/decryptor/mysql/column_field.go index 36d22f4c0..33fc7c0a3 100644 --- a/decryptor/mysql/column_field.go +++ b/decryptor/mysql/column_field.go @@ -19,6 +19,8 @@ package mysql import ( "encoding/binary" "errors" + + "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" ) @@ -111,7 +113,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { var err error //skip catalog, always def pos := 0 - n, err = SkipLengthEncodedString(packet.data) + n, err = base.SkipLengthEncodedString(packet.data) if err != nil { return nil, err @@ -119,35 +121,35 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { pos += n //schema - field.Schema, n, err = LengthEncodedString(packet.data[pos:]) + field.Schema, n, err = base.LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //table - field.Table, n, err = LengthEncodedString(packet.data[pos:]) + field.Table, n, err = base.LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //org_table - field.OrgTable, n, err = LengthEncodedString(packet.data[pos:]) + field.OrgTable, n, err = base.LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //name - field.Name, n, err = LengthEncodedString(packet.data[pos:]) + field.Name, n, err = base.LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //org_name - field.OrgName, n, err = LengthEncodedString(packet.data[pos:]) + field.OrgName, n, err = base.LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } @@ -183,7 +185,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { //if more data, command was field list if len(packet.data) > pos { //length of default value lenenc-int - field.DefaultValueLength, _, n, err = LengthEncodedInt(packet.data[pos:]) + field.DefaultValueLength, _, n, err = base.LengthEncodedInt(packet.data[pos:]) if err != nil { log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).WithError(err).Errorln("Can't get length encoded integer of default value length") return nil, err @@ -192,7 +194,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { if pos+int(field.DefaultValueLength) > len(packet.data) { log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).Errorln("Incorrect position, malformed packet") - err = ErrMalformPacket + err = base.ErrMalformPacket return nil, err } @@ -218,30 +220,30 @@ func (field *ColumnDescription) Dump() []byte { data := make([]byte, 0, l) - data = append(data, PutLengthEncodedString([]byte("def"))...) + data = append(data, base.PutLengthEncodedString([]byte("def"))...) - data = append(data, PutLengthEncodedString(field.Schema)...) + data = append(data, base.PutLengthEncodedString(field.Schema)...) - data = append(data, PutLengthEncodedString(field.Table)...) - data = append(data, PutLengthEncodedString(field.OrgTable)...) + data = append(data, base.PutLengthEncodedString(field.Table)...) + data = append(data, base.PutLengthEncodedString(field.OrgTable)...) - data = append(data, PutLengthEncodedString(field.Name)...) - data = append(data, PutLengthEncodedString(field.OrgName)...) + data = append(data, base.PutLengthEncodedString(field.Name)...) + data = append(data, base.PutLengthEncodedString(field.OrgName)...) // length of fixed-length fields // https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition data = append(data, 0x0c) - data = append(data, Uint16ToBytes(field.Charset)...) - data = append(data, Uint32ToBytes(field.ColumnLength)...) + data = append(data, base.Uint16ToBytes(field.Charset)...) + data = append(data, base.Uint32ToBytes(field.ColumnLength)...) data = append(data, byte(field.Type)) - data = append(data, Uint16ToBytes(field.Flag)...) + data = append(data, base.Uint16ToBytes(field.Flag)...) data = append(data, field.Decimal) // filler data = append(data, 0, 0) if field.DefaultValue != nil { - data = append(data, Uint64ToBytes(field.DefaultValueLength)...) + data = append(data, base.Uint64ToBytes(field.DefaultValueLength)...) data = append(data, field.DefaultValue...) } diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 44728114a..f034cb7dc 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -6,9 +6,10 @@ import ( "encoding/base64" "encoding/binary" "errors" - "github.com/cossacklabs/acra/utils" "strconv" + "github.com/cossacklabs/acra/utils" + "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/encryptor" "github.com/cossacklabs/acra/encryptor/config" @@ -69,7 +70,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, if len(data) == 0 { // we still need to encode result data as it might be null field in db - return ctx, PutLengthEncodedString(data), nil + return ctx, base.PutLengthEncodedString(data), nil } dataTypeEncoded, isEncoded, err := p.encodeBinaryWithDataType(ctx, data, setting) @@ -153,7 +154,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, encoded, err } - return ctx, PutLengthEncodedString(data), nil + return ctx, base.PutLengthEncodedString(data), nil } func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting) ([]byte, bool, error) { @@ -165,7 +166,7 @@ func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, d if err != nil { return data, false, err } - return PutLengthEncodedString(binValue), true, nil + return base.PutLengthEncodedString(binValue), true, nil } return data, false, ErrConvertToDataType } @@ -173,7 +174,7 @@ func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, d case common.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { if value := setting.GetDefaultDataValue(); value != nil { - return PutLengthEncodedString([]byte(*value)), true, nil + return base.PutLengthEncodedString([]byte(*value)), true, nil } return data, false, ErrConvertToDataType } @@ -219,7 +220,7 @@ func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, dat case common.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { if value := setting.GetDefaultDataValue(); value != nil { - return PutLengthEncodedString([]byte(*value)), true, nil + return base.PutLengthEncodedString([]byte(*value)), true, nil } return data, false, ErrConvertToDataType } @@ -231,7 +232,7 @@ func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, dat if err != nil { return nil, false, err } - return PutLengthEncodedString(binValue), true, nil + return base.PutLengthEncodedString(binValue), true, nil } return data, false, ErrConvertToDataType } @@ -240,12 +241,12 @@ func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, dat _, err := strconv.ParseInt(utils.BytesToString(data), 10, 64) // if it's valid string literal and decrypted, return as is if err == nil { - return PutLengthEncodedString(data), true, nil + return base.PutLengthEncodedString(data), true, nil } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { if newVal := setting.GetDefaultDataValue(); newVal != nil { - return PutLengthEncodedString([]byte(*newVal)), true, nil + return base.PutLengthEncodedString([]byte(*newVal)), true, nil } return data, false, ErrConvertToDataType } @@ -259,7 +260,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se logger.Debugln("Encode text") if len(data) == 0 { // we still need to encode result data as it might be null field in db - return ctx, PutLengthEncodedString(data), nil + return ctx, base.PutLengthEncodedString(data), nil } dataTypeEncoded, isEncoded, err := p.encodeTextWithDataType(ctx, data, setting) @@ -277,7 +278,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se ctx = base.MarkErrorConvertedDataTypeContext(ctx) } - return ctx, PutLengthEncodedString(data), nil + return ctx, base.PutLengthEncodedString(data), nil } func (p *BaseMySQLDataProcessor) decodeBinary(ctx context.Context, encoded []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, []byte, error) { diff --git a/decryptor/mysql/prepared_statements.go b/decryptor/mysql/prepared_statements.go index 8c440cd52..a0b71ff0e 100644 --- a/decryptor/mysql/prepared_statements.go +++ b/decryptor/mysql/prepared_statements.go @@ -112,7 +112,7 @@ func NewMysqlBoundValue(data []byte, format base.BoundValueFormat, paramType Typ // if we cant find amount of stored bytes for the paramType assume that it is length encoded string storageBytes, ok := NumericTypesStorageBytes[paramType] if !ok { - value, n, err := LengthEncodedString(data) + value, n, err := base.LengthEncodedString(data) if err != nil { return nil, 0, err } @@ -226,7 +226,7 @@ func (m *mysqlBoundValue) GetData(_ config.ColumnEncryptionSetting) ([]byte, err func (m *mysqlBoundValue) Encode() (encoded []byte, err error) { storageBytes, ok := NumericTypesStorageBytes[m.paramType] if !ok { - return PutLengthEncodedString(m.data), nil + return base.PutLengthEncodedString(m.data), nil } // separate error variable for output error from case statements // to not overlap with new err variables inside diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go index 3f7ad8dc6..3e680ddac 100644 --- a/decryptor/mysql/response_proxy.go +++ b/decryptor/mysql/response_proxy.go @@ -20,15 +20,16 @@ import ( "context" "encoding/binary" "errors" - "github.com/cossacklabs/acra/keystore/filesystem" - "github.com/cossacklabs/acra/sqlparser" - "go.opencensus.io/trace" "io" "net" "strconv" "time" - "github.com/cossacklabs/acra/acra-censor" + "github.com/cossacklabs/acra/keystore/filesystem" + "github.com/cossacklabs/acra/sqlparser" + "go.opencensus.io/trace" + + acracensor "github.com/cossacklabs/acra/acra-censor" "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" @@ -496,7 +497,7 @@ func (handler *Handler) processTextDataRow(ctx context.Context, rowData []byte, handler.logger.Debugln("Process data rows in text protocol") for i := range fields { fieldLogger = handler.logger.WithField("field_index", i) - value, n, err := LengthEncodedString(rowData[pos:]) + value, n, err := base.LengthEncodedString(rowData[pos:]) if err != nil { return nil, err } @@ -534,7 +535,7 @@ func (handler *Handler) processBinaryDataRow(ctx context.Context, rowData []byte } if rowData[0] != OkPacket { - return nil, ErrMalformPacket + return nil, base.ErrMalformPacket } // https://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html @@ -609,7 +610,7 @@ func (handler *Handler) extractData(pos int, rowData []byte, field *ColumnDescri return rowData[pos : pos+8], 8, nil case TypeDecimal, TypeNewDecimal, TypeBit, TypeEnum, TypeSet, TypeGeometry, TypeDate, TypeNewDate, TypeTimestamp, TypeDatetime, TypeTime, TypeVarchar, TypeTinyBlob, TypeMediumBlob, TypeLongBlob, TypeBlob, TypeVarString, TypeString: - value, n, err := LengthEncodedString(rowData[pos:]) + value, n, err := base.LengthEncodedString(rowData[pos:]) if err != nil { handler.logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). Errorln("Can't handle length encoded string non binary value") @@ -653,7 +654,7 @@ func (handler *Handler) QueryResponseHandler(ctx context.Context, packet *Packet if fieldPacket.IsEOF() { if i != fieldCount { handler.logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).Errorln("EOF and field count != current row packet count") - return ErrMalformPacket + return base.ErrMalformPacket } output = append(output, fieldPacket) break From 7bde560957181616e89d852ff11b8edb2a5a5dce Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 19:13:22 +0300 Subject: [PATCH 03/19] Add OnFail handler into text encoder --- decryptor/base/encoder.go | 21 +++++++++++++++++++++ decryptor/mysql/data_encoder.go | 29 +++++++++++++++++------------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index a5ba64f54..d5b32bd22 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -47,6 +47,9 @@ type EncodingValue interface { AsPostgresBinary() []byte // AsPostgresText returns value encoded in postgres text format AsPostgresText() []byte + + // AsMysqlText returns value encoded in mysql text format + AsMysqlText() []byte } // ByteSequenceValue is an abstraction over all byte-sequence values -- strings @@ -73,6 +76,12 @@ func (v *ByteSequenceValue) AsPostgresText() []byte { return utils.PgEncodeToHex(v.seq) } +// AsMysqlText returns value encoded in mysql text format +// For a byte sequence value (string or []byte) this is a length encoded string +func (v *ByteSequenceValue) AsMysqlText() []byte { + return PutLengthEncodedString(v.seq) +} + // IntValue represents a {size*8}-bit integer ready for encoding type IntValue struct { size int @@ -104,6 +113,12 @@ func (v *IntValue) AsPostgresText() []byte { return []byte(v.strValue) } +// AsMysqlText returns value encoded in mysql text format +// For an int this is a length encoded string of that integer +func (v *IntValue) AsMysqlText() []byte { + return PutLengthEncodedString([]byte(v.strValue)) +} + // IdentityValue is an encodingValue that just returns data as is type IdentityValue struct { data []byte @@ -126,6 +141,12 @@ func (v *IdentityValue) AsPostgresText() []byte { return v.data } +// AsMysqlText returns value encoded in mysql text format +// For identity value this means returning value as it is +func (v *IdentityValue) AsMysqlText() []byte { + return v.data +} + // EncodeDefault returns wrapped default value from settings ready for encoding // returns nil if something went wrong, which in many cases indicates that the // original value should be returned as it is diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index f034cb7dc..3e9788109 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -215,24 +215,26 @@ func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, d return data, false, nil } -func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting) ([]byte, bool, error) { +func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) ([]byte, bool, error) { switch setting.GetEncryptedDataType() { case common.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { - if value := setting.GetDefaultDataValue(); value != nil { - return base.PutLengthEncodedString([]byte(*value)), true, nil + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlText(), true, nil } return data, false, ErrConvertToDataType } return data, false, nil case common.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - if newValue := setting.GetDefaultDataValue(); newValue != nil { - binValue, err := base64.StdEncoding.DecodeString(*newValue) - if err != nil { - return nil, false, err - } - return base.PutLengthEncodedString(binValue), true, nil + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlText(), true, nil } return data, false, ErrConvertToDataType } @@ -245,8 +247,11 @@ func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, dat } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - if newVal := setting.GetDefaultDataValue(); newVal != nil { - return base.PutLengthEncodedString([]byte(*newVal)), true, nil + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlText(), true, nil } return data, false, ErrConvertToDataType } @@ -263,7 +268,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se return ctx, base.PutLengthEncodedString(data), nil } - dataTypeEncoded, isEncoded, err := p.encodeTextWithDataType(ctx, data, setting) + dataTypeEncoded, isEncoded, err := p.encodeTextWithDataType(ctx, data, setting, logger) if err != nil && err != ErrConvertToDataType { return nil, nil, err } From c0dca3c0b2a615f1b350aa630a940bec2dfffe50 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 19:24:11 +0300 Subject: [PATCH 04/19] Add OnFail handler for a binary encoding --- decryptor/base/encoder.go | 28 +++++++++++++++++++ decryptor/mysql/data_encoder.go | 49 ++++++++++++++++----------------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index d5b32bd22..44e192403 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -48,6 +48,8 @@ type EncodingValue interface { // AsPostgresText returns value encoded in postgres text format AsPostgresText() []byte + // AsMysqlBinary returns value encoded in mysql binary format + AsMysqlBinary() []byte // AsMysqlText returns value encoded in mysql text format AsMysqlText() []byte } @@ -76,6 +78,13 @@ func (v *ByteSequenceValue) AsPostgresText() []byte { return utils.PgEncodeToHex(v.seq) } +// AsMysqlBinary returns value encoded in mysql binary format +// For a byte sequence value (string or []byte) this is the same as text +// encoding +func (v *ByteSequenceValue) AsMysqlBinary() []byte { + return v.AsMysqlText() +} + // AsMysqlText returns value encoded in mysql text format // For a byte sequence value (string or []byte) this is a length encoded string func (v *ByteSequenceValue) AsMysqlText() []byte { @@ -113,6 +122,19 @@ func (v *IntValue) AsPostgresText() []byte { return []byte(v.strValue) } +// AsMysqlBinary returns value encoded in mysql binary format +// For an int value it is a little endian encoded integer +func (v *IntValue) AsMysqlBinary() []byte { + newData := make([]byte, v.size) + switch v.size { + case 4: + binary.LittleEndian.PutUint32(newData, uint32(v.value)) + case 8: + binary.LittleEndian.PutUint64(newData, uint64(v.value)) + } + return newData +} + // AsMysqlText returns value encoded in mysql text format // For an int this is a length encoded string of that integer func (v *IntValue) AsMysqlText() []byte { @@ -141,6 +163,12 @@ func (v *IdentityValue) AsPostgresText() []byte { return v.data } +// AsMysqlBinary returns value encoded in mysql binary format +// For identity value this means returning value as it is +func (v *IdentityValue) AsMysqlBinary() []byte { + return v.data +} + // AsMysqlText returns value encoded in mysql text format // For identity value this means returning value as it is func (v *IdentityValue) AsMysqlText() []byte { diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 3e9788109..06a341aed 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -3,7 +3,6 @@ package mysql import ( "bytes" "context" - "encoding/base64" "encoding/binary" "errors" "strconv" @@ -73,7 +72,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, base.PutLengthEncodedString(data), nil } - dataTypeEncoded, isEncoded, err := p.encodeBinaryWithDataType(ctx, data, setting) + dataTypeEncoded, isEncoded, err := p.encodeBinaryWithDataType(ctx, data, setting, logger) if err != nil && err != ErrConvertToDataType { return nil, nil, err } @@ -157,24 +156,26 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, base.PutLengthEncodedString(data), nil } -func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting) ([]byte, bool, error) { +func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) ([]byte, bool, error) { switch setting.GetEncryptedDataType() { case common.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - if newValue := setting.GetDefaultDataValue(); newValue != nil { - binValue, err := base64.StdEncoding.DecodeString(*newValue) - if err != nil { - return data, false, err - } - return base.PutLengthEncodedString(binValue), true, nil + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlBinary(), true, nil } return data, false, ErrConvertToDataType } return data, false, nil case common.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { - if value := setting.GetDefaultDataValue(); value != nil { - return base.PutLengthEncodedString([]byte(*value)), true, nil + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlBinary(), true, nil } return data, false, ErrConvertToDataType } @@ -183,14 +184,13 @@ func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, d encoded := make([]byte, 4) intValue, err := strconv.ParseInt(utils.BytesToString(data), 10, 32) if err != nil { - if newVal := setting.GetDefaultDataValue(); newVal != nil { - intValue, err = strconv.ParseInt(*newVal, 10, 32) - if err != nil { - return data, false, ErrConvertToDataType - } - } else { - return data, false, ErrConvertToDataType + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlBinary(), true, nil } + return data, false, ErrConvertToDataType } err = binary.Write(bytes.NewBuffer(encoded[:0]), binary.LittleEndian, int32(intValue)) return encoded, true, err @@ -199,14 +199,13 @@ func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, d encoded := make([]byte, 8) intValue, err := strconv.ParseInt(utils.BytesToString(data), 10, 64) if err != nil { - if newVal := setting.GetDefaultDataValue(); newVal != nil { - intValue, err = strconv.ParseInt(*newVal, 10, 64) - if err != nil { - return data, false, ErrConvertToDataType - } - } else { - return data, false, ErrConvertToDataType + value, err := base.EncodeOnFail(setting, logger) + if err != nil { + return data, false, err + } else if value != nil { + return value.AsMysqlBinary(), true, nil } + return data, false, ErrConvertToDataType } err = binary.Write(bytes.NewBuffer(encoded[:0]), binary.LittleEndian, intValue) return encoded, true, err From 89bb2e6dfda1a83a445f578b0cb3c77af5093984 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 19:48:04 +0300 Subject: [PATCH 05/19] Merge text and binary encoding into one function So it looks like in the postgres version. It also simplifies and reduces the code. --- decryptor/mysql/data_encoder.go | 113 +++++++------------------------- 1 file changed, 25 insertions(+), 88 deletions(-) diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 06a341aed..3c11ba22a 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -72,14 +72,14 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, base.PutLengthEncodedString(data), nil } - dataTypeEncoded, isEncoded, err := p.encodeBinaryWithDataType(ctx, data, setting, logger) + encodingValue, err := p.encodeValueWithDataType(ctx, data, setting, logger) if err != nil && err != ErrConvertToDataType { return nil, nil, err } // in case of successful encoding with defined data type return encoded data - if isEncoded { - return ctx, dataTypeEncoded, nil + if encodingValue != nil { + return ctx, encodingValue.AsMysqlBinary(), nil } var columnType = columnInfo.DataBinaryType() @@ -156,107 +156,44 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, base.PutLengthEncodedString(data), nil } -func (p *BaseMySQLDataProcessor) encodeBinaryWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) ([]byte, bool, error) { - switch setting.GetEncryptedDataType() { - case common.EncryptedType_Bytes: +func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) (base.EncodingValue, error) { + dataType := setting.GetEncryptedDataType() + switch dataType { + case common.EncryptedType_String, common.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { value, err := base.EncodeOnFail(setting, logger) if err != nil { - return data, false, err + return nil, err } else if value != nil { - return value.AsMysqlBinary(), true, nil + return value, nil } - return data, false, ErrConvertToDataType + return nil, ErrConvertToDataType } - return data, false, nil - case common.EncryptedType_String: - if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) - if err != nil { - return data, false, err - } else if value != nil { - return value.AsMysqlBinary(), true, nil - } - return data, false, ErrConvertToDataType - } - return data, false, nil - case common.EncryptedType_Int32: - encoded := make([]byte, 4) - intValue, err := strconv.ParseInt(utils.BytesToString(data), 10, 32) - if err != nil { - value, err := base.EncodeOnFail(setting, logger) - if err != nil { - return data, false, err - } else if value != nil { - return value.AsMysqlBinary(), true, nil - } - return data, false, ErrConvertToDataType - } - err = binary.Write(bytes.NewBuffer(encoded[:0]), binary.LittleEndian, int32(intValue)) - return encoded, true, err - - case common.EncryptedType_Int64: - encoded := make([]byte, 8) - intValue, err := strconv.ParseInt(utils.BytesToString(data), 10, 64) - if err != nil { - value, err := base.EncodeOnFail(setting, logger) - if err != nil { - return data, false, err - } else if value != nil { - return value.AsMysqlBinary(), true, nil - } - return data, false, ErrConvertToDataType - } - err = binary.Write(bytes.NewBuffer(encoded[:0]), binary.LittleEndian, intValue) - return encoded, true, err - } - - return data, false, nil -} - -func (p *BaseMySQLDataProcessor) encodeTextWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) ([]byte, bool, error) { - switch setting.GetEncryptedDataType() { - case common.EncryptedType_String: - if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) - if err != nil { - return data, false, err - } else if value != nil { - return value.AsMysqlText(), true, nil - } - return data, false, ErrConvertToDataType - } - return data, false, nil - case common.EncryptedType_Bytes: - if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) - if err != nil { - return data, false, err - } else if value != nil { - return value.AsMysqlText(), true, nil - } - return data, false, ErrConvertToDataType - } - return data, false, nil + return nil, nil case common.EncryptedType_Int32, common.EncryptedType_Int64: - _, err := strconv.ParseInt(utils.BytesToString(data), 10, 64) + strValue := utils.BytesToString(data) + intValue, err := strconv.ParseInt(strValue, 10, 64) // if it's valid string literal and decrypted, return as is if err == nil { - return base.PutLengthEncodedString(data), true, nil + size := 4 + if dataType == common.EncryptedType_Int64 { + size = 8 + } + return base.NewIntValue(size, intValue, strValue), nil } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { value, err := base.EncodeOnFail(setting, logger) if err != nil { - return data, false, err + return nil, err } else if value != nil { - return value.AsMysqlText(), true, nil + return value, nil } - return data, false, ErrConvertToDataType + return nil, ErrConvertToDataType } } - return data, false, nil + return nil, nil } func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, []byte, error) { @@ -267,14 +204,14 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se return ctx, base.PutLengthEncodedString(data), nil } - dataTypeEncoded, isEncoded, err := p.encodeTextWithDataType(ctx, data, setting, logger) + encodingValue, err := p.encodeValueWithDataType(ctx, data, setting, logger) if err != nil && err != ErrConvertToDataType { return nil, nil, err } // in case of successful encoding with defined data type return encoded data - if isEncoded { - return ctx, dataTypeEncoded, nil + if encodingValue != nil { + return ctx, encodingValue.AsMysqlText(), nil } // in case of error on converting to defined type we should roll back field type and encode it as it was originally From 0faff6b6ec01b2bdd1c372144324ded37b4069fc Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 21:43:49 +0300 Subject: [PATCH 06/19] Add tests for successful, default and error paths --- decryptor/base/encoder.go | 4 +- decryptor/mysql/data_encoder_test.go | 280 +++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 decryptor/mysql/data_encoder_test.go diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index 44e192403..3276c61e8 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -166,13 +166,13 @@ func (v *IdentityValue) AsPostgresText() []byte { // AsMysqlBinary returns value encoded in mysql binary format // For identity value this means returning value as it is func (v *IdentityValue) AsMysqlBinary() []byte { - return v.data + return PutLengthEncodedString(v.data) } // AsMysqlText returns value encoded in mysql text format // For identity value this means returning value as it is func (v *IdentityValue) AsMysqlText() []byte { - return v.data + return PutLengthEncodedString(v.data) } // EncodeDefault returns wrapped default value from settings ready for encoding diff --git a/decryptor/mysql/data_encoder_test.go b/decryptor/mysql/data_encoder_test.go new file mode 100644 index 000000000..59fdd840e --- /dev/null +++ b/decryptor/mysql/data_encoder_test.go @@ -0,0 +1,280 @@ +package mysql + +import ( + "bytes" + "context" + "errors" + "fmt" + "testing" + + "github.com/cossacklabs/acra/decryptor/base" + "github.com/cossacklabs/acra/encryptor/config" + "github.com/cossacklabs/acra/encryptor/config/common" + "github.com/sirupsen/logrus" +) + +const binaryFormat = true +const textFormat = false + +func TestSuccessfulTextEncoding(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + expected []byte + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String, []byte("\x06string")}, + {[]byte("bytes"), common.EncryptedType_Bytes, []byte("\x05bytes")}, + {[]byte("3200"), common.EncryptedType_Int32, []byte("\x043200")}, + {[]byte("64000000"), common.EncryptedType_Int64, []byte("\b64000000")}, + } + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", textFormat, -1, 0, 0) + // mark context as decrypted + ctx := base.MarkDecryptedContext(context.Background()) + + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + setting := &config.BasicColumnEncryptionSetting{ + DataType: dataType, + } + logger := logrus.NewEntry(logrus.New()) + + _, encoded, err := encoder.encodeText(ctx, testcase.input, setting, info, logger) + + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(encoded, testcase.expected) { + t.Fatalf("incorrect encoding: %q but expected %q\n", encoded, testcase.expected) + } + } +} + +func TestSuccessfulBinaryEncoding(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + expected []byte + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String, []byte("\x06string")}, + {[]byte("bytes"), common.EncryptedType_Bytes, []byte("\x05bytes")}, + {[]byte("3200"), common.EncryptedType_Int32, []byte("\x80\f\x00\x00")}, + {[]byte("64000000"), common.EncryptedType_Int64, []byte("\x00\x90\xd0\x03\x00\x00\x00\x00")}, + } + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", binaryFormat, -1, 0, 0) + // mark context as decrypted + ctx := base.MarkDecryptedContext(context.Background()) + + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + + setting := &config.BasicColumnEncryptionSetting{ + DataType: dataType, + } + logger := logrus.NewEntry(logrus.New()) + + _, encoded, err := encoder.encodeBinary(ctx, testcase.input, setting, info, logger) + + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(encoded, testcase.expected) { + t.Fatalf("incorrect encoding: %q but expected %q\n", encoded, testcase.expected) + } + } +} + +func TestFailingTextEncodingWithDefault(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + defaultValue string + expected []byte + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String, "default_string", []byte("\x0edefault_string")}, + {[]byte("bytes"), common.EncryptedType_Bytes, "ZGVmYXVsdF9ieXRlcw==", []byte("\rdefault_bytes")}, + {[]byte("invalid_int32"), common.EncryptedType_Int32, "25519", []byte("\x0525519")}, + {[]byte("invalid_int64"), common.EncryptedType_Int64, "448", []byte("\x03448")}, + } + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", textFormat, -1, 0, 0) + ctx := context.Background() + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + setting := &config.BasicColumnEncryptionSetting{ + DataType: dataType, + ResponseOnFail: common.ResponseOnFailDefault, + DefaultDataValue: &testcase.defaultValue, + } + logger := logrus.NewEntry(logrus.New()) + + _, encoded, err := encoder.encodeText(ctx, testcase.input, setting, info, logger) + + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(encoded, testcase.expected) { + t.Fatalf("incorrect encoding: %q but expected %q\n", encoded, testcase.expected) + } + } +} + +func TestFailingBinaryEncodingWithDefault(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + defaultValue string + expected []byte + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String, "default_string", []byte("\x0edefault_string")}, + {[]byte("bytes"), common.EncryptedType_Bytes, "ZGVmYXVsdF9ieXRlcw==", []byte("\rdefault_bytes")}, + {[]byte("invalid_int32"), common.EncryptedType_Int32, "25519", []byte("\xafc\x00\x00")}, + {[]byte("invalid_int64"), common.EncryptedType_Int64, "448", []byte("\xc0\x01\x00\x00\x00\x00\x00\x00")}, + } + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", binaryFormat, -1, 0, 0) + ctx := context.Background() + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + setting := &config.BasicColumnEncryptionSetting{ + DataType: dataType, + ResponseOnFail: common.ResponseOnFailDefault, + DefaultDataValue: &testcase.defaultValue, + } + logger := logrus.NewEntry(logrus.New()) + + _, encoded, err := encoder.encodeBinary(ctx, testcase.input, setting, info, logger) + + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(encoded, testcase.expected) { + t.Fatalf("incorrect encoding: %q but expected %q\n", encoded, testcase.expected) + } + } +} + +func TestFailingTextEncodingWithEncodingError(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String}, + {[]byte("bytes"), common.EncryptedType_Bytes}, + {[]byte("invalid_int32"), common.EncryptedType_Int32}, + {[]byte("invalid_int64"), common.EncryptedType_Int64}, + } + + column := "cossack_column" + expectedError := base.NewEncodingError(column) + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", textFormat, -1, 0, 0) + ctx := context.Background() + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + setting := &config.BasicColumnEncryptionSetting{ + Name: column, + DataType: dataType, + ResponseOnFail: common.ResponseOnFailError, + } + logger := logrus.NewEntry(logrus.New()) + + _, _, err = encoder.encodeText(ctx, testcase.input, setting, info, logger) + + if !errors.Is(err, expectedError) { + t.Fatalf("expected error %q, but found %q", expectedError, err) + } + } +} + +func TestFailingBinaryEncodingWithEncodingError(t *testing.T) { + type testcase struct { + input []byte + dataType common.EncryptedType + } + + testcases := []testcase{ + {[]byte("string"), common.EncryptedType_String}, + {[]byte("bytes"), common.EncryptedType_Bytes}, + {[]byte("invalid_int32"), common.EncryptedType_Int32}, + {[]byte("invalid_int64"), common.EncryptedType_Int64}, + } + + column := "cossack_column" + expectedError := base.NewEncodingError(column) + + for _, testcase := range testcases { + fmt.Printf("-- case %q\n", testcase.input) + + encoder := NewDataEncoderProcessor() + + info := base.NewColumnInfo(0, "", binaryFormat, -1, 0, 0) + ctx := context.Background() + dataType, err := testcase.dataType.ToConfigString() + if err != nil { + t.Fatal(err) + } + setting := &config.BasicColumnEncryptionSetting{ + Name: column, + DataType: dataType, + ResponseOnFail: common.ResponseOnFailError, + } + logger := logrus.NewEntry(logrus.New()) + + _, _, err = encoder.encodeBinary(ctx, testcase.input, setting, info, logger) + + if !errors.Is(err, expectedError) { + t.Fatalf("expected error %q, but found %q", expectedError, err) + } + } +} From 240253b04a9dd432570a23828a867e5e5349f299 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 21:47:17 +0300 Subject: [PATCH 07/19] Rename IdentityValue into StringValue It makes sense, because: a) It is used mostly in string context. b) It is an identity operation only in postgres case. In mysql it encodes string as length encoded one. --- decryptor/base/encoder.go | 28 ++++++++++++++-------------- decryptor/postgresql/data_encoder.go | 10 +++++----- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index 3276c61e8..59958dfe1 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -141,37 +141,37 @@ func (v *IntValue) AsMysqlText() []byte { return PutLengthEncodedString([]byte(v.strValue)) } -// IdentityValue is an encodingValue that just returns data as is -type IdentityValue struct { +// StringValue is an EncodingValue that encodes data into string format +type StringValue struct { data []byte } -// NewIdentityValue returns EncodingValue as identity value -func NewIdentityValue(data []byte) EncodingValue { - return &IdentityValue{data} +// NewStringValue returns string EncodingValue +func NewStringValue(data []byte) EncodingValue { + return &StringValue{data} } // AsPostgresBinary returns value encoded in postgres binary format -// For identity value this means returning value as it is -func (v *IdentityValue) AsPostgresBinary() []byte { +// In other words, it returns data as it is +func (v *StringValue) AsPostgresBinary() []byte { return v.data } // AsPostgresText returns value encoded in postgres text format -// For identity value this means returning value as it is -func (v *IdentityValue) AsPostgresText() []byte { +// In other words, it returns data as it is +func (v *StringValue) AsPostgresText() []byte { return v.data } // AsMysqlBinary returns value encoded in mysql binary format -// For identity value this means returning value as it is -func (v *IdentityValue) AsMysqlBinary() []byte { +// In other words, it encodes data into length encoded string +func (v *StringValue) AsMysqlBinary() []byte { return PutLengthEncodedString(v.data) } // AsMysqlText returns value encoded in mysql text format -// For identity value this means returning value as it is -func (v *IdentityValue) AsMysqlText() []byte { +// In other words, it encodes data into length encoded string +func (v *StringValue) AsMysqlText() []byte { return PutLengthEncodedString(v.data) } @@ -189,7 +189,7 @@ func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) switch dataType { case common.EncryptedType_String: - return &IdentityValue{[]byte(*strValue)} + return NewStringValue([]byte(*strValue)) case common.EncryptedType_Bytes: binValue, err := base64.StdEncoding.DecodeString(*strValue) if err != nil { diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index 0ea6bf235..aa59eeb1c 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -30,7 +30,7 @@ func (p *PgSQLDataEncoderProcessor) ID() string { func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, base.EncodingValue, error) { logger = logger.WithField("column", setting.ColumnName()).WithField("decrypted", base.IsDecryptedFromContext(ctx)) if len(data) == 0 { - return ctx, base.NewIdentityValue(data), nil + return ctx, base.NewStringValue(data), nil } switch setting.GetEncryptedDataType() { case common2.EncryptedType_String: @@ -43,7 +43,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } // decrypted values return as is, without any encoding - return ctx, base.NewIdentityValue(data), nil + return ctx, base.NewStringValue(data), nil case common2.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { value, err := base.EncodeOnFail(setting, logger) @@ -77,7 +77,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } logger.Warningln("Can't decode int value and no default value") - return ctx, base.NewIdentityValue(data), nil + return ctx, base.NewStringValue(data), nil } // here we process AcraStruct/AcraBlock decryption without any encryptor config that defines data_type/token_type // values. If it was decrypted then we return it as valid bytea value @@ -88,9 +88,9 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by // as it come to us. encodedValue, ok := getEncodedValueFromContext(ctx) if ok { - return ctx, base.NewIdentityValue(encodedValue), nil + return ctx, base.NewStringValue(encodedValue), nil } - return ctx, base.NewIdentityValue(data), nil + return ctx, base.NewStringValue(data), nil } // OnColumn encode binary value to text and back. Should be before and after tokenizer processor From 28b8b6a27f429c6914686bf7984e1b1b893d6c26 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Tue, 3 May 2022 21:50:46 +0300 Subject: [PATCH 08/19] Rename ByteSequenceValue into BytesValue To be more consistent. --- decryptor/base/encoder.go | 30 +++++++++++++--------------- decryptor/postgresql/data_encoder.go | 4 ++-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index 59958dfe1..499ba29ca 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -54,40 +54,38 @@ type EncodingValue interface { AsMysqlText() []byte } -// ByteSequenceValue is an abstraction over all byte-sequence values -- strings -// and []byte (because they are encoded in the same way) -type ByteSequenceValue struct { +// BytesValue is an EncodingValue that represents byte array +type BytesValue struct { seq []byte } -// NewByteSequenceValue returns EncodingValue from a byte-string-like data -func NewByteSequenceValue(seq []byte) EncodingValue { - return &ByteSequenceValue{seq} +// NewBytesValue returns EncodingValue from a byte array +func NewBytesValue(seq []byte) EncodingValue { + return &BytesValue{seq} } // AsPostgresBinary returns value encoded in postgres binary format -// For a byte sequence value (string or []byte) this is an identity operation -func (v *ByteSequenceValue) AsPostgresBinary() []byte { +// For a byte sequence value this is an identity operation +func (v *BytesValue) AsPostgresBinary() []byte { return v.seq } // AsPostgresText returns value encoded in postgres text format -// For a byte sequence value (string or []byte) this is a hex encoded string -func (v *ByteSequenceValue) AsPostgresText() []byte { +// For a byte sequence value this is a hex encoded string +func (v *BytesValue) AsPostgresText() []byte { // all bytes should be encoded as valid bytea value return utils.PgEncodeToHex(v.seq) } // AsMysqlBinary returns value encoded in mysql binary format -// For a byte sequence value (string or []byte) this is the same as text -// encoding -func (v *ByteSequenceValue) AsMysqlBinary() []byte { +// For a byte sequence value this is the same as text encoding +func (v *BytesValue) AsMysqlBinary() []byte { return v.AsMysqlText() } // AsMysqlText returns value encoded in mysql text format -// For a byte sequence value (string or []byte) this is a length encoded string -func (v *ByteSequenceValue) AsMysqlText() []byte { +// For a byte sequence value this is a length encoded string +func (v *BytesValue) AsMysqlText() []byte { return PutLengthEncodedString(v.seq) } @@ -196,7 +194,7 @@ func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) logger.WithError(err).Errorln("Can't decode base64 default value") return nil } - return &ByteSequenceValue{seq: binValue} + return &BytesValue{seq: binValue} case common.EncryptedType_Int32, common.EncryptedType_Int64: size := 8 if dataType == common.EncryptedType_Int32 { diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index aa59eeb1c..9f487f91a 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -53,7 +53,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by return ctx, value, nil } } - return ctx, base.NewByteSequenceValue(data), nil + return ctx, base.NewBytesValue(data), nil case common2.EncryptedType_Int32, common2.EncryptedType_Int64: size := 8 if setting.GetEncryptedDataType() == common2.EncryptedType_Int32 { @@ -82,7 +82,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by // here we process AcraStruct/AcraBlock decryption without any encryptor config that defines data_type/token_type // values. If it was decrypted then we return it as valid bytea value if base.IsDecryptedFromContext(ctx) { - return ctx, base.NewByteSequenceValue(data), nil + return ctx, base.NewBytesValue(data), nil } // If it wasn't decrypted (due to inappropriate keys or not AcraStructs as payload) then we return it in same way // as it come to us. From db9583bc686789a0333c78389fcc9c97f55a3071 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Wed, 4 May 2022 12:19:42 +0300 Subject: [PATCH 09/19] Add integration tests for `ciphertext` option Just copy-paste from `WithoutDefaults` --- tests/test.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/tests/test.py b/tests/test.py index 44c5a1851..d107b3acb 100644 --- a/tests/test.py +++ b/tests/test.py @@ -9420,6 +9420,158 @@ def testClientIDRead(self): self.assertIsInstance(value, bytearray, column) self.assertNotEqual(data[column], value, column) +class TestMySQLTextTypeAwareDecryptionWithСiphertext(BaseBinaryMySQLTestCase, BaseTransparentEncryption): + # test table used for queries and data mapping into python types + test_table = sa.Table( + # use new object of metadata to avoid name conflict + 'test_type_aware_decryption_with_ciphertext', sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('value_str', sa.Text), + sa.Column('value_bytes', sa.LargeBinary), + sa.Column('value_int32', sa.Integer), + sa.Column('value_int64', sa.BigInteger), + sa.Column('value_null_str', sa.Text, nullable=True, default=None), + sa.Column('value_null_int32', sa.Integer, nullable=True, default=None), + sa.Column('value_empty_str', sa.Text, nullable=False, default=''), + extend_existing=True + ) + # schema table used to generate table in the database with binary column types + schema_table = sa.Table( + + 'test_type_aware_decryption_with_ciphertext', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('value_str', sa.LargeBinary), + sa.Column('value_bytes', sa.LargeBinary), + sa.Column('value_int32', sa.LargeBinary), + sa.Column('value_int64', sa.LargeBinary), + sa.Column('value_null_str', sa.LargeBinary, nullable=True, default=None), + sa.Column('value_null_int32', sa.LargeBinary, nullable=True, default=None), + sa.Column('value_empty_str', sa.LargeBinary, nullable=False, default=b''), + extend_existing=True + ) + ENCRYPTOR_CONFIG = get_encryptor_config('tests/encryptor_configs/transparent_type_aware_decryption.yaml') + + def setUp(self): + super().setUp() + + # switch off raw mode to be able to convert result rows to python types + def raw_executor_with_ssl(ssl_key, ssl_cert): + args = ConnectionArgs( + host=get_db_host(), port=self.ACRASERVER_PORT, dbname=DB_NAME, + user=DB_USER, password=DB_USER_PASSWORD, + ssl_ca=TEST_TLS_CA, + ssl_key=ssl_key, + ssl_cert=ssl_cert, + raw=False, + ) + return MysqlExecutor(args) + + self.executor1 = raw_executor_with_ssl(TEST_TLS_CLIENT_KEY, TEST_TLS_CLIENT_CERT) + self.executor2 = raw_executor_with_ssl(TEST_TLS_CLIENT_2_KEY, TEST_TLS_CLIENT_2_CERT) + + def checkSkip(self): + if not (TEST_MYSQL and TEST_WITH_TLS): + self.skipTest("Test only for MySQL with TLS") + + def testClientIDRead(self): + """test decrypting with correct clientID and not decrypting with + incorrect clientID or using direct connection to db + All result data should be valid for application. Not decrypted data should be returned as is and DB driver + should cause error + + MySQL decoder should roll back FieldType as well. + """ + data = { + 'id': get_random_id(), + 'value_str': random_str(), + 'value_bytes': random_bytes(), + 'value_int32': random_int32(), + 'value_int64': random_int64(), + 'value_null_str': None, + 'value_null_int32': None, + 'value_empty_str': '' + } + self.schema_table.create(bind=self.engine_raw, checkfirst=True) + self.engine1.execute(self.test_table.insert(), data) + columns = ('value_str', 'value_bytes', 'value_int32', 'value_int64', 'value_empty_str') + null_columns = ('value_null_str', 'value_null_int32') + + compile_kwargs = {"literal_binds": True} + query = sa.select([self.test_table]).where(self.test_table.c.id == data['id']) + query = str(query.compile(compile_kwargs=compile_kwargs)) + + row = self.executor1.execute(query)[0] + for column in columns: + self.assertEqual(data[column], row[column]) + self.assertIsInstance(row[column], type(data[column])) + + # mysql.connector represent null value as empty string + for column in null_columns: + self.assertEqual(row[column], '') + + # field types should be rollbacked in case of invalid encoding + row = self.executor2.execute(query)[0] + + # direct connection should receive binary data according to real scheme + result = self.engine_raw.execute( + sa.select([self.test_table]) + .where(self.test_table.c.id == data['id'])) + row = result.fetchone() + for column in columns: + if 'null' in column or 'empty' in column: + # asyncpg decodes None values as empty str/bytes value + self.assertFalse(row[column]) + continue + value = utils.memoryview_to_bytes(row[column]) + self.assertIsInstance(value, bytes, column) + self.assertNotEqual(data[column], value, column) + +class TestMySQLBinaryTypeAwareDecryptionWithСiphertext(TestMySQLTextTypeAwareDecryptionWithСiphertext): + def checkSkip(self): + if not (TEST_MYSQL and TEST_WITH_TLS): + self.skipTest("Test only for MySQL with TLS") + + def testClientIDRead(self): + """test decrypting with correct clientID and not decrypting with + incorrect clientID or using direct connection to db + All result data should be valid for application. Not decrypted data should be returned as is and DB driver + should cause error + + MySQL decoder should roll back FieldType as well. + """ + data = { + 'id': get_random_id(), + 'value_str': random_str(), + 'value_bytes': random_bytes(), + 'value_int32': random_int32(), + 'value_int64': random_int64(), + 'value_null_str': None, + 'value_null_int32': None, + 'value_empty_str': '' + } + self.schema_table.create(bind=self.engine_raw, checkfirst=True) + ###### + columns = ('value_str', 'value_bytes', 'value_int32', 'value_int64', 'value_null_str', 'value_null_int32', + 'value_empty_str') + query, args = self.compileQuery(self.test_table.insert(), data) + self.executor1.execute_prepared_statement_no_result(query, args) + + query, args = self.compileQuery( + sa.select([self.test_table]) + .where(self.test_table.c.id == sa.bindparam('id')), {'id': data['id']}) + + # just make sure that it is not failing meant that decoder rollback field types + row = self.executor2.execute_prepared_statement(query, args)[0] + + for column in columns: + if 'null' in column or 'empty' in column: + # asyncpg decodes None values as empty str/bytes value + self.assertFalse(row[column]) + continue + value = utils.memoryview_to_bytes(row[column]) + self.assertIsInstance(value, bytearray, column) + self.assertNotEqual(data[column], value, column) + class TestPostgresqlConnectWithTLSPrefer(BaseTestCase): def checkSkip(self): if TEST_WITH_TLS or not TEST_POSTGRESQL: From d455624ce35ea72cc76535eab3b8814c81d47519 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Wed, 4 May 2022 19:45:39 +0300 Subject: [PATCH 10/19] Send interrupted error with custom msg to client In case of EncodingError, forward the it to the client as `QueryInterruptedError` but with custom message. --- decryptor/mysql/error.go | 12 ++++++++---- decryptor/mysql/response_proxy.go | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/decryptor/mysql/error.go b/decryptor/mysql/error.go index 4b35fdf39..c46708011 100644 --- a/decryptor/mysql/error.go +++ b/decryptor/mysql/error.go @@ -30,18 +30,22 @@ const ( ErQueryInterruptedState = "70100" ) -func newQueryInterruptedError() *SQLError { +// QueryExecutionWasInterrupted is a default message of the mysql's Query +// interrupted error +const QueryExecutionWasInterrupted = "Query execution was interrupted" + +func newQueryInterruptedError(msg string) *SQLError { e := new(SQLError) e.Code = ErQueryInterruptedCode e.State = ErQueryInterruptedState - e.Message = "Query execution was interrupted" + e.Message = msg return e } // NewQueryInterruptedError return packed QueryInterrupted error // https://dev.mysql.com/doc/internals/en/packet-ERR_Packet.html -func NewQueryInterruptedError(isProtocol41 bool) []byte { - mysqlError := newQueryInterruptedError() +func NewQueryInterruptedError(isProtocol41 bool, msg string) []byte { + mysqlError := newQueryInterruptedError(msg) var data []byte if isProtocol41 { // 1 byte ErrPacket flag + 2 bytes of error code = 3 diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go index 3e680ddac..5141a70d2 100644 --- a/decryptor/mysql/response_proxy.go +++ b/decryptor/mysql/response_proxy.go @@ -296,7 +296,7 @@ func (handler *Handler) ProxyClientConnection(ctx context.Context, errCh chan<- "for connections AcraServer->Database and CA certificate which will be used to verify certificate " + "from database") handler.logger.Debugln("Send error to db") - errPacket := NewQueryInterruptedError(handler.clientProtocol41) + errPacket := NewQueryInterruptedError(handler.clientProtocol41, QueryExecutionWasInterrupted) packet.SetData(errPacket) if _, err := handler.clientConnection.Write(packet.Dump()); err != nil { handler.logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseConnectorCantWriteToClient). @@ -395,7 +395,7 @@ func (handler *Handler) ProxyClientConnection(ctx context.Context, errCh chan<- if err := handler.acracensor.HandleQuery(query); err != nil { censorSpan.End() clientLog.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCensorQueryIsNotAllowed).Errorln("Error on AcraCensor check") - errPacket := NewQueryInterruptedError(handler.clientProtocol41) + errPacket := NewQueryInterruptedError(handler.clientProtocol41, QueryExecutionWasInterrupted) packet.SetData(errPacket) if _, err := handler.clientConnection.Write(packet.Dump()); err != nil { handler.logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseConnectorCantWriteToClient). @@ -857,6 +857,21 @@ func (handler *Handler) ProxyDatabaseConnection(ctx context.Context, errCh chan< accessContext.SetZoneID(nil) responseHandler = handler.getResponseHandler() err = responseHandler(ctx, packet, handler.dbConnection, handler.clientConnection) + + // EncodingError is the only one that we should forward to the client + if encodingError, ok := err.(*base.EncodingError); ok { + handler.logger.WithError(encodingError).Debugln("Sending encoding error to the client") + errPacket := NewQueryInterruptedError(handler.clientProtocol41, encodingError.Error()) + packet.SetData(errPacket) + if _, err := handler.clientConnection.Write(packet.Dump()); err != nil { + handler.logger.WithError(err). + WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseConnectorCantWriteToClient). + Debugln("Can't write response with error to client") + errCh <- base.NewDBProxyError(err) + } + return + } + if err != nil { handler.resetQueryHandler() errCh <- base.NewDBProxyError(err) From 96325a94fc97801fd99e8cab1d8cfbbaaf5542b6 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Wed, 4 May 2022 20:45:35 +0300 Subject: [PATCH 11/19] Add integration tests for response_on_fail: error --- tests/test.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/tests/test.py b/tests/test.py index d107b3acb..8da858641 100644 --- a/tests/test.py +++ b/tests/test.py @@ -235,6 +235,8 @@ 'sslmode': 'require', }) +# THe code for mysql "Query execution was interrupted" error +MYSQL_ERR_QUERY_INTERRUPTED_CODE = 1317 def get_tls_connection_args(client_key, client_cert, for_mysql=TEST_MYSQL): if for_mysql: @@ -9572,6 +9574,145 @@ def testClientIDRead(self): self.assertIsInstance(value, bytearray, column) self.assertNotEqual(data[column], value, column) + +class TestMySQLTextTypeAwareDecryptionWithError(BaseBinaryMySQLTestCase, BaseTransparentEncryption): + # test table used for queries and data mapping into python types + test_table = sa.Table( + # use new object of metadata to avoid name conflict + 'test_type_aware_decryption_with_error', sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('value_str', sa.Text), + sa.Column('value_bytes', sa.LargeBinary), + sa.Column('value_int32', sa.Integer), + sa.Column('value_int64', sa.BigInteger), + sa.Column('value_null_str', sa.Text, nullable=True, default=None), + sa.Column('value_null_int32', sa.Integer, nullable=True, default=None), + sa.Column('value_empty_str', sa.Text, nullable=False, default=''), + extend_existing=True + ) + # schema table used to generate table in the database with binary column types + schema_table = sa.Table( + 'test_type_aware_decryption_with_error', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('value_str', sa.LargeBinary), + sa.Column('value_bytes', sa.LargeBinary), + sa.Column('value_int32', sa.LargeBinary), + sa.Column('value_int64', sa.LargeBinary), + sa.Column('value_null_str', sa.LargeBinary, nullable=True, default=None), + sa.Column('value_null_int32', sa.LargeBinary, nullable=True, default=None), + sa.Column('value_empty_str', sa.LargeBinary, nullable=False, default=b''), + extend_existing=True + ) + ENCRYPTOR_CONFIG = get_encryptor_config('tests/encryptor_configs/transparent_type_aware_decryption.yaml') + + def setUp(self): + super().setUp() + + # switch off raw mode to be able to convert result rows to python types + def raw_executor_with_ssl(ssl_key, ssl_cert): + args = ConnectionArgs( + host=get_db_host(), port=self.ACRASERVER_PORT, dbname=DB_NAME, + user=DB_USER, password=DB_USER_PASSWORD, + ssl_ca=TEST_TLS_CA, + ssl_key=ssl_key, + ssl_cert=ssl_cert, + raw=False, + ) + return MysqlExecutor(args) + + self.executor1 = raw_executor_with_ssl(TEST_TLS_CLIENT_KEY, TEST_TLS_CLIENT_CERT) + self.executor2 = raw_executor_with_ssl(TEST_TLS_CLIENT_2_KEY, TEST_TLS_CLIENT_2_CERT) + + def checkSkip(self): + if not (TEST_MYSQL and TEST_WITH_TLS): + self.skipTest("Test only for MySQL with TLS") + + def testClientIDRead(self): + """test decrypting with correct clientID and not decrypting with + incorrect clientID or using direct connection to db + All result data should be valid for application. Not decrypted data should be returned as is and DB driver + should cause error + + MySQL decoder should roll back FieldType as well. + """ + data = { + 'id': get_random_id(), + 'value_str': random_str(), + 'value_bytes': random_bytes(), + 'value_int32': random_int32(), + 'value_int64': random_int64(), + 'value_null_str': None, + 'value_null_int32': None, + 'value_empty_str': '' + } + self.schema_table.create(bind=self.engine_raw, checkfirst=True) + self.engine1.execute(self.test_table.insert(), data) + columns = ('value_str', 'value_bytes', 'value_int32', 'value_int64', 'value_empty_str') + null_columns = ('value_null_str', 'value_null_int32') + + compile_kwargs = {"literal_binds": True} + query = sa.select([self.test_table]).where(self.test_table.c.id == data['id']) + query = str(query.compile(compile_kwargs=compile_kwargs)) + + row = self.executor1.execute(query)[0] + for column in columns: + self.assertEqual(data[column], row[column]) + self.assertIsInstance(row[column], type(data[column])) + + # mysql.connector represent null value as empty string + for column in null_columns: + self.assertEqual(row[column], '') + + # we expect an exception because of decryption error + with self.assertRaises(mysql.connector.errors.DatabaseError) as ex: + self.executor2.execute(query)[0] + + self.assertEqual('encoding error in column "value_str"', ex.exception.msg) + self.assertEqual(ex.exception.errno, MYSQL_ERR_QUERY_INTERRUPTED_CODE) + +class TestMySQLBinaryTypeAwareDecryptionWithError(TestMySQLTextTypeAwareDecryptionWithError): + def checkSkip(self): + if not (TEST_MYSQL and TEST_WITH_TLS): + self.skipTest("Test only for MySQL with TLS") + + def testClientIDRead(self): + """test decrypting with correct clientID and not decrypting with + incorrect clientID or using direct connection to db + All result data should be valid for application. Not decrypted data should be returned as is and DB driver + should cause error + + MySQL decoder should roll back FieldType as well. + """ + data = { + 'id': get_random_id(), + 'value_str': random_str(), + 'value_bytes': random_bytes(), + 'value_int32': random_int32(), + 'value_int64': random_int64(), + 'value_null_str': None, + 'value_null_int32': None, + 'value_empty_str': '' + } + self.schema_table.create(bind=self.engine_raw, checkfirst=True) + ###### + columns = ('value_str', 'value_bytes', 'value_int32', 'value_int64', 'value_null_str', 'value_null_int32', + 'value_empty_str') + query, args = self.compileQuery(self.test_table.insert(), data) + self.executor1.execute_prepared_statement_no_result(query, args) + + query, args = self.compileQuery( + sa.select([self.test_table]) + .where(self.test_table.c.id == sa.bindparam('id')), {'id': data['id']}) + + # we expect an exception because of decryption error + with self.assertRaises(mysql.connector.errors.DatabaseError) as ex: + self.executor2.execute_prepared_statement(query, args)[0] + + self.assertEqual('encoding error in column "value_str"', ex.exception.msg) + self.assertEqual(ex.exception.errno, MYSQL_ERR_QUERY_INTERRUPTED_CODE) + + + class TestPostgresqlConnectWithTLSPrefer(BaseTestCase): def checkSkip(self): if TEST_WITH_TLS or not TEST_POSTGRESQL: From a4ead1dcd8e2763bd116d1b7d7ef4ab904e6aab5 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Wed, 4 May 2022 20:39:51 +0300 Subject: [PATCH 12/19] Update changelog --- CHANGELOG_DEV.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG_DEV.md b/CHANGELOG_DEV.md index 1e8408926..5025f0a2f 100644 --- a/CHANGELOG_DEV.md +++ b/CHANGELOG_DEV.md @@ -1,3 +1,6 @@ +# 0.93.0 - 2022-05-04 +- Add mysql support for `response_on_fail` options. + # 0.93.0 - 2022-04-20 - Add `make install_dev_deps` for development dependencies installation. From 7daf4c88e5e63936a9209f1d71c74f3475d749b5 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Thu, 5 May 2022 15:07:15 +0300 Subject: [PATCH 13/19] Add ValueFactory and a temporary plug ValueFactory will be implemented by each backend to support their own encoding. --- decryptor/base/encoder.go | 58 +++++++++++++++++++++++----- decryptor/mysql/data_encoder.go | 4 +- decryptor/postgresql/data_encoder.go | 6 +-- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index 499ba29ca..9c36a3082 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -54,6 +54,19 @@ type EncodingValue interface { AsMysqlText() []byte } +// EncodingValueFactory represents a factory that produces ready for encoding +// value. +type EncodingValueFactory interface { + // NewStringValue creates a value that encodes as a str + NewStringValue(str []byte) EncodingValue + // NewBytesValue creates a value that encodes as bytes + NewBytesValue(bytes []byte) EncodingValue + // NewInt32Value creates a value that encodes as int32 + NewInt32Value(intVal int32, strVal []byte) EncodingValue + // NewInt64Value creates a value that encodes as int64 + NewInt64Value(intVal int64, strVal []byte) EncodingValue +} + // BytesValue is an EncodingValue that represents byte array type BytesValue struct { seq []byte @@ -176,7 +189,7 @@ func (v *StringValue) AsMysqlText() []byte { // EncodeDefault returns wrapped default value from settings ready for encoding // returns nil if something went wrong, which in many cases indicates that the // original value should be returned as it is -func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) EncodingValue { +func EncodeDefault(setting config.ColumnEncryptionSetting, valueFactory EncodingValueFactory, logger *logrus.Entry) EncodingValue { strValue := setting.GetDefaultDataValue() if strValue == nil { logger.Errorln("Default value is not specified") @@ -187,26 +200,29 @@ func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) switch dataType { case common.EncryptedType_String: - return NewStringValue([]byte(*strValue)) + return valueFactory.NewStringValue([]byte(*strValue)) case common.EncryptedType_Bytes: binValue, err := base64.StdEncoding.DecodeString(*strValue) if err != nil { logger.WithError(err).Errorln("Can't decode base64 default value") return nil } - return &BytesValue{seq: binValue} + return valueFactory.NewBytesValue(binValue) case common.EncryptedType_Int32, common.EncryptedType_Int64: - size := 8 + size := 64 if dataType == common.EncryptedType_Int32 { - size = 4 + size = 32 } - value, err := strconv.ParseInt(*strValue, 10, 64) + value, err := strconv.ParseInt(*strValue, 10, size) if err != nil { logger.WithError(err).Errorln("Can't parse default integer value") return nil } - return &IntValue{size: size, value: value, strValue: *strValue} + if dataType == common.EncryptedType_Int32 { + return valueFactory.NewInt32Value(int32(value), []byte(*strValue)) + } + return valueFactory.NewInt64Value(value, []byte(*strValue)) } return nil } @@ -214,14 +230,14 @@ func EncodeDefault(setting config.ColumnEncryptionSetting, logger *logrus.Entry) // EncodeOnFail returns either an error, which should be returned, or value, which // should be encoded, because there is some problem with original, or `nil` // which indicates that original value should be returned as is. -func EncodeOnFail(setting config.ColumnEncryptionSetting, logger *logrus.Entry) (EncodingValue, error) { +func EncodeOnFail(setting config.ColumnEncryptionSetting, valueFactory EncodingValueFactory, logger *logrus.Entry) (EncodingValue, error) { action := setting.GetResponseOnFail() switch action { case common.ResponseOnFailEmpty, common.ResponseOnFailCiphertext: return nil, nil case common.ResponseOnFailDefault: - return EncodeDefault(setting, logger), nil + return EncodeDefault(setting, valueFactory, logger), nil case common.ResponseOnFailError: return nil, NewEncodingError(setting.ColumnName()) @@ -229,3 +245,27 @@ func EncodeOnFail(setting config.ColumnEncryptionSetting, logger *logrus.Entry) return nil, fmt.Errorf("unknown action: %q", action) } + +// ValueFactoryPlug is a temporary factory which will be removed when +// each backend (postgres or mysql) implements its own one +type ValueFactoryPlug struct{} + +// NewStringValue creates a value that encodes as a str +func (plug *ValueFactoryPlug) NewStringValue(str []byte) EncodingValue { + return NewStringValue(str) +} + +// NewBytesValue creates a value that encodes as bytes +func (plug *ValueFactoryPlug) NewBytesValue(bytes []byte) EncodingValue { + return NewBytesValue(bytes) +} + +// NewInt32Value creates a value that encodes as int32 +func (plug *ValueFactoryPlug) NewInt32Value(intVal int32, strVal []byte) EncodingValue { + return NewIntValue(4, int64(intVal), string(strVal)) +} + +// NewInt64Value creates a value that encodes as int64 +func (plug *ValueFactoryPlug) NewInt64Value(intVal int64, strVal []byte) EncodingValue { + return NewIntValue(8, int64(intVal), string(strVal)) +} diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 3c11ba22a..5a3900d14 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -161,7 +161,7 @@ func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, da switch dataType { case common.EncryptedType_String, common.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) if err != nil { return nil, err } else if value != nil { @@ -183,7 +183,7 @@ func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, da } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) if err != nil { return nil, err } else if value != nil { diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index 9f487f91a..7762372ef 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -35,7 +35,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by switch setting.GetEncryptedDataType() { case common2.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -46,7 +46,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by return ctx, base.NewStringValue(data), nil case common2.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -69,7 +69,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, logger) + value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) if err != nil { return ctx, nil, err } else if value != nil { From 8b447d1401b0e5e6f03a5fa7897e13fa5abbc689 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Thu, 5 May 2022 15:18:21 +0300 Subject: [PATCH 14/19] Implement postgresValueFactory --- decryptor/postgresql/data_encoder.go | 127 ++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index 7762372ef..f2eeb7ecd 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -35,7 +35,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by switch setting.GetEncryptedDataType() { case common2.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) + value, err := base.EncodeOnFail(setting, &postgresValueFactory{}, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -46,7 +46,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by return ctx, base.NewStringValue(data), nil case common2.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) + value, err := base.EncodeOnFail(setting, &postgresValueFactory{}, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -69,7 +69,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) + value, err := base.EncodeOnFail(setting, &postgresValueFactory{}, logger) if err != nil { return ctx, nil, err } else if value != nil { @@ -231,3 +231,124 @@ func (p *PgSQLDataDecoderProcessor) OnColumn(ctx context.Context, data []byte) ( } return p.decodeText(ctx, data, columnSetting, columnInfo, logger) } + +// bytesValue is an EncodingValue that represents byte array +type bytesValue struct { + bytes []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For a byte sequence value this is an identity operation +func (v *bytesValue) AsPostgresBinary() []byte { + return v.bytes +} + +// AsPostgresText returns value encoded in postgres text format +// For a byte sequence value this is a hex encoded string +func (v *bytesValue) AsPostgresText() []byte { + // all bytes should be encoded as valid bytea value + return utils.PgEncodeToHex(v.bytes) +} + +// AsMysqlBinary returns value encoded in mysql binary format +// For a byte sequence value this is the same as text encoding +func (v *bytesValue) AsMysqlBinary() []byte { + panic("REMOVE THIS") +} + +// AsMysqlText returns value encoded in mysql text format +// For a byte sequence value this is a length encoded string +func (v *bytesValue) AsMysqlText() []byte { + panic("REMOVE THIS") +} + +// intValue represents a {size*8}-bit integer ready for encoding +type intValue struct { + size int + intValue int64 + strValue []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For an int value it is a big endian encoded integer +func (v *intValue) AsPostgresBinary() []byte { + newData := make([]byte, v.size) + switch v.size { + case 4: + binary.BigEndian.PutUint32(newData, uint32(v.intValue)) + case 8: + binary.BigEndian.PutUint64(newData, uint64(v.intValue)) + } + return newData +} + +// AsPostgresText returns value encoded in postgres text format +// For an int this means returning textual representation of the integer +func (v *intValue) AsPostgresText() []byte { + return v.strValue +} + +// AsMysqlBinary returns value encoded in mysql binary format +// For an int value it is a little endian encoded integer +func (v *intValue) AsMysqlBinary() []byte { + panic("REMOVE THIS") +} + +// AsMysqlText returns value encoded in mysql text format +// For an int this is a length encoded string of that integer +func (v *intValue) AsMysqlText() []byte { + panic("REMOVE THIS") +} + +// stringValue is an EncodingValue that encodes data into string format +type stringValue struct { + data []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// In other words, it returns data as it is +func (v *stringValue) AsPostgresBinary() []byte { + return v.data +} + +// AsPostgresText returns value encoded in postgres text format +// In other words, it returns data as it is +func (v *stringValue) AsPostgresText() []byte { + return v.data +} + +// AsMysqlBinary returns value encoded in mysql binary format +// In other words, it encodes data into length encoded string +func (v *stringValue) AsMysqlBinary() []byte { + panic("REMOVE THIS") +} + +// AsMysqlText returns value encoded in mysql text format +// In other words, it encodes data into length encoded string +func (v *stringValue) AsMysqlText() []byte { + panic("REMOVE THIS") +} + +// postgresValueFactory is a factory that produces values that can encode into +// postgres format +type postgresValueFactory struct{} + +// NewStringValue creates a value that encodes as a str +func (*postgresValueFactory) NewStringValue(str []byte) base.EncodingValue { + return &stringValue{data: str} +} + +// NewBytesValue creates a value that encodes as bytes +func (*postgresValueFactory) NewBytesValue(bytes []byte) base.EncodingValue { + return &bytesValue{bytes} +} + +// NewInt32Value creates a value that encodes as int32 +func (*postgresValueFactory) NewInt32Value(intVal int32, strVal []byte) base.EncodingValue { + return &intValue{size: 4, intValue: int64(intVal), strValue: strVal} +} + +// NewInt64Value creates a value that encodes as int64 +func (*postgresValueFactory) NewInt64Value(intVal int64, strVal []byte) base.EncodingValue { + return &intValue{size: 8, intValue: intVal, strValue: strVal} +} From 03725d33dbf8f7df62ddd386186bcef0cdde2093 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Thu, 5 May 2022 15:26:31 +0300 Subject: [PATCH 15/19] Implement mysqlValueFactory --- decryptor/mysql/data_encoder.go | 133 +++++++++++++++++++++++++-- decryptor/postgresql/data_encoder.go | 30 +++--- 2 files changed, 143 insertions(+), 20 deletions(-) diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 5a3900d14..5c803cf94 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -20,6 +20,8 @@ import ( // ErrConvertToDataType error that indicates if data type conversion was failed var ErrConvertToDataType = errors.New("error on converting to data type") +var valueFactory base.EncodingValueFactory = &mysqlValueFactory{} + // BaseMySQLDataProcessor implements processor and encode/decode binary intX values to text format which acceptable by Tokenizer type BaseMySQLDataProcessor struct{} @@ -161,7 +163,7 @@ func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, da switch dataType { case common.EncryptedType_String, common.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) + value, err := base.EncodeOnFail(setting, &mysqlValueFactory{}, logger) if err != nil { return nil, err } else if value != nil { @@ -175,15 +177,14 @@ func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, da intValue, err := strconv.ParseInt(strValue, 10, 64) // if it's valid string literal and decrypted, return as is if err == nil { - size := 4 - if dataType == common.EncryptedType_Int64 { - size = 8 + if dataType == common.EncryptedType_Int32 { + return valueFactory.NewInt32Value(int32(intValue), data), nil } - return base.NewIntValue(size, intValue, strValue), nil + return valueFactory.NewInt64Value(intValue, data), nil } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { - value, err := base.EncodeOnFail(setting, &base.ValueFactoryPlug{}, logger) + value, err := base.EncodeOnFail(setting, valueFactory, logger) if err != nil { return nil, err } else if value != nil { @@ -325,3 +326,123 @@ func (p *DataDecoderProcessor) OnColumn(ctx context.Context, data []byte) (conte } return ctx, data, nil } + +// bytesValue is an EncodingValue that represents byte array +type bytesValue struct { + bytes []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For a byte sequence value this is an identity operation +func (v *bytesValue) AsPostgresBinary() []byte { + panic("REMOVE THIS") +} + +// AsPostgresText returns value encoded in postgres text format +// For a byte sequence value this is a hex encoded string +func (v *bytesValue) AsPostgresText() []byte { + panic("REMOVE THIS") +} + +// AsMysqlBinary returns value encoded in mysql binary format +// For a byte sequence value this is the same as text encoding +func (v *bytesValue) AsMysqlBinary() []byte { + return v.AsMysqlText() +} + +// AsMysqlText returns value encoded in mysql text format +// For a byte sequence value this is a length encoded string +func (v *bytesValue) AsMysqlText() []byte { + return base.PutLengthEncodedString(v.bytes) +} + +// intValue represents a {size*8}-bit integer ready for encoding +type intValue struct { + size int + intValue int64 + strValue []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// For an int value it is a big endian encoded integer +func (v *intValue) AsPostgresBinary() []byte { + panic("REMOVE THIS") +} + +// AsPostgresText returns value encoded in postgres text format +// For an int this means returning textual representation of the integer +func (v *intValue) AsPostgresText() []byte { + panic("REMOVE THIS") +} + +// AsMysqlBinary returns value encoded in mysql binary format +// For an int value it is a little endian encoded integer +func (v *intValue) AsMysqlBinary() []byte { + newData := make([]byte, v.size) + switch v.size { + case 4: + binary.LittleEndian.PutUint32(newData, uint32(v.intValue)) + case 8: + binary.LittleEndian.PutUint64(newData, uint64(v.intValue)) + } + return newData +} + +// AsMysqlText returns value encoded in mysql text format +// For an int this is a length encoded string of that integer +func (v *intValue) AsMysqlText() []byte { + return base.PutLengthEncodedString(v.strValue) +} + +// stringValue is an EncodingValue that encodes data into string format +type stringValue struct { + data []byte +} + +// AsPostgresBinary returns value encoded in postgres binary format +// In other words, it returns data as it is +func (v *stringValue) AsPostgresBinary() []byte { + panic("REMOVE THIS") +} + +// AsPostgresText returns value encoded in postgres text format +// In other words, it returns data as it is +func (v *stringValue) AsPostgresText() []byte { + panic("REMOVE THIS") +} + +// AsMysqlBinary returns value encoded in mysql binary format +// In other words, it encodes data into length encoded string +func (v *stringValue) AsMysqlBinary() []byte { + return base.PutLengthEncodedString(v.data) +} + +// AsMysqlText returns value encoded in mysql text format +// In other words, it encodes data into length encoded string +func (v *stringValue) AsMysqlText() []byte { + return base.PutLengthEncodedString(v.data) +} + +// mysqlValueFactory is a factory that produces values that can encode into +// mysql format +type mysqlValueFactory struct{} + +// NewStringValue creates a value that encodes as a str +func (*mysqlValueFactory) NewStringValue(str []byte) base.EncodingValue { + return &stringValue{data: str} +} + +// NewBytesValue creates a value that encodes as bytes +func (*mysqlValueFactory) NewBytesValue(bytes []byte) base.EncodingValue { + return &bytesValue{bytes} +} + +// NewInt32Value creates a value that encodes as int32 +func (*mysqlValueFactory) NewInt32Value(intVal int32, strVal []byte) base.EncodingValue { + return &intValue{size: 4, intValue: int64(intVal), strValue: strVal} +} + +// NewInt64Value creates a value that encodes as int64 +func (*mysqlValueFactory) NewInt64Value(intVal int64, strVal []byte) base.EncodingValue { + return &intValue{size: 8, intValue: intVal, strValue: strVal} +} diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index f2eeb7ecd..e2ef38d65 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -14,6 +14,8 @@ import ( "github.com/sirupsen/logrus" ) +var valueFactory base.EncodingValueFactory = &postgresValueFactory{} + // PgSQLDataEncoderProcessor implements processor and encode binary/text values before sending to app type PgSQLDataEncoderProcessor struct{} @@ -30,9 +32,10 @@ func (p *PgSQLDataEncoderProcessor) ID() string { func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, base.EncodingValue, error) { logger = logger.WithField("column", setting.ColumnName()).WithField("decrypted", base.IsDecryptedFromContext(ctx)) if len(data) == 0 { - return ctx, base.NewStringValue(data), nil + return ctx, valueFactory.NewStringValue(data), nil } - switch setting.GetEncryptedDataType() { + dataType := setting.GetEncryptedDataType() + switch dataType { case common2.EncryptedType_String: if !base.IsDecryptedFromContext(ctx) { value, err := base.EncodeOnFail(setting, &postgresValueFactory{}, logger) @@ -43,7 +46,7 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } // decrypted values return as is, without any encoding - return ctx, base.NewStringValue(data), nil + return ctx, valueFactory.NewStringValue(data), nil case common2.EncryptedType_Bytes: if !base.IsDecryptedFromContext(ctx) { value, err := base.EncodeOnFail(setting, &postgresValueFactory{}, logger) @@ -53,19 +56,18 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by return ctx, value, nil } } - return ctx, base.NewBytesValue(data), nil + return ctx, valueFactory.NewBytesValue(data), nil case common2.EncryptedType_Int32, common2.EncryptedType_Int64: - size := 8 - if setting.GetEncryptedDataType() == common2.EncryptedType_Int32 { - size = 4 - } + // convert back from text to binary strValue := string(data) // if it's valid string literal and decrypted, return as is value, err := strconv.ParseInt(strValue, 10, 64) if err == nil { - val := base.NewIntValue(size, value, strValue) - return ctx, val, nil + if dataType == common2.EncryptedType_Int32 { + return ctx, valueFactory.NewInt32Value(int32(value), data), nil + } + return ctx, valueFactory.NewInt64Value(value, data), nil } // if it's encrypted binary, then it is binary array that is invalid int literal if !base.IsDecryptedFromContext(ctx) { @@ -77,20 +79,20 @@ func (p *PgSQLDataEncoderProcessor) encodeToValue(ctx context.Context, data []by } } logger.Warningln("Can't decode int value and no default value") - return ctx, base.NewStringValue(data), nil + return ctx, valueFactory.NewStringValue(data), nil } // here we process AcraStruct/AcraBlock decryption without any encryptor config that defines data_type/token_type // values. If it was decrypted then we return it as valid bytea value if base.IsDecryptedFromContext(ctx) { - return ctx, base.NewBytesValue(data), nil + return ctx, valueFactory.NewBytesValue(data), nil } // If it wasn't decrypted (due to inappropriate keys or not AcraStructs as payload) then we return it in same way // as it come to us. encodedValue, ok := getEncodedValueFromContext(ctx) if ok { - return ctx, base.NewStringValue(encodedValue), nil + return ctx, valueFactory.NewStringValue(encodedValue), nil } - return ctx, base.NewStringValue(data), nil + return ctx, valueFactory.NewStringValue(data), nil } // OnColumn encode binary value to text and back. Should be before and after tokenizer processor From 1e867e7b9f2a6b4986867d103bb5630942fe95a6 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Thu, 5 May 2022 15:39:40 +0300 Subject: [PATCH 16/19] Canonize EncodingValue and remove plugs --- decryptor/base/encoder.go | 158 +-------------------------- decryptor/mysql/data_encoder.go | 66 +++-------- decryptor/postgresql/data_encoder.go | 64 +++-------- 3 files changed, 33 insertions(+), 255 deletions(-) diff --git a/decryptor/base/encoder.go b/decryptor/base/encoder.go index 9c36a3082..a920ed844 100644 --- a/decryptor/base/encoder.go +++ b/decryptor/base/encoder.go @@ -2,12 +2,10 @@ package base import ( "encoding/base64" - "encoding/binary" "fmt" "strconv" "github.com/cossacklabs/acra/encryptor/config" - "github.com/cossacklabs/acra/utils" "github.com/sirupsen/logrus" "github.com/cossacklabs/acra/encryptor/config/common" @@ -43,15 +41,10 @@ func NewEncodingError(column string) error { // EncodingValue represents a (possibly parsed and prepared) value that is // ready to be encoded type EncodingValue interface { - // AsPostgresBinary returns value encoded in postgres binary format - AsPostgresBinary() []byte - // AsPostgresText returns value encoded in postgres text format - AsPostgresText() []byte - - // AsMysqlBinary returns value encoded in mysql binary format - AsMysqlBinary() []byte - // AsMysqlText returns value encoded in mysql text format - AsMysqlText() []byte + // AssBinary returns value encoded in a binary format + AsBinary() []byte + // AsText returns value encoded in a text format + AsText() []byte } // EncodingValueFactory represents a factory that produces ready for encoding @@ -67,125 +60,6 @@ type EncodingValueFactory interface { NewInt64Value(intVal int64, strVal []byte) EncodingValue } -// BytesValue is an EncodingValue that represents byte array -type BytesValue struct { - seq []byte -} - -// NewBytesValue returns EncodingValue from a byte array -func NewBytesValue(seq []byte) EncodingValue { - return &BytesValue{seq} -} - -// AsPostgresBinary returns value encoded in postgres binary format -// For a byte sequence value this is an identity operation -func (v *BytesValue) AsPostgresBinary() []byte { - return v.seq -} - -// AsPostgresText returns value encoded in postgres text format -// For a byte sequence value this is a hex encoded string -func (v *BytesValue) AsPostgresText() []byte { - // all bytes should be encoded as valid bytea value - return utils.PgEncodeToHex(v.seq) -} - -// AsMysqlBinary returns value encoded in mysql binary format -// For a byte sequence value this is the same as text encoding -func (v *BytesValue) AsMysqlBinary() []byte { - return v.AsMysqlText() -} - -// AsMysqlText returns value encoded in mysql text format -// For a byte sequence value this is a length encoded string -func (v *BytesValue) AsMysqlText() []byte { - return PutLengthEncodedString(v.seq) -} - -// IntValue represents a {size*8}-bit integer ready for encoding -type IntValue struct { - size int - value int64 - strValue string -} - -// NewIntValue returns EncodingValue from integer with size*8 bits -func NewIntValue(size int, value int64, strValue string) EncodingValue { - return &IntValue{size, value, strValue} -} - -// AsPostgresBinary returns value encoded in postgres binary format -// For an int value it is a big endian encoded integer -func (v *IntValue) AsPostgresBinary() []byte { - newData := make([]byte, v.size) - switch v.size { - case 4: - binary.BigEndian.PutUint32(newData, uint32(v.value)) - case 8: - binary.BigEndian.PutUint64(newData, uint64(v.value)) - } - return newData -} - -// AsPostgresText returns value encoded in postgres text format -// For an int this means returning textual representation of the integer -func (v *IntValue) AsPostgresText() []byte { - return []byte(v.strValue) -} - -// AsMysqlBinary returns value encoded in mysql binary format -// For an int value it is a little endian encoded integer -func (v *IntValue) AsMysqlBinary() []byte { - newData := make([]byte, v.size) - switch v.size { - case 4: - binary.LittleEndian.PutUint32(newData, uint32(v.value)) - case 8: - binary.LittleEndian.PutUint64(newData, uint64(v.value)) - } - return newData -} - -// AsMysqlText returns value encoded in mysql text format -// For an int this is a length encoded string of that integer -func (v *IntValue) AsMysqlText() []byte { - return PutLengthEncodedString([]byte(v.strValue)) -} - -// StringValue is an EncodingValue that encodes data into string format -type StringValue struct { - data []byte -} - -// NewStringValue returns string EncodingValue -func NewStringValue(data []byte) EncodingValue { - return &StringValue{data} -} - -// AsPostgresBinary returns value encoded in postgres binary format -// In other words, it returns data as it is -func (v *StringValue) AsPostgresBinary() []byte { - return v.data -} - -// AsPostgresText returns value encoded in postgres text format -// In other words, it returns data as it is -func (v *StringValue) AsPostgresText() []byte { - return v.data -} - -// AsMysqlBinary returns value encoded in mysql binary format -// In other words, it encodes data into length encoded string -func (v *StringValue) AsMysqlBinary() []byte { - return PutLengthEncodedString(v.data) -} - -// AsMysqlText returns value encoded in mysql text format -// In other words, it encodes data into length encoded string -func (v *StringValue) AsMysqlText() []byte { - return PutLengthEncodedString(v.data) -} - // EncodeDefault returns wrapped default value from settings ready for encoding // returns nil if something went wrong, which in many cases indicates that the // original value should be returned as it is @@ -245,27 +119,3 @@ func EncodeOnFail(setting config.ColumnEncryptionSetting, valueFactory EncodingV return nil, fmt.Errorf("unknown action: %q", action) } - -// ValueFactoryPlug is a temporary factory which will be removed when -// each backend (postgres or mysql) implements its own one -type ValueFactoryPlug struct{} - -// NewStringValue creates a value that encodes as a str -func (plug *ValueFactoryPlug) NewStringValue(str []byte) EncodingValue { - return NewStringValue(str) -} - -// NewBytesValue creates a value that encodes as bytes -func (plug *ValueFactoryPlug) NewBytesValue(bytes []byte) EncodingValue { - return NewBytesValue(bytes) -} - -// NewInt32Value creates a value that encodes as int32 -func (plug *ValueFactoryPlug) NewInt32Value(intVal int32, strVal []byte) EncodingValue { - return NewIntValue(4, int64(intVal), string(strVal)) -} - -// NewInt64Value creates a value that encodes as int64 -func (plug *ValueFactoryPlug) NewInt64Value(intVal int64, strVal []byte) EncodingValue { - return NewIntValue(8, int64(intVal), string(strVal)) -} diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 5c803cf94..254410f48 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -81,7 +81,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, // in case of successful encoding with defined data type return encoded data if encodingValue != nil { - return ctx, encodingValue.AsMysqlBinary(), nil + return ctx, encodingValue.AsBinary(), nil } var columnType = columnInfo.DataBinaryType() @@ -212,7 +212,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se // in case of successful encoding with defined data type return encoded data if encodingValue != nil { - return ctx, encodingValue.AsMysqlText(), nil + return ctx, encodingValue.AsText(), nil } // in case of error on converting to defined type we should roll back field type and encode it as it was originally @@ -332,27 +332,15 @@ type bytesValue struct { bytes []byte } -// AsPostgresBinary returns value encoded in postgres binary format -// For a byte sequence value this is an identity operation -func (v *bytesValue) AsPostgresBinary() []byte { - panic("REMOVE THIS") -} - -// AsPostgresText returns value encoded in postgres text format -// For a byte sequence value this is a hex encoded string -func (v *bytesValue) AsPostgresText() []byte { - panic("REMOVE THIS") -} - -// AsMysqlBinary returns value encoded in mysql binary format +// AsBinary returns value encoded in mysql binary format // For a byte sequence value this is the same as text encoding -func (v *bytesValue) AsMysqlBinary() []byte { - return v.AsMysqlText() +func (v *bytesValue) AsBinary() []byte { + return v.AsText() } -// AsMysqlText returns value encoded in mysql text format +// AsText returns value encoded in mysql text format // For a byte sequence value this is a length encoded string -func (v *bytesValue) AsMysqlText() []byte { +func (v *bytesValue) AsText() []byte { return base.PutLengthEncodedString(v.bytes) } @@ -363,21 +351,9 @@ type intValue struct { strValue []byte } -// AsPostgresBinary returns value encoded in postgres binary format -// For an int value it is a big endian encoded integer -func (v *intValue) AsPostgresBinary() []byte { - panic("REMOVE THIS") -} - -// AsPostgresText returns value encoded in postgres text format -// For an int this means returning textual representation of the integer -func (v *intValue) AsPostgresText() []byte { - panic("REMOVE THIS") -} - -// AsMysqlBinary returns value encoded in mysql binary format +// AsBinary returns value encoded in mysql binary format // For an int value it is a little endian encoded integer -func (v *intValue) AsMysqlBinary() []byte { +func (v *intValue) AsBinary() []byte { newData := make([]byte, v.size) switch v.size { case 4: @@ -388,9 +364,9 @@ func (v *intValue) AsMysqlBinary() []byte { return newData } -// AsMysqlText returns value encoded in mysql text format +// AsText returns value encoded in mysql text format // For an int this is a length encoded string of that integer -func (v *intValue) AsMysqlText() []byte { +func (v *intValue) AsText() []byte { return base.PutLengthEncodedString(v.strValue) } @@ -399,27 +375,15 @@ type stringValue struct { data []byte } -// AsPostgresBinary returns value encoded in postgres binary format -// In other words, it returns data as it is -func (v *stringValue) AsPostgresBinary() []byte { - panic("REMOVE THIS") -} - -// AsPostgresText returns value encoded in postgres text format -// In other words, it returns data as it is -func (v *stringValue) AsPostgresText() []byte { - panic("REMOVE THIS") -} - -// AsMysqlBinary returns value encoded in mysql binary format +// AsBinary returns value encoded in mysql binary format // In other words, it encodes data into length encoded string -func (v *stringValue) AsMysqlBinary() []byte { +func (v *stringValue) AsBinary() []byte { return base.PutLengthEncodedString(v.data) } -// AsMysqlText returns value encoded in mysql text format +// AsText returns value encoded in mysql text format // In other words, it encodes data into length encoded string -func (v *stringValue) AsMysqlText() []byte { +func (v *stringValue) AsText() []byte { return base.PutLengthEncodedString(v.data) } diff --git a/decryptor/postgresql/data_encoder.go b/decryptor/postgresql/data_encoder.go index e2ef38d65..8d5c9f333 100644 --- a/decryptor/postgresql/data_encoder.go +++ b/decryptor/postgresql/data_encoder.go @@ -117,9 +117,9 @@ func (p *PgSQLDataEncoderProcessor) OnColumn(ctx context.Context, data []byte) ( } if columnInfo.IsBinaryFormat() { - return ctx, value.AsPostgresBinary(), nil + return ctx, value.AsBinary(), nil } - return ctx, value.AsPostgresText(), nil + return ctx, value.AsText(), nil } // PgSQLDataDecoderProcessor implements processor and decode binary/text values from DB @@ -239,31 +239,19 @@ type bytesValue struct { bytes []byte } -// AsPostgresBinary returns value encoded in postgres binary format +// AsBinary returns value encoded in postgres binary format // For a byte sequence value this is an identity operation -func (v *bytesValue) AsPostgresBinary() []byte { +func (v *bytesValue) AsBinary() []byte { return v.bytes } -// AsPostgresText returns value encoded in postgres text format +// AsText returns value encoded in postgres text format // For a byte sequence value this is a hex encoded string -func (v *bytesValue) AsPostgresText() []byte { +func (v *bytesValue) AsText() []byte { // all bytes should be encoded as valid bytea value return utils.PgEncodeToHex(v.bytes) } -// AsMysqlBinary returns value encoded in mysql binary format -// For a byte sequence value this is the same as text encoding -func (v *bytesValue) AsMysqlBinary() []byte { - panic("REMOVE THIS") -} - -// AsMysqlText returns value encoded in mysql text format -// For a byte sequence value this is a length encoded string -func (v *bytesValue) AsMysqlText() []byte { - panic("REMOVE THIS") -} - // intValue represents a {size*8}-bit integer ready for encoding type intValue struct { size int @@ -271,9 +259,9 @@ type intValue struct { strValue []byte } -// AsPostgresBinary returns value encoded in postgres binary format +// AsBinary returns value encoded in postgres binary format // For an int value it is a big endian encoded integer -func (v *intValue) AsPostgresBinary() []byte { +func (v *intValue) AsBinary() []byte { newData := make([]byte, v.size) switch v.size { case 4: @@ -284,53 +272,29 @@ func (v *intValue) AsPostgresBinary() []byte { return newData } -// AsPostgresText returns value encoded in postgres text format +// AsText returns value encoded in postgres text format // For an int this means returning textual representation of the integer -func (v *intValue) AsPostgresText() []byte { +func (v *intValue) AsText() []byte { return v.strValue } -// AsMysqlBinary returns value encoded in mysql binary format -// For an int value it is a little endian encoded integer -func (v *intValue) AsMysqlBinary() []byte { - panic("REMOVE THIS") -} - -// AsMysqlText returns value encoded in mysql text format -// For an int this is a length encoded string of that integer -func (v *intValue) AsMysqlText() []byte { - panic("REMOVE THIS") -} - // stringValue is an EncodingValue that encodes data into string format type stringValue struct { data []byte } -// AsPostgresBinary returns value encoded in postgres binary format +// AsBinary returns value encoded in postgres binary format // In other words, it returns data as it is -func (v *stringValue) AsPostgresBinary() []byte { +func (v *stringValue) AsBinary() []byte { return v.data } -// AsPostgresText returns value encoded in postgres text format +// AsText returns value encoded in postgres text format // In other words, it returns data as it is -func (v *stringValue) AsPostgresText() []byte { +func (v *stringValue) AsText() []byte { return v.data } -// AsMysqlBinary returns value encoded in mysql binary format -// In other words, it encodes data into length encoded string -func (v *stringValue) AsMysqlBinary() []byte { - panic("REMOVE THIS") -} - -// AsMysqlText returns value encoded in mysql text format -// In other words, it encodes data into length encoded string -func (v *stringValue) AsMysqlText() []byte { - panic("REMOVE THIS") -} - // postgresValueFactory is a factory that produces values that can encode into // postgres format type postgresValueFactory struct{} From a638607966cc0dc599ff81318b1d4a8d44257ef6 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Thu, 5 May 2022 15:44:20 +0300 Subject: [PATCH 17/19] Revert "Move mysql.utils into base.utils" This reverts commit 1a913c17f02eca16f8f7025182be6656f5167a56. Because as @lagovas mentioned: > it's mysql related functions. they should be stored in mysql package, not in base) --- decryptor/mysql/column_field.go | 38 ++++++++++++-------------- decryptor/mysql/data_encoder.go | 16 +++++------ decryptor/mysql/prepared_statements.go | 4 +-- decryptor/mysql/response_proxy.go | 17 ++++++------ decryptor/{base => mysql}/utils.go | 5 ++-- 5 files changed, 38 insertions(+), 42 deletions(-) rename decryptor/{base => mysql}/utils.go (99%) diff --git a/decryptor/mysql/column_field.go b/decryptor/mysql/column_field.go index 33fc7c0a3..36d22f4c0 100644 --- a/decryptor/mysql/column_field.go +++ b/decryptor/mysql/column_field.go @@ -19,8 +19,6 @@ package mysql import ( "encoding/binary" "errors" - - "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" ) @@ -113,7 +111,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { var err error //skip catalog, always def pos := 0 - n, err = base.SkipLengthEncodedString(packet.data) + n, err = SkipLengthEncodedString(packet.data) if err != nil { return nil, err @@ -121,35 +119,35 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { pos += n //schema - field.Schema, n, err = base.LengthEncodedString(packet.data[pos:]) + field.Schema, n, err = LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //table - field.Table, n, err = base.LengthEncodedString(packet.data[pos:]) + field.Table, n, err = LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //org_table - field.OrgTable, n, err = base.LengthEncodedString(packet.data[pos:]) + field.OrgTable, n, err = LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //name - field.Name, n, err = base.LengthEncodedString(packet.data[pos:]) + field.Name, n, err = LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } pos += n //org_name - field.OrgName, n, err = base.LengthEncodedString(packet.data[pos:]) + field.OrgName, n, err = LengthEncodedString(packet.data[pos:]) if err != nil { return nil, err } @@ -185,7 +183,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { //if more data, command was field list if len(packet.data) > pos { //length of default value lenenc-int - field.DefaultValueLength, _, n, err = base.LengthEncodedInt(packet.data[pos:]) + field.DefaultValueLength, _, n, err = LengthEncodedInt(packet.data[pos:]) if err != nil { log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).WithError(err).Errorln("Can't get length encoded integer of default value length") return nil, err @@ -194,7 +192,7 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) { if pos+int(field.DefaultValueLength) > len(packet.data) { log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).Errorln("Incorrect position, malformed packet") - err = base.ErrMalformPacket + err = ErrMalformPacket return nil, err } @@ -220,30 +218,30 @@ func (field *ColumnDescription) Dump() []byte { data := make([]byte, 0, l) - data = append(data, base.PutLengthEncodedString([]byte("def"))...) + data = append(data, PutLengthEncodedString([]byte("def"))...) - data = append(data, base.PutLengthEncodedString(field.Schema)...) + data = append(data, PutLengthEncodedString(field.Schema)...) - data = append(data, base.PutLengthEncodedString(field.Table)...) - data = append(data, base.PutLengthEncodedString(field.OrgTable)...) + data = append(data, PutLengthEncodedString(field.Table)...) + data = append(data, PutLengthEncodedString(field.OrgTable)...) - data = append(data, base.PutLengthEncodedString(field.Name)...) - data = append(data, base.PutLengthEncodedString(field.OrgName)...) + data = append(data, PutLengthEncodedString(field.Name)...) + data = append(data, PutLengthEncodedString(field.OrgName)...) // length of fixed-length fields // https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition data = append(data, 0x0c) - data = append(data, base.Uint16ToBytes(field.Charset)...) - data = append(data, base.Uint32ToBytes(field.ColumnLength)...) + data = append(data, Uint16ToBytes(field.Charset)...) + data = append(data, Uint32ToBytes(field.ColumnLength)...) data = append(data, byte(field.Type)) - data = append(data, base.Uint16ToBytes(field.Flag)...) + data = append(data, Uint16ToBytes(field.Flag)...) data = append(data, field.Decimal) // filler data = append(data, 0, 0) if field.DefaultValue != nil { - data = append(data, base.Uint64ToBytes(field.DefaultValueLength)...) + data = append(data, Uint64ToBytes(field.DefaultValueLength)...) data = append(data, field.DefaultValue...) } diff --git a/decryptor/mysql/data_encoder.go b/decryptor/mysql/data_encoder.go index 254410f48..0e51ffba0 100644 --- a/decryptor/mysql/data_encoder.go +++ b/decryptor/mysql/data_encoder.go @@ -71,7 +71,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, if len(data) == 0 { // we still need to encode result data as it might be null field in db - return ctx, base.PutLengthEncodedString(data), nil + return ctx, PutLengthEncodedString(data), nil } encodingValue, err := p.encodeValueWithDataType(ctx, data, setting, logger) @@ -155,7 +155,7 @@ func (p *BaseMySQLDataProcessor) encodeBinary(ctx context.Context, data []byte, return ctx, encoded, err } - return ctx, base.PutLengthEncodedString(data), nil + return ctx, PutLengthEncodedString(data), nil } func (p *BaseMySQLDataProcessor) encodeValueWithDataType(ctx context.Context, data []byte, setting config.ColumnEncryptionSetting, logger *logrus.Entry) (base.EncodingValue, error) { @@ -202,7 +202,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se logger.Debugln("Encode text") if len(data) == 0 { // we still need to encode result data as it might be null field in db - return ctx, base.PutLengthEncodedString(data), nil + return ctx, PutLengthEncodedString(data), nil } encodingValue, err := p.encodeValueWithDataType(ctx, data, setting, logger) @@ -220,7 +220,7 @@ func (p *BaseMySQLDataProcessor) encodeText(ctx context.Context, data []byte, se ctx = base.MarkErrorConvertedDataTypeContext(ctx) } - return ctx, base.PutLengthEncodedString(data), nil + return ctx, PutLengthEncodedString(data), nil } func (p *BaseMySQLDataProcessor) decodeBinary(ctx context.Context, encoded []byte, setting config.ColumnEncryptionSetting, columnInfo base.ColumnInfo, logger *logrus.Entry) (context.Context, []byte, error) { @@ -341,7 +341,7 @@ func (v *bytesValue) AsBinary() []byte { // AsText returns value encoded in mysql text format // For a byte sequence value this is a length encoded string func (v *bytesValue) AsText() []byte { - return base.PutLengthEncodedString(v.bytes) + return PutLengthEncodedString(v.bytes) } // intValue represents a {size*8}-bit integer ready for encoding @@ -367,7 +367,7 @@ func (v *intValue) AsBinary() []byte { // AsText returns value encoded in mysql text format // For an int this is a length encoded string of that integer func (v *intValue) AsText() []byte { - return base.PutLengthEncodedString(v.strValue) + return PutLengthEncodedString(v.strValue) } // stringValue is an EncodingValue that encodes data into string format @@ -378,13 +378,13 @@ type stringValue struct { // AsBinary returns value encoded in mysql binary format // In other words, it encodes data into length encoded string func (v *stringValue) AsBinary() []byte { - return base.PutLengthEncodedString(v.data) + return PutLengthEncodedString(v.data) } // AsText returns value encoded in mysql text format // In other words, it encodes data into length encoded string func (v *stringValue) AsText() []byte { - return base.PutLengthEncodedString(v.data) + return PutLengthEncodedString(v.data) } // mysqlValueFactory is a factory that produces values that can encode into diff --git a/decryptor/mysql/prepared_statements.go b/decryptor/mysql/prepared_statements.go index a0b71ff0e..8c440cd52 100644 --- a/decryptor/mysql/prepared_statements.go +++ b/decryptor/mysql/prepared_statements.go @@ -112,7 +112,7 @@ func NewMysqlBoundValue(data []byte, format base.BoundValueFormat, paramType Typ // if we cant find amount of stored bytes for the paramType assume that it is length encoded string storageBytes, ok := NumericTypesStorageBytes[paramType] if !ok { - value, n, err := base.LengthEncodedString(data) + value, n, err := LengthEncodedString(data) if err != nil { return nil, 0, err } @@ -226,7 +226,7 @@ func (m *mysqlBoundValue) GetData(_ config.ColumnEncryptionSetting) ([]byte, err func (m *mysqlBoundValue) Encode() (encoded []byte, err error) { storageBytes, ok := NumericTypesStorageBytes[m.paramType] if !ok { - return base.PutLengthEncodedString(m.data), nil + return PutLengthEncodedString(m.data), nil } // separate error variable for output error from case statements // to not overlap with new err variables inside diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go index 5141a70d2..f1bff2d65 100644 --- a/decryptor/mysql/response_proxy.go +++ b/decryptor/mysql/response_proxy.go @@ -20,16 +20,15 @@ import ( "context" "encoding/binary" "errors" + "github.com/cossacklabs/acra/keystore/filesystem" + "github.com/cossacklabs/acra/sqlparser" + "go.opencensus.io/trace" "io" "net" "strconv" "time" - "github.com/cossacklabs/acra/keystore/filesystem" - "github.com/cossacklabs/acra/sqlparser" - "go.opencensus.io/trace" - - acracensor "github.com/cossacklabs/acra/acra-censor" + "github.com/cossacklabs/acra/acra-censor" "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" @@ -497,7 +496,7 @@ func (handler *Handler) processTextDataRow(ctx context.Context, rowData []byte, handler.logger.Debugln("Process data rows in text protocol") for i := range fields { fieldLogger = handler.logger.WithField("field_index", i) - value, n, err := base.LengthEncodedString(rowData[pos:]) + value, n, err := LengthEncodedString(rowData[pos:]) if err != nil { return nil, err } @@ -535,7 +534,7 @@ func (handler *Handler) processBinaryDataRow(ctx context.Context, rowData []byte } if rowData[0] != OkPacket { - return nil, base.ErrMalformPacket + return nil, ErrMalformPacket } // https://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html @@ -610,7 +609,7 @@ func (handler *Handler) extractData(pos int, rowData []byte, field *ColumnDescri return rowData[pos : pos+8], 8, nil case TypeDecimal, TypeNewDecimal, TypeBit, TypeEnum, TypeSet, TypeGeometry, TypeDate, TypeNewDate, TypeTimestamp, TypeDatetime, TypeTime, TypeVarchar, TypeTinyBlob, TypeMediumBlob, TypeLongBlob, TypeBlob, TypeVarString, TypeString: - value, n, err := base.LengthEncodedString(rowData[pos:]) + value, n, err := LengthEncodedString(rowData[pos:]) if err != nil { handler.logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). Errorln("Can't handle length encoded string non binary value") @@ -654,7 +653,7 @@ func (handler *Handler) QueryResponseHandler(ctx context.Context, packet *Packet if fieldPacket.IsEOF() { if i != fieldCount { handler.logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorProtocolProcessing).Errorln("EOF and field count != current row packet count") - return base.ErrMalformPacket + return ErrMalformPacket } output = append(output, fieldPacket) break diff --git a/decryptor/base/utils.go b/decryptor/mysql/utils.go similarity index 99% rename from decryptor/base/utils.go rename to decryptor/mysql/utils.go index 70eba4005..99307f5ce 100644 --- a/decryptor/base/utils.go +++ b/decryptor/mysql/utils.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package base +package mysql import ( "errors" - "io" - "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" + "io" ) // ErrMalformPacket if packet parsing failed From 28d9ace8b85e499e243257d24e9dee80cbf25e4c Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Fri, 6 May 2022 13:24:13 +0300 Subject: [PATCH 18/19] Close the connection in case of encoding error This is a temporary solution, before proper error handling lands on. --- decryptor/mysql/response_proxy.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go index f1bff2d65..6d808a295 100644 --- a/decryptor/mysql/response_proxy.go +++ b/decryptor/mysql/response_proxy.go @@ -20,15 +20,16 @@ import ( "context" "encoding/binary" "errors" - "github.com/cossacklabs/acra/keystore/filesystem" - "github.com/cossacklabs/acra/sqlparser" - "go.opencensus.io/trace" "io" "net" "strconv" "time" - "github.com/cossacklabs/acra/acra-censor" + "github.com/cossacklabs/acra/keystore/filesystem" + "github.com/cossacklabs/acra/sqlparser" + "go.opencensus.io/trace" + + acracensor "github.com/cossacklabs/acra/acra-censor" "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" @@ -867,7 +868,10 @@ func (handler *Handler) ProxyDatabaseConnection(ctx context.Context, errCh chan< WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseConnectorCantWriteToClient). Debugln("Can't write response with error to client") errCh <- base.NewDBProxyError(err) + return } + // Close the connections + errCh <- base.NewDBProxyError(nil) return } From 9dfb3342d314320934d1f7c9280326bd29e5ce05 Mon Sep 17 00:00:00 2001 From: G1gg1L3s Date: Fri, 6 May 2022 16:07:08 +0300 Subject: [PATCH 19/19] Continue serving packets in case of encoding error This is temporary, until we implement proper flushing of db packets. Right now, it data is sent after our encoding error, it would probably trigger the "unexpected sequence number" error on the client side. --- decryptor/mysql/response_proxy.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go index 6d808a295..3d7df61e6 100644 --- a/decryptor/mysql/response_proxy.go +++ b/decryptor/mysql/response_proxy.go @@ -870,9 +870,9 @@ func (handler *Handler) ProxyDatabaseConnection(ctx context.Context, errCh chan< errCh <- base.NewDBProxyError(err) return } - // Close the connections - errCh <- base.NewDBProxyError(nil) - return + // Continue serving packet, though we should skip them till the end + // of the response. + continue } if err != nil {