cossacklabs · Lagovas · Jul 26, 2023 · Jul 26, 2023 · Jul 26, 2023 · Jul 26, 2023
diff --git a/CHANGELOG_DEV.md b/CHANGELOG_DEV.md
@@ -1,3 +1,6 @@
+# 0.96.0 - 2023-07-26
+- Improve unicode support
+
 # 0.96.0 - 2023-03-06
 - Added support of PreparedStatement from SQL for PostgreSQL;
 

diff --git a/encryptor/dbDataCoder.go b/encryptor/dbDataCoder.go
@@ -20,9 +20,11 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
+	"github.com/cossacklabs/acra/encryptor/config"
 	"github.com/cossacklabs/acra/logging"
 	"github.com/cossacklabs/acra/sqlparser"
 	"github.com/cossacklabs/acra/utils"
+	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/sirupsen/logrus"
 	"strconv"
 	"unicode/utf8"
@@ -34,8 +36,8 @@ var hexNumPrefix = []byte{48, 120}
 
 // DBDataCoder encode/decode binary data to correct string form for specific db
 type DBDataCoder interface {
-	Decode(sqlparser.Expr) ([]byte, error)
-	Encode(sqlparser.Expr, []byte) ([]byte, error)
+	Decode(sqlparser.Expr, config.ColumnEncryptionSetting) ([]byte, error)
+	Encode(sqlparser.Expr, []byte, config.ColumnEncryptionSetting) ([]byte, error)
 }
 
 // errUnsupportedExpression unsupported type of literal to binary encode/decode
@@ -45,7 +47,7 @@ var errUnsupportedExpression = errors.New("unsupported expression")
 type MysqlDBDataCoder struct{}
 
 // Decode decode literals from string to byte slice
-func (*MysqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
+func (*MysqlDBDataCoder) Decode(expr sqlparser.Expr, _ config.ColumnEncryptionSetting) ([]byte, error) {
 	switch val := expr.(type) {
 	case *sqlparser.SQLVal:
 		switch val.Type {
@@ -76,7 +78,7 @@ func (*MysqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
 }
 
 // Encode data to correct literal from binary data for this expression
-func (*MysqlDBDataCoder) Encode(expr sqlparser.Expr, data []byte) ([]byte, error) {
+func (*MysqlDBDataCoder) Encode(expr sqlparser.Expr, data []byte, _ config.ColumnEncryptionSetting) ([]byte, error) {
 	encodeDataToHex := func(val *sqlparser.SQLVal, data []byte) ([]byte, error) {
 		output := make([]byte, hex.EncodedLen(len(data)))
 		hex.Encode(output, data)
@@ -140,7 +142,7 @@ type PostgresqlDBDataCoder struct{}
 // Decode hex/escaped literals to raw binary values for encryption/decryption. String values left as is because it
 // doesn't need any decoding. Historically Int values had support only for tokenization and operated over string SQL
 // literals.
-func (*PostgresqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
+func (*PostgresqlDBDataCoder) Decode(expr sqlparser.Expr, setting config.ColumnEncryptionSetting) ([]byte, error) {
 	switch val := expr.(type) {
 	case *sqlparser.SQLVal:
 		switch val.Type {
@@ -154,7 +156,30 @@ func (*PostgresqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
 				return nil, err
 			}
 			return binValue, err
-		case sqlparser.PgEscapeString, sqlparser.StrVal:
+		case sqlparser.PgEscapeString:
+			// try to decode hex/octal encoding
+			binValue, err := utils.DecodeEscaped(val.Val)
+			if err != nil && err != utils.ErrDecodeOctalString {
+				// return error on hex decode
+				if _, ok := err.(hex.InvalidByteError); err == hex.ErrLength || ok {
+					return nil, err
+				} else if err == utils.ErrDecodeOctalString {
+					return nil, err
+				}
+
+				logrus.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCodingCantDecodeSQLValue).Warningln("Can't decode value, process as unescaped string")
+				// return value as is because it may be string with printable characters that wasn't encoded on client
+				return val.Val, nil
+			}
+			return binValue, nil
+		case sqlparser.StrVal:
+			// simple strings should be handled as is
+			typeID := setting.GetDBDataTypeID()
+			if typeID != 0 && typeID != pgtype.ByteaOID {
+				return val.Val, nil
+			}
+			// bytea strings are escaped with \x hex value or with octal encoding
+
 			// try to decode hex/octal encoding
 			binValue, err := utils.DecodeEscaped(val.Val)
 			if err != nil && err != utils.ErrDecodeOctalString {
@@ -176,7 +201,7 @@ func (*PostgresqlDBDataCoder) Decode(expr sqlparser.Expr) ([]byte, error) {
 }
 
 // Encode data to correct literal from binary data for this expression
-func (*PostgresqlDBDataCoder) Encode(expr sqlparser.Expr, data []byte) ([]byte, error) {
+func (*PostgresqlDBDataCoder) Encode(expr sqlparser.Expr, data []byte, setting config.ColumnEncryptionSetting) ([]byte, error) {
 	switch val := expr.(type) {
 	case *sqlparser.SQLVal:
 		switch val.Type {
@@ -197,24 +222,25 @@ func (*PostgresqlDBDataCoder) Encode(expr sqlparser.Expr, data []byte) ([]byte,
 			// otherwise change type and pass it below for hex encoding
 			val.Type = sqlparser.PgEscapeString
 			fallthrough
-		case sqlparser.StrVal:
-			binValue, err := utils.DecodeEscaped(data)
-			// Check is encryption/tokenization operation returned string in "binary string" format instead of UTF8 string.
-			// If we can decode it as binary value and get another value, then escape it as hex binary value.
-			if err != nil || !bytes.Equal(binValue, data) {
-				return PgEncodeToHexString(data), nil
-			}
-			// If the binValue the same to data means data is Printable, return as is
-			if bytes.Equal(binValue, data) {
-				return data, nil
-			}
-			val.Type = sqlparser.PgEscapeString
-			fallthrough
 		case sqlparser.PgEscapeString:
-			// valid strings we pass as is without extra encoding
-			if utils.IsPrintablePostgresqlString(data) {
-				return data, nil
+			// if type is not byte array, then it probably string or int and we pass printable strings
+			if setting.GetDBDataTypeID() != 0 && setting.GetDBDataTypeID() != pgtype.ByteaOID {
+				// valid strings we pass as is without extra encoding
+				if utils.IsPrintablePostgresqlString(data) {
+					return data, nil
+				}
+			}
+			// valid string can contain escaped symbols, or tokenizer may generate string with symbols that should be escaped
+			return utils.EncodeToOctal(data), nil
+		case sqlparser.StrVal:
+			// if type is not byte array, then it probably string or int and we pass printable strings
+			if setting.GetDBDataTypeID() != 0 && setting.GetDBDataTypeID() != pgtype.ByteaOID {
+				// valid strings we pass as is without extra encoding
+				if utils.IsPrintablePostgresqlString(data) {
+					return data, nil
+				}
 			}
+			// byte array can be valid hex/octal encoded value, eventually we should encode it as binary data
 			return PgEncodeToHexString(data), nil
 		}
 	}

diff --git a/encryptor/dbDataCoder_test.go b/encryptor/dbDataCoder_test.go
@@ -21,6 +21,7 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
+	"github.com/cossacklabs/acra/encryptor/config"
 	"github.com/cossacklabs/acra/sqlparser"
 	"github.com/cossacklabs/acra/utils"
 	"testing"
@@ -51,7 +52,7 @@ func TestMysqlDBDataCoder_Decode(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
-		data, err := coder.Decode(testCase.Input)
+		data, err := coder.Decode(testCase.Input, &config.BasicColumnEncryptionSetting{})
 		if err != testCase.Err {
 			t.Errorf("Expr: %s\nUnexpected error\nExpected: %v\nActual: %v", sqlparser.String(testCase.Input), testCase.Err, err)
 			continue
@@ -93,7 +94,7 @@ func TestMysqlDBDataCoder_Encode(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
-		coded, err := coder.Encode(testCase.Expr, testCase.Input)
+		coded, err := coder.Encode(testCase.Expr, testCase.Input, &config.BasicColumnEncryptionSetting{})
 		if err != testCase.Err {
 			t.Errorf("Expr: %s\nUnexpected error\nExpected: %v\nActual: %v", sqlparser.String(testCase.Expr), testCase.Err, err)
 			continue
@@ -114,7 +115,7 @@ func TestPostgresqlDBDataCoder_Decode(t *testing.T) {
 		sqlparser.NewStrVal([]byte(fmt.Sprintf("\\x%s", hex.EncodeToString(testData)))),
 	}
 	for _, expr := range testCases {
-		data, err := coder.Decode(expr)
+		data, err := coder.Decode(expr, &config.BasicColumnEncryptionSetting{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -158,7 +159,7 @@ func TestPostgresqlDBDataCoder_Decode(t *testing.T) {
 		},
 	}
 	for i, testCase := range errTestCases {
-		_, err := coder.Decode(testCase.Expr)
+		_, err := coder.Decode(testCase.Expr, &config.BasicColumnEncryptionSetting{})
 		if err != testCase.Err {
 			t.Fatalf("[%d] Incorrect error. Took: %s; Expected: %s", i, err, testCase.Err.Error())
 		}
@@ -182,20 +183,20 @@ func TestPostgresqlDBDataCoder_Encode(t *testing.T) {
 			Expr:   sqlparser.NewStrVal(testData),
 		},
 		{
-			Output: []byte(fmt.Sprintf("\\x%s", hex.EncodeToString(testData))),
-			Expr:   sqlparser.NewPgEscapeString(testData),
+			Output: utils.EncodeToOctal(testData),
+			Expr:   sqlparser.NewPgEscapeString(utils.EncodeToOctal(testData)),
 		},
 	}
 	for _, testCase := range testCases {
-		coded, err := coder.Encode(testCase.Expr, testData)
+		coded, err := coder.Encode(testCase.Expr, testData, &config.BasicColumnEncryptionSetting{})
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !bytes.Equal(coded, testCase.Output) {
 			t.Fatalf("Expr: %s\nTook: %s\nExpected: %s", sqlparser.String(testCase.Expr), string(coded), string(testCase.Output))
 		}
 	}
-	if _, err := coder.Encode(sqlparser.NewFloatVal([]byte{1}), testData); err != errUnsupportedExpression {
+	if _, err := coder.Encode(sqlparser.NewFloatVal([]byte{1}), testData, &config.BasicColumnEncryptionSetting{}); err != errUnsupportedExpression {
 		t.Fatalf("Incorrect error. Took: %s; Expected: %s", err.Error(), errUnsupportedExpression.Error())
 	}
 }
diff --git a/encryptor/queryDataEncryptor.go b/encryptor/queryDataEncryptor.go
@@ -161,18 +161,18 @@ func (encryptor *QueryDataEncryptor) encryptInsertQuery(ctx context.Context, ins
 var ErrUpdateLeaveDataUnchanged = errors.New("updateFunc didn't change data")
 
 // UpdateExpressionValue decode value from DB related string to binary format, call updateFunc, encode to DB string format and replace value in expression with new
-func UpdateExpressionValue(ctx context.Context, expr sqlparser.Expr, coder DBDataCoder, updateFunc func(context.Context, []byte) ([]byte, error)) error {
+func UpdateExpressionValue(ctx context.Context, expr sqlparser.Expr, coder DBDataCoder, setting config.ColumnEncryptionSetting, updateFunc func(context.Context, []byte) ([]byte, error)) error {
 	switch val := expr.(type) {
 	case *sqlparser.UnaryExpr:
-		return UpdateUnaryExpressionValue(ctx, expr.(*sqlparser.UnaryExpr), coder, updateFunc)
+		return UpdateUnaryExpressionValue(ctx, expr.(*sqlparser.UnaryExpr), coder, setting, updateFunc)
 	// Update Parenthese expression like  `('AAAA')` just by processing inner
 	// expression 'AAAA'.
 	case *sqlparser.ParenExpr:
-		return UpdateExpressionValue(ctx, expr.(*sqlparser.ParenExpr).Expr, coder, updateFunc)
+		return UpdateExpressionValue(ctx, expr.(*sqlparser.ParenExpr).Expr, coder, setting, updateFunc)
 	case *sqlparser.SQLVal:
 		switch val.Type {
 		case sqlparser.StrVal, sqlparser.HexVal, sqlparser.PgEscapeString, sqlparser.IntVal, sqlparser.HexNum:
-			rawData, err := coder.Decode(val)
+			rawData, err := coder.Decode(val, setting)
 			if err != nil {
 				if err == utils.ErrDecodeOctalString || err == errUnsupportedExpression {
 					logrus.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCodingCantDecodeSQLValue).
@@ -190,7 +190,7 @@ func UpdateExpressionValue(ctx context.Context, expr sqlparser.Expr, coder DBDat
 			if len(newData) == len(rawData) && bytes.Equal(newData, rawData) {
 				return ErrUpdateLeaveDataUnchanged
 			}
-			coded, err := coder.Encode(expr, newData)
+			coded, err := coder.Encode(expr, newData, setting)
 			if err != nil {
 				return err
 			}
@@ -202,12 +202,12 @@ func UpdateExpressionValue(ctx context.Context, expr sqlparser.Expr, coder DBDat
 
 // UpdateUnaryExpressionValue updates supported unary expression
 // By now, supported are only `_binary` charsets, that are parsed as unary expr.
-func UpdateUnaryExpressionValue(ctx context.Context, expr *sqlparser.UnaryExpr, coder DBDataCoder, updateFunc func(context.Context, []byte) ([]byte, error)) error {
+func UpdateUnaryExpressionValue(ctx context.Context, expr *sqlparser.UnaryExpr, coder DBDataCoder, setting config.ColumnEncryptionSetting, updateFunc func(context.Context, []byte) ([]byte, error)) error {
 	switch unaryVal := expr.Expr.(type) {
 	case *sqlparser.SQLVal:
 		switch strings.TrimSpace(expr.Operator) {
 		case "_binary":
-			return UpdateExpressionValue(ctx, unaryVal, coder, updateFunc)
+			return UpdateExpressionValue(ctx, unaryVal, coder, setting, updateFunc)
 		}
 	}
 	return nil
@@ -216,14 +216,14 @@ func UpdateUnaryExpressionValue(ctx context.Context, expr *sqlparser.UnaryExpr,
 // encryptExpression check that expr is SQLVal and has Hexval then try to encrypt
 func (encryptor *QueryDataEncryptor) encryptExpression(ctx context.Context, expr sqlparser.Expr, schema config.TableSchema, columnName string, bindPlaceholder map[int]config.ColumnEncryptionSetting) (bool, error) {
 	if schema.NeedToEncrypt(columnName) {
+		setting := schema.GetColumnEncryptionSettings(columnName)
 		if sqlVal, ok := expr.(*sqlparser.SQLVal); ok {
 			placeholderIndex, err := ParsePlaceholderIndex(sqlVal)
 			if err == nil {
-				setting := schema.GetColumnEncryptionSettings(columnName)
 				bindPlaceholder[placeholderIndex] = setting
 			}
 		}
-		err := UpdateExpressionValue(ctx, expr, encryptor.dataCoder, func(ctx context.Context, data []byte) ([]byte, error) {
+		err := UpdateExpressionValue(ctx, expr, encryptor.dataCoder, setting, func(ctx context.Context, data []byte) ([]byte, error) {
 			if len(data) == 0 {
 				return data, nil
 			}

diff --git a/encryptor/queryDataEncryptor_test.go b/encryptor/queryDataEncryptor_test.go
@@ -431,7 +431,7 @@ schemas:
 		{
 			Query:             `INSERT INTO "tablewithoutcolumnschema" ("specified_client_id", "other_column", "default_client_id") VALUES ('%s', 1, '%s')`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},
@@ -442,7 +442,7 @@ schemas:
 		{
 			Query:             `UPDATE "tablewithoutcolumnschema" as "t" set "other_column"='%s', "specified_client_id"='%s', "default_client_id"='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},
@@ -496,7 +496,7 @@ schemas:
 		{
 			Query:             `UPDATE lowercasetable set other_column='%s', specified_client_id='%s', "DEFAULT_client_id"='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},
@@ -507,7 +507,7 @@ schemas:
 		{
 			Query:             `UPDATE lowercasetable set other_column='%s', specified_client_id='%s', DEFAULT_client_id='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, simpleStringData},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), simpleStringData},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID},
@@ -518,7 +518,7 @@ schemas:
 		{
 			Query:             `UPDATE "lowercasetable" set other_column='%s', specified_client_id='%s', "DEFAULT_client_id"='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},
@@ -529,7 +529,7 @@ schemas:
 		{
 			Query:             `UPDATE LOWERCASETABLE set other_column='%s', specified_client_id='%s', "DEFAULT_client_id"='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},
@@ -584,7 +584,7 @@ schemas:
 		{
 			Query:             `UPDATE "UPPERCASETABLE" set "other_column"='%s', "specified_client_id"='%s', "DEFAULT_client_id"='%s'`,
 			QueryData:         []interface{}{simpleStringData, simpleStringData, simpleStringData},
-			ExpectedQueryData: []interface{}{simpleStringData, encryptedValue, encryptedValue},
+			ExpectedQueryData: []interface{}{simpleStringData, PgEncodeToHexString(encryptedValue), PgEncodeToHexString(encryptedValue)},
 			Normalized:        true,
 			Changed:           true,
 			ExpectedIDS:       [][]byte{specifiedClientID, defaultClientID},

diff --git a/hmac/decryptor/hashQuery.go b/hmac/decryptor/hashQuery.go
@@ -138,7 +138,7 @@ func (encryptor *HashQuery) OnQuery(ctx context.Context, query base.OnQueryObjec
 
 		// substring(column, 1, <HMAC_size>) = 'value' ===> substring(column, 1, <HMAC_size>) = <HMAC('value')>
 		// substring(column, 1, <HMAC_size>) = $1      ===> no changes
-		err := queryEncryptor.UpdateExpressionValue(ctx, item.Expr.Right, encryptor.coder, encryptor.calculateHmac)
+		err := queryEncryptor.UpdateExpressionValue(ctx, item.Expr.Right, encryptor.coder, item.Setting, encryptor.calculateHmac)
 		if err != nil {
 			logrus.WithError(err).Debugln("Failed to update expression")
 			return query, false, err

diff --git a/hmac/decryptor/hashQuery_test.go b/hmac/decryptor/hashQuery_test.go
@@ -199,7 +199,7 @@ func TestSearchableWithTextFormat(t *testing.T) {
 		hmacValue, err := encryptor.calculateHmac(ctx, []byte(dataQueryPart))
 		assert.NoError(t, err)
 
-		newData, err := coder.Encode(rightVal, hmacValue)
+		newData, err := coder.Encode(rightVal, hmacValue, &config.BasicColumnEncryptionSetting{})
 		assert.NoError(t, err)
 		assert.Equal(t, len(rightVal.Val), len(newData))
 	}

diff --git a/pseudonymization/tokenizeQuery.go b/pseudonymization/tokenizeQuery.go
@@ -85,7 +85,7 @@ func (encryptor *TokenizeQuery) OnQuery(ctx context.Context, query base.OnQueryO
 
 		encryptor.searchableQueryFilter.ChangeSearchableOperator(item.Expr)
 
-		err = queryEncryptor.UpdateExpressionValue(ctx, item.Expr.Right, encryptor.coder, encryptor.getTokenizerDataWithSetting(item.Setting))
+		err = queryEncryptor.UpdateExpressionValue(ctx, item.Expr.Right, encryptor.coder, item.Setting, encryptor.getTokenizerDataWithSetting(item.Setting))
 		if err != nil {
 			logrus.WithError(err).Debugln("Failed to update expression")
 			return query, false, err