Skip to content

Commit

Permalink
Base64 encode/decode control settings refactoring.
Browse files Browse the repository at this point in the history
  • Loading branch information
@vglavnyy
vglavnyy committed Jan 21, 2018
1 parent 994a392 commit c5c6864
Show file tree
Hide file tree
Showing 6 changed files with 124 additions and 107 deletions.
2 changes: 1 addition & 1 deletion include/flatbuffers/idl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -647,7 +647,7 @@ class Parser : public ParserState {
FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(
size_t &count, ParseVectorDelimitersBody body, void *state);
FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
FLATBUFFERS_CHECKED_ERROR ParseVectorBase64(const Type &type, FieldDef *field,
FLATBUFFERS_CHECKED_ERROR ParseVectorBase64(const Type &type, int base64_mode,
uoffset_t *ovalue);
FLATBUFFERS_CHECKED_ERROR ParseNestedFlatbuffer(Value &val, FieldDef *field,
size_t fieldn,
Expand Down
41 changes: 26 additions & 15 deletions include/flatbuffers/util_base64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,50 @@

namespace flatbuffers {

enum Base64Mode {
// RFC4648-url, padding is mandatory
kBase64Url = 0,
// RFC4648, padding is mandatory
kBase64Strict = 1,
// Cancel padding for decode/encode.
kBase64CancelPadding = 2
};

inline Base64Mode operator+(Base64Mode a, Base64Mode b) {
return static_cast<Base64Mode>(static_cast<int>(a) | static_cast<int>(b));
// a Field doesn't have base64 attribute
static const int kBase64Undefined = 0;
// attribute (base64): strict RFC4648
static const int kBase64Strict = 1;
// attribute (base64url): RFC4648 with URL and Filename Safe Alphabet
static const int kBase64Url = 2;
// Cancel padding for encoder (by RFC4648 padding ('=') is mandatory)
static const int kBase64CancelPadding = 4;

// FDT is <idl.h>::FieldDef type
template<typename FDT>
int field_base64_mode(const FDT *fd, bool cancel_padding = false) {
auto mode = kBase64Undefined;
if (fd) {
if (fd->attributes.Lookup("base64")) {
mode = kBase64Strict;
} else if (fd->attributes.Lookup("base64url")) {
mode = kBase64Url;
}
if (mode && cancel_padding) { mode |= kBase64CancelPadding; }
}
return mode;
}

// calculate number of bytes required to save Decoded result
// return zero if error
size_t Base64DecodedSize(Base64Mode mode, const char *src, size_t src_size,
size_t Base64DecodedSize(int base64_mode, const char *src, size_t src_size,
size_t *error_position = nullptr);

// Decode input base64 sequence.
// expects: dst_size >= Base64DecodedSize(mode, src, src_size)
// return number of written bytes or zero
size_t Base64Decode(Base64Mode mode, const char *src, size_t src_size,
size_t Base64Decode(int base64_mode, const char *src, size_t src_size,
uint8_t *dst, size_t dst_size,
size_t *error_position = nullptr);

// calculate number of bytes required to save Encoded result
// return zero if error
size_t Base64EncodedSize(Base64Mode mode, const uint8_t *src, size_t src_size);
size_t Base64EncodedSize(int base64_mode, const uint8_t *src, size_t src_size);

// encode input byte sequence to base64 sequence.
// expects: dst_size >= Base64EncodedSize(mode, src, src_size)
// return number of written bytes or zero
size_t Base64Encode(Base64Mode mode, const uint8_t *src, size_t src_size,
size_t Base64Encode(int base64_mode, const uint8_t *src, size_t src_size,
char *dst, size_t dst_size);

// internal unit test
Expand Down
16 changes: 5 additions & 11 deletions src/idl_gen_text.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,13 @@ bool Print(T val, Type type, int /*indent*/, Type * /*union_type*/,
template<typename T>
bool PrintVectorBase64(const Vector<T> &v, Type type, int indent,
const IDLOptions &opts, std::string *_text,
bool b64_strict) {
int b64mode) {
(void)type;
(void)indent;
(void)opts;
// only byte array supported
if (type.base_type != BASE_TYPE_UCHAR) return false;

Base64Mode b64mode = b64_strict ? kBase64Strict : kBase64Url;
if (opts.output_base64_cancel_padding) {
b64mode = b64mode + kBase64CancelPadding;
}
// Print a sequence of [uint8_t] as base64-encoded string
std::string &text = *_text;
const auto *src = reinterpret_cast<const uint8_t *>(v.data());
Expand All @@ -112,12 +109,9 @@ template<typename T>
bool PrintVector(const Vector<T> &v, Type type, int indent,
const IDLOptions &opts, std::string *_text,
const FieldDef *fd = nullptr) {
if (fd &&
(fd->attributes.Lookup("base64") || fd->attributes.Lookup("base64url"))) {
const bool b64_strict = !!fd->attributes.Lookup("base64");
if (PrintVectorBase64<T>(v, type, indent, opts, _text, b64_strict))
return true; // leave PrintVector
}
auto base64 = field_base64_mode(fd, opts.output_base64_cancel_padding);
if (base64 && PrintVectorBase64<T>(v, type, indent, opts, _text, base64))
return true;

std::string &text = *_text;
text += "[";
Expand Down
35 changes: 17 additions & 18 deletions src/idl_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -857,16 +857,20 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
break;
}
case BASE_TYPE_VECTOR: {
if (Is('[')) {
if (token_ == kTokenStringConstant) {
// base64 padding doesn't requered by default at decode
auto b64mode = field_base64_mode(field, true);
if (b64mode) {
uoffset_t off;
ECHECK(ParseVectorBase64(val.type.VectorType(), b64mode, &off));
val.constant = NumToString(off);
} else {
EXPECT('[');
}
} else if (Is('[')) {
uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off));
val.constant = NumToString(off);
} else if (token_ == kTokenStringConstant && field &&
(field->attributes.Lookup("base64") ||
field->attributes.Lookup("base64url"))) {
uoffset_t off;
ECHECK(ParseVectorBase64(val.type.VectorType(), field, &off));
val.constant = NumToString(off);
} else {
EXPECT('[');
}
Expand Down Expand Up @@ -1156,29 +1160,24 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
return NoError();
}

CheckedError Parser::ParseVectorBase64(const Type &type, FieldDef *field,
CheckedError Parser::ParseVectorBase64(const Type &type, const int base64_mode,
uoffset_t *ovalue) {
if (type.base_type != BASE_TYPE_UCHAR) {
return Error(
"The base64[url] attribute is only allowed for an [ubyte] array.");
}
const auto b64_strict = !!field->attributes.Lookup("base64");
const std::string &str = attribute_;
const auto src_data = str.c_str();
const auto src_size = str.size();
// cancel padding at decode, by default
const auto b64mode =
kBase64CancelPadding + (b64_strict ? kBase64Strict : kBase64Url);
const auto src_data = attribute_.c_str();
const auto src_size = attribute_.size();
size_t err_pos = 0;
// calculate number of required bytes
auto decoded_size =
Base64DecodedSize(b64mode, src_data, src_size, &err_pos);
Base64DecodedSize(base64_mode, src_data, src_size, &err_pos);
if (decoded_size > 0) {
uint8_t *dst = nullptr;
*ovalue = builder_.CreateUninitializedVector(decoded_size, 1, &dst);
// Base64Decode return number of written bytes or zero if error detected
decoded_size =
Base64Decode(b64mode, src_data, src_size, dst, decoded_size, &err_pos);
decoded_size = Base64Decode(base64_mode, src_data, src_size, dst,
decoded_size, &err_pos);
}
if ((0 == decoded_size) && (src_size > 0)) {
return Error(
Expand Down
82 changes: 43 additions & 39 deletions src/util_base64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,26 @@

#include "flatbuffers/util_base64.h"

namespace flatbuffers {
// BASE64 - https://tools.ietf.org/html/rfc4648

static inline bool Base64ModeIsStrict(Base64Mode m) {
return (m & kBase64Strict);
namespace flatbuffers {

static inline bool Base64ModeIsStrict(int base64_mode) {
return 0 == (base64_mode & kBase64Url);
}

static inline bool Base64PadingdIsOptional(Base64Mode m) {
return !!(m & kBase64CancelPadding);
static inline bool Base64PadingdIsOptional(int base64_mode) {
return 0 != (base64_mode & kBase64CancelPadding);
}

// return number of written bytes if success or zero if an error.
// error condition: (src_size!=0 && ret==0)
// return number of bytes requred for decoding if:
// ((nullptr == dst) && (0 == dst_size))
static inline size_t decode_b64(const Base64Mode mode, const char *const src,
const size_t src_size, uint8_t *const dst,
const size_t dst_size,
size_t *const error_position = nullptr) {
static size_t decode_b64(const int mode, const char *const src,
const size_t src_size, uint8_t *const dst,
const size_t dst_size,
size_t *const error_position = nullptr) {
// strict base64 table (RFC 4648)
static const uint8_t strict_table[256] = {
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
Expand Down Expand Up @@ -105,10 +106,10 @@ static inline size_t decode_b64(const Base64Mode mode, const char *const src,
// loop counter: the last block plus all perfect blocks
auto loop_cnt = (1 + C4full);
// error mask can take only two states: (0)-ok and (1)-fail
// type of <err_mask> must match with <loop_cnt>
// type of <err_mask> must match with unsigned <loop_cnt>
auto err_mask = loop_cnt * 0;
// use (err_mask - 1) = {0,~0} as conditional gate for the loop_cnt
for (auto k = 0; loop_cnt & (err_mask - 1); loop_cnt--, k++) {
for (size_t k = 0; loop_cnt & (err_mask-1); loop_cnt--, k++) {
const auto a0 = b64_tbl[static_cast<uint8_t>(src_[0])];
const auto a1 = b64_tbl[static_cast<uint8_t>(src_[1])];
const auto a2 = b64_tbl[static_cast<uint8_t>(src_[2])];
Expand Down Expand Up @@ -140,9 +141,9 @@ static inline size_t decode_b64(const Base64Mode mode, const char *const src,
// return number of written bytes or zero if error.
// return number of bytes requred for decoding if:
// ((nullptr == dst) && (0 == dst_size))
static inline size_t encode_b64(const Base64Mode mode, const uint8_t *const src,
const size_t src_size, char *const dst,
const size_t dst_size) {
static size_t encode_b64(const int mode, const uint8_t *const src,
const size_t src_size, char *const dst,
const size_t dst_size) {
static const char strict_table[64] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
Expand Down Expand Up @@ -201,46 +202,52 @@ static inline size_t encode_b64(const Base64Mode mode, const uint8_t *const src,
return encoded_len;
}

size_t Base64DecodedSize(Base64Mode mode, const char *src, size_t src_size,
size_t Base64DecodedSize(int base64_mode, const char *src, size_t src_size,
size_t *error_position) {
return decode_b64(mode, src, src_size, nullptr, 0, error_position);
return decode_b64(base64_mode, src, src_size, nullptr, 0, error_position);
}

size_t Base64Decode(Base64Mode mode, const char *src, size_t src_size,
size_t Base64Decode(int base64_mode, const char *src, size_t src_size,
uint8_t *dst, size_t dst_size, size_t *error_position) {
if (error_position) *error_position = 0;
if (0 == dst_size) return 0;
return decode_b64(mode, src, src_size, dst, dst_size, error_position);
return decode_b64(base64_mode, src, src_size, dst, dst_size, error_position);
}

size_t Base64EncodedSize(Base64Mode mode, const uint8_t *src, size_t src_size) {
return encode_b64(mode, src, src_size, nullptr, 0);
size_t Base64EncodedSize(int base64_mode, const uint8_t *src, size_t src_size) {
return encode_b64(base64_mode, src, src_size, nullptr, 0);
}

size_t Base64Encode(Base64Mode mode, const uint8_t *src, size_t src_size,
size_t Base64Encode(int base64_mode, const uint8_t *src, size_t src_size,
char *dst, size_t dst_size) {
if (0 == dst_size) return 0;
return encode_b64(mode, src, src_size, dst, dst_size);
return encode_b64(base64_mode, src, src_size, dst, dst_size);
}
} // namespace flatbuffers

//#define B64_TEST_PERFORMANCE

#if defined(B64_TEST_PERFORMANCE)
#include <chrono>
#include <iostream>

bool Base64PerformanceTest() {
const size_t Nsamp = 1024 * 1024;
const size_t Mrep = 256;
std::vector<uint8_t> bin(Nsamp);
for (size_t k = 0; k < bin.size(); k++) { bin[k] = (k + 7) % 0x100; }
// preallocate
size_t enc_size = 0;
if (Base64EncodedSize(Base64Mode::kBase64Strict, bin.data(), bin.size(),
&enc_size)) {
return false;
}
const auto enc_size = flatbuffers::Base64EncodedSize(
flatbuffers::kBase64Strict, bin.data(), bin.size());
if (!enc_size) return false;

std::vector<char> enc(enc_size);
// start encode test
auto enc_start = std::chrono::steady_clock::now();
for (size_t rep_cnt = 0; rep_cnt < Mrep; rep_cnt++) {
size_t enc_rc = Base64Encode(Base64Mode::kBase64Strict, bin.data(),
bin.size(), enc.data(), enc.size());
size_t enc_rc =
flatbuffers::Base64Encode(flatbuffers::kBase64Strict, bin.data(),
bin.size(), enc.data(), enc.size());
if (enc_rc != enc_size) return false;
}
auto enc_end = std::chrono::steady_clock::now();
Expand All @@ -249,17 +256,17 @@ bool Base64PerformanceTest() {
.count();

// start decode test
size_t dec_size = 0;
if (Base64DecodedSize(Base64Mode::kBase64Strict, enc.data(), enc.size(),
&dec_size)) {
return false;
}
const auto dec_size = flatbuffers::Base64DecodedSize(
flatbuffers::kBase64Strict, enc.data(), enc.size());
if (!dec_size) return false;

bin.clear();
bin.resize(dec_size);
auto dec_start = std::chrono::steady_clock::now();
for (size_t rep_cnt = 0; rep_cnt < Mrep; rep_cnt++) {
size_t dec_rc = Base64Decode(Base64Mode::kBase64Strict, enc.data(),
enc.size(), bin.data(), bin.size());
size_t dec_rc =
flatbuffers::Base64Decode(flatbuffers::kBase64Strict, enc.data(),
enc.size(), bin.data(), bin.size());
if (dec_rc != dec_size) return false;
}
auto dec_end = std::chrono::steady_clock::now();
Expand All @@ -277,7 +284,4 @@ bool Base64PerformanceTest() {

return true;
}
#else
int Base64SelfTest() { return 0; }
#endif
} // namespace flatbuffers
Loading

0 comments on commit c5c6864

Please sign in to comment.