Skip to content

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@vglavnyy
vglavnyy committed Jan 23, 2018
1 parent c5c6864 commit 23a0871
Show file tree
Hide file tree
Showing 6 changed files with 124 additions and 234 deletions.
2 changes: 0 additions & 2 deletions include/flatbuffers/idl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,6 @@ struct IDLOptions {
std::string go_namespace;
bool reexport_ts_modules;
bool protobuf_ascii_alike;
bool output_base64_cancel_padding;

// Possible options for the more general generator below.
enum Language {
Expand Down Expand Up @@ -441,7 +440,6 @@ struct IDLOptions {
skip_flatbuffers_import(false),
reexport_ts_modules(true),
protobuf_ascii_alike(false),
output_base64_cancel_padding(false),
lang(IDLOptions::kJava),
mini_reflect(IDLOptions::kNone),
lang_to_generate(0) {}
Expand Down
26 changes: 14 additions & 12 deletions include/flatbuffers/util_base64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,58 @@
#define FLATBUFFERS_UTIL_B64_H_

namespace flatbuffers {
// [BASE64]{https://tools.ietf.org/html/rfc4648}

// a Field doesn't have base64 attribute
static const int kBase64Undefined = 0;

// attribute (base64): strict RFC4648
// general base64 alphabet, padding mandatory both for encoder and decoder
static const int kBase64Strict = 1;

// attribute (base64url): RFC4648 with URL and Filename Safe Alphabet
// URL safe alphabet, padding optional both for decoder and skipped at encoder
static const int kBase64Url = 2;
// Cancel padding for encoder (by RFC4648 padding ('=') is mandatory)
static const int kBase64CancelPadding = 4;

// FDT is <idl.h>::FieldDef type
template<typename FDT>
int field_base64_mode(const FDT *fd, bool cancel_padding = false) {
// Helper: FDT is <idl.h>::FieldDef type
template<typename FDT> inline int field_base64_mode(const FDT *fd) {
auto mode = kBase64Undefined;
if (fd) {
if (fd->attributes.Lookup("base64")) {
mode = kBase64Strict;
} else if (fd->attributes.Lookup("base64url")) {
mode = kBase64Url;
}
if (mode && cancel_padding) { mode |= kBase64CancelPadding; }
}
return mode;
}

// calculate number of bytes required to save Decoded result
// return zero if error
// return zero if error.
// error condition : (src_size != 0 && ret == 0)
size_t Base64DecodedSize(int base64_mode, const char *src, size_t src_size,
size_t *error_position = nullptr);

// Decode input base64 sequence.
// expects: dst_size >= Base64DecodedSize(mode, src, src_size)
// return number of written bytes or zero
// error condition : (src_size != 0 && ret == 0)
size_t Base64Decode(int base64_mode, const char *src, size_t src_size,
uint8_t *dst, size_t dst_size,
size_t *error_position = nullptr);

// calculate number of bytes required to save Encoded result
// return zero if error
// return zero if error.
// error condition : (src_size != 0 && ret == 0)
size_t Base64EncodedSize(int base64_mode, const uint8_t *src, size_t src_size);

// encode input byte sequence to base64 sequence.
// expects: dst_size >= Base64EncodedSize(mode, src, src_size)
// return number of written bytes or zero
// return number of written bytes or zero.
// error condition : (src_size != 0 && ret == 0)
size_t Base64Encode(int base64_mode, const uint8_t *src, size_t src_size,
char *dst, size_t dst_size);

// internal unit test
int Base64SelfTest();

} // namespace flatbuffers

#endif // FLATBUFFERS_UTIL_B64_H_
17 changes: 7 additions & 10 deletions src/idl_gen_text.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,20 @@ bool PrintVectorBase64(const Vector<T> &v, Type type, int indent,
(void)type;
(void)indent;
(void)opts;
// Print a sequence of [uint8_t] as base64-encoded string
// only byte array supported
if (type.base_type != BASE_TYPE_UCHAR) return false;

// Print a sequence of [uint8_t] as base64-encoded string
std::string &text = *_text;
// type T can by any, not only uint8_t
const auto *src = reinterpret_cast<const uint8_t *>(v.data());
const auto src_size = v.size();
const auto req_size = Base64EncodedSize(b64mode, src, src_size);
if ((0 == req_size) && (src_size > 0)) return false;
// reserve text-space before temporary allocation
text.reserve(text.size() + req_size + 64);
std::vector<char> buf(req_size); // temporary buffer
const auto b64_len = Base64Encode(b64mode, src, src_size,
flatbuffers::vector_data(buf), buf.size());
auto b64_len = Base64Encode(b64mode, src, src_size,
flatbuffers::vector_data(buf), buf.size());
if ((0 == b64_len) && (src_size > 0)) return false;
buf.resize(b64_len); // fit to actual size
// copy data from temporary to the string
std::string &text = *_text;
text += "\"";
text.append(flatbuffers::vector_data(buf), buf.size());
text += "\"";
Expand All @@ -109,8 +106,8 @@ template<typename T>
bool PrintVector(const Vector<T> &v, Type type, int indent,
const IDLOptions &opts, std::string *_text,
const FieldDef *fd = nullptr) {
auto base64 = field_base64_mode(fd, opts.output_base64_cancel_padding);
if (base64 && PrintVectorBase64<T>(v, type, indent, opts, _text, base64))
auto b64mode = field_base64_mode(fd);
if (b64mode && PrintVectorBase64<T>(v, type, indent, opts, _text, b64mode))
return true;

std::string &text = *_text;
Expand Down
15 changes: 5 additions & 10 deletions src/idl_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -857,16 +857,11 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
break;
}
case BASE_TYPE_VECTOR: {
if (token_ == kTokenStringConstant) {
// base64 padding doesn't requered by default at decode
auto b64mode = field_base64_mode(field, true);
if (b64mode) {
uoffset_t off;
ECHECK(ParseVectorBase64(val.type.VectorType(), b64mode, &off));
val.constant = NumToString(off);
} else {
EXPECT('[');
}
if ((token_ == kTokenStringConstant) && field_base64_mode(field)) {
uoffset_t off;
ECHECK(ParseVectorBase64(val.type.VectorType(),
field_base64_mode(field), &off));
val.constant = NumToString(off);
} else if (Is('[')) {
uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off));
Expand Down
119 changes: 34 additions & 85 deletions src/util_base64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,14 @@ static inline bool Base64ModeIsStrict(int base64_mode) {
return 0 == (base64_mode & kBase64Url);
}

static inline bool Base64PadingdIsOptional(int base64_mode) {
return 0 != (base64_mode & kBase64CancelPadding);
}

// return number of written bytes if success or zero if an error.
// error condition: (src_size!=0 && ret==0)
// return number of bytes requred for decoding if:
// return number of bytes requred for decoding if condition:
// ((nullptr == dst) && (0 == dst_size))
// <error_position> MUST be not-null
static size_t decode_b64(const int mode, const char *const src,
const size_t src_size, uint8_t *const dst,
const size_t dst_size,
size_t *const error_position = nullptr) {
const size_t dst_size, size_t *const error_position) {
// strict base64 table (RFC 4648)
static const uint8_t strict_table[256] = {
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
Expand Down Expand Up @@ -59,6 +55,8 @@ static size_t decode_b64(const int mode, const char *const src,
};
if (error_position) *error_position = 0;
const auto b64_tbl = Base64ModeIsStrict(mode) ? strict_table : url_table;
// padding mandatory for strict base64, and optional for base64url
const auto padding_mandatory = Base64ModeIsStrict(mode);
// base64 decode transform 4 encoded charaters to 3 data bytes
if (0 == src_size) return 0;
// decode prepare stage
Expand All @@ -77,7 +75,7 @@ static size_t decode_b64(const int mode, const char *const src,
// if it is optional, sets to zero symbol ('A')
char last_enc4[4];
uint8_t last_dec3[3];
memset(&last_enc4[0], Base64PadingdIsOptional(mode) ? 'A' : '#', 4);
memset(&last_enc4[0], padding_mandatory ? '#' : 'A', 4);
// copy available bytes
memcpy(&last_enc4[0], &src[C4full * 4], C4rem);
// guest for size of last decoded block, can be 1,2 or 3
Expand Down Expand Up @@ -109,7 +107,7 @@ static size_t decode_b64(const int mode, const char *const src,
// type of <err_mask> must match with unsigned <loop_cnt>
auto err_mask = loop_cnt * 0;
// use (err_mask - 1) = {0,~0} as conditional gate for the loop_cnt
for (size_t k = 0; loop_cnt & (err_mask-1); loop_cnt--, k++) {
for (size_t k = 0; loop_cnt & (err_mask - 1); loop_cnt--, k++) {
const auto a0 = b64_tbl[static_cast<uint8_t>(src_[0])];
const auto a1 = b64_tbl[static_cast<uint8_t>(src_[1])];
const auto a2 = b64_tbl[static_cast<uint8_t>(src_[2])];
Expand All @@ -123,17 +121,18 @@ static size_t decode_b64(const int mode, const char *const src,
src_ = src + (k * 4);
dst_ = dst + (k * 3);
}
// copy the last decoded block to destanation
memcpy(&dst[C4full * 3], &last_dec3[0], last_dec_len);
// check errors and return number of written bytes
const auto done = (0 == err_mask);
if (error_position) {
if (done) {
*error_position = (src_size + 1);
} else {
*error_position =
4 * (C4full - ((loop_cnt < C4full) ? (loop_cnt + 1) : 0));
}
if (done) {
// copy the last decoded block to destanation
memcpy(&dst[C4full * 3], &last_dec3[0], last_dec_len);
*error_position = (src_size + 1);
} else {
// number of blocks processed and written to [src] memory
auto indx = C4full - loop_cnt;
// reject writen data
memset(dst, 0, indx * 3);
// seve position of corrupted characters
*error_position = 4 * (indx ? (indx - 1) : C4full);
}
return done ? decoded_len : 0;
}
Expand Down Expand Up @@ -161,6 +160,8 @@ static size_t encode_b64(const int mode, const uint8_t *const src,
};

const auto tbl64 = Base64ModeIsStrict(mode) ? strict_table : url_table;
// padding mandatory for strict base64, and skipped for base64url
const auto force_padding = Base64ModeIsStrict(mode);
// base64 encode transform 3 data bytes to 4 encoded charaters
if (0 == src_size) return 0;
// number of complete blocks
Expand All @@ -173,7 +174,7 @@ static size_t encode_b64(const int mode, const uint8_t *const src,
B3rem = 3;
}
// if pad is mandatory, the remainder shall be encoded to the complete char[4]
const auto last_enc_len = Base64PadingdIsOptional(mode) ? (B3rem + 1) : 4;
const auto last_enc_len = force_padding ? 4 : (B3rem + 1);
const auto encoded_len = (B3full * 4) + last_enc_len;
// if requested the only number of required bytes for encoding, return it
if ((nullptr == dst) && (0 == dst_size)) return encoded_len;
Expand Down Expand Up @@ -204,14 +205,23 @@ static size_t encode_b64(const int mode, const uint8_t *const src,

size_t Base64DecodedSize(int base64_mode, const char *src, size_t src_size,
size_t *error_position) {
return decode_b64(base64_mode, src, src_size, nullptr, 0, error_position);
size_t err_pos = 0;
auto rc = decode_b64(base64_mode, src, src_size, nullptr, 0, &err_pos);
if (error_position) *error_position = err_pos;
return rc;
}

size_t Base64Decode(int base64_mode, const char *src, size_t src_size,
uint8_t *dst, size_t dst_size, size_t *error_position) {
if (error_position) *error_position = 0;
if (0 == dst_size) return 0;
return decode_b64(base64_mode, src, src_size, dst, dst_size, error_position);
size_t err_pos = 0;
size_t rc = 0;
// if 0==dst_size && nullptr==src function decode_b64 return required size
// avoid this by check dst_size>0
if (dst_size > 0) {
rc = decode_b64(base64_mode, src, src_size, dst, dst_size, &err_pos);
}
if (error_position) *error_position = err_pos;
return rc;
}

size_t Base64EncodedSize(int base64_mode, const uint8_t *src, size_t src_size) {
Expand All @@ -224,64 +234,3 @@ size_t Base64Encode(int base64_mode, const uint8_t *src, size_t src_size,
return encode_b64(base64_mode, src, src_size, dst, dst_size);
}
} // namespace flatbuffers

//#define B64_TEST_PERFORMANCE

#if defined(B64_TEST_PERFORMANCE)
#include <chrono>
#include <iostream>

bool Base64PerformanceTest() {
const size_t Nsamp = 1024 * 1024;
const size_t Mrep = 256;
std::vector<uint8_t> bin(Nsamp);
for (size_t k = 0; k < bin.size(); k++) { bin[k] = (k + 7) % 0x100; }
// preallocate
const auto enc_size = flatbuffers::Base64EncodedSize(
flatbuffers::kBase64Strict, bin.data(), bin.size());
if (!enc_size) return false;

std::vector<char> enc(enc_size);
// start encode test
auto enc_start = std::chrono::steady_clock::now();
for (size_t rep_cnt = 0; rep_cnt < Mrep; rep_cnt++) {
size_t enc_rc =
flatbuffers::Base64Encode(flatbuffers::kBase64Strict, bin.data(),
bin.size(), enc.data(), enc.size());
if (enc_rc != enc_size) return false;
}
auto enc_end = std::chrono::steady_clock::now();
auto enc_pref =
std::chrono::duration_cast<std::chrono::nanoseconds>(enc_end - enc_start)
.count();

// start decode test
const auto dec_size = flatbuffers::Base64DecodedSize(
flatbuffers::kBase64Strict, enc.data(), enc.size());
if (!dec_size) return false;

bin.clear();
bin.resize(dec_size);
auto dec_start = std::chrono::steady_clock::now();
for (size_t rep_cnt = 0; rep_cnt < Mrep; rep_cnt++) {
size_t dec_rc =
flatbuffers::Base64Decode(flatbuffers::kBase64Strict, enc.data(),
enc.size(), bin.data(), bin.size());
if (dec_rc != dec_size) return false;
}
auto dec_end = std::chrono::steady_clock::now();
auto dec_pref =
std::chrono::duration_cast<std::chrono::nanoseconds>(dec_end - dec_start)
.count();

(void)dec_pref;

std::cout << "Encode performance [ns/KB]: "
<< (enc_pref / Mrep * 1024 / Nsamp) << std::endl;

std::cout << "Decode performance [ns/KB]: "
<< (dec_pref / Mrep * 1024 / Nsamp) << std::endl;

return true;
}
#endif
Loading

0 comments on commit 23a0871

Please sign in to comment.