Skip to content

Add log.h #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 4, 2024
Merged

Add log.h #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acad533
Add log.h
larryliu0820 Dec 4, 2024
a560ee7
Patch __has_buitin
larryliu0820 Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ project(Tokenizers)

option(TOKENIZERS_BUILD_TEST "Build tests" OFF)

# Ignore weak attribute warning
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")

set(ABSL_ENABLE_INSTALL ON)
set(ABSL_PROPAGATE_CXX_STD ON)
set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
Expand Down
30 changes: 11 additions & 19 deletions include/base64.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,8 @@ inline Error validate(uint32_t v) {
}

inline Error decode(const std::string_view &input, std::string &output) {
if (input.size() != 4) {
fprintf(stderr, "input length must be 4, got %zu", input.size());
return Error::Base64DecodeFailure;
}
TK_CHECK_OR_RETURN_ERROR(input.size() == 4, Base64DecodeFailure,
"input length must be 4, got %zu", input.size());

uint32_t val = 0;

Expand Down Expand Up @@ -104,10 +102,8 @@ inline Error decode(const std::string_view &input, std::string &output) {

inline Error decode_1_padding(const std::string_view &input,
std::string &output) {
if (input.size() != 3) {
fprintf(stderr, "input length must be 3, got %zu", input.size());
return Error::Base64DecodeFailure;
}
TK_CHECK_OR_RETURN_ERROR(input.size() == 3, Base64DecodeFailure,
"input length must be 3, got %zu", input.size());

uint32_t val = 0;

Expand All @@ -133,7 +129,8 @@ inline Error decode_1_padding(const std::string_view &input,

inline Error decode_2_padding(const std::string_view &input,
std::string &output) {
TK_CHECK_OR_RETURN_ERROR(input.size() == 2, Base64DecodeFailure);
TK_CHECK_OR_RETURN_ERROR(input.size() == 2, Base64DecodeFailure,
"input length must be 2, got %zu", input.size());

uint32_t val = 0;

Expand All @@ -154,18 +151,13 @@ inline Error decode_2_padding(const std::string_view &input,
} // namespace detail

inline tokenizers::Result<std::string> decode(const std::string_view &input) {
if (input.empty()) {
fprintf(stderr, "empty input");
return Error::Base64DecodeFailure;
}
TK_CHECK_OR_RETURN_ERROR(!input.empty(), Base64DecodeFailure, "empty input");

// Faster than `input.size() % 4`.
if ((input.size() & 3) != 0 || input.size() < 4) {
fprintf(stderr,
"input length must be larger than 4 and is multiple of 4, got %zu",
input.size());
return Error::Base64DecodeFailure;
}
TK_CHECK_OR_RETURN_ERROR(
(input.size() & 3) == 0 && input.size() >= 4, Base64DecodeFailure,
"input length must be larger than 4 and is multiple of 4, got %zu",
input.size());

std::string output;
output.reserve(input.size() / 4 * 3);
Expand Down
89 changes: 81 additions & 8 deletions include/error.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#pragma once

#include "log.h"
#include <stdint.h>

namespace tokenizers {
Expand Down Expand Up @@ -59,11 +60,14 @@ enum class Error : error_code_t {
* TODO: Add logging support
* @param[in] cond__ The condition to be checked, asserted as true.
* @param[in] error__ Error enum value to return without the `Error::` prefix,
* like `InvalidArgument`.
* like `Base64DecodeFailure`.
* @param[in] message__ Format string for the log error message.
* @param[in] ... Optional additional arguments for the format string.
*/
#define TK_CHECK_OR_RETURN_ERROR(cond__, error__) \
#define TK_CHECK_OR_RETURN_ERROR(cond__, error__, message__, ...) \
{ \
if (!(cond__)) { \
TK_LOG(Error, message__, ##__VA_ARGS__); \
return ::tokenizers::Error::error__; \
} \
}
Expand All @@ -72,11 +76,80 @@ enum class Error : error_code_t {
* If error__ is not Error::Ok, return the specified Error
* TODO: Add logging support
* @param[in] error__ Error enum value to return without the `Error::` prefix,
* like `InvalidArgument`.
* like `Base64DecodeFailure`.
* @param[in] ... Optional format string for the log error message and its
* arguments.
*/
#define TK_CHECK_OK_OR_RETURN_ERROR(error__) \
{ \
if (error__ != ::tokenizers::Error::Ok) { \
return error__; \
#define TK_CHECK_OK_OR_RETURN_ERROR(error__, ...) \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR(error__, ##__VA_ARGS__)

// Internal only: Use ET_CHECK_OK_OR_RETURN_ERROR() instead.
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR(...) \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_SELECT(__VA_ARGS__, 10, 9, 8, 7, 6, 5, \
4, 3, 2, 1) \
(__VA_ARGS__)

/**
* Internal only: Use TK_CHECK_OK_OR_RETURN_ERROR() instead.
* This macro selects the correct version of
* TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR based on the number of arguments passed.
* It uses a trick with the preprocessor to count the number of arguments and
* then selects the appropriate macro.
*
* The macro expansion uses __VA_ARGS__ to accept any number of arguments and
* then appends them to TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_, followed by the
* count of arguments. The count is determined by the macro
* TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_SELECT which takes the arguments and
* passes them along with a sequence of numbers (2, 1). The preprocessor then
* matches this sequence to the correct number of arguments provided.
*
* If two arguments are passed, TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2 is
* selected, suitable for cases where an error code and a custom message are
* provided. If only one argument is passed,
* TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_1 is selected, which is used for cases
* with just an error code.
*
* Usage:
* TK_CHECK_OK_OR_RETURN_ERROR(error_code); // Calls v1
* TK_CHECK_OK_OR_RETURN_ERROR(error_code, "Error message", ...); // Calls v2
*/
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_SELECT(_1, _2, _3, _4, _5, _6, \
_7, _8, _9, _10, N, ...) \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_##N

// Internal only: Use ET_CHECK_OK_OR_RETURN_ERROR() instead.
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_1(error__) \
do { \
const auto et_error__ = (error__); \
if (et_error__ != ::tokenizers::Error::Ok) { \
return et_error__; \
} \
}
} while (0)

// Internal only: Use ET_CHECK_OK_OR_RETURN_ERROR() instead.
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2(error__, message__, ...) \
do { \
const auto et_error__ = (error__); \
if (et_error__ != ::tokenizers::Error::Ok) { \
TK_LOG(Error, message__, ##__VA_ARGS__); \
return et_error__; \
} \
} while (0)

// Internal only: Use ET_CHECK_OK_OR_RETURN_ERROR() instead.
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_3 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_4 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_5 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_6 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_7 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_8 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_9 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
#define TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_10 \
TK_INTERNAL_CHECK_OK_OR_RETURN_ERROR_2
Loading