From f4023551346bd9e6f63e0c9811906a7f32453132 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 17 Jul 2024 10:47:39 -0700 Subject: [PATCH 1/6] Update MacOS CI to Arm64 (#1136) --- .github/workflows/ci.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1919e2d48..daaf93d56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,8 +174,8 @@ jobs: echo "Starting to run AppVerifier on all tests found by CTest" python .\aws-c-common\scripts\appverifier_ctest.py --build_directory .\aws-c-common\build\aws-c-common - osx: - runs-on: macos-12 # latest + macos-x64: + runs-on: macos-14-large # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -183,8 +183,17 @@ jobs: chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} - osx-no-cpu-extensions: - runs-on: macos-12 # latest + macos: + runs-on: macos-14 # latest + steps: + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + macos-no-cpu-extensions: + runs-on: macos-14 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -255,8 +264,8 @@ jobs: python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --config Debug - osx-debug: - runs-on: macos-12 # latest + macos-debug: + runs-on: macos-14 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | From 852f8ce17c739cc7c1b5585345ddcd913f78ecc8 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 17 Jul 2024 15:00:01 -0700 Subject: [PATCH 2/6] Update builder to fix macos arm64 CI (#1137) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daaf93d56..0f8b49cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: - 'main' env: - BUILDER_VERSION: v0.9.55 + BUILDER_VERSION: v0.9.62 BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net BUILDER_SOURCE: releases PACKAGE_NAME: aws-c-common From 67601bbbce6ea1c26191f235afc50007a4e796c5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 24 Jul 2024 10:17:15 -0700 Subject: [PATCH 3/6] Bump the minimum stack size to at least 1MB (#1139) --- source/posix/thread.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/source/posix/thread.c b/source/posix/thread.c index af7fac84c..34b5dbe94 100644 --- a/source/posix/thread.c +++ b/source/posix/thread.c @@ -275,6 +275,25 @@ int aws_thread_launch( if (attr_return) { goto cleanup; } + } else if (!options->stack_size) { + /** + * On some systems, the default stack size is too low (128KB on musl at the time of writing this), which can + * cause stack overflow when the dependency chain is long. Increase the stack size to at + * least 1MB, which is the default on Windows. + */ + size_t min_stack_size = (size_t)1 * 1024 * 1024; + size_t current_stack_size; + attr_return = pthread_attr_getstacksize(attributes_ptr, ¤t_stack_size); + if (attr_return) { + goto cleanup; + } + + if (current_stack_size < min_stack_size) { + attr_return = pthread_attr_setstacksize(attributes_ptr, min_stack_size); + if (attr_return) { + goto cleanup; + } + } } /* AFAIK you can't set thread affinity on apple platforms, and it doesn't really matter since all memory From 622853ccc415874f60c67f62b8108556bd8e2273 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Wed, 24 Jul 2024 11:48:22 -0700 Subject: [PATCH 4/6] Run proofs with CBMC 6.1.0 (#1140) --- .github/workflows/proof_ci_resources/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proof_ci_resources/config.yaml b/.github/workflows/proof_ci_resources/config.yaml index 4a362f2db..83407e8d9 100644 --- a/.github/workflows/proof_ci_resources/config.yaml +++ b/.github/workflows/proof_ci_resources/config.yaml @@ -1,7 +1,7 @@ # Use exact versions (instead of "latest") so we're not broken by surprise upgrades. cadical-tag: "rel-2.0.0" # tag of latest release: https://github.com/arminbiere/cadical/releases -cbmc-version: "6.0.0" # semver of latest release: https://github.com/diffblue/cbmc/releases -cbmc-viewer-version: "3.8" # semver of latest release: https://github.com/model-checking/cbmc-viewer/releases +cbmc-version: "6.1.0" # semver of latest release: https://github.com/diffblue/cbmc/releases +cbmc-viewer-version: "3.9" # semver of latest release: https://github.com/model-checking/cbmc-viewer/releases kissat-tag: "rel-3.1.1" # tag of latest release: https://github.com/arminbiere/kissat/releases litani-version: "1.29.0" # semver of latest release: https://github.com/awslabs/aws-build-accumulator/releases proofs-dir: verification/cbmc/proofs From c9ead7594928798964e8c85bd14eab71d01389d5 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 30 Jul 2024 01:29:41 +0300 Subject: [PATCH 5/6] Avoiding allocating a handle in the Windows RNG. (#1046) --- source/windows/device_random.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/source/windows/device_random.c b/source/windows/device_random.c index 6cb92d43e..0398241dd 100644 --- a/source/windows/device_random.c +++ b/source/windows/device_random.c @@ -5,26 +5,11 @@ #include #include -#include #include #include -static BCRYPT_ALG_HANDLE s_alg_handle = NULL; -static aws_thread_once s_rand_init = AWS_THREAD_ONCE_STATIC_INIT; - -static void s_init_rand(void *user_data) { - (void)user_data; - NTSTATUS status = 0; - - status = BCryptOpenAlgorithmProvider(&s_alg_handle, BCRYPT_RNG_ALGORITHM, NULL, 0); - - if (!BCRYPT_SUCCESS(status)) { - abort(); - } -} - int aws_device_random_buffer(struct aws_byte_buf *output) { return aws_device_random_buffer_append(output, output->capacity - output->len); } @@ -32,8 +17,6 @@ int aws_device_random_buffer(struct aws_byte_buf *output) { int aws_device_random_buffer_append(struct aws_byte_buf *output, size_t n) { AWS_PRECONDITION(aws_byte_buf_is_valid(output)); - aws_thread_call_once(&s_rand_init, s_init_rand, NULL); - size_t space_available = output->capacity - output->len; if (space_available < n) { AWS_POSTCONDITION(aws_byte_buf_is_valid(output)); @@ -47,7 +30,8 @@ int aws_device_random_buffer_append(struct aws_byte_buf *output, size_t n) { while (n > 0) { uint32_t capped_n = (uint32_t)aws_min_size(n, UINT32_MAX); - NTSTATUS status = BCryptGenRandom(s_alg_handle, output->buffer + output->len, capped_n, 0 /*flags*/); + NTSTATUS status = + BCryptGenRandom(NULL, output->buffer + output->len, capped_n, BCRYPT_USE_SYSTEM_PREFERRED_RNG); if (!BCRYPT_SUCCESS(status)) { output->len = original_len; From 0a98aa0b44dfb6c0bc07d9fa6811135c9a579eb9 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Tue, 30 Jul 2024 08:32:46 -0700 Subject: [PATCH 6/6] Add no copy api variants to json interface (#1138) --- include/aws/common/json.h | 69 ++++++++++++++++++++++++++++- source/json.c | 91 +++++++++++++++++++++------------------ tests/json_test.c | 19 +++++--- 3 files changed, 130 insertions(+), 49 deletions(-) diff --git a/include/aws/common/json.h b/include/aws/common/json.h index b8c4e6cfe..cfda4cf18 100644 --- a/include/aws/common/json.h +++ b/include/aws/common/json.h @@ -23,13 +23,26 @@ AWS_EXTERN_C_BEGIN * * Note: You will need to free the memory for the aws_json_value using aws_json_destroy on the aws_json_value or * on the object/array containing the aws_json_value. - * @param string A byte pointer to the string you want to store in the aws_json_value + * Note: might be slower than c_str version due to internal copy + * @param string A byte cursor you want to store in the aws_json_value * @param allocator The allocator to use when creating the value * @return A new string aws_json_value */ AWS_COMMON_API struct aws_json_value *aws_json_value_new_string(struct aws_allocator *allocator, struct aws_byte_cursor string); +/** + * Creates a new string aws_json_value with the given string and returns a pointer to it. + * + * Note: You will need to free the memory for the aws_json_value using aws_json_destroy on the aws_json_value or + * on the object/array containing the aws_json_value. + * @param string c string pointer you want to store in the aws_json_value + * @param allocator The allocator to use when creating the value + * @return A new string aws_json_value + */ +AWS_COMMON_API +struct aws_json_value *aws_json_value_new_string_from_c_str(struct aws_allocator *allocator, const char *string); + /** * Creates a new number aws_json_value with the given number and returns a pointer to it. * @@ -129,6 +142,7 @@ int aws_json_value_get_boolean(const struct aws_json_value *value, bool *output) * * Note that the aws_json_value will be destroyed when the aws_json_value object is destroyed * by calling "aws_json_destroy()" + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to add a value to. * @param key The key to add the aws_json_value at. * @param value The aws_json_value you want to add. @@ -142,8 +156,24 @@ int aws_json_value_add_to_object( struct aws_byte_cursor key, struct aws_json_value *value); +/** + * Adds a aws_json_value to a object aws_json_value. + * + * Note that the aws_json_value will be destroyed when the aws_json_value object is destroyed + * by calling "aws_json_destroy()" + * @param object The object aws_json_value you want to add a value to. + * @param key The key to add the aws_json_value at. + * @param value The aws_json_value you want to add. + * @return AWS_OP_SUCCESS if adding was successful. + * Will return AWS_OP_ERROR if the object passed is invalid or if the passed key + * is already in use in the object. + */ +AWS_COMMON_API +int aws_json_value_add_to_object_c_str(struct aws_json_value *object, const char *key, struct aws_json_value *value); + /** * Returns the aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to get the value from. * @param key The key that the aws_json_value is at. Is case sensitive. * @return The aws_json_value at the given key, otherwise NULL. @@ -151,8 +181,20 @@ int aws_json_value_add_to_object( AWS_COMMON_API struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Returns the aws_json_value at the given key. + * Note: same as aws_json_value_get_from_object but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The object aws_json_value you want to get the value from. + * @param key The key that the aws_json_value is at. Is case sensitive. + * @return The aws_json_value at the given key, otherwise NULL. + */ +AWS_COMMON_API +struct aws_json_value *aws_json_value_get_from_object_c_str(const struct aws_json_value *object, const char *key); + /** * Checks if there is a aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The value aws_json_value you want to check a key in. * @param key The key that you want to check. Is case sensitive. * @return True if a aws_json_value is found. @@ -160,8 +202,20 @@ struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_valu AWS_COMMON_API bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Checks if there is a aws_json_value at the given key. + * Note: same as aws_json_value_has_key but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The value aws_json_value you want to check a key in. + * @param key The key that you want to check. Is case sensitive. + * @return True if a aws_json_value is found. + */ +AWS_COMMON_API +bool aws_json_value_has_key_c_str(const struct aws_json_value *object, const char *key); + /** * Removes the aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to remove a aws_json_value in. * @param key The key that the aws_json_value is at. Is case sensitive. * @return AWS_OP_SUCCESS if the aws_json_value was removed. @@ -171,6 +225,19 @@ bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte AWS_COMMON_API int aws_json_value_remove_from_object(struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Removes the aws_json_value at the given key. + * Note: same as aws_json_value_remove_from_object but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The object aws_json_value you want to remove a aws_json_value in. + * @param key The key that the aws_json_value is at. Is case sensitive. + * @return AWS_OP_SUCCESS if the aws_json_value was removed. + * Will return AWS_OP_ERR if the object passed is invalid or if the value + * at the key cannot be found. + */ +AWS_COMMON_API +int aws_json_value_remove_from_object_c_str(struct aws_json_value *object, const char *key); + /** * @brief callback for iterating members of an object * Iteration can be controlled as follows: diff --git a/source/json.c b/source/json.c index 28524a88c..2f1630ea2 100644 --- a/source/json.c +++ b/source/json.c @@ -21,6 +21,12 @@ struct aws_json_value *aws_json_value_new_string(struct aws_allocator *allocator return ret_val; } +struct aws_json_value *aws_json_value_new_string_from_c_str(struct aws_allocator *allocator, const char *string) { + (void)allocator; /* No need for allocator. It is overriden through hooks. */ + void *ret_val = cJSON_CreateString(string); + return ret_val; +} + struct aws_json_value *aws_json_value_new_number(struct aws_allocator *allocator, double number) { (void)allocator; // prevent warnings over unused parameter return (void *)cJSON_CreateNumber(number); @@ -78,92 +84,95 @@ int aws_json_value_add_to_object( struct aws_byte_cursor key, struct aws_json_value *value) { - int result = AWS_OP_ERR; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + int result = aws_json_value_add_to_object_c_str(object, aws_string_c_str(tmp), value); + + aws_string_destroy_secure(tmp); + return result; +} + +int aws_json_value_add_to_object_c_str(struct aws_json_value *object, const char *key, struct aws_json_value *value) { struct cJSON *cjson = (struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } struct cJSON *cjson_value = (struct cJSON *)value; if (cJSON_IsInvalid(cjson_value)) { - result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } - if (cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (cJSON_HasObjectItem(cjson, key)) { + return AWS_OP_ERR; } - cJSON_AddItemToObject(cjson, aws_string_c_str(tmp), cjson_value); - result = AWS_OP_SUCCESS; - -done: - aws_string_destroy_secure(tmp); - return result; + cJSON_AddItemToObject(cjson, key, cjson_value); + return AWS_OP_SUCCESS; } struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_value *object, struct aws_byte_cursor key) { - void *return_value = NULL; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + void *return_value = aws_json_value_get_from_object_c_str(object, aws_string_c_str(tmp)); + aws_string_destroy_secure(tmp); + return return_value; +} + +struct aws_json_value *aws_json_value_get_from_object_c_str(const struct aws_json_value *object, const char *key) { const struct cJSON *cjson = (const struct cJSON *)object; if (!cJSON_IsObject(cjson)) { aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return NULL; } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return NULL; } - return_value = (void *)cJSON_GetObjectItem(cjson, aws_string_c_str(tmp)); - -done: - aws_string_destroy_secure(tmp); - return return_value; + return (void *)cJSON_GetObjectItem(cjson, key); } bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte_cursor key) { struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); - bool result = false; + bool result = aws_json_value_has_key_c_str(object, aws_string_c_str(tmp)); + + aws_string_destroy_secure(tmp); + return result; +} +bool aws_json_value_has_key_c_str(const struct aws_json_value *object, const char *key) { const struct cJSON *cjson = (const struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - goto done; + return false; } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return false; } - result = true; -done: - aws_string_destroy_secure(tmp); - return result; + return true; } int aws_json_value_remove_from_object(struct aws_json_value *object, struct aws_byte_cursor key) { - int result = AWS_OP_ERR; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + int result = aws_json_value_remove_from_object_c_str(object, aws_string_c_str(tmp)); + aws_string_destroy_secure(tmp); + return result; +} + +int aws_json_value_remove_from_object_c_str(struct aws_json_value *object, const char *key) { struct cJSON *cjson = (struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return AWS_OP_ERR; } - cJSON_DeleteItemFromObject(cjson, aws_string_c_str(tmp)); - result = AWS_OP_SUCCESS; - -done: - aws_string_destroy_secure(tmp); - return result; + cJSON_DeleteItemFromObject(cjson, key); + return AWS_OP_SUCCESS; } int aws_json_const_iterate_object( diff --git a/tests/json_test.c b/tests/json_test.c index 8ba2a1c07..ed10fbee6 100644 --- a/tests/json_test.c +++ b/tests/json_test.c @@ -114,6 +114,7 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * // Testing valid array struct aws_json_value *array_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("array")); + ASSERT_PTR_EQUALS(array_node, aws_json_value_get_from_object_c_str(root, "array")); ASSERT_NOT_NULL(array_node); ASSERT_TRUE(aws_json_value_is_array(array_node)); ASSERT_TRUE(aws_json_get_array_size(array_node) == 3); @@ -163,12 +164,14 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * aws_string_destroy_secure(tmp_str); // Testing valid number - struct aws_json_value *number_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("number")); + struct aws_json_value *number_node = aws_json_value_get_from_object_c_str(root, "number"); ASSERT_NOT_NULL(number_node); ASSERT_TRUE(aws_json_value_is_number(number_node)); double double_test_two = 0; aws_json_value_get_number(number_node, &double_test_two); ASSERT_TRUE(double_test_two == (double)123); + ASSERT_TRUE(aws_json_value_has_key_c_str(root, "number")); + ASSERT_TRUE(aws_json_value_has_key(root, aws_byte_cursor_from_c_str("number"))); // Testing valid object struct aws_json_value *object_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("object")); @@ -210,6 +213,12 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * // Test getting invalid type of data ASSERT_INT_EQUALS(aws_json_value_get_number(string_node, NULL), AWS_OP_ERR); + ASSERT_SUCCESS(aws_json_value_remove_from_object(root, aws_byte_cursor_from_c_str("number"))); + ASSERT_FALSE(aws_json_value_has_key_c_str(root, "number")); + + ASSERT_SUCCESS(aws_json_value_remove_from_object_c_str(root, "object")); + ASSERT_FALSE(aws_json_value_has_key_c_str(root, "object")); + aws_json_value_destroy(root); // Make sure that destroying NULL does not have any bad effects. @@ -233,12 +242,8 @@ static int s_test_json_parse_to_string(struct aws_allocator *allocator, void *ct aws_json_value_add_array_element(array, aws_json_value_new_number(allocator, 3)); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("array"), array); - aws_json_value_add_to_object( - root, aws_byte_cursor_from_c_str("boolean"), aws_json_value_new_boolean(allocator, true)); - aws_json_value_add_to_object( - root, - aws_byte_cursor_from_c_str("color"), - aws_json_value_new_string(allocator, aws_byte_cursor_from_c_str("gold"))); + aws_json_value_add_to_object_c_str(root, "boolean", aws_json_value_new_boolean(allocator, true)); + aws_json_value_add_to_object_c_str(root, "color", aws_json_value_new_string_from_c_str(allocator, "gold")); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("null"), aws_json_value_new_null(allocator)); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("number"), aws_json_value_new_number(allocator, 123));