diff --git a/Makefile b/Makefile index 97d481012909..640b59fa9a68 100644 --- a/Makefile +++ b/Makefile @@ -1144,6 +1144,7 @@ CORRECTNESS_TESTS = $(shell ls $(ROOT_DIR)/test/correctness/*.cpp) $(shell ls $( PERFORMANCE_TESTS = $(shell ls $(ROOT_DIR)/test/performance/*.cpp) ERROR_TESTS = $(shell ls $(ROOT_DIR)/test/error/*.cpp) WARNING_TESTS = $(shell ls $(ROOT_DIR)/test/warning/*.cpp) +RUNTIME_TESTS = $(shell ls $(ROOT_DIR)/test/runtime/*.cpp) GENERATOR_EXTERNAL_TESTS := $(shell ls $(ROOT_DIR)/test/generator/*test.cpp) GENERATOR_EXTERNAL_TEST_GENERATOR := $(shell ls $(ROOT_DIR)/test/generator/*_generator.cpp) TUTORIALS = $(filter-out %_generate.cpp, $(shell ls $(ROOT_DIR)/tutorial/*.cpp)) @@ -1153,6 +1154,7 @@ test_correctness: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=quiet_c test_performance: $(PERFORMANCE_TESTS:$(ROOT_DIR)/test/performance/%.cpp=performance_%) test_error: $(ERROR_TESTS:$(ROOT_DIR)/test/error/%.cpp=error_%) test_warning: $(WARNING_TESTS:$(ROOT_DIR)/test/warning/%.cpp=warning_%) +test_runtime: $(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=runtime_%) test_tutorial: $(TUTORIALS:$(ROOT_DIR)/tutorial/%.cpp=tutorial_%) test_valgrind: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=valgrind_%) test_avx512: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=avx512_%) @@ -1239,7 +1241,7 @@ test_generator: $(GENERATOR_AOT_TESTS) $(GENERATOR_AOTCPP_TESTS) $(GENERATOR_JIT $(FILTERS_DIR)/rungen_test $(FILTERS_DIR)/registration_test -ALL_TESTS = test_internal test_correctness test_error test_tutorial test_warning test_generator +ALL_TESTS = test_internal test_correctness test_error test_tutorial test_warning test_runtime test_generator # These targets perform timings of each test. For most tests this includes Halide JIT compile times, and run times. # For generator tests they time the compile time only. The times are recorded in CSV files. @@ -1260,6 +1262,7 @@ build_tests: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=$(BIN_DIR)/c $(PERFORMANCE_TESTS:$(ROOT_DIR)/test/performance/%.cpp=$(BIN_DIR)/performance_%) \ $(ERROR_TESTS:$(ROOT_DIR)/test/error/%.cpp=$(BIN_DIR)/error_%) \ $(WARNING_TESTS:$(ROOT_DIR)/test/warning/%.cpp=$(BIN_DIR)/warning_%) \ + $(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=$(BIN_DIR)/runtime_%) \ $(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_aottest.cpp=$(BIN_DIR)/$(TARGET)/generator_aot_%) \ $(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_jittest.cpp=$(BIN_DIR)/generator_jit_%) \ $(AUTO_SCHEDULE_TESTS:$(ROOT_DIR)/test/auto_schedule/%.cpp=$(BIN_DIR)/auto_schedule_%) @@ -1332,6 +1335,11 @@ $(BIN_DIR)/error_%: $(ROOT_DIR)/test/error/%.cpp $(BIN_DIR)/libHalide.$(SHARED_E $(BIN_DIR)/warning_%: $(ROOT_DIR)/test/warning/%.cpp $(BIN_DIR)/libHalide.$(SHARED_EXT) $(INCLUDE_DIR)/Halide.h $(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@ +# Runtime tests that test internals +RUNTIME_TESTS_CXXFLAGS = -fno-rtti -fno-exceptions -fno-threadsafe-statics -Wno-builtin-declaration-mismatch -DCOMPILING_HALIDE_RUNTIME -DCOMPILING_HALIDE_RUNTIME_TESTS +$(BIN_DIR)/runtime_%: $(ROOT_DIR)/test/runtime/%.cpp $(ROOT_DIR)/test/runtime/common.h + $(CXX) $(TEST_CXX_FLAGS) $(RUNTIME_TESTS_CXXFLAGS) -I$(ROOT_DIR)/test/runtime -I$(ROOT_DIR)/src/runtime $(OPTIMIZE_FOR_BUILD_TIME) $< $(COMMON_LD_FLAGS) -o $@ + # Auto schedule tests that link against libHalide $(BIN_DIR)/auto_schedule_%: $(ROOT_DIR)/test/auto_schedule/%.cpp $(BIN_DIR)/libHalide.$(SHARED_EXT) $(INCLUDE_DIR)/Halide.h $(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@ @@ -1929,6 +1937,11 @@ warning_%: $(BIN_DIR)/warning_% cd $(TMP_DIR) ; $(CURDIR)/$< 2>&1 | egrep --q "^Warning" @-echo +runtime_%: $(BIN_DIR)/runtime_% + @-mkdir -p $(TMP_DIR) + cd $(TMP_DIR) ; $(CURDIR)/$< + @-echo + generator_jit_%: $(BIN_DIR)/generator_jit_% @-mkdir -p $(TMP_DIR) cd $(TMP_DIR) ; $(CURDIR)/$< diff --git a/src/AbstractGenerator.cpp b/src/AbstractGenerator.cpp index 59c5ff097ea0..52bd89553e38 100644 --- a/src/AbstractGenerator.cpp +++ b/src/AbstractGenerator.cpp @@ -30,7 +30,7 @@ Module AbstractGenerator::build_module(const std::string &function_name) { auto_schedule_results = pipeline.auto_schedule(context.target(), context.machine_params()); } #else - const auto asp = context.autoscheduler_params(); + const auto &asp = context.autoscheduler_params(); if (!asp.name.empty()) { debug(1) << "Applying autoscheduler " << asp.name << " to Generator " << name() << " ...\n"; auto_schedule_results = pipeline.apply_autoscheduler(context.target(), asp); @@ -230,7 +230,7 @@ Module AbstractGenerator::build_gradient_module(const std::string &function_name auto_schedule_results = grad_pipeline.auto_schedule(context.target(), context.machine_params()); } #else - const auto asp = context.autoscheduler_params(); + const auto &asp = context.autoscheduler_params(); if (!asp.name.empty()) { auto_schedule_results = grad_pipeline.apply_autoscheduler(context.target(), asp); } diff --git a/src/runtime/internal/block_allocator.h b/src/runtime/internal/block_allocator.h new file mode 100644 index 000000000000..f7f0247e441f --- /dev/null +++ b/src/runtime/internal/block_allocator.h @@ -0,0 +1,478 @@ +#ifndef HALIDE_RUNTIME_BLOCK_ALLOCATOR_H +#define HALIDE_RUNTIME_BLOCK_ALLOCATOR_H + +#include "linked_list.h" +#include "memory_resources.h" +#include "region_allocator.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// -- + +/** Allocator class interface for managing large contiguous blocks + * of memory, which are then sub-allocated into smaller regions of + * memory. This class only manages the address creation for the + * regions -- allocation callback functions are used to request the + * memory from the necessary system or API calls. This class is + * intended to be used inside of a higher level memory management + * class that provides thread safety, policy management and API + * integration for a specific runtime API (eg Vulkan, OpenCL, etc) + */ +class BlockAllocator { +public: + // disable copy constructors and assignment + BlockAllocator(const BlockAllocator &) = delete; + BlockAllocator &operator=(const BlockAllocator &) = delete; + + // disable non-factory based construction + BlockAllocator() = delete; + ~BlockAllocator() = delete; + + // Allocators for the different types of memory we need to allocate + struct MemoryAllocators { + SystemMemoryAllocatorFns system; + MemoryBlockAllocatorFns block; + MemoryRegionAllocatorFns region; + }; + + // Runtime configuration parameters to adjust the behaviour of the block allocator + struct Config { + size_t initial_capacity = 0; + size_t minimum_block_size = 0; + size_t maximum_block_size = 0; + size_t maximum_block_count = 0; + }; + + // Factory methods for creation / destruction + static BlockAllocator *create(void *user_context, const Config &config, const MemoryAllocators &allocators); + static void destroy(void *user_context, BlockAllocator *block_allocator); + + // Public interface methods + MemoryRegion *reserve(void *user_context, const MemoryRequest &request); + void reclaim(void *user_context, MemoryRegion *region); + bool collect(void *user_context); //< returns true if any blocks were removed + void release(void *user_context); + void destroy(void *user_context); + + // Access methods + const MemoryAllocators ¤t_allocators() const; + const Config ¤t_config() const; + const Config &default_config() const; + size_t block_count() const; + +private: + // Linked-list for storing the block resources + typedef LinkedList::EntryType BlockEntry; + + // Initializes a new instance + void initialize(void *user_context, const Config &config, const MemoryAllocators &allocators); + + // Reserves a region of memory using the given allocator for the given block resource, returns nullptr on failure + MemoryRegion *reserve_memory_region(void *user_context, RegionAllocator *allocator, const MemoryRequest &request); + + // Creates a new region allocator for the given block resource + RegionAllocator *create_region_allocator(void *user_context, BlockResource *block); + + // Destroys the given region allocator and all associated memory regions + void destroy_region_allocator(void *user_context, RegionAllocator *region_allocator); + + // Reserves a block of memory for the requested size and returns the corresponding block entry, or nullptr on failure + BlockEntry *reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated); + + // Locates the "best-fit" block entry for the requested size, or nullptr if none was found + BlockEntry *find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated); + + // Creates a new block entry and int the list + BlockEntry *create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated); + + // Releases the block entry from being used, and makes it available for further allocations + void release_block_entry(void *user_context, BlockEntry *block_entry); + + // Destroys the block entry and removes it from the list + void destroy_block_entry(void *user_context, BlockEntry *block_entry); + + // Invokes the allocation callback to allocate memory for the block region + void alloc_memory_block(void *user_context, BlockResource *block); + + // Invokes the deallocation callback to free memory for the memory block + void free_memory_block(void *user_context, BlockResource *block); + + // Returns a constrained size for the requested size based on config parameters + size_t constrain_requested_size(size_t size) const; + + // Returns true if the given block is compatible with the given properties + bool is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const; + + Config config; + LinkedList block_list; + MemoryAllocators allocators; +}; + +BlockAllocator *BlockAllocator::create(void *user_context, const Config &cfg, const MemoryAllocators &allocators) { + halide_abort_if_false(user_context, allocators.system.allocate != nullptr); + BlockAllocator *result = reinterpret_cast( + allocators.system.allocate(user_context, sizeof(BlockAllocator))); + + if (result == nullptr) { + error(user_context) << "BlockAllocator: Failed to create instance! Out of memory!\n"; + return nullptr; + } + + result->initialize(user_context, cfg, allocators); + return result; +} + +void BlockAllocator::destroy(void *user_context, BlockAllocator *instance) { + halide_abort_if_false(user_context, instance != nullptr); + const MemoryAllocators &allocators = instance->allocators; + instance->destroy(user_context); + halide_abort_if_false(user_context, allocators.system.deallocate != nullptr); + allocators.system.deallocate(user_context, instance); +} + +void BlockAllocator::initialize(void *user_context, const Config &cfg, const MemoryAllocators &ma) { + config = cfg; + allocators = ma; + block_list.initialize(user_context, + sizeof(BlockResource), + config.initial_capacity, + allocators.system); +} + +MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &request) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Reserve (" + << "user_context=" << (void *)(user_context) << " " + << "offset=" << (uint32_t)request.offset << " " + << "size=" << (uint32_t)request.size << " " + << "dedicated=" << (request.dedicated ? "true" : "false") << " " + << "usage=" << halide_memory_usage_name(request.properties.usage) << " " + << "caching=" << halide_memory_caching_name(request.properties.caching) << " " + << "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...\n"; +#endif + BlockEntry *block_entry = reserve_block_entry(user_context, request.properties, request.size, request.dedicated); + if (block_entry == nullptr) { + debug(user_context) << "BlockAllocator: Failed to allocate new empty block of requested size (" + << (int32_t)(request.size) << " bytes)!\n"; + return nullptr; + } + + BlockResource *block = static_cast(block_entry->value); + halide_abort_if_false(user_context, block != nullptr); + halide_abort_if_false(user_context, block->allocator != nullptr); + + MemoryRegion *result = reserve_memory_region(user_context, block->allocator, request); + if (result == nullptr) { + + // Unable to reserve region in an existing block ... create a new block and try again. + size_t actual_size = constrain_requested_size(request.size); + block_entry = create_block_entry(user_context, request.properties, actual_size, request.dedicated); + if (block_entry == nullptr) { + debug(user_context) << "BlockAllocator: Out of memory! Failed to allocate empty block of size (" + << (int32_t)(actual_size) << " bytes)!\n"; + return nullptr; + } + + block = static_cast(block_entry->value); + if (block->allocator == nullptr) { + block->allocator = create_region_allocator(user_context, block); + } + + result = reserve_memory_region(user_context, block->allocator, request); + } + return result; +} + +void BlockAllocator::reclaim(void *user_context, MemoryRegion *memory_region) { + halide_abort_if_false(user_context, memory_region != nullptr); + RegionAllocator *allocator = RegionAllocator::find_allocator(user_context, memory_region); + if (allocator == nullptr) { return; } + allocator->reclaim(user_context, memory_region); +} + +bool BlockAllocator::collect(void *user_context) { + bool result = false; + BlockEntry *block_entry = block_list.back(); + while (block_entry != nullptr) { + BlockEntry *prev_entry = block_entry->prev_ptr; + + const BlockResource *block = static_cast(block_entry->value); + if (block->allocator == nullptr) { + block_entry = prev_entry; + continue; + } + + block->allocator->collect(user_context); + if (block->reserved == 0) { + destroy_block_entry(user_context, block_entry); + result = true; + } + + block_entry = prev_entry; + } + return result; +} + +void BlockAllocator::release(void *user_context) { + BlockEntry *block_entry = block_list.back(); + while (block_entry != nullptr) { + BlockEntry *prev_entry = block_entry->prev_ptr; + release_block_entry(user_context, block_entry); + block_entry = prev_entry; + } +} + +void BlockAllocator::destroy(void *user_context) { + BlockEntry *block_entry = block_list.back(); + while (block_entry != nullptr) { + BlockEntry *prev_entry = block_entry->prev_ptr; + destroy_block_entry(user_context, block_entry); + block_entry = prev_entry; + } +} + +MemoryRegion *BlockAllocator::reserve_memory_region(void *user_context, RegionAllocator *allocator, const MemoryRequest &request) { + MemoryRegion *result = allocator->reserve(user_context, request); + if (result == nullptr) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Failed to allocate region of size (" + << (int32_t)(request.size) << " bytes)!\n"; +#endif + // allocator has enough free space, but not enough contiguous space + // -- collect and try to reallocate + if (allocator->collect(user_context)) { + result = allocator->reserve(user_context, request); + } + } + return result; +} + +BlockAllocator::BlockEntry * +BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) { + BlockEntry *block_entry = nullptr; + for (block_entry = block_list.front(); block_entry != nullptr; block_entry = block_entry->next_ptr) { + + const BlockResource *block = static_cast(block_entry->value); + if (!is_compatible_block(block, properties)) { + continue; + } + + // skip blocks that can't be dedicated to a single allocation + if (dedicated && (block->reserved > 0)) { + continue; + } + + // skip dedicated blocks that are already allocated + if (block->memory.dedicated && (block->reserved > 0)) { + continue; + } + + size_t available = (block->memory.size - block->reserved); + if (available >= size) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: find_block_entry (FOUND) (" + << "user_context=" << (void *)(user_context) << " " + << "block_entry=" << (void *)(block_entry) << " " + << "size=" << (uint32_t)size << " " + << "dedicated=" << (dedicated ? "true" : "false") << " " + << "usage=" << halide_memory_usage_name(properties.usage) << " " + << "caching=" << halide_memory_caching_name(properties.caching) << " " + << "visibility=" << halide_memory_visibility_name(properties.visibility) << ") ...\n"; +#endif + break; + } + } + + return block_entry; +} + +BlockAllocator::BlockEntry * +BlockAllocator::reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) { + BlockEntry *block_entry = find_block_entry(user_context, properties, size, dedicated); + if (block_entry == nullptr) { + size_t actual_size = constrain_requested_size(size); + block_entry = create_block_entry(user_context, properties, actual_size, dedicated); + } + + if (block_entry) { + BlockResource *block = static_cast(block_entry->value); + if (block->allocator == nullptr) { + block->allocator = create_region_allocator(user_context, block); + } + } + return block_entry; +} + +RegionAllocator * +BlockAllocator::create_region_allocator(void *user_context, BlockResource *block) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Creating region allocator (" + << "user_context=" << (void *)(user_context) << " " + << "block_resource=" << (void *)(block) << ")...\n"; +#endif + halide_abort_if_false(user_context, block != nullptr); + RegionAllocator *region_allocator = RegionAllocator::create( + user_context, block, {allocators.system, allocators.region}); + + if (region_allocator == nullptr) { + error(user_context) << "BlockAllocator: Failed to create new region allocator!\n"; + return nullptr; + } + + return region_allocator; +} + +void BlockAllocator::destroy_region_allocator(void *user_context, RegionAllocator *region_allocator) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Destroying region allocator (" + << "user_context=" << (void *)(user_context) << " " + << "region_allocator=" << (void *)(region_allocator) << ")...\n"; +#endif + if (region_allocator == nullptr) { return; } + region_allocator->destroy(user_context); + RegionAllocator::destroy(user_context, region_allocator); +} + +BlockAllocator::BlockEntry * +BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) { + if (config.maximum_block_count && (block_count() >= config.maximum_block_count)) { + debug(user_context) << "BlockAllocator: No free blocks found! Maximum block count reached (" + << (int32_t)(config.maximum_block_count) << ")!\n"; + return nullptr; + } + + BlockEntry *block_entry = block_list.append(user_context); + if (block_entry == nullptr) { + debug(user_context) << "BlockAllocator: Failed to allocate new block entry!\n"; + return nullptr; + } + +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Creating block entry (" + << "block_entry=" << (void *)(block_entry) << " " + << "block=" << (void *)(block_entry->value) << " " + << "allocator=" << (void *)(allocators.block.allocate) << ")...\n"; +#endif + + BlockResource *block = static_cast(block_entry->value); + block->memory.size = size; + block->memory.properties = properties; + block->memory.dedicated = dedicated; + block->reserved = 0; + block->allocator = create_region_allocator(user_context, block); + alloc_memory_block(user_context, block); + return block_entry; +} + +void BlockAllocator::release_block_entry(void *user_context, BlockAllocator::BlockEntry *block_entry) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Releasing block entry (" + << "block_entry=" << (void *)(block_entry) << " " + << "block=" << (void *)(block_entry->value) << ")...\n"; +#endif + BlockResource *block = static_cast(block_entry->value); + if (block->allocator) { + block->allocator->release(user_context); + } +} + +void BlockAllocator::destroy_block_entry(void *user_context, BlockAllocator::BlockEntry *block_entry) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Destroying block entry (" + << "block_entry=" << (void *)(block_entry) << " " + << "block=" << (void *)(block_entry->value) << " " + << "deallocator=" << (void *)(allocators.block.deallocate) << ")...\n"; +#endif + BlockResource *block = static_cast(block_entry->value); + if (block->allocator) { + destroy_region_allocator(user_context, block->allocator); + block->allocator = nullptr; + } + free_memory_block(user_context, block); + block_list.remove(user_context, block_entry); +} + +void BlockAllocator::alloc_memory_block(void *user_context, BlockResource *block) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Allocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.allocate << ")...\n"; +#endif + halide_abort_if_false(user_context, allocators.block.allocate != nullptr); + MemoryBlock *memory_block = &(block->memory); + allocators.block.allocate(user_context, memory_block); + block->reserved = 0; +} + +void BlockAllocator::free_memory_block(void *user_context, BlockResource *block) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "BlockAllocator: Deallocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.deallocate << ")...\n"; +#endif + halide_abort_if_false(user_context, allocators.block.deallocate != nullptr); + MemoryBlock *memory_block = &(block->memory); + allocators.block.deallocate(user_context, memory_block); + block->reserved = 0; + block->memory.size = 0; +} + +size_t BlockAllocator::constrain_requested_size(size_t size) const { + size_t actual_size = size; + if (config.minimum_block_size) { + actual_size = ((actual_size < config.minimum_block_size) ? + config.minimum_block_size : + actual_size); + } + if (config.maximum_block_size) { + actual_size = ((actual_size > config.maximum_block_size) ? + config.maximum_block_size : + actual_size); + } + return actual_size; +} + +bool BlockAllocator::is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const { + if (properties.caching != MemoryCaching::DefaultCaching) { + if (properties.caching != block->memory.properties.caching) { + return false; + } + } + + if (properties.visibility != MemoryVisibility::DefaultVisibility) { + if (properties.visibility != block->memory.properties.visibility) { + return false; + } + } + + if (properties.usage != MemoryUsage::DefaultUsage) { + if (properties.usage != block->memory.properties.usage) { + return false; + } + } + + return true; +} + +const BlockAllocator::MemoryAllocators &BlockAllocator::current_allocators() const { + return allocators; +} + +const BlockAllocator::Config &BlockAllocator::current_config() const { + return config; +} + +const BlockAllocator::Config &BlockAllocator::default_config() const { + static Config result; + return result; +} + +size_t BlockAllocator::block_count() const { + return block_list.size(); +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_BLOCK_ALLOCATOR_H diff --git a/src/runtime/internal/block_storage.h b/src/runtime/internal/block_storage.h new file mode 100644 index 000000000000..648f10a84846 --- /dev/null +++ b/src/runtime/internal/block_storage.h @@ -0,0 +1,425 @@ +#ifndef HALIDE_RUNTIME_BLOCK_STORAGE_H +#define HALIDE_RUNTIME_BLOCK_STORAGE_H + +#include "memory_resources.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// Dynamically resizable array for block storage (eg plain old data) +// -- No usage of constructors/destructors for value type +// -- Assumes all elements stored are uniformly the same fixed size +// -- Allocations are done in blocks of a fixed size +// -- Implementation uses memcpy/memmove for copying +// -- Customizable allocator ... default uses NativeSystemAllocator +class BlockStorage { +public: + static constexpr size_t default_capacity = 32; // smallish + + // Configurable parameters + struct Config { + uint32_t entry_size = 1; // bytes per entry + uint32_t block_size = 32; // bytes per each allocation block + uint32_t minimum_capacity = default_capacity; + }; + + BlockStorage(void *user_context, const Config &cfg, const SystemMemoryAllocatorFns &sma = default_allocator()); + BlockStorage(const BlockStorage &other); + ~BlockStorage(); + + void initialize(void *user_context, const Config &cfg, const SystemMemoryAllocatorFns &sma = default_allocator()); + + BlockStorage &operator=(const BlockStorage &other); + bool operator==(const BlockStorage &other) const; + bool operator!=(const BlockStorage &other) const; + + void reserve(void *user_context, size_t capacity, bool free_existing = false); + void resize(void *user_context, size_t entry_count, bool realloc = true); + + void assign(void *user_context, size_t index, const void *entry_ptr); + void insert(void *user_context, size_t index, const void *entry_ptr); + void prepend(void *user_context, const void *entry_ptr); + void append(void *user_context, const void *entry_ptr); + void remove(void *user_context, size_t index); + + void fill(void *user_context, const void *array, size_t array_size); + void insert(void *user_context, size_t index, const void *array, size_t array_size); + void replace(void *user_context, size_t index, const void *array, size_t array_size); + void prepend(void *user_context, const void *array, size_t array_size); + void append(void *user_context, const void *array, size_t array_size); + void remove(void *user_context, size_t index, size_t entry_count); + + void pop_front(void *user_context); + void pop_back(void *user_context); + void shrink_to_fit(void *user_context); + void clear(void *user_context); + void destroy(void *user_context); + + bool empty() const; + size_t stride() const; + size_t size() const; + + void *operator[](size_t index); ///< logical entry index (returns ptr = data() + (index * stride()) + const void *operator[](size_t index) const; + + void *data(); + void *front(); + void *back(); + + const void *data() const; + const void *front() const; + const void *back() const; + + const Config ¤t_config() const; + static const Config &default_config(); + + const SystemMemoryAllocatorFns ¤t_allocator() const; + static const SystemMemoryAllocatorFns &default_allocator(); + +private: + void allocate(void *user_context, size_t capacity); + + void *ptr = nullptr; + size_t count = 0; + size_t capacity = 0; + Config config; + SystemMemoryAllocatorFns allocator; +}; + +BlockStorage::BlockStorage(void *user_context, const Config &cfg, const SystemMemoryAllocatorFns &sma) + : config(cfg), allocator(sma) { + halide_abort_if_false(user_context, config.entry_size != 0); + halide_abort_if_false(user_context, allocator.allocate != nullptr); + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + if (config.minimum_capacity) { + reserve(user_context, config.minimum_capacity); + } +} + +BlockStorage::BlockStorage(const BlockStorage &other) + : BlockStorage(nullptr, other.config, other.allocator) { + if (other.count) { + resize(nullptr, other.count); + memcpy(this->ptr, other.ptr, count * config.entry_size); + } +} + +BlockStorage::~BlockStorage() { + destroy(nullptr); +} + +void BlockStorage::destroy(void *user_context) { + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + if (ptr != nullptr) { + allocator.deallocate(user_context, ptr); + } + capacity = count = 0; + ptr = nullptr; +} + +void BlockStorage::initialize(void *user_context, const Config &cfg, const SystemMemoryAllocatorFns &sma) { + allocator = sma; + config = cfg; + capacity = count = 0; + ptr = nullptr; + if (config.minimum_capacity) { + reserve(user_context, config.minimum_capacity); + } +} + +BlockStorage &BlockStorage::operator=(const BlockStorage &other) { + if (&other != this) { + config = other.config; + resize(nullptr, other.count); + if (count != 0 && other.ptr != nullptr) { + memcpy(ptr, other.ptr, count * config.entry_size); + } + } + return *this; +} + +bool BlockStorage::operator==(const BlockStorage &other) const { + if (config.entry_size != other.config.entry_size) { return false; } + if (count != other.count) { return false; } + return memcmp(this->ptr, other.ptr, this->size() * config.entry_size) == 0; +} + +bool BlockStorage::operator!=(const BlockStorage &other) const { + return !(*this == other); +} + +void BlockStorage::fill(void *user_context, const void *array, size_t array_size) { + if (array_size != 0) { + resize(user_context, array_size); + memcpy(this->ptr, array, array_size * config.entry_size); + count = array_size; + } +} + +void BlockStorage::assign(void *user_context, size_t index, const void *entry_ptr) { + replace(user_context, index, entry_ptr, 1); +} + +void BlockStorage::prepend(void *user_context, const void *entry_ptr) { + insert(user_context, 0, entry_ptr, 1); +} + +void BlockStorage::append(void *user_context, const void *entry_ptr) { + append(user_context, entry_ptr, 1); +} + +void BlockStorage::pop_front(void *user_context) { + halide_debug_assert(user_context, count > 0); + remove(user_context, 0); +} + +void BlockStorage::pop_back(void *user_context) { + halide_debug_assert(user_context, count > 0); + resize(user_context, size() - 1); +} + +void BlockStorage::clear(void *user_context) { + resize(user_context, 0); +} + +void BlockStorage::reserve(void *user_context, size_t new_capacity, bool free_existing) { + new_capacity = max(new_capacity, count); + + if ((new_capacity < capacity) && !free_existing) { + new_capacity = capacity; + } + + allocate(user_context, new_capacity); +} + +void BlockStorage::resize(void *user_context, size_t entry_count, bool realloc) { + size_t current_size = capacity; + size_t requested_size = entry_count; + size_t minimum_size = config.minimum_capacity; + size_t actual_size = current_size; + count = requested_size; + + // increase capacity upto 1.5x existing (or at least min_capacity) + if (requested_size > current_size) { + actual_size = max(requested_size, max(current_size * 3 / 2, minimum_size)); + } else if (!realloc) { + return; + } + +#if DEBUG + debug(user_context) << "BlockStorage: Resize (" + << "requested_size=" << (int32_t)requested_size << " " + << "current_size=" << (int32_t)current_size << " " + << "minimum_size=" << (int32_t)minimum_size << " " + << "actual_size=" << (int32_t)actual_size << " " + << "entry_size=" << (int32_t)config.entry_size << " " + << "realloc=" << (realloc ? "true" : "false") << ")...\n"; +#endif + + allocate(user_context, actual_size); +} + +void BlockStorage::shrink_to_fit(void *user_context) { + if (capacity > count) { + void *new_ptr = nullptr; + if (count > 0) { + size_t actual_bytes = count * config.entry_size; + new_ptr = allocator.allocate(user_context, actual_bytes); + memcpy(new_ptr, ptr, actual_bytes); + } + allocator.deallocate(user_context, ptr); + capacity = count; + ptr = new_ptr; + } +} + +void BlockStorage::insert(void *user_context, size_t index, const void *entry_ptr) { + insert(user_context, index, entry_ptr, 1); +} + +void BlockStorage::remove(void *user_context, size_t index) { + remove(user_context, index, 1); +} + +void BlockStorage::remove(void *user_context, size_t index, size_t entry_count) { + halide_debug_assert(user_context, index < count); + const size_t last_index = size(); + if (index < (last_index - entry_count)) { + size_t dst_offset = index * config.entry_size; + size_t src_offset = (index + entry_count) * config.entry_size; + size_t bytes = (last_index - index - entry_count) * config.entry_size; + +#if DEBUG + debug(0) << "BlockStorage: Remove (" + << "index=" << (int32_t)index << " " + << "entry_count=" << (int32_t)entry_count << " " + << "entry_size=" << (int32_t)config.entry_size << " " + << "last_index=" << (int32_t)last_index << " " + << "src_offset=" << (int32_t)src_offset << " " + << "dst_offset=" << (int32_t)dst_offset << " " + << "bytes=" << (int32_t)bytes << ")...\n"; +#endif + void *dst_ptr = offset_address(ptr, dst_offset); + void *src_ptr = offset_address(ptr, src_offset); + memmove(dst_ptr, src_ptr, bytes); + } + resize(user_context, last_index - entry_count); +} + +void BlockStorage::replace(void *user_context, size_t index, const void *array, size_t array_size) { + halide_debug_assert(user_context, index < count); + size_t offset = index * config.entry_size; + size_t remaining = count - index; + +#if DEBUG + debug(0) << "BlockStorage: Replace (" + << "index=" << (int32_t)index << " " + << "array_size=" << (int32_t)array_size << " " + << "entry_size=" << (int32_t)config.entry_size << " " + << "offset=" << (int32_t)offset << " " + << "remaining=" << (int32_t)remaining << " " + << "capacity=" << (int32_t)capacity << ")...\n"; +#endif + + halide_debug_assert(user_context, remaining > 0); + size_t copy_count = min(remaining, array_size); + void *dst_ptr = offset_address(ptr, offset); + memcpy(dst_ptr, array, copy_count * config.entry_size); + count = max(count, index + copy_count); +} + +void BlockStorage::insert(void *user_context, size_t index, const void *array, size_t array_size) { + halide_debug_assert(user_context, index <= count); + const size_t last_index = size(); + resize(user_context, last_index + array_size); + if (index < last_index) { + size_t src_offset = index * config.entry_size; + size_t dst_offset = (index + array_size) * config.entry_size; + size_t bytes = (last_index - index) * config.entry_size; + void *src_ptr = offset_address(ptr, src_offset); + void *dst_ptr = offset_address(ptr, dst_offset); + memmove(dst_ptr, src_ptr, bytes); + } + replace(user_context, index, array, array_size); +} + +void BlockStorage::prepend(void *user_context, const void *array, size_t array_size) { + insert(user_context, 0, array, array_size); +} + +void BlockStorage::append(void *user_context, const void *array, size_t array_size) { + const size_t last_index = size(); + insert(user_context, last_index, array, array_size); +} + +bool BlockStorage::empty() const { + return count == 0; +} + +size_t BlockStorage::size() const { + return count; +} + +size_t BlockStorage::stride() const { + return config.entry_size; +} + +void *BlockStorage::operator[](size_t index) { + halide_debug_assert(nullptr, index < capacity); + return offset_address(ptr, index * config.entry_size); +} + +const void *BlockStorage::operator[](size_t index) const { + halide_debug_assert(nullptr, index < capacity); + return offset_address(ptr, index * config.entry_size); +} + +void *BlockStorage::data() { + return ptr; +} + +void *BlockStorage::front() { + halide_debug_assert(nullptr, count > 0); + return ptr; +} + +void *BlockStorage::back() { + halide_debug_assert(nullptr, count > 0); + size_t index = count - 1; + return offset_address(ptr, index * config.entry_size); +} + +const void *BlockStorage::data() const { + return ptr; +} + +const void *BlockStorage::front() const { + halide_debug_assert(nullptr, count > 0); + return ptr; +} + +const void *BlockStorage::back() const { + halide_debug_assert(nullptr, count > 0); + size_t index = count - 1; + return offset_address(ptr, index * config.entry_size); +} + +void BlockStorage::allocate(void *user_context, size_t new_capacity) { + if (new_capacity != capacity) { + halide_abort_if_false(user_context, allocator.allocate != nullptr); + size_t requested_bytes = new_capacity * config.entry_size; + size_t block_size = max(config.block_size, config.entry_size); + size_t block_count = (requested_bytes / block_size); + block_count += (requested_bytes % block_size) ? 1 : 0; + size_t alloc_size = block_count * block_size; +#if DEBUG + debug(0) << "BlockStorage: Allocating (" + << "requested_bytes=" << (int32_t)requested_bytes << " " + << "block_size=" << (int32_t)block_size << " " + << "block_count=" << (int32_t)block_count << " " + << "alloc_size=" << (int32_t)alloc_size << ") ...\n"; +#endif + void *new_ptr = alloc_size ? allocator.allocate(user_context, alloc_size) : nullptr; + if (count != 0 && ptr != nullptr && new_ptr != nullptr) { + memcpy(new_ptr, ptr, count * config.entry_size); + } + if (ptr != nullptr) { + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + allocator.deallocate(user_context, ptr); + } + capacity = new_capacity; + ptr = new_ptr; + } +} + +const SystemMemoryAllocatorFns & +BlockStorage::current_allocator() const { + return this->allocator; +} + +const BlockStorage::Config & +BlockStorage::default_config() { + static Config default_cfg; + return default_cfg; +} + +const BlockStorage::Config & +BlockStorage::current_config() const { + return this->config; +} + +const SystemMemoryAllocatorFns & +BlockStorage::default_allocator() { + static SystemMemoryAllocatorFns native_allocator = { + native_system_malloc, native_system_free}; + return native_allocator; +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_BLOCK_STORAGE_H diff --git a/src/runtime/internal/linked_list.h b/src/runtime/internal/linked_list.h new file mode 100644 index 000000000000..dea22c13285e --- /dev/null +++ b/src/runtime/internal/linked_list.h @@ -0,0 +1,333 @@ +#ifndef HALIDE_RUNTIME_LINKED_LIST_H +#define HALIDE_RUNTIME_LINKED_LIST_H + +#include "memory_arena.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// Doubly linked list container +// -- Implemented using MemoryArena for allocation +class LinkedList { +public: + // Disable copy support + LinkedList(const LinkedList &) = delete; + LinkedList &operator=(const LinkedList &) = delete; + + // Default initial capacity + static constexpr uint32_t default_capacity = uint32_t(32); // smallish + + // List entry + struct EntryType { + void *value = nullptr; + EntryType *prev_ptr = nullptr; + EntryType *next_ptr = nullptr; + }; + + LinkedList(void *user_context, uint32_t entry_size, uint32_t capacity = default_capacity, + const SystemMemoryAllocatorFns &allocator = default_allocator()); + ~LinkedList(); + + void initialize(void *user_context, uint32_t entry_size, uint32_t capacity = default_capacity, + const SystemMemoryAllocatorFns &allocator = default_allocator()); + + EntryType *front(); + EntryType *back(); + + const EntryType *front() const; + const EntryType *back() const; + + EntryType *prepend(void *user_context); + EntryType *prepend(void *user_context, const void *value); + + EntryType *append(void *user_context); + EntryType *append(void *user_context, const void *value); + + void pop_front(void *user_context); + void pop_back(void *user_context); + + EntryType *insert_before(void *user_context, EntryType *entry_ptr); + EntryType *insert_before(void *user_context, EntryType *entry_ptr, const void *value); + + EntryType *insert_after(void *user_context, EntryType *entry_ptr); + EntryType *insert_after(void *user_context, EntryType *entry_ptr, const void *value); + + void remove(void *user_context, EntryType *entry_ptr); + void clear(void *user_context); + void destroy(void *user_context); + + size_t size() const; + bool empty() const; + + const SystemMemoryAllocatorFns ¤t_allocator() const; + static const SystemMemoryAllocatorFns &default_allocator(); + +private: + EntryType *reserve(void *user_context); + void reclaim(void *user_context, EntryType *entry_ptr); + + MemoryArena *link_arena = nullptr; + MemoryArena *data_arena = nullptr; + EntryType *front_ptr = nullptr; + EntryType *back_ptr = nullptr; + size_t entry_count = 0; +}; + +LinkedList::LinkedList(void *user_context, uint32_t entry_size, uint32_t capacity, + const SystemMemoryAllocatorFns &sma) { + uint32_t arena_capacity = max(capacity, MemoryArena::default_capacity); + link_arena = MemoryArena::create(user_context, {sizeof(EntryType), arena_capacity, 0}, sma); + data_arena = MemoryArena::create(user_context, {entry_size, arena_capacity, 0}, sma); + front_ptr = nullptr; + back_ptr = nullptr; + entry_count = 0; +} + +LinkedList::~LinkedList() { + destroy(nullptr); +} + +void LinkedList::initialize(void *user_context, uint32_t entry_size, uint32_t capacity, + const SystemMemoryAllocatorFns &sma) { + uint32_t arena_capacity = max(capacity, MemoryArena::default_capacity); + link_arena = MemoryArena::create(user_context, {sizeof(EntryType), arena_capacity, 0}, sma); + data_arena = MemoryArena::create(user_context, {entry_size, arena_capacity, 0}, sma); + front_ptr = nullptr; + back_ptr = nullptr; + entry_count = 0; +} + +void LinkedList::destroy(void *user_context) { + clear(nullptr); + if (link_arena) { MemoryArena::destroy(nullptr, link_arena); } + if (data_arena) { MemoryArena::destroy(nullptr, data_arena); } + link_arena = nullptr; + data_arena = nullptr; + front_ptr = nullptr; + back_ptr = nullptr; + entry_count = 0; +} + +typename LinkedList::EntryType *LinkedList::front() { + return front_ptr; +} + +typename LinkedList::EntryType *LinkedList::back() { + return back_ptr; +} + +const typename LinkedList::EntryType *LinkedList::front() const { + return front_ptr; +} + +const typename LinkedList::EntryType *LinkedList::back() const { + return back_ptr; +} + +typename LinkedList::EntryType * +LinkedList::prepend(void *user_context) { + EntryType *entry_ptr = reserve(user_context); + if (empty()) { + front_ptr = entry_ptr; + back_ptr = entry_ptr; + entry_count = 1; + } else { + entry_ptr->next_ptr = front_ptr; + front_ptr->prev_ptr = entry_ptr; + front_ptr = entry_ptr; + ++entry_count; + } + return entry_ptr; +} + +typename LinkedList::EntryType * +LinkedList::append(void *user_context) { + EntryType *entry_ptr = reserve(user_context); + if (empty()) { + front_ptr = entry_ptr; + back_ptr = entry_ptr; + entry_count = 1; + } else { + entry_ptr->prev_ptr = back_ptr; + back_ptr->next_ptr = entry_ptr; + back_ptr = entry_ptr; + ++entry_count; + } + return entry_ptr; +} + +typename LinkedList::EntryType * +LinkedList::prepend(void *user_context, const void *value) { + EntryType *entry_ptr = prepend(user_context); + memcpy(entry_ptr->value, value, data_arena->current_config().entry_size); + return entry_ptr; +} + +typename LinkedList::EntryType * +LinkedList::append(void *user_context, const void *value) { + EntryType *entry_ptr = append(user_context); + memcpy(entry_ptr->value, value, data_arena->current_config().entry_size); + return entry_ptr; +} + +void LinkedList::pop_front(void *user_context) { + halide_abort_if_false(user_context, (entry_count > 0)); + EntryType *remove_ptr = front_ptr; + EntryType *next_ptr = remove_ptr->next_ptr; + if (next_ptr != nullptr) { + next_ptr->prev_ptr = nullptr; + } + front_ptr = next_ptr; + reclaim(user_context, remove_ptr); + --entry_count; +} + +void LinkedList::pop_back(void *user_context) { + halide_abort_if_false(user_context, (entry_count > 0)); + EntryType *remove_ptr = back_ptr; + EntryType *prev_ptr = remove_ptr->prev_ptr; + if (prev_ptr != nullptr) { + prev_ptr->next_ptr = nullptr; + } + back_ptr = prev_ptr; + reclaim(user_context, remove_ptr); + --entry_count; +} + +void LinkedList::clear(void *user_context) { + if (empty() == false) { + EntryType *remove_ptr = back_ptr; + while (remove_ptr != nullptr) { + EntryType *prev_ptr = remove_ptr->prev_ptr; + reclaim(user_context, remove_ptr); + remove_ptr = prev_ptr; + } + front_ptr = nullptr; + back_ptr = nullptr; + entry_count = 0; + } +} + +void LinkedList::remove(void *user_context, EntryType *entry_ptr) { + halide_abort_if_false(user_context, (entry_ptr != nullptr)); + halide_abort_if_false(user_context, (entry_count > 0)); + + if (entry_ptr->prev_ptr != nullptr) { + entry_ptr->prev_ptr->next_ptr = entry_ptr->next_ptr; + } else { + halide_abort_if_false(user_context, (front_ptr == entry_ptr)); + front_ptr = entry_ptr->next_ptr; + } + + if (entry_ptr->next_ptr != nullptr) { + entry_ptr->next_ptr->prev_ptr = entry_ptr->prev_ptr; + } else { + halide_abort_if_false(user_context, (back_ptr == entry_ptr)); + back_ptr = entry_ptr->prev_ptr; + } + + reclaim(user_context, entry_ptr); + --entry_count; +} + +typename LinkedList::EntryType * +LinkedList::insert_before(void *user_context, EntryType *entry_ptr) { + if (entry_ptr != nullptr) { + EntryType *prev_ptr = entry_ptr->prev_ptr; + EntryType *new_ptr = reserve(user_context); + new_ptr->prev_ptr = prev_ptr; + new_ptr->next_ptr = entry_ptr; + entry_ptr->prev_ptr = new_ptr; + if (prev_ptr != nullptr) { + prev_ptr->next_ptr = new_ptr; + } else { + halide_abort_if_false(user_context, (front_ptr == entry_ptr)); + front_ptr = new_ptr; + } + ++entry_count; + return new_ptr; + } else { + return append(user_context); + } +} + +typename LinkedList::EntryType * +LinkedList::insert_after(void *user_context, EntryType *entry_ptr) { + if (entry_ptr != nullptr) { + EntryType *next_ptr = entry_ptr->next_ptr; + EntryType *new_ptr = reserve(user_context); + new_ptr->next_ptr = next_ptr; + new_ptr->prev_ptr = entry_ptr; + entry_ptr->next_ptr = new_ptr; + if (next_ptr != nullptr) { + next_ptr->prev_ptr = new_ptr; + } else { + halide_abort_if_false(user_context, (back_ptr == entry_ptr)); + back_ptr = new_ptr; + } + ++entry_count; + return new_ptr; + } else { + return prepend(user_context); + } +} + +typename LinkedList::EntryType * +LinkedList::insert_before(void *user_context, EntryType *entry_ptr, const void *value) { + EntryType *new_ptr = insert_before(user_context, entry_ptr); + memcpy(new_ptr->value, value, data_arena->current_config().entry_size); + return new_ptr; +} + +typename LinkedList::EntryType * +LinkedList::insert_after(void *user_context, EntryType *entry_ptr, const void *value) { + EntryType *new_ptr = insert_after(user_context, entry_ptr); + memcpy(new_ptr->value, value, data_arena->current_config().entry_size); + return new_ptr; +} + +size_t LinkedList::size() const { + return entry_count; +} + +bool LinkedList::empty() const { + return entry_count == 0; +} + +const SystemMemoryAllocatorFns & +LinkedList::current_allocator() const { + return link_arena->current_allocator(); +} + +const SystemMemoryAllocatorFns & +LinkedList::default_allocator() { + return MemoryArena::default_allocator(); +} + +typename LinkedList::EntryType * +LinkedList::reserve(void *user_context) { + EntryType *entry_ptr = static_cast( + link_arena->reserve(user_context, true)); + entry_ptr->value = data_arena->reserve(user_context, true); + entry_ptr->next_ptr = nullptr; + entry_ptr->prev_ptr = nullptr; + return entry_ptr; +} + +void LinkedList::reclaim(void *user_context, EntryType *entry_ptr) { + void *value_ptr = entry_ptr->value; + entry_ptr->value = nullptr; + entry_ptr->next_ptr = nullptr; + entry_ptr->prev_ptr = nullptr; + data_arena->reclaim(user_context, value_ptr); + link_arena->reclaim(user_context, entry_ptr); +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_LINKED_LIST_H diff --git a/src/runtime/internal/memory_arena.h b/src/runtime/internal/memory_arena.h new file mode 100644 index 000000000000..27c3d871dccf --- /dev/null +++ b/src/runtime/internal/memory_arena.h @@ -0,0 +1,310 @@ +#ifndef HALIDE_RUNTIME_MEMORY_ARENA_H +#define HALIDE_RUNTIME_MEMORY_ARENA_H + +#include "block_storage.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// -- +// Memory Arena class for region based allocations and caching of same-type data +// -- Implementation uses block_storage, and internally manages lists of allocated entries +// -- Customizable allocator (defaults to BlockStorage::default_allocator()) +// -- Not thread safe ... locking must be done by client +// +class MemoryArena { +public: + // Disable copy constructors and assignment + MemoryArena(const MemoryArena &) = delete; + MemoryArena &operator=(const MemoryArena &) = delete; + + // Default initial capacity + static constexpr uint32_t default_capacity = uint32_t(32); // smallish + + // Configurable parameters + struct Config { + uint32_t entry_size = 1; + uint32_t minimum_block_capacity = default_capacity; + uint32_t maximum_block_count = 0; + }; + + MemoryArena(void *user_context, const Config &config = default_config(), + const SystemMemoryAllocatorFns &allocator = default_allocator()); + + ~MemoryArena(); + + // Factory methods for creation / destruction + static MemoryArena *create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator = default_allocator()); + static void destroy(void *user_context, MemoryArena *arena); + + // Initialize a newly created instance + void initialize(void *user_context, const Config &config, + const SystemMemoryAllocatorFns &allocator = default_allocator()); + + // Public interface methods + void *reserve(void *user_context, bool initialize = false); + void reclaim(void *user_context, void *ptr); + bool collect(void *user_context); //< returns true if any blocks were removed + void destroy(void *user_context); + + // Access methods + const Config ¤t_config() const; + static const Config &default_config(); + + const SystemMemoryAllocatorFns ¤t_allocator() const; + static const SystemMemoryAllocatorFns &default_allocator(); + +private: + // Sentinal invalid entry value + static const uint32_t invalid_entry = uint32_t(-1); + + // Each block contains: + // - an array of entries + // - an array of indices (for the free list) + // - an array of status flags (indicating usage) + // - free index points to next available entry for the block (or invalid_entry if block is full) + struct Block { + void *entries = nullptr; + uint32_t *indices = nullptr; + AllocationStatus *status = nullptr; + uint32_t capacity = 0; + uint32_t free_index = 0; + }; + + Block *create_block(void *user_context); + bool collect_block(void *user_context, Block *block); //< returns true if any blocks were removed + void destroy_block(void *user_context, Block *block); + Block *lookup_block(void *user_context, uint32_t index); + + void *create_entry(void *user_context, Block *block, uint32_t index); + void destroy_entry(void *user_context, Block *block, uint32_t index); + void *lookup_entry(void *user_context, Block *block, uint32_t index); + + Config config; + BlockStorage blocks; +}; + +MemoryArena::MemoryArena(void *user_context, + const Config &cfg, + const SystemMemoryAllocatorFns &alloc) + : config(cfg), + blocks(user_context, {sizeof(MemoryArena::Block), 32, 32}, alloc) { + halide_debug_assert(user_context, config.minimum_block_capacity > 1); +} + +MemoryArena::~MemoryArena() { + destroy(nullptr); +} + +MemoryArena *MemoryArena::create(void *user_context, const Config &cfg, const SystemMemoryAllocatorFns &system_allocator) { + halide_abort_if_false(user_context, system_allocator.allocate != nullptr); + MemoryArena *result = reinterpret_cast( + system_allocator.allocate(user_context, sizeof(MemoryArena))); + + if (result == nullptr) { + halide_error(user_context, "MemoryArena: Failed to create instance! Out of memory!\n"); + return nullptr; + } + + result->initialize(user_context, cfg, system_allocator); + return result; +} + +void MemoryArena::destroy(void *user_context, MemoryArena *instance) { + halide_abort_if_false(user_context, instance != nullptr); + const SystemMemoryAllocatorFns &system_allocator = instance->blocks.current_allocator(); + instance->destroy(user_context); + halide_abort_if_false(user_context, system_allocator.deallocate != nullptr); + system_allocator.deallocate(user_context, instance); +} + +void MemoryArena::initialize(void *user_context, + const Config &cfg, + const SystemMemoryAllocatorFns &system_allocator) { + config = cfg; + blocks.initialize(user_context, {sizeof(MemoryArena::Block), 32, 32}, system_allocator); + halide_debug_assert(user_context, config.minimum_block_capacity > 1); +} + +void MemoryArena::destroy(void *user_context) { + for (size_t i = blocks.size(); i--;) { + Block *block = lookup_block(user_context, i); + halide_abort_if_false(user_context, block != nullptr); + destroy_block(user_context, block); + } + blocks.destroy(user_context); +} + +bool MemoryArena::collect(void *user_context) { + bool result = false; + for (size_t i = blocks.size(); i--;) { + Block *block = lookup_block(user_context, i); + halide_abort_if_false(user_context, block != nullptr); + if (collect_block(user_context, block)) { + blocks.remove(user_context, i); + result = true; + } + } + return result; +} + +void *MemoryArena::reserve(void *user_context, bool initialize) { + // Scan blocks for a free entry + for (size_t i = blocks.size(); i--;) { + Block *block = lookup_block(user_context, i); + halide_abort_if_false(user_context, block != nullptr); + if (block->free_index != invalid_entry) { + return create_entry(user_context, block, block->free_index); + } + } + + if (config.maximum_block_count && (blocks.size() >= config.maximum_block_count)) { + halide_error(user_context, "MemoryArena: Failed to reserve new entry! Maxmimum blocks reached!\n"); + return nullptr; + } + + // All blocks full ... create a new one + uint32_t index = 0; + Block *block = create_block(user_context); + void *entry_ptr = create_entry(user_context, block, index); + + // Optionally clear the allocation if requested + if (initialize) { + memset(entry_ptr, 0, config.entry_size); + } + return entry_ptr; +} + +void MemoryArena::reclaim(void *user_context, void *entry_ptr) { + for (size_t i = blocks.size(); i--;) { + Block *block = lookup_block(user_context, i); + halide_abort_if_false(user_context, block != nullptr); + + // is entry_ptr in the address range of this block. + uint8_t *offset_ptr = static_cast(entry_ptr); + uint8_t *base_ptr = static_cast(block->entries); + uint8_t *end_ptr = static_cast(offset_address(block->entries, block->capacity * config.entry_size)); + if ((entry_ptr >= base_ptr) && (entry_ptr < end_ptr)) { + const uint32_t offset = static_cast(offset_ptr - base_ptr); + const uint32_t index = offset / config.entry_size; + destroy_entry(user_context, block, index); + return; + } + } + halide_error(user_context, "MemoryArena: Pointer address doesn't belong to this memory pool!\n"); +} + +typename MemoryArena::Block *MemoryArena::create_block(void *user_context) { + // resize capacity starting with initial up to 1.5 last capacity + uint32_t new_capacity = config.minimum_block_capacity; + if (!blocks.empty()) { + const Block *last_block = static_cast(blocks.back()); + new_capacity = (last_block->capacity * 3 / 2); + } + + halide_abort_if_false(user_context, current_allocator().allocate != nullptr); + void *new_entries = current_allocator().allocate(user_context, config.entry_size * new_capacity); + memset(new_entries, 0, config.entry_size * new_capacity); + + uint32_t *new_indices = (uint32_t *)current_allocator().allocate(user_context, sizeof(uint32_t) * new_capacity); + AllocationStatus *new_status = (AllocationStatus *)current_allocator().allocate(user_context, sizeof(AllocationStatus) * new_capacity); + + for (uint32_t i = 0; i < new_capacity - 1; ++i) { + new_indices[i] = i + 1; // singly-linked list of all free entries in the block + new_status[i] = AllocationStatus::Available; // usage status + } + + new_indices[new_capacity - 1] = invalid_entry; + new_status[new_capacity - 1] = AllocationStatus::InvalidStatus; + + const Block new_block = {new_entries, new_indices, new_status, new_capacity, 0}; + blocks.append(user_context, &new_block); + return static_cast(blocks.back()); +} + +void MemoryArena::destroy_block(void *user_context, Block *block) { + halide_abort_if_false(user_context, block != nullptr); + if (block->entries != nullptr) { + halide_abort_if_false(user_context, current_allocator().deallocate != nullptr); + current_allocator().deallocate(user_context, block->entries); + current_allocator().deallocate(user_context, block->indices); + current_allocator().deallocate(user_context, block->status); + block->entries = nullptr; + block->indices = nullptr; + block->status = nullptr; + } +} + +bool MemoryArena::collect_block(void *user_context, Block *block) { + halide_abort_if_false(user_context, block != nullptr); + if (block->entries != nullptr) { + bool can_collect = true; + for (size_t i = block->capacity; i--;) { + if (block->status[i] == AllocationStatus::InUse) { + can_collect = false; + break; + } + } + if (can_collect) { + destroy_block(user_context, block); + return true; + } + } + return false; +} + +MemoryArena::Block *MemoryArena::lookup_block(void *user_context, uint32_t index) { + return static_cast(blocks[index]); +} + +void *MemoryArena::lookup_entry(void *user_context, Block *block, uint32_t index) { + halide_abort_if_false(user_context, block != nullptr); + halide_abort_if_false(user_context, block->entries != nullptr); + return offset_address(block->entries, index * config.entry_size); +} + +void *MemoryArena::create_entry(void *user_context, Block *block, uint32_t index) { + void *entry_ptr = lookup_entry(user_context, block, index); + block->free_index = block->indices[index]; + block->status[index] = AllocationStatus::InUse; +#if DEBUG_RUNTIME + memset(entry_ptr, 0, config.entry_size); +#endif + return entry_ptr; +} + +void MemoryArena::destroy_entry(void *user_context, Block *block, uint32_t index) { + block->status[index] = AllocationStatus::Available; + block->indices[index] = block->free_index; + block->free_index = index; +} + +const typename MemoryArena::Config & +MemoryArena::current_config() const { + return config; +} + +const typename MemoryArena::Config & +MemoryArena::default_config() { + static Config result; + return result; +} + +const SystemMemoryAllocatorFns & +MemoryArena::current_allocator() const { + return blocks.current_allocator(); +} + +const SystemMemoryAllocatorFns & +MemoryArena::default_allocator() { + return BlockStorage::default_allocator(); +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_MEMORY_ARENA_H diff --git a/src/runtime/internal/memory_resources.h b/src/runtime/internal/memory_resources.h new file mode 100644 index 000000000000..513892922530 --- /dev/null +++ b/src/runtime/internal/memory_resources.h @@ -0,0 +1,280 @@ +#ifndef HALIDE_RUNTIME_MEMORY_RESOURCES_H +#define HALIDE_RUNTIME_MEMORY_RESOURCES_H + +namespace Halide { +namespace Runtime { +namespace Internal { + +// -- + +// Hint for allocation usage indicating whether or not the resource +// is in use, available, or dedicated (and can't be split or shared) +enum class AllocationStatus { + InvalidStatus, + InUse, + Available, + Dedicated +}; + +// Hint for allocation requests indicating intended usage +// required between host and device address space mappings +enum class MemoryVisibility { + InvalidVisibility, //< invalid enum value + HostOnly, //< host local + DeviceOnly, //< device local + DeviceToHost, //< transfer from device to host + HostToDevice, //< transfer from host to device + DefaultVisibility, //< default visibility (use any valid visibility -- unable to determine prior to usage) +}; + +// Hint for allocation requests indicating intended update +// frequency for modifying the contents of the allocation +enum class MemoryUsage { + InvalidUsage, //< invalid enum value + StaticStorage, //< intended for static storage, whereby the contents will be set once and remain unchanged + DynamicStorage, //< intended for dyanmic storage, whereby the contents will be set frequently and change constantly + UniformStorage, //< intended for fast & small fixed read-only uniform storage (intended for passing shader parameters), whereby the contents will be set once and remain unchanged + TransferSrc, //< intended for staging storage updates, whereby the contents will be used as the source of a transfer + TransferDst, //< intended for staging storage updates, whereby the contents will be used as the destination of a transfer + TransferSrcDst, //< intended for staging storage updates, whereby the contents will be used either as a source or destination of a transfer + DefaultUsage //< default usage (use any valid usage -- unable to determine prior to usage) +}; + +// Hint for allocation requests indicating ideal caching support (if available) +enum class MemoryCaching { + InvalidCaching, //< invalid enum value + Cached, //< cached + Uncached, //< uncached + CachedCoherent, //< cached and coherent + UncachedCoherent, //< uncached but still coherent + DefaultCaching //< default caching (use any valid caching behaviour -- unable to determine prior to usage) +}; + +struct MemoryProperties { + MemoryVisibility visibility = MemoryVisibility::InvalidVisibility; + MemoryUsage usage = MemoryUsage::InvalidUsage; + MemoryCaching caching = MemoryCaching::InvalidCaching; +}; + +// Client-facing struct for exchanging memory block allocation requests +struct MemoryBlock { + void *handle = nullptr; //< client data storing native handle (managed by alloc_block_region/free_block_region) + size_t size = 0; //< allocated size (in bytes) + bool dedicated = false; //< flag indicating whether allocation is one dedicated resource (or split/shared into other resources) + MemoryProperties properties; //< properties for the allocated block +}; + +// Client-facing struct for exchanging memory region allocation requests +struct MemoryRegion { + void *handle = nullptr; //< client data storing native handle (managed by alloc_block_region/free_block_region) + size_t offset = 0; //< offset from base address in block (in bytes) + size_t size = 0; //< allocated size (in bytes) + bool dedicated = false; //< flag indicating whether allocation is one dedicated resource (or split/shared into other resources) + MemoryProperties properties; //< properties for the allocated region +}; + +// Client-facing struct for issuing memory allocation requests +struct MemoryRequest { + size_t offset = 0; //< offset from base address in block (in bytes) + size_t size = 0; //< allocated size (in bytes) + size_t alignment = 0; //< alignment constraint for address + bool dedicated = false; //< flag indicating whether allocation is one dedicated resource (or split/shared into other resources) + MemoryProperties properties; //< properties for the allocated region +}; + +class RegionAllocator; +struct BlockRegion; + +// Internal struct for block resource state +// -- Note: first field must MemoryBlock +struct BlockResource { + MemoryBlock memory; //< memory info for the allocated block + RegionAllocator *allocator = nullptr; //< designated allocator for the block + BlockRegion *regions = nullptr; //< head of linked list of memory regions + size_t reserved = 0; //< number of bytes already reserved to regions +}; + +// Internal struct for block region state +// -- Note: first field must MemoryRegion +struct BlockRegion { + MemoryRegion memory; //< memory info for the allocated region + AllocationStatus status = AllocationStatus::InvalidStatus; //< allocation status indicator + BlockRegion *next_ptr = nullptr; //< pointer to next block region in linked list + BlockRegion *prev_ptr = nullptr; //< pointer to prev block region in linked list + BlockResource *block_ptr = nullptr; //< pointer to parent block resource +}; + +// Returns an aligned byte offset to adjust the given offset based on alignment constraints +// -- Alignment must be power of two! +ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment) { + return (offset + (alignment - 1)) & ~(alignment - 1); +} + +// Returns a padded size to accomodate an adjusted offset due to alignment constraints +// -- Alignment must be power of two! +ALWAYS_INLINE size_t aligned_size(size_t offset, size_t size, size_t alignment) { + size_t actual_offset = aligned_offset(offset, alignment); + size_t padding = actual_offset - offset; + size_t actual_size = padding + size; + return actual_size; +} + +// Clamps the given value to be within the [min_value, max_value] range +ALWAYS_INLINE size_t clamped_size(size_t value, size_t min_value, size_t max_value) { + size_t result = (value < min_value) ? min_value : value; + return (result > max_value) ? max_value : result; +} + +// Offset the untyped pointer by the given number of bytes +ALWAYS_INLINE const void *offset_address(const void *address, size_t byte_offset) { + const uintptr_t base = reinterpret_cast(address); + return reinterpret_cast(base + byte_offset); +} + +// Offset the untyped pointer by the given number of bytes +ALWAYS_INLINE void *offset_address(void *address, size_t byte_offset) { + const uintptr_t base = reinterpret_cast(address); + return reinterpret_cast(base + byte_offset); +} + +// -- + +typedef void *(*AllocateSystemFn)(void *, size_t); +typedef void (*DeallocateSystemFn)(void *, void *); + +ALWAYS_INLINE void *native_system_malloc(void *user_context, size_t bytes) { + return malloc(bytes); +} + +ALWAYS_INLINE void native_system_free(void *user_context, void *ptr) { + free(ptr); +} + +struct SystemMemoryAllocatorFns { + AllocateSystemFn allocate = nullptr; + DeallocateSystemFn deallocate = nullptr; +}; + +struct HalideSystemAllocatorFns { + AllocateSystemFn allocate = halide_malloc; + DeallocateSystemFn deallocate = halide_free; +}; + +typedef void (*AllocateBlockFn)(void *, MemoryBlock *); +typedef void (*DeallocateBlockFn)(void *, MemoryBlock *); + +struct MemoryBlockAllocatorFns { + AllocateBlockFn allocate = nullptr; + DeallocateBlockFn deallocate = nullptr; +}; + +typedef void (*AllocateRegionFn)(void *, MemoryRegion *); +typedef void (*DeallocateRegionFn)(void *, MemoryRegion *); + +struct MemoryRegionAllocatorFns { + AllocateRegionFn allocate = nullptr; + DeallocateRegionFn deallocate = nullptr; +}; + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +// -- + +extern "C" { + +WEAK const char *halide_memory_visibility_name(MemoryVisibility value) { + switch (value) { + case MemoryVisibility::InvalidVisibility: { + return "InvalidVisibility"; + } + case MemoryVisibility::DefaultVisibility: { + return "DefaultVisibility"; + } + case MemoryVisibility::HostOnly: { + return "HostOnly"; + } + case MemoryVisibility::DeviceOnly: { + return "DeviceOnly"; + } + case MemoryVisibility::HostToDevice: { + return "HostToDevice"; + } + case MemoryVisibility::DeviceToHost: { + return "DeviceToHost"; + } + default: { + return ""; + } + }; + return ""; +} + +WEAK const char *halide_memory_usage_name(MemoryUsage value) { + switch (value) { + case MemoryUsage::InvalidUsage: { + return "InvalidUsage"; + } + case MemoryUsage::DefaultUsage: { + return "DefaultUsage"; + } + case MemoryUsage::StaticStorage: { + return "StaticStorage"; + } + case MemoryUsage::DynamicStorage: { + return "DynamicStorage"; + } + case MemoryUsage::UniformStorage: { + return "UniformStorage"; + } + case MemoryUsage::TransferSrc: { + return "TransferSrc"; + } + case MemoryUsage::TransferDst: { + return "TransferDst"; + } + case MemoryUsage::TransferSrcDst: { + return "TransferSrcDst"; + } + default: { + return ""; + } + }; + return ""; +} + +WEAK const char *halide_memory_caching_name(MemoryCaching value) { + switch (value) { + case MemoryCaching::InvalidCaching: { + return "InvalidCaching"; + } + case MemoryCaching::DefaultCaching: { + return "DefaultCaching"; + } + case MemoryCaching::Cached: { + return "Cached"; + } + case MemoryCaching::Uncached: { + return "Uncached"; + } + case MemoryCaching::CachedCoherent: { + return "CachedCoherent"; + } + case MemoryCaching::UncachedCoherent: { + return "UncachedCoherent"; + } + default: { + return ""; + } + }; + return ""; +} + +} // extern "C" + +// -- + +#endif // HALIDE_RUNTIME_MEMORY_RESOURCES_H diff --git a/src/runtime/internal/pointer_table.h b/src/runtime/internal/pointer_table.h new file mode 100644 index 000000000000..b5ff3bfd6f7c --- /dev/null +++ b/src/runtime/internal/pointer_table.h @@ -0,0 +1,366 @@ +#ifndef HALIDE_RUNTIME_POINTER_TABLE_H +#define HALIDE_RUNTIME_POINTER_TABLE_H + +#include "memory_resources.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// Dynamically resizable array for storing untyped pointers +// -- Implementation uses memcpy/memmove for copying +// -- Customizable allocator ... default uses NativeSystemAllocator +class PointerTable { +public: + static constexpr size_t default_capacity = 32; // smallish + + PointerTable(void *user_context, size_t initial_capacity = 0, const SystemMemoryAllocatorFns &sma = default_allocator()); + PointerTable(const PointerTable &other); + ~PointerTable(); + + void initialize(void *user_context, size_t initial_capacity = 0, const SystemMemoryAllocatorFns &sma = default_allocator()); + + PointerTable &operator=(const PointerTable &other); + bool operator==(const PointerTable &other) const; + bool operator!=(const PointerTable &other) const; + + void reserve(void *user_context, size_t capacity, bool free_existing = false); + void resize(void *user_context, size_t entry_count, bool realloc = true); + + void assign(void *user_context, size_t index, const void *entry_ptr); + void insert(void *user_context, size_t index, const void *entry_ptr); + void prepend(void *user_context, const void *entry_ptr); + void append(void *user_context, const void *entry_ptr); + void remove(void *user_context, size_t index); + + void fill(void *user_context, const void **array, size_t array_size); + void insert(void *user_context, size_t index, const void **array, size_t array_size); + void replace(void *user_context, size_t index, const void **array, size_t array_size); + void prepend(void *user_context, const void **array, size_t array_size); + void append(void *user_context, const void **array, size_t array_size); + void remove(void *user_context, size_t index, size_t entry_count); + + void pop_front(void *user_context); + void pop_back(void *user_context); + void shrink_to_fit(void *user_context); + void clear(void *user_context); + void destroy(void *user_context); + + bool empty() const; + size_t size() const; + + void *operator[](size_t index); + void *operator[](size_t index) const; + + void **data(); + const void **data() const; + + void *front(); + void *back(); + + const SystemMemoryAllocatorFns ¤t_allocator() const; + static const SystemMemoryAllocatorFns &default_allocator(); + +private: + void allocate(void *user_context, size_t capacity); + + void **ptr = nullptr; + size_t count = 0; + size_t capacity = 0; + SystemMemoryAllocatorFns allocator; +}; + +PointerTable::PointerTable(void *user_context, size_t initial_capacity, const SystemMemoryAllocatorFns &sma) + : allocator(sma) { + halide_abort_if_false(user_context, allocator.allocate != nullptr); + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + if (initial_capacity) { reserve(user_context, initial_capacity); } +} + +PointerTable::PointerTable(const PointerTable &other) + : PointerTable(nullptr, 0, other.allocator) { + if (other.capacity) { + ptr = static_cast(allocator.allocate(nullptr, other.capacity * sizeof(void *))); + capacity = other.capacity; + } + if (ptr && other.count != 0) { + count = other.count; + memcpy(this->ptr, other.ptr, count * sizeof(void *)); + } +} + +PointerTable::~PointerTable() { + destroy(nullptr); +} + +void PointerTable::destroy(void *user_context) { + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + if (ptr != nullptr) { + allocator.deallocate(user_context, ptr); + } + capacity = count = 0; + ptr = nullptr; +} + +void PointerTable::initialize(void *user_context, size_t initial_capacity, const SystemMemoryAllocatorFns &sma) { + allocator = sma; + capacity = count = 0; + ptr = nullptr; + if (initial_capacity) { + reserve(user_context, initial_capacity); + } +} + +PointerTable &PointerTable::operator=(const PointerTable &other) { + if (&other != this) { + resize(nullptr, other.count); + if (count != 0 && other.ptr != nullptr) { + memcpy(ptr, other.ptr, count * sizeof(void *)); + } + } + return *this; +} + +bool PointerTable::operator==(const PointerTable &other) const { + if (count != other.count) { return false; } + return memcmp(this->ptr, other.ptr, this->size() * sizeof(void *)) == 0; +} + +bool PointerTable::operator!=(const PointerTable &other) const { + return !(*this == other); +} + +void PointerTable::fill(void *user_context, const void **array, size_t array_size) { + if (array_size != 0) { + resize(user_context, array_size); + memcpy(this->ptr, array, array_size * sizeof(void *)); + count = array_size; + } +} + +void PointerTable::assign(void *user_context, size_t index, const void *entry_ptr) { + halide_debug_assert(user_context, index < count); + ptr[index] = const_cast(entry_ptr); +} + +void PointerTable::prepend(void *user_context, const void *entry_ptr) { + insert(user_context, 0, &entry_ptr, 1); +} + +void PointerTable::append(void *user_context, const void *entry_ptr) { + append(user_context, &entry_ptr, 1); +} + +void PointerTable::pop_front(void *user_context) { + halide_debug_assert(user_context, count > 0); + remove(user_context, 0); +} + +void PointerTable::pop_back(void *user_context) { + halide_debug_assert(user_context, count > 0); + resize(user_context, size() - 1); +} + +void PointerTable::clear(void *user_context) { + resize(user_context, 0); +} + +void PointerTable::reserve(void *user_context, size_t new_capacity, bool free_existing) { + new_capacity = max(new_capacity, count); + if ((new_capacity < capacity) && !free_existing) { + new_capacity = capacity; + } + allocate(user_context, new_capacity); +} + +void PointerTable::resize(void *user_context, size_t entry_count, bool realloc) { + size_t current_size = capacity; + size_t requested_size = entry_count; + size_t minimum_size = default_capacity; + size_t actual_size = current_size; + count = requested_size; + +#ifdef DEBUG_RUNTIME + debug(user_context) << "PointerTable: Resize (" + << "requested_size=" << (int32_t)requested_size << " " + << "current_size=" << (int32_t)current_size << " " + << "minimum_size=" << (int32_t)minimum_size << " " + << "sizeof(void*)=" << (int32_t)sizeof(void *) << " " + << "realloc=" << (realloc ? "true" : "false") << ")...\n"; +#endif + + // increase capacity upto 1.5x existing (or at least min_capacity) + if (requested_size > current_size) { + actual_size = max(requested_size, max(current_size * 3 / 2, minimum_size)); + } else if (!realloc) { + return; + } + + allocate(user_context, actual_size); +} + +void PointerTable::shrink_to_fit(void *user_context) { + if (capacity > count) { + void *new_ptr = nullptr; + if (count > 0) { + size_t bytes = count * sizeof(void *); + new_ptr = allocator.allocate(user_context, bytes); + memcpy(new_ptr, ptr, bytes); + } + allocator.deallocate(user_context, ptr); + capacity = count; + ptr = static_cast(new_ptr); + } +} + +void PointerTable::insert(void *user_context, size_t index, const void *entry_ptr) { + const void *addr = reinterpret_cast(entry_ptr); + insert(user_context, index, &addr, 1); +} + +void PointerTable::remove(void *user_context, size_t index) { + remove(user_context, index, 1); +} + +void PointerTable::remove(void *user_context, size_t index, size_t entry_count) { + halide_debug_assert(user_context, index < count); + const size_t last_index = size(); + if (index < (last_index - entry_count)) { + size_t dst_offset = index * sizeof(void *); + size_t src_offset = (index + entry_count) * sizeof(void *); + size_t bytes = (last_index - index - entry_count) * sizeof(void *); + +#ifdef DEBUG_RUNTIME + debug(user_context) << "PointerTable: Remove (" + << "index=" << (int32_t)index << " " + << "entry_count=" << (int32_t)entry_count << " " + << "last_index=" << (int32_t)last_index << " " + << "src_offset=" << (int32_t)src_offset << " " + << "dst_offset=" << (int32_t)dst_offset << " " + << "bytes=" << (int32_t)bytes << ")...\n"; +#endif + memmove(ptr + dst_offset, ptr + src_offset, bytes); + } + resize(user_context, last_index - entry_count); +} + +void PointerTable::replace(void *user_context, size_t index, const void **array, size_t array_size) { + halide_debug_assert(user_context, index < count); + size_t remaining = count - index; + size_t copy_count = min(remaining, array_size); + +#ifdef DEBUG_RUNTIME + + debug(user_context) << "PointerTable: Replace (" + << "index=" << (int32_t)index << " " + << "array_size=" << (int32_t)array_size << " " + << "remaining=" << (int32_t)remaining << " " + << "copy_count=" << (int32_t)copy_count << " " + << "capacity=" << (int32_t)capacity << ")...\n"; +#endif + + halide_debug_assert(user_context, remaining > 0); + memcpy(ptr + index, array, copy_count * sizeof(void *)); + count = max(count, index + copy_count); +} + +void PointerTable::insert(void *user_context, size_t index, const void **array, size_t array_size) { + halide_debug_assert(user_context, index <= count); + const size_t last_index = size(); + resize(user_context, last_index + array_size); + if (index < last_index) { + size_t src_offset = index * sizeof(void *); + size_t dst_offset = (index + array_size) * sizeof(void *); + size_t bytes = (last_index - index) * sizeof(void *); + memmove(ptr + dst_offset, ptr + src_offset, bytes); + } + replace(user_context, index, array, array_size); +} + +void PointerTable::prepend(void *user_context, const void **array, size_t array_size) { + insert(user_context, 0, array, array_size); +} + +void PointerTable::append(void *user_context, const void **array, size_t array_size) { + const size_t last_index = size(); + insert(user_context, last_index, array, array_size); +} + +bool PointerTable::empty() const { + return count == 0; +} + +size_t PointerTable::size() const { + return count; +} + +void *PointerTable::operator[](size_t index) { + halide_debug_assert(nullptr, index < capacity); + return ptr[index]; +} + +void *PointerTable::operator[](size_t index) const { + halide_debug_assert(nullptr, index < capacity); + return ptr[index]; +} + +void **PointerTable::data() { + return ptr; +} + +void *PointerTable::front() { + halide_debug_assert(nullptr, count > 0); + return ptr[0]; +} + +void *PointerTable::back() { + halide_debug_assert(nullptr, count > 0); + size_t index = count - 1; + return ptr[index]; +} + +const void **PointerTable::data() const { + return const_cast(ptr); +} + +void PointerTable::allocate(void *user_context, size_t new_capacity) { + if (new_capacity != capacity) { + halide_abort_if_false(user_context, allocator.allocate != nullptr); + size_t bytes = new_capacity * sizeof(void *); + +#ifdef DEBUG_RUNTIME + debug(user_context) << "PointerTable: Allocating (bytes=" << (int32_t)bytes << " allocator=" << (void *)allocator.allocate << ")...\n"; +#endif + + void *new_ptr = bytes ? allocator.allocate(user_context, bytes) : nullptr; + if (count != 0 && ptr != nullptr && new_ptr != nullptr) { + memcpy(new_ptr, ptr, count * sizeof(void *)); + } + if (ptr != nullptr) { + halide_abort_if_false(user_context, allocator.deallocate != nullptr); + allocator.deallocate(user_context, ptr); + } + capacity = new_capacity; + ptr = static_cast(new_ptr); + } +} + +const SystemMemoryAllocatorFns & +PointerTable::current_allocator() const { + return this->allocator; +} + +const SystemMemoryAllocatorFns & +PointerTable::default_allocator() { + static SystemMemoryAllocatorFns native_allocator = { + native_system_malloc, native_system_free}; + return native_allocator; +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_POINTER_TABLE_H diff --git a/src/runtime/internal/region_allocator.h b/src/runtime/internal/region_allocator.h new file mode 100644 index 000000000000..8c7f8602abe7 --- /dev/null +++ b/src/runtime/internal/region_allocator.h @@ -0,0 +1,462 @@ +#ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H +#define HALIDE_RUNTIME_REGION_ALLOCATOR_H + +#include "memory_arena.h" +#include "memory_resources.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// -- + +/** Allocator class interface for sub-allocating a contiguous + * memory block into smaller regions of memory. This class only + * manages the address creation for the regions -- allocation + * callback functions are used to request the memory from the + * necessary system or API calls. This class is intended to be + * used inside of a higher level memory management class that + * provides thread safety, policy management and API + * integration for a specific runtime API (eg Vulkan, OpenCL, etc) + */ +class RegionAllocator { +public: + // disable copy constructors and assignment + RegionAllocator(const RegionAllocator &) = delete; + RegionAllocator &operator=(const RegionAllocator &) = delete; + + // disable non-factory based construction + RegionAllocator() = delete; + ~RegionAllocator() = delete; + + // Allocators for the different types of memory we need to allocate + struct MemoryAllocators { + SystemMemoryAllocatorFns system; + MemoryRegionAllocatorFns region; + }; + + // Factory methods for creation / destruction + static RegionAllocator *create(void *user_context, BlockResource *block, const MemoryAllocators &ma); + static void destroy(void *user_context, RegionAllocator *region_allocator); + + // Returns the allocator class instance for the given allocation (or nullptr) + static RegionAllocator *find_allocator(void *user_context, MemoryRegion *memory_region); + + // Public interface methods + MemoryRegion *reserve(void *user_context, const MemoryRequest &request); + void reclaim(void *user_context, MemoryRegion *memory_region); + bool collect(void *user_context); //< returns true if any blocks were removed + void release(void *user_context); + void destroy(void *user_context); + + // Returns the currently managed block resource + BlockResource *block_resource() const; + +private: + // Initializes a new instance + void initialize(void *user_context, BlockResource *block, const MemoryAllocators &ma); + + // Search through allocated block regions (Best-Fit) + BlockRegion *find_block_region(void *user_context, const MemoryRequest &request); + + // Returns true if neighbouring block regions to the given region can be coalesced into one + bool can_coalesce(BlockRegion *region); + + // Merges available neighbouring block regions into the given region + BlockRegion *coalesce_block_regions(void *user_context, BlockRegion *region); + + // Returns true if the given region can be split to accomadate the given size + bool can_split(BlockRegion *region, size_t size); + + // Splits the given block region into a smaller region to accomadate the given size, followed by empty space for the remaining + BlockRegion *split_block_region(void *user_context, BlockRegion *region, size_t size, size_t alignment); + + // Creates a new block region and adds it to the region list + BlockRegion *create_block_region(void *user_context, const MemoryProperties &properties, size_t offset, size_t size, bool dedicated); + + // Creates a new block region and adds it to the region list + void destroy_block_region(void *user_context, BlockRegion *region); + + // Invokes the allocation callback to allocate memory for the block region + void alloc_block_region(void *user_context, BlockRegion *region); + + // Releases a block region and leaves it in the list for further allocations + void release_block_region(void *user_context, BlockRegion *region); + + // Invokes the deallocation callback to free memory for the block region + void free_block_region(void *user_context, BlockRegion *region); + + // Returns true if the given block region is compatible with the given properties + bool is_compatible_block_region(const BlockRegion *region, const MemoryProperties &properties) const; + + BlockResource *block = nullptr; + MemoryArena *arena = nullptr; + MemoryAllocators allocators; +}; + +RegionAllocator *RegionAllocator::create(void *user_context, BlockResource *block_resource, const MemoryAllocators &allocators) { + halide_abort_if_false(user_context, allocators.system.allocate != nullptr); + RegionAllocator *result = reinterpret_cast( + allocators.system.allocate(user_context, sizeof(RegionAllocator))); + + if (result == nullptr) { + halide_error(user_context, "RegionAllocator: Failed to create instance! Out of memory!\n"); + return nullptr; + } + + result->initialize(user_context, block_resource, allocators); + return result; +} + +void RegionAllocator::destroy(void *user_context, RegionAllocator *instance) { + halide_abort_if_false(user_context, instance != nullptr); + const MemoryAllocators &allocators = instance->allocators; + instance->destroy(user_context); + halide_abort_if_false(user_context, allocators.system.deallocate != nullptr); + allocators.system.deallocate(user_context, instance); +} + +void RegionAllocator::initialize(void *user_context, BlockResource *mb, const MemoryAllocators &ma) { + block = mb; + allocators = ma; + arena = MemoryArena::create(user_context, {sizeof(BlockRegion), MemoryArena::default_capacity, 0}, allocators.system); + halide_abort_if_false(user_context, arena != nullptr); + block->allocator = this; + block->regions = create_block_region( + user_context, + block->memory.properties, + 0, block->memory.size, + block->memory.dedicated); +} + +MemoryRegion *RegionAllocator::reserve(void *user_context, const MemoryRequest &request) { + halide_abort_if_false(user_context, request.size > 0); + size_t remaining = block->memory.size - block->reserved; + if (remaining < request.size) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Unable to reserve more memory from block " + << "-- requested size (" << (int32_t)(request.size) << " bytes) " + << "greater than available (" << (int32_t)(remaining) << " bytes)!\n"; +#endif + return nullptr; + } + + BlockRegion *block_region = find_block_region(user_context, request); + if (block_region == nullptr) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Failed to locate region for requested size (" + << (int32_t)(request.size) << " bytes)!\n"; +#endif + return nullptr; + } + + if (can_split(block_region, request.size)) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Splitting region of size ( " << (int32_t)(block_region->memory.size) << ") " + << "to accomodate requested size (" << (int32_t)(request.size) << " bytes)!\n"; +#endif + split_block_region(user_context, block_region, request.size, request.alignment); + } + + alloc_block_region(user_context, block_region); + return reinterpret_cast(block_region); +} + +void RegionAllocator::reclaim(void *user_context, MemoryRegion *memory_region) { + BlockRegion *block_region = reinterpret_cast(memory_region); + halide_abort_if_false(user_context, block_region != nullptr); + halide_abort_if_false(user_context, block_region->block_ptr == block); + free_block_region(user_context, block_region); + if (can_coalesce(block_region)) { + block_region = coalesce_block_regions(user_context, block_region); + } +} + +RegionAllocator *RegionAllocator::find_allocator(void *user_context, MemoryRegion *memory_region) { + BlockRegion *block_region = reinterpret_cast(memory_region); + halide_abort_if_false(user_context, block_region != nullptr); + halide_abort_if_false(user_context, block_region->block_ptr != nullptr); + return block_region->block_ptr->allocator; +} + +BlockRegion *RegionAllocator::find_block_region(void *user_context, const MemoryRequest &request) { + BlockRegion *result = nullptr; + for (BlockRegion *block_region = block->regions; block_region != nullptr; block_region = block_region->next_ptr) { + + if (block_region->status != AllocationStatus::Available) { + continue; + } + + // skip incompatible block regions for this request + if (!is_compatible_block_region(block_region, request.properties)) { + continue; + } + + // is the requested size larger than the current region? + if (request.size > block_region->memory.size) { + continue; + } + + size_t actual_size = aligned_size(block_region->memory.offset, request.size, request.alignment); + + // is the adjusted size larger than the current region? + if (actual_size > block_region->memory.size) { + continue; + } + + // will the adjusted size fit within the remaining unallocated space? + if ((actual_size + block->reserved) < block->memory.size) { + result = block_region; // best-fit! + break; + } + } + return result; +} + +bool RegionAllocator::can_coalesce(BlockRegion *block_region) { + if (block_region == nullptr) { return false; } + if (block_region->prev_ptr && (block_region->prev_ptr->status == AllocationStatus::Available)) { + return true; + } + if (block_region->next_ptr && (block_region->next_ptr->status == AllocationStatus::Available)) { + return true; + } + return false; +} + +BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) { + if (block_region->prev_ptr && (block_region->prev_ptr->status == AllocationStatus::Available)) { + BlockRegion *prev_region = block_region->prev_ptr; + +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Coalescing " + << "previous region (offset=" << (int32_t)prev_region->memory.offset << " size=" << (int32_t)(prev_region->memory.size) << " bytes) " + << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)\n!"; +#endif + + prev_region->next_ptr = block_region->next_ptr; + if (block_region->next_ptr) { + block_region->next_ptr->prev_ptr = prev_region; + } + prev_region->memory.size += block_region->memory.size; + destroy_block_region(user_context, block_region); + block_region = prev_region; + } + + if (block_region->next_ptr && (block_region->next_ptr->status == AllocationStatus::Available)) { + BlockRegion *next_region = block_region->next_ptr; + +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Coalescing " + << "next region (offset=" << (int32_t)next_region->memory.offset << " size=" << (int32_t)(next_region->memory.size) << " bytes) " + << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)!\n"; +#endif + + if (next_region->next_ptr) { + next_region->next_ptr->prev_ptr = block_region; + } + block_region->next_ptr = next_region->next_ptr; + block_region->memory.size += next_region->memory.size; + destroy_block_region(user_context, next_region); + } + + return block_region; +} + +bool RegionAllocator::can_split(BlockRegion *block_region, size_t size) { + return (block_region && (block_region->memory.size > size)); +} + +BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, size_t size, size_t alignment) { + size_t adjusted_size = aligned_size(block_region->memory.offset, size, alignment); + size_t adjusted_offset = aligned_offset(block_region->memory.offset, alignment); + + size_t empty_offset = adjusted_offset + size; + size_t empty_size = block_region->memory.size - adjusted_size; + +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Splitting " + << "current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes) " + << "to create empty region (offset=" << (int32_t)empty_offset << " size=" << (int32_t)(empty_size) << " bytes)!\n"; +#endif + + BlockRegion *next_region = block_region->next_ptr; + BlockRegion *empty_region = create_block_region(user_context, + block_region->memory.properties, + empty_offset, empty_size, + block_region->memory.dedicated); + halide_abort_if_false(user_context, empty_region != nullptr); + + empty_region->next_ptr = next_region; + if (next_region) { + next_region->prev_ptr = empty_region; + } + block_region->next_ptr = empty_region; + block_region->memory.size = size; + return empty_region; +} + +BlockRegion *RegionAllocator::create_block_region(void *user_context, const MemoryProperties &properties, size_t offset, size_t size, bool dedicated) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Creating block region (" + << "user_context=" << (void *)(user_context) << " " + << "offset=" << (uint32_t)offset << " " + << "size=" << (uint32_t)size << " " + << "dedicated=" << (dedicated ? "true" : "false") << " " + << "usage=" << halide_memory_usage_name(properties.usage) << " " + << "caching=" << halide_memory_caching_name(properties.caching) << " " + << "visibility=" << halide_memory_visibility_name(properties.visibility) << ") ...\n"; +#endif + + BlockRegion *block_region = static_cast(arena->reserve(user_context, true)); + + if (block_region == nullptr) { + error(user_context) << "RegionAllocator: Failed to allocate new block region!\n"; + return nullptr; + } + +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Added block region (" + << "user_context=" << (void *)(user_context) << " " + << "block_region=" << (void *)(block_region) << ") ...\n"; +#endif + + block_region->memory.offset = offset; + block_region->memory.size = size; + block_region->memory.properties = properties; + block_region->memory.dedicated = dedicated; + block_region->status = AllocationStatus::Available; + block_region->block_ptr = block; + return block_region; +} + +void RegionAllocator::release_block_region(void *user_context, BlockRegion *block_region) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Releasing block region (" + << "user_context=" << (void *)(user_context) << " " + << "block_region=" << (void *)(block_region) << ") ...\n"; +#endif + free_block_region(user_context, block_region); +} + +void RegionAllocator::destroy_block_region(void *user_context, BlockRegion *block_region) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Destroying block region (" + << "user_context=" << (void *)(user_context) << " " + << "block_region=" << (void *)(block_region) << ") ...\n"; +#endif + + free_block_region(user_context, block_region); + arena->reclaim(user_context, block_region); +} + +void RegionAllocator::alloc_block_region(void *user_context, BlockRegion *block_region) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Allocating region (size=" << (int32_t)(block_region->memory.size) << ", offset=" << (int32_t)block_region->memory.offset << ")!\n"; +#endif + halide_abort_if_false(user_context, allocators.region.allocate != nullptr); + halide_abort_if_false(user_context, block_region->status == AllocationStatus::Available); + MemoryRegion *memory_region = &(block_region->memory); + allocators.region.allocate(user_context, memory_region); + block_region->status = block_region->memory.dedicated ? AllocationStatus::Dedicated : AllocationStatus::InUse; + block->reserved += block_region->memory.size; +} + +void RegionAllocator::free_block_region(void *user_context, BlockRegion *block_region) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Freeing block region (" + << "user_context=" << (void *)(user_context) << " " + << "block_region=" << (void *)(block_region) << ") ...\n"; +#endif + if ((block_region->status == AllocationStatus::InUse) || + (block_region->status == AllocationStatus::Dedicated)) { + debug(user_context) << "RegionAllocator: Deallocating region (size=" << (int32_t)(block_region->memory.size) << ", offset=" << (int32_t)block_region->memory.offset << ")!\n"; + halide_abort_if_false(user_context, allocators.region.deallocate != nullptr); + MemoryRegion *memory_region = &(block_region->memory); + allocators.region.deallocate(user_context, memory_region); + block->reserved -= block_region->memory.size; + } + block_region->status = AllocationStatus::Available; +} + +void RegionAllocator::release(void *user_context) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Releasing all regions (" + << "user_context=" << (void *)(user_context) << ") ...\n"; +#endif + for (BlockRegion *block_region = block->regions; block_region != nullptr; block_region = block_region->next_ptr) { + release_block_region(user_context, block_region); + } +} + +bool RegionAllocator::collect(void *user_context) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Collecting free block regions (" + << "user_context=" << (void *)(user_context) << ") ...\n"; +#endif + bool result = false; + for (BlockRegion *block_region = block->regions; block_region != nullptr; block_region = block_region->next_ptr) { + if (block_region->status == AllocationStatus::Available) { + if (can_coalesce(block_region)) { + block_region = coalesce_block_regions(user_context, block_region); + result = true; + } + } + } + return result; +} + +void RegionAllocator::destroy(void *user_context) { +#ifdef DEBUG_RUNTIME + debug(user_context) << "RegionAllocator: Destroying all block regions (" + << "user_context=" << (void *)(user_context) << ") ...\n"; +#endif + for (BlockRegion *block_region = block->regions; block_region != nullptr;) { + + if (block_region->next_ptr == nullptr) { + destroy_block_region(user_context, block_region); + block_region = nullptr; + } else { + BlockRegion *prev_region = block_region; + block_region = block_region->next_ptr; + destroy_block_region(user_context, prev_region); + } + } + block->regions = nullptr; + block->reserved = 0; + arena->destroy(user_context); +} + +bool RegionAllocator::is_compatible_block_region(const BlockRegion *block_region, const MemoryProperties &properties) const { + if (properties.caching != MemoryCaching::DefaultCaching) { + if (properties.caching != block_region->memory.properties.caching) { + return false; + } + } + + if (properties.visibility != MemoryVisibility::DefaultVisibility) { + if (properties.visibility != block_region->memory.properties.visibility) { + return false; + } + } + + if (properties.usage != MemoryUsage::DefaultUsage) { + if (properties.usage != block_region->memory.properties.usage) { + return false; + } + } + + return true; +} + +BlockResource *RegionAllocator::block_resource() const { + return block; +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H diff --git a/src/runtime/internal/string_storage.h b/src/runtime/internal/string_storage.h new file mode 100644 index 000000000000..6b4daa95ac0a --- /dev/null +++ b/src/runtime/internal/string_storage.h @@ -0,0 +1,216 @@ +#ifndef HALIDE_RUNTIME_STRING_STORAGE_H +#define HALIDE_RUNTIME_STRING_STORAGE_H + +#include "block_storage.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// Static utility functions for dealing with string data +struct StringUtils { + static bool is_empty(const char *str) { + if (str == nullptr) { return true; } + if (str[0] == '\0') { return true; } + return false; + } + + // count the number of delimited string tokens + static size_t count_tokens(const char *str, const char *delim) { + if (StringUtils::is_empty(str)) { return 0; } + if (StringUtils::is_empty(delim)) { return 1; } // no delim ... string is one token + + size_t count = 0; + const char *ptr = str; + size_t delim_length = strlen(delim); + while (!StringUtils::is_empty(ptr)) { + const char *next_delim = strstr(ptr, delim); + ptr = (next_delim != nullptr) ? (next_delim + delim_length) : nullptr; + ++count; + } + return count; + } + + static size_t count_length(const char *str) { + const char *ptr = str; + while (!StringUtils::is_empty(ptr)) { + ++ptr; + } + return size_t(ptr - str); + } +}; + +// -- +// Storage class for handling c-string data (based on block storage) +// -- Intended for building and maintaining string data w/8-bit chars +// +class StringStorage { +public: + StringStorage(void *user_context = nullptr, uint32_t capacity = 0, const SystemMemoryAllocatorFns &sma = default_allocator()); + StringStorage(const StringStorage &other) = default; + ~StringStorage(); + + void initialize(void *user_context, uint32_t capacity = 0, const SystemMemoryAllocatorFns &sma = default_allocator()); + void destroy(void *user_context); + + StringStorage &operator=(const StringStorage &other); + bool operator==(const StringStorage &other) const; + bool operator!=(const StringStorage &other) const; + + bool contains(const char *str) const; + bool contains(const StringStorage &other) const; + + void reserve(void *user_context, size_t length); + void assign(void *user_context, char ch); + void assign(void *user_context, const char *str, size_t length = 0); // if length is zero, strlen is used + void append(void *user_context, char ch); + void append(void *user_context, const char *str, size_t length = 0); // if length is zero, strlen is used + void prepend(void *user_context, char ch); + void prepend(void *user_context, const char *str, size_t length = 0); // if length is zero, strlen is used + void clear(void *user_context); + void terminate(void *user_context, size_t length); + + size_t length() const; + const char *data() const; + + const SystemMemoryAllocatorFns ¤t_allocator() const; + static const SystemMemoryAllocatorFns &default_allocator(); + +private: + BlockStorage contents; +}; + +StringStorage::StringStorage(void *user_context, uint32_t capacity, const SystemMemoryAllocatorFns &sma) + : contents(user_context, {sizeof(char), 32, 32}, sma) { + if (capacity) { contents.reserve(user_context, capacity); } +} + +StringStorage::~StringStorage() { + destroy(nullptr); +} + +StringStorage &StringStorage::operator=(const StringStorage &other) { + if (&other != this) { + assign(nullptr, other.data(), other.length()); + } + return *this; +} + +bool StringStorage::contains(const char *str) const { + const char *this_str = static_cast(contents.data()); + return strstr(this_str, str) != nullptr; +} + +bool StringStorage::contains(const StringStorage &other) const { + const char *this_str = static_cast(contents.data()); + const char *other_str = static_cast(other.contents.data()); + return strstr(this_str, other_str) != nullptr; +} + +bool StringStorage::operator==(const StringStorage &other) const { + if (contents.size() != other.contents.size()) { return false; } + const char *this_str = static_cast(contents.data()); + const char *other_str = static_cast(other.contents.data()); + return strncmp(this_str, other_str, contents.size()) == 0; +} + +bool StringStorage::operator!=(const StringStorage &other) const { + return !(*this == other); +} + +void StringStorage::reserve(void *user_context, size_t length) { + contents.reserve(user_context, length + 1); // leave room for termination + contents.resize(user_context, length, false); + terminate(user_context, length); +} + +void StringStorage::assign(void *user_context, char ch) { + contents.resize(user_context, 1); + char *ptr = static_cast(contents[0]); + (*ptr) = ch; +} + +void StringStorage::assign(void *user_context, const char *str, size_t length) { + if (StringUtils::is_empty(str)) { return; } + if (length == 0) { length = strlen(str); } + char *this_str = static_cast(contents.data()); + reserve(user_context, length); + memcpy(this_str, str, length); + terminate(user_context, length); +} + +void StringStorage::append(void *user_context, const char *str, size_t length) { + if (StringUtils::is_empty(str)) { return; } + if (length == 0) { length = strlen(str); } + const size_t old_size = contents.size(); + size_t new_length = old_size + length; + char *this_str = static_cast(contents[old_size]); + reserve(user_context, length); + memcpy(this_str, str, length); + terminate(user_context, new_length); +} + +void StringStorage::append(void *user_context, char ch) { + contents.append(user_context, &ch); +} + +void StringStorage::prepend(void *user_context, const char *str, size_t length) { + if (StringUtils::is_empty(str)) { return; } + if (length == 0) { length = strlen(str); } + const size_t old_size = contents.size(); + size_t new_length = old_size + length; + char *this_str = static_cast(contents.data()); + reserve(user_context, new_length); + memcpy(this_str + length, this_str, old_size); + memcpy(this_str, str, length); + terminate(user_context, new_length); +} + +void StringStorage::prepend(void *user_context, char ch) { + contents.prepend(user_context, &ch); +} + +void StringStorage::terminate(void *user_context, size_t length) { + char *end_ptr = static_cast(contents[length]); + (*end_ptr) = '\0'; +} + +void StringStorage::clear(void *user_context) { + contents.clear(user_context); + if (contents.data()) { terminate(user_context, 0); } +} + +void StringStorage::initialize(void *user_context, uint32_t capacity, const SystemMemoryAllocatorFns &sma) { + contents.initialize(user_context, {sizeof(char), 32, 32}, sma); + if (capacity) { contents.reserve(user_context, capacity); } +} + +void StringStorage::destroy(void *user_context) { + contents.destroy(user_context); +} + +size_t StringStorage::length() const { + return StringUtils::count_length(data()); +} + +const char *StringStorage::data() const { + return static_cast(contents.data()); +} + +const SystemMemoryAllocatorFns & +StringStorage::current_allocator() const { + return contents.current_allocator(); +} + +const SystemMemoryAllocatorFns & +StringStorage::default_allocator() { + return BlockStorage::default_allocator(); +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_STRING_STORAGE_H diff --git a/src/runtime/internal/string_table.h b/src/runtime/internal/string_table.h new file mode 100644 index 000000000000..07e09f5f97b2 --- /dev/null +++ b/src/runtime/internal/string_table.h @@ -0,0 +1,217 @@ +#ifndef HALIDE_RUNTIME_STRING_TABLE_H +#define HALIDE_RUNTIME_STRING_TABLE_H + +#include "linked_list.h" +#include "pointer_table.h" +#include "string_storage.h" + +namespace Halide { +namespace Runtime { +namespace Internal { + +// Storage class for an array of strings (based on block storage) +// -- Intended for building and maintaining tables of strings +class StringTable { +public: + // Disable copy constructors + StringTable(const StringTable &) = delete; + StringTable &operator=(const StringTable &) = delete; + + StringTable(const SystemMemoryAllocatorFns &allocator = StringStorage::default_allocator()); + StringTable(void *user_context, size_t capacity, const SystemMemoryAllocatorFns &allocator = StringStorage::default_allocator()); + StringTable(void *user_context, const char **array, size_t count, const SystemMemoryAllocatorFns &allocator = StringStorage::default_allocator()); + ~StringTable(); + + void resize(void *user_context, size_t capacity); + void destroy(void *user_context); + void clear(void *user_context); + + // fills the contents of the table (copies strings from given array) + void fill(void *user_context, const char **array, size_t coun); + + // assign the entry at given index the given string + void assign(void *user_context, size_t index, const char *str, size_t length = 0); // if length is zero, strlen is used + + // appends the given string to the end of the table + void append(void *user_context, const char *str, size_t length = 0); // if length is zero, strlen is used + + // prepend the given string to the end of the table + void prepend(void *user_context, const char *str, size_t length = 0); // if length is zero, strlen is used + + // parses the given c-string based on given delimiter, stores each substring in the resulting table + size_t parse(void *user_context, const char *str, const char *delim); + + // index-based access operator + const char *operator[](size_t index) const; + + // returns the raw string table pointer + const char **data() const; + + // scans the table for existance of the given string within any entry (linear scan w/string compare!) + bool contains(const char *str) const; + + size_t size() const { + return contents.size(); + } + +private: + LinkedList contents; //< owns string data + PointerTable pointers; //< stores pointers +}; + +// -- + +StringTable::StringTable(const SystemMemoryAllocatorFns &sma) + : contents(nullptr, sizeof(StringStorage), 0, sma), + pointers(nullptr, 0, sma) { + // EMPTY! +} + +StringTable::StringTable(void *user_context, size_t capacity, const SystemMemoryAllocatorFns &sma) + : contents(user_context, sizeof(StringStorage), capacity, sma), + pointers(user_context, capacity, sma) { + if (capacity) { resize(user_context, capacity); } +} + +StringTable::StringTable(void *user_context, const char **array, size_t count, const SystemMemoryAllocatorFns &sma) + : contents(user_context, sizeof(StringStorage), count, sma), + pointers(user_context, count, sma) { + fill(user_context, array, count); +} + +StringTable::~StringTable() { + destroy(nullptr); +} + +void StringTable::resize(void *user_context, size_t capacity) { + for (size_t n = contents.size(); n < capacity; ++n) { + LinkedList::EntryType *entry_ptr = contents.append(user_context); + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->initialize(user_context, 0, contents.current_allocator()); + } + pointers.resize(user_context, capacity); +} + +void StringTable::clear(void *user_context) { + for (size_t n = 0; n < contents.size(); ++n) { + LinkedList::EntryType *entry_ptr = contents.front(); + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->clear(user_context); + contents.pop_front(user_context); + } + contents.clear(user_context); + pointers.clear(user_context); +} + +void StringTable::destroy(void *user_context) { + for (size_t n = 0; n < contents.size(); ++n) { + LinkedList::EntryType *entry_ptr = contents.front(); + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->destroy(user_context); + contents.pop_front(user_context); + } + contents.destroy(user_context); + pointers.destroy(user_context); +} + +const char *StringTable::operator[](size_t index) const { + return static_cast(pointers[index]); +} + +void StringTable::fill(void *user_context, const char **array, size_t count) { + resize(user_context, count); + LinkedList::EntryType *entry_ptr = contents.front(); + for (size_t n = 0; n < count && n < contents.size() && entry_ptr != nullptr; ++n) { + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->assign(user_context, array[n]); + pointers.assign(user_context, n, storage_ptr->data()); + entry_ptr = entry_ptr->next_ptr; + } +} + +void StringTable::assign(void *user_context, size_t index, const char *str, size_t length) { + if (length == 0) { length = strlen(str); } + LinkedList::EntryType *entry_ptr = contents.front(); + for (size_t n = 0; n < contents.size() && entry_ptr != nullptr; ++n) { + if (n == index) { + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->assign(user_context, str, length); + pointers.assign(user_context, n, storage_ptr->data()); + break; + } + entry_ptr = entry_ptr->next_ptr; + } +} + +void StringTable::append(void *user_context, const char *str, size_t length) { + LinkedList::EntryType *entry_ptr = contents.append(user_context); + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->initialize(user_context, 0, contents.current_allocator()); + storage_ptr->assign(user_context, str, length); + pointers.append(user_context, storage_ptr->data()); +} + +void StringTable::prepend(void *user_context, const char *str, size_t length) { + LinkedList::EntryType *entry_ptr = contents.prepend(user_context); + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->initialize(user_context, 0, contents.current_allocator()); + storage_ptr->assign(user_context, str, length); + pointers.prepend(user_context, storage_ptr->data()); +} + +size_t StringTable::parse(void *user_context, const char *str, const char *delim) { + if (StringUtils::is_empty(str)) { return 0; } + + size_t delim_length = strlen(delim); + size_t total_length = strlen(str); + size_t entry_count = StringUtils::count_tokens(str, delim); + if (entry_count < 1) { return 0; } + + resize(user_context, entry_count); + + // save each entry into the table + size_t index = 0; + const char *ptr = str; + LinkedList::EntryType *entry_ptr = contents.front(); + while (!StringUtils::is_empty(ptr) && (index < entry_count)) { + size_t ptr_offset = ptr - str; + const char *next_delim = strstr(ptr, delim); + size_t token_length = (next_delim == nullptr) ? (total_length - ptr_offset) : (next_delim - ptr); + if (token_length > 0 && entry_ptr != nullptr) { + StringStorage *storage_ptr = static_cast(entry_ptr->value); + storage_ptr->assign(user_context, ptr, token_length); + pointers.assign(user_context, index, storage_ptr->data()); + entry_ptr = entry_ptr->next_ptr; + ++index; + } + ptr = (next_delim != nullptr) ? (next_delim + delim_length) : nullptr; + } + return entry_count; +} + +bool StringTable::contains(const char *str) const { + if (StringUtils::is_empty(str)) { return false; } + + const LinkedList::EntryType *entry_ptr = contents.front(); + for (size_t n = 0; n < contents.size() && entry_ptr != nullptr; ++n) { + StringStorage *storage_ptr = static_cast(entry_ptr->value); + if (storage_ptr->contains(str)) { + return true; + } + entry_ptr = entry_ptr->next_ptr; + } + + return false; +} + +const char **StringTable::data() const { + return reinterpret_cast(pointers.data()); +} + +// -- + +} // namespace Internal +} // namespace Runtime +} // namespace Halide + +#endif // HALIDE_RUNTIME_STRING_STORAGE_H diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index e551d080613b..2801f9bfedc5 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -1,9 +1,13 @@ #ifndef HALIDE_RUNTIME_INTERNAL_H #define HALIDE_RUNTIME_INTERNAL_H +#ifdef COMPILING_HALIDE_RUNTIME_TESTS +// Only allowed if building Halide runtime tests ... since they use system compiler which may be GCC or MSVS +#else #if __STDC_HOSTED__ #error "Halide runtime files must be compiled with clang in freestanding mode." #endif +#endif #ifdef __UINT8_TYPE__ typedef __INT64_TYPE__ int64_t; @@ -92,6 +96,7 @@ int strncmp(const char *s, const char *t, size_t n); size_t strlen(const char *s); const char *strchr(const char *s, int c); void *memcpy(void *s1, const void *s2, size_t n); +void *memmove(void *dest, const void *src, size_t n); int memcmp(const void *s1, const void *s2, size_t n); void *memset(void *s, int val, size_t n); // Use fopen+fileno+fclose instead of open+close - the value of the diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4130f8fddaf3..ca1e3f46acf8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,4 +39,25 @@ if (WITH_TEST_GENERATOR) add_subdirectory(generator) endif () +# FIXME: Disable the runtime tests for MSVC until we have a MS compatible header. +# +# The runtime tests include src/runtime/runtime_internal.h which was written +# to only support clang (GCC's front end is close enough it works fine as well). +# We originally setup the tests to compile with clang (in the same way as the actual +# runtime bitcode files), but that wasn't very clean and didn't integrate well with +# the other tests, so we switched to just using the native system compiler. +# Sadly MSVC isn't compatible with the current runtime_internal.h which would need +# some platform specific ifdefs for attributes and types that are causing compile +# errors. +# +cmake_dependent_option(WITH_TEST_RUNTIME "Build runtime tests" ON + "NOT MSVC" OFF) + +if (WITH_TEST_RUNTIME) + message(STATUS "Building internal runtime tests enabled") + add_subdirectory(runtime) +else () + message(STATUS "Building internal runtime tests disabled") +endif () + # FIXME: failing_with_issue is dead code :) diff --git a/test/runtime/CMakeLists.txt b/test/runtime/CMakeLists.txt new file mode 100644 index 000000000000..54c219ffa392 --- /dev/null +++ b/test/runtime/CMakeLists.txt @@ -0,0 +1,32 @@ +function(halide_define_runtime_internal_test NAME) + add_executable(runtime_internal_${NAME} ${NAME}.cpp) + target_link_libraries(runtime_internal_${NAME} PRIVATE Halide::Test) + target_include_directories(runtime_internal_${NAME} PRIVATE "${Halide_SOURCE_DIR}/src") + target_include_directories(runtime_internal_${NAME} PRIVATE "${Halide_SOURCE_DIR}/src/runtime") + target_link_libraries(runtime_internal_${NAME} PRIVATE Halide::Runtime) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Halide runtime lib has declarations for memcmp etc that conflict with GNU stdlib + target_compile_options(runtime_internal_${NAME} PRIVATE -Wno-builtin-declaration-mismatch ) + endif() + target_compile_definitions( + runtime_internal_${NAME} + PRIVATE + HALIDE_VERSION=${Halide_VERSION} + HALIDE_VERSION_MAJOR=${Halide_VERSION_MAJOR} + HALIDE_VERSION_MINOR=${Halide_VERSION_MINOR} + HALIDE_VERSION_PATCH=${Halide_VERSION_PATCH} + COMPILING_HALIDE_RUNTIME + COMPILING_HALIDE_RUNTIME_TESTS + ) + add_halide_test(runtime_internal_${NAME} GROUPS runtime_internal) +endfunction() + +# NOTE: These tests directly include runtime_internal.h which isn't compatible with MSVC +if(NOT MSVC) + halide_define_runtime_internal_test(block_allocator) + halide_define_runtime_internal_test(block_storage) + halide_define_runtime_internal_test(linked_list) + halide_define_runtime_internal_test(memory_arena) + halide_define_runtime_internal_test(string_storage) + halide_define_runtime_internal_test(string_table) +endif() \ No newline at end of file diff --git a/test/runtime/block_allocator.cpp b/test/runtime/block_allocator.cpp new file mode 100644 index 000000000000..69479901fa95 --- /dev/null +++ b/test/runtime/block_allocator.cpp @@ -0,0 +1,140 @@ +#include "common.h" + +#include "internal/block_allocator.h" +#include "internal/pointer_table.h" + +using namespace Halide::Runtime::Internal; + +namespace { + +size_t allocated_block_memory = 0; +size_t allocated_region_memory = 0; + +void allocate_block(void *user_context, MemoryBlock *block) { + block->handle = native_system_malloc(user_context, block->size); + allocated_block_memory += block->size; + + debug(user_context) << "Test : allocate_block (" + << "block=" << (void *)(block) << " " + << "block_size=" << int32_t(block->size) << " " + << "allocated_block_memory=" << int32_t(allocated_block_memory) << " " + << ") !\n"; +} + +void deallocate_block(void *user_context, MemoryBlock *block) { + native_system_free(user_context, block->handle); + allocated_block_memory -= block->size; + + debug(user_context) << "Test : deallocate_block (" + << "block=" << (void *)(block) << " " + << "block_size=" << int32_t(block->size) << " " + << "allocated_block_memory=" << int32_t(allocated_block_memory) << " " + << ") !\n"; +} + +void allocate_region(void *user_context, MemoryRegion *region) { + region->handle = (void *)1; + allocated_region_memory += region->size; + + debug(user_context) << "Test : allocate_region (" + << "region=" << (void *)(region) << " " + << "region_size=" << int32_t(region->size) << " " + << "allocated_region_memory=" << int32_t(allocated_region_memory) << " " + << ") !\n"; +} + +void deallocate_region(void *user_context, MemoryRegion *region) { + region->handle = (void *)0; + allocated_region_memory -= region->size; + + debug(user_context) << "Test : deallocate_region (" + << "region=" << (void *)(region) << " " + << "region_size=" << int32_t(region->size) << " " + << "allocated_region_memory=" << int32_t(allocated_region_memory) << " " + << ") !\n"; +} + +} // end namespace + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + SystemMemoryAllocatorFns system_allocator = {native_system_malloc, native_system_free}; + MemoryBlockAllocatorFns block_allocator = {allocate_block, deallocate_block}; + MemoryRegionAllocatorFns region_allocator = {allocate_region, deallocate_region}; + + // test class interface + { + BlockAllocator::Config config = {0}; + config.minimum_block_size = 1024; + + BlockAllocator::MemoryAllocators allocators = {system_allocator, block_allocator, region_allocator}; + BlockAllocator *instance = BlockAllocator::create(user_context, config, allocators); + + MemoryRequest request = {0}; + request.size = sizeof(int); + request.alignment = sizeof(int); + request.properties.visibility = MemoryVisibility::DefaultVisibility; + request.properties.caching = MemoryCaching::DefaultCaching; + request.properties.usage = MemoryUsage::DefaultUsage; + + MemoryRegion *r1 = instance->reserve(user_context, request); + halide_abort_if_false(user_context, r1 != nullptr); + halide_abort_if_false(user_context, allocated_block_memory == config.minimum_block_size); + halide_abort_if_false(user_context, allocated_region_memory == request.size); + + MemoryRegion *r2 = instance->reserve(user_context, request); + halide_abort_if_false(user_context, r2 != nullptr); + halide_abort_if_false(user_context, allocated_block_memory == config.minimum_block_size); + halide_abort_if_false(user_context, allocated_region_memory == (2 * request.size)); + + instance->reclaim(user_context, r1); + halide_abort_if_false(user_context, allocated_region_memory == (1 * request.size)); + + instance->destroy(user_context); + halide_abort_if_false(user_context, allocated_block_memory == 0); + halide_abort_if_false(user_context, allocated_region_memory == 0); + + BlockAllocator::destroy(user_context, instance); + } + + // stress test + { + BlockAllocator::Config config = {0}; + config.minimum_block_size = 1024; + + BlockAllocator::MemoryAllocators allocators = {system_allocator, block_allocator, region_allocator}; + BlockAllocator *instance = BlockAllocator::create(user_context, config, allocators); + + MemoryRequest request = {0}; + request.size = sizeof(int); + request.alignment = sizeof(int); + request.properties.visibility = MemoryVisibility::DefaultVisibility; + request.properties.caching = MemoryCaching::DefaultCaching; + request.properties.usage = MemoryUsage::DefaultUsage; + + static size_t test_allocations = 1000; + PointerTable pointers(user_context, test_allocations, system_allocator); + for (size_t n = 0; n < test_allocations; ++n) { + size_t count = n % 32; + count = count > 1 ? count : 1; + request.size = count * sizeof(int); + MemoryRegion *region = instance->reserve(user_context, request); + pointers.append(user_context, region); + } + + for (size_t n = 0; n < pointers.size(); ++n) { + MemoryRegion *region = static_cast(pointers[n]); + instance->reclaim(user_context, region); + } + halide_abort_if_false(user_context, allocated_region_memory == 0); + + instance->destroy(user_context); + halide_abort_if_false(user_context, allocated_block_memory == 0); + + BlockAllocator::destroy(user_context, instance); + } + + print(user_context) << "Success!\n"; + return 0; +} diff --git a/test/runtime/block_storage.cpp b/test/runtime/block_storage.cpp new file mode 100644 index 000000000000..ad7499f84378 --- /dev/null +++ b/test/runtime/block_storage.cpp @@ -0,0 +1,148 @@ +#include "common.h" + +#include "internal/block_storage.h" + +using namespace Halide::Runtime::Internal; + +struct TestStruct { + int8_t i8; + uint16_t ui16; + float f32; +}; + +template +T read_as(const BlockStorage &bs, size_t index) { + const T *ptr = static_cast(bs[index]); + return *ptr; +} + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + // test class interface + { + BlockStorage::Config config = BlockStorage::default_config(); + config.entry_size = sizeof(int); + + BlockStorage bs(user_context, config); + bs.reserve(user_context, 256); + halide_abort_if_false(user_context, bs.size() == 0); + + int a1[4] = {12, 34, 56, 78}; + bs.append(user_context, &a1[0]); + halide_abort_if_false(user_context, bs.size() == 1); + halide_abort_if_false(user_context, read_as(bs, 0) == a1[0]); + + bs.append(user_context, &a1[1]); + halide_abort_if_false(user_context, bs.size() == 2); + halide_abort_if_false(user_context, read_as(bs, 1) == a1[1]); + + bs.insert(user_context, 1, &a1[2]); + halide_abort_if_false(user_context, bs.size() == 3); + halide_abort_if_false(user_context, read_as(bs, 0) == a1[0]); + halide_abort_if_false(user_context, read_as(bs, 1) == a1[2]); // inserted here + halide_abort_if_false(user_context, read_as(bs, 2) == a1[1]); + + bs.prepend(user_context, &a1[3]); + halide_abort_if_false(user_context, bs.size() == 4); + halide_abort_if_false(user_context, read_as(bs, 0) == a1[3]); + + int a2[] = {98, 76, 54, 32, 10}; + size_t a2_size = 5; + bs.fill(user_context, a2, a2_size); + halide_abort_if_false(user_context, bs.size() == a2_size); + halide_abort_if_false(user_context, read_as(bs, 0) == a2[0]); + halide_abort_if_false(user_context, read_as(bs, 1) == a2[1]); + halide_abort_if_false(user_context, read_as(bs, 2) == a2[2]); + halide_abort_if_false(user_context, read_as(bs, 3) == a2[3]); + halide_abort_if_false(user_context, read_as(bs, 4) == a2[4]); + + int a3[] = {77, 66, 55}; + size_t a3_size = 3; + bs.insert(user_context, 2, a3, a3_size); + halide_abort_if_false(user_context, bs.size() == (a2_size + a3_size)); + halide_abort_if_false(user_context, read_as(bs, 0) == a2[0]); + halide_abort_if_false(user_context, read_as(bs, 1) == a2[1]); + halide_abort_if_false(user_context, read_as(bs, 2) == a3[0]); // a3 inserted here + halide_abort_if_false(user_context, read_as(bs, 3) == a3[1]); + halide_abort_if_false(user_context, read_as(bs, 4) == a3[2]); + halide_abort_if_false(user_context, read_as(bs, 5) == a2[2]); // a2 resumes here + halide_abort_if_false(user_context, read_as(bs, 6) == a2[3]); + halide_abort_if_false(user_context, read_as(bs, 7) == a2[4]); + + bs.pop_front(user_context); + bs.pop_front(user_context); + + bs.pop_back(user_context); + bs.pop_back(user_context); + + halide_abort_if_false(user_context, bs.size() == (a2_size + a3_size - 4)); + halide_abort_if_false(user_context, read_as(bs, 0) == a3[0]); + halide_abort_if_false(user_context, read_as(bs, 1) == a3[1]); + halide_abort_if_false(user_context, read_as(bs, 2) == a3[2]); + halide_abort_if_false(user_context, read_as(bs, 3) == a2[2]); + + bs.clear(user_context); + halide_abort_if_false(user_context, bs.size() == 0); + } + + // test copy and equality + { + BlockStorage::Config config = BlockStorage::default_config(); + config.entry_size = sizeof(int); + + int a1[] = {98, 76, 54, 32, 10}; + size_t a1_size = 5; + + int a2[] = {77, 66, 55}; + size_t a2_size = 3; + + BlockStorage bs1(user_context, config); + bs1.fill(user_context, a1, a1_size); + + BlockStorage bs2(user_context, config); + bs2.fill(user_context, a2, a2_size); + + BlockStorage bs3(bs1); + + halide_abort_if_false(user_context, bs1.size() == (a1_size)); + halide_abort_if_false(user_context, bs2.size() == (a2_size)); + halide_abort_if_false(user_context, bs3.size() == bs1.size()); + + halide_abort_if_false(user_context, bs1 != bs2); + halide_abort_if_false(user_context, bs1 == bs3); + + bs2 = bs1; + halide_abort_if_false(user_context, bs1 == bs2); + } + + // test struct storage + { + BlockStorage::Config config = BlockStorage::default_config(); + config.entry_size = sizeof(TestStruct); + + BlockStorage bs(user_context, config); + halide_abort_if_false(user_context, bs.size() == 0); + + TestStruct s1 = {8, 16, 32.0f}; + bs.append(user_context, &s1); + halide_abort_if_false(user_context, bs.size() == 1); + + const TestStruct e1 = read_as(bs, 0); + halide_abort_if_false(user_context, e1.i8 == s1.i8); + halide_abort_if_false(user_context, e1.ui16 == s1.ui16); + halide_abort_if_false(user_context, e1.f32 == s1.f32); + + TestStruct s2 = {1, 2, 3.0f}; + bs.prepend(user_context, &s2); + halide_abort_if_false(user_context, bs.size() == 2); + + const TestStruct e2 = read_as(bs, 0); + halide_abort_if_false(user_context, e2.i8 == s2.i8); + halide_abort_if_false(user_context, e2.ui16 == s2.ui16); + halide_abort_if_false(user_context, e2.f32 == s2.f32); + } + + print(user_context) << "Success!\n"; + return 0; +} diff --git a/test/runtime/common.h b/test/runtime/common.h new file mode 100644 index 000000000000..523e3b7e6797 --- /dev/null +++ b/test/runtime/common.h @@ -0,0 +1,29 @@ +#include +#include + +#include "HalideRuntime.h" +#include "msan_stubs.cpp" +#include "runtime_internal.h" +#include "to_string.cpp" + +extern "C" { + +extern int printf(const char *format, ...); + +void halide_print(void *user_context, const char *str) { + printf("%s", str); +} + +void halide_error(void *user_context, const char *msg) { + halide_print(user_context, msg); +} + +void halide_profiler_report(void *user_context) { +} + +void halide_profiler_reset() { +} + +} // extern "C" + +#include "printer.h" diff --git a/test/runtime/linked_list.cpp b/test/runtime/linked_list.cpp new file mode 100644 index 000000000000..4e2ab51da685 --- /dev/null +++ b/test/runtime/linked_list.cpp @@ -0,0 +1,91 @@ +#include "common.h" + +#include "internal/linked_list.h" + +using namespace Halide::Runtime::Internal; + +struct TestStruct { + int8_t i8; + uint16_t ui16; + float f32; +}; + +template +T read_as(const LinkedList::EntryType *entry_ptr) { + const T *ptr = static_cast(entry_ptr->value); + return *ptr; +} + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + // test class interface + { + LinkedList list(user_context, sizeof(int), 64); + halide_abort_if_false(user_context, list.size() == 0); + + const int i0 = 12; + list.append(user_context, &i0); // contents: 12 + halide_abort_if_false(user_context, list.size() == 1); + halide_abort_if_false(user_context, (list.front() != nullptr)); + halide_abort_if_false(user_context, (list.back() != nullptr)); + halide_abort_if_false(user_context, read_as(list.front()) == i0); + halide_abort_if_false(user_context, read_as(list.back()) == i0); + + const int i1 = 34; + list.append(user_context, &i1); // contents: 12, 34 + halide_abort_if_false(user_context, list.size() == 2); + halide_abort_if_false(user_context, read_as(list.back()) == i1); + + const int i2 = 56; + list.insert_before(user_context, list.back(), &i2); // contents: 12, 56, 34 + halide_abort_if_false(user_context, list.size() == 3); + halide_abort_if_false(user_context, read_as(list.back()) == i1); + + const int i3 = 78; + list.prepend(user_context, &i3); // contents: 78, 12, 56, 34 + halide_abort_if_false(user_context, list.size() == 4); + halide_abort_if_false(user_context, read_as(list.front()) == i3); + halide_abort_if_false(user_context, read_as(list.back()) == i1); + + list.pop_front(user_context); // contents: 12, 56, 34 + halide_abort_if_false(user_context, list.size() == 3); + halide_abort_if_false(user_context, read_as(list.front()) == i0); + halide_abort_if_false(user_context, read_as(list.back()) == i1); + + list.pop_back(user_context); // contents: 12, 56 + halide_abort_if_false(user_context, list.size() == 2); + halide_abort_if_false(user_context, read_as(list.front()) == i0); + halide_abort_if_false(user_context, read_as(list.back()) == i2); + + list.clear(user_context); + halide_abort_if_false(user_context, list.size() == 0); + } + + // test struct storage + { + LinkedList list(user_context, sizeof(TestStruct)); + halide_abort_if_false(user_context, list.size() == 0); + + TestStruct s1 = {8, 16, 32.0f}; + list.append(user_context, &s1); + halide_abort_if_false(user_context, list.size() == 1); + + const TestStruct e1 = read_as(list.front()); + halide_abort_if_false(user_context, e1.i8 == s1.i8); + halide_abort_if_false(user_context, e1.ui16 == s1.ui16); + halide_abort_if_false(user_context, e1.f32 == s1.f32); + + TestStruct s2 = {1, 2, 3.0f}; + list.prepend(user_context, &s2); + halide_abort_if_false(user_context, list.size() == 2); + + TestStruct e2 = read_as(list.front()); + halide_abort_if_false(user_context, e2.i8 == s2.i8); + halide_abort_if_false(user_context, e2.ui16 == s2.ui16); + halide_abort_if_false(user_context, e2.f32 == s2.f32); + } + + print(user_context) << "Success!\n"; + return 0; +} diff --git a/test/runtime/memory_arena.cpp b/test/runtime/memory_arena.cpp new file mode 100644 index 000000000000..cce3c7bf1c02 --- /dev/null +++ b/test/runtime/memory_arena.cpp @@ -0,0 +1,88 @@ +#include "common.h" + +#include "internal/memory_arena.h" + +using namespace Halide::Runtime::Internal; + +namespace { + +size_t counter = 0; + +void *allocate_system(void *user_context, size_t bytes) { + ++counter; + return native_system_malloc(user_context, bytes); +} + +void deallocate_system(void *user_context, void *ptr) { + native_system_free(user_context, ptr); + --counter; +} + +} // namespace + +struct TestStruct { + int8_t i8; + uint16_t ui16; + float f32; +}; + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + // test class interface + { + SystemMemoryAllocatorFns test_allocator = {allocate_system, deallocate_system}; + + MemoryArena::Config config = {sizeof(int), 32, 0}; + MemoryArena arena(user_context, config, test_allocator); + void *p1 = arena.reserve(user_context); + halide_abort_if_false(user_context, counter > 1); + halide_abort_if_false(user_context, p1 != nullptr); + + void *p2 = arena.reserve(user_context, true); + halide_abort_if_false(user_context, counter > 2); + halide_abort_if_false(user_context, p2 != nullptr); + halide_abort_if_false(user_context, (*static_cast(p2)) == 0); + + arena.reclaim(user_context, p1); + arena.destroy(user_context); + + halide_abort_if_false(user_context, counter == 0); + } + + // test struct allocations + { + SystemMemoryAllocatorFns test_allocator = {allocate_system, deallocate_system}; + MemoryArena::Config config = {sizeof(TestStruct), 32, 0}; + MemoryArena arena(user_context, config, test_allocator); + void *s1 = arena.reserve(user_context, true); + halide_abort_if_false(user_context, s1 != nullptr); + halide_abort_if_false(user_context, counter > 1); + halide_abort_if_false(user_context, ((TestStruct *)s1)->i8 == int8_t(0)); + halide_abort_if_false(user_context, ((TestStruct *)s1)->ui16 == uint16_t(0)); + halide_abort_if_false(user_context, ((TestStruct *)s1)->f32 == float(0)); + + arena.destroy(user_context); + + size_t count = 4 * 1024; + void *pointers[count]; + for (size_t n = 0; n < count; ++n) { + pointers[n] = arena.reserve(user_context, true); + } + + for (size_t n = 0; n < count; ++n) { + void *s1 = pointers[n]; + halide_abort_if_false(user_context, s1 != nullptr); + halide_abort_if_false(user_context, ((TestStruct *)s1)->i8 == int8_t(0)); + halide_abort_if_false(user_context, ((TestStruct *)s1)->ui16 == uint16_t(0)); + halide_abort_if_false(user_context, ((TestStruct *)s1)->f32 == float(0)); + } + + arena.destroy(user_context); + + halide_abort_if_false(user_context, counter == 0); + } + + print(user_context) << "Success!\n"; + return 0; +} diff --git a/test/runtime/string_storage.cpp b/test/runtime/string_storage.cpp new file mode 100644 index 000000000000..b7428d4440a3 --- /dev/null +++ b/test/runtime/string_storage.cpp @@ -0,0 +1,63 @@ +#include "common.h" + +#include "internal/string_storage.h" + +using namespace Halide::Runtime::Internal; + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + // test class interface + { + StringStorage ss; + halide_abort_if_false(user_context, ss.length() == 0); + + const char *ts1 = "Testing!"; + const size_t ts1_length = strlen(ts1); + ss.assign(user_context, ts1); + halide_abort_if_false(user_context, ss.length() == ts1_length); + halide_abort_if_false(user_context, ss.contains(ts1)); + + const char *ts2 = "More "; + const size_t ts2_length = strlen(ts2); + ss.prepend(user_context, ts2); + halide_abort_if_false(user_context, ss.length() == (ts1_length + ts2_length)); + halide_abort_if_false(user_context, ss.contains(ts2)); + halide_abort_if_false(user_context, ss.contains(ts1)); + + ss.append(user_context, '!'); + halide_abort_if_false(user_context, ss.length() == (ts1_length + ts2_length + 1)); + + ss.clear(user_context); + halide_abort_if_false(user_context, ss.length() == 0); + } + + // test copy and equality + { + const char *ts1 = "Test One!"; + const size_t ts1_length = strlen(ts1); + + const char *ts2 = "Test Two!"; + const size_t ts2_length = strlen(ts2); + + StringStorage ss1; + ss1.assign(user_context, ts1, ts1_length); + + StringStorage ss2; + ss2.assign(user_context, ts2, ts2_length); + + StringStorage ss3(ss1); + + halide_abort_if_false(user_context, ss1.length() == (ts1_length)); + halide_abort_if_false(user_context, ss2.length() == (ts2_length)); + halide_abort_if_false(user_context, ss3.length() == ss1.length()); + + halide_abort_if_false(user_context, ss1 != ss2); + halide_abort_if_false(user_context, ss1 == ss3); + + ss2 = ss1; + halide_abort_if_false(user_context, ss1 == ss2); + } + print(user_context) << "Success!\n"; + return 0; +} diff --git a/test/runtime/string_table.cpp b/test/runtime/string_table.cpp new file mode 100644 index 000000000000..82d0525d02f3 --- /dev/null +++ b/test/runtime/string_table.cpp @@ -0,0 +1,44 @@ +#include "common.h" + +#include "internal/string_table.h" + +using namespace Halide::Runtime::Internal; + +int main(int argc, char **argv) { + void *user_context = (void *)1; + + // test class interface + { + size_t data_size = 4; + const char *data[] = { + "one", "two", "three", "four"}; + + StringTable st1; + halide_abort_if_false(user_context, st1.size() == 0); + + st1.fill(user_context, data, data_size); + halide_abort_if_false(user_context, st1.size() == data_size); + halide_abort_if_false(user_context, strncmp(st1[0], data[0], strlen(data[0])) == 0); + halide_abort_if_false(user_context, strncmp(st1[1], data[1], strlen(data[1])) == 0); + halide_abort_if_false(user_context, strncmp(st1[2], data[2], strlen(data[2])) == 0); + halide_abort_if_false(user_context, strncmp(st1[3], data[3], strlen(data[3])) == 0); + halide_abort_if_false(user_context, st1.contains(data[0])); + halide_abort_if_false(user_context, st1.contains(data[1])); + halide_abort_if_false(user_context, st1.contains(data[2])); + halide_abort_if_false(user_context, st1.contains(data[3])); + + st1.clear(user_context); + halide_abort_if_false(user_context, st1.size() == 0); + + size_t entry_count = st1.parse(user_context, "one:two:three:four", ":"); + halide_abort_if_false(user_context, entry_count == data_size); + halide_abort_if_false(user_context, st1.size() == data_size); + halide_abort_if_false(user_context, st1.contains(data[0])); + halide_abort_if_false(user_context, st1.contains(data[1])); + halide_abort_if_false(user_context, st1.contains(data[2])); + halide_abort_if_false(user_context, st1.contains(data[3])); + } + + print(user_context) << "Success!\n"; + return 0; +}