diff --git a/CMakeLists.txt b/CMakeLists.txt index 504f487..3c06671 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,10 +40,12 @@ add_library(${PROJECT_NAME} src/ftrees.cpp src/key_file.cpp src/ptree.cpp + src/quota_area.cpp src/recovery.cpp src/rtree.cpp src/structs.cpp src/sub_block_allocator.cpp + src/transactions_area.cpp src/wfs_device.cpp src/wfs_item.cpp ) diff --git a/include/wfslib/directory.h b/include/wfslib/directory.h index 0b385e7..ea570b9 100644 --- a/include/wfslib/directory.h +++ b/include/wfslib/directory.h @@ -24,7 +24,7 @@ struct DirectoryTreeNode; class Directory : public WfsItem, public std::enable_shared_from_this { public: // TODO: Replace name with tree iterator? - Directory(std::string name, AttributesRef attributes, std::shared_ptr area, std::shared_ptr block); + Directory(std::string name, AttributesRef attributes, std::shared_ptr quota, std::shared_ptr block); std::expected, WfsError> GetObject(const std::string& name) const; std::expected, WfsError> GetDirectory(const std::string& name) const; @@ -34,14 +34,14 @@ class Directory : public WfsItem, public std::enable_shared_from_this DirectoryItemsIterator begin() const; DirectoryItemsIterator end() const; - const std::shared_ptr& area() const { return area_; } + const std::shared_ptr& quota() const { return quota_; } private: friend DirectoryItemsIterator; friend class Recovery; // TODO: We may have cyclic reference here if we do cache in area. - std::shared_ptr area_; + std::shared_ptr quota_; std::shared_ptr block_; diff --git a/include/wfslib/file.h b/include/wfslib/file.h index 777ed46..2f678d6 100644 --- a/include/wfslib/file.h +++ b/include/wfslib/file.h @@ -14,7 +14,7 @@ #include #include "wfs_item.h" -class Area; +class QuotaArea; class File : public WfsItem, public std::enable_shared_from_this { public: @@ -26,8 +26,8 @@ class File : public WfsItem, public std::enable_shared_from_this { class DataCategory3Reader; class DataCategory4Reader; - File(std::string name, AttributesRef attributes, std::shared_ptr area) - : WfsItem(std::move(name), std::move(attributes)), area_(std::move(area)) {} + File(std::string name, AttributesRef attributes, std::shared_ptr quota) + : WfsItem(std::move(name), std::move(attributes)), quota_(std::move(quota)) {} uint32_t Size() const; uint32_t SizeOnDisk() const; @@ -54,10 +54,10 @@ class File : public WfsItem, public std::enable_shared_from_this { typedef boost::iostreams::stream stream; private: - std::shared_ptr area() const { return area_; } + std::shared_ptr quota() const { return quota_; } // TODO: We may have cyclic reference here if we do cache in area. - std::shared_ptr area_; + std::shared_ptr quota_; static std::shared_ptr CreateReader(std::shared_ptr file); }; diff --git a/include/wfslib/link.h b/include/wfslib/link.h index 5d743a1..52992fc 100644 --- a/include/wfslib/link.h +++ b/include/wfslib/link.h @@ -11,14 +11,14 @@ #include "wfs_item.h" -class Area; +class QuotaArea; class Link : public WfsItem, public std::enable_shared_from_this { public: - Link(std::string name, AttributesRef attributes, std::shared_ptr area) - : WfsItem(std::move(name), std::move(attributes)), area_(std::move(area)) {} + Link(std::string name, AttributesRef attributes, std::shared_ptr quota) + : WfsItem(std::move(name), std::move(attributes)), quota_(std::move(quota)) {} private: // TODO: We may have cyclic reference here if we do cache in area. - std::shared_ptr area_; + std::shared_ptr quota_; }; diff --git a/include/wfslib/wfs_device.h b/include/wfslib/wfs_device.h index 1a3bf68..f2dd127 100644 --- a/include/wfslib/wfs_device.h +++ b/include/wfslib/wfs_device.h @@ -20,6 +20,8 @@ class BlocksDevice; class Area; +class QuotaArea; +class TransactionsArea; class WfsItem; class File; class Directory; @@ -36,12 +38,12 @@ class WfsDevice : public std::enable_shared_from_this { std::shared_ptr GetFile(const std::string& filename); std::shared_ptr GetDirectory(const std::string& filename); - std::shared_ptr GetRootArea(); + std::shared_ptr GetRootArea(); std::expected, WfsError> GetRootDirectory(); void Flush(); - std::expected, WfsError> GetTransactionsArea(bool backup_area = false); + std::expected, WfsError> GetTransactionsArea(bool backup_area = false); std::expected, WfsError> LoadMetadataBlock(const Area* area, uint32_t device_block_number, @@ -55,19 +57,24 @@ class WfsDevice : public std::enable_shared_from_this { bool encrypted, bool new_block = false) const; - uint32_t CalcIV(const Area* area, uint32_t device_block_number) const; - static std::expected, WfsError> Open(std::shared_ptr device); - // Create + static std::expected, WfsError> Create(std::shared_ptr device); private: friend class Area; + friend class QuotaArea; + + void Init(); + + uint32_t CalcIV(const Area* area, uint32_t device_block_number) const; static constexpr uint16_t header_offset() { return sizeof(MetadataBlockHeader); } auto* mutable_header() { return root_block_->get_mutable_object(header_offset()); } const auto* header() const { return root_block_->get_object(header_offset()); } + const std::shared_ptr& root_block() const { return root_block_; } + std::shared_ptr device_; std::shared_ptr root_block_; }; diff --git a/include/wfslib/wfs_item.h b/include/wfslib/wfs_item.h index adff014..2635684 100644 --- a/include/wfslib/wfs_item.h +++ b/include/wfslib/wfs_item.h @@ -13,7 +13,7 @@ #include "block.h" #include "structs.h" -class Area; +class QuotaArea; struct AttributesRef { std::shared_ptr block; @@ -34,7 +34,7 @@ class WfsItem { bool is_link() const { return attributes()->is_link(); } bool is_quota() const { return attributes()->is_directory() && attributes()->is_quota(); } - static std::expected, WfsError> Load(std::shared_ptr area, + static std::expected, WfsError> Load(std::shared_ptr quota, std::string name, AttributesRef attributes_ref); diff --git a/src/area.cpp b/src/area.cpp index 8fe87eb..74f1803 100644 --- a/src/area.cpp +++ b/src/area.cpp @@ -10,102 +10,28 @@ #include #include -#include "directory.h" -#include "free_blocks_allocator.h" #include "wfs_device.h" Area::Area(std::shared_ptr wfs_device, std::shared_ptr header_block) : wfs_device_(std::move(wfs_device)), header_block_(std::move(header_block)) {} -// static -#if 0 -std::expected, WfsError> Area::CreateRootArea(const std::shared_ptr& device) { - constexpr uint32_t kTransactionsAreaEnd = 0x1000; - - auto block = MetadataBlock::LoadBlock(device, /*block_number=*/0, Block::BlockSize::Regular, /*iv=*/0, - /*check_hash=*/false, /*load_data=*/false); - if (!block.has_value()) { - return std::unexpected(block.error()); - } - - uint32_t blocks_count = - device->device()->SectorsCount() >> (Block::BlockSize::Regular - device->device()->Log2SectorSize()); - - AttributesBlock attributes{*block, sizeof(MetadataBlockHeader) + offsetof(WfsHeader, root_area_attributes)}; - auto area = std::make_shared(device, nullptr, *block, "", attributes); - +void Area::Init(std::shared_ptr parent_area, uint32_t blocks_count, Block::BlockSize block_size) { std::random_device rand_device; std::default_random_engine rand_engine{rand_device()}; std::uniform_int_distribution random_iv_generator(std::numeric_limits::min(), std::numeric_limits::max()); - auto* header = area->mutable_header(); + auto* header = mutable_header(); std::fill(reinterpret_cast(header), reinterpret_cast(header + 1), std::byte{0}); header->iv = random_iv_generator(rand_engine); header->blocks_count = blocks_count; - header->root_directory_block_number = kRootDirectoryBlockNumber; - header->shadow_directory_block_number_1 = kShadowDirectory1BlockNumber; - header->shadow_directory_block_number_2 = kShadowDirectory2BlockNumber; - header->depth = 0; - header->block_size_log2 = static_cast(Block::BlockSize::Regular); + header->depth = parent_area ? parent_area->header()->depth.value() + 1 : 0; + header->block_size_log2 = static_cast(block_size); header->large_block_size_log2 = header->block_size_log2.value() + static_cast(Block::BlockSizeType::Large); header->large_block_cluster_size_log2 = header->block_size_log2.value() + static_cast(Block::BlockSizeType::LargeCluster); - header->area_type = static_cast(WfsAreaHeader::AreaType::QuotaArea); header->maybe_always_zero = 0; header->remainder_blocks_count = 0; - header->first_fragments[0].block_number = 0; - header->first_fragments[0].blocks_count = area->ToAbsoluteBlocksCount(blocks_count); - header->fragments_log2_block_size = static_cast(Block::BlockSize::Basic); - - auto* wfs_header = area->mutable_wfs_header(); - std::fill(reinterpret_cast(wfs_header), reinterpret_cast(wfs_header + 1), std::byte{0}); - wfs_header->iv = random_iv_generator(rand_engine); - wfs_header->device_type = static_cast(DeviceType::USB); // TODO - wfs_header->version = WFS_VERSION; - wfs_header->root_area_attributes.flags = Attributes::DIRECTORY | Attributes::AREA_SIZE_REGULAR | Attributes::QUOTA; - wfs_header->root_area_attributes.blocks_count = blocks_count; - wfs_header->transactions_area_block_number = area->ToAbsoluteBlockNumber(kTransactionsBlockNumber); - wfs_header->transactions_area_blocks_count = - kTransactionsAreaEnd - wfs_header->transactions_area_block_number.value(); - - // Initialize FreeBlocksAllocator: - auto free_blocks_allocator_block = area->GetMetadataBlock(kFreeBlocksAllocatorBlockNumber, /*new_block=*/true); - if (!free_blocks_allocator_block.has_value()) { - return std::unexpected(free_blocks_allocator_block.error()); - } - auto free_blocks_allocator = std::make_unique(area, std::move(*free_blocks_allocator_block)); - free_blocks_allocator->Init(); - - // TODO: Initialize: - // 1. Root directory - // 2. shadow directories - // 3. Transactions area - - return area; -} -#endif - -std::expected, WfsError> Area::LoadDirectory(uint32_t area_block_number, - std::string name, - AttributesRef attributes) { - auto block = LoadMetadataBlock(area_block_number); - if (!block.has_value()) - return std::unexpected(WfsError::kDirectoryCorrupted); - return std::make_shared(std::move(name), std::move(attributes), shared_from_this(), std::move(*block)); -} - -std::expected, WfsError> Area::LoadRootDirectory(std::string name, - AttributesRef attributes) { - return LoadDirectory(header()->root_directory_block_number.value(), std::move(name), std::move(attributes)); -} - -std::expected, WfsError> Area::GetShadowDirectory1() { - return LoadDirectory(header()->shadow_directory_block_number_1.value(), ".shadow_dir_1", {}); -} - -std::expected, WfsError> Area::GetShadowDirectory2() { - return LoadDirectory(header()->shadow_directory_block_number_2.value(), ".shadow_dir_2", {}); } std::expected, WfsError> Area::GetArea(uint32_t area_block_number, Block::BlockSize size) { @@ -135,47 +61,3 @@ std::expected, WfsError> Area::LoadDataBlock(uint32_t are return wfs_device_->LoadDataBlock(this, to_device_block_number(area_block_number), size, data_size, std::move(data_hash), encrypted, new_block); } - -std::expected, WfsError> Area::GetFreeBlocksAllocator() { - auto block = LoadMetadataBlock(kFreeBlocksAllocatorBlockNumber); - if (!block.has_value()) - return std::unexpected(WfsError::kFreeBlocksAllocatorCorrupted); - return std::make_unique(shared_from_this(), std::move(*block)); -} - -uint32_t Area::ReservedBlocksCount() const { - uint32_t reserved_blocks = kReservedAreaBlocks; - if (is_root_area()) { - // Root area also reserve for transaction - reserved_blocks += to_area_blocks_count(wfs_device_->header()->transactions_area_blocks_count.value()); - } - return reserved_blocks; -} - -std::expected, WfsError> Area::AllocMetadataBlock() { - auto allocator = GetFreeBlocksAllocator(); - if (!allocator) - return std::unexpected(allocator.error()); - auto res = (*allocator)->AllocBlocks(1, Block::BlockSizeType::Single, true); - if (!res) - return std::unexpected(kNoSpace); - return LoadMetadataBlock((*res)[0], /*new_block=*/true); -} - -std::expected, WfsError> Area::AllocDataBlocks(uint32_t chunks_count, - Block::BlockSizeType chunk_size) { - auto allocator = GetFreeBlocksAllocator(); - if (!allocator) - return std::unexpected(allocator.error()); - auto res = (*allocator)->AllocBlocks(chunks_count, chunk_size, false); - if (!res) - return std::unexpected(kNoSpace); - return *res; -} - -bool Area::DeleteBlocks(uint32_t block_number, uint32_t blocks_count) { - auto allocator = GetFreeBlocksAllocator(); - if (!allocator) - return false; - return (*allocator)->AddFreeBlocks({block_number, blocks_count}); -} diff --git a/src/area.h b/src/area.h index 8b87482..8353657 100644 --- a/src/area.h +++ b/src/area.h @@ -15,26 +15,15 @@ #include "structs.h" class WfsDevice; -class Directory; -class FreeBlocksAllocator; -struct AttributesRef; -class Area : public std::enable_shared_from_this { +class Area { public: Area(std::shared_ptr wfs_device, std::shared_ptr header_block); - // static std::expected, WfsError> CreateRootArea(const std::shared_ptr& device); + // TODO Create transactions area std::expected, WfsError> GetArea(uint32_t area_block_number, Block::BlockSize size); - std::expected, WfsError> LoadRootDirectory(std::string name, AttributesRef attributes); - std::expected, WfsError> GetShadowDirectory1(); - std::expected, WfsError> GetShadowDirectory2(); - - std::expected, WfsError> LoadDirectory(uint32_t area_block_number, - std::string name, - AttributesRef attributes); - std::expected, WfsError> LoadMetadataBlock(uint32_t area_block_number, bool new_block = false) const; std::expected, WfsError> LoadMetadataBlock(uint32_t area_block_number, @@ -47,11 +36,6 @@ class Area : public std::enable_shared_from_this { bool encrypted, bool new_block = false) const; - std::expected, WfsError> AllocMetadataBlock(); - std::expected, WfsError> AllocDataBlocks(uint32_t chunks_count, - Block::BlockSizeType chunk_size); - bool DeleteBlocks(uint32_t area_block_number, uint32_t area_blocks_count); - uint32_t to_area_block_number(uint32_t device_block_number) const { return to_area_blocks_count(device_block_number - header_block_->device_block_number()); } @@ -76,25 +60,14 @@ class Area : public std::enable_shared_from_this { // In area blocks count uint32_t blocks_count() const { return header()->blocks_count.value(); } - uint32_t ReservedBlocksCount() const; - - std::expected, WfsError> GetFreeBlocksAllocator(); - - private: + protected: friend class Recovery; friend class WfsDevice; - friend class TestArea; - static constexpr uint32_t kFreeBlocksAllocatorBlockNumber = 1; - static constexpr uint32_t kFreeBlocksAllocatorInitialFTreeBlockNumber = 2; - static constexpr uint32_t kRootDirectoryBlockNumber = 3; - static constexpr uint32_t kShadowDirectory1BlockNumber = 4; - static constexpr uint32_t kShadowDirectory2BlockNumber = 5; - static constexpr uint32_t kReservedAreaBlocks = 6; + void Init(std::shared_ptr parent_area, uint32_t blocks_count, Block::BlockSize block_size); - static constexpr uint32_t kTransactionsBlockNumber = 6; + const std::shared_ptr& header_block() const { return header_block_; }; - bool is_root_area() const { return device_block_number() == 0; } uint16_t header_offset() const { return sizeof(MetadataBlockHeader) + (is_root_area() ? sizeof(WfsDeviceHeader) : 0); } @@ -102,8 +75,11 @@ class Area : public std::enable_shared_from_this { WfsAreaHeader* mutable_header() { return header_block_->get_mutable_object(header_offset()); } const WfsAreaHeader* header() const { return header_block_->get_object(header_offset()); } - uint32_t IV(uint32_t block_number) const; + const std::shared_ptr& wfs_device() { return wfs_device_; } + bool is_root_area() const { return device_block_number() == 0; } + + private: std::shared_ptr wfs_device_; std::shared_ptr header_block_; }; diff --git a/src/directory.cpp b/src/directory.cpp index c70c915..e1422fc 100644 --- a/src/directory.cpp +++ b/src/directory.cpp @@ -9,8 +9,8 @@ #include -#include "area.h" #include "file.h" +#include "quota_area.h" #include "structs.h" #include "sub_block_allocator.h" @@ -19,15 +19,15 @@ using DirectoryTree = SubBlockAllocator; Directory::Directory(std::string name, AttributesRef attributes, - std::shared_ptr area, + std::shared_ptr quota, std::shared_ptr block) - : WfsItem(std::move(name), std::move(attributes)), area_(std::move(area)), block_(std::move(block)) {} + : WfsItem(std::move(name), std::move(attributes)), quota_(std::move(quota)), block_(std::move(block)) {} std::expected, WfsError> Directory::GetObject(const std::string& name) const { const auto attributes_block = FindObjectAttributes(block_, name); if (!attributes_block.has_value()) return std::unexpected(attributes_block.error()); - return WfsItem::Load(area_, name, std::move(*attributes_block)); + return WfsItem::Load(quota_, name, std::move(*attributes_block)); } std::expected, WfsError> Directory::GetDirectory(const std::string& name) const { @@ -134,7 +134,7 @@ std::expected Directory::FindObjectAttributes(const std // Not found return std::unexpected(kItemNotFound); } - auto next_block = area_->LoadMetadataBlock(last_block_number); + auto next_block = quota_->LoadMetadataBlock(last_block_number); if (!next_block.has_value()) return std::unexpected(kDirectoryCorrupted); return FindObjectAttributes(std::move(*next_block), name); diff --git a/src/directory_items_iterator.cpp b/src/directory_items_iterator.cpp index 2444b24..6b82153 100644 --- a/src/directory_items_iterator.cpp +++ b/src/directory_items_iterator.cpp @@ -10,8 +10,8 @@ #include #include -#include "area.h" #include "directory.h" +#include "quota_area.h" #include "structs.h" #include "sub_block_allocator.h" @@ -54,7 +54,7 @@ DirectoryItemsIterator& DirectoryItemsIterator::operator++() { auto* header = node_state_->block->get_object(0); if (!(header->block_flags.value() & header->Flags::EXTERNAL_DIRECTORY_TREE)) { // This is just internal node (in the directories trees tree), it just point to another tree - auto block = directory_->area()->LoadMetadataBlock( + auto block = directory_->quota()->LoadMetadataBlock( static_cast(node_state_->node)->get_next_allocator_block_number().value()); if (!block.has_value()) throw WfsException(WfsError::kDirectoryCorrupted); @@ -98,7 +98,7 @@ DirectoryItemsIterator::value_type DirectoryItemsIterator::operator*() { auto external_node = static_cast(node_state_->node); const AttributesRef attributes{block, external_node->get_item(node_state_->current_index).value()}; auto name = attributes.get()->GetCaseSensitiveName(node_state_->path); - return {name, WfsItem::Load(directory_->area(), name, std::move(attributes))}; + return {name, WfsItem::Load(directory_->quota(), name, std::move(attributes))}; } else { // Should not happen (can't happen, the iterator should stop only at external trees) throw std::logic_error("Should not happen!"); diff --git a/src/file.cpp b/src/file.cpp index 11354be..868e19a 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -8,8 +8,8 @@ #include "file.h" #include -#include "area.h" #include "block.h" +#include "quota_area.h" #include "structs.h" struct FileDataChunkInfo { @@ -149,16 +149,16 @@ class File::RegularDataCategoryReader : public File::DataCategoryReader { protected: virtual size_t GetBlocksLog2CountInDataBlock() const = 0; virtual Block::BlockSize GetDataBlockSize() const { - return static_cast(file_->area()->block_size_log2() + GetBlocksLog2CountInDataBlock()); + return static_cast(file_->quota()->block_size_log2() + GetBlocksLog2CountInDataBlock()); } std::shared_ptr current_data_block; void LoadDataBlock(uint32_t block_number, uint32_t data_size, Block::HashRef data_hash) { if (current_data_block && - file_->area()->to_area_block_number(current_data_block->device_block_number()) == block_number) + file_->quota()->to_area_block_number(current_data_block->device_block_number()) == block_number) return; - auto block = file_->area()->LoadDataBlock(block_number, GetDataBlockSize(), data_size, std::move(data_hash), - !(file_->attributes()->flags.value() & Attributes::UNENCRYPTED_FILE)); + auto block = file_->quota()->LoadDataBlock(block_number, GetDataBlockSize(), data_size, std::move(data_hash), + !(file_->attributes()->flags.value() & Attributes::UNENCRYPTED_FILE)); if (!block.has_value()) throw WfsException(WfsError::kFileDataCorrupted); current_data_block = std::move(*block); @@ -260,9 +260,9 @@ class File::DataCategory4Reader : public File::DataCategory3Reader { void LoadMetadataBlock(uint32_t block_number) { if (current_metadata_block && - file_->area()->to_area_block_number(current_metadata_block->device_block_number()) == block_number) + file_->quota()->to_area_block_number(current_metadata_block->device_block_number()) == block_number) return; - auto metadata_block = file_->area()->LoadMetadataBlock(block_number); + auto metadata_block = file_->quota()->LoadMetadataBlock(block_number); if (!metadata_block.has_value()) throw WfsException(WfsError::kFileMetadataCorrupted); current_metadata_block = std::move(*metadata_block); @@ -270,7 +270,7 @@ class File::DataCategory4Reader : public File::DataCategory3Reader { size_t ClustersInBlock() const { size_t clusters_in_block = - (file_->area()->block_size() - sizeof(MetadataBlockHeader)) / sizeof(DataBlocksClusterMetadata); + (file_->quota()->block_size() - sizeof(MetadataBlockHeader)) / sizeof(DataBlocksClusterMetadata); clusters_in_block = std::min(clusters_in_block, static_cast(48)); return clusters_in_block; } diff --git a/src/free_blocks_allocator.cpp b/src/free_blocks_allocator.cpp index 8fb28a7..21052ef 100644 --- a/src/free_blocks_allocator.cpp +++ b/src/free_blocks_allocator.cpp @@ -41,30 +41,28 @@ size_t BlockSizeToIndex(Block::BlockSizeType size) { FreeBlocksAllocator::FreeBlocksAllocator(std::shared_ptr area, std::shared_ptr block) : area_(std::move(area)), block_(std::move(block)) {} -void FreeBlocksAllocator::Init() { - uint32_t initial_free_blocks_number = area_->ReservedBlocksCount(); - uint32_t free_blocks_count = area_->blocks_count() - initial_free_blocks_number; - +void FreeBlocksAllocator::Init(std::vector initial_free_blocks) { // Init cach info auto* header = mutable_header(); - header->free_blocks_count = free_blocks_count; + header->free_blocks_count = 0; header->always_one = 1; if (BlocksCacheSizeLog2()) { - uint32_t cache_end_block_number = initial_free_blocks_number + (1 << BlocksCacheSizeLog2()); + uint32_t cache_end_block_number = initial_free_blocks[0].block_number + (1 << BlocksCacheSizeLog2()); cache_end_block_number = area_->to_device_block_number(cache_end_block_number); cache_end_block_number = align_ceil_pow2( cache_end_block_number, BlocksCacheSizeLog2() + area_->block_size_log2() - Block::BlockSize::Basic); cache_end_block_number = area_->to_area_block_number(cache_end_block_number); - uint32_t cache_free_blocks_count = cache_end_block_number - initial_free_blocks_number; - if (cache_free_blocks_count > free_blocks_count) { - cache_free_blocks_count = free_blocks_count; + uint32_t cache_free_blocks_count = cache_end_block_number - initial_free_blocks[0].block_number; + if (cache_free_blocks_count > initial_free_blocks[0].blocks_count) { + cache_free_blocks_count = initial_free_blocks[0].blocks_count; } - header->free_blocks_cache = initial_free_blocks_number; + header->free_blocks_cache = initial_free_blocks[0].block_number; header->free_blocks_cache_count = cache_free_blocks_count; + header->free_blocks_count += cache_free_blocks_count; - initial_free_blocks_number += cache_free_blocks_count; - free_blocks_count -= cache_free_blocks_count; + initial_free_blocks[0].block_number += cache_free_blocks_count; + initial_free_blocks[0].blocks_count -= cache_free_blocks_count; } else { header->free_blocks_cache = 0; header->free_blocks_cache_count = 0; @@ -78,8 +76,8 @@ void FreeBlocksAllocator::Init() { FTrees ftrees{std::move(ftree_block)}; ftrees.Init(); eptree.insert({0, 2}); - if (free_blocks_count) - AddFreeBlocks({initial_free_blocks_number, free_blocks_count}); + for (const auto& free_range : initial_free_blocks) + AddFreeBlocks(free_range); } uint32_t FreeBlocksAllocator::AllocFreeBlockFromCache() { diff --git a/src/free_blocks_allocator.h b/src/free_blocks_allocator.h index 86e4923..ee275ef 100644 --- a/src/free_blocks_allocator.h +++ b/src/free_blocks_allocator.h @@ -47,7 +47,7 @@ class FreeBlocksAllocator { FreeBlocksAllocator(std::shared_ptr area, std::shared_ptr block); virtual ~FreeBlocksAllocator() = default; - void Init(); + void Init(std::vector initial_free_blocks); uint32_t AllocFreeBlockFromCache(); uint32_t FindSmallestFreeBlockExtent(uint32_t near, std::vector& allocated); diff --git a/src/quota_area.cpp b/src/quota_area.cpp new file mode 100644 index 0000000..a81d853 --- /dev/null +++ b/src/quota_area.cpp @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2024 koolkdev + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ + +#include "quota_area.h" + +#include +#include +#include + +#include "directory.h" +#include "free_blocks_allocator.h" +#include "utils.h" +#include "wfs_device.h" + +QuotaArea::QuotaArea(std::shared_ptr wfs_device, std::shared_ptr header_block) + : Area(std::move(wfs_device), std::move(header_block)) {} + +// static +std::expected, WfsError> QuotaArea::Create(std::shared_ptr wfs_device, + std::shared_ptr parent_area, + uint32_t blocks_count, + Block::BlockSize block_size, + const std::vector& fragments) { + // TODO: size + std::shared_ptr block; + if (parent_area) { + auto loaded_block = parent_area->LoadMetadataBlock(fragments[0].block_number); + if (!loaded_block.has_value()) + return std::unexpected(loaded_block.error()); + } else { + block = wfs_device->root_block(); + } + auto quota = std::make_shared(wfs_device, std::move(block)); + quota->Init(parent_area, blocks_count, block_size, fragments); + return quota; +} + +std::expected, WfsError> QuotaArea::LoadDirectory(uint32_t area_block_number, + std::string name, + AttributesRef attributes) { + auto block = LoadMetadataBlock(area_block_number); + if (!block.has_value()) + return std::unexpected(WfsError::kDirectoryCorrupted); + return std::make_shared(std::move(name), std::move(attributes), shared_from_this(), std::move(*block)); +} + +std::expected, WfsError> QuotaArea::LoadRootDirectory(std::string name, + AttributesRef attributes) { + return LoadDirectory(header()->root_directory_block_number.value(), std::move(name), std::move(attributes)); +} + +std::expected, WfsError> QuotaArea::GetShadowDirectory1() { + return LoadDirectory(header()->shadow_directory_block_number_1.value(), ".shadow_dir_1", {}); +} + +std::expected, WfsError> QuotaArea::GetShadowDirectory2() { + return LoadDirectory(header()->shadow_directory_block_number_2.value(), ".shadow_dir_2", {}); +} + +std::expected, WfsError> QuotaArea::LoadQuotaArea(uint32_t area_block_number, + Block::BlockSize size) { + auto area_metadata_block = LoadMetadataBlock(area_block_number, size); + if (!area_metadata_block.has_value()) + return std::unexpected(WfsError::kAreaHeaderCorrupted); + return std::make_shared(wfs_device(), std::move(*area_metadata_block)); +} + +std::expected, WfsError> QuotaArea::GetFreeBlocksAllocator() { + auto block = LoadMetadataBlock(kFreeBlocksAllocatorBlockNumber); + if (!block.has_value()) + return std::unexpected(WfsError::kFreeBlocksAllocatorCorrupted); + return std::make_unique(shared_from_this(), std::move(*block)); +} + +std::expected, WfsError> QuotaArea::AllocMetadataBlock() { + auto allocator = GetFreeBlocksAllocator(); + if (!allocator) + return std::unexpected(allocator.error()); + auto res = (*allocator)->AllocBlocks(1, Block::BlockSizeType::Single, true); + if (!res) + return std::unexpected(kNoSpace); + return LoadMetadataBlock((*res)[0], /*new_block=*/true); +} + +std::expected, WfsError> QuotaArea::AllocDataBlocks(uint32_t chunks_count, + Block::BlockSizeType chunk_size) { + auto allocator = GetFreeBlocksAllocator(); + if (!allocator) + return std::unexpected(allocator.error()); + auto res = (*allocator)->AllocBlocks(chunks_count, chunk_size, false); + if (!res) + return std::unexpected(kNoSpace); + return *res; +} + +std::expected, WfsError> QuotaArea::AllocAreaBlocks(uint32_t blocks_count) { + uint32_t chunks_count = + (blocks_count + (1 << Block::BlockSizeType::LargeCluster) - 1) >> Block::BlockSizeType::LargeCluster; + auto allocator = GetFreeBlocksAllocator(); + if (!allocator) + return std::unexpected(allocator.error()); + auto res = (*allocator)->AllocAreaBlocks(chunks_count, Block::BlockSizeType::LargeCluster); + if (!res) + return std::unexpected(kNoSpace); + return *res | std::views::transform([](const auto& frag) { + return QuotaFragment{frag.block_number, frag.blocks_count}; + }) | std::ranges::to(); +} + +bool QuotaArea::DeleteBlocks(uint32_t block_number, uint32_t blocks_count) { + auto allocator = GetFreeBlocksAllocator(); + if (!allocator) + return false; + return (*allocator)->AddFreeBlocks({block_number, blocks_count}); +} + +void QuotaArea::Init(std::shared_ptr parent_area, + uint32_t blocks_count, + Block::BlockSize block_size, + const std::vector& fragments) { + Area::Init(parent_area, blocks_count, block_size); + + auto* header = mutable_header(); + header->root_directory_block_number = kRootDirectoryBlockNumber; + header->shadow_directory_block_number_1 = kShadowDirectory1BlockNumber; + header->shadow_directory_block_number_2 = kShadowDirectory2BlockNumber; + header->area_type = static_cast(WfsAreaHeader::AreaType::QuotaArea); + + for (const auto& [dst, src] : std::views::zip(header->first_fragments, fragments)) { + dst.block_number = src.block_number; + dst.blocks_count = src.blocks_count; + } + + header->fragments_log2_block_size = + static_cast(parent_area ? parent_area->block_size_log2() : size_t{Block::BlockSize::Basic}); + uint32_t blocks_count_in_parent_size = to_device_blocks_count(blocks_count); + if (parent_area) + blocks_count_in_parent_size = parent_area->to_area_blocks_count(blocks_count_in_parent_size); + uint32_t total_blocks_count_in_parent_size = + std::accumulate(fragments.begin(), fragments.end(), uint32_t{0}, + [](auto acc, const auto& frag) { return acc + frag.blocks_count; }); + header->remainder_blocks_count = + static_cast(blocks_count_in_parent_size - total_blocks_count_in_parent_size); + + auto* quota_header = mutable_quota_header(); + quota_header->max_fragments_count = static_cast(std::size(quota_header->fragments)); + quota_header->fragments_log2_block_size = static_cast(header->fragments_log2_block_size.value()); + for (const auto& [dst, src] : std::views::zip(quota_header->fragments, fragments)) { + dst.block_number = src.block_number; + dst.blocks_count = src.blocks_count; + } + + // Initialize FreeBlocksAllocator: + auto free_blocks_allocator_block = + throw_if_error(LoadMetadataBlock(kFreeBlocksAllocatorBlockNumber, /*new_block=*/true)); + + auto free_blocks_allocator = + std::make_unique(shared_from_this(), std::move(free_blocks_allocator_block)); + auto quota_free_blocks = + fragments | std::views::transform([&](const auto& frag) { + return FreeBlocksRangeInfo{ + to_area_block_number(parent_area ? parent_area->to_device_block_number(frag.block_number) + : frag.block_number), + to_area_blocks_count(parent_area ? parent_area->to_device_blocks_count(frag.blocks_count) + : frag.blocks_count)}; + }) | + std::ranges::to(); + + // Decrease reserved blocks from first block + uint32_t reserved_blocks = kReservedAreaBlocks; + if (is_root_area()) { + // Root area also reserve for transaction + reserved_blocks += to_area_blocks_count(wfs_device()->header()->transactions_area_blocks_count.value()); + } + quota_free_blocks.front().block_number += reserved_blocks; + quota_free_blocks.front().blocks_count -= reserved_blocks; + + // Decrease spare blocks from last block + quota_free_blocks.back().blocks_count -= + to_area_blocks_count(parent_area ? parent_area->to_device_blocks_count(header->remainder_blocks_count.value()) + : header->remainder_blocks_count.value()); + free_blocks_allocator->Init(std::move(quota_free_blocks)); + + // TODO: Initialize: + // 1. Root directory + // 2. shadow directories +} diff --git a/src/quota_area.h b/src/quota_area.h new file mode 100644 index 0000000..8faf6f6 --- /dev/null +++ b/src/quota_area.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2017 koolkdev + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ + +#pragma once + +#include + +#include "area.h" +#include "errors.h" + +class Directory; +class FreeBlocksAllocator; +struct AttributesRef; + +class QuotaArea : public Area, public std::enable_shared_from_this { + public: + struct QuotaFragment { + uint32_t block_number; + uint32_t blocks_count; + }; + + QuotaArea(std::shared_ptr wfs_device, std::shared_ptr header_block); + + // If parent_area null it is root area + static std::expected, WfsError> Create(std::shared_ptr wfs_device, + std::shared_ptr parent_area, + uint32_t blocks_count, + Block::BlockSize block_size, + const std::vector& fragments); + + std::expected, WfsError> LoadQuotaArea(uint32_t area_block_number, Block::BlockSize size); + + std::expected, WfsError> LoadRootDirectory(std::string name, AttributesRef attributes); + std::expected, WfsError> GetShadowDirectory1(); + std::expected, WfsError> GetShadowDirectory2(); + + std::expected, WfsError> LoadDirectory(uint32_t area_block_number, + std::string name, + AttributesRef attributes); + + std::expected, WfsError> AllocMetadataBlock(); + std::expected, WfsError> AllocDataBlocks(uint32_t chunks_count, + Block::BlockSizeType chunk_size); + std::expected, WfsError> AllocAreaBlocks(uint32_t blocks_count); + bool DeleteBlocks(uint32_t area_block_number, uint32_t area_blocks_count); + + // TODO: Private + std::expected, WfsError> GetFreeBlocksAllocator(); + + private: + friend class Recovery; + friend class WfsDevice; + friend class TestArea; + + enum ReservedBlocks : uint32_t { + kAreaHader = 0, + kFreeBlocksAllocatorBlockNumber = 1, + kFreeBlocksAllocatorInitialFTreeBlockNumber = 2, + kRootDirectoryBlockNumber = 3, + kShadowDirectory1BlockNumber = 4, + kShadowDirectory2BlockNumber = 5, + kReservedAreaBlocks = 6, + }; + + uint16_t quota_header_offset() const { return header_offset() + sizeof(*header()); }; + auto* mutable_quota_header() { return header_block()->get_mutable_object(quota_header_offset()); } + const auto* quota_header() const { return header_block()->get_object(quota_header_offset()); } + + void Init(std::shared_ptr parent_area, + uint32_t blocks_count, + Block::BlockSize block_size, + const std::vector& fragments); +}; diff --git a/src/recovery.cpp b/src/recovery.cpp index 5c0541a..3cd3270 100644 --- a/src/recovery.cpp +++ b/src/recovery.cpp @@ -9,11 +9,11 @@ #include -#include "area.h" #include "blocks_device.h" #include "device_encryption.h" #include "directory.h" #include "file_device.h" +#include "quota_area.h" #include "structs.h" #include "wfs_device.h" @@ -233,11 +233,11 @@ std::expected, WfsError> Recovery::OpenUsrDirectoryWi if (!attributes.has_value() || !attributes->get()->is_quota()) continue; // ok this is quota - auto new_area = - system_save_dir->area()->GetArea(attributes->get()->directory_block_number.value(), Block::BlockSize::Regular); - if (!new_area.has_value()) - return std::unexpected(new_area.error()); - sub_area = std::move(*new_area); + auto new_quota = system_save_dir->quota()->LoadQuotaArea(attributes->get()->directory_block_number.value(), + Block::BlockSize::Regular); + if (!new_quota.has_value()) + return std::unexpected(new_quota.error()); + sub_area = std::move(*new_quota); break; } if (!sub_area) { diff --git a/src/structs.h b/src/structs.h index a638f4f..f770e2d 100644 --- a/src/structs.h +++ b/src/structs.h @@ -149,7 +149,7 @@ struct WfsAreaHeader { static_assert(sizeof(WfsAreaHeader) == 0x60, "Incorrect sizeof WfsAreaHeader"); // sizof 0x8 -struct WfsAreaFragmentsInfo { +struct WfsQuotaAreaHeader { uint16_be_t max_fragments_count; // 480 uint16_be_t fragments_log2_block_size; // block size of parent area. In root area it is the minimum block size (12) uint32_be_t fragments_count; @@ -157,25 +157,16 @@ struct WfsAreaFragmentsInfo { // sequentials blocks it will framgnet the area). parent area blocks, so in // parent area block size. }; -static_assert(sizeof(WfsAreaFragmentsInfo) == 0xF08, "Incorrect sizeof WfsAreaFragmentsInfo"); - -struct WfsQuotaArea { - WfsAreaHeader header; - - WfsAreaFragmentsInfo framgnets; -}; -static_assert(sizeof(WfsQuotaArea) == 0x60 + 0xF08, "Incorrect sizeof WfsQuotaArea"); - -struct WfsTransactionsArea { - WfsAreaHeader header; +static_assert(sizeof(WfsQuotaAreaHeader) == 0xF08, "Incorrect sizeof WfsQuotaAreaHeader"); +struct WfsTransactionsAreaHeader { // This is the area that responsible for creating transaction when making changes to the file system (until the quota // is flushed to the disk). // Most of the structures of this area are known, but it isn't interesting since it is cleared every time the WiiU // mount the file system. (transactions from previous boot that weren't flushed to the disk are lost) uint8_be_t unknown[0x3B4]; }; -static_assert(sizeof(WfsTransactionsArea) == 0x60 + 0x3B4, "Incorrect sizeof WfsTransactionsArea"); +static_assert(sizeof(WfsTransactionsAreaHeader) == 0x3B4, "Incorrect sizeof WfsTransactionsAreaHeader"); struct SubBlockAllocatorFreeListEntry { static const uint16_t FREE_MARK_CONST = 0xFEDC; diff --git a/src/transactions_area.cpp b/src/transactions_area.cpp new file mode 100644 index 0000000..f24c6b7 --- /dev/null +++ b/src/transactions_area.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 koolkdev + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ + +#include "transactions_area.h" + +#include "wfs_device.h" + +TransactionsArea::TransactionsArea(std::shared_ptr wfs_device, std::shared_ptr header_block) + : Area(std::move(wfs_device), std::move(header_block)) {} + +// static +std::expected, WfsError> TransactionsArea::Create( + std::shared_ptr wfs_device, + std::shared_ptr parent_area, + uint32_t device_block_number, + uint32_t device_blocks_count) { + auto block = wfs_device->LoadMetadataBlock(parent_area.get(), device_block_number, Block::BlockSize::Basic, + /*new_block=*/true); + if (!block.has_value()) + return std::unexpected(block.error()); + auto transactions = std::make_shared(std::move(wfs_device), std::move(*block)); + transactions->Init(parent_area, device_blocks_count); + return transactions; +} + +void TransactionsArea::Init(std::shared_ptr parent_area, uint32_t blocks_count) { + Area::Init(parent_area, blocks_count, Block::BlockSize::Basic); + + mutable_header()->area_type = static_cast(WfsAreaHeader::AreaType::TransactionsArea); + + // Initializaing transactinos area header not implemented because it is wiped on mount on the console anyway. +} diff --git a/src/transactions_area.h b/src/transactions_area.h new file mode 100644 index 0000000..c6b9af7 --- /dev/null +++ b/src/transactions_area.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2017 koolkdev + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ + +#pragma once + +#include "area.h" + +class TransactionsArea : public Area { + public: + TransactionsArea(std::shared_ptr wfs_device, std::shared_ptr header_block); + + static std::expected, WfsError> Create(std::shared_ptr wfs_device, + std::shared_ptr parent_area, + uint32_t device_block_number, + uint32_t device_blocks_count); + + private: + void Init(std::shared_ptr parent_area, uint32_t device_blocks_count); +}; diff --git a/src/wfs_device.cpp b/src/wfs_device.cpp index 2e9ac4b..74d4536 100644 --- a/src/wfs_device.cpp +++ b/src/wfs_device.cpp @@ -10,12 +10,15 @@ #include #include #include +#include #include "area.h" #include "blocks_device.h" #include "device.h" #include "directory.h" +#include "quota_area.h" #include "structs.h" +#include "transactions_area.h" WfsDevice::WfsDevice(std::shared_ptr device, std::shared_ptr root_block) : device_(std::move(device)), root_block_(std::move(root_block)) {} @@ -85,8 +88,8 @@ void WfsDevice::Flush() { device_->FlushAll(); } -std::shared_ptr WfsDevice::GetRootArea() { - return std::make_shared(shared_from_this(), root_block_); +std::shared_ptr WfsDevice::GetRootArea() { + return std::make_shared(shared_from_this(), root_block_); } std::expected, WfsError> WfsDevice::GetRootDirectory() { @@ -107,6 +110,19 @@ std::expected, WfsError> WfsDevice::Open(std::shared_ return std::make_shared(std::move(device), std::move(*block)); } +// static +std::expected, WfsError> WfsDevice::Create(std::shared_ptr device) { + auto block = Block::LoadMetadataBlock(device, /*dvice_block_number=*/0, Block::BlockSize::Regular, /*iv=*/0, + /*load_data=*/false); + if (!block.has_value()) { + return std::unexpected(block.error()); + } + + auto wfs_device = std::make_shared(std::move(device), std::move(*block)); + wfs_device->Init(); + return wfs_device; +} + std::expected, WfsError> WfsDevice::LoadMetadataBlock(const Area* area, uint32_t device_block_number, Block::BlockSize size, @@ -131,12 +147,46 @@ uint32_t WfsDevice::CalcIV(const Area* area, uint32_t device_block_number) const << (Block::BlockSize::Basic - device_->device()->Log2SectorSize())); } -std::expected, WfsError> WfsDevice::GetTransactionsArea(bool backup_area) { +std::expected, WfsError> WfsDevice::GetTransactionsArea(bool backup_area) { auto root_area = GetRootArea(); auto block = LoadMetadataBlock(root_area.get(), header()->transactions_area_block_number.value() + (backup_area ? 1 : 0), Block::BlockSize::Basic); if (!block.has_value()) return std::unexpected(WfsError::kTransactionsAreaCorrupted); - return std::make_shared(shared_from_this(), *block); + return std::make_shared(shared_from_this(), std::move(*block)); +} + +void WfsDevice::Init() { + constexpr uint32_t kTransactionsAreaEnd = 0x1000; + + uint32_t blocks_count = + device_->device()->SectorsCount() >> (Block::BlockSize::Regular - device_->device()->Log2SectorSize()); + + std::random_device rand_device; + std::default_random_engine rand_engine{rand_device()}; + std::uniform_int_distribution random_iv_generator(std::numeric_limits::min(), + std::numeric_limits::max()); + + // Initialize device header + auto* header = mutable_header(); + std::fill(reinterpret_cast(header), reinterpret_cast(header + 1), std::byte{0}); + header->iv = random_iv_generator(rand_engine); + header->device_type = static_cast(DeviceType::USB); // TODO + header->version = WFS_VERSION; + header->root_quota_attributes.flags = Attributes::DIRECTORY | Attributes::AREA_SIZE_REGULAR | Attributes::QUOTA; + header->root_quota_attributes.quota_blocks_count = blocks_count; + header->transactions_area_block_number = QuotaArea::kReservedAreaBlocks + << (Block::BlockSize::Regular - Block::BlockSize::Basic); + header->transactions_area_blocks_count = kTransactionsAreaEnd - header->transactions_area_block_number.value(); + + // Initialize root area + auto root_area = + throw_if_error(QuotaArea::Create(shared_from_this(), /*parent_area=*/nullptr, + blocks_count >> (Block::BlockSize::Regular - Block::BlockSize::Basic), + Block::BlockSize::Regular, {{0, blocks_count}})); + + auto transactions_area = throw_if_error(TransactionsArea::Create(shared_from_this(), root_area, + header->transactions_area_block_number.value(), + header->transactions_area_blocks_count.value())); } diff --git a/src/wfs_item.cpp b/src/wfs_item.cpp index 9044588..5395f78 100644 --- a/src/wfs_item.cpp +++ b/src/wfs_item.cpp @@ -7,10 +7,10 @@ #include "wfs_item.h" -#include "area.h" #include "directory.h" #include "file.h" #include "link.h" +#include "quota_area.h" WfsItem::WfsItem(std::string name, AttributesRef attributes) : name_(std::move(name)), attributes_(std::move(attributes)) {} @@ -18,13 +18,13 @@ WfsItem::WfsItem(std::string name, AttributesRef attributes) WfsItem::~WfsItem() = default; // static -std::expected, WfsError> WfsItem::Load(std::shared_ptr area, +std::expected, WfsError> WfsItem::Load(std::shared_ptr quota, std::string name, AttributesRef attributes_ref) { auto* attributes = attributes_ref.get(); if (attributes->is_link()) { // TODO, I think that the link info is in the attributes metadata - return std::make_shared(std::move(name), std::move(attributes_ref), std::move(area)); + return std::make_shared(std::move(name), std::move(attributes_ref), std::move(quota)); } else if (attributes->is_directory()) { if (attributes->flags.value() & attributes->Flags::QUOTA) { // The directory is quota, aka new area @@ -32,16 +32,16 @@ std::expected, WfsError> WfsItem::Load(std::shared_ptr< if (!(attributes->flags.value() & attributes->Flags::AREA_SIZE_BASIC) && (attributes->flags.value() & attributes->Flags::AREA_SIZE_REGULAR)) block_size = Block::BlockSize::Regular; - auto new_area = area->GetArea(attributes->directory_block_number.value(), block_size); - if (!new_area.has_value()) - return std::unexpected(new_area.error()); - return (*new_area)->LoadRootDirectory(std::move(name), std::move(attributes_ref)); + auto new_quota = quota->LoadQuotaArea(attributes->directory_block_number.value(), block_size); + if (!new_quota.has_value()) + return std::unexpected(new_quota.error()); + return (*new_quota)->LoadRootDirectory(std::move(name), std::move(attributes_ref)); } else { - return area->LoadDirectory(attributes->directory_block_number.value(), std::move(name), - std::move(attributes_ref)); + return quota->LoadDirectory(attributes->directory_block_number.value(), std::move(name), + std::move(attributes_ref)); } } else { // IsFile() - return std::make_shared(std::move(name), std::move(attributes_ref), std::move(area)); + return std::make_shared(std::move(name), std::move(attributes_ref), std::move(quota)); } }