diff --git a/cmd/traffic_cache_tool/CacheDefs.cc b/cmd/traffic_cache_tool/CacheDefs.cc new file mode 100644 index 00000000000..f924a2d5da0 --- /dev/null +++ b/cmd/traffic_cache_tool/CacheDefs.cc @@ -0,0 +1,135 @@ +/** @file + + Main program file for Cache Tool. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "CacheDefs.h" +#include +using namespace std; +using namespace ts; + +using ts::Errata; +namespace ts +{ +std::ostream & +operator<<(std::ostream &s, Bytes const &n) +{ + return s << n.count() << " bytes"; +} +std::ostream & +operator<<(std::ostream &s, Kilobytes const &n) +{ + return s << n.count() << " KB"; +} +std::ostream & +operator<<(std::ostream &s, Megabytes const &n) +{ + return s << n.count() << " MB"; +} +std::ostream & +operator<<(std::ostream &s, Gigabytes const &n) +{ + return s << n.count() << " GB"; +} +std::ostream & +operator<<(std::ostream &s, Terabytes const &n) +{ + return s << n.count() << " TB"; +} + +std::ostream & +operator<<(std::ostream &s, CacheStripeBlocks const &n) +{ + return s << n.count() << " stripe blocks"; +} +std::ostream & +operator<<(std::ostream &s, CacheStoreBlocks const &n) +{ + return s << n.count() << " store blocks"; +} +std::ostream & +operator<<(std::ostream &s, CacheDataBlocks const &n) +{ + return s << n.count() << " data blocks"; +} + +Errata +URLparser::parseURL(StringView URI) +{ + Errata zret; + static const StringView HTTP("http"); + static const StringView HTTPS("https"); + StringView scheme = URI.splitPrefix(':'); + if ((strcasecmp(scheme, HTTP) == 0) || (strcasecmp(scheme, HTTPS) == 0)) { + StringView hostname = URI.splitPrefix(':'); + if (!hostname) // i.e. port not present + { + } + } + + return zret; +} + +int +URLparser::getPort(std::string &fullURL, int &port_ptr, int &port_len) +{ + url_matcher matcher; + int n_port = -1; + int u_pos = -1; + + if (fullURL.find("https") == 0) { + u_pos = 8; + n_port = 443; + } else if (fullURL.find("http") == 0) { + u_pos = 7; + n_port = 80; + } + if (u_pos != -1) { + fullURL.insert(u_pos, ":@"); + static const StringView HTTP("http"); + static const StringView HTTPS("https"); + StringView url(fullURL.data(), (int)fullURL.size()); + + url += 9; + + StringView hostPort = url.splitPrefix(':'); + if (hostPort) // i.e. port is present + { + StringView port = url.splitPrefix('/'); + if (!port) // i.e. backslash is not present, then the rest of the url must be just port + port = url; + if (matcher.portmatch(port.begin(), port.size())) { + StringView text; + n_port = svtoi(port, &text); + if (text == port) { + port_ptr = fullURL.find(':', 9); + port_len = port.size(); + return n_port; + } + } + } + return n_port; + } else { + std::cout << "No scheme provided for: " << fullURL << std::endl; + return -1; + } +} +} \ No newline at end of file diff --git a/cmd/traffic_cache_tool/CacheDefs.h b/cmd/traffic_cache_tool/CacheDefs.h index bc589da1940..6ebffb54438 100644 --- a/cmd/traffic_cache_tool/CacheDefs.h +++ b/cmd/traffic_cache_tool/CacheDefs.h @@ -21,11 +21,15 @@ limitations under the License. */ -#if !defined(CACHE_DEFS_H) +#ifndef CACHE_DEFS_H #define CACHE_DEFS_H #include #include - +#include +#include +#include +#include "tsconfig/Errata.h" +#include namespace tag { struct bytes { @@ -49,32 +53,6 @@ typedef Scalar<1024 * Kilobytes::SCALE, off_t, tag::bytes> Megabytes; typedef Scalar<1024 * Megabytes::SCALE, off_t, tag::bytes> Gigabytes; typedef Scalar<1024 * Gigabytes::SCALE, off_t, tag::bytes> Terabytes; -std::ostream & -operator<<(std::ostream &s, Bytes const &n) -{ - return s << n.count() << " bytes"; -} -std::ostream & -operator<<(std::ostream &s, Kilobytes const &n) -{ - return s << n.count() << " KB"; -} -std::ostream & -operator<<(std::ostream &s, Megabytes const &n) -{ - return s << n.count() << " MB"; -} -std::ostream & -operator<<(std::ostream &s, Gigabytes const &n) -{ - return s << n.count() << " GB"; -} -std::ostream & -operator<<(std::ostream &s, Terabytes const &n) -{ - return s << n.count() << " TB"; -} - // Units of allocation for stripes. typedef Scalar<128 * Megabytes::SCALE, int64_t, tag::bytes> CacheStripeBlocks; // Size measurement of cache storage. @@ -83,22 +61,6 @@ typedef Scalar<8 * Kilobytes::SCALE, int64_t, tag::bytes> CacheStoreBlocks; // Size unit for content stored in cache. typedef Scalar<512, int64_t, tag::bytes> CacheDataBlocks; -std::ostream & -operator<<(std::ostream &s, CacheStripeBlocks const &n) -{ - return s << n.count() << " stripe blocks"; -} -std::ostream & -operator<<(std::ostream &s, CacheStoreBlocks const &n) -{ - return s << n.count() << " store blocks"; -} -std::ostream & -operator<<(std::ostream &s, CacheDataBlocks const &n) -{ - return s << n.count() << " data blocks"; -} - /** A cache span is a representation of raw storage. It corresponds to a raw disk, disk partition, file, or directory. */ @@ -148,6 +110,7 @@ struct SpanHeader { @internal nee VolHeadFooter */ +// the counterpart of this structure in ATS is called VolHeaderFooter class StripeMeta { public: @@ -167,10 +130,18 @@ class StripeMeta uint32_t dirty; uint32_t sector_size; uint32_t unused; // pad out to 8 byte boundary + uint16_t freelist[1]; }; +/* + @internal struct Dir in P_CacheDir.h + * size: 10bytes + */ + class CacheDirEntry { +public: +#if 0 unsigned int offset : 24; unsigned int big : 2; unsigned int size : 6; @@ -181,11 +152,108 @@ class CacheDirEntry unsigned int token : 1; unsigned int next : 16; uint16_t offset_high; +#else + uint16_t w[5]; +#endif }; class CacheVolume { }; + +class URLparser +{ +public: + bool verifyURL(std::string &url1); + Errata parseURL(StringView URI); + int getPort(std::string &fullURL, int &port_ptr, int &port_len); + +private: + // DFA regex; +}; + +class CacheURL +{ +public: + in_port_t port; + std::string scheme; + std::string url; + std::string hostname; + std::string path; + std::string query; + std::string params; + std::string fragments; + std::string user; + std::string password; + CacheURL(int port_, ts::StringView b_hostname, ts::StringView b_path, ts::StringView b_params, ts::StringView b_query, + ts::StringView b_fragments) + { + hostname.assign(b_hostname.begin(), b_hostname.size()); + port = port_; + path.assign(b_path.begin(), b_path.size()); + params.assign(b_params.begin(), b_params.size()); + query.assign(b_query.begin(), b_query.size()); + fragments.assign(b_fragments.begin(), b_fragments.size()); + } + + CacheURL(ts::StringView blob, int port_) + { + url.assign(blob.begin(), blob.size()); + port = port_; + } + + void + setCredential(char *p_user, int user_len, char *p_pass, int pass_len) + { + user.assign(p_user, user_len); + password.assign(p_pass, pass_len); + } +}; } +class DFA; +// this class matches url of the format : scheme://hostname:port/path;params?query +struct url_matcher { + // R"(^https?\:\/\/^[a-z A-Z 0-9]\.[a-z A-Z 0-9 \.]+)" + url_matcher() + { + /*if (regex.compile(R"(^https?\:\/\/^[a-z A-Z 0-9][\. a-z A-Z 0-9 ]+(\:[0-9]\/)?.*))") != 0) { + std::cout<<"Check your regular expression"< #include #include -#include #include #include #include @@ -43,6 +42,11 @@ #include #include #include +#include + +#define __STDC_FORMAT_MACROS +#include + using ts::Bytes; using ts::Megabytes; @@ -55,10 +59,19 @@ using ts::FilePath; using ts::MemView; using ts::CacheDirEntry; -constexpr int VOL_HASH_TABLE_SIZE = 32707; +constexpr int ESTIMATED_OBJECT_SIZE = 8000; +constexpr int VOL_HASH_TABLE_SIZE = 32707; +int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; CacheStoreBlocks Vol_hash_alloc_size(1024); constexpr unsigned short VOL_HASH_EMPTY = 65535; constexpr int DIR_TAG_WIDTH = 12; +constexpr int DIR_DEPTH = 4; +constexpr int SIZEOF_DIR = 10; +constexpr int MAX_ENTRIES_PER_SEGMENT = (1 << 16); +constexpr int DIR_SIZE_WIDTH = 6; +constexpr int DIR_BLOCK_SIZES = 4; +constexpr int CACHE_BLOCK_SHIFT = 9; +constexpr int CACHE_BLOCK_SIZE = (1 << CACHE_BLOCK_SHIFT); // 512, smallest sector size const Bytes ts::CacheSpan::OFFSET{CacheStoreBlocks{1}}; enum { SILENT = 0, NORMAL, VERBOSE } Verbosity = NORMAL; @@ -150,22 +163,25 @@ struct Stripe { /// Load metadata for this stripe. Errata loadMeta(); + void dir_check(); /// Initialize the live data from the loaded serialized data. void updateLiveData(enum Copy c); Span *_span; ///< Hosting span. INK_MD5 hash_id; /// hash_id - Bytes _start; ///< Offset of first byte of stripe. + Bytes _start; ///< Offset of first byte of stripe metadata. Bytes _content; ///< Start of content. CacheStoreBlocks _len; ///< Length of stripe. uint8_t _vol_idx = 0; ///< Volume index. uint8_t _type = 0; ///< Stripe type. int8_t _idx = -1; ///< Stripe index in span. + int agg_buf_pos = 0; int64_t _buckets; ///< Number of buckets per segment. int64_t _segments; ///< Number of segments. - const char *hashText = nullptr; + + std::string hashText; /// Meta copies, indexed by A/B then HEAD/FOOT. StripeMeta _meta[2][2]; @@ -173,6 +189,18 @@ struct Stripe { CacheStoreBlocks _meta_pos[2][2]; /// Directory. Chunk _directory; + CacheDirEntry const *dir = nullptr; + uint16_t *freelist = nullptr; + int dir_freelist_length(int s); + TS_INLINE CacheDirEntry *dir_segment(int s); + TS_INLINE CacheDirEntry *vol_dir_segment(int s); + size_t vol_dirlen(); + TS_INLINE int vol_headerlen(); + void vol_init_data_internal(); + void vol_init_data(); + int dir_bucket_loop_fix(CacheDirEntry *start_dir, int s); + void dir_init_segment(int s); + void dir_free_entry(CacheDirEntry *e, int s); }; Stripe::Chunk::~Chunk() @@ -203,7 +231,7 @@ Stripe::Stripe(Span *span, Bytes start, CacheStoreBlocks len) : _span(span), _st (uint64_t)_len.count()); printf("hash id of stripe is hash of %s\n", hash_text); ink_code_md5((unsigned char *)hash_text, strlen(hash_text), (unsigned char *)&hash_id); - hashText = hash_text; + hashText.assign(hash_text, strlen(hash_text)); } bool @@ -238,6 +266,44 @@ Stripe::probeMeta(MemView &mem, StripeMeta const *base_meta) return false; } +/* INK_ALIGN() is only to be used to align on a power of 2 boundary */ +#define INK_ALIGN(size, boundary) (((size) + ((boundary)-1)) & ~((boundary)-1)) + +#define ROUND_TO_STORE_BLOCK(_x) INK_ALIGN((_x), 8192) + +TS_INLINE int +Stripe::vol_headerlen() +{ + return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (this->_segments - 1)); +} + +size_t +Stripe::vol_dirlen() +{ + return vol_headerlen() + ROUND_TO_STORE_BLOCK(((size_t)this->_buckets) * DIR_DEPTH * this->_segments * SIZEOF_DIR) + + ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); +} + +void +Stripe::vol_init_data_internal() +{ + int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; + this->_buckets = + ((this->_len.count() * 8192 - (this->_content - this->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; + this->_segments = (this->_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); + this->_buckets = (this->_buckets + this->_segments - 1) / this->_segments; + this->_content = this->_start + Bytes(2 * vol_dirlen()); +} + +void +Stripe::vol_init_data() +{ + // iteratively calculate start + buckets + this->vol_init_data_internal(); + this->vol_init_data_internal(); + this->vol_init_data_internal(); +} + void Stripe::updateLiveData(enum Copy c) { @@ -246,6 +312,10 @@ Stripe::updateLiveData(enum Copy c) int64_t n_buckets; int64_t n_segments; + _content = _start; + this->vol_init_data(); + /* + * COMMENTING THIS SECTION FOR NOW TO USE THE EXACT LOGIN USED IN ATS TO CALCULATE THE NUMBER OF SEGMENTS AND BUCKETS // Past the header is the segment free list heads which if sufficiently long (> ~4K) can take // more than 1 store block. Start with a guess of 1 and adjust upwards as needed. A 2TB stripe // with an AOS of 8000 has roughly 3700 segments meaning that for even 10TB drives this loop @@ -261,9 +331,365 @@ Stripe::updateLiveData(enum Copy c) _buckets = n_buckets / n_segments; _segments = n_segments; + */ _directory._skip = header_len; } +#define dir_big(_e) ((uint32_t)((((_e)->w[1]) >> 8) & 0x3)) +#define dir_bit(_e, _w, _b) ((uint32_t)(((_e)->w[_w] >> (_b)) & 1)) +#define dir_size(_e) ((uint32_t)(((_e)->w[1]) >> 10)) +#define dir_approx_size(_e) ((dir_size(_e) + 1) * DIR_BLOCK_SIZE(dir_big(_e))) +#define dir_head(_e) dir_bit(_e, 2, 13) +#define dir_valid(_d, _e) (_d->_meta[0][0].phase == dir_phase(_e) ? vol_in_phase_valid(_d, _e) : vol_out_of_phase_valid(_d, _e)) +#define dir_tag(_e) ((uint32_t)((_e)->w[2] & ((1 << DIR_TAG_WIDTH) - 1))) +#define dir_offset(_e) \ + ((int64_t)(((uint64_t)(_e)->w[0]) | (((uint64_t)((_e)->w[1] & 0xFF)) << 16) | (((uint64_t)(_e)->w[4]) << 24))) +#define dir_set_offset(_e, _o) \ + do { \ + (_e)->w[0] = (uint16_t)_o; \ + (_e)->w[1] = (uint16_t)((((_o) >> 16) & 0xFF) | ((_e)->w[1] & 0xFF00)); \ + (_e)->w[4] = (uint16_t)((_o) >> 24); \ + } while (0) +//#define dir_segment(_s, _d) vol_dir_segment(_d, _s) +#define dir_in_seg(_s, _i) ((CacheDirEntry *)(((char *)(_s)) + (SIZEOF_DIR * (_i)))) +#define dir_next(_e) (_e)->w[3] +#define dir_phase(_e) dir_bit(_e, 2, 12) +#define DIR_BLOCK_SHIFT(_i) (3 * (_i)) +#define DIR_BLOCK_SIZE(_i) (CACHE_BLOCK_SIZE << DIR_BLOCK_SHIFT(_i)) +#define dir_set_prev(_e, _o) (_e)->w[2] = (uint16_t)(_o) +#define dir_set_next(_e, _o) (_e)->w[3] = (uint16_t)(_o) + +TS_INLINE CacheDirEntry * +Stripe::dir_segment(int s) +{ + return vol_dir_segment(s); +} + +TS_INLINE CacheDirEntry * +Stripe::vol_dir_segment(int s) +{ + return (CacheDirEntry *)(((char *)this->dir) + (s * this->_buckets) * DIR_DEPTH * SIZEOF_DIR); +} + +TS_INLINE CacheDirEntry * +dir_bucket(int64_t b, CacheDirEntry *seg) +{ + return dir_in_seg(seg, b * DIR_DEPTH); +} + +TS_INLINE CacheDirEntry * +dir_from_offset(int64_t i, CacheDirEntry *seg) +{ +#if DIR_DEPTH < 5 + if (!i) + return 0; + return dir_in_seg(seg, i); +#else + i = i + ((i - 1) / (DIR_DEPTH - 1)); + return dir_in_seg(seg, i); +#endif +} + +TS_INLINE int +vol_in_phase_valid(Stripe *d, CacheDirEntry *e) +{ + return (dir_offset(e) - 1 < ((d->_meta[0][0].write_pos + d->agg_buf_pos - d->_start) / CACHE_BLOCK_SIZE)); +} + +TS_INLINE int +vol_out_of_phase_valid(Stripe *d, CacheDirEntry *e) +{ + return (dir_offset(e) - 1 >= ((d->_meta[0][0].agg_pos - d->_start) / CACHE_BLOCK_SIZE)); +} + +TS_INLINE CacheDirEntry * +next_dir(CacheDirEntry *d, CacheDirEntry *seg) +{ + int i = dir_next(d); + return dir_from_offset(i, seg); +} +#define dir_offset(_e) \ + ((int64_t)(((uint64_t)(_e)->w[0]) | (((uint64_t)((_e)->w[1] & 0xFF)) << 16) | (((uint64_t)(_e)->w[4]) << 24))) + +TS_INLINE CacheDirEntry * +dir_bucket_row(CacheDirEntry *b, int64_t i) +{ + return dir_in_seg(b, i); +} + +TS_INLINE int64_t +dir_to_offset(const CacheDirEntry *d, const CacheDirEntry *seg) +{ +#if DIR_DEPTH < 5 + return (((char *)d) - ((char *)seg)) / SIZEOF_DIR; +#else + int64_t i = (int64_t)((((char *)d) - ((char *)seg)) / SIZEOF_DIR); + i = i - (i / DIR_DEPTH); + return i; +#endif +} + +void +Stripe::dir_free_entry(CacheDirEntry *e, int s) +{ + CacheDirEntry *seg = this->dir_segment(s); + unsigned int fo = this->_meta[0][0].freelist[s]; + unsigned int eo = dir_to_offset(e, seg); + dir_set_next(e, fo); + if (fo) { + dir_set_prev(dir_from_offset(fo, seg), eo); + } + this->_meta[0][0].freelist[s] = eo; +} + +// adds all the directory entries +// in a segment to the segment freelist +void +Stripe::dir_init_segment(int s) +{ + this->_meta[0][0].freelist[s] = 0; + CacheDirEntry *seg = this->dir_segment(s); + int l, b; + memset(seg, 0, SIZEOF_DIR * DIR_DEPTH * this->_buckets); + for (l = 1; l < DIR_DEPTH; l++) { + for (b = 0; b < this->_buckets; b++) { + CacheDirEntry *bucket = dir_bucket(b, seg); + this->dir_free_entry(dir_bucket_row(bucket, l), s); + } + } +} + +// +// Cache Directory +// + +// return value 1 means no loop +// zero indicates loop +int +dir_bucket_loop_check(CacheDirEntry *start_dir, CacheDirEntry *seg) +{ + if (start_dir == nullptr) { + return 1; + } + + CacheDirEntry *p1 = start_dir; + CacheDirEntry *p2 = start_dir; + + while (p2) { + // p1 moves by one entry per iteration + assert(p1); + p1 = next_dir(p1, seg); + // p2 moves by two entries per iteration + p2 = next_dir(p2, seg); + if (p2) { + p2 = next_dir(p2, seg); + } else { + return 1; + } + + if (p2 == p1) { + return 0; // we have a loop + } + } + return 1; +} + +// break the infinite loop in directory entries +// Note : abuse of the token bit in dir entries + +int +Stripe::dir_bucket_loop_fix(CacheDirEntry *start_dir, int s) +{ + if (!dir_bucket_loop_check(start_dir, this->dir_segment(s))) { + this->dir_init_segment(s); + return 1; + } + return 0; +} + +int +Stripe::dir_freelist_length(int s) +{ + int free = 0; + CacheDirEntry *seg = this->dir_segment(s); + // TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- + CacheDirEntry *e = dir_from_offset(this->freelist[s], seg); + if (this->dir_bucket_loop_fix(e, s)) { + return (DIR_DEPTH - 1) * this->_buckets; + } + while (e) { + free++; + e = next_dir(e, seg); + } + return free; +} + +int +compare_ushort(void const *a, void const *b) +{ + return *static_cast(a) - *static_cast(b); +} + +void +Stripe::dir_check() +{ + static int const SEGMENT_HISTOGRAM_WIDTH = 16; + int hist[SEGMENT_HISTOGRAM_WIDTH + 1] = {0}; + unsigned short chain_tag[MAX_ENTRIES_PER_SEGMENT]; + int32_t chain_mark[MAX_ENTRIES_PER_SEGMENT]; + + this->loadMeta(); + // create raw_dir pointing at the first ever dir in the stripe; + char *raw_dir = (char *)ats_memalign(ats_pagesize(), this->vol_dirlen()); + dir = (CacheDirEntry *)(raw_dir + this->vol_headerlen()); + uint64_t total_buckets = _segments * _buckets; + uint64_t total_entries = total_buckets * DIR_DEPTH; + int frag_demographics[1 << DIR_SIZE_WIDTH][DIR_BLOCK_SIZES]; + int j; + int stale = 0, in_use = 0, empty = 0; + int free = 0, head = 0, buckets_in_use = 0; + + int max_chain_length = 0; + int64_t bytes_in_use = 0; + std::cout << "Stripe '[" << hashText << "]'" << std::endl; + std::cout << " Directory Bytes: " << _segments * _buckets * SIZEOF_DIR << std::endl; + std::cout << " Segments: " << _segments << std::endl; + std::cout << " Buckets per segment: " << _buckets << std::endl; + std::cout << " Entries: " << _segments * _buckets * DIR_DEPTH << std::endl; + int fd = _span->_fd; + // read directory + pread(fd, raw_dir, this->vol_dirlen(), this->_start); + for (int s = 0; s < _segments; s++) { + CacheDirEntry *seg = this->dir_segment(s); + int seg_chain_max = 0; + int seg_empty = 0; + int seg_in_use = 0; + int seg_stale = 0; + int seg_bytes_in_use = 0; + int seg_dups = 0; + int seg_buckets_in_use = 0; + + ink_zero(chain_tag); + memset(chain_mark, -1, sizeof(chain_mark)); + for (int b = 0; b < _buckets; b++) { + CacheDirEntry *root = dir_bucket(b, seg); + int h = 0; + int chain_idx = 0; + int mark = 0; + ++seg_buckets_in_use; + // walking through the directories + for (CacheDirEntry *e = root; e; e = next_dir(e, seg)) { + if (!dir_offset(e)) { + ++seg_empty; + --seg_buckets_in_use; + // this should only happen on the first dir in a bucket + assert(nullptr == next_dir(e, seg)); + break; + } else { + int e_idx = e - seg; + ++h; + chain_tag[chain_idx++] = dir_tag(e); + if (chain_mark[e_idx] == mark) { + printf(" - Cycle of length %d detected for bucket %d\n", h, b); + } else if (chain_mark[e_idx] >= 0) { + printf(" - Entry %d is in chain %d and %d", e_idx, chain_mark[e_idx], mark); + } else { + chain_mark[e_idx] = mark; + } + + if (!dir_valid(this, e)) { + ++seg_stale; + } else { + uint64_t size = dir_approx_size(e); + if (dir_head(e)) { + ++head; + } + ++seg_in_use; + seg_bytes_in_use += size; + ++frag_demographics[dir_size(e)][dir_big(e)]; + } + } + e = next_dir(e, seg); + if (!e) { + break; + } + } + + // Check for duplicates (identical tags in the same bucket). + if (h > 1) { + unsigned short last; + qsort(chain_tag, h, sizeof(chain_tag[0]), &compare_ushort); + last = chain_tag[0]; + for (int k = 1; k < h; ++k) { + if (last == chain_tag[k]) { + ++seg_dups; + } + last = chain_tag[k]; + } + } + ++hist[std::min(h, SEGMENT_HISTOGRAM_WIDTH)]; + seg_chain_max = std::max(seg_chain_max, h); + } + int fl_size = dir_freelist_length(s); + in_use += seg_in_use; + empty += seg_empty; + stale += seg_stale; + free += fl_size; + buckets_in_use += seg_buckets_in_use; + max_chain_length = std::max(max_chain_length, seg_chain_max); + bytes_in_use += seg_bytes_in_use; + + printf(" - Segment-%d | Entries: used=%d stale=%d free=%d disk-bytes=%d Buckets: used=%d empty=%d max=%d avg=%.2f dups=%d\n", + s, seg_in_use, seg_stale, fl_size, seg_bytes_in_use, seg_buckets_in_use, seg_empty, seg_chain_max, + seg_buckets_in_use ? static_cast(seg_in_use + seg_stale) / seg_buckets_in_use : 0.0, seg_dups); + } + ////////////////// + + printf(" - Stripe | Entries: in-use=%d stale=%d free=%d Buckets: empty=%d max=%d avg=%.2f\n", in_use, stale, free, empty, + max_chain_length, buckets_in_use ? static_cast(in_use + stale) / buckets_in_use : 0); + + printf(" Chain lengths: "); + for (j = 0; j < SEGMENT_HISTOGRAM_WIDTH; ++j) { + printf(" %d=%d ", j, hist[j]); + } + printf(" %d>=%d\n", SEGMENT_HISTOGRAM_WIDTH, hist[SEGMENT_HISTOGRAM_WIDTH]); + + char tt[256]; + printf(" Total Size: %" PRIu64 "\n", static_cast(_len.count())); + printf(" Bytes in Use: %" PRIu64 " [%0.2f%%]\n", bytes_in_use, 100.0 * (static_cast(bytes_in_use) / _len.count())); + printf(" Objects: %d\n", head); + printf(" Average Size: %" PRIu64 "\n", head ? (bytes_in_use / head) : 0); + printf(" Average Frags: %.2f\n", head ? static_cast(in_use) / head : 0); + printf(" Write Position: %" PRIu64 "\n", _meta[0][0].write_pos - _content.count()); + printf(" Wrap Count: %d\n", _meta[0][0].cycle); + printf(" Phase: %s\n", _meta[0][0].phase ? "true" : "false"); + ctime_r(&_meta[0][0].create_time, tt); + tt[strlen(tt) - 1] = 0; + printf(" Sync Serial: %u\n", _meta[0][0].sync_serial); + printf(" Write Serial: %u\n", _meta[0][0].write_serial); + printf(" Create Time: %s\n", tt); + printf("\n"); + printf(" Fragment size demographics\n"); + for (int b = 0; b < DIR_BLOCK_SIZES; ++b) { + int block_size = DIR_BLOCK_SIZE(b); + int s = 0; + while (s < 1 << DIR_SIZE_WIDTH) { + for (int j = 0; j < 8; ++j, ++s) { + // The size markings are redundant. Low values (less than DIR_SHIFT_WIDTH) for larger + // base block sizes should never be used. Such entries should use the next smaller base block size. + if (b > 0 && s < 1 << DIR_BLOCK_SHIFT(1)) { + assert(frag_demographics[s][b] == 0); + continue; + } + printf(" %8d[%2d:%1d]:%06d", (s + 1) * block_size, s, b, frag_demographics[s][b]); + } + printf("\n"); + } + } + printf("\n"); + //////////////// +} + Errata Stripe::loadMeta() { @@ -290,7 +716,7 @@ Stripe::loadMeta() std::unique_ptr bulk_buff; // Buffer for bulk reads. static const size_t SBSIZE = CacheStoreBlocks::SCALE; // save some typing. alignas(SBSIZE) char stripe_buff[SBSIZE]; // Use when reading a single stripe block. - + alignas(SBSIZE) char stripe_buff2[SBSIZE]; // use to save the stripe freelist if (io_align > SBSIZE) return Errata::Message(0, 1, "Cannot load stripe ", _idx, " on span ", _span->_path, " because the I/O block alignment ", io_align, " is larger than the buffer alignment ", SBSIZE); @@ -299,11 +725,13 @@ Stripe::loadMeta() // Header A must be at the start of the stripe block. // Todo: really need to check pread() for failure. - n.assign(pread(fd, stripe_buff, SBSIZE, pos)); - data.setView(stripe_buff, n); + ssize_t headerbyteCount = pread(fd, stripe_buff2, SBSIZE, pos); + n.assign(headerbyteCount); + data.setView(stripe_buff2, n); meta = data.template at_ptr(0); + // TODO:: We need to read more data at this point to populate dir if (this->validateMeta(meta)) { - delta = Bytes(data.template at_ptr(0) - stripe_buff); + delta = Bytes(data.template at_ptr(0) - stripe_buff2); _meta[A][HEAD] = *meta; _meta_pos[A][HEAD] = round_down(pos + Bytes(delta)); pos += round_up(SBSIZE); @@ -376,6 +804,14 @@ Stripe::loadMeta() } } + n.assign(headerbyteCount); + data.setView(stripe_buff2, n); + meta = data.template at_ptr(0); + // copy freelist + freelist = (uint16_t *)malloc(_segments * sizeof(uint16_t)); + for (int i = 0; i < _segments; i++) + freelist[i] = meta->freelist[i]; + if (!zret) _directory.clear(); return zret; @@ -516,7 +952,7 @@ struct Cache { std::list _spans; std::map _volumes; std::vector globalVec_stripe; - std::unordered_set URLset; + std::unordered_set URLset; unsigned short *stripes_hash_table; }; @@ -821,7 +1257,7 @@ Errata Cache::loadURLs(FilePath const &path) { static const ts::StringView TAG_VOL("url"); - + ts::URLparser parser; Errata zret; ts::BulkFile cfile(path); @@ -836,7 +1272,13 @@ Cache::loadURLs(FilePath const &path) } else if (0 == strcasecmp(tag, TAG_VOL)) { std::string url; url.assign(blob.begin(), blob.size()); - URLset.insert(url); + int port_ptr = -1, port_len = -1; + int port = parser.getPort(url, port_ptr, port_len); + if (port_ptr >= 0 && port_len > 0) + url.erase(port_ptr, port_len + 1); // get rid of :PORT + std::cout << "port # " << port << ":" << port_ptr << ":" << port_len << ":" << url << std::endl; + ts::CacheURL *curl = new ts::CacheURL(url, port); + URLset.emplace(curl); } } } else { @@ -853,9 +1295,9 @@ Cache::dumpSpans(SpanDumpDepth depth) if (nullptr == span->_header) { std::cout << "Span: " << span->_path << " is uninitialized" << std::endl; } else { - std::cout << "Span: " << span->_path << " " << span->_header->num_volumes << " Volumes " << span->_header->num_used - << " in use " << span->_header->num_free << " free " << span->_header->num_diskvol_blks << " stripes " - << span->_header->num_blocks.value() << " blocks" << std::endl; + std::cout << "Span: " << span->_path << " #Volumes: " << span->_header->num_volumes <<" #in use: "<< span->_header->num_used + << " #free: " << span->_header->num_free << " #stripes: " << span->_header->num_diskvol_blks << " Len(bytes): " + << span->_header->num_blocks.value() << std::endl; for (auto stripe : span->_stripes) { std::cout << " : " @@ -1320,7 +1762,8 @@ VolumeConfig::load(FilePath const &path) } /* --------------------------------------------------------------------------------------- */ struct option Options[] = {{"help", 0, nullptr, 'h'}, {"spans", 1, nullptr, 's'}, {"volumes", 1, nullptr, 'v'}, - {"write", 0, nullptr, 'w'}, {"input", 1, nullptr, 'i'}, {nullptr, 0, nullptr, 0}}; + {"write", 0, nullptr, 'w'}, {"input", 1, nullptr, 'i'}, {"device", 1, nullptr, 'd'}, + {nullptr, 0, nullptr, 0}}; } Errata @@ -1390,6 +1833,18 @@ Clear_Spans(int argc, char *argv[]) Errata Find_Stripe(FilePath const &input_file_path) { + // scheme=http user=u password=p host=172.28.56.109 path=somepath query=somequery port=1234 + // input file format: scheme://hostname:port/somepath;params?somequery user=USER password=PASS + // user, password, are optional; scheme and host are required + + char hashStr[33]; + // char* h= http://user:pass@IPADDRESS/path_to_file;?port <== this is the format we need + // url_matcher matcher; + // if (matcher.match(h)) + // std::cout << h << " : is valid" << std::endl; + // else + // std::cout << h << " : is NOT valid" << std::endl; + Errata zret; Cache cache; if (input_file_path) @@ -1399,17 +1854,58 @@ Find_Stripe(FilePath const &input_file_path) cache.dumpSpans(Cache::SpanDumpDepth::SPAN); cache.build_stripe_hash_table(); for (auto host : cache.URLset) { - INK_MD5 hash; - char hashStr[33]; - ink_code_md5((unsigned char *)host.data(), host.size(), (unsigned char *)&hash); - Stripe *stripe_ = cache.key_to_stripe(&hash, host.data(), host.size()); - printf("hash of %.*s is %s: Stripe %s \n", (int)host.size(), host.data(), - ink_code_to_hex_str(hashStr, (unsigned char *)&hash), stripe_->hashText); + MD5Context ctx; + INK_MD5 hashT; + ctx.update(host->url.data(), host->url.size()); + ctx.update(&host->port, sizeof(host->port)); + ctx.finalize(hashT); + Stripe *stripe_ = cache.key_to_stripe(&hashT, host->url.data(), host->url.size()); + printf("hash of %.*s is %s: Stripe %s \n", (int)host->url.size(), host->url.data(), + ink_code_to_hex_str(hashStr, (unsigned char *)&hashT), stripe_->hashText.data()); } } return zret; } +Errata +dir_check() +{ + Errata zret; + Cache cache; + if ((zret = cache.loadSpan(SpanFile))) { + cache.dumpSpans(Cache::SpanDumpDepth::SPAN); + for (auto &stripe : cache.globalVec_stripe) { + stripe->dir_check(); + } + } + printf("\nCHECK succeeded\n"); + return zret; +} + +Errata +Clear_Span(std::string devicePath) +{ + Errata zret; + Cache cache; + if ((zret = cache.loadSpan(SpanFile))) { + cache.dumpSpans(Cache::SpanDumpDepth::SPAN); + for (auto sp : cache._spans) { + if (devicePath.size()>0 && 0 == strncmp(sp->_path.path(), devicePath.data(), devicePath.size())) { + printf("clearing %s\n", devicePath.data()); + sp->clearPermanently(); + } + } + } + return zret; +} + +Errata +Check_Freelist(std::string devicePath) +{ + Errata zret; + printf("cache or cash %s\n",devicePath.data()); + return zret; +} int main(int argc, char *argv[]) @@ -1418,6 +1914,7 @@ main(int argc, char *argv[]) int opt_val; bool help = false; FilePath input_url_file; + std::string inputFile; while (-1 != (opt_val = getopt_long(argc, argv, "h", Options, &opt_idx))) { switch (opt_val) { case 'h': @@ -1437,20 +1934,27 @@ main(int argc, char *argv[]) case 'i': input_url_file = optarg; break; + case 'd': + char *inp = strdup(optarg); + inputFile.assign(inp, strlen(inp)); + break; } } - Commands - .add("list", "List elements of the cache", - []() { return List_Stripes(Cache::SpanDumpDepth::SPAN); }) + Commands.add("list", "List elements of the cache", []() { return List_Stripes(Cache::SpanDumpDepth::SPAN); }) .subCommand(std::string("stripes"), std::string("List the stripes"), []() { return List_Stripes(Cache::SpanDumpDepth::STRIPE); }); Commands.add(std::string("clear"), std::string("Clear spans"), &Clear_Spans); + Commands.add(std::string("dir_check"), std::string("cache check")) + .subCommand(std::string("full"), std::string("Full report of the cache storage"), &dir_check) + .subCommand(std::string("freelist"),std::string("check the freelist for loop"), [&](int, char *argv[]) {return Check_Freelist(inputFile); }); Commands.add(std::string("volumes"), std::string("Volumes"), &Simulate_Span_Allocation); Commands.add(std::string("alloc"), std::string("Storage allocation")) .subCommand(std::string("free"), std::string("Allocate storage on free (empty) spans"), &Cmd_Allocate_Empty_Spans); Commands.add(std::string("find"), std::string("Find Stripe Assignment")) .subCommand(std::string("url"), std::string("URL"), [&](int, char *argv[]) { return Find_Stripe(input_url_file); }); + Commands.add(std::string("clearspan"), std::string("clear specific span")) + .subCommand(std::string("span"), std::string("device path"), [&](int, char *argv[]) { return Clear_Span(inputFile); }); Commands.setArgIndex(optind); diff --git a/cmd/traffic_cache_tool/Command.cc b/cmd/traffic_cache_tool/Command.cc index c7a98d1310f..91912b9f9b9 100644 --- a/cmd/traffic_cache_tool/Command.cc +++ b/cmd/traffic_cache_tool/Command.cc @@ -42,25 +42,26 @@ ERR_COMMAND_TAG_NOT_FOUND(char const *tag) return ts::Errata(s.str()); } -CommandTable::Command::Command() {} +CommandTable::Command::Command() +{ +} CommandTable::Command::Command(std::string const &name, std::string const &help) : _name(name), _help(help) { } -CommandTable::Command::Command(std::string const &name, std::string const &help, LeafAction const &f) - : _name(name), _help(help) +CommandTable::Command::Command(std::string const &name, std::string const &help, LeafAction const &f) : _name(name), _help(help) { _action = f; } -CommandTable::Command::Command(std::string const &name, std::string const &help, NullaryAction const &f) - : _name(name), _help(help) +CommandTable::Command::Command(std::string const &name, std::string const &help, NullaryAction const &f) : _name(name), _help(help) { _action = f; } -CommandTable::Command::~Command() { +CommandTable::Command::~Command() +{ } CommandTable::Command & diff --git a/cmd/traffic_cache_tool/Command.h b/cmd/traffic_cache_tool/Command.h index 582f46766e7..c454721c7fd 100644 --- a/cmd/traffic_cache_tool/Command.h +++ b/cmd/traffic_cache_tool/Command.h @@ -48,7 +48,7 @@ class CommandTable /// Signature for a leaf command. using LeafAction = std::function; /// Signature for a argumentless command. - using NullaryAction = std::function; + using NullaryAction = std::function; CommandTable(); @@ -59,7 +59,7 @@ class CommandTable { typedef Command self; ///< Self reference type. public: - Command(Command && that) = default; + Command(Command &&that) = default; ~Command(); /** Add a subcommand to this command. @@ -90,60 +90,84 @@ class CommandTable /** Class to hold varying types of functions. - @internal A bit ugly, I need to do better wrapping and type erasure. + @internal A bit ugly, I need to do better wrapping and type erasure. */ - class Action { + class Action + { public: /// Type of the function stored. enum Type { - NIL, ///< Nothing / empty - LEAF, ///< Leaf action (arguments) - NULLARY, ///< Nullary action. + NIL, ///< Nothing / empty + LEAF, ///< Leaf action (arguments) + NULLARY, ///< Nullary action. }; - Action() { } - Action(Action && that) { - _type = that._type; - memcpy(_data, that._data, sizeof(_data)); - that._type = NIL; + Action() {} + Action(Action &&that) + { + _type = that._type; + memcpy(_data, that._data, sizeof(_data)); + that._type = NIL; } ~Action() { this->clear(); } - Action& operator = (LeafAction const& a) { - this->clear(); - _type = LEAF; - new (_data) LeafAction(a); - return *this; + Action & + operator=(LeafAction const &a) + { + this->clear(); + _type = LEAF; + new (_data) LeafAction(a); + return *this; } - Action& operator = (NullaryAction const& a) { - this->clear(); - _type = NULLARY; - new (_data) NullaryAction(a); - return *this; + Action & + operator=(NullaryAction const &a) + { + this->clear(); + _type = NULLARY; + new (_data) NullaryAction(a); + return *this; } - Errata invoke(int argc, char *argv[]) { - assert(LEAF == _type); - return (*reinterpret_cast(_data))(argc, argv); + Errata + invoke(int argc, char *argv[]) + { + assert(LEAF == _type); + return (*reinterpret_cast(_data))(argc, argv); } - Errata invoke() { - assert(NULLARY == _type); - return (*reinterpret_cast(_data))(); + Errata + invoke() + { + assert(NULLARY == _type); + return (*reinterpret_cast(_data))(); } - bool is_leaf() const { return LEAF == _type; } - bool is_nullary() const { return NULLARY == _type; } + bool + is_leaf() const + { + return LEAF == _type; + } + bool + is_nullary() const + { + return NULLARY == _type; + } protected: - - void clear() { - switch (_type) { - case NIL: break; - case LEAF: reinterpret_cast(_data)->~LeafAction(); break; - case NULLARY: reinterpret_cast(_data)->~NullaryAction(); break; - } - _type = NIL; + void + clear() + { + switch (_type) { + case NIL: + break; + case LEAF: + reinterpret_cast(_data)->~LeafAction(); + break; + case NULLARY: + reinterpret_cast(_data)->~NullaryAction(); + break; + } + _type = NIL; } Type _type = NIL; ///< Type of function stored. diff --git a/cmd/traffic_cache_tool/Makefile.am b/cmd/traffic_cache_tool/Makefile.am index fdb20b489b1..e0d4ac8afdf 100644 --- a/cmd/traffic_cache_tool/Makefile.am +++ b/cmd/traffic_cache_tool/Makefile.am @@ -21,7 +21,7 @@ AM_CPPFLAGS = -I $(srcdir)/lib/ts noinst_PROGRAMS = traffic_cache_tool -traffic_cache_tool_SOURCES = CacheDefs.h CacheTool.cc File.h File.cc Command.h Command.cc +traffic_cache_tool_SOURCES = CacheDefs.h CacheDefs.cc CacheTool.cc File.h File.cc Command.h Command.cc traffic_cache_tool_LDADD = \ $(top_builddir)/lib/ts/.libs/MemView.o \ $(top_builddir)/lib/ts/.libs/ink_memory.o \ @@ -31,6 +31,7 @@ traffic_cache_tool_LDADD = \ $(top_builddir)/lib/ts/.libs/ink_assert.o \ $(top_builddir)/lib/ts/.libs/ink_code.o \ $(top_builddir)/lib/tsconfig/.libs/Errata.o \ - @OPENSSL_LIBS@ + $(top_builddir)/lib/ts/.libs/Regex.o \ + @OPENSSL_LIBS@ @LIBPCRE@ all-am: Makefile $(PROGRAMS) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index b84f4634e1f..e284ed17bb3 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2491,7 +2491,10 @@ Cache::lookup(Continuation *cont, const CacheKey *key, CacheFragType type, const return ACTION_RESULT_DONE; } - Vol *vol = key_to_vol(key, hostname, host_len); + Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); ProxyMutex *mutex = cont->mutex.get(); CacheVC *c = new_CacheVC(cont); SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead); @@ -2611,6 +2614,9 @@ Cache::remove(Continuation *cont, const CacheKey *key, CacheFragType type, const CACHE_TRY_LOCK(lock, cont->mutex, this_ethread()); ink_assert(lock.is_locked()); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); // coverity[var_decl] Dir result; dir_clear(&result); // initialized here, set result empty so we can recognize missed lock diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 2616d818fcb..9833da37d49 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -1209,7 +1209,7 @@ int Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this ink_zero(frag_demographics); - printf("Stripe '[%s]'\n", hash_text.get()); + printf(" Stripe '[%s]'\n", hash_text.get()); printf(" Directory Bytes: %" PRIu64 "\n", total_buckets * SIZEOF_DIR); printf(" Segments: %d\n", segments); printf(" Buckets per segment: %" PRIu64 "\n", buckets); diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index 4b3f9f30ed3..9f44d07ee80 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -41,6 +41,9 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->earliest_key = *to; c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); #ifdef DEBUG Doc *doc = (Doc *)c->buf->data(); memcpy(doc->data(), to, sizeof(*to)); // doublecheck @@ -82,11 +85,13 @@ Cache::deref(Continuation *cont, const CacheKey *key, CacheFragType type, const } ink_assert(caches[type] == this); - Vol *vol = key_to_vol(key, hostname, host_len); Dir result; Dir *last_collision = nullptr; - CacheVC *c = nullptr; + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); + CacheVC *c = nullptr; { MUTEX_TRY_LOCK(lock, vol->mutex, cont->mutex->thread_holding); if (lock.is_locked()) { diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index b9a6ccd196e..3bd585a5ac9 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -37,6 +37,9 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co ink_assert(caches[type] == this); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; @@ -103,6 +106,9 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, ink_assert(caches[type] == this); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 58719803910..7b8f9b8e47d 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1653,7 +1653,10 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheFragType frag_ty c->vio.op = VIO::WRITE; c->base_stat = cache_write_active_stat; c->vol = key_to_vol(key, hostname, host_len); - Vol *vol = c->vol; + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); + Vol *vol = c->vol; CACHE_INCREMENT_DYN_STAT(c->base_stat + CACHE_STAT_ACTIVE); c->first_key = c->key = *key; c->frag_type = frag_type; @@ -1730,8 +1733,11 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *info, c->earliest_key = c->key; c->frag_type = CACHE_FRAG_TYPE_HTTP; c->vol = key_to_vol(key, hostname, host_len); - Vol *vol = c->vol; - c->info = info; + char hashStr[33]; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); + Vol *vol = c->vol; + c->info = info; if (c->info && (uintptr_t)info != CACHE_ALLOW_MULTIPLE_WRITES) { /* Update has the following code paths : diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 372c2f3e529..1eb5061255a 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1696,8 +1696,9 @@ url_MD5_get_fast(const URLImpl *url, CryptoContext &ctx, CryptoHash *hash, cache if (generation != -1) { ctx.update(&generation, sizeof(generation)); } - ctx.finalize(hash); + char hashStr[33]; + Debug("cache", "fast string to be hashed: %s%d => %s", buffer, generation, ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } static inline void @@ -1755,6 +1756,7 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p == e) { + char hashStr[33]; ctx.update(buffer, BUFSIZE); p = buffer; } @@ -1763,6 +1765,7 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p != buffer) { + char hashStr[33]; ctx.update(buffer, p - buffer); } @@ -1772,8 +1775,10 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca if (generation != -1) { ctx.update(&generation, sizeof(generation)); } - ctx.finalize(hash); + char hashStr[33]; + Debug("cache", "string to be hashed: %.*s%d%d => %s", (p - buffer), buffer, port, generation, + ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } void @@ -1821,6 +1826,7 @@ url_host_MD5_get(URLImpl *url, INK_MD5 *md5) // Especially since it's in_port_t for url_MD5_get. int port = url_canonicalize_port(url->m_url_type, url->m_port); ctx.update(&port, sizeof(port)); + Debug("cache", "%s://%s:%d", url->m_ptr_scheme, url->m_ptr_host, port); ctx.finalize(*md5); }