diff --git a/cmd/traffic_ctl/traffic_ctl.cc b/cmd/traffic_ctl/traffic_ctl.cc index 32f59d94aac..f8c458828fa 100644 --- a/cmd/traffic_ctl/traffic_ctl.cc +++ b/cmd/traffic_ctl/traffic_ctl.cc @@ -23,9 +23,9 @@ #include "traffic_ctl.h" -#include "ts/I_Layout.h" #include "I_RecProcess.h" #include "RecordsConfig.h" +#include "ts/I_Layout.h" #include "ts/runroot.h" AppVersionInfo CtrlVersionInfo; @@ -211,6 +211,15 @@ CtrlGenericSubcommand(const char *name, const subcommand *cmds, unsigned ncmds, return CtrlSubcommandUsage(name, cmds, ncmds, nullptr, 0); } +static const subcommand commands[] = { + {subcommand_alarm, "alarm", "Manipulate alarms"}, + {subcommand_config, "config", "Manipulate configuration records"}, + {subcommand_metric, "metric", "Manipulate performance metrics"}, + {subcommand_server, "server", "Stop, restart and examine the server"}, + {subcommand_storage, "storage", "Manipulate cache storage"}, + {subcommand_plugin, "plugin", "Interact with plugins"}, +}; + int main(int argc, const char **argv) { @@ -222,20 +231,14 @@ main(int argc, const char **argv) ArgumentDescription argument_descriptions[] = { {"debug", '-', "Enable debugging output", "F", &debug, nullptr, nullptr}, - HELP_ARGUMENT_DESCRIPTION(), + {"help", 'h', "Print usage information", nullptr, nullptr, nullptr, + [](const ArgumentDescription *args, unsigned nargs, const char *arg_unused) { + CtrlSubcommandUsage(nullptr, commands, countof(commands), args, nargs); + }}, VERSION_ARGUMENT_DESCRIPTION(), RUNROOT_ARGUMENT_DESCRIPTION(), }; - const subcommand commands[] = { - {subcommand_alarm, "alarm", "Manipulate alarms"}, - {subcommand_config, "config", "Manipulate configuration records"}, - {subcommand_metric, "metric", "Manipulate performance metrics"}, - {subcommand_server, "server", "Stop, restart and examine the server"}, - {subcommand_storage, "storage", "Manipulate cache storage"}, - {subcommand_plugin, "plugin", "Interact with plugins"}, - }; - BaseLogFile *base_log_file = new BaseLogFile("stderr"); diags = new Diags(program_name, "" /* tags */, "" /* actions */, base_log_file); @@ -261,8 +264,10 @@ main(int argc, const char **argv) ats_scoped_str rundir(RecConfigReadRuntimeDir()); - // Make a best effort to connect the control socket. If it turns out we are just displaying help or something then it - // doesn't matter that we failed. If we end up performing some operation then that operation will fail and display the + // Make a best effort to connect the control socket. If it turns out we are + // just displaying help or something then it + // doesn't matter that we failed. If we end up performing some operation then + // that operation will fail and display the // error. TSInit(rundir, static_cast(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); diff --git a/doc/admin-guide/plugins/s3_auth.en.rst b/doc/admin-guide/plugins/s3_auth.en.rst index 7293188f81c..6445e13feb4 100644 --- a/doc/admin-guide/plugins/s3_auth.en.rst +++ b/doc/admin-guide/plugins/s3_auth.en.rst @@ -34,16 +34,16 @@ Using the plugin in a remap rule would be e.g.:: # remap.config - ... @plugin=s3_auth @pparam=--access_key @pparam=my-key \ - @pparam=--secret_key @pparam=my-secret \ - @pparam=--virtual_host + ... @plugin=s3_auth.so @pparam=--access_key @pparam=my-key \ + @pparam=--secret_key @pparam=my-secret \ + @pparam=--virtual_host Alternatively, you can store the access key and secret in an external configuration file, and point the remap rule(s) to it:: # remap.config - ... @plugin=s3_auth @pparam=--config @pparam=s3_auth_v2.config + ... @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v2.config Where ``s3.config`` could look like:: diff --git a/doc/admin-guide/plugins/ts_lua.en.rst b/doc/admin-guide/plugins/ts_lua.en.rst index 342a859de88..cf8dff613e6 100644 --- a/doc/admin-guide/plugins/ts_lua.en.rst +++ b/doc/admin-guide/plugins/ts_lua.en.rst @@ -196,6 +196,22 @@ We should write this TAG in records.config(If TAG is missing, default TAG will b `TOP <#ts-lua-plugin>`_ +ts.error +-------- +**syntax:** *ts.error(MESSAGE)* + +**context:** global + +**description**: Log the MESSAGE to error.log + +Here is an example: + +:: + + ts.error('This is an error message') + +`TOP <#ts-lua-plugin>`_ + Remap status constants ---------------------- **context:** do_remap @@ -2373,6 +2389,44 @@ Here is an example `TOP <#ts-lua-plugin>`_ +ts.http.get_remap_from_url +-------------------------- +**syntax:** *ts.http.get_remap_from_url()* + +**context:** do_global_post_remap + +**description:** This function can be used to get the *from* URL in the matching line in :file:`remap.config`. + +Here is an example + +:: + + function do_global_post_remap() + local from_url = ts.http.get_remap_from_url() + ts.debug(from_url) + end + +`TOP <#ts-lua-plugin>`_ + +ts.http.get_remap_to_url +------------------------ +**syntax:** *ts.http.get_remap_to_url()* + +**context:** do_global_post_remap + +**description:** This function can be used to get the *to* URL in the matching line in :file:`remap.config`. + +Here is an example + +:: + + function do_global_post_remap() + local to_url = ts.http.get_remap_to_url() + ts.debug(to_url) + end + +`TOP <#ts-lua-plugin>`_ + Server state constants ---------------------- **context:** global diff --git a/doc/admin-guide/plugins/xdebug.en.rst b/doc/admin-guide/plugins/xdebug.en.rst index 45920afddfe..32cc1dfae42 100644 --- a/doc/admin-guide/plugins/xdebug.en.rst +++ b/doc/admin-guide/plugins/xdebug.en.rst @@ -54,6 +54,12 @@ Diags transaction specific diagnostics for the transaction. This also requires that :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``. +log-headers + If the ``log-headers`` is requested while :ts:cv:`proxy.config.diags.debug.tags` + is set to ``xdebug.headers`` and :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``, + then all client and server, request and response headers are logged. + Also, the ``X-Debug: log-headers`` header is always added to the upstream request. + X-Cache-Key The ``X-Cache-Key`` header contains the URL that identifies the HTTP object in the Traffic Server cache. This header is particularly useful if a custom cache diff --git a/doc/developer-guide/api/types/SystemTypes.en.rst b/doc/developer-guide/api/types/SystemTypes.en.rst index f086e24b049..aed63901b01 100644 --- a/doc/developer-guide/api/types/SystemTypes.en.rst +++ b/doc/developer-guide/api/types/SystemTypes.en.rst @@ -35,11 +35,15 @@ These types are provided by the compiler ("built-in") or from a required operati .. c:type:: off_t - `Reference `__. + `Reference `__. .. cpp:type:: off_t - `Reference `__. + `Reference `__. + +.. cpp:type:: ptrdiff_t + + The difference between two pointers. `Reference `__. .. cpp:type:: uint64_t diff --git a/doc/developer-guide/cache-architecture/architecture.en.rst b/doc/developer-guide/cache-architecture/architecture.en.rst index 2b6cf5d738f..6442c0e9579 100644 --- a/doc/developer-guide/cache-architecture/architecture.en.rst +++ b/doc/developer-guide/cache-architecture/architecture.en.rst @@ -353,7 +353,7 @@ the client as the object. Objects are rooted in a :cpp:class:`Doc` structure stored in the cache. :cpp:class:`Doc` serves as the header data for a :term:`cache fragment` and is contained at the start of every fragment. The first fragment for an object is -termed the *first Doc* and always contains the object metadata. Any +termed the *First Doc* and always contains the object metadata. Any operation on the object will read this fragment first. The fragment is located by converting the :term:`cache key` for the object to a :term:`cache ID` and then doing a lookup for a :term:`directory entry` with that key. The directory @@ -465,8 +465,6 @@ default). Objects which are in use when the write cursor is near use the same underlying evacuation mechanism but are handled automatically and not via the explicit ``pinned`` bit in :cpp:class:`Dir`. -.. [#multiple-alternates] It could, under certain circumstances, be accurate for none of the alternates. - Additional Notes ---------------- @@ -601,7 +599,7 @@ The target fragment size can set with the :file:`records.config` value This value should be chosen so that it is a multiple of a :ref:`cache entry multiplier `. It is not necessary to make it a -power of two[#cache-mult-value]_. Larger fragments increase I/O efficiency but +power of two [#cache-mult-value]_. Larger fragments increase I/O efficiency but lead to more wasted space. The default size (1M, 2^20) is a reasonable choice in most circumstances, altough in very specific cases there can be benefit from tuning this parameter. |TS| imposes an internal maximum of a 4,194,232 bytes, @@ -628,12 +626,6 @@ Index entries in a segment are grouped :term:`buckets `, each of ``DIR_DEPTH`` (currently 4) entries. These are handled in the standard hash table manner, giving somewhat less than 2^14 buckets per segment. -.. [#cache-mult-value] - - The comment in earlier versions of the :file:`records.config` documentation - which indicated that this value must be a power of two were, unfortunately, - mistaken and have been corrected. - .. _cache-directory-probe: Directory Probing @@ -762,14 +754,7 @@ The checks that are done are: A plugin can call :c:func:`TSHttpTxnReqCacheableSet()` to force the request to be viewed as cache valid. -.. [#cacheability-overrides] - - The code appears to check :file:`cache.config` in this logic by setting the - ``does_config_permit_lookup`` in the ``cache_info.directives`` of the state - machine instance but I can find no place where the value is used. The - directive ``does_config_permit_storing`` is set and later checked so the - directive (from the administrator point of view) is effective in preventing - caching of the object. +.. _cache-lookup: Cache Lookup ------------------------- @@ -789,9 +774,9 @@ The basic steps to a cache lookup are: #. The cache stripe is determined (based on the cache key). - The :term:`cache key` is used as a hash key in to an array of - :cpp:class:`Vol` instances. The construction and arrangement of this array - is the essence of how volumes are assigned. + The :term:`cache key` is used as a hash key in to an array of :cpp:class:`Vol` instances by + :func:`Cache::key_to_vol`. The construction and arrangement of this array is the essence of how + volumes are assigned. #. The cache stripe directory :ref:`is probed ` using the index key computed from the cache key. @@ -1095,96 +1080,21 @@ fragment while it remains open (as all open objects are evacuated). If, when the object is closed, the fragment is still marked then it is placed in the appropriate evacuation bucket. -.. _cache-initialization: - -Initialization -------------------------- - -Initialization starts with an instance of :cpp:class:`Store` reading the storage -configuration file, by default :file:`storage.config`. For each valid element in -the file an instance of :cpp:class:`Span` is created. These are of basically -four types: - -* File - -* Directory - -* Disk - -* Raw device - -After creating all the :cpp:class:`Span` instances, they are grouped by device -ID to internal linked lists attached to the :cpp:member:`Store::disk` -array[#store-disk-array]_. Spans that refer to the same directory, disk, or raw -device are coalesced in to a single span. Spans that refer to the same file -with overlapping offsets are also coalesced [#coalesced-spans]_. This is all done in -:func:`ink_cache_init` called during startup. - -.. note:: - - The span logic is also used by the HostDB and more than one otherwise - inexplicable feature is provided by the span logic for that module. - -After configuration initialization, the cache processor is started by calling -:cpp:member:`CacheProcessor::start()`. This does a number of things: - -For each valid span, an instance of :cpp:class:`CacheDisk` is created. This -class is a :term:`continuation` and so can be used to perform potentially -blocking operations on the span. The primary use of these is to be passed to -the AIO threads as the callback when an I/O operation completes. These are then -dispatched to AIO threads to perform :term:`storage unit` initialization. After -all of those have completed, the resulting storage is distributed across the -:term:`volumes ` in :func:`cplist_reconfigure`. The -:cpp:class:`CacheVol` instances are created at this time. - -:term:`Cache stripe ` assignment setup is done once all stripes -have initialized (that is, the stripe header information has been successfully -read from disk for all stripes). The assignment information is stored as an -array of indices. These are indices in to an array of stripes. Both the -assignment and the stripe arrays are stored in an instance of :cpp:class:`CacheHostRecord`. -Assignment initialization consists of populating the assignment array, which is -much larger than the stripe array. - -There is an instance of :cpp:class:`CacheHostRecord` for each line in -:file:`hosting.config` and one generic record. For the configured instances, the -set of stripes is determined from the cache volume specified in the line. If no -lines are specified, all stripes are placed in the generic record, otherwise -only those stripes marked as default are placed in the generic record. - -.. note:: - - If hosting records are specified, it is an error to not specify at least one - default cache volume. - -The assignment table is initialized in :func:`build_vol_hash_table` which is -called for each :cpp:class:`CacheHostRecord` instance. For each stripe in the -host record, a sequence of pseudo-random numbers is generated. This begins with -the folded hash of the stripe hash identifier, which is the device path followed -by the ``skip`` and ``size`` values for that stripe, making it unique. This -also makes the sequence deterministic for any particular stripe. - -Each stripe gets one number in its sequence for every `VOL_HASH_ALLOC_SIZE` (8 -MB currently) of storage. These numbers are paired with the stripe index, -combined across all stripes, then sorted by the random values. The resulting -array is sampled for every slot in the stripe assignment table by dividing the -maximum random value by the size of the assignment table and using the value -midway between each multiple of the result of the division. The coalesced -pseudo-random sequence is scanned for each sample in turn and the first number -not greater than the sample is found. The stripe associated with that value is -used for that assignment table entry. - -While this procedure is deterministic, it is sensitive to initial conditions, -including the size of each stripe. - .. rubric:: Footnotes -.. [#store-disk-array] +.. [#multiple-alternates] It could, under certain circumstances, be accurate for none of the alternates. + +.. [#cache-mult-value] - `Work is under way `_ on - extending this to include objects that are in the memory cache. + The comment in earlier versions of the :file:`records.config` documentation + which indicated that this value must be a power of two were, unfortunately, + mistaken and have been corrected. -.. [#coalesced-spans] +.. [#cacheability-overrides] - This linked list is mostly ignored in later processing, causing all but one - file or directory storage units on the same device to be ignored. See - `TS-1869 `_. + The code appears to check :file:`cache.config` in this logic by setting the + ``does_config_permit_lookup`` in the ``cache_info.directives`` of the state + machine instance but I can find no place where the value is used. The + directive ``does_config_permit_storing`` is set and later checked so the + directive (from the administrator point of view) is effective in preventing + caching of the object. diff --git a/doc/developer-guide/cache-architecture/cache-initialization.en.rst b/doc/developer-guide/cache-architecture/cache-initialization.en.rst new file mode 100644 index 00000000000..7e5b90249e0 --- /dev/null +++ b/doc/developer-guide/cache-architecture/cache-initialization.en.rst @@ -0,0 +1,164 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs +.. default-domain:: cpp +.. _cache-initialization: + +Cache Initialization +******************** + +Initialization starts with an instance of :cpp:class:`Store` reading the storage +configuration file, by default :file:`storage.config`. For each valid element in +the file an instance of :cpp:class:`Span` is created. These are of basically +four types: + +* File + +* Directory + +* Disk + +* Raw device + +After creating all the :cpp:class:`Span` instances, they are grouped by device +ID to internal linked lists attached to the :cpp:member:`Store::disk` +array [#store-disk-array]_. Spans that refer to the same directory, disk, or raw +device are coalesced in to a single span. Spans that refer to the same file +with overlapping offsets are also coalesced [#coalesced-spans]_. This is all done in +:func:`ink_cache_init` called during startup. + +.. note:: + + The span logic is also used by the HostDB and more than one otherwise + inexplicable feature is provided by the span logic for that module. + +After configuration initialization, the cache processor is started by calling +:cpp:member:`CacheProcessor::start()`. This does a number of things: + +For each valid span, an instance of :cpp:class:`CacheDisk` is created. This +class is a :term:`continuation` and so can be used to perform potentially +blocking operations on the span. The primary use of these is to be passed to +the AIO threads as the callback when an I/O operation completes. These are then +dispatched to AIO threads to perform :term:`storage unit` initialization. After +all of those have completed, the resulting storage is distributed across the +:term:`volumes ` in :func:`cplist_reconfigure`. The +:cpp:class:`CacheVol` instances are created at this time. + +Stripe Assignment +================= + +Every object that is stored in stored in a single, specific stripe. Stripe assignment is determing +for an object what stripe that is. The logic described here sets up the stripe assignment table maps +from the cache key to a specific stripe. + +:term:`Cache stripe ` assignment setup is done once all stripes have initialized (that +is, all stripe header read operations have completed). There is an instance of +:cpp:class:`CacheHostRecord` for each line in :file:`hosting.config` and one generic instance +(:member:`Cache::hosttable`) for all cache volumes that are not explicitly assigned. If +:file:`hosting.config` is empty then all cache volumes will be in the generic record. + +:func:`build_vol_hash_table` in :ts:git:`iocore/cache/Cache.cc` does the work of setting up the +stripe assignment and is called for each :class:`CacheHostRecord` and the generic host record. The +stripes to be assigned are in :member:`CacheHostRecord::vols`. + +.. figure:: images/cache-init-cachehostrecord.png + :align: left + + :member:`CacheHostRecord::vols` is the union of all the stripes in the :class:`CacheVol` instances in :member:`CacheHostRecord::cp`. + +An indirect index mapping is created to account for stripes that are not available. The total size +of the stripes is computed at the same time. The :code:`forvol` and :code:`getvol` arrays are used +for debugging, they are not essential to the assignment setup. :code:`rtable_entries` is filled with +stripe size divided by :code:`VOL_HASH_ALLOC_SIZE`. These values are used to determine the number of +assignment slots given to each stripe. For each stripe a seed for a 32 bit pseudo random number +generator is created based on stripe properties. Another array of pairs of value and stripe index is +filled using these. For each :code:`VOL_HASH_ALLOC_SIZE` amount of space in a stripe, a pair is +generated containing the stripe index and the next random number from that stripe's generator. This +array is then sorted in ascending order. + +.. figure:: images/cache-init-rtable-setup.png + + In this example the total size of the stripes is 10 and an `8 bit pseudo random number generator + `__ is used. + +The result is sampled in sections, the size of the sections selected to yield +:code:`VOL_HASH_TABLE_SIZE` sections. For each section the sample value is the midpoint of the +section.For the example, the number of sections is set to 17 (because the number of sections should +be a prime number). This yields 17 sections each of width 15 with a sample value equal to 7 more +than the initial value. The results of applying this to the :code:`rtable` is + +.. figure:: images/cache-init-rtable-result.png + + Sampling results. + +This process can be viewed as dividing a number line in to sampling sections, each section corresponding +to a stripe assignment slot. + +.. figure:: images/cache-init-sampling.png + + Sample sections. + +Each random value for a stripe in the :code:`rtable` array can be considered a node in this space. +For one stripe this might look like + +.. figure:: images/cache-init-slots-single.png + + Random value nodes for a single stripe. + +The full array contains random value nodes for all the stripes. + +.. figure:: images/cache-init-selection.png + + Random value nodes for all (four) stripes. + +The stripe for that section (assignment slot) is the first node at or past the sample value. This +can be seen as the arrow color in the previous figure. This provides a reasonably proportioned to +size assignment of slots to stripes. It is also a consistent hash in that if a stripe is removed, +the recomputation will tend to distribute assignments for the missing stripe across the other +stripes in proportion to their sizes while not changing the assignment of any slot not assigned to +the missing stripe. In essence for each sample point (assignment slot) a permuation of the stripes +is implied by the order of the random value nodes past that sample point. The randomization serves +to spread re-assigned slots to various stripes instead of a single one. + +.. figure:: images/cache-init-slots-minus-1.png + + Remove the blue stripe. + +If the stripe is restored the assignments will be the same as before the stripe was removed. The +assignment is very sensitive to the properties of each stripe - changing a stripe size or location +will effectively be as if it were a new stripe. In the figure the two blue stripe assignments are +changed to purple and green respectively. If the blue stripe were added back those assignments and +only those would revert to blue. This is because for each stripe the node sequence as generated by +the pseudo random number generator depends only the properties of the stripes. + +At runtime stripe selection is done by :func:`Cache::key_to_vol` which selects the +:class:`CacheHostRecord` instance then picks the stripe assignment slot in the array which +determines the stripe for the object. + +.. rubric:: Footnotes + +.. [#store-disk-array] + + `Work is under way `_ on + extending this to include objects that are in the memory cache. + +.. [#coalesced-spans] + + This linked list is mostly ignored in later processing, causing all but one + file or directory storage units on the same device to be ignored. See + `TS-1869 `_. diff --git a/doc/developer-guide/cache-architecture/data-structures.en.rst b/doc/developer-guide/cache-architecture/data-structures.en.rst index 73c6a379b01..1f80a446711 100644 --- a/doc/developer-guide/cache-architecture/data-structures.en.rst +++ b/doc/developer-guide/cache-architecture/data-structures.en.rst @@ -22,6 +22,50 @@ Data Structures *************** +.. uml:: + :align: center + + hide empty members + + CacheHostRecord *-- "*" Stripe : Vol > + CacheHostRecord *-- "*" CacheVol : cp > + CacheVol *-- "*" Stripe : Vol > + +.. class:: CacheHostTable + + A container that maps from a FQDN to a :class:`CacheHostRecord`. This is constructed from + the contents of :file:`hosting.config`. + + .. function:: void Match(char const * fqdn, int len, CacheHostResult * result) + + Search the table for a match for the hostname :arg:`fqdn`, a string of length :arg:`len`. + If found the result is placed in :arg:`result`. + +.. class:: CacheHostResult + + A wrapper for :class:`CacheHostRecord` used by :func:`CacheHostTable::Match`. This contains + the set of cache volumes for the cache host record and is used to perform stripe assignment. + +.. class:: CacheHostRecord + + A cache hosting record from :file:`hosting.config`. + + .. member:: CacheVol ** cp + + The cache volumes that are part of this cache host record. + + .. member:: Vol ** vols + + The stripes that are part of the cache volumes. This is the union over the stripes of + :member:`CacheHostRecord::cp` + + .. member:: unsigned short * vol_hash_table + + The stripe assignment table. This is an array of indices in to + :member:`CacheHostRecord::vols`. + + .. see: :class:`CacheHostTable`. + .. class:: OpenDir An open directory entry. It contains all the information of a @@ -31,13 +75,13 @@ Data Structures A virtual connection class which accepts input for writing to cache. -.. function:: int CacheVC::openReadStartHead(int event, Event* e) + .. function:: int openReadStartHead(int event, Event* e) - Performs the initial read for a cached object. + Performs the initial read for a cached object. -.. function:: int CacheVC::openReadStartEarliest(int event, Event* e) + .. function:: int openReadStartEarliest(int event, Event* e) - Performs the initial read for an :term:`alternate` of an object. + Performs the initial read for an :term:`alternate` of an object. .. class:: HttpTunnel @@ -96,7 +140,7 @@ Data Structures .. cpp:member:: int aggWrite(int event, void * e) Schedule the aggregation buffer to be written to disk. - + .. member:: off_t segments The number of segments in the volume. This will be roughly the total @@ -305,7 +349,7 @@ Data Structures .. class:: DiskVolBlockQueue .. member:: DiskVolBlock* b - + .. member:: int new_block Indicates if this is a new stripe rather than an existing one. In case a stripe is new ATS decides to clear that stripe(:class:`Vol`) @@ -335,11 +379,11 @@ Data Structures The disk containing the stripe .. member:: Queue dpb_queue - + .. enum:: CacheType .. enumerator:: HTTP - + .. enumerator:: Stream .. class:: CacheVol @@ -347,7 +391,7 @@ Data Structures A :term:`cache volume` as described in :file:`volume.config`. This class represents a single volume. :class:`CacheVol` comprises of stripes spread across Spans(disks) .. member:: int volume_number - + indentification number of this volume .. member:: int scheme @@ -362,17 +406,17 @@ Data Structures Number of stripes(:class:`Vol`) contained in this volume .. member:: Vol** vols - + :class:`Vol` represents a single stripe in the disk. vols contains all the stripes this volume is made up of - + .. member:: DiskVol** disk_vols disk_vols contain references to the disks of all the stripes in this volume - + .. member:: LINK link .. member:: RecRawStatBlock vol_rsb - + per volume stat .. class:: ConfigVol @@ -386,13 +430,13 @@ Data Structures .. member:: CacheType scheme .. member:: off_t size - + .. member:: bool in_percent Used as an indicator if the volume is part of the overall volumes created by ATS .. member:: int percent - + .. member:: CacheVol* cachep .. member:: LINK link @@ -413,6 +457,23 @@ Data Structures .. member:: Queue cp_queue +.. class:: Cache + + Base object for a cache. + + .. member:: CacheHostRecord hosttable + + A generic class:`CacheHostRecord` that contains all cache volumes that are not explicitly + assigned in :file:`hosting.config`. + + .. function:: Vol * key_to_vol(const char * key, const char * host, int host_len) + + Compute the stripe (:code:`Vol*`) for a cache :arg:`key` and :arg:`host`. The :arg:`host` is + used to find the appropriate :class:`CacheHostRecord` instance. From there the stripe + assignment slot is determined by taking bits 64..83 (20 bits) of the cache :arg:`key` modulo + the stripe assignment array count (:code:`VOL_HASH_TABLE_SIZE`). These bits are the third 32 + bit slice of the :arg:`key` less the bottom :code:`DIR_TAG_WIDTH` (12) bits. + .. rubric:: Footnotes .. [#fragment-offset-table] diff --git a/doc/developer-guide/cache-architecture/images/cache-init-cachehostrecord.png b/doc/developer-guide/cache-architecture/images/cache-init-cachehostrecord.png new file mode 100644 index 00000000000..6892fbfef27 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-cachehostrecord.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-rtable-result.png b/doc/developer-guide/cache-architecture/images/cache-init-rtable-result.png new file mode 100644 index 00000000000..dff45adcd13 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-rtable-result.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-rtable-setup.png b/doc/developer-guide/cache-architecture/images/cache-init-rtable-setup.png new file mode 100644 index 00000000000..82e646150e4 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-rtable-setup.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-sampling.png b/doc/developer-guide/cache-architecture/images/cache-init-sampling.png new file mode 100644 index 00000000000..d4303a33157 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-sampling.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-selection.png b/doc/developer-guide/cache-architecture/images/cache-init-selection.png new file mode 100644 index 00000000000..ca4d76a36c5 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-selection.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-slots-minus-1.png b/doc/developer-guide/cache-architecture/images/cache-init-slots-minus-1.png new file mode 100644 index 00000000000..0c9cf623809 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-slots-minus-1.png differ diff --git a/doc/developer-guide/cache-architecture/images/cache-init-slots-single.png b/doc/developer-guide/cache-architecture/images/cache-init-slots-single.png new file mode 100644 index 00000000000..e18ba41b985 Binary files /dev/null and b/doc/developer-guide/cache-architecture/images/cache-init-slots-single.png differ diff --git a/doc/developer-guide/cache-architecture/images/stripe-header.svg b/doc/developer-guide/cache-architecture/images/stripe-header.svg old mode 100755 new mode 100644 diff --git a/doc/developer-guide/cache-architecture/index.en.rst b/doc/developer-guide/cache-architecture/index.en.rst index 4c30059be06..e4cb401e151 100644 --- a/doc/developer-guide/cache-architecture/index.en.rst +++ b/doc/developer-guide/cache-architecture/index.en.rst @@ -35,6 +35,7 @@ understanding and modifying the source. :maxdepth: 2 architecture.en + cache-initialization.en core-cache-functions.en data-structures.en api-functions.en diff --git a/doc/developer-guide/internal-libraries/MemSpan.en.rst b/doc/developer-guide/internal-libraries/MemSpan.en.rst new file mode 100644 index 00000000000..a9b6775e2ac --- /dev/null +++ b/doc/developer-guide/internal-libraries/MemSpan.en.rst @@ -0,0 +1,99 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. default-domain:: cpp + +MemSpan +******* + +Synopsis +======== + +:code:`#include ` + +:class:`MemSpan` is a view on a contiguous section of writeable memory. A view does not own the memory +and neither allocates nor de-allocates. The memory in the view is always owned by some other container +and it is the responsibility of the code to make sure the lifetime of the view is no more than that +of the owning container [#vector]_. + + +Description +=========== + +A :class:`MemSpan` is generally constructed on either an array or an allocated buffer. This allows +the buffer to be passed around with intrinsic length information. The buffer can also be treated as +an array of varying types, which makes working with serialized data much easier. + +Reference +========= + +.. class:: MemSpan + + A span of writable memory. Because this is a chunk of memory, conceptually delimited by start and + end pointers, the sizing type is :code:`ptrdiff_t` so that all of the sizing is consistent with + differences between pointers. The memory is owned by some other object and that object must + maintain the memory as long as the span references it. + + .. function:: MemSpan(void * ptr, ptrdiff_t size) + + Construct a view starting at :arg:`ptr` for :arg:`size` bytes. + + .. function:: void * data() const + + Return a pointer to the first byte of the span. + + .. function:: ptrdiff_t size() const + + Return the size of the span. + + .. function:: bool operator == (MemSpan const& that) const + + Check the equality of two spans, which are equal if they contain the same number of bytes of the same values. + + .. function:: bool is_same(MemSpan const& that) const + + Check if :arg:`that` is the same span as :arg:`this`, that is the spans contain the exact same bytes. + + .. function:: template < typename V > V at(ptrdiff_t n) const + + Return a value of type :arg:`V` as if the span were are array of type :arg:`V`. + + .. function:: template < typename V > V * ptr(ptrdiff_t n) const + + Return a pointer to a value of type :arg:`V` as if the span were are array of type :arg:`V`. + + .. function:: MemSpan prefix(ptrdiff_t n) const + + Return a new instance that contains the first :arg:`n` bytes of the current span. If :arg:`n` + is larger than the number of bytes in the span, only that many bytes are returned. + + .. function:: MemSpan& remove_prefix(ptrdiff_t n) + + Remove the first :arg:`n` bytes of the span. If :arg:`n` is more than the number of bytes in + the span the result is an empty span. A reference to the instance is returned. + +.. rubric:: Footnotes + +.. [#vector] + + Strong caution must be used with containers such as :code:`std::vector` or :code:`std::string` + because the lifetime of the memory can be much less than the lifetime of the container. In + particular, adding or removing any element from a :code:`std::vector` can cause a re-allocation, + invalidating any view of the original memory. In general views should be treated like iterators, + suitable for passing to nested function calls but not for storing. diff --git a/doc/developer-guide/internal-libraries/memview.en.rst b/doc/developer-guide/internal-libraries/TextView.en.rst similarity index 65% rename from doc/developer-guide/internal-libraries/memview.en.rst rename to doc/developer-guide/internal-libraries/TextView.en.rst index 75f57f88ddd..8cb00d4c0ec 100644 --- a/doc/developer-guide/internal-libraries/memview.en.rst +++ b/doc/developer-guide/internal-libraries/TextView.en.rst @@ -19,60 +19,62 @@ .. default-domain:: cpp -MemView +TextView ************* Synopsis ======== -:code:`#include ` - -.. class:: MemView - -.. class:: StringView +:code:`#include ` .. class:: TextView -These classes act as views in to already allocated memory. Internally in |TS| work must be done with -string or memory entities that are embedded in larger pre-existing memory structures. These classes -are designed to make that easier, more efficient, and less error prone. +This class acts as a view in to memory allocated / owned elsewhere. It is in effect a pointer and +should be treated as such (e.g. care must be taken to avoid dangling references by knowing where the +memory really is). The purpose is to provide string manipulation that is fast, efficient, and +non-modifying, particularly when temporary "copies" are needed. + Description =========== -The term "view" will be used to mean an instance of :class:`MemView` or :class:`StringView`. -Fundamentally both classes do the same thing, maintain a read only view of a contiguous region of -memory. They differ in the methods and return types due to the conflicting requirements of raw -memory operations and string based operations. +:class:`TextView` is a subclass of :class:`string_view` and has all of those methods. In addition it +provides a number of ancillary methods of common string manipulation methods. -A view is constructed by providing a contiguous region of memory, either based on a start pointer -and a length or a pair of pointers in the usual STL half open range style where the view starts at -the first pointer and ends one short of the second pointer. A view can be empty and refer to no -memory (which what default construction yields). A view attempts to act like a normal pointer in -most situations. A view is only somewhat more expensive than a raw pointer but in most cases a count -is needed as well making a view not any more costly than existing code. Any code that already keeps -a pointer and a count is a good candidate for using :class:`MemView` or :class:`StringView`. +A :class:`TextView` should be treated as an enhanced character pointer that both a location and a +size. This is when makes it possible to pass sub strings around without having to make copies or +allocation additional memory. This comes at the cost of keeping track of the actual owner of the +string memory and making sure the :class:`TextView` does not outlive the memory owner, just as with +a normal pointer type. Internal for |TS| any place that passes a :code:`char *` and a size is an +excellent candidate for using a :class:`TextView` as it is more convinient and no more risky than +the existing arguments. -:class:`MemView` and :class:`StringView` inter-convert because the difference between them is simply -the API to access the underingly memory in the view, the actual class internal data is identical. +In deciding between :class:`string_view` and :class:`TextView` remember that these easily and +cheaply cross convert. In general if the string is treated as a block of data, :class:`string_view` +is better. If the contents of the string are to be examined / parsed non-uniformly then +:class:`TextView` is better. For example, if the string is used simply as a key or a hash source, +use :class:`string_view`. Or, if the string may contain sustrings of interests such as key / value +pairs, then use a :class:`TextView`. -:class:`StringView` provides a variety of methods for manipulating the view as a string. These are provided as families of overloads differentiated by how characters are compared. There are four flavors. +:class:`TextView` provides a variety of methods for manipulating the view as a string. These are +provided as families of overloads differentiated by how characters are compared. There are four +flavors. * Direct, a pointer to the target character. * Comparison, an explicit character value to compare. -* Set, a set of characters (described by a :class:`StringView`) which are compared, any one of which matches. +* Set, a set of characters (described by a :class:`TextView`) which are compared, any one of which matches. * Predicate, a function that takes a single character argument and returns a bool to indicate a match. If the latter three are inadequate the first, the direct pointer, can be used after finding the appropriate character through some other mechanism. -The increment operator for :class:`StringView` shrinks the view by one character from the front +The increment operator for :class:`TextView` shrinks the view by one character from the front which allows stepping through the view in normal way, although the string view itself should be the loop condition, not a dereference of it. .. code-block:: cpp - StringView v; + TextView v; size_t hash = 0; for ( ; v ; ++v) hash = hash * 13 + *v; @@ -80,21 +82,21 @@ Or, because the view acts as a container of characters, this can be done non-des .. code-block:: cpp - StringView v; + TextView v; size_t hash = 0; for (char c : v) hash = hash * 13 + c; Views are cheap to construct therefore making a copy to use destructively is very inexpensive. -:class:`MemView` provides a :code:`find` method that searches for a matching value. The type of this +:class:`MemSpan` provides a :code:`find` method that searches for a matching value. The type of this value can be anything that is fixed sized and supports the equality operator. The view is treated as an array of the type and searched sequentially for a matching value. The value type is treated as having no identity and cheap to copy, in the manner of a integral type. -Parsing with StringView +Parsing with TextView ----------------------- -A primary use of :class:`StringView` is to do field oriented parsing. It is easy and fast to split +A primary use of :class:`TextView` is to do field oriented parsing. It is easy and fast to split strings in to fields without modifying the original data. For example, assume that :arg:`value` contains a null terminated string which is possibly several tokens separated by commas. @@ -102,9 +104,9 @@ contains a null terminated string which is possibly several tokens separated by #include parse_token(const char* value) { - StringView v(value); // construct assuming null terminated string. + TextView v(value); // construct assuming null terminated string. while (v) { - StringView token(v.extractPrefix(',')); + TextView token(v.extractPrefix(',')); token.trim(&isspace); if (token) { // process token @@ -125,10 +127,10 @@ What if the tokens were key / value pairs, of the form `key=value`? This is can #include parse_token(const char* source) { - StringView in(source); // construct assuming null terminated string. + TextView in(source); // construct assuming null terminated string. while (in) { - StringView value(in.extractPrefix(',')); - StringView key(value.trim(&isspace).splitPrefix('=').rtrim(&isspace)); + TextView value(in.extractPrefix(',')); + TextView key(value.trim(&isspace).splitPrefix('=').rtrim(&isspace)); if (key) { // it's a key=value token with key and value set appropriately. value.ltrim(&isspace); // clip potential space after '='. @@ -159,9 +161,9 @@ proved to provide little of the functionality available in :code:`ts::ConstBuffe reworking was required in any case, it seemed better to start from scratch and build just what was useful in the |TS| context. -The next step was the :code:`StringView` class which turned out reasonably well. It was then +The next step was the :code:`TextView` class which turned out reasonably well. It was then suggested that more support for raw memory (as opposed to memory presumed to contain printable ASCII data) would be useful. An attempt was made to do this but the differences in arguments, subtle -method differences, and return types made that infeasible. Instead :class:`MemView` was split off to +method differences, and return types made that infeasible. Instead :class:`MemSpan` was split off to provide a :code:`void*` oriented view. String specific methods were stripped out and a few non-character based methods added. diff --git a/doc/developer-guide/internal-libraries/buffer-writer.en.rst b/doc/developer-guide/internal-libraries/buffer-writer.en.rst index d72d7baafca..7e05547721d 100644 --- a/doc/developer-guide/internal-libraries/buffer-writer.en.rst +++ b/doc/developer-guide/internal-libraries/buffer-writer.en.rst @@ -25,11 +25,6 @@ BufferWriter ************* -:class:`BufferWriter` is designed to make writing text to a buffer fast and safe. The output buffer -can have a size and :class:`BufferWriter` will prevent writing past the end, while tracking the -theoretical output to enable buffer resizing after the fact. This also lets a :class:`BufferWriter` -instance write into the middle of a larger buffer, making nested output logic easy to build. - Synopsis ++++++++ @@ -40,6 +35,17 @@ Synopsis Description +++++++++++ +:class:`BufferWriter` is designed to make writing text to a buffer fast, convenient, and safe. It is +easier and less error-prone than using a combination of :code:`sprintf` and :code:`memcpy` as is +done in many places in the code.. A :class:`BufferWriter` can have a size and will prevent writing +past the end, while tracking the theoretical output to enable buffer resizing after the fact. This +also lets a :class:`BufferWriter` instance write into the middle of a larger buffer, making nested +output logic easy to build. + +The header files are divided in to two variants. ``BufferWriter.h`` provides the basic capabilities +of buffer output control. ``BufferWriterFormat.h`` provides formatted output mechanisms, primarily +the implementation and ancillary classes for :func:`BufferWriter::print`. + :class:`BufferWriter` is an abstract base class, in the style of :code:`std::ostream`. There are several subclasses for various use cases. When passing around this is the common type. @@ -84,7 +90,7 @@ Several basic types are overloaded and it is easy to extend to additional types. .. code-block:: cpp - ts::BufferWriter & operator << (ts::BufferWriter & w, ts::TextView const & sv) { + ts::BufferWriter & operator << (ts::BufferWriter & w, TextView const & sv) { w.write(sv.data(), sv.size()); return w; } @@ -263,6 +269,10 @@ Reference well with the standard "try before you buy" approach of attempting to write output, counting the characters needed, then allocating a sufficiently sized buffer and actually writing. + .. function:: BufferWriter & print(TextView fmt, ...) + + Print the arguments according to the format. See `bw-formatting`_. + .. class:: FixedBufferWriter : public BufferWriter This is a class that implements :class:`BufferWriter` on a fixed buffer, passed in to the constructor. @@ -294,6 +304,114 @@ Reference Construct an instance with a capacity of :arg:`N`. +.. _bw-formatting: + +Formatted Output +++++++++++++++++ + +:class:`BufferWriter` supports formatting output in a style similar to Python formatting via +:func:`BufferWriter::print`. This takes a format string which then controls the use of subsquent +arguments in generating out in the buffer. The basic format is divided in to three parts, separated by colons. + +.. productionList:: BufferWriterFormat + Format: "{" [name] [":" [specifier] [":" extension]] "}" + name: index | name + extension: * + +:arg:`name` + The name of the argument to use. This can be a number in which case it is the zero based index of the argument to the method call. E.g. ``{0}`` means the first argument and ``{2}`` is the third argument after the format. + + ``bw.print("{0} {1}", 'a', 'b')`` => ``a b`` + + ``bw.print("{1} {0}", 'a', 'b')`` => ``b a`` + + The name can be omitted in which case it is treated as an index in parallel to the position in + the format string. Only the position in the format string matters, not what names those other + format elements may have used. + + ``bw.print("{0} {2} {}", 'a', 'b', 'c')`` => ``a c c`` + + ``bw.print("{0} {2} {2}", 'a', 'b', 'c')`` => ``a c c`` + + Note that an argument can be printed more than once if the name is used more than once. + + ``bw.print("{0} {} {0}", 'a', 'b')`` => ``a b a`` + + ``bw.print("{0} {1} {0}", 'a', 'b')`` => ``a b a`` + + Alphanumeric names refer to values in a global table. These will be described in more detail someday. + +:arg:`specifier` + Basic formatting control. + + .. productionList:: specifier + specifier: [[fill]align][sign]["#"]["0"][[min][.precision][,max][type]] + fill: | URI-char + URI-char: "%" hex-digit hex-digit + align: "<" | ">" | "=" | "^" + sign: "+" | "-" | " " + min: integer + precision: integer + max: integer + type: "x" | "o" | "b" + + The output is placed in a field that is at least :token:`min` wide and no more than :token:`max` wide. If + the output is less than :token:`min` then + + * The :token:`fill` character is used for the extra space required. This can be an explicit + character or a URI encoded one (to allow otherwise reserved characters). + * The output is shifted according to the :token:`align`. + + < + Align to the left, fill to the right. + + > + Align to the right, fill to the left. + + ^ + Align in the middle, fill to left and right. + + = + Numerically align, putting the fill between the output and the sign character. + + The output is clipped by :token:`max` width characters or the end of the buffer. :token:`precision` is used by + floating point values to specify the number of places of precision. The precense of the ``#`` character is used for + integer values and causes a radix indicator to be used (one of ``0xb``, ``0``, ``0x``). + + :token:`type` is used to indicate type specific formatting. For integers it indicates the output + radix. If ``#`` is present the radix is prefix is generated with case matching that of the type + (e.g. type ``x`` causes ``0x`` and type ``X`` causes ``0X``). + + = =============== + b binary + o octal + x hexadecimal + = =============== + + +:arg:`extension` + Text (excluding braces) that is passed to the formatting function. This can be used to provide + extensions for specific argument types (e.g., IP addresses). The base logic ignores it but passes + it on to the formatting function for the corresponding argument type which can then behave + different based on the extension. + +User Defined Formatting ++++++++++++++++++++++++ + +When an value needs to be formatted an overloaded function for type :code:`V` is called. + +.. code-block:: cpp + + BufferWriter& ts::bwformat(BufferWriter& w, BWFSpec const& spec, V const& v) + +This can (and should be) overloaded for user defined types. This makes it easier and cheaper to +build one overload on another by tweaking the :arg:`spec` as it passed through. The calling +framework will handle basic alignment, the overload does not need to unless the alignment +requirements are more detailed (e.g. integer alignment operations). + +The output stream operator :code:`operator<<` is defined to call this function with a default +constructed :code:`BWFSpec` instance. + Futures +++++++ diff --git a/doc/developer-guide/internal-libraries/index.en.rst b/doc/developer-guide/internal-libraries/index.en.rst index 0542b079aa8..80582fdb94a 100644 --- a/doc/developer-guide/internal-libraries/index.en.rst +++ b/doc/developer-guide/internal-libraries/index.en.rst @@ -29,6 +29,7 @@ development team. :maxdepth: 1 string_view.en - memview.en + TextView.en + MemSpan.en scalar.en buffer-writer.en diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/s3_auth.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/s3_auth.en.po index 4d30ba432ee..3ff82295374 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/s3_auth.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/s3_auth.en.po @@ -59,7 +59,7 @@ msgid "" msgstr "" #: ../../admin-guide/plugins/s3_auth.en.rst:50 -msgid "... @plugin=s3_auth @pparam=--config @pparam=s3.config" +msgid "... @plugin=s3_auth.so @pparam=--config @pparam=s3.config" msgstr "" #: ../../admin-guide/plugins/s3_auth.en.rst:53 diff --git a/example/cppapi/delay_transformation_plugin/DelayTransformationPlugin.cc b/example/cppapi/delay_transformation_plugin/DelayTransformationPlugin.cc index bd834457314..b1fcca74744 100644 --- a/example/cppapi/delay_transformation_plugin/DelayTransformationPlugin.cc +++ b/example/cppapi/delay_transformation_plugin/DelayTransformationPlugin.cc @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -26,7 +27,6 @@ #include "ts/ts.h" using namespace atscppapi; -using std::string; /* * This example demonstrates how you can pause a transformation and resume it @@ -65,7 +65,7 @@ class DelayTransformationPlugin : public TransformationPlugin } void - consume(const string &data) + consume(ts::string_view data) { TS_DEBUG(TAG, "Consuming..."); produce(data); diff --git a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc index b4babb5d7dd..e7e59407022 100644 --- a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc +++ b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -103,7 +104,7 @@ class SomeTransformationPlugin : public TransformationPlugin } void - consume(const string &data) override + consume(ts::string_view data) override { produce(data); } diff --git a/example/cppapi/null_transformation_plugin/NullTransformationPlugin.cc b/example/cppapi/null_transformation_plugin/NullTransformationPlugin.cc index d85732e1c35..0fa649c78c7 100644 --- a/example/cppapi/null_transformation_plugin/NullTransformationPlugin.cc +++ b/example/cppapi/null_transformation_plugin/NullTransformationPlugin.cc @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -24,7 +25,6 @@ #include using namespace atscppapi; -using std::string; namespace { @@ -57,7 +57,7 @@ class NullTransformationPlugin : public TransformationPlugin } void - consume(const string &data) override + consume(ts::string_view data) override { produce(data); } diff --git a/example/cppapi/post_buffer/PostBuffer.cc b/example/cppapi/post_buffer/PostBuffer.cc index 941de6b560b..feda07a9992 100644 --- a/example/cppapi/post_buffer/PostBuffer.cc +++ b/example/cppapi/post_buffer/PostBuffer.cc @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -43,9 +44,9 @@ class PostBufferTransformationPlugin : public TransformationPlugin } void - consume(const string &data) override + consume(ts::string_view data) override { - buffer_.append(data); + buffer_.append(data.data(), data.length()); } void diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index ccf292631e6..b93ca1ecd25 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -306,9 +306,9 @@ CacheVC::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *abuf) vio.nbytes = nbytes; vio.vc_server = this; #ifdef DEBUG - ink_assert(c->mutex->thread_holding); + ink_assert(!c || c->mutex->thread_holding); #endif - if (!trigger && !recursive) { + if (c && !trigger && !recursive) { trigger = c->mutex->thread_holding->schedule_imm_local(this); } return &vio; @@ -344,9 +344,9 @@ CacheVC::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuf, bool vio.nbytes = nbytes; vio.vc_server = this; #ifdef DEBUG - ink_assert(c->mutex->thread_holding); + ink_assert(!c || c->mutex->thread_holding); #endif - if (!trigger && !recursive) { + if (c && !trigger && !recursive) { trigger = c->mutex->thread_holding->schedule_imm_local(this); } return &vio; diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index dbd4e84507d..a7ecbd3a459 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -116,9 +116,17 @@ class Continuation : private force_VFPT_to_top lock is initialized in the constructor and should not be set directly. + TODO: make this private. + */ Ptr mutex; + ProxyMutex * + getMutex() const + { + return mutex.get(); + } + /** Link to other continuations. diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index a7310b1df0a..9bcc81179fe 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -684,7 +684,7 @@ UnixNetVConnection::do_io_shutdown(ShutdownHowTo_t howto) read.vio.buffer.clear(); read.vio.nbytes = 0; read.vio._cont = nullptr; - f.shutdown = NET_VC_SHUTDOWN_READ; + f.shutdown |= NET_VC_SHUTDOWN_READ; break; case IO_SHUTDOWN_WRITE: socketManager.shutdown(((UnixNetVConnection *)this)->con.fd, 1); @@ -692,7 +692,7 @@ UnixNetVConnection::do_io_shutdown(ShutdownHowTo_t howto) write.vio.buffer.clear(); write.vio.nbytes = 0; write.vio._cont = nullptr; - f.shutdown = NET_VC_SHUTDOWN_WRITE; + f.shutdown |= NET_VC_SHUTDOWN_WRITE; break; case IO_SHUTDOWN_READWRITE: socketManager.shutdown(((UnixNetVConnection *)this)->con.fd, 2); diff --git a/lib/cppapi/GzipDeflateTransformation.cc b/lib/cppapi/GzipDeflateTransformation.cc index 43aa71c90e5..0695489cc25 100644 --- a/lib/cppapi/GzipDeflateTransformation.cc +++ b/lib/cppapi/GzipDeflateTransformation.cc @@ -19,17 +19,16 @@ * @file GzipDeflateTransformation.cc */ -#include #include #include #include #include +#include #include "atscppapi/TransformationPlugin.h" #include "atscppapi/GzipDeflateTransformation.h" #include "logging_internal.h" using namespace atscppapi::transformations; -using std::string; using std::vector; namespace @@ -81,7 +80,7 @@ GzipDeflateTransformation::~GzipDeflateTransformation() } void -GzipDeflateTransformation::consume(const string &data) +GzipDeflateTransformation::consume(ts::string_view data) { if (data.size() == 0) { return; @@ -94,7 +93,7 @@ GzipDeflateTransformation::consume(const string &data) int iteration = 0; state_->z_stream_.data_type = Z_ASCII; - state_->z_stream_.next_in = reinterpret_cast(const_cast(data.c_str())); + state_->z_stream_.next_in = reinterpret_cast(const_cast(data.data())); state_->z_stream_.avail_in = data.length(); // For small payloads the size can actually be greater than the original input @@ -119,7 +118,7 @@ GzipDeflateTransformation::consume(const string &data) LOG_DEBUG("Iteration %d: Deflate compressed %ld bytes to %d bytes, producing output...", iteration, data.size(), bytes_to_write); - produce(string(reinterpret_cast(&buffer[0]), static_cast(bytes_to_write))); + produce(ts::string_view(reinterpret_cast(&buffer[0]), static_cast(bytes_to_write))); } while (state_->z_stream_.avail_out == 0); state_->z_stream_.next_out = nullptr; @@ -153,7 +152,7 @@ GzipDeflateTransformation::handleInputComplete() if (status == Z_OK || status == Z_STREAM_END) { LOG_DEBUG("Iteration %d: Gzip deflate finalize had an extra %d bytes to process, status '%d'. Producing output...", iteration, bytes_to_write, status); - produce(string(reinterpret_cast(buffer), static_cast(bytes_to_write))); + produce(ts::string_view(reinterpret_cast(buffer), static_cast(bytes_to_write))); } else if (status != Z_STREAM_END) { LOG_ERROR("Iteration %d: Gzip deflinate finalize produced an error '%d'", iteration, status); } diff --git a/lib/cppapi/GzipInflateTransformation.cc b/lib/cppapi/GzipInflateTransformation.cc index bd73b3f19a2..e0c8f1297d5 100644 --- a/lib/cppapi/GzipInflateTransformation.cc +++ b/lib/cppapi/GzipInflateTransformation.cc @@ -20,17 +20,16 @@ * @file GzipInflateTransformation.cc */ -#include #include #include #include #include +#include #include "atscppapi/TransformationPlugin.h" #include "atscppapi/GzipInflateTransformation.h" #include "logging_internal.h" using namespace atscppapi::transformations; -using std::string; using std::vector; namespace @@ -84,7 +83,7 @@ GzipInflateTransformation::~GzipInflateTransformation() } void -GzipInflateTransformation::consume(const string &data) +GzipInflateTransformation::consume(ts::string_view data) { if (data.size() == 0) { return; @@ -101,7 +100,7 @@ GzipInflateTransformation::consume(const string &data) vector buffer(inflate_block_size); // Setup the compressed input - state_->z_stream_.next_in = reinterpret_cast(const_cast(data.c_str())); + state_->z_stream_.next_in = reinterpret_cast(const_cast(data.data())); state_->z_stream_.avail_in = data.length(); // Loop while we have more data to inflate @@ -123,7 +122,7 @@ GzipInflateTransformation::consume(const string &data) LOG_DEBUG("Iteration %d: Gzip inflated a total of %d bytes, producingOutput...", iteration, (inflate_block_size - state_->z_stream_.avail_out)); - produce(string(&buffer[0], (inflate_block_size - state_->z_stream_.avail_out))); + produce(ts::string_view(&buffer[0], (inflate_block_size - state_->z_stream_.avail_out))); state_->bytes_produced_ += (inflate_block_size - state_->z_stream_.avail_out); } state_->z_stream_.next_out = nullptr; diff --git a/lib/cppapi/TransformationPlugin.cc b/lib/cppapi/TransformationPlugin.cc index fa9c6b94ae9..2746d7c53a6 100644 --- a/lib/cppapi/TransformationPlugin.cc +++ b/lib/cppapi/TransformationPlugin.cc @@ -302,7 +302,7 @@ TransformationPlugin::resumeCallback(TSCont cont, TSEvent event, void *edata) } size_t -TransformationPlugin::doProduce(const std::string &data) +TransformationPlugin::doProduce(ts::string_view data) { LOG_DEBUG("TransformationPlugin=%p tshttptxn=%p producing output with length=%ld", this, state_->txn_, data.length()); int64_t write_length = static_cast(data.length()); @@ -331,7 +331,7 @@ TransformationPlugin::doProduce(const std::string &data) } // Finally we can copy this data into the output_buffer - int64_t bytes_written = TSIOBufferWrite(state_->output_buffer_, data.c_str(), write_length); + int64_t bytes_written = TSIOBufferWrite(state_->output_buffer_, data.data(), write_length); state_->bytes_written_ += bytes_written; // So we can set BytesDone on outputComplete(). LOG_DEBUG("TransformationPlugin=%p tshttptxn=%p write to TSIOBuffer %" PRId64 " bytes total bytes written %" PRId64, this, state_->txn_, bytes_written, state_->bytes_written_); @@ -358,10 +358,10 @@ TransformationPlugin::doProduce(const std::string &data) } size_t -TransformationPlugin::produce(const std::string &data) +TransformationPlugin::produce(ts::string_view data) { if (state_->type_ == REQUEST_TRANSFORMATION) { - state_->request_xform_output_.append(data); + state_->request_xform_output_.append(data.data(), data.length()); return data.size(); } else if (state_->type_ == SINK_TRANSFORMATION) { LOG_DEBUG("produce TransformationPlugin=%p tshttptxn=%p : This is a sink transform. Not producing any output", this, diff --git a/lib/cppapi/include/atscppapi/GzipDeflateTransformation.h b/lib/cppapi/include/atscppapi/GzipDeflateTransformation.h index f25b00954b3..dd235614326 100644 --- a/lib/cppapi/include/atscppapi/GzipDeflateTransformation.h +++ b/lib/cppapi/include/atscppapi/GzipDeflateTransformation.h @@ -22,10 +22,9 @@ */ #pragma once -#ifndef ATSCPPAPI_GZIPDEFLATETRANSFORMATION_H_ -#define ATSCPPAPI_GZIPDEFLATETRANSFORMATION_H_ #include +#include #include "atscppapi/TransformationPlugin.h" namespace atscppapi @@ -71,13 +70,13 @@ namespace transformations * * @param data the input data to compress */ - void consume(const std::string &data); + void consume(ts::string_view data) override; /** * Any TransformationPlugin must implement handleInputComplete(), this method will * finalize the gzip compression and flush any remaining data and the epilouge. */ - void handleInputComplete(); + void handleInputComplete() override; virtual ~GzipDeflateTransformation(); @@ -86,5 +85,3 @@ namespace transformations }; } } - -#endif /* ATSCPPAPI_GZIPDEFLATETRANSFORMATION_H_ */ diff --git a/lib/cppapi/include/atscppapi/GzipInflateTransformation.h b/lib/cppapi/include/atscppapi/GzipInflateTransformation.h index d0e47136c5e..425d1ed4447 100644 --- a/lib/cppapi/include/atscppapi/GzipInflateTransformation.h +++ b/lib/cppapi/include/atscppapi/GzipInflateTransformation.h @@ -22,10 +22,8 @@ */ #pragma once -#ifndef ATSCPPAPI_GZIPINFLATETRANSFORMATION_H_ -#define ATSCPPAPI_GZIPINFLATETRANSFORMATION_H_ -#include +#include #include "atscppapi/TransformationPlugin.h" namespace atscppapi @@ -73,13 +71,13 @@ namespace transformations * * @param data the input data to decompress */ - void consume(const std::string &); + void consume(ts::string_view) override; /** * Any TransformationPlugin must implement handleInputComplete(), this method will * finalize the gzip decompression. */ - void handleInputComplete(); + void handleInputComplete() override; virtual ~GzipInflateTransformation(); @@ -90,5 +88,3 @@ namespace transformations } /* transformations */ } /* atscppapi */ - -#endif /* ATSCPPAPI_GZIPINFLATETRANSFORMATION_H_ */ diff --git a/lib/cppapi/include/atscppapi/TransformationPlugin.h b/lib/cppapi/include/atscppapi/TransformationPlugin.h index 714d8e2ab96..7c640fcbf33 100644 --- a/lib/cppapi/include/atscppapi/TransformationPlugin.h +++ b/lib/cppapi/include/atscppapi/TransformationPlugin.h @@ -21,10 +21,8 @@ */ #pragma once -#ifndef ATSCPPAPI_TRANSFORMATIONPLUGIN_H_ -#define ATSCPPAPI_TRANSFORMATIONPLUGIN_H_ -#include +#include #include #include @@ -64,7 +62,7 @@ struct TransformationPluginState; * transaction.getClientResponse().getHeaders().set("X-Content-Transformed", "1"); * transaction.resume(); * } - * void consume(const string &data) { + * void consume(ts::string_view data) { * produce(data); * } * void handleInputComplete() { @@ -95,7 +93,7 @@ class TransformationPlugin : public TransactionPlugin * A method that you must implement when writing a TransformationPlugin, this method will be * fired whenever an upstream TransformationPlugin has produced output. */ - virtual void consume(const std::string &data) = 0; + virtual void consume(ts::string_view data) = 0; /** * Call this method if you wish to pause the transformation. @@ -117,11 +115,9 @@ class TransformationPlugin : public TransactionPlugin protected: /** * This method is how a TransformationPlugin will produce output for the downstream - * transformation plugin, if you need to produce binary data this can still be - * done with strings by a call to string::assign() or by constructing a string - * with string::string(char *, size_t). + * transformation plugin. */ - size_t produce(const std::string &); + size_t produce(ts::string_view); /** * This is the method that you must call when you're done producing output for @@ -134,10 +130,8 @@ class TransformationPlugin : public TransactionPlugin private: TransformationPluginState *state_; /** Internal state for a TransformationPlugin */ - size_t doProduce(const std::string &); + size_t doProduce(ts::string_view); static int resumeCallback(TSCont cont, TSEvent event, void *edata); /** Resume callback*/ }; } /* atscppapi */ - -#endif /* ATSCPPAPI_TRANSFORMATIONPLUGIN_H_ */ diff --git a/lib/ts/BufferWriter.h b/lib/ts/BufferWriter.h index ab6a6fa02b4..66f2aad302c 100644 --- a/lib/ts/BufferWriter.h +++ b/lib/ts/BufferWriter.h @@ -27,9 +27,13 @@ #include #include #include +#include +#include +#include -#include +#include #include +#include namespace ts { @@ -170,6 +174,22 @@ class BufferWriter // Force virtual destructor. virtual ~BufferWriter() {} + + /** BufferWriter print. + + This prints its arguments to the @c BufferWriter @a w according to the format @a fmt. The format + string is based on Python style formating, each argument substitution marked by braces, {}. Each + specification has three parts, a @a name, a @a specifier, and an @a extention. These are + separated by colons. The name should be either omitted or a number, the index of the argument to + use. If omitted the place in the format string is used as the argument index. E.g. "{} {} {}", + "{} {1} {}", and "{0} {1} {2}" are equivalent. Using an explicit index does not reset the + position of subsequent substiations, therefore "{} {0} {}" is equivalent to "{0} {0} {2}". + */ + template BufferWriter &print(TextView fmt, Rest... rest); + + template BufferWriter &print(BWFormat const &fmt, Rest... rest); + + // bwprint(*this, fmt, std::forward(rest)...); }; /** A @c BufferWrite concrete subclass to write to a fixed size buffer. @@ -412,64 +432,254 @@ template class LocalBufferWriter : public FixedBufferWriter char _arr[N]; ///< output buffer. }; -// Define stream operators for built in @c write overloads. +// --------------- Implementation -------------------- +/** Overridable formatting for type @a V. + + This is the output generator for data to a @c BufferWriter. Default stream operators call this with + the default format specification (although those can be overloaded specifically for performance). + User types should overload this function to format output for that type. + + @code + BufferWriter & + bwformat(BufferWriter &w, BWFSpec &, V const &v) + { + // generate output on @a w + } + @endcode + */ + +namespace bw_fmt +{ + template using ArgFormatterSignature = BufferWriter &(*)(BufferWriter &w, BWFSpec const &, TUPLE const &args); + + /// Internal error / reporting message generators + void Err_Bad_Arg_Index(BufferWriter &w, int i, size_t n); + + // MSVC will expand the parameter pack inside a lambda but not gcc, so this indirection is required. + + /// This selects the @a I th argument in the @a TUPLE arg pack and calls the formatter on it. This + /// (or the equivalent lambda) is needed because the array of formatters must have a homogenous + /// signature, not vary per argument. Effectively this indirection erases the type of the specific + /// argument being formatter. + template + BufferWriter & + Arg_Formatter(BufferWriter &w, BWFSpec const &spec, TUPLE const &args) + { + return bwformat(w, spec, std::get(args)); + } + + /// This exists only to expand the index sequence into an array of formatters for the tuple type + /// @a TUPLE. Due to langauge limitations it cannot be done directly. The formatters can be + /// access via standard array access in constrast to templated tuple access. The actual array is + /// static and therefore at run time the only operation is loading the address of the array. + template + ArgFormatterSignature * + Get_Arg_Formatter_Array(std::index_sequence) + { + static ArgFormatterSignature fa[sizeof...(N)] = {&bw_fmt::Arg_Formatter...}; + return fa; + } + + /// Perform alignment adjustments / fill on @a w of the content in @a lw. + void Do_Alignment(BWFSpec const &spec, BufferWriter &w, BufferWriter &lw); + + /// Global named argument table. + using GlobalSignature = void (*)(BufferWriter &, BWFSpec const &); + using GlobalTable = std::map; + extern GlobalTable BWF_GLOBAL_TABLE; + extern GlobalSignature Global_Table_Find(string_view name); + + /// Generic integral conversion. + BufferWriter &Format_Integer(BufferWriter &w, BWFSpec const &spec, uintmax_t n, bool negative_p); + +} // bw_fmt + +/** Compiled BufferWriter format + */ +class BWFormat +{ +public: + /// Construct from a format string @a fmt. + BWFormat(TextView fmt); + ~BWFormat(); + + /** Parse elements of a format string. + + @param fmt The format string [in|out] + @param literal A literal if found + @param spec A specifier if found (less enclosing braces) + @return @c true if a specifier was found, @c false if not. + + Pull off the next literal and/or specifier from @a fmt. The return value distinguishes + the case of no specifier found (@c false) or an empty specifier (@c true). + + */ + static bool parse(TextView &fmt, string_view &literal, string_view &spec); + + /** Parsed items from the format string. + + Literals are handled by putting the literal text in the extension field and setting the + global formatter @a _gf to @c LiteralFormatter, which writes out the extension as a literal. + */ + struct Item { + BWFSpec _spec; ///< Specification. + /// If the spec has a global formatter name, cache it here. + mutable bw_fmt::GlobalSignature _gf = nullptr; + + Item() {} + Item(BWFSpec const &spec, bw_fmt::GlobalSignature gf) : _spec(spec), _gf(gf) {} + }; + + using Items = std::vector; + Items _items; ///< Items from format string. + +protected: + /// Handles literals by writing the contents of the extension directly to @a w. + static void Format_Literal(BufferWriter &w, BWFSpec const &spec); +}; + +template +BufferWriter & +BufferWriter::print(TextView fmt, Rest... rest) +{ + static constexpr int N = sizeof...(Rest); + auto args(std::forward_as_tuple(rest...)); + auto fa = bw_fmt::Get_Arg_Formatter_Array(std::index_sequence_for{}); + int arg_idx = 0; + size_t base = this->size(); + + while (fmt.size()) { + string_view lit_v; + string_view spec_v; + bool spec_p = BWFormat::parse(fmt, lit_v, spec_v); + + if (lit_v.size()) { + this->write(lit_v); + } + if (spec_p) { + BWFSpec spec{spec_v}; + size_t width = this->remaining(); + if (spec._max > 0) + width = std::min(width, static_cast(spec._max)); + FixedBufferWriter lw{this->auxBuffer(), width}; + + if (spec._name.size() == 0) { + spec._idx = arg_idx; + } + if (0 <= spec._idx) { + if (spec._idx < N) { + fa[spec._idx](lw, spec, args); + } else { + bw_fmt::Err_Bad_Arg_Index(lw, spec._idx, N); + } + } else if (spec._name.size()) { + auto gf = bw_fmt::Global_Table_Find(spec._name); + if (gf) { + gf(lw, spec); + } else { + static constexpr TextView msg{"{invalid name:"}; + lw.write(msg).write(spec._name).write('}'); + } + } + if (lw.size()) { + bw_fmt::Do_Alignment(spec, *this, lw); + } + ++arg_idx; + } + } + return *this; +} + +template +BufferWriter & +BufferWriter::print(BWFormat const &fmt, Rest... rest) +{ + static constexpr int N = sizeof...(Rest); + auto const args(std::forward_as_tuple(rest...)); + static const auto fa = bw_fmt::Get_Arg_Formatter_Array(std::index_sequence_for{}); + + for (BWFormat::Item const &item : fmt._items) { + size_t width = this->remaining(); + size_t max = item._spec._max; + if (max && max < width) { + width = max; + } + FixedBufferWriter lw{this->auxBuffer(), width}; + if (item._gf) { + item._gf(lw, item._spec); + } else { + auto idx = item._spec._idx; + if (0 <= idx && idx < N) { + fa[idx](lw, item._spec, args); + } else if (item._spec._name.size() && (nullptr != (item._gf = bw_fmt::Global_Table_Find(item._spec._name)))) { + item._gf(lw, item._spec); + } + } + bw_fmt::Do_Alignment(item._spec, *this, lw); + } + return *this; +} + +// Generically a stream operator is a formatter with the default specification. +template +BufferWriter & +operator<<(BufferWriter &w, V &&v) +{ + return bwformat(w, BWFSpec::DEFAULT, std::forward(v)); +} + +// -- Common formatters -- + +BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, string_view sv); inline BufferWriter & -operator<<(BufferWriter &b, char c) +bwformat(BufferWriter &w, BWFSpec const &, char c) { - return b.write(c); + return w.write(c); } inline BufferWriter & -operator<<(BufferWriter &b, const string_view &sv) +bwformat(BufferWriter &w, BWFSpec const &spec, const char *v) { - return b.write(sv); + return bwformat(w, spec, string_view(v)); } inline BufferWriter & -operator<<(BufferWriter &w, intmax_t i) +bwformat(BufferWriter &w, BWFSpec const &spec, TextView const &tv) { - if (i) { - char txt[std::numeric_limits::digits10 + 1]; - int n = sizeof(txt); - while (i) { - txt[--n] = '0' + i % 10; - i /= 10; - } - return w.write(txt + n, sizeof(txt) - n); - } else { - return w.write('0'); - } + return bwformat(w, spec, static_cast(tv)); } -// Annoying but otherwise ambiguous. +//-- Integral types inline BufferWriter & -operator<<(BufferWriter &w, int i) +bwformat(BufferWriter &w, BWFSpec const &spec, uintmax_t const &i) { - return w << static_cast(i); + return bw_fmt::Format_Integer(w, spec, i, false); } inline BufferWriter & -operator<<(BufferWriter &w, uintmax_t i) +bwformat(BufferWriter &w, BWFSpec const &spec, intmax_t const &i) { - if (i) { - char txt[std::numeric_limits::digits10 + 1]; - int n = sizeof(txt); - while (i) { - txt[--n] = '0' + i % 10; - i /= 10; - } - return w.write(txt + n, sizeof(txt) - n); - } else { - return w.write('0'); - } + return i < 0 ? bw_fmt::Format_Integer(w, spec, -i, true) : bw_fmt::Format_Integer(w, spec, i, false); +} + +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, unsigned int const &i) +{ + return bw_fmt::Format_Integer(w, spec, i, false); +} + +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, int const &i) +{ + return i < 0 ? bw_fmt::Format_Integer(w, spec, -i, true) : bw_fmt::Format_Integer(w, spec, i, false); } -// Annoying but otherwise ambiguous. +// Annoying but otherwise ambiguous with char inline BufferWriter & -operator<<(BufferWriter &w, unsigned int i) +operator<<(BufferWriter &w, int const &i) { - return w << static_cast(i); + return bwformat(w, BWFSpec::DEFAULT, static_cast(i)); } } // end namespace ts diff --git a/lib/ts/BufferWriterFormat.cc b/lib/ts/BufferWriterFormat.cc new file mode 100644 index 00000000000..921664d8976 --- /dev/null +++ b/lib/ts/BufferWriterFormat.cc @@ -0,0 +1,489 @@ +#include +#include +#include + +ts::bw_fmt::GlobalTable ts::bw_fmt::BWF_GLOBAL_TABLE; +const ts::BWFSpec ts::BWFSpec::DEFAULT; + +namespace +{ +// Customized version of string to int. Using this instead of the general @c svtoi function +// made @c bwprint performance test run in < 30% of the time, changing it from about 2.5 +// times slower than snprintf to the same speed. This version handles only positive integers +// in decimal. +inline int +tv_to_positive_decimal(ts::TextView src, ts::TextView *out) +{ + int zret = 0; + + if (out) { + out->clear(); + } + src.ltrim_if(&isspace); + if (src.size()) { + const char *start = src.data(); + const char *limit = start + src.size(); + while (start < limit && ('0' <= *start && *start <= '9')) { + zret = zret * 10 + *start - '0'; + ++start; + } + if (out && (start > src.data())) { + out->set_view(src.data(), start); + } + } + return zret; +} +} + +void +ts::bw_fmt::Err_Bad_Arg_Index(BufferWriter &w, int i, size_t n) +{ + static const BWFormat fmt{"{{BAD_ARG_INDEX:{} of {}}}"_sv}; + w.print(fmt, i, n); +} + +/// Parse a format specification. +ts::BWFSpec::BWFSpec(TextView fmt) +{ + TextView num; + intmax_t n; + + _name = fmt.take_prefix_at(':'); + // if it's parsable as a number, treat it as an index. + n = tv_to_positive_decimal(_name, &num); + if (num.size()) + _idx = static_cast(n); + + if (fmt.size()) { + TextView sz = fmt.take_prefix_at(':'); // the format specifier. + _ext = fmt; // anything past the second ':' is the extension. + if (sz.size()) { + // fill and alignment + if ('%' == *sz) { // enable URI encoding of the fill character so metasyntactic chars can be used if needed. + if (sz.size() < 4) { + throw std::invalid_argument("Fill URI encoding without 2 hex characters and align mark"); + } + if (Align::NONE == (_align = align_of(sz[3]))) { + throw std::invalid_argument("Fill URI without alignment mark"); + } + char d1 = sz[1], d0 = sz[2]; + if (!isxdigit(d0) || !isxdigit(d1)) { + throw std::invalid_argument("URI encoding with non-hex characters"); + } + _fill = isdigit(d0) ? d0 - '0' : tolower(d0) - 'a' + 10; + _fill += (isdigit(d1) ? d1 - '0' : tolower(d1) - 'a' + 10) << 4; + sz += 4; + } else if (sz.size() > 1 && Align::NONE != (_align = align_of(sz[1]))) { + _fill = *sz; + sz += 2; + } else if (Align::NONE != (_align = align_of(*sz))) { + ++sz; + } + if (!sz.size()) + return; + // sign + if (is_sign(*sz)) { + _sign = *sz; + if (!(++sz).size()) + return; + } + // radix prefix + if ('#' == *sz) { + _radix_lead_p = true; + if (!(++sz).size()) + return; + } + // 0 fill for integers + if ('0' == *sz) { + if (Align::NONE == _align) + _align = Align::SIGN; + _fill = '0'; + ++sz; + } + n = tv_to_positive_decimal(sz, &num); + if (num.size()) { + _min = static_cast(n); + sz.remove_prefix(num.size()); + if (!sz.size()) + return; + } + // precision + if ('.' == *sz) { + n = tv_to_positive_decimal(++sz, &num); + if (num.size()) { + _prec = static_cast(n); + sz.remove_prefix(num.size()); + if (!sz.size()) + return; + } else { + throw std::invalid_argument("Precision mark without precision"); + } + } + // style (type). Hex, octal, etc. + if (is_type(*sz)) { + _type = *sz; + if (!(++sz).size()) + return; + } + // maximum width + if (',' == *sz) { + n = tv_to_positive_decimal(++sz, &num); + if (num.size()) { + _max = static_cast(n); + sz.remove_prefix(num.size()); + if (!sz.size()) + return; + } else { + throw std::invalid_argument("Maximum width mark without width"); + } + // Can only have a type indicator here if there was a max width. + if (is_type(*sz)) { + _type = *sz; + if (!(++sz).size()) + return; + } + } + } + } +} + +/** This performs generic alignment operations. + + If a formatter specialization performs this operation instead, that should result in output that + is at least @a spec._min characters wide, which will cause this function to make no further + adjustments. + */ +void +ts::bw_fmt::Do_Alignment(BWFSpec const &spec, BufferWriter &w, BufferWriter &lw) +{ + size_t size = lw.size(); + size_t min = spec._min; + if (size < min) { + size_t delta = min - size; // note - size <= extent -> size < min + switch (spec._align) { + case BWFSpec::Align::NONE: // same as LEFT for output. + case BWFSpec::Align::LEFT: + w.fill(size); + while (delta--) + w.write(spec._fill); + break; + case BWFSpec::Align::RIGHT: + std::memmove(w.auxBuffer() + delta, w.auxBuffer(), size); + while (delta--) + w.write(spec._fill); + w.fill(size); + break; + case BWFSpec::Align::CENTER: + if (delta > 1) { + size_t d2 = delta / 2; + std::memmove(w.auxBuffer() + (delta / 2), w.auxBuffer(), size); + while (d2--) + w.write(spec._fill); + } + w.fill(size); + delta = (delta + 1) / 2; + while (delta--) + w.write(spec._fill); + break; + case BWFSpec::Align::SIGN: + w.fill(size); + break; + } + } else { + w.fill(size); + } +} + +// Numeric conversions +namespace ts::bw_fmt +{ +// Conversions from remainder to character, in upper and lower case versions. +// Really only useful for hexadecimal currently. +namespace +{ + char UPPER_DIGITS[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + char LOWER_DIGITS[] = "0123456789abcdefghijklmnopqrstuvwxyz"; +} +/// Templated radix based conversions. Only a small number of radix are supported +/// and providing a template minimizes cut and paste code while also enabling +/// compiler optimizations (e.g. for power of 2 radix the modulo / divide become +/// bit operations). +template +size_t +To_Radix(uintmax_t n, char *buff, size_t width, char *digits) +{ + static_assert(1 < RADIX && RADIX <= 36); + char *out = buff + width; + if (n) { + while (n) { + *--out = digits[n % RADIX]; + n /= RADIX; + } + } else { + *--out = '0'; + } + return (buff + width) - out; +} + +BufferWriter & +Format_Integer(BufferWriter &w, BWFSpec const &spec, uintmax_t i, bool neg_p) +{ + size_t n = 0; + int width = static_cast(spec._min); // amount left to fill. + string_view prefix; + char neg = 0; + char prefix1 = spec._radix_lead_p ? '0' : 0; + char prefix2 = 0; + char buff[std::numeric_limits::digits + 1]; + + if (neg_p) { + neg = '-'; + } else if (spec._sign != '-') { + neg = spec._sign; + } + + switch (spec._type) { + case 'x': + prefix2 = 'x'; + n = bw_fmt::To_Radix<16>(i, buff, sizeof(buff), bw_fmt::LOWER_DIGITS); + break; + case 'X': + prefix2 = 'X'; + n = bw_fmt::To_Radix<16>(i, buff, sizeof(buff), bw_fmt::UPPER_DIGITS); + break; + case 'b': + prefix2 = 'b'; + n = bw_fmt::To_Radix<2>(i, buff, sizeof(buff), bw_fmt::LOWER_DIGITS); + break; + case 'B': + prefix2 = 'B'; + n = bw_fmt::To_Radix<2>(i, buff, sizeof(buff), bw_fmt::UPPER_DIGITS); + break; + case 'o': + n = bw_fmt::To_Radix<8>(i, buff, sizeof(buff), bw_fmt::LOWER_DIGITS); + break; + default: + prefix1 = 0; + n = bw_fmt::To_Radix<10>(i, buff, sizeof(buff), bw_fmt::LOWER_DIGITS); + break; + } + // Clip fill width by stuff that's already committed to be written. + if (neg) + --width; + if (prefix1) { + --width; + if (prefix2) + --width; + } + width -= static_cast(n); + string_view digits{buff + sizeof(buff) - n, n}; + + // The idea here is the various pieces have all been assembled, the only difference + // is the order in which they are written to the output. + switch (spec._align) { + case BWFSpec::Align::LEFT: + if (neg) + w.write(neg); + if (prefix1) { + w.write(prefix1); + if (prefix2) + w.write(prefix2); + } + w.write(digits); + while (width-- > 0) + w.write(spec._fill); + break; + case BWFSpec::Align::RIGHT: + while (width-- > 0) + w.write(spec._fill); + if (neg) + w.write(neg); + if (prefix1) { + w.write(prefix1); + if (prefix2) + w.write(prefix2); + } + w.write(digits); + break; + case BWFSpec::Align::CENTER: + for (int i = width / 2; i > 0; --i) + w.write(spec._fill); + if (neg) + w.write(neg); + if (prefix1) { + w.write(prefix1); + if (prefix2) + w.write(prefix2); + } + w.write(digits); + for (int i = (width + 1) / 2; i > 0; --i) + w.write(spec._fill); + break; + case BWFSpec::Align::SIGN: + if (neg) + w.write(neg); + if (prefix1) { + w.write(prefix1); + if (prefix2) + w.write(prefix2); + } + while (width-- > 0) + w.write(spec._fill); + w.write(digits); + break; + default: + if (neg) + w.write(neg); + if (prefix1) { + w.write(prefix1); + if (prefix2) + w.write(prefix2); + } + w.write(digits); + break; + } + return w; +} + +} // ts::detail + +ts::BufferWriter & +ts::bwformat(BufferWriter &w, BWFSpec const &spec, string_view sv) +{ + int width = static_cast(spec._min); // amount left to fill. + if (spec._prec > 0) + sv.remove_prefix(spec._prec); + + width -= sv.size(); + switch (spec._align) { + case BWFSpec::Align::LEFT: + case BWFSpec::Align::SIGN: + w.write(sv); + while (width-- > 0) + w.write(spec._fill); + break; + case BWFSpec::Align::RIGHT: + while (width-- > 0) + w.write(spec._fill); + w.write(sv); + break; + case BWFSpec::Align::CENTER: + for (int i = width / 2; i > 0; --i) + w.write(spec._fill); + w.write(sv); + for (int i = (width + 1) / 2; i > 0; --i) + w.write(spec._fill); + break; + default: + w.write(sv); + break; + } + return w; +} + +/// Preparse format string for later use. +ts::BWFormat::BWFormat(ts::TextView fmt) +{ + BWFSpec lit_spec{BWFSpec::DEFAULT}; + int arg_idx = 0; + + while (fmt) { + string_view lit_str; + string_view spec_str; + bool spec_p = this->parse(fmt, lit_str, spec_str); + + if (lit_str.size()) { + lit_spec._ext = lit_str; + _items.emplace_back(lit_spec, &Format_Literal); + } + if (spec_p) { + bw_fmt::GlobalSignature gf = nullptr; + BWFSpec parsed_spec{spec_str}; + if (parsed_spec._name.size() == 0) { + parsed_spec._idx = arg_idx; + } + if (parsed_spec._idx < 0) { + gf = bw_fmt::Global_Table_Find(parsed_spec._name); + } + _items.emplace_back(parsed_spec, gf); + ++arg_idx; + } + } +} + +ts::BWFormat::~BWFormat() +{ +} + +bool +ts::BWFormat::parse(ts::TextView &fmt, string_view &literal, string_view &specifier) +{ + TextView::size_type off; + + off = fmt.find_if([](char c) { return '{' == c || '}' == c; }); + if (off == TextView::npos) { + literal = fmt; + fmt.remove_prefix(literal.size()); + return false; + } + + if (fmt.size() > off + 1) { + char c1 = fmt[off]; + char c2 = fmt[off + 1]; + if (c1 == c2) { + literal = fmt.take_prefix_at(off + 1); + return false; + } else if ('}' == c1) { + throw std::invalid_argument("Unopened }"); + } else { + literal = string_view{fmt.data(), off}; + fmt.remove_prefix(off + 1); + } + } else { + throw std::invalid_argument("Invalid trailing character"); + } + + if (fmt.size()) { + // Need to be careful, because an empty format is OK and it's hard to tell if + // take_prefix_at failed to find the delimiter or found it as the first byte. + off = fmt.find('}'); + if (off == TextView::npos) { + throw std::invalid_argument("Unclosed {"); + } + specifier = fmt.take_prefix_at(off); + return true; + } + return false; +} + +void +ts::BWFormat::Format_Literal(BufferWriter &w, BWFSpec const &spec) +{ + w.write(spec._ext); +} + +ts::bw_fmt::GlobalSignature +ts::bw_fmt::Global_Table_Find(string_view name) +{ + if (name.size()) { + auto spot = bw_fmt::BWF_GLOBAL_TABLE.find(name); + if (spot != bw_fmt::BWF_GLOBAL_TABLE.end()) + return spot->second; + } + return nullptr; +} + +namespace +{ +void +BWF_Now(ts::BufferWriter &w, ts::BWFSpec const &spec) +{ + std::time_t t = std::time(nullptr); + w.fill(std::strftime(w.auxBuffer(), w.remaining(), "%Y%b%d:%H%M%S", std::localtime(&t))); +} + +static bool BW_INITIALIZED = []() -> bool { + ts::bw_fmt::BWF_GLOBAL_TABLE.emplace("now", &BWF_Now); + return true; +}(); +} diff --git a/lib/ts/BufferWriterForward.h b/lib/ts/BufferWriterForward.h new file mode 100644 index 00000000000..aca64e8f440 --- /dev/null +++ b/lib/ts/BufferWriterForward.h @@ -0,0 +1,102 @@ +/** @file + + Forward definitions for BufferWriter formatting. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace ts +{ +/** A parsed version of a format specifier. + */ +struct BWFSpec { + using self_type = BWFSpec; ///< Self reference type. + /// Constructor a default instance. + constexpr BWFSpec() {} + + /// Construct by parsing @a fmt. + BWFSpec(TextView fmt); + + char _fill = ' '; ///< Fill character. + char _sign = '-'; ///< Numeric sign style, space + - + enum class Align : char { + NONE, ///< No alignment. + LEFT, ///< Left alignment '<'. + RIGHT, ///< Right alignment '>'. + CENTER, ///< Center alignment '='. + SIGN ///< Align plus/minus sign before numeric fill. '^' + } _align = Align::NONE; ///< Output field alignment. + char _type = 'g'; ///< Type / radix indicator. + bool _radix_lead_p = false; ///< Print leading radix indication. + // @a _min is unsigned because there's no point in an invalid default, 0 works fine. + unsigned int _min = 0; ///< Minimum width. + int _prec = -1; ///< Precision + unsigned int _max = 0; ///< Maxium width + int _idx = -1; ///< Positional "name" of the specification. + string_view _name; ///< Name of the specification. + string_view _ext; ///< Extension if provided. + + static const self_type DEFAULT; + +protected: + /// Validate character is alignment character and return the appropriate enum value. + Align align_of(char c); + + /// Validate is sign indicator. + bool is_sign(char c); + + /// Validate @a c is a specifier type indicator. + bool is_type(char c); +}; + +inline BWFSpec::Align +BWFSpec::align_of(char c) +{ + return '<' == c ? Align::LEFT : '>' == c ? Align::RIGHT : '=' == c ? Align::CENTER : '^' == c ? Align::SIGN : Align::NONE; +} + +inline bool +BWFSpec::is_sign(char c) +{ + return '+' == c || '-' == c || ' ' == c; +} + +inline bool +BWFSpec::is_type(char c) +{ + return 'x' == c || 'X' == c || 'o' == c || 'b' == c || 'B' == c || 'd' == c; +} + +class BWFormat; + +class BufferWriter; + +} // ts diff --git a/lib/ts/Makefile.am b/lib/ts/Makefile.am index 7175ac84c08..f87bd10d20f 100644 --- a/lib/ts/Makefile.am +++ b/lib/ts/Makefile.am @@ -23,7 +23,7 @@ library_includedir=$(includedir)/ts library_include_HEADERS = apidefs.h string_view.h noinst_PROGRAMS = mkdfa CompileParseRules -check_PROGRAMS = test_tsutil test_arena test_atomic test_freelist test_geometry test_List test_Map test_Vec test_X509HostnameValidator test_Scalar test_tslib +check_PROGRAMS = test_tsutil test_arena test_atomic test_freelist test_geometry test_List test_Map test_Vec test_X509HostnameValidator test_tslib TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=suppression.txt @@ -56,6 +56,10 @@ libtsutil_la_SOURCES = \ BaseLogFile.h \ Bitops.cc \ Bitops.h \ + BufferWriter.h \ + BufferWriterForward.h \ + BufferWriterFormat.cc \ + c14_utility.h \ ConsistentHash.cc \ ConsistentHash.h \ ContFlags.cc \ @@ -170,6 +174,7 @@ libtsutil_la_SOURCES = \ Map.h \ MatcherUtils.cc \ MatcherUtils.h \ + MemSpan.h \ MMH.cc \ MMH.h \ MT_hashtable.h \ @@ -194,7 +199,6 @@ libtsutil_la_SOURCES = \ SourceLocation.cc \ SourceLocation.h \ string_view.h \ - BufferWriter.h \ TestBox.h \ TextBuffer.cc \ TextBuffer.h \ @@ -251,8 +255,6 @@ test_tsutil_SOURCES = \ test_Regex.cc \ tests.cc -test_Scalar_SOURCES = test_Scalar.cc Scalar.h - test_tslib_CPPFLAGS = $(AM_CPPFLAGS)\ -I$(abs_top_srcdir)/tests/include @@ -262,12 +264,15 @@ test_tslib_LDADD = libtsutil.la $(top_builddir)/iocore/eventsystem/libinkevent.a test_tslib_SOURCES = \ unit-tests/unit_test_main.cc \ unit-tests/test_BufferWriter.cc \ + unit-tests/test_BufferWriterFormat.cc \ unit-tests/test_ink_inet.cc \ unit-tests/test_IpMap.cc \ unit-tests/test_layout.cc \ unit-tests/test_MT_hashtable.cc \ + unit-tests/test_Scalar.cc \ unit-tests/test_string_view.cc \ - unit-tests/test_TextView.cc + unit-tests/test_TextView.cc \ + unit-tests/test_MemSpan.cc CompileParseRules_SOURCES = CompileParseRules.cc diff --git a/lib/ts/MemSpan.h b/lib/ts/MemSpan.h new file mode 100644 index 00000000000..8f4f873851f --- /dev/null +++ b/lib/ts/MemSpan.h @@ -0,0 +1,572 @@ +/** @file + + Spans of memory. This is similar but independently developed from @c std::span. The goal is + to provide convenient handling for chunks of memory. These chunks can be treated as arrays + of arbitrary types via template methods. + + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once +#include +#include + +/// Apache Traffic Server commons. +namespace ts +{ +/** A span of contiguous piece of memory. + + A @c MemSpan does not own the memory to which it refers, it is simply a span of part of some + (presumably) larger memory object. The purpose is that frequently code needs to work on a specific + part of the memory. This can avoid copying or allocation by allocating all needed memory at once + and then working with it via instances of this class. + */ +class MemSpan +{ + using self_type = MemSpan; ///< Self reference type. + +protected: + void *_data = nullptr; ///< Pointer to base of memory chunk. + ptrdiff_t _size = 0; ///< Size of memory chunk. + +public: + /// Default constructor (empty buffer). + constexpr MemSpan(); + + /** Construct explicitly with a pointer and size. + */ + constexpr MemSpan(void *ptr, ///< Pointer to buffer. + ptrdiff_t n ///< Size of buffer. + ); + + /** Construct from a half open range of two pointers. + @note The instance at @start is in the span but the instance at @a end is not. + */ + template + constexpr MemSpan(T *start, ///< First byte in the span. + T *end ///< First byte not in the span. + ); + + /** Construct from a half open range of two pointers. + @note The instance at @start is in the span but the instance at @a end is not. + */ + MemSpan(void *start, ///< First byte in the span. + void *end ///< First byte not in the span. + ); + + /** Construct to cover an array. + * + * @tparam T Array element type. + * @tparam N Number of elements in the array. + * @param a The array. + */ + template MemSpan(T (&a)[N]); + + /** Construct from nullptr. + This implicitly makes the length 0. + */ + constexpr MemSpan(std::nullptr_t); + + /** Equality. + + Compare the span contents. + + @return @c true if the contents of @a that are the same as the content of @a this, + @c false otherwise. + */ + bool operator==(self_type const &that) const; + + /** Identical. + + Check if the spans refer to the same span of memory. + @return @c true if @a this and @a that refer to the same span, @c false if not. + */ + bool is_same(self_type const &that) const; + + /** Inequality. + @return @c true if @a that does not refer to the same span as @a this, + @c false otherwise. + */ + bool operator!=(self_type const &that) const; + + /// Assignment - the span is copied, not the content. + self_type &operator=(self_type const &that); + + /** Shift the span to discard the first byte. + @return @a this. + */ + self_type &operator++(); + + /** Shift the span to discard the leading @a n bytes. + @return @a this + */ + self_type &operator+=(ptrdiff_t n); + + /// Check for empty span. + /// @return @c true if the span is empty (no contents), @c false otherwise. + bool operator!() const; + + /// Check for non-empty span. + /// @return @c true if the span contains bytes. + explicit operator bool() const; + + /// Check for empty span (no content). + /// @see operator bool + bool empty() const; + + /// @name Accessors. + //@{ + /// Pointer to the first byte in the span. + void *begin() const; + + /// Pointer to first byte not in the span. + void *end() const; + + /// Number of bytes in the span. + constexpr ptrdiff_t size() const; + + /// Memory pointer. + /// @note This is equivalent to @c begin currently but it's probably good to have separation. + constexpr void *data() const; + + /// Memory pointer, one past the last element of the span. + void *data_end() const; + + /// @return the @a V value at index @a n. + template V at(ptrdiff_t n) const; + + /// @return a pointer to the @a V value at index @a n. + template V const *ptr(ptrdiff_t n) const; + //@} + + /// Set the span. + /// This is faster but equivalent to constructing a new span with the same + /// arguments and assigning it. + /// @return @c this. + self_type &assign(void *ptr, ///< Buffer address. + ptrdiff_t n = 0 ///< Buffer size. + ); + + /// Set the span. + /// This is faster but equivalent to constructing a new span with the same + /// arguments and assigning it. + /// @return @c this. + self_type &assign(void *start, ///< First valid character. + void const *end ///< First invalid character. + ); + + /// Clear the span (become an empty span). + self_type &clear(); + + /// @return @c true if the byte at @a *p is in the span. + bool contains(const void *p) const; + + /** Find a value. + The memory is searched as if it were an array of the value type @a V. + + @return A pointer to the first occurrence of @a v in @a this + or @c nullptr if @a v is not found. + */ + template V *find(V v) const; + + /** Find a value. + The memory is searched as if it were an array of type @a V. + + @return A pointer to the first value for which @a pred is @c true otherwise + @c nullptr. + */ + template V *find_if(F const &pred); + + /** Get the initial segment of the span before @a p. + + The byte at @a p is not included. If @a p is not in the span an empty span + is returned. + + @return A buffer that contains all data before @a p. + */ + self_type prefix(const void *p) const; + + /** Get the first @a n bytes of the span. + + @return A span with the first @a n bytes of this span. + */ + self_type prefix(ptrdiff_t n) const; + + /** Shrink the span from the front. + * + * @param p The limit of the removed span. + * @return @c *this + * + * The byte at @a p is not removed. + */ + + self_type remove_prefix(void const *p); + /** Shringt the span from the front. + * + * @param n The number of bytes to remove. + * @return @c *this + */ + self_type &remove_prefix(ptrdiff_t n); + + /** Get the trailing segment of the span after @a p. + + The byte at @a p is not included. If @a p is not in the span an empty span is returned. + + @return A buffer that contains all data after @a p. + */ + self_type suffix(const void *p) const; + + /** Get the trailing @a n bytes. + + @return A span with @a n bytes of the current span. + */ + self_type suffix(ptrdiff_t p) const; + + /** Shrink the span from the back. + * + * @param p The limit of the removed span. + * @return @c *this + * + * The byte at @a p is not removed. + */ + self_type &remove_suffix(void const *p); + + /** Shringt the span from the back. + * + * @param n The number of bytes to remove. + * @return @c *this + */ + self_type &remove_suffix(ptrdiff_t n); + + /// Internal utility for computing the difference of two void pointers. + /// @return the byte (char) difference between the pointers, @a lhs - @a rhs + static ptrdiff_t distance(void const *lhs, void const *rhs); +}; + +// -- Implementation -- + +inline int +memcmp(MemSpan const &lhs, MemSpan const &rhs) +{ + int zret = 0; + ptrdiff_t n = lhs.size(); + + // Seems a bit ugly but size comparisons must be done anyway to get the memcmp args. + if (lhs.size() < rhs.size()) { + zret = 1; + } else if (lhs.size() > rhs.size()) { + zret = -1; + n = rhs.size(); + } + // else the sizes are equal therefore @a n and @a zret are already correct. + + int r = std::memcmp(lhs.data(), rhs.data(), n); + if (0 != r) { // If we got a not-equal, override the size based result. + zret = r; + } + + return zret; +} +// need to bring memcmp in so this is an overload, not an override. +using std::memcmp; + +inline constexpr MemSpan::MemSpan() +{ +} + +inline constexpr MemSpan::MemSpan(void *ptr, ptrdiff_t n) : _data(ptr), _size(n) +{ +} + +template constexpr MemSpan::MemSpan(T *start, T *end) : _data(start), _size((end - start) * sizeof(T)) +{ +} + +// is magic, handle that specially. +// No constexpr because the spec specifically forbids casting from to a typed pointer. +inline MemSpan::MemSpan(void *start, void *end) : _data(start), _size(static_cast(end) - static_cast(start)) +{ +} + +template MemSpan::MemSpan(T (&a)[N]) : _data(a), _size(N * sizeof(T)) +{ +} + +inline constexpr MemSpan::MemSpan(std::nullptr_t) : _data(nullptr), _size(0) +{ +} + +inline ptrdiff_t +MemSpan::distance(void const *lhs, void const *rhs) +{ + return static_cast(lhs) - static_cast(rhs); +} + +inline MemSpan & +MemSpan::assign(void *ptr, ptrdiff_t n) +{ + _data = ptr; + _size = n; + return *this; +} + +inline MemSpan & +MemSpan::assign(void *ptr, void const *limit) +{ + _data = ptr; + _size = static_cast(limit) - static_cast(ptr); + return *this; +} + +inline MemSpan & +MemSpan::clear() +{ + _data = 0; + _size = 0; + return *this; +} + +inline bool +MemSpan::is_same(self_type const &that) const +{ + return _data == that._data && _size == that._size; +} + +inline bool +MemSpan::operator==(self_type const &that) const +{ + return _size == that._size && (_data == that._data || 0 == memcmp(this->data(), that.data(), _size)); +} + +inline bool +MemSpan::operator!=(self_type const &that) const +{ + return !(*this == that); +} + +inline bool MemSpan::operator!() const +{ + return _size == 0; +} + +inline MemSpan::operator bool() const +{ + return _size != 0; +} + +inline bool +MemSpan::empty() const +{ + return _size == 0; +} + +inline MemSpan &MemSpan::operator++() +{ + _data = static_cast(_data) + 1; + --_size; + return *this; +} + +inline MemSpan & +MemSpan::operator+=(ptrdiff_t n) +{ + if (n > _size) { + this->clear(); + } else { + _data = static_cast(_data) + n; + _size -= n; + } + return *this; +} + +inline void * +MemSpan::begin() const +{ + return _data; +} + +inline constexpr void * +MemSpan::data() const +{ + return _data; +} + +inline void * +MemSpan::end() const +{ + return static_cast(_data) + _size; +} + +inline void * +MemSpan::data_end() const +{ + return static_cast(_data) + _size; +} + +inline constexpr ptrdiff_t +MemSpan::size() const +{ + return _size; +} + +inline MemSpan & +MemSpan::operator=(MemSpan const &that) +{ + _data = that._data; + _size = that._size; + return *this; +} + +inline bool +MemSpan::contains(const void *p) const +{ + return !this->empty() && _data <= p && p < this->data_end(); +} + +inline MemSpan +MemSpan::prefix(const void *p) const +{ + self_type zret; + if (_data <= p && p <= this->data_end()) + zret.assign(_data, p); + return zret; +} + +inline MemSpan +MemSpan::prefix(ptrdiff_t n) const +{ + return {_data, std::min(n, _size)}; +} + +inline MemSpan & +MemSpan::remove_prefix(ptrdiff_t n) +{ + if (n < 0) { + } else if (n <= _size) { + _size -= n; + _data = static_cast(_data) + n; + } else { + this->clear(); + } + return *this; +} + +inline MemSpan +MemSpan::suffix(void const *p) const +{ + self_type zret; + if (_data <= p && p <= this->data_end()) { + zret.assign(const_cast(p), this->data_end()); + } + return zret; +} + +inline MemSpan +MemSpan::suffix(ptrdiff_t n) const +{ + self_type zret; + if (n < 0) { + n = std::max(ptrdiff_t{0}, n + _size); + } + if (n <= _size) { + zret.assign(static_cast(_data) + n, _size - n); + } + return zret; +} + +inline MemSpan & +MemSpan::remove_suffix(void const *p) +{ + if (_data <= p && p <= this->data_end()) { + _size -= distance(this->data_end(), p); + } + return *this; +} + +inline MemSpan & +MemSpan::remove_suffix(ptrdiff_t n) +{ + if (n < 0) { + n = std::max(ptrdiff_t{0}, n + _size); + } + if (n <= _size) { + _size -= n; + _data = static_cast(_data) + n; + } + return *this; +} + +template +inline V +MemSpan::at(ptrdiff_t n) const +{ + return static_cast(_data)[n]; +} + +template +inline V const * +MemSpan::ptr(ptrdiff_t n) const +{ + return static_cast(_data) + n; +} + +template +inline V * +MemSpan::find(V v) const +{ + for (V *spot = static_cast(_data), *limit = spot + (_size / sizeof(V)); spot < limit; ++spot) + if (v == *spot) + return spot; + return nullptr; +} + +// Specialize char for performance. +template <> +inline char * +MemSpan::find(char v) const +{ + return static_cast(memchr(_data, v, _size)); +} + +template +inline V * +MemSpan::find_if(F const &pred) +{ + for (V *p = static_cast(_data), *limit = p + (_size / sizeof(V)); p < limit; ++p) + if (pred(*p)) + return p; + return nullptr; +} + +} // namespace ts + +namespace std +{ +ostream & +operator<<(ostream &os, const ts::MemSpan &b) +{ + if (os.good()) { + ostringstream out; + out << b.size() << '@' << hex << b.data(); + os << out.str(); + } + return os; +} +} // std diff --git a/lib/ts/Scalar.h b/lib/ts/Scalar.h index 92b2665223c..2142ed33275 100644 --- a/lib/ts/Scalar.h +++ b/lib/ts/Scalar.h @@ -38,7 +38,7 @@ namespace tag struct generic; } -namespace ApacheTrafficServer +namespace ts { template class Scalar; @@ -493,21 +493,14 @@ Scalar::scale() } // --- Compare operators -// These optimize nicely due to dead code elimination. +// These optimize nicely because if R::num or R::den is 1 the compiler will drop it. template bool operator<(Scalar const &lhs, Scalar const &rhs) { typedef std::ratio R; - if (N == S) - return lhs.count() < rhs.count(); - else if (R::den == 1) - return lhs.count() * R::num < rhs.count(); - else if (R::num == 1) - return lhs.count() < rhs.count() * R::den; - else - return lhs.value() < rhs.value(); + return lhs.count() * R::num < rhs.count() * R::den; } template @@ -515,14 +508,7 @@ bool operator==(Scalar const &lhs, Scalar const &rhs) { typedef std::ratio R; - if (N == S) - return lhs.count() == rhs.count(); - else if (R::den == 1) - return lhs.count() * R::num == rhs.count(); - else if (R::num == 1) - return lhs.count() == rhs.count() * R::den; - else - return lhs.value() == rhs.value(); + return lhs.count() * R::num == rhs.count() * R::den; } template @@ -530,222 +516,24 @@ bool operator<=(Scalar const &lhs, Scalar const &rhs) { typedef std::ratio R; - if (N == S) - return lhs.count() <= rhs.count(); - else if (R::den == 1) - return lhs.count() * R::num <= rhs.count(); - else if (R::num == 1) - return lhs.count() <= rhs.count() * R::den; - else - return lhs.value() <= rhs.value(); + return lhs.count() * R::num <= rhs.count() * R::den; } // Derived compares. - -template +template bool -operator>(Scalar const &lhs, Scalar const &rhs) +operator>(Scalar const &lhs, Scalar const &rhs) { return rhs < lhs; } -template +template bool -operator>=(Scalar const &lhs, Scalar const &rhs) +operator>=(Scalar const &lhs, Scalar const &rhs) { return rhs <= lhs; } -// Do the integer compares. -// A bit ugly to handle the issue that integers without explicit type are . Therefore suppport -// must be provided for comparison not just to the counter type C but also explicitly , otherwise -// function template argument deduction may fail (because it can't figure out what to use for ). - -template -bool -operator<(Scalar const &lhs, C n) -{ - return lhs.value() < n; -} -template -bool -operator<(C n, Scalar const &rhs) -{ - return n < rhs.value(); -} -template -bool -operator<(Scalar const &lhs, int n) -{ - return lhs.value() < static_cast(n); -} -template -bool -operator<(int n, Scalar const &rhs) -{ - return static_cast(n) < rhs.value(); -} -template -bool -operator<(Scalar const &lhs, int n) -{ - return lhs.value() < n; -} -template -bool -operator<(int n, Scalar const &rhs) -{ - return n < rhs.value(); -} - -template -bool -operator==(Scalar const &lhs, C n) -{ - return lhs.value() == n; -} -template -bool -operator==(C n, Scalar const &rhs) -{ - return n == rhs.value(); -} -template -bool -operator==(Scalar const &lhs, int n) -{ - return lhs.value() == static_cast(n); -} -template -bool -operator==(int n, Scalar const &rhs) -{ - return static_cast(n) == rhs.value(); -} -template -bool -operator==(Scalar const &lhs, int n) -{ - return lhs.value() == n; -} -template -bool -operator==(int n, Scalar const &rhs) -{ - return n == rhs.value(); -} - -template -bool -operator>(Scalar const &lhs, C n) -{ - return lhs.value() > n; -} -template -bool -operator>(C n, Scalar const &rhs) -{ - return n > rhs.value(); -} -template -bool -operator>(Scalar const &lhs, int n) -{ - return lhs.value() > static_cast(n); -} -template -bool -operator>(int n, Scalar const &rhs) -{ - return static_cast(n) > rhs.value(); -} -template -bool -operator>(Scalar const &lhs, int n) -{ - return lhs.value() > n; -} -template -bool -operator>(int n, Scalar const &rhs) -{ - return n > rhs.value(); -} - -template -bool -operator<=(Scalar const &lhs, C n) -{ - return lhs.value() <= n; -} -template -bool -operator<=(C n, Scalar const &rhs) -{ - return n <= rhs.value(); -} -template -bool -operator<=(Scalar const &lhs, int n) -{ - return lhs.value() <= static_cast(n); -} -template -bool -operator<=(int n, Scalar const &rhs) -{ - return static_cast(n) <= rhs.value(); -} -template -bool -operator<=(Scalar const &lhs, int n) -{ - return lhs.value() <= n; -} -template -bool -operator<=(int n, Scalar const &rhs) -{ - return n <= rhs.value(); -} - -template -bool -operator>=(Scalar const &lhs, C n) -{ - return lhs.value() >= n; -} -template -bool -operator>=(C n, Scalar const &rhs) -{ - return n >= rhs.value(); -} -template -bool -operator>=(Scalar const &lhs, int n) -{ - return lhs.value() >= static_cast(n); -} -template -bool -operator>=(int n, Scalar const &rhs) -{ - return static_cast(n) >= rhs.value(); -} -template -bool -operator>=(Scalar const &lhs, int n) -{ - return lhs.value() >= n; -} -template -bool -operator>=(int n, Scalar const &rhs) -{ - return n >= rhs.value(); -} - // Arithmetic operators template auto @@ -816,27 +604,27 @@ template Scalar operator+(detail::scalar_unit_round_up_t lhs, Scalar const &rhs) { - return Scalar(rhs) += lhs.template scale(); + return Scalar(rhs) += lhs; } template Scalar operator+(Scalar const &lhs, detail::scalar_unit_round_up_t rhs) { - return Scalar(lhs) += rhs.template scale(); + return Scalar(lhs) += rhs; } template Scalar operator+(detail::scalar_unit_round_down_t lhs, Scalar const &rhs) { - return Scalar(rhs) += lhs.template scale(); + return Scalar(rhs) += lhs; } template Scalar operator+(Scalar const &lhs, detail::scalar_unit_round_down_t rhs) { - return Scalar(lhs) += rhs.template scale(); + return Scalar(lhs) += rhs; } template Scalar @@ -938,7 +726,7 @@ template Scalar operator-(Scalar const &lhs, detail::scalar_unit_round_up_t rhs) { - return Scalar(lhs) -= rhs.template scale(); + return Scalar(lhs) -= rhs; } template @@ -952,7 +740,7 @@ template Scalar operator-(Scalar const &lhs, detail::scalar_unit_round_down_t rhs) { - return Scalar(lhs) -= rhs.template scale(); + return Scalar(lhs) -= rhs; } template @@ -1066,27 +854,22 @@ Scalar::operator/=(C n) -> self & return *this; } -template -Scalar -operator/(Scalar const &lhs, C n) +template +auto +operator/(Scalar lhs, Scalar rhs) -> typename std::common_type::type { - return Scalar(lhs) /= n; + using R = std::ratio; + return (lhs.count() * R::num) / (rhs.count() * R::den); } -template +template Scalar -operator/(Scalar const &lhs, int n) +operator/(Scalar lhs, I n) { + static_assert(std::is_integral::value, "Scalar divsion only support integral types."); return Scalar(lhs) /= n; } -template -Scalar -operator/(Scalar const &lhs, int n) -{ - return Scalar(lhs) /= n; -} - template auto Scalar::operator()(Counter n) const -> self @@ -1141,20 +924,19 @@ namespace std { template ostream & -operator<<(ostream &s, ApacheTrafficServer::Scalar const &x) +operator<<(ostream &s, ts::Scalar const &x) { - static ApacheTrafficServer::detail::tag_label_B b; // Can't be const or the compiler gets upset. + static ts::detail::tag_label_B b; // Can't be const or the compiler gets upset. s << x.value(); - return ApacheTrafficServer::detail::tag_label(s, b); + return ts::detail::tag_label(s, b); } /// Compute common type of two scalars. /// In `std` to overload the base definition. This yields a type that has the common type of the /// counter type and a scale that is the GCF of the input scales. -template -struct common_type, ApacheTrafficServer::Scalar> { +template struct common_type, ts::Scalar> { typedef std::ratio R; - typedef ApacheTrafficServer::Scalar::type, T> type; + typedef ts::Scalar::type, T> type; }; } #endif // TS_SCALAR_H diff --git a/lib/ts/TextView.h b/lib/ts/TextView.h index dce8e51fc5a..5554dd9c014 100644 --- a/lib/ts/TextView.h +++ b/lib/ts/TextView.h @@ -1033,479 +1033,3 @@ namespace std { ostream &operator<<(ostream &os, const ts::TextView &b); } - -#if 0 -// Preserved for now, I may want this back later. -/** A read only view of contiguous piece of memory. - - A @c MemView does not own the memory to which it refers, it is simply a view of part of some - (presumably) larger memory object. The purpose is to allow working in a read only way a specific - part of the memory. This can avoid copying or allocation by allocating all needed memory at once - and then working with it via instances of this class. - - MemView is based on an earlier class ConstBuffer and influenced by Boost.string_ref. Neither - of these were adequate for how use of @c ConstBuffer evolved and so @c MemView is @c - ConstBuffer with some additional stylistic changes based on Boost.string_ref. - - This class is closely integrated with @c StringView. These classes have the same underlying - implementation and are differentiated only because of the return types and a few string oriented - methods. - */ -class MemView -{ - typedef MemView self; ///< Self reference type. - -protected: - const void *_ptr = nullptr; ///< Pointer to base of memory chunk. - size_t _size = 0; ///< Size of memory chunk. - -public: - /// Default constructor (empty buffer). - constexpr MemView(); - - /** Construct explicitly with a pointer and size. - */ - constexpr MemView(const void *ptr, ///< Pointer to buffer. - size_t n ///< Size of buffer. - ); - - /** Construct from a half open range of two pointers. - @note The instance at @start is in the view but the instance at @a end is not. - */ - template - constexpr MemView(T const *start, ///< First byte in the view. - T const *end ///< First byte not in the view. - ); - - /** Construct from a half open range of two pointers. - @note The instance at @start is in the view but the instance at @a end is not. - */ - MemView(void const *start, ///< First byte in the view. - void const *end ///< First byte not in the view. - ); - - /** Construct from nullptr. - This implicitly makes the length 0. - */ - constexpr MemView(std::nullptr_t); - - /// Convert from StringView. - constexpr MemView(StringView const &that); - - /** Equality. - - This is effectively a pointer comparison, buffer contents are not compared. - - @return @c true if @a that refers to the same view as @a this, - @c false otherwise. - */ - bool operator==(self const &that) const; - - /** Inequality. - @return @c true if @a that does not refer to the same view as @a this, - @c false otherwise. - */ - bool operator!=(self const &that) const; - - /// Assignment - the view is copied, not the content. - self &operator=(self const &that); - - /** Shift the view to discard the first byte. - @return @a this. - */ - self &operator++(); - - /** Shift the view to discard the leading @a n bytes. - @return @a this - */ - self &operator+=(size_t n); - - /// Check for empty view. - /// @return @c true if the view has a zero pointer @b or size. - bool operator!() const; - - /// Check for non-empty view. - /// @return @c true if the view refers to a non-empty range of bytes. - explicit operator bool() const; - - /// Check for empty view (no content). - /// @see operator bool - bool isEmpty() const; - - /// @name Accessors. - //@{ - /// Pointer to the first byte in the view. - const void *begin() const; - /// Pointer to first byte not in the view. - const void *end() const; - /// Number of bytes in the view. - constexpr size_t size() const; - /// Memory pointer. - /// @note This is equivalent to @c begin currently but it's probably good to have separation. - constexpr const void *ptr() const; - /// @return the @a V value at index @a n. - template V at(ssize_t n) const; - /// @return a pointer to the @a V value at index @a n. - template V const *at_ptr(ssize_t n) const; - //@} - - /// Set the view. - /// This is faster but equivalent to constructing a new view with the same - /// arguments and assigning it. - /// @return @c this. - self &setView(const void *ptr, ///< Buffer address. - size_t n = 0 ///< Buffer size. - ); - - /// Set the view. - /// This is faster but equivalent to constructing a new view with the same - /// arguments and assigning it. - /// @return @c this. - self &setView(const void *start, ///< First valid character. - const void *end ///< First invalid character. - ); - - /// Clear the view (become an empty view). - self &clear(); - - /// @return @c true if the byte at @a *p is in the view. - bool contains(const void *p) const; - - /** Find a value. - The memory is searched as if it were an array of the value type @a T. - - @return A pointer to the first occurrence of @a v in @a this - or @c nullptr if @a v is not found. - */ - template const V *find(V v) const; - - /** Find a value. - The memory is searched as if it were an array of the value type @a V. - - @return A pointer to the first value for which @a pred is @c true otherwise - @c nullptr. - */ - template const V *find(std::function const &pred); - - /** Get the initial segment of the view before @a p. - - The byte at @a p is not included. If @a p is not in the view an empty view - is returned. - - @return A buffer that contains all data before @a p. - */ - self prefix(const void *p) const; - - /** Split the view at @a p. - - The view is split in to two parts at @a p and the prefix is returned. The view is updated to - contain the bytes not returned in the prefix. The prefix will not contain @a p. - - @note If @a *p refers to a byte that is not in @a this then @a this is not changed and an empty - buffer is returned. Therefore this method can be safely called with the return value of - calling @c find. - - @return A buffer containing data up to but not including @a p. - - @see extractPrefix - */ - self splitPrefix(const void *p); - - /** Extract a prefix delimited by @a p. - - A prefix of @a this is removed from the view and returned. If @a p is not in the view then the - entire view is extracted and returned. - - If @a p points at a byte in the view this is identical to @c splitPrefix. If not then the - entire view in @a this will be returned and @a this will become an empty view. - - @return The prefix bounded at @a p or the entire view if @a p is not a byte in the view. - - @see splitPrefix - */ - self extractPrefix(const void *p); - - /** Get the trailing segment of the view after @a p. - - The byte at @a p is not included. If @a p is not in the view an empty view is returned. - - @return A buffer that contains all data after @a p. - */ - self suffix(const void *p) const; - - /** Split the view at @a p. - - The view is split in to two parts and the suffix is returned. The view is updated to contain - the bytes not returned in the suffix. The suffix will not contain @a p. - - @note If @a p does not refer to a byte in the view, an empty view is returned and @a this is - unchanged. - - @return @a this. - */ - self splitSuffix(const void *p); -}; - -inline constexpr MemView::MemView() -{ -} -inline constexpr MemView::MemView(void const *ptr, size_t n) : _ptr(ptr), _size(n) -{ -} -template constexpr MemView::MemView(const T *start, const T *end) : _ptr(start), _size((end - start) * sizeof(T)) -{ -} -// is magic, handle that specially. -// No constexpr because the spec specifically forbids casting from to a typed pointer. -inline MemView::MemView(void const *start, void const *end) - : _ptr(start), _size(static_cast(end) - static_cast(start)) -{ -} -inline constexpr MemView::MemView(std::nullptr_t) : _ptr(nullptr), _size(0) -{ -} -inline constexpr MemView::MemView(StringView const &that) : _ptr(that.ptr()), _size(that.size()) -{ -} - -inline MemView & -MemView::setView(const void *ptr, size_t n) -{ - _ptr = ptr; - _size = n; - return *this; -} - -inline MemView & -MemView::setView(const void *ptr, const void *limit) -{ - _ptr = ptr; - _size = static_cast(limit) - static_cast(ptr); - return *this; -} - -inline MemView & -MemView::clear() -{ - _ptr = 0; - _size = 0; - return *this; -} - -inline bool -MemView::operator==(self const &that) const -{ - return _size == that._size && _ptr == that._ptr; -} - -inline bool -MemView::operator!=(self const &that) const -{ - return !(*this == that); -} - -inline bool MemView::operator!() const -{ - return !(_ptr && _size); -} - -inline MemView::operator bool() const -{ - return _ptr && _size; -} - -inline bool -MemView::isEmpty() const -{ - return !(_ptr && _size); -} - -inline MemView &MemView::operator++() -{ - _ptr = static_cast(_ptr) + 1; - --_size; - return *this; -} - -inline MemView & -MemView::operator+=(size_t n) -{ - if (n > _size) { - _ptr = nullptr; - _size = 0; - } else { - _ptr = static_cast(_ptr) + n; - _size -= n; - } - return *this; -} - -inline const void * -MemView::begin() const -{ - return _ptr; -} -inline constexpr const void * -MemView::ptr() const -{ - return _ptr; -} - -inline const void * -MemView::end() const -{ - return static_cast(_ptr) + _size; -} - -inline constexpr size_t -MemView::size() const -{ - return _size; -} - -inline MemView & -MemView::operator=(MemView const &that) -{ - _ptr = that._ptr; - _size = that._size; - return *this; -} - -inline bool -MemView::contains(const void *p) const -{ - return _ptr <= this->begin() && p < this->end(); -} - -inline MemView -MemView::prefix(const void *p) const -{ - self zret; - if (this->contains(p)) - zret.setView(_ptr, p); - return zret; -} - -inline MemView -MemView::splitPrefix(const void *p) -{ - self zret; // default to empty return. - if (this->contains(p)) { - zret.setView(_ptr, p); - this->setView(p, this->end()); - } - return zret; -} - -inline MemView -MemView::extractPrefix(const void *p) -{ - self zret{this->splitPrefix(p)}; - - // For extraction if zret is empty, use up all of @a this - if (!zret) { - zret = *this; - this->clear(); - } - - return zret; -} - -inline MemView -MemView::suffix(const void *p) const -{ - self zret; - if (this->contains(p)) - zret.setView(p, this->end()); - return zret; -} - -inline MemView -MemView::splitSuffix(const void *p) -{ - self zret; - if (this->contains(p)) { - zret.setView(p, this->end()); - this->setView(_ptr, p); - } - return zret; -} - -template -inline V -MemView::at(ssize_t n) const -{ - return static_cast(_ptr)[n]; -} - -template -inline V const * -MemView::at_ptr(ssize_t n) const -{ - return static_cast(_ptr) + n; -} - -template -inline const V * -MemView::find(V v) const -{ - for (const V *spot = static_cast(_ptr), limit = spot + (_size / sizeof(V)); spot < limit; ++spot) - if (v == *spot) - return spot; - return nullptr; -} - -// Specialize char for performance. -template <> -inline const char * -MemView::find(char v) const -{ - return static_cast(memchr(_ptr, v, _size)); -} - -template -inline const V * -MemView::find(std::function const &pred) -{ - for (const V *p = static_cast(_ptr), *limit = p + (_size / sizeof(V)); p < limit; ++p) - if (pred(*p)) - return p; - return nullptr; -} - -inline int -memcmp(MemView const &lhs, MemView const &rhs) -{ - int zret; - size_t n; - - // Seems a bit ugly but size comparisons must be done anyway to get the memcmp args. - if (lhs.size() < rhs.size()) { - zret = 1, n = lhs.size(); - } else { - n = rhs.size(); - zret = rhs.size() < lhs.size() ? -1 : 0; - } - - int r = ::memcmp(lhs.ptr(), rhs.ptr(), n); - if (0 != r) { // If we got a not-equal, override the size based result. - zret = r; - } - - return zret; -} - -namespace std -{ -ostream & -operator<<(ostream &os, const ts::MemView &b) -{ - if (os.good()) { - ostringstream out; - out << b.size() << '@' << hex << b.ptr(); - os << out.str(); - } - return os; -} -} -#endif diff --git a/lib/ts/ink_std_compat.h b/lib/ts/ink_std_compat.h index ca972a6ebe3..e62d36e0e7f 100644 --- a/lib/ts/ink_std_compat.h +++ b/lib/ts/ink_std_compat.h @@ -25,6 +25,8 @@ #if __cplusplus < 201402L #include +#include + namespace std { template @@ -33,5 +35,99 @@ make_unique(Args &&... args) { return std::unique_ptr(new T(std::forward(args)...)); } -} + +// Local implementation of integer sequence templates from in C++14. +// Drop once we move to C++14. + +template struct integer_sequence { + typedef T value_type; + static_assert(std::is_integral::value, "std::integer_sequence requires an integral type"); + + static inline std::size_t + size() + { + return (sizeof...(N)); + } +}; + +template using index_sequence = integer_sequence; + +namespace sequence_expander_detail +{ + // Expand a sequence (4 ways) + template struct seq_expand; + + template struct seq_expand, _Extra...> { + typedef integer_sequence type; + }; + + template struct modulus; + template struct construct : modulus::template modular_construct { + }; + + // 4 base cases (e.g. modulo 4) + template <> struct construct<0> { + typedef integer_sequence type; + }; + template <> struct construct<1> { + typedef integer_sequence type; + }; + template <> struct construct<2> { + typedef integer_sequence type; + }; + template <> struct construct<3> { + typedef integer_sequence type; + }; + + // Modulus cases - split 4 ways and pick up the remainder explicitly. + template <> struct modulus<0> { + template struct modular_construct : seq_expand::type> { + }; + }; + template <> struct modulus<1> { + template struct modular_construct : seq_expand::type, N - 1> { + }; + }; + template <> struct modulus<2> { + template struct modular_construct : seq_expand::type, N - 2, N - 1> { + }; + }; + template <> struct modulus<3> { + template struct modular_construct : seq_expand::type, N - 3, N - 2, N - 1> { + }; + }; + + template struct convert { + template struct result; + + template struct result> { + typedef integer_sequence type; + }; + }; + + template struct convert { + template struct result { + typedef U type; + }; + }; + + template + using make_integer_sequence_unchecked = typename convert::template result::type>::type; + + template struct make_integer_sequence { + static_assert(std::is_integral::value, "std::make_integer_sequence can only be instantiated with an integral type"); + static_assert(0 <= N, "std::make_integer_sequence input shall not be negative"); + + typedef make_integer_sequence_unchecked type; + }; + +} // namespace sequence_expander_detail + +template using make_integer_sequence = typename sequence_expander_detail::make_integer_sequence::type; + +template using make_index_sequence = make_integer_sequence; + +template using index_sequence_for = make_index_sequence; + +} // std #endif diff --git a/lib/ts/test_Scalar.cc b/lib/ts/test_Scalar.cc deleted file mode 100644 index 369610325b9..00000000000 --- a/lib/ts/test_Scalar.cc +++ /dev/null @@ -1,407 +0,0 @@ -/** @file - - Intrusive pointer test. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include -#include -#include -#include - -namespace ts -{ -using namespace ApacheTrafficServer; -} - -typedef ts::Scalar<1, off_t> Bytes; -typedef ts::Scalar<16, off_t> Paragraphs; -typedef ts::Scalar<1024, off_t> KB; -typedef ts::Scalar MB; - -#define FAIL_LINE " line ", __LINE__ - -struct TestBox { - using self = TestBox; ///< Self reference type. - - std::string _name; - - static int _count; - static int _fail; - - TestBox(char const *name) : _name(name) {} - TestBox(std::string const &name) : _name(name) {} - template - bool - result(bool r, Rest &&... rest) - { - ++_count; - if (!r) { - std::cout << "FAIL: [" << _name << ":" << _count << "] "; - (void)(int[]){0, ((std::cout << rest), 0)...}; - std::cout << std::endl; - ++_fail; - } - return r; - } - - template - bool - equal(A const &got, B const &expected, Rest const &... rest) - { - return result(expected == got, "Expected ", expected, " got ", got, rest...); - } - - template - bool - lt(A const &lhs, B const &rhs, Rest const &... rest) - { - return result(lhs < rhs, "Expected {", lhs, " < ", rhs, "} ", rest...); - } - - template - bool - le(A const &lhs, B const &rhs, Rest const &... rest) - { - return result(lhs <= rhs, "Expected {", lhs, " <= ", rhs, "} ", rest...); - } - - template - bool - gt(A const &lhs, B const &rhs, Rest const &... rest) - { - return result(lhs > rhs, "Expected {", lhs, " > ", rhs, "} ", rest...); - } - - template - bool - ge(A const &lhs, B const &rhs, Rest const &... rest) - { - return result(lhs >= rhs, "Expected {", lhs, " >= ", rhs, "} ", rest...); - } - - static void - print_summary() - { - printf("Tests: %d of %d passed - %s\n", (_count - _fail), _count, _fail ? "FAIL" : "SUCCESS"); - } -}; - -int TestBox::_count = 0; -int TestBox::_fail = 0; - -// Extremely simple test. -void -Test_1() -{ - constexpr static int SCALE = 4096; - using PageSize = ts::Scalar; - - TestBox test("TS.Scalar basic"); - PageSize pg1(1); - - test.equal(pg1.count(), 1, "Count wrong", FAIL_LINE); - test.equal(pg1.value(), SCALE, "Units wrong", FAIL_LINE); -} - -// Test multiples. -void -Test_2() -{ - constexpr static int SCALE_1 = 8192; - constexpr static int SCALE_2 = 512; - - using Size_1 = ts::Scalar; - using Size_2 = ts::Scalar; - - TestBox test("TS.Scalar Conversion of scales of multiples"); - Size_2 sz_a(2); - Size_2 sz_b(57); - Size_2 sz_c(SCALE_1 / SCALE_2); - Size_2 sz_d(29 * SCALE_1 / SCALE_2); - - Size_1 sz = ts::round_up(sz_a); - test.equal(sz.count(), 1, FAIL_LINE); - sz = ts::round_down(sz_a); - test.equal(sz.count(), 0, FAIL_LINE); - - sz = ts::round_up(sz_b); - test.equal(sz.count(), 4, FAIL_LINE); - sz = ts::round_down(sz_b); - test.equal(sz.count(), 3, FAIL_LINE); - - sz = ts::round_up(sz_c); - test.equal(sz.count(), 1, FAIL_LINE); - sz = ts::round_down(sz_c); - test.equal(sz.count(), 1, FAIL_LINE); - - sz = ts::round_up(sz_d); - test.equal(sz.count(), 29, FAIL_LINE); - sz = ts::round_down(sz_d); - test.equal(sz.count(), 29, FAIL_LINE); - - sz.assign(119); - sz_b = sz; // Should be OK because SCALE_1 is an integer multiple of SCALE_2 - // sz = sz_b; // Should not compile. - test.equal(sz_b.count(), 119 * (SCALE_1 / SCALE_2), FAIL_LINE); -} - -// Test common factor. -void -Test_3() -{ - constexpr static int SCALE_1 = 30; - constexpr static int SCALE_2 = 20; - - using Size_1 = ts::Scalar; - using Size_2 = ts::Scalar; - - TestBox test("TS.Scalar common factor conversions"); - Size_2 sz_a(2); - Size_2 sz_b(97); - - Size_1 sz = round_up(sz_a); - test.equal(sz.count(), 2, FAIL_LINE); - sz = round_down(sz_a); - test.equal(sz.count(), 1, FAIL_LINE); - - sz = ts::round_up(sz_b); - test.equal(sz.count(), 65, FAIL_LINE); - sz = ts::round_down(sz_b); - test.equal(sz.count(), 64, FAIL_LINE); -} - -void -Test_4() -{ - TestBox test("TS.Scalar: relatively prime tests"); - - ts::Scalar<9> m_9; - ts::Scalar<4> m_4, m_test; - - m_9.assign(95); - // m_4 = m_9; // Should fail to compile with static assert. - // m_9 = m_4; // Should fail to compile with static assert. - - m_4 = ts::round_up(m_9); - test.equal(m_4.count(), 214, "Rounding up 9->4", FAIL_LINE); - m_4 = ts::round_down(m_9); - test.equal(m_4.count(), 213, "Rounding down 9->4", FAIL_LINE); - - m_4.assign(213); - m_9 = ts::round_up(m_4); - test.equal(m_9.count(), 95, FAIL_LINE); - m_9 = ts::round_down(m_4); - test.equal(m_9.count(), 94, FAIL_LINE); - - m_test = m_4; // Verify assignment of identical scale values compiles. - test.equal(m_test.count(), 213, FAIL_LINE); -} - -void -Test_5() -{ - TestBox test("TS.Scalar: arithmetics"); - - using KBytes = ts::Scalar<1024>; - typedef ts::Scalar<1024, long int> KiBytes; - typedef ts::Scalar<1, int64_t> Bytes; - using MBytes = ts::Scalar<1024 * KBytes::SCALE>; - - Bytes bytes(96); - KBytes kbytes(2); - MBytes mbytes(5); - - Bytes z1 = ts::round_up(bytes + 128); - test.equal(z1.count(), 224, FAIL_LINE); - KBytes z2 = kbytes + kbytes(3); - test.equal(z2.count(), 5, FAIL_LINE); - Bytes z3(bytes); - z3 += kbytes; - test.equal(z3.value(), 2048 + 96, FAIL_LINE); - MBytes z4 = mbytes; - z4.inc(5); - z2 += z4; - test.equal(z2.value(), (10 << 20) + (5 << 10), FAIL_LINE); - - z1.inc(128); - test.equal(z1.count(), 352, FAIL_LINE); - - z2.assign(2); - z1 = 3 * z2; - test.equal(z1.count(), 6144, FAIL_LINE); - z1 *= 5; - test.equal(z1.count(), 30720, FAIL_LINE); - z1 /= 3; - test.equal(z1.count(), 10240, FAIL_LINE); - - z2.assign(3148); - auto x = z2 + MBytes(1); - test.equal(x.scale(), z2.scale(), FAIL_LINE); - test.equal(x.count(), 4172, FAIL_LINE); - - z2 = ts::round_down(262150); - test.equal(z2.count(), 256, FAIL_LINE); - - z2 = ts::round_up(262150); - test.equal(z2.count(), 257, FAIL_LINE); - - KBytes q(ts::round_down(262150)); - test.equal(q.count(), 256, FAIL_LINE); - - z2 += ts::round_up(97384); - test.equal(z2.count(), 353, FAIL_LINE); - - decltype(z2) a = ts::round_down(z2 + 167229); - test.equal(a.count(), 516, FAIL_LINE); - - KiBytes k(3148); - auto kx = k + MBytes(1); - test.equal(kx.scale(), k.scale(), FAIL_LINE); - test.equal(kx.count(), 4172, FAIL_LINE); - - k = ts::round_down(262150); - test.equal(k.count(), 256, FAIL_LINE); - - k = ts::round_up(262150); - test.equal(k.count(), 257, FAIL_LINE); - - KBytes kq(ts::round_down(262150)); - test.equal(kq.count(), 256, FAIL_LINE); - - k += ts::round_up(97384); - test.equal(k.count(), 353, FAIL_LINE); - - decltype(k) ka = ts::round_down(k + 167229); - test.equal(ka.count(), 516, FAIL_LINE); -} - -// test comparisons -void -Test_6() -{ - using ts::Scalar; - typedef Scalar<8 * KB::SCALE, off_t> StoreBlocks; - typedef Scalar<127 * MB::SCALE, off_t> SpanBlocks; - - TestBox test("TS.Scalar: comparisons"); - - StoreBlocks a(80759700); - SpanBlocks b(4968); - SpanBlocks delta(1); - - test.lt(a, b, FAIL_LINE); - test.lt(b, a + delta, FAIL_LINE); -} - -void -Test_7() -{ - using ts::Scalar; - TestBox test("TS.Scalar: constructor tests"); - - static const off_t N = 7 * 1024; - Bytes b(N + 384); - KB kb(round_down(b)); - - test.equal(kb, N, "Round down wrong ", FAIL_LINE); - test.lt(kb, N + 1, FAIL_LINE); - test.gt(kb, N - 1, FAIL_LINE); - - test.lt(kb, b, FAIL_LINE); - test.le(kb, b, FAIL_LINE); - test.gt(b, kb, FAIL_LINE); - test.ge(b, kb, FAIL_LINE); - - ++kb; - - test.lt(b, kb, FAIL_LINE); - test.le(b, kb, FAIL_LINE); - test.gt(kb, b, FAIL_LINE); - test.ge(kb, b, FAIL_LINE); -} - -struct KBytes_tag { - static std::string const label; -}; -std::string const KBytes_tag::label(" bytes"); - -void -Test_IO() -{ - typedef ts::Scalar<1024, long int, KBytes_tag> KBytes; - typedef ts::Scalar<1024, int> KiBytes; - - KBytes x(12); - KiBytes y(12); - - std::cout << "Testing" << std::endl; - std::cout << "x is " << x << std::endl; - std::cout << "y is " << y << std::endl; -} - -void -test_Compile() -{ - // These tests aren't normally run, they exist to detect compiler issues. - - typedef ts::Scalar<1024, short> KBytes; - typedef ts::Scalar<1024, int> KiBytes; - int delta = 10; - - KBytes x(12); - KiBytes y(12); - - if (x > 12) { - std::cout << "Operator > works" << std::endl; - } - if (y > 12) { - std::cout << "Operator > works" << std::endl; - } - - (void)(x.inc(10)); - (void)(x.inc(static_cast(10))); - (void)(x.inc(static_cast(10))); - (void)(x.inc(delta)); - (void)(y.inc(10)); - (void)(y.inc(static_cast(10))); - (void)(y.inc(static_cast(10))); - (void)(y.inc(delta)); - - (void)(x.dec(10)); - (void)(x.dec(static_cast(10))); - (void)(x.dec(static_cast(10))); - (void)(x.dec(delta)); -} - -int -main(int, char **) -{ - Test_1(); - Test_2(); - Test_3(); - Test_4(); - Test_5(); - Test_6(); - Test_7(); - Test_IO(); - TestBox::print_summary(); - return 0; -} diff --git a/lib/ts/unit-tests/test_BufferWriter.cc b/lib/ts/unit-tests/test_BufferWriter.cc index 3d6b1c7caa7..cfccfd26e7a 100644 --- a/lib/ts/unit-tests/test_BufferWriter.cc +++ b/lib/ts/unit-tests/test_BufferWriter.cc @@ -21,12 +21,9 @@ limitations under the License. */ -#include "BufferWriter.h" - #include "catch.hpp" - -#include "string_view.h" - +#include +#include #include namespace @@ -333,19 +330,6 @@ TEST_CASE("Discard Buffer Writer", "[BWD]") REQUIRE(scratch[0] == '!'); } -TEST_CASE("Buffer Writer << operator", "[BW<<]") -{ - ts::LocalBufferWriter<50> bw; - - bw << "The" << ' ' << "quick" << ' ' << "brown fox"; - - REQUIRE(bw.view() == "The quick brown fox"); - - bw.reduce(0); - bw << "x=" << bw.capacity(); - REQUIRE(bw.view() == "x=50"); -} - TEST_CASE("LocalBufferWriter clip and extend") { ts::LocalBufferWriter<10> bw; diff --git a/lib/ts/unit-tests/test_BufferWriterFormat.cc b/lib/ts/unit-tests/test_BufferWriterFormat.cc new file mode 100644 index 00000000000..76065f7c00c --- /dev/null +++ b/lib/ts/unit-tests/test_BufferWriterFormat.cc @@ -0,0 +1,198 @@ +/** @file + + Unit tests for BufferFormat and bwprint. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "catch.hpp" +#include +#include +#include + +TEST_CASE("Buffer Writer << operator", "[bufferwriter][stream]") +{ + ts::LocalBufferWriter<50> bw; + + bw << "The" << ' ' << "quick" << ' ' << "brown fox"; + + REQUIRE(bw.view() == "The quick brown fox"); + + bw.reduce(0); + bw << "x=" << bw.capacity(); + REQUIRE(bw.view() == "x=50"); +} + +TEST_CASE("bwprint basics", "[bwprint]") +{ + ts::LocalBufferWriter<256> bw; + auto fmt1{"Some text"_sv}; + + bw.print(fmt1); + REQUIRE(bw.view() == fmt1); + bw.reduce(0); + bw.print("Arg {}", 1); + REQUIRE(bw.view() == "Arg 1"); + bw.reduce(0); + bw.print("arg 1 {1} and 2 {2} and 0 {0}", "zero", "one", "two"); + REQUIRE(bw.view() == "arg 1 one and 2 two and 0 zero"); + bw.reduce(0); + bw.print("args {2}{0}{1}", "zero", "one", "two"); + REQUIRE(bw.view() == "args twozeroone"); + bw.reduce(0); + bw.print("left |{:<10}|", "text"); + REQUIRE(bw.view() == "left |text |"); + bw.reduce(0); + bw.print("right |{:>10}|", "text"); + REQUIRE(bw.view() == "right | text|"); + bw.reduce(0); + bw.print("right |{:.>10}|", "text"); + REQUIRE(bw.view() == "right |......text|"); + bw.reduce(0); + bw.print("center |{:.=10}|", "text"); + REQUIRE(bw.view() == "center |...text...|"); + bw.reduce(0); + bw.print("center |{:.=11}|", "text"); + REQUIRE(bw.view() == "center |...text....|"); + bw.reduce(0); + bw.print("center |{:==10}|", "text"); + REQUIRE(bw.view() == "center |===text===|"); + bw.reduce(0); + bw.print("center |{:%3A=10}|", "text"); + REQUIRE(bw.view() == "center |:::text:::|"); + bw.reduce(0); + bw.print("left >{0:<9}< right >{0:>9}< center >{0:=9}<", 956); + REQUIRE(bw.view() == "left >956 < right > 956< center > 956 <"); + + bw.reduce(0); + bw.print("Format |{:>#010x}|", -956); + REQUIRE(bw.view() == "Format |0000-0x3bc|"); + bw.reduce(0); + bw.print("Format |{:<#010x}|", -956); + REQUIRE(bw.view() == "Format |-0x3bc0000|"); + bw.reduce(0); + bw.print("Format |{:#010x}|", -956); + REQUIRE(bw.view() == "Format |-0x00003bc|"); + + bw.reduce(0); + bw.print("{{BAD_ARG_INDEX:{} of {}}}", 17, 23); + REQUIRE(bw.view() == "{BAD_ARG_INDEX:17 of 23}"); + + bw.reduce(0); + bw.print("Arg {0} Arg {3}", 1, 2); + REQUIRE(bw.view() == "Arg 1 Arg {BAD_ARG_INDEX:3 of 2}"); + + bw.reduce(0); + bw.print("{{stuff}} Arg {0} Arg {}", 1, 2); + REQUIRE(bw.view() == "{stuff} Arg 1 Arg 2"); + bw.reduce(0); + bw.print("Arg {0} Arg {} and {{stuff}}", 3, 4); + REQUIRE(bw.view() == "Arg 3 Arg 4 and {stuff}"); + bw.reduce(0); + bw.print("Arg {{{0}}} Arg {} and {{stuff}}", 5, 6); + REQUIRE(bw.view() == "Arg {5} Arg 6 and {stuff}"); + bw.reduce(0); + bw.print("Arg {0} Arg {{}}{{}} {} and {{stuff}}", 7, 8); + REQUIRE(bw.view() == "Arg 7 Arg {}{} 8 and {stuff}"); + bw.reduce(0); + bw.print("Arg {0} Arg {{{{}}}} {}", 9, 10); + REQUIRE(bw.view() == "Arg 9 Arg {{}} 10"); + + bw.reduce(0); + bw.print("Arg {0} Arg {{{{}}}} {}", 9, 10); + REQUIRE(bw.view() == "Arg 9 Arg {{}} 10"); + bw.reduce(0); + bw.print("Time is {now}"); + // REQUIRE(bw.view() == "Time is"); +} + +TEST_CASE("BWFormat", "[bwprint][bwformat]") +{ + ts::LocalBufferWriter<256> bw; + ts::BWFormat fmt("left >{0:<9}< right >{0:>9}< center >{0:=9}<"); + ts::string_view text{"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + + bw.reduce(0); + static const ts::BWFormat bad_arg_fmt{"{{BAD_ARG_INDEX:{} of {}}}"}; + bw.print(bad_arg_fmt, 17, 23); + REQUIRE(bw.view() == "{BAD_ARG_INDEX:17 of 23}"); + + bw.reduce(0); + bw.print(fmt, 956); + REQUIRE(bw.view() == "left >956 < right > 956< center > 956 <"); + + bw.reduce(0); + bw.print("Text: _{0:.10,20}_", text); + REQUIRE(bw.view() == "Text: _abcdefghijklmnopqrst_"); + bw.reduce(0); + bw.print("Text: _{0:-<20.52,20}_", text); + REQUIRE(bw.view() == "Text: _QRSTUVWXYZ----------_"); +} + +#if 0 +TEST_CASE("bwperf", "[bwprint][performance]") +{ + // Force these so I can easily change the set of tests. + auto start = std::chrono::high_resolution_clock::now(); + auto delta = std::chrono::high_resolution_clock::now() - start; + constexpr int N_LOOPS = 1000000; + + ts::string_view text{"Format |"}; + ts::LocalBufferWriter<256> bw; + + ts::BWFSpec spec; + + start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < N_LOOPS; ++i) { + bw.reduce(0); + bw.print( "Format |{:#010x}|", -956); + } + delta = std::chrono::high_resolution_clock::now() - start; + std::cout << "BW Timing is " << delta.count() << "ns or " << std::chrono::duration_cast(delta).count() + << "ms" << std::endl; + + start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < N_LOOPS; ++i) { + bw.reduce(0); + bw.print("Format |{:#010x}|", -956); + } + delta = std::chrono::high_resolution_clock::now() - start; + std::cout << "bw.print() " << delta.count() << "ns or " << std::chrono::duration_cast(delta).count() + << "ms" << std::endl; + + ts::BWFormat fmt("Format |{:#010x}|"); + start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < N_LOOPS; ++i) { + bw.reduce(0); + bw.print( fmt, -956); + } + delta = std::chrono::high_resolution_clock::now() - start; + std::cout << "Preformatted: " << delta.count() << "ns or " + << std::chrono::duration_cast(delta).count() << "ms" << std::endl; + + char buff[256]; + start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < N_LOOPS; ++i) { + snprintf(buff, sizeof(buff), "Format |%#0x10|", -956); + } + delta = std::chrono::high_resolution_clock::now() - start; + std::cout << "snprint Timing is " << delta.count() << "ns or " + << std::chrono::duration_cast(delta).count() << "ms" << std::endl; +} +#endif diff --git a/lib/ts/unit-tests/test_MemSpan.cc b/lib/ts/unit-tests/test_MemSpan.cc new file mode 100644 index 00000000000..d5949c1be05 --- /dev/null +++ b/lib/ts/unit-tests/test_MemSpan.cc @@ -0,0 +1,56 @@ +/** @file + + TextView unit tests. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include + +using ts::MemSpan; + +TEST_CASE("MemSpan", "[libts][MemSpan]") +{ + int idx[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + char buff[1024]; + MemSpan span(buff, sizeof(buff)); + MemSpan left = span.prefix(512); + REQUIRE(left.size() == 512); + REQUIRE(span.size() == 1024); + span.remove_prefix(512); + REQUIRE(span.size() == 512); + REQUIRE(left.data_end() == span.data()); + + MemSpan idx_span(idx); + REQUIRE(idx_span.size() == sizeof(idx)); + REQUIRE(idx_span.data() == idx); + REQUIRE(idx_span.find(4) == idx + 4); + REQUIRE(idx_span.find(8) == idx + 8); + MemSpan a = idx_span.suffix(idx_span.find(7)); + REQUIRE(a.at(0) == 7); + MemSpan b = idx_span.suffix(-(4 * sizeof(int))); + REQUIRE(b.size() == 4 * sizeof(int)); + REQUIRE(b.at(0) == 7); + REQUIRE(a == b); + MemSpan c = idx_span.prefix(3 * sizeof(int)); + REQUIRE(c.size() == 3 * sizeof(int)); + REQUIRE(c.ptr(2) == idx + 2); +} diff --git a/lib/ts/unit-tests/test_Scalar.cc b/lib/ts/unit-tests/test_Scalar.cc new file mode 100644 index 00000000000..d0629f6041e --- /dev/null +++ b/lib/ts/unit-tests/test_Scalar.cc @@ -0,0 +1,285 @@ +/** @file + + Scalar unit testing. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +//#include +//#include +//#include + +using Bytes = ts::Scalar<1, off_t>; +using Paragraphs = ts::Scalar<16, off_t>; +using KB = ts::Scalar<1024, off_t>; +using MB = ts::Scalar; + +TEST_CASE("Scalar", "[libts][Scalar]") +{ + constexpr static int SCALE = 4096; + constexpr static int SCALE_1 = 8192; + constexpr static int SCALE_2 = 512; + + using PageSize = ts::Scalar; + + PageSize pg1(1); + REQUIRE(pg1.count() == 1); + REQUIRE(pg1.value() == SCALE); + + using Size_1 = ts::Scalar; + using Size_2 = ts::Scalar; + + Size_2 sz_a(2); + Size_2 sz_b(57); + Size_2 sz_c(SCALE_1 / SCALE_2); + Size_2 sz_d(29 * SCALE_1 / SCALE_2); + + Size_1 sz = ts::round_up(sz_a); + REQUIRE(sz.count() == 1); + sz = ts::round_down(sz_a); + REQUIRE(sz.count() == 0); + + sz = ts::round_up(sz_b); + REQUIRE(sz.count() == 4); + sz = ts::round_down(sz_b); + REQUIRE(sz.count() == 3); + + sz = ts::round_up(sz_c); + REQUIRE(sz.count() == 1); + sz = ts::round_down(sz_c); + REQUIRE(sz.count() == 1); + + sz = ts::round_up(sz_d); + REQUIRE(sz.count() == 29); + sz = ts::round_down(sz_d); + REQUIRE(sz.count() == 29); + + sz.assign(119); + sz_b = sz; // Should be OK because SCALE_1 is an integer multiple of SCALE_2 + // sz = sz_b; // Should not compile. + REQUIRE(sz_b.count() == 119 * (SCALE_1 / SCALE_2)); +} + +TEST_CASE("Scalar Factors", "[libts][Scalar][factors]") +{ + constexpr static int SCALE_1 = 30; + constexpr static int SCALE_2 = 20; + + using Size_1 = ts::Scalar; + using Size_2 = ts::Scalar; + + Size_2 sz_a(2); + Size_2 sz_b(97); + + Size_1 sz = round_up(sz_a); + REQUIRE(sz.count() == 2); + sz = round_down(sz_a); + REQUIRE(sz.count() == 1); + + sz = ts::round_up(sz_b); + REQUIRE(sz.count() == 65); + sz = ts::round_down(sz_b); + REQUIRE(sz.count() == 64); + + ts::Scalar<9> m_9; + ts::Scalar<4> m_4, m_test; + + m_9.assign(95); + // m_4 = m_9; // Should fail to compile with static assert. + // m_9 = m_4; // Should fail to compile with static assert. + + m_4 = ts::round_up(m_9); + REQUIRE(m_4.count() == 214); + m_4 = ts::round_down(m_9); + REQUIRE(m_4.count() == 213); + + m_4.assign(213); + m_9 = ts::round_up(m_4); + REQUIRE(m_9.count() == 95); + m_9 = ts::round_down(m_4); + REQUIRE(m_9.count() == 94); + + m_test = m_4; // Verify assignment of identical scale values compiles. + REQUIRE(m_test.count() == 213); +} + +TEST_CASE("Scalar Arithmetic", "[libts][Scalar][arithmetic]") +{ + using KBytes = ts::Scalar<1024>; + using KiBytes = ts::Scalar<1024, long int>; + using Bytes = ts::Scalar<1, int64_t>; + using MBytes = ts::Scalar<1024 * KBytes::SCALE>; + + Bytes bytes(96); + KBytes kbytes(2); + MBytes mbytes(5); + + Bytes z1 = ts::round_up(bytes + 128); + REQUIRE(z1.count() == 224); + KBytes z2 = kbytes + kbytes(3); + REQUIRE(z2.count() == 5); + Bytes z3(bytes); + z3 += kbytes; + REQUIRE(z3.value() == 2048 + 96); + MBytes z4 = mbytes; + z4.inc(5); + z2 += z4; + REQUIRE(z2.value() == (10 << 20) + (5 << 10)); + + z1.inc(128); + REQUIRE(z1.count() == 352); + + z2.assign(2); + z1 = 3 * z2; + REQUIRE(z1.count() == 6144); + z1 *= 5; + REQUIRE(z1.count() == 30720); + z1 /= 3; + REQUIRE(z1.count() == 10240); + + z2.assign(3148); + auto x = z2 + MBytes(1); + REQUIRE(x.scale() == z2.scale()); + REQUIRE(x.count() == 4172); + + z2 = ts::round_down(262150); + REQUIRE(z2.count() == 256); + + z2 = ts::round_up(262150); + REQUIRE(z2.count() == 257); + + KBytes q(ts::round_down(262150)); + REQUIRE(q.count() == 256); + + z2 += ts::round_up(97384); + REQUIRE(z2.count() == 353); + + decltype(z2) a = ts::round_down(z2 + 167229); + REQUIRE(a.count() == 516); + + KiBytes k(3148); + auto kx = k + MBytes(1); + REQUIRE(kx.scale() == k.scale()); + REQUIRE(kx.count() == 4172); + + k = ts::round_down(262150); + REQUIRE(k.count() == 256); + + k = ts::round_up(262150); + REQUIRE(k.count() == 257); + + KBytes kq(ts::round_down(262150)); + REQUIRE(kq.count() == 256); + + k += ts::round_up(97384); + REQUIRE(k.count() == 353); + + decltype(k) ka = ts::round_down(k + 167229); + REQUIRE(ka.count() == 516); + + using StoreBlocks = ts::Scalar<8 * KB::SCALE, off_t>; + using SpanBlocks = ts::Scalar<127 * MB::SCALE, off_t>; + + StoreBlocks store_b(80759700); + SpanBlocks span_b(4968); + SpanBlocks delta(1); + + REQUIRE(store_b < span_b); + REQUIRE(span_b < store_b + delta); + store_b += delta; + REQUIRE(span_b < store_b); + + static const off_t N = 7 * 1024; + Bytes b(N + 384); + KB kb(round_down(b)); + + REQUIRE(kb == N); + REQUIRE(kb < N + 1); + REQUIRE(kb > N - 1); + + REQUIRE(kb < b); + REQUIRE(kb <= b); + REQUIRE(b > kb); + REQUIRE(b >= kb); + + ++kb; + + REQUIRE(b < kb); + REQUIRE(b <= kb); + REQUIRE(kb > b); + REQUIRE(kb >= b); +} + +#if 0 +struct KBytes_tag { + static std::string const label; +}; +std::string const KBytes_tag::label(" bytes"); + +void +Test_IO() +{ + typedef ts::Scalar<1024, long int, KBytes_tag> KBytes; + typedef ts::Scalar<1024, int> KiBytes; + + KBytes x(12); + KiBytes y(12); + + std::cout << "Testing" << std::endl; + std::cout << "x is " << x << std::endl; + std::cout << "y is " << y << std::endl; +} + +void +test_Compile() +{ + // These tests aren't normally run, they exist to detect compiler issues. + + typedef ts::Scalar<1024, short> KBytes; + typedef ts::Scalar<1024, int> KiBytes; + int delta = 10; + + KBytes x(12); + KiBytes y(12); + + if (x > 12) { + std::cout << "Operator > works" << std::endl; + } + if (y > 12) { + std::cout << "Operator > works" << std::endl; + } + + (void)(x.inc(10)); + (void)(x.inc(static_cast(10))); + (void)(x.inc(static_cast(10))); + (void)(x.inc(delta)); + (void)(y.inc(10)); + (void)(y.inc(static_cast(10))); + (void)(y.inc(static_cast(10))); + (void)(y.inc(delta)); + + (void)(x.dec(10)); + (void)(x.dec(static_cast(10))); + (void)(x.dec(static_cast(10))); + (void)(x.dec(delta)); +} + +#endif diff --git a/lib/tsconfig/Errata.cc b/lib/tsconfig/Errata.cc index 488c2dca555..0d1c5e9faf9 100644 --- a/lib/tsconfig/Errata.cc +++ b/lib/tsconfig/Errata.cc @@ -53,6 +53,11 @@ Errata::Data::push(Message const& msg) { m_items.push_back(msg); } +void +Errata::Data::push(Message && msg) { + m_items.push_back(std::move(msg)); +} + Errata::Message const& Errata::Data::top() const { return m_items.size() ? m_items.back() : NIL_MESSAGE ; @@ -65,21 +70,16 @@ inline Errata::Errata(ImpPtr const& ptr) Errata::Data::~Data() { if (m_log_on_delete) { Errata tmp(this); // because client API requires a wrapper. - std::deque::iterator spot, limit; - for ( spot = Sink_List.begin(), limit = Sink_List.end(); - spot != limit; - ++spot - ) { - (**spot)(tmp); - } + for ( auto& f : Sink_List ) (*f)(tmp); tmp.m_data.release(); // don't delete this again. } } -Errata::Errata() { +Errata::Errata(self const& that) + : m_data(that.m_data) { } -Errata::Errata(self const& that) +Errata::Errata(self && that) : m_data(that.m_data) { } @@ -111,7 +111,7 @@ Errata::pre_write() { } // Just create an instance if needed. -Errata::Data* +Errata::Data const* Errata::instance() { if (!m_data) { m_data = new Data; } @@ -124,6 +124,12 @@ Errata::push(Message const& msg) { return *this; } +Errata& +Errata::push(Message && msg) { + this->pre_write()->push(std::move(msg)); + return *this; +} + Errata& Errata::operator=(self const& that) { m_data = that.m_data; @@ -143,6 +149,12 @@ Errata::operator = (Message const& msg) { return *this; } +Errata& +Errata::operator = (self && that) { + m_data = that.m_data; + return *this; +} + Errata& Errata::pull(self& that) { if (that.m_data) { @@ -217,21 +229,18 @@ Errata::write( int shift, char const* lead ) const { - for ( const_iterator spot = this->begin(), limit = this->end(); - spot != limit; - ++spot - ) { + + for ( auto m : *this ) { if ((offset + indent) > 0) { out << std::setw(indent + offset) << std::setfill(' ') << ((indent > 0 && lead) ? lead : " "); -} + } - out << spot->m_id << " [" << spot->m_code << "]: " << spot->m_text + out << m.m_id << " [" << m.m_code << "]: " << m.m_text << std::endl ; - if (spot->getErrata().size()) { - spot->getErrata().write(out, offset, indent+shift, shift, lead); -} + if (m.getErrata().size()) + m.getErrata().write(out, offset, indent+shift, shift, lead); } return out; @@ -259,4 +268,3 @@ std::ostream& operator<< (std::ostream& os, Errata const& err) { } } // namespace ts - diff --git a/lib/tsconfig/Errata.h b/lib/tsconfig/Errata.h index 8cb50da40ff..278697ac103 100644 --- a/lib/tsconfig/Errata.h +++ b/lib/tsconfig/Errata.h @@ -68,6 +68,7 @@ # include # include # include +# include # include # include "NumericType.h" # include "IntrusivePtr.h" @@ -141,6 +142,11 @@ class Errata { Message const& msg ///< Message to push ); + /// Move constructor. + Errata(self && that); + /// Move constructor from @c Message. + Errata(Message && msg); + /// destructor ~Errata(); @@ -150,6 +156,9 @@ class Errata { const self& that ///< Source instance. ); + /// Move assignment. + self& operator = (self && that); + /** Assign message. All other messages are discarded. @return A reference to this object. @@ -181,6 +190,15 @@ class Errata { @return A reference to this object. */ self& push(Message const& msg); + self& push(Message && msg); + + /** Push a constructed @c Message. + The @c Message is set to have the @a id and @a code. The other arguments are converted + to strings and concatenated to form the messsage text. + @return A reference to this object. + */ + template < typename ... Args > + self& push(Id id, Code code, Args const& ... args); /** Push a nested status. @a err becomes the top item. @@ -342,7 +360,7 @@ class Errata { Data* pre_write(); /// Force and return an implementation instance. /// Does not follow copy on write. - Data* instance(); + Data const* instance(); /// Used for returns when no data is present. static Message const NIL_MESSAGE; @@ -382,6 +400,16 @@ struct Errata::Message { std::string const& text ///< Final text for message. ); + /// Construct with an @a id, @a code, and a @a message. + /// The message contents are created by converting the variable arguments + /// to strings using the stream operator and concatenated in order. + template < typename ... Args> + Message( + Id id, ///< Message Id. + Code code, ///< Message Code. + Args const& ... text + ); + /// Reset to the message to default state. self& clear(); @@ -452,6 +480,8 @@ struct Errata::Message { static SuccessTest const DEFAULT_SUCCESS_TEST; + template < typename ... Args> static std::string stringify(Args const& ... items); + Id m_id; ///< Message ID. Code m_code; ///< Message code. std::string m_text; ///< Final text. @@ -484,9 +514,10 @@ struct Errata::Data : public IntrusivePtrCounter { /// Put a message on top of the stack. void push(Message const& msg); + void push(Message && msg); /// Log this when it is deleted. - bool m_log_on_delete; + mutable bool m_log_on_delete; //! The message stack. Container m_items; @@ -730,6 +761,12 @@ inline Errata::Message::Message(Id id, std::string const& text) inline Errata::Message::Message(Id id, Code code, std::string const& text) : m_id(id), m_code(code), m_text(text) { } +template < typename ... Args> +Errata::Message::Message(Id id, Code code, Args const& ... text) + : m_id(id), m_code(code), m_text(stringify(text ...)) +{ +} + inline Errata::Message& Errata::Message::clear() { m_id = 0; m_code = Default_Code; @@ -764,12 +801,24 @@ inline Errata::Message& Errata::Message::set(Errata const& err) { return *this; } +template < typename ... Args> +std::string Errata::Message::stringify(Args const& ... items) +{ + std::ostringstream s; + (void)(int[]){0, ( (s << items) , 0 ) ... }; + return s.str(); +} + +inline Errata::Errata() {} inline Errata::Errata(Id id, Code code, std::string const& text) { this->push(Message(id, code, text)); } inline Errata::Errata(Message const& msg) { this->push(msg); } +inline Errata::Errata(Message && msg) { + this->push(std::move(msg)); +} inline Errata::operator bool() const { return this->isOK(); } @@ -802,6 +851,13 @@ Errata::push(Id id, Code code, std::string const& text) { return *this; } +template < typename ... Args > +auto Errata::push(Id id, Code code, Args const& ... args) -> self& +{ + this->push(Message(id, code, args ...)); + return *this; +} + inline Errata::Message const& Errata::top() const { return m_data ? m_data->top() : NIL_MESSAGE; diff --git a/plugins/Makefile.am b/plugins/Makefile.am index f7046fdf2e2..3e12147826a 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -42,6 +42,7 @@ include s3_auth/Makefile.inc include stats_over_http/Makefile.inc include tcpinfo/Makefile.inc include xdebug/Makefile.inc +include test_cppapi/Makefile.inc if BUILD_EXPERIMENTAL_PLUGINS diff --git a/plugins/experimental/cachekey/Makefile.inc b/plugins/experimental/cachekey/Makefile.inc index aca2adb4c7e..6bc2e33b5eb 100644 --- a/plugins/experimental/cachekey/Makefile.inc +++ b/plugins/experimental/cachekey/Makefile.inc @@ -21,7 +21,3 @@ experimental_cachekey_cachekey_la_SOURCES = \ experimental/cachekey/configs.cc \ experimental/cachekey/pattern.cc \ experimental/cachekey/plugin.cc - -VIRTUALENV_DIR = ../../../ci/tsqa/virtualenv -tsqa: $(VIRTUALENV_DIR) - @. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/nosetests --with-xunit -sv --logging-level=INFO diff --git a/plugins/experimental/cachekey/tests/test_cachekey.py b/plugins/experimental/cachekey/tests/test_cachekey.py deleted file mode 100644 index e061b2e9759..00000000000 --- a/plugins/experimental/cachekey/tests/test_cachekey.py +++ /dev/null @@ -1,721 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests -import logging - -#import helpers -import tsqa.test_cases -import tsqa.utils -import tsqa.endpoint -import os - - -log = logging.getLogger(__name__) - -# Since at this moment tha plan is to treat all query, headers and cookie related -# plugin parameters a similar way - include | exclude | remove-all | sort decided to create -# a 'meta' bench and then use it to create / adjust the corresponding query, headers and cookie -# related test benches and use to to validate the plugin behavoior. TBD how well that works. -meta_bench = [ - # Testing empty parametes and defaults. - {"args": "", - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('include', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('exclude', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('exclude', []), ('include', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('remove-all', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - - # Testing the removal of query parameters from the cache key. - {"args": [('remove-all', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('remove-all', ['false'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('remove-all', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": []}, - - # Testing the sorting of the query parameters in the cache key. - {"args": [('sort', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('sort', ['false'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - {"args": [('sort', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('a', '1'), ('b', '2'), ('c', '1'), ('k', '1'), ('u', '1'), ('x', '1'), ('y', '1')]}, - {"args": [('sort', []), ('remove-all', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - - # Testing the exclusion of query parameters from the cache key. - {"args": [('exclude', ['x', 'y', 'z'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('k', '1'), ('u', '1')]}, - {"args": [('exclude', ['x', 'y', 'z']), ('include', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('k', '1'), ('u', '1')]}, - {"args": [('exclude', ['x', 'y', 'z']), ('include', []), ('sort', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('a', '1'), ('b', '2'), ('c', '1'), ('k', '1'), ('u', '1')]}, - - # Testing the inclusion of query parameters in the cache key. - {"args": [('include', ['x', 'y', 'b', 'c'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2'), ('x', '1'), ('y', '1')]}, - {"args": [('include', ['x', 'y', 'b', 'c', 'g'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2'), ('x', '1'), ('y', '1')]}, - {"args": [('include', ['x', 'y', 'b', 'c']), ('exclude', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2'), ('x', '1'), ('y', '1')]}, - {"args": [('include', ['x', 'y', 'b', 'c']), ('sort', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('b', '2'), ('c', '1'), ('x', '1'), ('y', '1')]}, - - # Testing various useful cases (combinations) to include/exclude/sort query parameters in the cache key. - {"args": [('exclude', ['x', 'y', 'z']), ('include', ['x', 'y', 'b', 'c'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2')]}, - {"args": [('exclude', ['x', 'y', 'z']), ('include', [])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('k', '1'), ('u', '1')]}, - {"args": [('exclude', ['x', 'y', 'z']), ('include', []), ('sort', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('a', '1'), ('b', '2'), ('c', '1'), ('k', '1'), ('u', '1')]}, - {"args": [('exclude', ['x', 'y', 'z']), ('include', ['x', 'y', 'b', 'c']), ('sort', ['true']), ('remove-all', ['true'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": []}, - {"args": [('exclude', ['x']), ('exclude', ['y']), ('exclude', ['z']), ('include', ['y', 'c']), ('include', ['x', 'b'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2')]}, - - # Testing regex include-match. - {"args": [('include-match', ['(a|b|c)']), ], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ]}, - # Testing multiple regex include-match with pattern that don't match ('k' and 'u'). - {"args": [('include-match', ['(a|b|c)']), ('include-match', ['(x|y|z)'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('y', '1')]}, - # Testing regex exclude match. - {"args": [('exclude-match', ['(a|b|c)']), ], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')]}, - # Testing multiple regex exclude-match with pattern that don't match ('k' and 'u'). - {"args": [('exclude-match', ['(a|b|c)']), ('exclude-match', ['(x|y|z)'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('k', '1'), ('u', '1')]}, - # Testing mixing exclude and include match - {"args": [('include-match', ['(a|b|c|x)']), ('exclude-match', ['(x|y|z)'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('a', '1'), ('b', '2')]}, - # Testing mixing exclude and include match - {"args": [('exclude-match', ['x']), ('exclude-match', ['y']), ('exclude-match', ['z']), ('include-match', ['(y|c)']), ('include-match', ['(x|b)'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2')]}, - # Testing mixing `--include-params`, `--exclude-params`, `--include-match-param` and `--exclude-match-param` - {"args": [('exclude', ['x']), ('exclude-match', ['y']), ('exclude-match', ['z']), ('include', ['y', 'c']), ('include-match', ['(x|b)'])], - "uri": [('c', '1'), ('a', '1'), ('b', '2'), ('x', '1'), ('k', '1'), ('u', '1'), ('y', '1')], - "key": [('c', '1'), ('b', '2')]}, -] - -# Query related bench - meta_bench is used to populate it. -query_bench = [] - -# Headers related bench - meta_bench is used to populate it. -headers_bench = [] - -# Cookies related bench - meta_bench is used to populate it. -cookies_bench = [] - -# Prefix related tests. Doesn't use the meta_bench. -prefix_bench = [ - # Testing not adding any custom prefix - {"args": "", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing using the option but with no value - {"args": "@pparam=--static-prefix=", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a static prefix to the cache key - {"args": "@pparam=--static-prefix=static_prefix", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/static_prefix/path/to/object?a=1&b=2&c=3" - }, - # Testing using the option but with no value - {"args": "@pparam=--capture-prefix=", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a capture prefix to the cache key - {"args": "@pparam=--capture-prefix=(test_prefix).*:([^\s\/$]*)", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/test_prefix/{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a capture prefix with replacement string defined - {"args": "@pparam=--capture-prefix=/(test_prefix).*:([^\s\/]*)/$1_$2/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/test_prefix_{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a capture prefix from URI to the cache key - {"args": "@pparam=--capture-prefix-uri=(test_prefix).*:.*(object).*$", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/test_prefix/object/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a capture prefix from with replacement string defined - {"args": "@pparam=--capture-prefix-uri=/(test_prefix).*:.*(object).*$/$1_$2/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/test_prefix_object/path/to/object?a=1&b=2&c=3" - }, - # Testing adding both static and capture prefix to the cache key - {"args": "@pparam=--static-prefix=static_prefix @pparam=--capture-prefix=(test_prefix).*", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/static_prefix/test_prefix/path/to/object?a=1&b=2&c=3" - }, - # Testing adding static and capture prefix and capture prefix from URI to the cache key - {"args": "@pparam=--static-prefix=static_prefix @pparam=--capture-prefix=(test_prefix).* @pparam=--capture-prefix-uri=(object).*", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/static_prefix/test_prefix/object/path/to/object?a=1&b=2&c=3" - }, -] -path_bench = [ - # Testing adding default path to the cache key - {"args": "", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/path/to/object?a=1&b=2&c=3" - }, - # Testing adding a path capture to the cache key - {"args": "@pparam=--capture-path=.*(object).*", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/object?a=1&b=2&c=3" - }, - # Testing adding a path capture/replacement to the cache key - {"args": "@pparam=--capture-path=/.*(object).*/const_path_$1/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/const_path_object?a=1&b=2&c=3" - }, - # Testing adding an URI capture to the cache key - {"args": "@pparam=--capture-path-uri=(test_path).*(object).*", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/test_path/object?a=1&b=2&c=3" - }, - # Testing adding an URI capture/replacement to the cache key - {"args": "@pparam=--capture-path-uri=/(test_path).*(object).*/$1_$2/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/test_path_object?a=1&b=2&c=3" - }, - # Testing adding an URI and path capture/replacement together to the cache key - {"args": "@pparam=--capture-path=/.*(object).*/const_path_$1/ @pparam=--capture-path-uri=/(test_path).*(object).*/$1_$2/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [], - "cookies": [], - "key": "/{0}/{1}/test_path_object/const_path_object?a=1&b=2&c=3" - }, -] - - -# User-Agent header capture related tests. Doesn't use the meta_bench. -ua_captures_bench = [ - # Testing single match without grouping. - {"args": "@pparam=--ua-capture=Mozilla\/[^\s]*", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0/path/to/object?a=1&b=2&c=3" - }, - # Testing single match with grouping. - {"args": "@pparam=--ua-capture=(Mozilla\/[^\s]*)", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0/path/to/object?a=1&b=2&c=3" - }, - # Testing multiple capturing group match. - {"args": "@pparam=--ua-capture=(Mozilla\/[^\s]*).*(AppleWebKit\/[^\s]*)", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0/AppleWebKit/537.75.14/path/to/object?a=1&b=2&c=3" - }, - # Testing multiple capturing group match with empty replacement string. - {"args": "@pparam=--ua-capture=/(Mozilla\/[^\s]*).*(AppleWebKit\/[^\s]*)//", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0/AppleWebKit/537.75.14/path/to/object?a=1&b=2&c=3" - }, - # Testing multiple capturing group match with the replacement. - {"args": "@pparam=--ua-capture=/(Mozilla\/[^\s]*).*(AppleWebKit\/[^\s]*)/$1_$2/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0_AppleWebKit/537.75.14/path/to/object?a=1&b=2&c=3" - }, - # Testing multiple capturing group match with $0 (zero group) in the replacement. - {"args": "@pparam=--ua-capture=/(Mozilla\/[^\s]*).*(AppleWebKit\/[^\s]*)/$0/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_9_3)%20AppleWebKit/537.75.14/path/to/object?a=1&b=2&c=3" - }, - # Testing an extra invalid variable in the replacement, the whole capture will be ignored (TODO verify the error message in the log). - {"args": "@pparam=--ua-capture=/(Mozilla\/[^\s]*).*(AppleWebKit\/[^\s]*)/$1_$2_$3/", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/path/to/object?a=1&b=2&c=3" - }, -] - -ua_classifier_bench = [ - # Testing ua-blacklist. - {"args": "@pparam=--ua-blacklist=class1:class1_blacklist.config", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Bozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/class1/path/to/object?a=1&b=2&c=3", - "files": [("class1_blacklist.config", "^Mozilla.*\n^AdSheet.*\n^iTube.*\n^TuneIn.*\n^iHeartRadio.*\n^Ruby.*\n^python.*\n^Twitter.*\n^Facebo.*\n")], - }, - # Testing ua-whitelist. - {"args": "@pparam=--ua-whitelist=class1:class1_blacklist.config", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/class1/path/to/object?a=1&b=2&c=3", - "files": [("class1_blacklist.config", "^Mozilla.*\n^AdSheet.*\n^iTube.*\n^TuneIn.*\n^iHeartRadio.*\n^Ruby.*\n^python.*\n^Twitter.*\n^Facebo.*\n")], - }, - # Testing ua-whitelist and ua-blacklist together, whitelist specified before blacklist. - {"args": "@pparam=--ua-whitelist=class1:class1_whitelist.config @pparam=--ua-blacklist=class2:class2_blacklist.config", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/class1/path/to/object?a=1&b=2&c=3", - "files": [("class1_whitelist.config", "^Mozilla.*\n^AdSheet.*\n^iTube.*\n^TuneIn.*\n"), - ("class2_blacklist.config", "^iHeartRadio.*\n^Ruby.*\n^python.*\n^Twitter.*\n^Facebo.*\n")], - }, - # Testing ua-whitelist and ua-blacklist together, blacklist specified before whitelist. - {"args": "@pparam=--ua-blacklist=class2:class2_blacklist.config @pparam=--ua-whitelist=class1:class1_whitelist.config", - "uri": "{0}:{1}/path/to/object?a=1&b=2&c=3", - "headers": [("User-Agent", "Bozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")], - "cookies": [], - "key": "/{0}/{1}/class2/path/to/object?a=1&b=2&c=3", - "files": [("class1_whitelist.config", "^Mozilla.*\n^AdSheet.*\n^iTube.*\n^TuneIn.*\n"), - ("class2_blacklist.config", "^iHeartRadio.*\n^Ruby.*\n^python.*\n^Twitter.*\n^Facebo.*\n")], - }, -] - - -def prepare_query_bench(bench): - new_bench = [] - for test in bench: - args = '' - - for arg in test['args']: - args += '@pparam=--{0}-params='.format(arg[0]) - args += ','.join(map(str, arg[1])) - args += ' ' - - uri = '{0}:{1}/?' - kvp_list = [] - for (k, v) in test['uri']: - kvp_list.append('{0}={1}'.format(k, v)) - uri += '&'.join(map(str, kvp_list)) - - key = '/{0}/{1}' - if len(test['key']) != 0: - key += '?' - kvp_list = [] - for (k, v) in test['key']: - kvp_list.append('{0}={1}'.format(k, v)) - key += '&'.join(map(str, kvp_list)) - - headers = [] - - new_test = {"args": args.strip(), "uri": uri.strip(), "headers": headers, "cookies": [], "key": key.strip()} - new_bench.append(new_test) - - return new_bench - - -def prepare_headers_bench(bench): - new_bench = [] - for test in bench: - args = '' - ignore_test = False - - include = [] - exclude = [] - - for arg in test['args']: - # 'exclude', 'exclude-match', 'sort', 'remove-all' don't make sense for headers as far cachekey is concerned. - # headers always sorted and never included by default. - if arg[0] == 'exclude' or arg[0] == 'sort' or arg[0] == 'remove-all' or arg[0] == 'include-match' or arg[0] == 'exclude-match': - ignore_test = True - break - - if arg[0] == 'include' and len(arg[1]) != 0: - include.append(arg[1]) - - args += '@pparam=--{0}-headers='.format(arg[0]) - args += ','.join(map(str, arg[1])) - args += ' ' - - if ignore_test: - continue - - uri = '{0}:{1}/' - - headers = test['uri'] - - key = '/{0}/{1}' - - # if there nothing to include and nothing to exclude don't add headers to the cache key. - if len(include) != 0 or len(exclude) != 0: - if len(test['key']) != 0: - key += '/' - kvp_list = [] - for (k, v) in test['key']: - kvp_list.append('{0}:{1}'.format(k, v)) - kvp_list.sort() - key += '/'.join(map(str, kvp_list)) - - new_test = {"args": args.strip(), "uri": uri.strip(), "headers": headers, "cookies": [], "key": key.strip()} - new_bench.append(new_test) - - return new_bench - - -def prepare_cookies_bench(bench): - new_bench = [] - for test in bench: - args = '' - ignore_test = False - - include = [] - exclude = [] - - for arg in test['args']: - # 'exclude', 'exclude-match', 'sort', 'remove-all' don't make sense for cookies as far cachekey is concerned. - # headers always sorted and never included by default. - if arg[0] == 'exclude' or arg[0] == 'sort' or arg[0] == 'remove-all' or arg[0] == 'include-match' or arg[0] == 'exclude-match': - ignore_test = True - break - - if arg[0] == 'include' and len(arg[1]) != 0: - include.append(arg[1]) - - args += '@pparam=--{0}-cookies='.format(arg[0]) - args += ','.join(map(str, arg[1])) - args += ' ' - - if ignore_test: - continue - - uri = '{0}:{1}/' - - cookies = test['uri'] - - key = '/{0}/{1}' - # if there nothing to include and nothing to exclude don't add headers to the cache key. - if len(include) != 0 or len(exclude) != 0: - if len(test['key']) != 0: - key += '/' - kvp_list = [] - for (k, v) in test['key']: - kvp_list.append('{0}={1}'.format(k, v)) - kvp_list.sort() - key += ';'.join(map(str, kvp_list)) - - new_test = {"args": args.strip(), "uri": uri.strip(), "headers": [], "cookies": cookies, "key": key.strip()} - new_bench.append(new_test) - - return new_bench - - -class StaticEnvironmentCase(tsqa.test_cases.EnvironmentCase): - ''' - Use static environment, to be able to experiment and speedup builds through ramdisk - Use this until it is merged into master (pull-request) then fall-back to helpers.EnvironmentCase class - ''' - @classmethod - def getEnv(cls): - layout = tsqa.environment.Layout('/opt/apache/trafficserver.TS-4183/') - env = tsqa.environment.Environment() - env.clone(layout=layout) - return env - - -class TestCacheKey(tsqa.test_cases.DynamicHTTPEndpointCase, StaticEnvironmentCase): - - @classmethod - def setUpEnv(cls, env): - global query_bench - global headers_bench - global cookies_bench - global meta_bench - - cls.configs['plugin.config'].add_line('xdebug.so') - - cls.configs['records.config']['CONFIG'].update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'cachekey.*', - 'proxy.config.url_remap.pristine_host_hdr': 1, - }) - - log.info("Initializing remap rules") - - def add_remap_rule(remap_prefix, remap_index, test): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = cls.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - args = test['args'] - remap_rule = 'map http://{0}:{1} http://127.0.0.1:{2} @plugin=cachekey.so {3}'.format( - host, port, cls.http_endpoint.address[1], args) - log.info(' {0}'.format(remap_rule)) - cls.configs['remap.config'].add_line(remap_rule) - - log.info("Preparing cache key query hadnling test bench") - query_bench = prepare_query_bench(meta_bench) - - log.info("Preparing cache key headers handling test bench") - headers_bench = prepare_headers_bench(meta_bench) - - log.info("Preparing cache key cookies handling test bench") - cookies_bench = prepare_cookies_bench(meta_bench) - - # Prepare query tests related remap rules. - i = 0 - for test in query_bench: - add_remap_rule("query", i, test) - i += 1 - - # Prepare headers tests related remap rules. - i = 0 - for test in headers_bench: - add_remap_rule("headers", i, test) - i += 1 - - # Prepare headers tests related remap rules. - i = 0 - for test in cookies_bench: - add_remap_rule("cookies", i, test) - i += 1 - - # Prepare prefix tests related remap rules. - i = 0 - for test in prefix_bench: - add_remap_rule("prefix", i, test) - i += 1 - - # Prepare path tests related remap rules. - i = 0 - for test in path_bench: - add_remap_rule("path", i, test) - i += 1 - - # Prepare ua-capture tests related remap rules. - i = 0 - for test in ua_captures_bench: - add_remap_rule("ua_captures", i, test) - i += 1 - - # Prepare ua-classifier tests related remap rules. - i = 0 - for test in ua_classifier_bench: - add_remap_rule("ua_classifier", i, test) - - # Create blacklist and white list files for User-Agent classification. - for file in test['files']: - filename = file[0] - content = file[1] - path = os.path.join(env.layout.prefix, 'etc/trafficserver', filename) - with open(path, 'w') as fh: - fh.write(content) - - i += 1 - - # Set up an origin server which returns OK all the time. - def handler(request): - return ('OK', 200, {"Cache-Control": "max-age=5 must-revalidate"}) - - cls.http_endpoint.add_handler('/', handler) - cls.http_endpoint.add_handler('/path/to/object', handler) - - def get_cachekey(self, host, port, uri, headers, cookies): - ''' - Sends a request to the traffic server and gets the cache key used while processing the request. - ''' - uri_req = uri.format('http://127.0.0.1', port) - s = requests.Session() - s.headers.update({'Host': '{0}:{1}'.format(host, port)}) - s.headers.update({'X-Debug': 'X-Cache-Key'}) - for header_name, header_value in headers: - s.headers.update({header_name: header_value}) - for cookie_name, cookie_value in cookies: - s.cookies.set(cookie_name, cookie_value) - response = s.get(uri_req) - self.assertEqual(response.status_code, 200) - return response.headers['X-Cache-Key'] - - def verify_key(self, remap_prefix, remap_index, test): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = self.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - expected_key = test['key'].format(host, port) - key = self.get_cachekey(host, port, test['uri'], test['headers'], test['cookies']) - log.info(" Test {0} / {1}".format(remap_prefix, remap_index)) - log.info(" map : cachekey.so {0}".format(test['args'])) - log.info(" uri :'{0}'".format(test['uri'])) - headers = '' - for name, value in test['headers']: - headers += "'{0}: {1}' ".format(name, value) - cookies = '' - for name, value in test['cookies']: - cookies += "'{0}: {1}' ".format(name, value) - log.info(" headers: {0}".format(headers)) - log.info(" cookies: {0}".format(cookies)) - log.info(" expected:'{0}'".format(expected_key)) - log.info(" received:'{0}'".format(key)) - - self.assertEqual(key, expected_key) - - def test_cachekey_query(self): - ''' - Testing cache key query parameters handling. - ''' - global query_bench - - log.info("Testing cache key query parameters handling.") - i = 0 - for test in query_bench: - self.verify_key('query', i, test) - i += 1 - - def test_cachekey_preffix(self): - ''' - Tests --static-prefix, --capture-prefix, --capture-prefix-uri plugin option in the cache key. - ''' - global prifix_bench - - log.info("Testing --static-prefix, --capture-prefix, --capture-prefix-uri plugin option in the cache key.") - i = 0 - for test in prefix_bench: - self.verify_key('prefix', i, test) - i += 1 - - def test_cachekey_path(self): - ''' - Tests --path-capture, --path-capture-uri plugin option for replacing path in the cache key. - ''' - global path_bench - - log.info("Testing --path-capture, --path-capture-uri plugin option for replacing path in the cache key.") - i = 0 - for test in path_bench: - self.verify_key('path', i, test) - i += 1 - - def test_cachekey_headers(self): - ''' - Testing cache key headers handling. - ''' - global headers_bench - - log.info("Testing cache key headers handling.") - i = 0 - for test in headers_bench: - self.verify_key('headers', i, test) - i += 1 - - def test_cachekey_cookies(self): - ''' - Testing cache key cookies handling. - ''' - global cookies_bench - - log.info("Testing cache key cookies handling.") - i = 0 - for test in cookies_bench: - self.verify_key('cookies', i, test) - i += 1 - - def test_cachekey_ua_capture(self): - ''' - Testing cache key User-Agent header capture handling. - ''' - global cookies_bench - - log.info("Testing cache key User-Agent header capture handling.") - i = 0 - for test in ua_captures_bench: - self.verify_key('ua_captures', i, test) - i += 1 - - def test_cachekey_ua_classifier(self): - ''' - Testing cache key User-Agent header classifier. - ''' - global cookies_bench - - log.info("Testing cache key User-Agent header capture handling.") - i = 0 - for test in ua_classifier_bench: - self.verify_key('ua_classifier', i, test) - i += 1 diff --git a/plugins/experimental/ts_lua/ts_lua_client_request.c b/plugins/experimental/ts_lua/ts_lua_client_request.c index d286bb5e16b..f1176912b67 100644 --- a/plugins/experimental/ts_lua/ts_lua_client_request.c +++ b/plugins/experimental/ts_lua/ts_lua_client_request.c @@ -527,10 +527,8 @@ ts_lua_client_request_set_url_scheme(lua_State *L) static int ts_lua_client_request_get_uri(lua_State *L) { - char uri[TS_LUA_MAX_URL_LENGTH]; const char *path; int path_len; - int uri_len; ts_lua_http_ctx *http_ctx; @@ -538,13 +536,9 @@ ts_lua_client_request_get_uri(lua_State *L) path = TSUrlPathGet(http_ctx->client_request_bufp, http_ctx->client_request_url, &path_len); - uri_len = snprintf(uri, TS_LUA_MAX_URL_LENGTH, "/%.*s", path_len, path); - - if (uri_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, uri, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, uri, uri_len); - } + lua_pushlstring(L, "/", 1); + lua_pushlstring(L, path, path_len >= TS_LUA_MAX_URL_LENGTH - 1 ? TS_LUA_MAX_URL_LENGTH - 2 : path_len); + lua_concat(L, 2); return 1; } diff --git a/plugins/experimental/ts_lua/ts_lua_http.c b/plugins/experimental/ts_lua/ts_lua_http.c index e59284c818e..9e9374cd810 100644 --- a/plugins/experimental/ts_lua/ts_lua_http.c +++ b/plugins/experimental/ts_lua/ts_lua_http.c @@ -99,6 +99,9 @@ static int ts_lua_http_transaction_count(lua_State *L); static int ts_lua_http_redirect_url_set(lua_State *L); static int ts_lua_http_get_server_state(lua_State *L); +static int ts_lua_http_get_remap_from_url(lua_State *L); +static int ts_lua_http_get_remap_to_url(lua_State *L); + static void ts_lua_inject_server_state_variables(lua_State *L); static void ts_lua_inject_http_resp_transform_api(lua_State *L); @@ -229,6 +232,12 @@ ts_lua_inject_http_misc_api(lua_State *L) lua_pushcfunction(L, ts_lua_http_get_server_state); lua_setfield(L, -2, "get_server_state"); + lua_pushcfunction(L, ts_lua_http_get_remap_from_url); + lua_setfield(L, -2, "get_remap_from_url"); + + lua_pushcfunction(L, ts_lua_http_get_remap_to_url); + lua_setfield(L, -2, "get_remap_to_url"); + ts_lua_inject_server_state_variables(L); } @@ -341,8 +350,6 @@ ts_lua_http_set_cache_lookup_status(lua_State *L) static int ts_lua_http_get_cache_lookup_url(lua_State *L) { - char output[TS_LUA_MAX_URL_LENGTH]; - int output_len; TSMLoc url = TS_NULL_MLOC; char *str = NULL; int len; @@ -363,12 +370,7 @@ ts_lua_http_get_cache_lookup_url(lua_State *L) str = TSUrlStringGet(http_ctx->client_request_bufp, url, &len); - output_len = snprintf(output, TS_LUA_MAX_URL_LENGTH, "%.*s", len, str); - if (output_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, output, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, output, output_len); - } + lua_pushlstring(L, str, len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : len); done: if (url != TS_NULL_MLOC) { @@ -460,8 +462,6 @@ ts_lua_http_set_parent_proxy(lua_State *L) static int ts_lua_http_get_parent_selection_url(lua_State *L) { - char output[TS_LUA_MAX_URL_LENGTH]; - int output_len; TSMLoc url = TS_NULL_MLOC; char *str = NULL; int len; @@ -482,12 +482,7 @@ ts_lua_http_get_parent_selection_url(lua_State *L) str = TSUrlStringGet(http_ctx->client_request_bufp, url, &len); - output_len = snprintf(output, TS_LUA_MAX_URL_LENGTH, "%.*s", len, str); - if (output_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, output, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, output, output_len); - } + lua_pushlstring(L, str, len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : len); done: if (url != TS_NULL_MLOC) { @@ -745,6 +740,60 @@ ts_lua_http_get_server_state(lua_State *L) return 1; } +static int +ts_lua_http_get_remap_from_url(lua_State *L) +{ + TSMLoc url = TS_NULL_MLOC; + char *str = NULL; + int len; + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + + if (TSRemapFromUrlGet(http_ctx->txnp, &url) != TS_SUCCESS) { + lua_pushnil(L); + goto done; + } + + str = TSUrlStringGet(NULL, url, &len); + + lua_pushlstring(L, str, len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : len); + +done: + if (str != NULL) { + TSfree(str); + } + + return 1; +} + +static int +ts_lua_http_get_remap_to_url(lua_State *L) +{ + TSMLoc url = TS_NULL_MLOC; + char *str = NULL; + int len; + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + + if (TSRemapToUrlGet(http_ctx->txnp, &url) != TS_SUCCESS) { + lua_pushnil(L); + goto done; + } + + str = TSUrlStringGet(NULL, url, &len); + + lua_pushlstring(L, str, len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : len); + +done: + if (str != NULL) { + TSfree(str); + } + + return 1; +} + static int ts_lua_http_resp_transform_get_upstream_bytes(lua_State *L) { diff --git a/plugins/experimental/ts_lua/ts_lua_remap.c b/plugins/experimental/ts_lua/ts_lua_remap.c index 61fd88c9ed0..b53b03a79d2 100644 --- a/plugins/experimental/ts_lua/ts_lua_remap.c +++ b/plugins/experimental/ts_lua/ts_lua_remap.c @@ -170,10 +170,8 @@ ts_lua_remap_get_from_url_scheme(lua_State *L) static int ts_lua_remap_get_from_uri(lua_State *L) { - char uri[TS_LUA_MAX_URL_LENGTH]; const char *path; int path_len; - int uri_len; ts_lua_http_ctx *http_ctx; @@ -182,13 +180,9 @@ ts_lua_remap_get_from_uri(lua_State *L) if (http_ctx->rri != NULL) { path = TSUrlPathGet(http_ctx->client_request_bufp, http_ctx->rri->mapFromUrl, &path_len); - uri_len = snprintf(uri, TS_LUA_MAX_URL_LENGTH, "/%.*s", path_len, path); - - if (uri_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, uri, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, uri, uri_len); - } + lua_pushlstring(L, "/", 1); + lua_pushlstring(L, path, path_len >= TS_LUA_MAX_URL_LENGTH - 1 ? TS_LUA_MAX_URL_LENGTH - 2 : path_len); + lua_concat(L, 2); } else { lua_pushnil(L); } @@ -199,10 +193,8 @@ ts_lua_remap_get_from_uri(lua_State *L) static int ts_lua_remap_get_from_url(lua_State *L) { - char output[TS_LUA_MAX_URL_LENGTH]; char *url; int url_len; - int output_len; ts_lua_http_ctx *http_ctx; @@ -211,13 +203,7 @@ ts_lua_remap_get_from_url(lua_State *L) if (http_ctx->rri != NULL) { url = TSUrlStringGet(http_ctx->client_request_bufp, http_ctx->rri->mapFromUrl, &url_len); - output_len = snprintf(output, TS_LUA_MAX_URL_LENGTH, "%.*s", url_len, url); - - if (output_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, output, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, output, output_len); - } + lua_pushlstring(L, url, url_len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : url_len); TSfree(url); } else { @@ -300,10 +286,8 @@ ts_lua_remap_get_to_url_scheme(lua_State *L) static int ts_lua_remap_get_to_uri(lua_State *L) { - char uri[TS_LUA_MAX_URL_LENGTH]; const char *path; int path_len; - int uri_len; ts_lua_http_ctx *http_ctx; @@ -312,13 +296,9 @@ ts_lua_remap_get_to_uri(lua_State *L) if (http_ctx->rri != NULL) { path = TSUrlPathGet(http_ctx->client_request_bufp, http_ctx->rri->mapToUrl, &path_len); - uri_len = snprintf(uri, TS_LUA_MAX_URL_LENGTH, "/%.*s", path_len, path); - - if (uri_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, uri, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, uri, uri_len); - } + lua_pushlstring(L, "/", 1); + lua_pushlstring(L, path, path_len >= TS_LUA_MAX_URL_LENGTH - 1 ? TS_LUA_MAX_URL_LENGTH - 2 : path_len); + lua_concat(L, 2); } else { lua_pushnil(L); } @@ -329,10 +309,8 @@ ts_lua_remap_get_to_uri(lua_State *L) static int ts_lua_remap_get_to_url(lua_State *L) { - char output[TS_LUA_MAX_URL_LENGTH]; char *url; int url_len; - int output_len; ts_lua_http_ctx *http_ctx; @@ -341,13 +319,7 @@ ts_lua_remap_get_to_url(lua_State *L) if (http_ctx->rri != NULL) { url = TSUrlStringGet(http_ctx->client_request_bufp, http_ctx->rri->mapToUrl, &url_len); - output_len = snprintf(output, TS_LUA_MAX_URL_LENGTH, "%.*s", url_len, url); - - if (output_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, output, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, output, output_len); - } + lua_pushlstring(L, url, url_len >= TS_LUA_MAX_URL_LENGTH ? TS_LUA_MAX_URL_LENGTH - 1 : url_len); TSfree(url); } else { diff --git a/plugins/experimental/ts_lua/ts_lua_server_request.c b/plugins/experimental/ts_lua/ts_lua_server_request.c index 7ba699556ca..67802d44c05 100644 --- a/plugins/experimental/ts_lua/ts_lua_server_request.c +++ b/plugins/experimental/ts_lua/ts_lua_server_request.c @@ -393,10 +393,8 @@ ts_lua_inject_server_request_uri_api(lua_State *L) static int ts_lua_server_request_get_uri(lua_State *L) { - char uri[TS_LUA_MAX_URL_LENGTH]; const char *path; int path_len; - int uri_len; ts_lua_http_ctx *http_ctx; @@ -406,13 +404,9 @@ ts_lua_server_request_get_uri(lua_State *L) path = TSUrlPathGet(http_ctx->server_request_bufp, http_ctx->server_request_url, &path_len); - uri_len = snprintf(uri, TS_LUA_MAX_URL_LENGTH, "/%.*s", path_len, path); - - if (uri_len >= TS_LUA_MAX_URL_LENGTH) { - lua_pushlstring(L, uri, TS_LUA_MAX_URL_LENGTH - 1); - } else { - lua_pushlstring(L, uri, uri_len); - } + lua_pushlstring(L, "/", 1); + lua_pushlstring(L, path, path_len >= TS_LUA_MAX_URL_LENGTH - 1 ? TS_LUA_MAX_URL_LENGTH - 2 : path_len); + lua_concat(L, 2); return 1; } diff --git a/plugins/experimental/webp_transform/ImageTransform.cc b/plugins/experimental/webp_transform/ImageTransform.cc index fe37f8cb175..967b4db09fd 100644 --- a/plugins/experimental/webp_transform/ImageTransform.cc +++ b/plugins/experimental/webp_transform/ImageTransform.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -29,10 +30,7 @@ using std::string; using namespace Magick; using namespace atscppapi; -namespace -{ #define TAG "webp_transform" -} class ImageTransform : public TransformationPlugin { @@ -53,9 +51,9 @@ class ImageTransform : public TransformationPlugin } void - consume(const string &data) override + consume(ts::string_view data) override { - _img.write(data.data(), data.size()); + _img.write(data.data(), data.length()); } void @@ -69,8 +67,7 @@ class ImageTransform : public TransformationPlugin Blob output_blob; image.magick("WEBP"); image.write(&output_blob); - string output_data(reinterpret_cast(output_blob.data()), output_blob.length()); - produce(output_data); + produce(ts::string_view(reinterpret_cast(output_blob.data()), output_blob.length())); setOutputComplete(); } diff --git a/plugins/gzip/tests/test_gzip.py b/plugins/gzip/tests/test_gzip.py deleted file mode 100644 index 8b9b4f6669c..00000000000 --- a/plugins/gzip/tests/test_gzip.py +++ /dev/null @@ -1,250 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests -import logging -import random -import string - -import tsqa.test_cases -import tsqa.utils -import tsqa.endpoint -import os - -origin_content_length = 0 -log = logging.getLogger(__name__) - -# Test positive cases of remap gzip plugin -gzip_remap_bench = [ - # Test gzip - {"args": "@pparam=gzip1.config", - "files": [("gzip1.config", "enabled true\nremove-accept-encoding true\ncache false\ncompressible-content-type text/*\n") - ], - }, - {"args": "@pparam=gzip2.config", - "files": [("gzip2.config", "enabled true\nremove-accept-encoding false\ncache false\ncompressible-content-type text/*\n") - ], - }, - {"args": "@pparam=gzip3.config", - "files": [("gzip3.config", "enabled true\nremove-accept-encoding true\ncache true\ncompressible-content-type text/*\n") - ], - }, - {"args": "@pparam=gzip4.config", - "files": [("gzip4.config", "enabled true\nremove-accept-encoding true\ncache true\ncompressible-content-type text/*\nflush true\n") - ], - }, - {"args": "@pparam=gzip5.config", - "files": [("gzip5.config", "enabled true\nremove-accept-encoding true\ncache true\ncompressible-content-type text/*\nflush false\n") - ], - }, -] - -# Test negative cases of remap gzip plugin -gzip_remap_negative_bench = [ - # Test when gzip is disabled - {"args": "@pparam=gzip_negative1.config", - "files": [("gzip_negative1.config", "enabled false\nremove-accept-encoding true\ncache false\ncompressible-content-type text/*\n") - ], - }, - # Test when compressible content doesn't match - {"args": "@pparam=gzip_negative2.config", - "files": [("gzip_negative2.config", "enabled true\nremove-accept-encoding true\ncache false\ncompressible-content-type !text/*\n") - ], - }, - # Test when disallow is configured to match some pattern - {"args": "@pparam=gzip_negative3.config", - "files": [("gzip_negative3.config", "enabled true\nremove-accept-encoding true\ncache false\ncompressible-content-type text/*\ndisallow *test*\n") - ], - }, -] - -# Test global gzip plugin -gzip_global_bench = [ - {"args": "gzip_global1.config", - "files": [("gzip_global1.config", "enabled true\nremove-accept-encoding true\ncache true\ncompressible-content-type text/*\n") - ], - }, -] - -# Set up an origin server which returns random string. - - -def handler(request): - global origin_content_length - rand_string = ''.join(random.choice(string.lowercase) for i in range(500)) - origin_content_length = len(rand_string) - return rand_string - - -def create_config_files(env, test): - # Create gzip config files. - for file in test['files']: - filename = file[0] - content = file[1] - path = os.path.join(env.layout.prefix, 'etc/trafficserver', filename) - with open(path, 'w') as fh: - fh.write(content) - - -class StaticEnvironmentCase(tsqa.test_cases.EnvironmentCase): - @classmethod - def getEnv(cls): - #layout = tsqa.environment.Layout('/opt/gitlab-gzip') - layout = tsqa.environment.Layout('/opt/apache/trafficserver.TS-4147') - env = tsqa.environment.Environment() - env.clone(layout=layout) - return env - -# Test gzip remap plugin - - -class TestGzipRemapPlugin(tsqa.test_cases.DynamicHTTPEndpointCase, StaticEnvironmentCase): - @classmethod - def setUpEnv(cls, env): - cls.configs['plugin.config'].add_line('xdebug.so') - cls.configs['records.config']['CONFIG'].update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': '.*', - 'proxy.config.diags.debug.tags': 'gzip.*', - 'proxy.config.url_remap.pristine_host_hdr': 1, }) - - cls.http_endpoint.add_handler('/path/to/object', handler) - - def add_remap_rule(remap_prefix, remap_index, test): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = cls.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - args = test['args'] - remap_rule = 'map http://{0}:{1} http://127.0.0.1:{2} @plugin=gzip.so {3}'.format( - host, port, cls.http_endpoint.address[1], args) - log.info(' {0}'.format(remap_rule)) - cls.configs['remap.config'].add_line(remap_rule) - - # Prepare gzip tests related remap rules. - i = 0 - for test in gzip_remap_bench: - add_remap_rule("gzip", i, test) - create_config_files(env, test) - i += 1 - - # Prepare negative gzip tests related remap rules. - i = 0 - for test in gzip_remap_negative_bench: - add_remap_rule("gzip_negative", i, test) - create_config_files(env, test) - i += 1 - - def send_request(self, remap_prefix, remap_index): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = self.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - url = 'http://127.0.0.1:{0}/path/to/object'.format(port) - log.info('host is {0}, port is {1}, url is {2}'.format(host, port, url)) - s = requests.Session() - s.headers.update({'Host': '{0}:{1}'.format(host, port)}) - s.headers.update({'Accept-Encoding:': 'gzip'}) - response = s.get(url) - log.info('Response headers obtained: {0}'.format(response.headers)) - return response - - def send_gzip_request(self, remap_prefix, remap_index): - ''' - Sends a gzip request to the traffic server - ''' - response = self.send_request(remap_prefix, remap_index) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.headers['Content-Encoding'], 'gzip') - self.assertLess(int(response.headers['Content-Length']), int(origin_content_length)) - - def send_gzip_request_negative(self, remap_prefix, remap_index): - ''' - Sends a gzip request to the traffic server - ''' - response = self.send_request(remap_prefix, remap_index) - self.assertEqual(response.status_code, 200) - self.assertEqual(int(response.headers['Content-Length']), int(origin_content_length)) - - def test_gzip_remap_plugin(self): - i = 0 - for test in gzip_remap_bench: - self.send_gzip_request('gzip', i) - i += 1 - - i = 0 - for test in gzip_remap_negative_bench: - self.send_gzip_request_negative('gzip_negative', i) - i += 1 - -# Test gzip global plugin - - -class TestGzipGlobalPlugin(tsqa.test_cases.DynamicHTTPEndpointCase, StaticEnvironmentCase): - @classmethod - def setUpEnv(cls, env): - cls.configs['plugin.config'].add_line('xdebug.so') - - cls.configs['records.config']['CONFIG'].update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'gzip.*', - 'proxy.config.url_remap.pristine_host_hdr': 1, }) - - cls.http_endpoint.add_handler('/path/to/object', handler) - - def add_remap_rule(remap_prefix, remap_index): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = cls.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - remap_rule = 'map http://{0}:{1} http://127.0.0.1:{2}'.format(host, port, cls.http_endpoint.address[1]) - log.info(' {0}'.format(remap_rule)) - cls.configs['remap.config'].add_line(remap_rule) - - def add_global_plugin_rule(test): - args = test['args'] - plugin_rule = 'gzip.so {0}'.format(args) - log.info(' {0}'.format(plugin_rule)) - cls.configs['plugin.config'].add_line(plugin_rule) - - # Prepare gzip plugin rules - i = 0 - for test in gzip_global_bench: - add_remap_rule("gzip_global", i) - add_global_plugin_rule(test) - create_config_files(env, test) - i += 1 - - def send_request(self, remap_prefix, remap_index): - host = 'test_{0}_{1}.example.com'.format(remap_prefix, remap_index) - port = self.configs['records.config']['CONFIG']['proxy.config.http.server_ports'] - url = 'http://127.0.0.1:{0}/path/to/object'.format(port) - log.info('host is {0}, port is {1}, url is {2}'.format(host, port, url)) - s = requests.Session() - s.headers.update({'Host': '{0}:{1}'.format(host, port)}) - s.headers.update({'Accept-Encoding:': 'gzip'}) - response = s.get(url) - log.info('Response headers obtained: {0}'.format(response.headers)) - return response - - def send_global_gzip_request(self, remap_prefix, remap_index): - ''' - Sends a gzip request to the traffic server - ''' - response = self.send_request(remap_prefix, remap_index) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.headers['Content-Encoding'], 'gzip') - self.assertLess(int(response.headers['Content-Length']), int(origin_content_length)) - - def test_gzip_global_plugin(self): - i = 0 - for test in gzip_global_bench: - self.send_global_gzip_request("gzip_global", i) - i += 1 diff --git a/plugins/test_cppapi/Makefile.inc b/plugins/test_cppapi/Makefile.inc new file mode 100644 index 00000000000..d1a7c970248 --- /dev/null +++ b/plugins/test_cppapi/Makefile.inc @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += test_cppapi/test_cppapi.la +test_cppapi_test_cppapi_la_SOURCES = test_cppapi/test_cppapi.cc diff --git a/plugins/test_cppapi/test_cppapi.cc b/plugins/test_cppapi/test_cppapi.cc new file mode 100644 index 00000000000..a2c782f6b25 --- /dev/null +++ b/plugins/test_cppapi/test_cppapi.cc @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ts/ts.h" + +// Placeholder test. +// +namespace Test1 +{ +void +x() +{ + TSReleaseAssert(true); +} + +} // end namespace Test1; + +void +TSPluginInit(int, const char **) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = "test_cppapi"; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + TSReleaseAssert(TSPluginRegister(&info) == TS_SUCCESS); + + Test1::x(); +} diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index a4c0a6222f2..7258a4d1846 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -320,6 +320,10 @@ InjectTxnUuidHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) void log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, const char *msg_type) { + if (!TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { + return; + } + TSIOBuffer output_buffer; TSIOBufferReader reader; TSIOBufferBlock block; @@ -458,7 +462,7 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) } else if (header_field_eq("diags", value, vsize)) { // Enable diagnostics for DebugTxn()'s only TSHttpTxnDebugSet(txn, 1); - } else if (header_field_eq("log-headers", value, vsize) && TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { + } else if (header_field_eq("log-headers", value, vsize)) { xheaders |= XHEADER_X_DUMP_HEADERS; log_headers(txn, buffer, hdr, "ClientRequest"); diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc index 5dd13199c6c..77c7b038e7d 100644 --- a/proxy/InkAPI.cc +++ b/proxy/InkAPI.cc @@ -6722,9 +6722,9 @@ TSVConn TSTransformCreate(TSEventFunc event_funcp, TSHttpTxn txnp) { sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - // TODO: This is somewhat of a leap of faith, but I think a TSHttpTxn is just another - // fancy continuation? - return TSVConnCreate(event_funcp, TSContMutexGet(reinterpret_cast(txnp))); + + return TSVConnCreate(event_funcp, + reinterpret_cast(static_cast(reinterpret_cast(txnp))->getMutex())); } TSVConn diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index d1dcd21338e..b32ddc0af8c 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1813,16 +1813,6 @@ HttpSM::state_read_server_response_header(int event, void *data) case VC_EVENT_EOS: server_entry->eos = true; - // If no bytes were transmitted, the parser treats - // as a good 0.9 response which is technically is - // but it's indistinguishable from an overloaded - // server closing the connection so don't accept - // zero length responses - if (vio->ndone == 0) { - // Error handling function - handle_server_setup_error(event, data); - return 0; - } // Fall through case VC_EVENT_READ_READY: case VC_EVENT_READ_COMPLETE: @@ -1863,12 +1853,11 @@ HttpSM::state_read_server_response_header(int event, void *data) server_response_hdr_bytes += bytes_used; - // Don't allow 0.9 (unparsable headers) on keep-alive connections after - // the connection has already served a transaction as what we are likely - // looking at is garbage on a keep-alive channel corrupted by the origin - // server - if (state == PARSE_RESULT_DONE && t_state.hdr_info.server_response.version_get() == HTTPVersion(0, 9) && - server_session->transact_count > 1) { + // Don't allow HTTP 0.9 (unparsable headers) on resued connections. + // And don't allow empty headers from closed connections + if ((state == PARSE_RESULT_DONE && t_state.hdr_info.server_response.version_get() == HTTPVersion(0, 9) && + server_session->transact_count > 1) || + (server_entry->eos && vio->ndone == 0)) { state = PARSE_RESULT_ERROR; } // Check to see if we are over the hdr size limit diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 9af7900ccc9..6085b1b236a 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -1123,7 +1123,7 @@ HttpTransact::HandleRequest(State *s) { TxnDebug("http_trans", "START HttpTransact::HandleRequest"); - if (!s->request_data.hdr) { + if (!s->state_machine->is_waiting_for_full_body) { ink_assert(!s->hdr_info.server_request.valid()); HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat); diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 606e1b00a0e..519010e5ad7 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -1245,6 +1245,9 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) if (p->alive) { p->alive = false; p->bytes_read = p->read_vio->ndone; + // Clear any outstanding reads so they don't + // collide with future tunnel IO's + p->vc->do_io_read(nullptr, 0, 0); // Interesting tunnel event, call SM jump_point = p->vc_handler; (sm->*jump_point)(event, p); diff --git a/tests/gold_tests/headers/syntax.200.gold b/tests/gold_tests/headers/syntax.200.gold index 3b224e6009e..a60ae6ef014 100644 --- a/tests/gold_tests/headers/syntax.200.gold +++ b/tests/gold_tests/headers/syntax.200.gold @@ -1,6 +1,6 @@ HTTP/1.1 200 OK Date: `` -Age: 0 +Age: `` Transfer-Encoding: chunked Connection: keep-alive Server: `` diff --git a/tests/gold_tests/pluginTest/cppapi/cppapi.test.py b/tests/gold_tests/pluginTest/cppapi/cppapi.test.py new file mode 100644 index 00000000000..1f4f0c312c0 --- /dev/null +++ b/tests/gold_tests/pluginTest/cppapi/cppapi.test.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Execute plugin with cppapi tests. +''' + +ts = Test.MakeATSProcess("ts") + +ts.Disk.plugin_config.AddLine('test_cppapi.so') + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = "echo run test_cppapi plugin" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.StillRunningAfter = ts