From 6066da991478fae9d08eef5d3471cda8e60e54b3 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Fri, 22 Mar 2019 22:53:48 +0000 Subject: [PATCH] autest for slice plugin --- plugins/experimental/slice/Makefile.tsxs | 24 ++- plugins/experimental/slice/README.md | 2 +- plugins/experimental/slice/client.cc | 5 + plugins/experimental/slice/server.cc | 2 +- tests/gold_tests/pluginTest/slice/curlsort.sh | 33 +++ .../slice/gold/slice_200.stderr.gold | 1 + .../slice/gold/slice_200.stdout.gold | 8 + .../slice/gold/slice_206.stderr.gold | 1 + .../slice/gold/slice_206.stdout.gold | 9 + .../slice/gold/slice_first.stderr.gold | 1 + .../slice/gold/slice_first.stdout.gold | 9 + .../slice/gold/slice_last.stderr.gold | 1 + .../slice/gold/slice_last.stdout.gold | 9 + .../slice/gold/slice_mid.stderr.gold | 1 + .../slice/gold/slice_mid.stdout.gold | 9 + .../gold_tests/pluginTest/slice/slice.test.py | 188 ++++++++++++++++++ 16 files changed, 290 insertions(+), 13 deletions(-) create mode 100755 tests/gold_tests/pluginTest/slice/curlsort.sh create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/slice.test.py diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs index b3b5d4f014d..0992d8cb27e 100644 --- a/plugins/experimental/slice/Makefile.tsxs +++ b/plugins/experimental/slice/Makefile.tsxs @@ -14,9 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: slice +TSXS?=tsxs +PLUGIN=slice -slice_la_SOURCES = \ +all: $(PLUGIN).so + +SOURCES = \ Config.cc \ ContentRange.cc \ Data.cc \ @@ -29,7 +32,7 @@ slice_la_SOURCES = \ slice.cc \ transfer.cc \ -slice_la_HEADERS = \ +HEADERS = \ Config.h \ ContentRange.h \ Data.h \ @@ -43,22 +46,21 @@ slice_la_HEADERS = \ slice.h \ transfer.h \ -slice: $(slice_la_SOURCES) $(slice_la_HEADERS) - tsxs -v -o slice.so $(slice_la_SOURCES) +$(PLUGIN).so: $(SOURCES) $(HEADERS) + $(TSXS) -v -o $(PLUGIN).so $(SOURCES) -install: slice $(slice_la_SOURCES) $(slice_la_HEADERS) - tsxs -v -o slice.so -i +install: all + $(TSXS) -v -o $(PLUGIN).so -i -CXX = c++ -std=c++11 -#CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Qunused-arguments -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 -CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 TSINCLUDE = $(shell tsxs -q INCLUDEDIR) +TSCXX = $(shell tsxs -q CXX) +TSCXXFLAGS = $(shell tsxs -q CXXFLAGS) #PREFIX = $(shell tsxs -q PREFIX) #LIBS = -L$(PREFIX)/lib -latscppapi #LIBS = $(PREFIX)/lib/libtsutil.la slice_test: slice_test.cc ContentRange.cc Range.cc - $(CXX) -o $@ $^ $(CXXFLAGS) -I$(TSINCLUDE) -DUNITTEST + $(TSCXX) -o $@ $^ $(TSCXXFLAGS) -I$(TSINCLUDE) -DUNITTEST clean: rm -fv *.lo *.so diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md index 731fd579241..dc56997ce56 100644 --- a/plugins/experimental/slice/README.md +++ b/plugins/experimental/slice/README.md @@ -31,7 +31,7 @@ cache_range_requests.so **Note**: cache_range_requests **MUST** follow slice.so Put these plugins at the end of the plugin list **Note**: blockbytes is defined in bytes. 1048576 (1MB) is the default. -For testing purposes an unchecked value of "blockbytestest" is also available. +For testing purposes an unchecked value of "bytesover" is also available. Debug output can be enable by setting the debug tag: **slice** diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index e29f9e715ac..d52887e5278 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -65,6 +65,11 @@ requestBlock(TSCont contp, Data *const data) TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); TSVIOReenable(data->m_upstream.m_write.m_vio); + /* + std::string const headerstr(header.toString()); + DEBUG_LOG("Headers\n%s", headerstr.c_str()); + */ + // get ready for data back from the server data->m_upstream.setupVioRead(contp); diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 508a455ecab..f2f4dee5af3 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -35,7 +35,7 @@ shutdown(TSCont const contp, Data *const data) } ContentRange -contentRangeFrom(HttpHeader &header) +contentRangeFrom(HttpHeader const &header) { ContentRange bcr; diff --git a/tests/gold_tests/pluginTest/slice/curlsort.sh b/tests/gold_tests/pluginTest/slice/curlsort.sh new file mode 100755 index 00000000000..eabf5618ff2 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/curlsort.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmd='curl' +for arg in "$@"; do + case "$arg" in + *\'*) +# arg=`printf '%s' "$arg" | sed s/'/'\"'\"'/g"` + arg=`printf '%s' "$arg"` + ;; + *) : ;; + esac + cmd="$cmd '$arg'" +done + +cmd="$cmd -s -D /dev/stdout -o /dev/stderr" + +eval " $cmd" | sort diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold new file mode 100644 index 00000000000..1e704c37b3d --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold @@ -0,0 +1,8 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Date: `` +Etag: "path" +HTTP/1.1 200 OK +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold new file mode 100644 index 00000000000..d4bd2de8782 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Content-Range: bytes 0-17/18 +Date: `` +Etag: "path" +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold new file mode 100644 index 00000000000..10893bb0688 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold @@ -0,0 +1 @@ +lets go`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold new file mode 100644 index 00000000000..235b074c618 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 7 +Content-Range: bytes 0-6/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold new file mode 100644 index 00000000000..6c9cadd2dfd --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold @@ -0,0 +1 @@ + now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold new file mode 100644 index 00000000000..88fa42bcfba --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 4 +Content-Range: bytes 14-17/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold new file mode 100644 index 00000000000..a9dc13c610e --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold @@ -0,0 +1 @@ +go surfin no`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold new file mode 100644 index 00000000000..5067f1cce49 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 12 +Content-Range: bytes 5-16/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py new file mode 100644 index 00000000000..a3e299a29d5 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -0,0 +1,188 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Basic slice plugin test +''' + +## Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +#block_bytes = 7 +body = "lets go surfin now" + +request_header = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, +} + +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Setup.CopyAs('curlsort.sh', Test.RunDirectory) +curl_and_args = 'sh curlsort.sh -H "Host: www.example.com"' + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# 0 Test - Prefetch entire asset into cache +tr = Test.AddTestRun("Fetch first slice range") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +block_bytes = 7 + +# 1 - Reconfigure remap.config with slice plugin +tr = Test.AddTestRun("Load Slice plugin") +remap_config_path = ts.Disk.remap_config.Name +tr.Disk.File(remap_config_path, typename="ats:config").AddLines([ + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=bytesover:{}'.format(block_bytes) +]) + +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 2 Test - First complete slice +tr = Test.AddTestRun("Fetch first slice range") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-6' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_first.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_first.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Test - Last slice auto +tr = Test.AddTestRun("Last slice -- 14-") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 4 Test - Last slice exact +tr = Test.AddTestRun("Last slice 14-17") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-17' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 5 Test - Last slice truncated +tr = Test.AddTestRun("Last truncated slice 14-20") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-20' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 6 Test - Whole asset via slices +tr = Test.AddTestRun("Whole asset via slices") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +# 7 Test - Whole asset via range +tr = Test.AddTestRun("Whole asset via range") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_206.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_206.stderr.gold" +tr.StillRunningAfter = ts + +# 8 Test - Non aligned slice request +tr = Test.AddTestRun("Non aligned slice request") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 5-16' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_mid.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_mid.stderr.gold" +tr.StillRunningAfter = ts +