Skip to content

Commit

Permalink
Fix inability to change httpclient's internal buffer size. (#531)
Browse files Browse the repository at this point in the history 
Add test.
Address #529.
  • Loading branch information
@cheatfate
cheatfate authored Apr 17, 2024
1 parent 0d050d5 commit e4cb480
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 23 deletions.
66 changes: 43 additions & 23 deletions chronos/apps/http/httpclient.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ type
redirectCount: int
timestamp*: Moment
duration*: Duration
headersBuffer: seq[byte]

HttpClientRequestRef* = ref HttpClientRequest

Expand Down Expand Up @@ -859,6 +860,7 @@ proc closeWait*(request: HttpClientRequestRef) {.async: (raises: []).} =
await noCancel(allFutures(pending))
request.session = nil
request.error = nil
request.headersBuffer.reset()
request.state = HttpReqRespState.Closed
untrackCounter(HttpClientRequestTrackerName)

Expand Down Expand Up @@ -992,38 +994,40 @@ proc prepareResponse(

proc getResponse(req: HttpClientRequestRef): Future[HttpClientResponseRef] {.
async: (raises: [CancelledError, HttpError]).} =
var buffer: array[HttpMaxHeadersSize, byte]
let timestamp = Moment.now()
req.connection.setTimestamp(timestamp)
let
bytesRead =
try:
await req.connection.reader.readUntil(addr buffer[0],
len(buffer), HeadersMark).wait(
await req.connection.reader.readUntil(addr req.headersBuffer[0],
len(req.headersBuffer),
HeadersMark).wait(
req.session.headersTimeout)
except AsyncTimeoutError:
raiseHttpReadError("Reading response headers timed out")
except AsyncStreamError as exc:
raiseHttpReadError(
"Could not read response headers, reason: " & $exc.msg)

let response = prepareResponse(req, buffer.toOpenArray(0, bytesRead - 1))
if response.isErr():
raiseHttpProtocolError(response.error())
let res = response.get()
res.setTimestamp(timestamp)
return res
let response =
prepareResponse(req,
req.headersBuffer.toOpenArray(0, bytesRead - 1)).valueOr:
raiseHttpProtocolError(error)
response.setTimestamp(timestamp)
response

proc new*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
ha: HttpAddress, meth: HttpMethod = MethodGet,
version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[byte] = []): HttpClientRequestRef =
let res = HttpClientRequestRef(
state: HttpReqRespState.Ready, session: session, meth: meth,
version: version, flags: flags, headers: HttpTable.init(headers),
address: ha, bodyFlag: HttpClientBodyFlag.Custom, buffer: @body
address: ha, bodyFlag: HttpClientBodyFlag.Custom, buffer: @body,
headersBuffer: newSeq[byte](max(maxResponseHeadersSize, HttpMaxHeadersSize))
)
trackCounter(HttpClientRequestTrackerName)
res
Expand All @@ -1032,62 +1036,74 @@ proc new*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
url: string, meth: HttpMethod = MethodGet,
version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[byte] = []): HttpResult[HttpClientRequestRef] =
let address = ? session.getAddress(parseUri(url))
let res = HttpClientRequestRef(
state: HttpReqRespState.Ready, session: session, meth: meth,
version: version, flags: flags, headers: HttpTable.init(headers),
address: address, bodyFlag: HttpClientBodyFlag.Custom, buffer: @body
address: address, bodyFlag: HttpClientBodyFlag.Custom, buffer: @body,
headersBuffer: newSeq[byte](max(maxResponseHeadersSize, HttpMaxHeadersSize))
)
trackCounter(HttpClientRequestTrackerName)
ok(res)

proc get*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
url: string, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = []
): HttpResult[HttpClientRequestRef] =
HttpClientRequestRef.new(session, url, MethodGet, version, flags, headers)
HttpClientRequestRef.new(session, url, MethodGet, version, flags,
maxResponseHeadersSize, headers)

proc get*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
ha: HttpAddress, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = []
): HttpClientRequestRef =
HttpClientRequestRef.new(session, ha, MethodGet, version, flags, headers)
HttpClientRequestRef.new(session, ha, MethodGet, version, flags,
maxResponseHeadersSize, headers)

proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
url: string, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[byte] = []
): HttpResult[HttpClientRequestRef] =
HttpClientRequestRef.new(session, url, MethodPost, version, flags, headers,
body)
HttpClientRequestRef.new(session, url, MethodPost, version, flags,
maxResponseHeadersSize, headers, body)

proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
url: string, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[char] = []): HttpResult[HttpClientRequestRef] =
HttpClientRequestRef.new(session, url, MethodPost, version, flags, headers,
HttpClientRequestRef.new(session, url, MethodPost, version, flags,
maxResponseHeadersSize, headers,
body.toOpenArrayByte(0, len(body) - 1))

proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
ha: HttpAddress, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[byte] = []): HttpClientRequestRef =
HttpClientRequestRef.new(session, ha, MethodPost, version, flags, headers,
body)
HttpClientRequestRef.new(session, ha, MethodPost, version, flags,
maxResponseHeadersSize, headers, body)

proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef,
ha: HttpAddress, version: HttpVersion = HttpVersion11,
flags: set[HttpClientRequestFlag] = {},
maxResponseHeadersSize: int = HttpMaxHeadersSize,
headers: openArray[HttpHeaderTuple] = [],
body: openArray[char] = []): HttpClientRequestRef =
HttpClientRequestRef.new(session, ha, MethodPost, version, flags, headers,
HttpClientRequestRef.new(session, ha, MethodPost, version, flags,
maxResponseHeadersSize, headers,
body.toOpenArrayByte(0, len(body) - 1))

proc prepareRequest(request: HttpClientRequestRef): string =
Expand Down Expand Up @@ -1454,8 +1470,10 @@ proc redirect*(request: HttpClientRequestRef,
var res = request.headers
res.set(HostHeader, ha.hostname)
res
var res = HttpClientRequestRef.new(request.session, ha, request.meth,
request.version, request.flags, headers.toList(), request.buffer)
var res =
HttpClientRequestRef.new(request.session, ha, request.meth,
request.version, request.flags, headers = headers.toList(),
body = request.buffer)
res.redirectCount = redirectCount
ok(res)

Expand All @@ -1478,8 +1496,10 @@ proc redirect*(request: HttpClientRequestRef,
var res = request.headers
res.set(HostHeader, address.hostname)
res
var res = HttpClientRequestRef.new(request.session, address, request.meth,
request.version, request.flags, headers.toList(), request.buffer)
var res =
HttpClientRequestRef.new(request.session, address, request.meth,
request.version, request.flags, headers = headers.toList(),
body = request.buffer)
res.redirectCount = redirectCount
ok(res)

Expand Down
60 changes: 60 additions & 0 deletions tests/testhttpclient.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1518,3 +1518,63 @@ suite "HTTP client testing suite":
res.isErr() and res.error == HttpAddressErrorType.NameLookupFailed
res.error.isRecoverableError()
not(res.error.isCriticalError())

asyncTest "HTTPS response headers buffer size test":
const HeadersSize = HttpMaxHeadersSize
let expectValue =
string.fromBytes(createBigMessage("HEADERSTEST", HeadersSize))
proc process(r: RequestFence): Future[HttpResponseRef] {.
async: (raises: [CancelledError]).} =
if r.isOk():
let request = r.get()
try:
case request.uri.path
of "/test":
let headers = HttpTable.init([("big-header", expectValue)])
await request.respond(Http200, "ok", headers)
else:
await request.respond(Http404, "Page not found")
except HttpWriteError as exc:
defaultResponse(exc)
else:
defaultResponse()

var server = createServer(initTAddress("127.0.0.1:0"), process, false)
server.start()
let
address = server.instance.localAddress()
ha = getAddress(address, HttpClientScheme.NonSecure, "/test")
session = HttpSessionRef.new()
let
req1 = HttpClientRequestRef.new(session, ha)
req2 =
HttpClientRequestRef.new(session, ha,
maxResponseHeadersSize = HttpMaxHeadersSize * 2)
res1 =
try:
let res {.used.} = await send(req1)
await closeWait(req1)
await closeWait(res)
false
except HttpReadError:
true
except HttpError:
await closeWait(req1)
false
except CancelledError:
await closeWait(req1)
false

res2 = await send(req2)

check:
res1 == true
res2.status == 200
res2.headers.getString("big-header") == expectValue

await req1.closeWait()
await req2.closeWait()
await res2.closeWait()
await session.closeWait()
await server.stop()
await server.closeWait()

0 comments on commit e4cb480

Please sign in to comment.