From 6fbe07ab480fc201db7eca19864210e4cbe100a9 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 00:16:24 +0300 Subject: [PATCH 01/20] pcapng --- .../scala/fs2/protocols/pcapng/Block.scala | 95 ++++++++++ .../fs2/protocols/pcapng/ByteOrderMagic.scala | 50 ++++++ .../scala/fs2/protocols/PcapMpegExample.scala | 2 +- .../scala/fs2/protocols/PcapNgExample.scala | 50 ++++++ .../fs2/protocols/pcapng/BlockTest.scala | 167 ++++++++++++++++++ 5 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala create mode 100644 protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala create mode 100644 protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala new file mode 100644 index 0000000000..dfcfbdb150 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import pcap._ +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ +import shapeless.ops.hlist.{Init, Last, Prepend} +import shapeless.{::, HList, HNil} + +sealed trait Block + +object Block { + + case class SectionHeaderBlock( + length: ByteVector, + ordering: ByteOrdering, + majorVersion: Int, + minorVersion: Int, + bytes: ByteVector + ) extends Block + + sealed trait BodyBlock extends Block + case class InterfaceDescriptionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + case class EnhancedPacketBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + + private val sectionHeaderConstant = hex"0A0D0D0A" + private def interfaceDescriptionHex(implicit ord: ByteOrdering) = + endiannessDependent(hex"00000001", hex"01000000") + private def enhancedPacketHex(implicit ord: ByteOrdering) = + endiannessDependent(hex"00000006", hex"06000000") + + private def endiannessDependent[T](big: ByteVector, little: ByteVector)(implicit ord: ByteOrdering) = + ord match { + case ByteOrdering.BigEndian => big + case ByteOrdering.LittleEndian => little + } + + // format: off + private def block[L <: HList, LB <: HList](const: ByteVector)(f: ByteVector => Codec[L])( + implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit] + ): Codec[Unit :: ByteVector :: LB] = + ("block_type" | constant(const)) :: + ("block_length" | bytes(4)).flatPrepend { length => + f(length) :+ constant(length) + } + + private def ignoredBlock(constant: ByteVector)(implicit ord: ByteOrdering) = + block(constant) { length => + ("block_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 12, bytes)) :: + Codec.deriveHNil + }.dropUnits + + private def sectionHeader(implicit ord: ByteOrdering) = { + ("version_major" | guint16 ) :: + ("version_minor" | guint16 ) :: + ("remaining" | bytes ) + } + + val sectionHeaderCodec: Codec[SectionHeaderBlock] = "SHB" | block(sectionHeaderConstant) { length => + ("magic_number" | ByteOrderMagic).flatPrepend { implicit ord => + ("body_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) + }}.dropUnits.as[SectionHeaderBlock] + // format: on + + def interfaceDescriptionBlock(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = + "IDB" | ignoredBlock(interfaceDescriptionHex).as[InterfaceDescriptionBlock] + + def enhancedPacketBlock(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = + "EPB" | ignoredBlock(enhancedPacketHex).as[EnhancedPacketBlock] +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala new file mode 100644 index 0000000000..5bdbc4ae4f --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols.pcapng + +import scodec._ +import scodec.bits._ +import scodec.codecs._ + +object ByteOrderMagic extends Codec[ByteOrdering] { + private val BigEndian = hex"1a2b3c4d" + private val LittleEndian = hex"4d3c2b1a" + + def sizeBound = SizeBound.exact(32) + + def encode(bo: ByteOrdering) = bo match { + case ByteOrdering.BigEndian => Attempt.successful(BigEndian.bits) + case ByteOrdering.LittleEndian => Attempt.successful(LittleEndian.bits) + } + + def decode(buf: BitVector) = + bytes(4).decode(buf).flatMap { + case DecodeResult(BigEndian, rest) => + Attempt.successful(DecodeResult(ByteOrdering.BigEndian, rest)) + case DecodeResult(LittleEndian, rest) => + Attempt.successful(DecodeResult(ByteOrdering.LittleEndian, rest)) + case DecodeResult(other, _) => + Attempt.failure( + Err(s"unable to detect byte ordering due to unrecognized magic number $other") + ) + } +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala index 2aae1d4d61..d604f724fa 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala @@ -38,7 +38,7 @@ import pcap.{CaptureFile, LinkType} * - of UDP datagrams * - containing MPEG transport stream packets */ -object PcapMpegExample extends IOApp.Simple { +object PcapMpegExample { case class CapturedPacket( source: SocketAddress[Ipv4Address], diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala new file mode 100644 index 0000000000..dc27ae6680 --- /dev/null +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package protocols + +import cats.effect.{IO, IOApp} +import fs2.interop.scodec.StreamDecoder +import fs2.io.file.{Files, Path} +import fs2.protocols.pcapng.Block +import fs2.protocols.pcapng.Block.{BodyBlock, SectionHeaderBlock} +import scodec.Decoder + +object PcapNgExample extends IOApp.Simple { + + def run: IO[Unit] = + Files[IO] + .readAll(Path("/Users/anikiforov/Downloads/dhcp.pcapng")) + .through(streamDecoder.toPipeByte) + .debug() + .compile + .drain + + private val streamDecoder: StreamDecoder[(SectionHeaderBlock, BodyBlock)] = + for { + header <- StreamDecoder.once(Block.sectionHeaderCodec) + idb = Block.interfaceDescriptionBlock(header.ordering) + epb = Block.enhancedPacketBlock(header.ordering) + decoder = Decoder.choiceDecoder(idb, epb) + block <- StreamDecoder.many(decoder) + } yield (header, block) +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala new file mode 100644 index 0000000000..4ee9f89735 --- /dev/null +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import fs2.protocols.pcapng.Block.{EnhancedPacketBlock, InterfaceDescriptionBlock, SectionHeaderBlock} +import scodec.Attempt.Successful +import scodec.DecodeResult +import scodec.bits.ByteOrdering.LittleEndian +import scodec.bits._ + +// https://wiki.wireshark.org/Development/PcapNg dhcp.pcapng +class BlockTest extends munit.FunSuite { + import BlockTest._ + + test("shb") { + val actual = Block.sectionHeaderCodec.decode(SHB.bytes.bits) + assertEquals(actual, fullyDecoded(SHB.expected)) + } + + test("interface") { + val actual = Block.interfaceDescriptionBlock(LittleEndian).decode(Interface.bytes.bits) + assertEquals(actual, fullyDecoded(Interface.expected)) + } + + test("enhanced1") { + val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced1.bytes.bits) + assertEquals(actual, fullyDecoded(Enhanced1.expected)) + } + + test("enhanced2") { + val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced2.bytes.bits) + assertEquals(actual, fullyDecoded(Enhanced2.expected)) + } + + test("enhanced3") { + val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced3.bytes.bits) + assertEquals(actual, fullyDecoded(Enhanced3.expected)) + } + + test("enhanced4") { + val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced4.bytes.bits) + assertEquals(actual, fullyDecoded(Enhanced4.expected)) + } + + private def fullyDecoded[V](v: V) = + Successful(DecodeResult(v, BitVector.empty)) +} + +private object BlockTest { + + object SHB { + val length = hex"1c000000" + val other = hex"ffffffffffffffff" + val expected = SectionHeaderBlock(length,LittleEndian,1,0,other) + val bytes = { + val header = hex"0a0d0d0a" + val magic = hex"4d3c2b1a" + val major = hex"0100" + val minor = hex"0000" + + header ++ length ++ magic ++ major ++ minor ++ other ++ length + } + } + + object Interface { + val header = hex"01000000" + val length = hex"20000000" + val other = hex"01000000ffff0000090001000600000000000000" + val expected = InterfaceDescriptionBlock(length, other) + val bytes = header ++ length ++ other ++ length + } + + val enhancedHeader = hex"06000000" + object Enhanced1 { + val length = hex"5c010000" + val other = + hex"""0000000083ea03000d8a33353a0100003a010000ffffffffffff000b + 8201fc4208004500012ca8360000fa11178b00000000ffffffff004400430118 + 591f0101060000003d1d0000000000000000000000000000000000000000000b + 8201fc4200000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000638253633501013d0701000b8201fc423204 + 0000000037040103062aff000000000000000000""" + val expected = EnhancedPacketBlock(length, other) + val bytes = enhancedHeader ++ length ++ other ++ length + } + + object Enhanced2 { + val length = hex"78010000" + val other = + hex"""0000000083ea0300348b33355601000056010000000b8201fc42000874adf19b + 0800450001480445000080110000c0a80001c0a8000a00430044013422330201 + 060000003d1d0000000000000000c0a8000ac0a8000100000000000b8201fc42 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 00000000000000000000638253633501020104ffffff003a04000007083b0400 + 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 + 000000000000000000000000""" + val expected = EnhancedPacketBlock(length, other) + val bytes = enhancedHeader ++ length ++ other ++ length + } + + object Enhanced3 { + val length = hex"5c010000" + val other = + hex"""0000000083ea03009c9b34353a0100003a010000ffffffff + ffff000b8201fc4208004500012ca8370000fa11178a00000000ffffffff0044 + 004301189fbd0101060000003d1e000000000000000000000000000000000000 + 0000000b8201fc42000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000638253633501033d0701000b8201 + fc423204c0a8000a3604c0a8000137040103062aff000000""" + val expected = EnhancedPacketBlock(length, other) + val bytes = enhancedHeader ++ length ++ other ++ length + } + + object Enhanced4 { + val length = hex"78010000" + val other = + hex"""0000000083ea0300d69c34355601000056010000000b8201fc420008 + 74adf19b0800450001480446000080110000c0a80001c0a8000a004300440134 + dfdb0201060000003d1e0000000000000000c0a8000a0000000000000000000b + 8201fc4200000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000638253633501053a04000007083b0400000c + 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 + 00000000000000000000000000000000""" + val expected = EnhancedPacketBlock(length, other) + val bytes = enhancedHeader ++ length ++ other ++ length + } +} From 5ccea1c5851835298b51026d01ad6b695f51e15d Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 02:15:39 +0300 Subject: [PATCH 02/20] pcapng - splitted to different classes --- .gitignore | 1 + .../scala/fs2/protocols/pcapng/Block.scala | 48 +++------------- .../fs2/protocols/pcapng/BodyBlock.scala | 34 ++++++++++++ .../fs2/protocols/pcapng/ByteOrderMagic.scala | 3 +- .../pcapng/EnhancedPacketBlock.scala | 38 +++++++++++++ .../pcapng/InterfaceDescriptionBlock.scala | 38 +++++++++++++ .../protocols/pcapng/SectionHeaderBlock.scala | 55 +++++++++++++++++++ .../scala/fs2/protocols/PcapNgExample.scala | 10 +--- .../fs2/protocols/pcapng/BlockTest.scala | 13 ++--- 9 files changed, 184 insertions(+), 56 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala diff --git a/.gitignore b/.gitignore index 596ac4fd40..5bad453f18 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ project/metals.sbt project/project .vscode .bsp +.java-version node_modules diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index dfcfbdb150..23953ac922 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -22,74 +22,40 @@ package fs2.protocols package pcapng -import pcap._ import scodec.Codec import scodec.bits._ import scodec.codecs._ import shapeless.ops.hlist.{Init, Last, Prepend} import shapeless.{::, HList, HNil} -sealed trait Block +trait Block object Block { - case class SectionHeaderBlock( - length: ByteVector, - ordering: ByteOrdering, - majorVersion: Int, - minorVersion: Int, - bytes: ByteVector - ) extends Block + type Length = ByteVector - sealed trait BodyBlock extends Block - case class InterfaceDescriptionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock - case class EnhancedPacketBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock - - private val sectionHeaderConstant = hex"0A0D0D0A" - private def interfaceDescriptionHex(implicit ord: ByteOrdering) = - endiannessDependent(hex"00000001", hex"01000000") - private def enhancedPacketHex(implicit ord: ByteOrdering) = - endiannessDependent(hex"00000006", hex"06000000") - - private def endiannessDependent[T](big: ByteVector, little: ByteVector)(implicit ord: ByteOrdering) = + def orderDependent[T](big: ByteVector, little: ByteVector)(implicit ord: ByteOrdering): ByteVector = ord match { case ByteOrdering.BigEndian => big case ByteOrdering.LittleEndian => little } // format: off - private def block[L <: HList, LB <: HList](const: ByteVector)(f: ByteVector => Codec[L])( + def block[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] ): Codec[Unit :: ByteVector :: LB] = - ("block_type" | constant(const)) :: + ("block_type" | constant(hexConstant)) :: ("block_length" | bytes(4)).flatPrepend { length => f(length) :+ constant(length) } - private def ignoredBlock(constant: ByteVector)(implicit ord: ByteOrdering) = - block(constant) { length => + def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = + block(hexConstant) { length => ("block_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 12, bytes)) :: Codec.deriveHNil }.dropUnits - - private def sectionHeader(implicit ord: ByteOrdering) = { - ("version_major" | guint16 ) :: - ("version_minor" | guint16 ) :: - ("remaining" | bytes ) - } - - val sectionHeaderCodec: Codec[SectionHeaderBlock] = "SHB" | block(sectionHeaderConstant) { length => - ("magic_number" | ByteOrderMagic).flatPrepend { implicit ord => - ("body_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) - }}.dropUnits.as[SectionHeaderBlock] // format: on - - def interfaceDescriptionBlock(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | ignoredBlock(interfaceDescriptionHex).as[InterfaceDescriptionBlock] - - def enhancedPacketBlock(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = - "EPB" | ignoredBlock(enhancedPacketHex).as[EnhancedPacketBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala new file mode 100644 index 0000000000..3ef026b3c0 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.Decoder +import scodec.bits.ByteOrdering + +trait BodyBlock extends Block + +object BodyBlock { + + def decoder(implicit ord: ByteOrdering): Decoder[BodyBlock] = + Decoder.choiceDecoder(InterfaceDescriptionBlock.codec, EnhancedPacketBlock.codec) +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala index 5bdbc4ae4f..97ee352949 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala @@ -19,7 +19,8 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package fs2.protocols.pcapng +package fs2.protocols +package pcapng import scodec._ import scodec.bits._ diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala new file mode 100644 index 0000000000..5c891221bd --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +case class EnhancedPacketBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + +object EnhancedPacketBlock { + + def hexConstant(implicit ord: ByteOrdering) = + Block.orderDependent(hex"00000006", hex"06000000") + + def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = + "EPB" | Block.ignoredBlock(hexConstant).as[EnhancedPacketBlock] +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala new file mode 100644 index 0000000000..baaef180a2 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.bits._ +import scodec.Codec +import scodec.codecs._ + +case class InterfaceDescriptionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + +object InterfaceDescriptionBlock { + + private def hexConstant(implicit ord: ByteOrdering) = + Block.orderDependent(hex"00000001", hex"01000000") + + def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = + "IDB" | Block.ignoredBlock(hexConstant).as[InterfaceDescriptionBlock] +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala new file mode 100644 index 0000000000..dd7eca9ec2 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import pcap._ +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ +import scodec.codecs.fixedSizeBytes + +case class SectionHeaderBlock( + length: ByteVector, + ordering: ByteOrdering, + majorVersion: Int, + minorVersion: Int, + bytes: ByteVector +) extends Block + +object SectionHeaderBlock { + + private val hexConstant = hex"0A0D0D0A" + + // format: off + private def sectionHeader(implicit ord: ByteOrdering) = { + ("version_major" | guint16 ) :: + ("version_minor" | guint16 ) :: + ("remaining" | bytes ) + } + + val codec: Codec[SectionHeaderBlock] = "SHB" | Block.block(hexConstant) { length => + ("magic_number" | ByteOrderMagic).flatPrepend { implicit ord => + ("body_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) + }}.dropUnits.as[SectionHeaderBlock] + // format: on +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index dc27ae6680..04d47b3ff9 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -25,9 +25,7 @@ package protocols import cats.effect.{IO, IOApp} import fs2.interop.scodec.StreamDecoder import fs2.io.file.{Files, Path} -import fs2.protocols.pcapng.Block -import fs2.protocols.pcapng.Block.{BodyBlock, SectionHeaderBlock} -import scodec.Decoder +import fs2.protocols.pcapng.{BodyBlock, SectionHeaderBlock} object PcapNgExample extends IOApp.Simple { @@ -41,10 +39,8 @@ object PcapNgExample extends IOApp.Simple { private val streamDecoder: StreamDecoder[(SectionHeaderBlock, BodyBlock)] = for { - header <- StreamDecoder.once(Block.sectionHeaderCodec) - idb = Block.interfaceDescriptionBlock(header.ordering) - epb = Block.enhancedPacketBlock(header.ordering) - decoder = Decoder.choiceDecoder(idb, epb) + header <- StreamDecoder.once(SectionHeaderBlock.codec) + decoder = BodyBlock.decoder(header.ordering) block <- StreamDecoder.many(decoder) } yield (header, block) } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 4ee9f89735..151c6ce6c9 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -22,7 +22,6 @@ package fs2.protocols package pcapng -import fs2.protocols.pcapng.Block.{EnhancedPacketBlock, InterfaceDescriptionBlock, SectionHeaderBlock} import scodec.Attempt.Successful import scodec.DecodeResult import scodec.bits.ByteOrdering.LittleEndian @@ -33,32 +32,32 @@ class BlockTest extends munit.FunSuite { import BlockTest._ test("shb") { - val actual = Block.sectionHeaderCodec.decode(SHB.bytes.bits) + val actual = SectionHeaderBlock.codec.decode(SHB.bytes.bits) assertEquals(actual, fullyDecoded(SHB.expected)) } test("interface") { - val actual = Block.interfaceDescriptionBlock(LittleEndian).decode(Interface.bytes.bits) + val actual = InterfaceDescriptionBlock.codec(LittleEndian).decode(Interface.bytes.bits) assertEquals(actual, fullyDecoded(Interface.expected)) } test("enhanced1") { - val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced1.bytes.bits) + val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced1.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced1.expected)) } test("enhanced2") { - val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced2.bytes.bits) + val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced2.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced2.expected)) } test("enhanced3") { - val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced3.bytes.bits) + val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced3.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced3.expected)) } test("enhanced4") { - val actual = Block.enhancedPacketBlock(LittleEndian).decode(Enhanced4.bytes.bits) + val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced4.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced4.expected)) } From 96d145467cccbcce2d64bd9f08cdc28617c51edb Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 02:19:29 +0300 Subject: [PATCH 03/20] pcapng - scalafmt --- .../src/main/scala/fs2/protocols/pcapng/Block.scala | 7 +++++-- .../scala/fs2/protocols/pcapng/ByteOrderMagic.scala | 2 +- .../fs2/protocols/pcapng/SectionHeaderBlock.scala | 12 ++++++------ .../test/scala/fs2/protocols/pcapng/BlockTest.scala | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index 23953ac922..3fafc0285a 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -34,9 +34,12 @@ object Block { type Length = ByteVector - def orderDependent[T](big: ByteVector, little: ByteVector)(implicit ord: ByteOrdering): ByteVector = + def orderDependent[T]( + big: ByteVector, + little: ByteVector + )(implicit ord: ByteOrdering): ByteVector = ord match { - case ByteOrdering.BigEndian => big + case ByteOrdering.BigEndian => big case ByteOrdering.LittleEndian => little } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala index 97ee352949..0aa1365f9c 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala @@ -33,7 +33,7 @@ object ByteOrderMagic extends Codec[ByteOrdering] { def sizeBound = SizeBound.exact(32) def encode(bo: ByteOrdering) = bo match { - case ByteOrdering.BigEndian => Attempt.successful(BigEndian.bits) + case ByteOrdering.BigEndian => Attempt.successful(BigEndian.bits) case ByteOrdering.LittleEndian => Attempt.successful(LittleEndian.bits) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index dd7eca9ec2..d03cab0349 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -29,11 +29,11 @@ import scodec.codecs._ import scodec.codecs.fixedSizeBytes case class SectionHeaderBlock( - length: ByteVector, - ordering: ByteOrdering, - majorVersion: Int, - minorVersion: Int, - bytes: ByteVector + length: ByteVector, + ordering: ByteOrdering, + majorVersion: Int, + minorVersion: Int, + bytes: ByteVector ) extends Block object SectionHeaderBlock { @@ -50,6 +50,6 @@ object SectionHeaderBlock { val codec: Codec[SectionHeaderBlock] = "SHB" | Block.block(hexConstant) { length => ("magic_number" | ByteOrderMagic).flatPrepend { implicit ord => ("body_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) - }}.dropUnits.as[SectionHeaderBlock] + }}.dropUnits.as[SectionHeaderBlock] // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 151c6ce6c9..cc147cc5b6 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -70,7 +70,7 @@ private object BlockTest { object SHB { val length = hex"1c000000" val other = hex"ffffffffffffffff" - val expected = SectionHeaderBlock(length,LittleEndian,1,0,other) + val expected = SectionHeaderBlock(length, LittleEndian, 1, 0, other) val bytes = { val header = hex"0a0d0d0a" val magic = hex"4d3c2b1a" From 4fe10627829d78167ba556b757f06b77a9943998 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 02:43:19 +0300 Subject: [PATCH 04/20] pcapng - granulate EnhancedPacketBlock --- .../scala/fs2/protocols/pcapng/Block.scala | 13 +++++---- .../pcapng/EnhancedPacketBlock.scala | 24 ++++++++++++++-- .../protocols/pcapng/SectionHeaderBlock.scala | 10 +++---- .../fs2/protocols/pcapng/BlockTest.scala | 28 +++++++++++-------- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index 3fafc0285a..a44adb95a1 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -35,14 +35,17 @@ object Block { type Length = ByteVector def orderDependent[T]( - big: ByteVector, - little: ByteVector + big: ByteVector, + little: ByteVector )(implicit ord: ByteOrdering): ByteVector = ord match { case ByteOrdering.BigEndian => big case ByteOrdering.LittleEndian => little } + def getLength(length: Length)(implicit ord: ByteOrdering): Long = + length.toLong(signed = false, ord) + // format: off def block[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( implicit @@ -50,14 +53,14 @@ object Block { init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] ): Codec[Unit :: ByteVector :: LB] = - ("block_type" | constant(hexConstant)) :: - ("block_length" | bytes(4)).flatPrepend { length => + ("Block Type" | constant(hexConstant)) :: + ("Block Total Length" | bytes(4)).flatPrepend { length => f(length) :+ constant(length) } def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = block(hexConstant) { length => - ("block_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 12, bytes)) :: + ("Block Bytes" | fixedSizeBytes(getLength(length) - 12, bytes)) :: Codec.deriveHNil }.dropUnits // format: on diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 5c891221bd..0017079f81 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -22,17 +22,35 @@ package fs2.protocols package pcapng +import pcap._ import scodec.Codec import scodec.bits._ import scodec.codecs._ -case class EnhancedPacketBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock +case class EnhancedPacketBlock( + length: ByteVector, + interfaceId: Long, + timestampHigh: Long, + timestampLow: Long, + capturedPacketLength: Long, + originalPacketLength: Long, + bytes: ByteVector +) extends BodyBlock object EnhancedPacketBlock { - def hexConstant(implicit ord: ByteOrdering) = + def hexConstant(implicit ord: ByteOrdering): ByteVector = Block.orderDependent(hex"00000006", hex"06000000") + // format: off def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = - "EPB" | Block.ignoredBlock(hexConstant).as[EnhancedPacketBlock] + "EPB" | Block.block(hexConstant) { length => + ("Interface ID" | guint32) :: + ("Timestamp (High)" | guint32) :: + ("Timestamp (Low)" | guint32) :: + ("Captured Packet Length" | guint32) :: + ("Original Packet Length" | guint32) :: + fixedSizeBytes(Block.getLength(length) - 32, bytes) + }.dropUnits.as[EnhancedPacketBlock] + // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index d03cab0349..871e3f83bf 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -42,14 +42,14 @@ object SectionHeaderBlock { // format: off private def sectionHeader(implicit ord: ByteOrdering) = { - ("version_major" | guint16 ) :: - ("version_minor" | guint16 ) :: - ("remaining" | bytes ) + ("Major Version" | guint16 ) :: + ("Minor Version" | guint16 ) :: + ("Remaining" | bytes ) } val codec: Codec[SectionHeaderBlock] = "SHB" | Block.block(hexConstant) { length => - ("magic_number" | ByteOrderMagic).flatPrepend { implicit ord => - ("body_bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) + ("Byte-Order Magic" | ByteOrderMagic).flatPrepend { implicit ord => + ("Block Bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) }}.dropUnits.as[SectionHeaderBlock] // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index cc147cc5b6..8cbba7ddec 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -92,8 +92,9 @@ private object BlockTest { val enhancedHeader = hex"06000000" object Enhanced1 { val length = hex"5c010000" + val body = hex"0000000083ea03000d8a33353a0100003a010000" val other = - hex"""0000000083ea03000d8a33353a0100003a010000ffffffffffff000b + hex"""ffffffffffff000b 8201fc4208004500012ca8360000fa11178b00000000ffffffff004400430118 591f0101060000003d1d0000000000000000000000000000000000000000000b 8201fc4200000000000000000000000000000000000000000000000000000000 @@ -104,14 +105,15 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501013d0701000b8201fc423204 0000000037040103062aff000000000000000000""" - val expected = EnhancedPacketBlock(length, other) - val bytes = enhancedHeader ++ length ++ other ++ length + val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, other) + val bytes = enhancedHeader ++ length ++ body ++ other ++ length } object Enhanced2 { val length = hex"78010000" + val body = hex"0000000083ea0300348b33355601000056010000" val other = - hex"""0000000083ea0300348b33355601000056010000000b8201fc42000874adf19b + hex"""000b8201fc42000874adf19b 0800450001480445000080110000c0a80001c0a8000a00430044013422330201 060000003d1d0000000000000000c0a8000ac0a8000100000000000b8201fc42 0000000000000000000000000000000000000000000000000000000000000000 @@ -123,14 +125,15 @@ private object BlockTest { 00000000000000000000638253633501020104ffffff003a04000007083b0400 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 000000000000000000000000""" - val expected = EnhancedPacketBlock(length, other) - val bytes = enhancedHeader ++ length ++ other ++ length + val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, other) + val bytes = enhancedHeader ++ length ++ body ++ other ++ length } object Enhanced3 { val length = hex"5c010000" + val body = hex"0000000083ea03009c9b34353a0100003a010000" val other = - hex"""0000000083ea03009c9b34353a0100003a010000ffffffff + hex"""ffffffff ffff000b8201fc4208004500012ca8370000fa11178a00000000ffffffff0044 004301189fbd0101060000003d1e000000000000000000000000000000000000 0000000b8201fc42000000000000000000000000000000000000000000000000 @@ -141,14 +144,15 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000638253633501033d0701000b8201 fc423204c0a8000a3604c0a8000137040103062aff000000""" - val expected = EnhancedPacketBlock(length, other) - val bytes = enhancedHeader ++ length ++ other ++ length + val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, other) + val bytes = enhancedHeader ++ length ++ body ++ other ++ length } object Enhanced4 { val length = hex"78010000" + val body = hex"0000000083ea0300d69c34355601000056010000" val other = - hex"""0000000083ea0300d69c34355601000056010000000b8201fc420008 + hex"""000b8201fc420008 74adf19b0800450001480446000080110000c0a80001c0a8000a004300440134 dfdb0201060000003d1e0000000000000000c0a8000a0000000000000000000b 8201fc4200000000000000000000000000000000000000000000000000000000 @@ -160,7 +164,7 @@ private object BlockTest { 0000000000000000000000000000638253633501053a04000007083b0400000c 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 00000000000000000000000000000000""" - val expected = EnhancedPacketBlock(length, other) - val bytes = enhancedHeader ++ length ++ other ++ length + val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, other) + val bytes = enhancedHeader ++ length ++ body ++ other ++ length } } From afbd381c6de3b7cf331165637038037c20344435 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 02:50:45 +0300 Subject: [PATCH 05/20] pcapng - rename variables in test --- .../fs2/protocols/pcapng/BlockTest.scala | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 8cbba7ddec..371a0991c7 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -68,32 +68,30 @@ class BlockTest extends munit.FunSuite { private object BlockTest { object SHB { + val header = hex"0a0d0d0a" val length = hex"1c000000" - val other = hex"ffffffffffffffff" - val expected = SectionHeaderBlock(length, LittleEndian, 1, 0, other) - val bytes = { - val header = hex"0a0d0d0a" - val magic = hex"4d3c2b1a" - val major = hex"0100" - val minor = hex"0000" - - header ++ length ++ magic ++ major ++ minor ++ other ++ length - } + val parsed = hex"4d3c2b1a01000000" + val nonParsed = hex"ffffffffffffffff" + val bytes = header ++ length ++ parsed ++ nonParsed ++ length + + val expected = SectionHeaderBlock(length, LittleEndian, 1, 0, nonParsed) } object Interface { val header = hex"01000000" val length = hex"20000000" - val other = hex"01000000ffff0000090001000600000000000000" - val expected = InterfaceDescriptionBlock(length, other) - val bytes = header ++ length ++ other ++ length + val nonParsed = hex"01000000ffff0000090001000600000000000000" + val bytes = header ++ length ++ nonParsed ++ length + + val expected = InterfaceDescriptionBlock(length, nonParsed) } val enhancedHeader = hex"06000000" + object Enhanced1 { val length = hex"5c010000" - val body = hex"0000000083ea03000d8a33353a0100003a010000" - val other = + val parsed = hex"0000000083ea03000d8a33353a0100003a010000" + val nonParsed = hex"""ffffffffffff000b 8201fc4208004500012ca8360000fa11178b00000000ffffffff004400430118 591f0101060000003d1d0000000000000000000000000000000000000000000b @@ -105,14 +103,15 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501013d0701000b8201fc423204 0000000037040103062aff000000000000000000""" - val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, other) - val bytes = enhancedHeader ++ length ++ body ++ other ++ length + val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + + val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, nonParsed) } object Enhanced2 { val length = hex"78010000" - val body = hex"0000000083ea0300348b33355601000056010000" - val other = + val parsed = hex"0000000083ea0300348b33355601000056010000" + val nonParsed = hex"""000b8201fc42000874adf19b 0800450001480445000080110000c0a80001c0a8000a00430044013422330201 060000003d1d0000000000000000c0a8000ac0a8000100000000000b8201fc42 @@ -125,14 +124,15 @@ private object BlockTest { 00000000000000000000638253633501020104ffffff003a04000007083b0400 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 000000000000000000000000""" - val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, other) - val bytes = enhancedHeader ++ length ++ body ++ other ++ length + val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + + val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, nonParsed) } object Enhanced3 { val length = hex"5c010000" - val body = hex"0000000083ea03009c9b34353a0100003a010000" - val other = + val pared = hex"0000000083ea03009c9b34353a0100003a010000" + val nonParsed = hex"""ffffffff ffff000b8201fc4208004500012ca8370000fa11178a00000000ffffffff0044 004301189fbd0101060000003d1e000000000000000000000000000000000000 @@ -144,14 +144,15 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000638253633501033d0701000b8201 fc423204c0a8000a3604c0a8000137040103062aff000000""" - val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, other) - val bytes = enhancedHeader ++ length ++ body ++ other ++ length + val bytes = enhancedHeader ++ length ++ pared ++ nonParsed ++ length + + val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, nonParsed) } object Enhanced4 { val length = hex"78010000" - val body = hex"0000000083ea0300d69c34355601000056010000" - val other = + val parsed = hex"0000000083ea0300d69c34355601000056010000" + val nonParsed = hex"""000b8201fc420008 74adf19b0800450001480446000080110000c0a80001c0a8000a004300440134 dfdb0201060000003d1e0000000000000000c0a8000a0000000000000000000b @@ -164,7 +165,8 @@ private object BlockTest { 0000000000000000000000000000638253633501053a04000007083b0400000c 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 00000000000000000000000000000000""" - val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, other) - val bytes = enhancedHeader ++ length ++ body ++ other ++ length + val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + + val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, nonParsed) } } From 17b7d1b032cb367097d3dcdcd371ccb1c3a31325 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 19:18:54 +0300 Subject: [PATCH 06/20] pcapng - enhanced packet block --- .../pcapng/EnhancedPacketBlock.scala | 26 ++++++++++---- .../scala/fs2/protocols/PcapNgExample.scala | 2 +- .../fs2/protocols/pcapng/BlockTest.scala | 36 ++++++++++--------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 0017079f81..48f4d51cf8 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -34,7 +34,8 @@ case class EnhancedPacketBlock( timestampLow: Long, capturedPacketLength: Long, originalPacketLength: Long, - bytes: ByteVector + packetData: ByteVector, + options: ByteVector ) extends BodyBlock object EnhancedPacketBlock { @@ -42,15 +43,26 @@ object EnhancedPacketBlock { def hexConstant(implicit ord: ByteOrdering): ByteVector = Block.orderDependent(hex"00000006", hex"06000000") + private def padTo32Bits(length: Int) = { + val rem = length % 4 + if(rem == 0) 0 + else 4 - rem + } + // format: off + def other(implicit ord: ByteOrdering) = + ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => + ("Original Packet Length" | guint32 ) :: + ("Packet Data" | bytes(packetLength.toInt) ) :: + ("Packet padding" | ignore(padTo32Bits(packetLength.toInt)) ) :: + ("Options" | bytes )} + def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = "EPB" | Block.block(hexConstant) { length => - ("Interface ID" | guint32) :: - ("Timestamp (High)" | guint32) :: - ("Timestamp (Low)" | guint32) :: - ("Captured Packet Length" | guint32) :: - ("Original Packet Length" | guint32) :: - fixedSizeBytes(Block.getLength(length) - 32, bytes) + ("Interface ID" | guint32 ) :: + ("Timestamp (High)" | guint32 ) :: + ("Timestamp (Low)" | guint32 ) :: + ("Block bytes" | fixedSizeBytes(Block.getLength(length) - 24, other)) }.dropUnits.as[EnhancedPacketBlock] // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 04d47b3ff9..cea1ea5708 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -31,7 +31,7 @@ object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = Files[IO] - .readAll(Path("/Users/anikiforov/Downloads/dhcp.pcapng")) + .readAll(Path("/Users/anikiforov/http-ex.pcap")) .through(streamDecoder.toPipeByte) .debug() .compile diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 371a0991c7..8a157b32e8 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -91,7 +91,7 @@ private object BlockTest { object Enhanced1 { val length = hex"5c010000" val parsed = hex"0000000083ea03000d8a33353a0100003a010000" - val nonParsed = + val data = hex"""ffffffffffff000b 8201fc4208004500012ca8360000fa11178b00000000ffffffff004400430118 591f0101060000003d1d0000000000000000000000000000000000000000000b @@ -102,16 +102,17 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501013d0701000b8201fc423204 - 0000000037040103062aff000000000000000000""" - val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + 0000000037040103062aff00000000000000""" + val options = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, nonParsed) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, options) } object Enhanced2 { val length = hex"78010000" val parsed = hex"0000000083ea0300348b33355601000056010000" - val nonParsed = + val data = hex"""000b8201fc42000874adf19b 0800450001480445000080110000c0a80001c0a8000a00430044013422330201 060000003d1d0000000000000000c0a8000ac0a8000100000000000b8201fc42 @@ -123,16 +124,17 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 00000000000000000000638253633501020104ffffff003a04000007083b0400 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 - 000000000000000000000000""" - val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + 00000000000000000000""" + val options = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, nonParsed) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, options) } object Enhanced3 { val length = hex"5c010000" val pared = hex"0000000083ea03009c9b34353a0100003a010000" - val nonParsed = + val data = hex"""ffffffff ffff000b8201fc4208004500012ca8370000fa11178a00000000ffffffff0044 004301189fbd0101060000003d1e000000000000000000000000000000000000 @@ -143,16 +145,17 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000638253633501033d0701000b8201 - fc423204c0a8000a3604c0a8000137040103062aff000000""" - val bytes = enhancedHeader ++ length ++ pared ++ nonParsed ++ length + fc423204c0a8000a3604c0a8000137040103062aff00""" + val options = hex"0000" + val bytes = enhancedHeader ++ length ++ pared ++ data ++ options ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, nonParsed) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, options) } object Enhanced4 { val length = hex"78010000" val parsed = hex"0000000083ea0300d69c34355601000056010000" - val nonParsed = + val data = hex"""000b8201fc420008 74adf19b0800450001480446000080110000c0a80001c0a8000a004300440134 dfdb0201060000003d1e0000000000000000c0a8000a0000000000000000000b @@ -164,9 +167,10 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501053a04000007083b0400000c 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 - 00000000000000000000000000000000""" - val bytes = enhancedHeader ++ length ++ parsed ++ nonParsed ++ length + 0000000000000000000000000000""" + val options = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, nonParsed) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, options) } } From c0ef279051659d9f4f16a7d765daf88f29b12235 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 19:48:36 +0300 Subject: [PATCH 07/20] pcapng - EPB --- .../scala/fs2/protocols/pcapng/Block.scala | 10 +++---- .../pcapng/EnhancedPacketBlock.scala | 23 ++++++++-------- .../fs2/protocols/pcapng/BlockTest.scala | 26 +++++++++---------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index a44adb95a1..9cd9cf6c97 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -53,15 +53,13 @@ object Block { init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] ): Codec[Unit :: ByteVector :: LB] = - ("Block Type" | constant(hexConstant)) :: - ("Block Total Length" | bytes(4)).flatPrepend { length => - f(length) :+ constant(length) - } + ("Block Type" | constant(hexConstant) ) :: + ("Block Total Length" | bytes(4) ).flatPrepend { length => + ("Block dependendent" | f(length) :+ constant(length) )} def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = block(hexConstant) { length => - ("Block Bytes" | fixedSizeBytes(getLength(length) - 12, bytes)) :: - Codec.deriveHNil + ("Block Bytes" | fixedSizeBytes(getLength(length) - 12, bytes)) :: Codec.deriveHNil }.dropUnits // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 48f4d51cf8..a34b9d0855 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -22,6 +22,7 @@ package fs2.protocols package pcapng +import fs2.protocols.pcapng.Block.Length import pcap._ import scodec.Codec import scodec.bits._ @@ -43,6 +44,9 @@ object EnhancedPacketBlock { def hexConstant(implicit ord: ByteOrdering): ByteVector = Block.orderDependent(hex"00000006", hex"06000000") + private def optionLength(length: Length, packetLength: Int)(implicit ord: ByteOrdering): Int = + Block.getLength(length).toInt - 32 - packetLength - padTo32Bits(packetLength) * 8 + private def padTo32Bits(length: Int) = { val rem = length % 4 if(rem == 0) 0 @@ -50,19 +54,16 @@ object EnhancedPacketBlock { } // format: off - def other(implicit ord: ByteOrdering) = - ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => - ("Original Packet Length" | guint32 ) :: - ("Packet Data" | bytes(packetLength.toInt) ) :: - ("Packet padding" | ignore(padTo32Bits(packetLength.toInt)) ) :: - ("Options" | bytes )} - def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = "EPB" | Block.block(hexConstant) { length => - ("Interface ID" | guint32 ) :: - ("Timestamp (High)" | guint32 ) :: - ("Timestamp (Low)" | guint32 ) :: - ("Block bytes" | fixedSizeBytes(Block.getLength(length) - 24, other)) + ("Interface ID" | guint32 ) :: + ("Timestamp (High)" | guint32 ) :: + ("Timestamp (Low)" | guint32 ) :: + ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => + ("Original Packet Length" | guint32 ) :: + ("Packet Data" | bytes(packetLength.toInt) ) :: + ("Packet padding" | ignore(padTo32Bits(packetLength.toInt) * 8) ) :: + ("Options" | bytes(optionLength(length, packetLength.toInt)) )} }.dropUnits.as[EnhancedPacketBlock] // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 8a157b32e8..ceb6b0db5a 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -27,7 +27,7 @@ import scodec.DecodeResult import scodec.bits.ByteOrdering.LittleEndian import scodec.bits._ -// https://wiki.wireshark.org/Development/PcapNg dhcp.pcapng +// https://gitlab.com/wireshark/wireshark/-/wikis/Development/PcapNg dhcp.pcapng class BlockTest extends munit.FunSuite { import BlockTest._ @@ -103,10 +103,10 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501013d0701000b8201fc423204 0000000037040103062aff00000000000000""" - val options = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, options) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, ByteVector.empty) } object Enhanced2 { @@ -125,10 +125,10 @@ private object BlockTest { 00000000000000000000638253633501020104ffffff003a04000007083b0400 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 00000000000000000000""" - val options = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, options) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, ByteVector.empty) } object Enhanced3 { @@ -146,10 +146,10 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000638253633501033d0701000b8201 fc423204c0a8000a3604c0a8000137040103062aff00""" - val options = hex"0000" - val bytes = enhancedHeader ++ length ++ pared ++ data ++ options ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ pared ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, options) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, ByteVector.empty) } object Enhanced4 { @@ -168,9 +168,9 @@ private object BlockTest { 0000000000000000000000000000638253633501053a04000007083b0400000c 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 0000000000000000000000000000""" - val options = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ options ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, options) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, ByteVector.empty) } } From 34ce8e8519f7e5e6766def4866180b04d1dcb8f1 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 20:02:12 +0300 Subject: [PATCH 08/20] pcapng - added block types --- .../fs2/protocols/pcapng/BodyBlock.scala | 7 +++- .../pcapng/EnhancedPacketBlock.scala | 2 +- .../pcapng/InterfaceStatisticsBlock.scala | 38 +++++++++++++++++++ .../pcapng/NameResolutionBlock.scala | 38 +++++++++++++++++++ .../scala/fs2/protocols/PcapNgExample.scala | 2 +- .../fs2/protocols/pcapng/BlockTest.scala | 10 +++-- 6 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index 3ef026b3c0..31606fbe81 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -30,5 +30,10 @@ trait BodyBlock extends Block object BodyBlock { def decoder(implicit ord: ByteOrdering): Decoder[BodyBlock] = - Decoder.choiceDecoder(InterfaceDescriptionBlock.codec, EnhancedPacketBlock.codec) + Decoder.choiceDecoder( + InterfaceDescriptionBlock.codec, + EnhancedPacketBlock.codec, + NameResolutionBlock.codec, + InterfaceStatisticsBlock.codec + ) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index a34b9d0855..492a9da216 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -49,7 +49,7 @@ object EnhancedPacketBlock { private def padTo32Bits(length: Int) = { val rem = length % 4 - if(rem == 0) 0 + if (rem == 0) 0 else 4 - rem } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala new file mode 100644 index 0000000000..5931ed59e9 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.bits._ +import scodec.codecs._ +import scodec.Codec + +case class InterfaceStatisticsBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + +object InterfaceStatisticsBlock { + + private def hexConstant(implicit ord: ByteOrdering) = + Block.orderDependent(hex"00000005", hex"05000000") + + def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = + "ISB" | Block.ignoredBlock(hexConstant).as[InterfaceStatisticsBlock] +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala new file mode 100644 index 0000000000..92bccbfe8b --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.bits._ +import scodec.codecs._ +import scodec.Codec + +case class NameResolutionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock + +object NameResolutionBlock { + + private def hexConstant(implicit ord: ByteOrdering) = + Block.orderDependent(hex"00000004", hex"04000000") + + def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = + "NRB" | Block.ignoredBlock(hexConstant).as[NameResolutionBlock] +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index cea1ea5708..157928dc39 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -31,7 +31,7 @@ object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = Files[IO] - .readAll(Path("/Users/anikiforov/http-ex.pcap")) + .readAll(Path("/Users/anikiforov/Downloads/many_interfaces.pcapng")) .through(streamDecoder.toPipeByte) .debug() .compile diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index ceb6b0db5a..cad5eb2b1f 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -88,6 +88,8 @@ private object BlockTest { val enhancedHeader = hex"06000000" + val opts = ByteVector.empty + object Enhanced1 { val length = hex"5c010000" val parsed = hex"0000000083ea03000d8a33353a0100003a010000" @@ -106,7 +108,7 @@ private object BlockTest { val padding = hex"0000" val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, ByteVector.empty) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, opts) } object Enhanced2 { @@ -128,7 +130,7 @@ private object BlockTest { val padding = hex"0000" val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, ByteVector.empty) + val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, opts) } object Enhanced3 { @@ -149,7 +151,7 @@ private object BlockTest { val padding = hex"0000" val bytes = enhancedHeader ++ length ++ pared ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, ByteVector.empty) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, opts) } object Enhanced4 { @@ -171,6 +173,6 @@ private object BlockTest { val padding = hex"0000" val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, ByteVector.empty) + val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, opts) } } From 55b82a5b45a4f452da8e0c060731e09cb453e1f9 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Fri, 3 Dec 2021 21:47:00 +0300 Subject: [PATCH 09/20] pcapng - parse of real data --- .../scala/fs2/protocols/pcapng/Block.scala | 10 ++--- .../fs2/protocols/pcapng/BodyBlock.scala | 13 +++++- .../pcapng/EnhancedPacketBlock.scala | 2 +- .../pcapng/InterfaceDescriptionBlock.scala | 2 +- .../pcapng/InterfaceStatisticsBlock.scala | 2 +- .../pcapng/NameResolutionBlock.scala | 2 +- .../protocols/pcapng/SectionHeaderBlock.scala | 2 +- .../scala/fs2/protocols/PcapNgExample.scala | 28 ++++++++---- .../fs2/protocols/pcapng/DummyBlock.scala | 45 +++++++++++++++++++ 9 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index 9cd9cf6c97..ba01f23552 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -47,7 +47,7 @@ object Block { length.toLong(signed = false, ord) // format: off - def block[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( + def codec[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], @@ -55,11 +55,7 @@ object Block { ): Codec[Unit :: ByteVector :: LB] = ("Block Type" | constant(hexConstant) ) :: ("Block Total Length" | bytes(4) ).flatPrepend { length => - ("Block dependendent" | f(length) :+ constant(length) )} - - def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = - block(hexConstant) { length => - ("Block Bytes" | fixedSizeBytes(getLength(length) - 12, bytes)) :: Codec.deriveHNil - }.dropUnits + ("Block Bytes" | f(length) ) :+ + ("Block Total Length" | constant(length) )} // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index 31606fbe81..3d639607f1 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -22,13 +22,22 @@ package fs2.protocols package pcapng -import scodec.Decoder -import scodec.bits.ByteOrdering +import scodec.bits._ +import scodec.codecs._ +import scodec.{Codec, Decoder} +import shapeless.{::, HNil} trait BodyBlock extends Block object BodyBlock { + // format: off + def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Block.Length :: ByteVector :: HNil] = + Block.codec(hexConstant) { length => + ("Block Bytes" | fixedSizeBytes(Block.getLength(length) - 12, bytes)) :: Codec.deriveHNil + }.dropUnits + // format: on + def decoder(implicit ord: ByteOrdering): Decoder[BodyBlock] = Decoder.choiceDecoder( InterfaceDescriptionBlock.codec, diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 492a9da216..5154d10070 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -55,7 +55,7 @@ object EnhancedPacketBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = - "EPB" | Block.block(hexConstant) { length => + "EPB" | Block.codec(hexConstant) { length => ("Interface ID" | guint32 ) :: ("Timestamp (High)" | guint32 ) :: ("Timestamp (Low)" | guint32 ) :: diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index baaef180a2..6f5ba261b7 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -34,5 +34,5 @@ object InterfaceDescriptionBlock { Block.orderDependent(hex"00000001", hex"01000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | Block.ignoredBlock(hexConstant).as[InterfaceDescriptionBlock] + "IDB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceDescriptionBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala index 5931ed59e9..0c8f8109ad 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -34,5 +34,5 @@ object InterfaceStatisticsBlock { Block.orderDependent(hex"00000005", hex"05000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = - "ISB" | Block.ignoredBlock(hexConstant).as[InterfaceStatisticsBlock] + "ISB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceStatisticsBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala index 92bccbfe8b..4eb7e60d78 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -34,5 +34,5 @@ object NameResolutionBlock { Block.orderDependent(hex"00000004", hex"04000000") def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = - "NRB" | Block.ignoredBlock(hexConstant).as[NameResolutionBlock] + "NRB" | BodyBlock.ignoredBlock(hexConstant).as[NameResolutionBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 871e3f83bf..71e78031ae 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -47,7 +47,7 @@ object SectionHeaderBlock { ("Remaining" | bytes ) } - val codec: Codec[SectionHeaderBlock] = "SHB" | Block.block(hexConstant) { length => + val codec: Codec[SectionHeaderBlock] = "SHB" | Block.codec(hexConstant) { length => ("Byte-Order Magic" | ByteOrderMagic).flatPrepend { implicit ord => ("Block Bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) }}.dropUnits.as[SectionHeaderBlock] diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 157928dc39..fb430c4a0b 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -25,22 +25,34 @@ package protocols import cats.effect.{IO, IOApp} import fs2.interop.scodec.StreamDecoder import fs2.io.file.{Files, Path} -import fs2.protocols.pcapng.{BodyBlock, SectionHeaderBlock} +import fs2.protocols.pcapng.{BodyBlock, DummyBlock, SectionHeaderBlock} +import scodec.Decoder +import scodec.bits._ object PcapNgExample extends IOApp.Simple { + private val EpbHex = hex"06000000" + def run: IO[Unit] = - Files[IO] - .readAll(Path("/Users/anikiforov/Downloads/many_interfaces.pcapng")) + byteStream .through(streamDecoder.toPipeByte) + .flatMap { + case epb @ DummyBlock(EpbHex, _, _) => Stream.emit(epb) + case _ => Stream.empty + } .debug() .compile - .drain + .count + .flatMap(IO.println) + + private def byteStream: Stream[IO, Byte] = + Files[IO].readAll(Path("/Users/anikiforov/pcapng/http.pcap")) - private val streamDecoder: StreamDecoder[(SectionHeaderBlock, BodyBlock)] = + private val streamDecoder: StreamDecoder[BodyBlock] = for { - header <- StreamDecoder.once(SectionHeaderBlock.codec) - decoder = BodyBlock.decoder(header.ordering) + hdr <- StreamDecoder.once(SectionHeaderBlock.codec) + fallback = DummyBlock.codec(hdr.ordering) + decoder = Decoder.choiceDecoder(BodyBlock.decoder(hdr.ordering), fallback) block <- StreamDecoder.many(decoder) - } yield (header, block) + } yield block } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala new file mode 100644 index 0000000000..f1bfcbbfb1 --- /dev/null +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +case class DummyBlock( + blockType: ByteVector, + totalLength: Block.Length, + body: ByteVector +) extends BodyBlock + +object DummyBlock { + + // format: off + def codec(implicit ord: ByteOrdering): Codec[DummyBlock] = "Dummy" | { + ("Block Type" | bytes(4) ) :: + ("Block Total Length" | bytes(4) ).flatPrepend { length => + ("Block Body" | bytes(Block.getLength(length).toInt - 12) ) :: + ("Block Total Length" | constant(length) )} + }.dropUnits.as[DummyBlock] + // format: on +} From a0fbca7318fb7d4f52f684af55ef4b433a707600 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Sat, 4 Dec 2021 14:41:00 +0300 Subject: [PATCH 10/20] pcapng - pacakge object + real data --- .../scala/fs2/protocols/pcapng/Block.scala | 24 +-- .../fs2/protocols/pcapng/BodyBlock.scala | 4 +- .../pcapng/EnhancedPacketBlock.scala | 7 +- .../pcapng/InterfaceDescriptionBlock.scala | 4 +- .../pcapng/InterfaceStatisticsBlock.scala | 4 +- .../pcapng/NameResolutionBlock.scala | 4 +- .../protocols/pcapng/SectionHeaderBlock.scala | 4 +- .../scala/fs2/protocols/pcapng/package.scala | 40 +++++ .../fs2/protocols/pcapng/BlockTest.scala | 146 +++++++++++------- .../fs2/protocols/pcapng/DummyBlock.scala | 8 +- 10 files changed, 148 insertions(+), 97 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index ba01f23552..9ff5774136 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -32,30 +32,16 @@ trait Block object Block { - type Length = ByteVector - - def orderDependent[T]( - big: ByteVector, - little: ByteVector - )(implicit ord: ByteOrdering): ByteVector = - ord match { - case ByteOrdering.BigEndian => big - case ByteOrdering.LittleEndian => little - } - - def getLength(length: Length)(implicit ord: ByteOrdering): Long = - length.toLong(signed = false, ord) - // format: off def codec[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] - ): Codec[Unit :: ByteVector :: LB] = - ("Block Type" | constant(hexConstant) ) :: - ("Block Total Length" | bytes(4) ).flatPrepend { length => - ("Block Bytes" | f(length) ) :+ - ("Block Total Length" | constant(length) )} + ): Codec[Unit :: Length :: LB] = + ("Block Type" | constant(hexConstant) ) :: + ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => + ("Block Bytes" | f(length) ) :+ + ("Block Total Length" | constant(length.bv) )} // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index 3d639607f1..9f1a96c0ec 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -32,9 +32,9 @@ trait BodyBlock extends Block object BodyBlock { // format: off - def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Block.Length :: ByteVector :: HNil] = + def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = Block.codec(hexConstant) { length => - ("Block Bytes" | fixedSizeBytes(Block.getLength(length) - 12, bytes)) :: Codec.deriveHNil + ("Block Bytes" | fixedSizeBytes(length.toLong - 12, bytes)) :: Codec.deriveHNil }.dropUnits // format: on diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 5154d10070..1998dbc92d 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -22,14 +22,13 @@ package fs2.protocols package pcapng -import fs2.protocols.pcapng.Block.Length import pcap._ import scodec.Codec import scodec.bits._ import scodec.codecs._ case class EnhancedPacketBlock( - length: ByteVector, + length: Length, interfaceId: Long, timestampHigh: Long, timestampLow: Long, @@ -42,10 +41,10 @@ case class EnhancedPacketBlock( object EnhancedPacketBlock { def hexConstant(implicit ord: ByteOrdering): ByteVector = - Block.orderDependent(hex"00000006", hex"06000000") + orderDependent(hex"00000006", hex"06000000") private def optionLength(length: Length, packetLength: Int)(implicit ord: ByteOrdering): Int = - Block.getLength(length).toInt - 32 - packetLength - padTo32Bits(packetLength) * 8 + length.toLong.toInt - 32 - packetLength - padTo32Bits(packetLength) * 8 private def padTo32Bits(length: Int) = { val rem = length % 4 diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 6f5ba261b7..7844892a32 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -26,12 +26,12 @@ import scodec.bits._ import scodec.Codec import scodec.codecs._ -case class InterfaceDescriptionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock +case class InterfaceDescriptionBlock(length: Length, bytes: ByteVector) extends BodyBlock object InterfaceDescriptionBlock { private def hexConstant(implicit ord: ByteOrdering) = - Block.orderDependent(hex"00000001", hex"01000000") + orderDependent(hex"00000001", hex"01000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = "IDB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceDescriptionBlock] diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala index 0c8f8109ad..80a52dc7cf 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -26,12 +26,12 @@ import scodec.bits._ import scodec.codecs._ import scodec.Codec -case class InterfaceStatisticsBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock +case class InterfaceStatisticsBlock(length: Length, bytes: ByteVector) extends BodyBlock object InterfaceStatisticsBlock { private def hexConstant(implicit ord: ByteOrdering) = - Block.orderDependent(hex"00000005", hex"05000000") + orderDependent(hex"00000005", hex"05000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = "ISB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceStatisticsBlock] diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala index 4eb7e60d78..c7c781ca3c 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -26,12 +26,12 @@ import scodec.bits._ import scodec.codecs._ import scodec.Codec -case class NameResolutionBlock(length: ByteVector, bytes: ByteVector) extends BodyBlock +case class NameResolutionBlock(length: Length, bytes: ByteVector) extends BodyBlock object NameResolutionBlock { private def hexConstant(implicit ord: ByteOrdering) = - Block.orderDependent(hex"00000004", hex"04000000") + orderDependent(hex"00000004", hex"04000000") def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = "NRB" | BodyBlock.ignoredBlock(hexConstant).as[NameResolutionBlock] diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 71e78031ae..12e770a294 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -29,7 +29,7 @@ import scodec.codecs._ import scodec.codecs.fixedSizeBytes case class SectionHeaderBlock( - length: ByteVector, + length: Length, ordering: ByteOrdering, majorVersion: Int, minorVersion: Int, @@ -49,7 +49,7 @@ object SectionHeaderBlock { val codec: Codec[SectionHeaderBlock] = "SHB" | Block.codec(hexConstant) { length => ("Byte-Order Magic" | ByteOrderMagic).flatPrepend { implicit ord => - ("Block Bytes" | fixedSizeBytes(length.toInt(signed = false, ord) - 16, sectionHeader)) + ("Block Bytes" | fixedSizeBytes(length.toLong - 16, sectionHeader)) }}.dropUnits.as[SectionHeaderBlock] // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala new file mode 100644 index 0000000000..cc3d97c0a3 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols + +import scodec.bits.{ByteOrdering, ByteVector} + +package object pcapng { + + case class Length(bv: ByteVector) extends AnyVal { + def toLong(implicit ord: ByteOrdering): Long = bv.toLong(signed = false, ord) + } + + def orderDependent[T]( + big: ByteVector, + little: ByteVector + )(implicit ord: ByteOrdering): ByteVector = + ord match { + case ByteOrdering.BigEndian => big + case ByteOrdering.LittleEndian => little + } +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index cad5eb2b1f..9a05e1d1e4 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -27,74 +27,81 @@ import scodec.DecodeResult import scodec.bits.ByteOrdering.LittleEndian import scodec.bits._ -// https://gitlab.com/wireshark/wireshark/-/wikis/Development/PcapNg dhcp.pcapng class BlockTest extends munit.FunSuite { - import BlockTest._ + import BlockTest.DHCP._ + import BlockTest.{RealData => RD} - test("shb") { + test("dhcp shb") { val actual = SectionHeaderBlock.codec.decode(SHB.bytes.bits) assertEquals(actual, fullyDecoded(SHB.expected)) } - test("interface") { + test("dhcp idb") { val actual = InterfaceDescriptionBlock.codec(LittleEndian).decode(Interface.bytes.bits) assertEquals(actual, fullyDecoded(Interface.expected)) } - test("enhanced1") { + test("dhcp epb1") { val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced1.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced1.expected)) } - test("enhanced2") { + test("dhcp epb2") { val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced2.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced2.expected)) } - test("enhanced3") { + test("dhcp epb3") { val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced3.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced3.expected)) } - test("enhanced4") { + test("dhcp epb4") { val actual = EnhancedPacketBlock.codec(LittleEndian).decode(Enhanced4.bytes.bits) assertEquals(actual, fullyDecoded(Enhanced4.expected)) } + test("real-data dummy") { + val actual = DummyBlock.codec(LittleEndian).decode(RD.bytes.bits) + assertEquals(actual, fullyDecoded(RD.expectedDummy)) + } + private def fullyDecoded[V](v: V) = Successful(DecodeResult(v, BitVector.empty)) } private object BlockTest { - object SHB { - val header = hex"0a0d0d0a" - val length = hex"1c000000" - val parsed = hex"4d3c2b1a01000000" - val nonParsed = hex"ffffffffffffffff" - val bytes = header ++ length ++ parsed ++ nonParsed ++ length + // https://gitlab.com/wireshark/wireshark/-/wikis/Development/PcapNg dhcp.pcapng + object DHCP { + object SHB { + val header = hex"0a0d0d0a" + val length = hex"1c000000" + val parsed = hex"4d3c2b1a01000000" + val nonParsed = hex"ffffffffffffffff" + val bytes = header ++ length ++ parsed ++ nonParsed ++ length - val expected = SectionHeaderBlock(length, LittleEndian, 1, 0, nonParsed) - } + val expected = SectionHeaderBlock(Length(length), LittleEndian, 1, 0, nonParsed) + } - object Interface { - val header = hex"01000000" - val length = hex"20000000" - val nonParsed = hex"01000000ffff0000090001000600000000000000" - val bytes = header ++ length ++ nonParsed ++ length + object Interface { + val header = hex"01000000" + val length = hex"20000000" + val nonParsed = hex"01000000ffff0000090001000600000000000000" + val bytes = header ++ length ++ nonParsed ++ length - val expected = InterfaceDescriptionBlock(length, nonParsed) - } + val expected = InterfaceDescriptionBlock(Length(length), nonParsed) + } - val enhancedHeader = hex"06000000" + val enhancedHeader = hex"06000000" - val opts = ByteVector.empty + val opts = ByteVector.empty - object Enhanced1 { - val length = hex"5c010000" - val parsed = hex"0000000083ea03000d8a33353a0100003a010000" - val data = - hex"""ffffffffffff000b + object Enhanced1 { + val length = hex"5c010000" + val parsed = hex"0000000083ea03000d8a33353a0100003a010000" + val data = + hex"""ffffffffffff000b 8201fc4208004500012ca8360000fa11178b00000000ffffffff004400430118 591f0101060000003d1d0000000000000000000000000000000000000000000b 8201fc4200000000000000000000000000000000000000000000000000000000 @@ -105,17 +112,17 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000638253633501013d0701000b8201fc423204 0000000037040103062aff00000000000000""" - val padding = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570125, 314, 314, data, opts) - } + val expected = EnhancedPacketBlock(Length(length), 0, 256643, 892570125, 314, 314, data, opts) + } - object Enhanced2 { - val length = hex"78010000" - val parsed = hex"0000000083ea0300348b33355601000056010000" - val data = - hex"""000b8201fc42000874adf19b + object Enhanced2 { + val length = hex"78010000" + val parsed = hex"0000000083ea0300348b33355601000056010000" + val data = + hex"""000b8201fc42000874adf19b 0800450001480445000080110000c0a80001c0a8000a00430044013422330201 060000003d1d0000000000000000c0a8000ac0a8000100000000000b8201fc42 0000000000000000000000000000000000000000000000000000000000000000 @@ -127,17 +134,17 @@ private object BlockTest { 00000000000000000000638253633501020104ffffff003a04000007083b0400 000c4e330400000e103604c0a80001ff00000000000000000000000000000000 00000000000000000000""" - val padding = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892570420, 342, 342, data, opts) - } + val expected = EnhancedPacketBlock(Length(length), 0, 256643, 892570420, 342, 342, data, opts) + } - object Enhanced3 { - val length = hex"5c010000" - val pared = hex"0000000083ea03009c9b34353a0100003a010000" - val data = - hex"""ffffffff + object Enhanced3 { + val length = hex"5c010000" + val pared = hex"0000000083ea03009c9b34353a0100003a010000" + val data = + hex"""ffffffff ffff000b8201fc4208004500012ca8370000fa11178a00000000ffffffff0044 004301189fbd0101060000003d1e000000000000000000000000000000000000 0000000b8201fc42000000000000000000000000000000000000000000000000 @@ -148,17 +155,17 @@ private object BlockTest { 0000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000638253633501033d0701000b8201 fc423204c0a8000a3604c0a8000137040103062aff00""" - val padding = hex"0000" - val bytes = enhancedHeader ++ length ++ pared ++ data ++ padding ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ pared ++ data ++ padding ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640156, 314, 314, data, opts) - } + val expected = EnhancedPacketBlock(Length(length), 0, 256643, 892640156, 314, 314, data, opts) + } - object Enhanced4 { - val length = hex"78010000" - val parsed = hex"0000000083ea0300d69c34355601000056010000" - val data = - hex"""000b8201fc420008 + object Enhanced4 { + val length = hex"78010000" + val parsed = hex"0000000083ea0300d69c34355601000056010000" + val data = + hex"""000b8201fc420008 74adf19b0800450001480446000080110000c0a80001c0a8000a004300440134 dfdb0201060000003d1e0000000000000000c0a8000a0000000000000000000b 8201fc4200000000000000000000000000000000000000000000000000000000 @@ -170,9 +177,28 @@ private object BlockTest { 0000000000000000000000000000638253633501053a04000007083b0400000c 4e330400000e103604c0a800010104ffffff00ff000000000000000000000000 0000000000000000000000000000""" - val padding = hex"0000" - val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length + val padding = hex"0000" + val bytes = enhancedHeader ++ length ++ parsed ++ data ++ padding ++ length + + val expected = EnhancedPacketBlock(Length(length), 0, 256643, 892640470, 342, 342, data, opts) + } + } + + object RealData { + val header = hex"06000000" + val length = hex"88000000" + val data = + hex"""0000000003d205008418cd174200000042000000 + + a483e7e0b1ad0200c9690a01080045000034ead7400022067cb336f4a8700a01 + 07d40050cc0c19b4409fe64f4fb1801000953a4800000101080a2ff46c4b386d + 1949 + + 0000 + + 018004000400000002000400010000000280040000000000048004000800000000000000""" + val bytes = header ++ length ++ data ++ length - val expected = EnhancedPacketBlock(length, 0, 256643, 892640470, 342, 342, data, opts) + val expectedDummy = DummyBlock(header, Length(length), data) } } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala index f1bfcbbfb1..103ee1bcd5 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala @@ -28,7 +28,7 @@ import scodec.codecs._ case class DummyBlock( blockType: ByteVector, - totalLength: Block.Length, + totalLength: Length, body: ByteVector ) extends BodyBlock @@ -37,9 +37,9 @@ object DummyBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[DummyBlock] = "Dummy" | { ("Block Type" | bytes(4) ) :: - ("Block Total Length" | bytes(4) ).flatPrepend { length => - ("Block Body" | bytes(Block.getLength(length).toInt - 12) ) :: - ("Block Total Length" | constant(length) )} + ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => + ("Block Body" | bytes(length.toLong.toInt - 12) ) :: + ("Block Total Length" | constant(length.bv) )} }.dropUnits.as[DummyBlock] // format: on } From fc1cf570053a41c891a3612011e81a880f097904 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Sat, 4 Dec 2021 15:28:42 +0300 Subject: [PATCH 11/20] pcapng - style tweaks --- .../src/main/scala/fs2/protocols/pcapng/Block.scala | 2 +- .../fs2/protocols/pcapng/SectionHeaderBlock.scala | 12 ++++-------- .../test/scala/fs2/protocols/pcapng/BlockTest.scala | 4 +--- .../test/scala/fs2/protocols/pcapng/DummyBlock.scala | 4 ++-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala index 9ff5774136..4d2f1195b0 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala @@ -40,7 +40,7 @@ object Block { last: Last.Aux[LB, Unit] ): Codec[Unit :: Length :: LB] = ("Block Type" | constant(hexConstant) ) :: - ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => + ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ ("Block Total Length" | constant(length.bv) )} // format: on diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 12e770a294..77919175b8 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -41,15 +41,11 @@ object SectionHeaderBlock { private val hexConstant = hex"0A0D0D0A" // format: off - private def sectionHeader(implicit ord: ByteOrdering) = { - ("Major Version" | guint16 ) :: - ("Minor Version" | guint16 ) :: - ("Remaining" | bytes ) - } - val codec: Codec[SectionHeaderBlock] = "SHB" | Block.codec(hexConstant) { length => - ("Byte-Order Magic" | ByteOrderMagic).flatPrepend { implicit ord => - ("Block Bytes" | fixedSizeBytes(length.toLong - 16, sectionHeader)) + ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit ord => + ("Major Version" | guint16 ) :: + ("Minor Version" | guint16 ) :: + ("Block Bytes" | bytes(length.toLong.toInt - 20) ) }}.dropUnits.as[SectionHeaderBlock] // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 9a05e1d1e4..7470b1a85c 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -192,9 +192,7 @@ private object BlockTest { a483e7e0b1ad0200c9690a01080045000034ead7400022067cb336f4a8700a01 07d40050cc0c19b4409fe64f4fb1801000953a4800000101080a2ff46c4b386d - 1949 - - 0000 + 1949 0000 018004000400000002000400010000000280040000000000048004000800000000000000""" val bytes = header ++ length ++ data ++ length diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala index 103ee1bcd5..ea387642f1 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala @@ -36,8 +36,8 @@ object DummyBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[DummyBlock] = "Dummy" | { - ("Block Type" | bytes(4) ) :: - ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => + ("Block Type" | bytes(4) ) :: + ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => ("Block Body" | bytes(length.toLong.toInt - 12) ) :: ("Block Total Length" | constant(length.bv) )} }.dropUnits.as[DummyBlock] From 48c4407f7a49b544fb27f0013bffe322708dec2c Mon Sep 17 00:00:00 2001 From: nikiforo Date: Sat, 4 Dec 2021 15:40:05 +0300 Subject: [PATCH 12/20] pcapng - real-data --- .../pcapng/EnhancedPacketBlock.scala | 2 +- .../fs2/protocols/pcapng/BlockTest.scala | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 1998dbc92d..d69fb389fa 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -44,7 +44,7 @@ object EnhancedPacketBlock { orderDependent(hex"00000006", hex"06000000") private def optionLength(length: Length, packetLength: Int)(implicit ord: ByteOrdering): Int = - length.toLong.toInt - 32 - packetLength - padTo32Bits(packetLength) * 8 + length.toLong.toInt - 32 - packetLength - padTo32Bits(packetLength) private def padTo32Bits(length: Int) = { val rem = length % 4 diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 7470b1a85c..0b2c158935 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -66,6 +66,11 @@ class BlockTest extends munit.FunSuite { assertEquals(actual, fullyDecoded(RD.expectedDummy)) } + test("real-data epb") { + val actual = EnhancedPacketBlock.codec(LittleEndian).decode(RD.bytes.bits) + assertEquals(actual, fullyDecoded(RD.expectedEPB)) + } + private def fullyDecoded[V](v: V) = Successful(DecodeResult(v, BitVector.empty)) } @@ -186,17 +191,17 @@ private object BlockTest { object RealData { val header = hex"06000000" - val length = hex"88000000" + val length = Length(hex"88000000") + val props = hex"0000000003d205008418cd174200000042000000" val data = - hex"""0000000003d205008418cd174200000042000000 - - a483e7e0b1ad0200c9690a01080045000034ead7400022067cb336f4a8700a01 + hex"""a483e7e0b1ad0200c9690a01080045000034ead7400022067cb336f4a8700a01 07d40050cc0c19b4409fe64f4fb1801000953a4800000101080a2ff46c4b386d - 1949 0000 - - 018004000400000002000400010000000280040000000000048004000800000000000000""" - val bytes = header ++ length ++ data ++ length + 1949""" + val padding = hex"0000" + val opts = hex"018004000400000002000400010000000280040000000000048004000800000000000000" + val bytes = header ++ length.bv ++ props ++ data ++ padding ++ opts ++ length.bv - val expectedDummy = DummyBlock(header, Length(length), data) + val expectedEPB = EnhancedPacketBlock(length, 0, 381443, 399317124, 66, 66, data, opts) + val expectedDummy = DummyBlock(header, length, props ++ data ++ padding ++ opts) } } From 5a4a7f5d7fb253209a84d51e95c13da59d9b3a22 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Sat, 4 Dec 2021 16:38:15 +0300 Subject: [PATCH 13/20] pcapng - PIB --- .../fs2/protocols/pcapng/BodyBlock.scala | 3 +- .../pcapng/ProcessInformationBlock.scala | 38 +++++++++++++++++++ .../scala/fs2/protocols/PcapNgExample.scala | 23 ++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index 9f1a96c0ec..fc24aa2c2e 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -43,6 +43,7 @@ object BodyBlock { InterfaceDescriptionBlock.codec, EnhancedPacketBlock.codec, NameResolutionBlock.codec, - InterfaceStatisticsBlock.codec + InterfaceStatisticsBlock.codec, + ProcessInformationBlock.codec ) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala new file mode 100644 index 0000000000..4bc7c864e7 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +case class ProcessInformationBlock(length: Length, bytes: ByteVector) extends BodyBlock + +object ProcessInformationBlock { + + private def hexConstant(implicit ord: ByteOrdering) = + orderDependent(hex"80000001", hex"01000080") + + def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = + "PIB" | BodyBlock.ignoredBlock(hexConstant).as[ProcessInformationBlock] +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index fb430c4a0b..5e85afb81f 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -27,26 +27,29 @@ import fs2.interop.scodec.StreamDecoder import fs2.io.file.{Files, Path} import fs2.protocols.pcapng.{BodyBlock, DummyBlock, SectionHeaderBlock} import scodec.Decoder -import scodec.bits._ object PcapNgExample extends IOApp.Simple { - private val EpbHex = hex"06000000" - def run: IO[Unit] = + revealFailed + .compile + .count + .flatMap(IO.println) + + private def byteStream: Stream[IO, Byte] = + Files[IO].readAll(Path("/Users/anikiforov/pcapng/http-ex.pcap")) + + private def revealFailed = byteStream .through(streamDecoder.toPipeByte) .flatMap { - case epb @ DummyBlock(EpbHex, _, _) => Stream.emit(epb) - case _ => Stream.empty + case dummy: DummyBlock => Stream.emit(dummy) + case _ => Stream.empty } .debug() - .compile - .count - .flatMap(IO.println) - private def byteStream: Stream[IO, Byte] = - Files[IO].readAll(Path("/Users/anikiforov/pcapng/http.pcap")) + private def decode = + byteStream.through(streamDecoder.toPipeByte) private val streamDecoder: StreamDecoder[BodyBlock] = for { From b23ff23ced3065543b0d112e2db91bb7accd5ce8 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Mon, 6 Dec 2021 17:47:24 +0300 Subject: [PATCH 14/20] pcapng - scala 2/3 compilation --- .../fs2/protocols/pcapng/Block.scala | 5 ++ protocols/shared/src/main/scala-3/Block.scala | 47 +++++++++++++++++++ .../fs2/protocols/pcapng/BodyBlock.scala | 11 +---- .../pcapng/InterfaceDescriptionBlock.scala | 2 +- .../pcapng/InterfaceStatisticsBlock.scala | 2 +- .../pcapng/NameResolutionBlock.scala | 2 +- .../pcapng/ProcessInformationBlock.scala | 2 +- .../protocols/pcapng/SectionHeaderBlock.scala | 1 - 8 files changed, 57 insertions(+), 15 deletions(-) rename protocols/shared/src/main/{scala => scala-2}/fs2/protocols/pcapng/Block.scala (88%) create mode 100644 protocols/shared/src/main/scala-3/Block.scala diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala similarity index 88% rename from protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala rename to protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala index 4d2f1195b0..b18a1b1df6 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala @@ -43,5 +43,10 @@ object Block { ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ ("Block Total Length" | constant(length.bv) )} + + def ignoredCodec(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = + Block.codec(hexConstant) { length => + ("Block Bytes" | fixedSizeBytes(length.toLong - 12, bytes)) :: Codec.deriveHNil + }.dropUnits // format: on } diff --git a/protocols/shared/src/main/scala-3/Block.scala b/protocols/shared/src/main/scala-3/Block.scala new file mode 100644 index 0000000000..739772ee60 --- /dev/null +++ b/protocols/shared/src/main/scala-3/Block.scala @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +trait Block + +object Block { + + // format: off + inline def codec[L <: Tuple]( + hexConstant: ByteVector + )(f: Length => Codec[L]): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = + ("Block Type" | constant(hexConstant) ) :: + ("Block Total Length" | bytes(4).xmapc(Length(_))(_.bv) ).flatPrepend { length => + ("Block Bytes" | f(length) ) :+ + ("Block Total Length" | constant(length.bv) )} + + def ignoredCodec(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length *: ByteVector *: EmptyTuple] = + codec(hexConstant) { length => + ("Block Bytes" | (fixedSizeBytes(length.toLong - 12, bytes))).tuple + }.dropUnits + // format: on +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index fc24aa2c2e..61fe256fa7 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -23,21 +23,12 @@ package fs2.protocols package pcapng import scodec.bits._ -import scodec.codecs._ -import scodec.{Codec, Decoder} -import shapeless.{::, HNil} +import scodec.Decoder trait BodyBlock extends Block object BodyBlock { - // format: off - def ignoredBlock(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = - Block.codec(hexConstant) { length => - ("Block Bytes" | fixedSizeBytes(length.toLong - 12, bytes)) :: Codec.deriveHNil - }.dropUnits - // format: on - def decoder(implicit ord: ByteOrdering): Decoder[BodyBlock] = Decoder.choiceDecoder( InterfaceDescriptionBlock.codec, diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 7844892a32..8ea113079a 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -34,5 +34,5 @@ object InterfaceDescriptionBlock { orderDependent(hex"00000001", hex"01000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceDescriptionBlock] + "IDB" | Block.ignoredCodec(hexConstant).as[InterfaceDescriptionBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala index 80a52dc7cf..135151bcaa 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -34,5 +34,5 @@ object InterfaceStatisticsBlock { orderDependent(hex"00000005", hex"05000000") def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = - "ISB" | BodyBlock.ignoredBlock(hexConstant).as[InterfaceStatisticsBlock] + "ISB" | Block.ignoredCodec(hexConstant).as[InterfaceStatisticsBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala index c7c781ca3c..75a2e59b13 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -34,5 +34,5 @@ object NameResolutionBlock { orderDependent(hex"00000004", hex"04000000") def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = - "NRB" | BodyBlock.ignoredBlock(hexConstant).as[NameResolutionBlock] + "NRB" | Block.ignoredCodec(hexConstant).as[NameResolutionBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala index 4bc7c864e7..37550b8b8e 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala @@ -34,5 +34,5 @@ object ProcessInformationBlock { orderDependent(hex"80000001", hex"01000080") def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = - "PIB" | BodyBlock.ignoredBlock(hexConstant).as[ProcessInformationBlock] + "PIB" | Block.ignoredCodec(hexConstant).as[ProcessInformationBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 77919175b8..7e4f5d9f33 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -26,7 +26,6 @@ import pcap._ import scodec.Codec import scodec.bits._ import scodec.codecs._ -import scodec.codecs.fixedSizeBytes case class SectionHeaderBlock( length: Length, From 9bbd722785bcfd67c2967c57504bf22622c700a1 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Tue, 7 Dec 2021 20:22:18 +0300 Subject: [PATCH 15/20] pcapng - scalafmt - turn scala3 dialect for scala-3 folder --- .scalafmt.conf | 5 +++++ .../src/test/scala/fs2/protocols/PcapNgExample.scala | 7 ++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index f8ac3945a9..90a6abaab8 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -3,6 +3,11 @@ version = "3.5.8" style = default runner.dialect = scala213source3 +fileOverride { + "glob:**/src/main/scala-3/**" { + runner.dialect = scala3 + } +} project.excludeFilters = [ "scalafix/*" diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 5e85afb81f..2cbffaeb84 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -31,13 +31,10 @@ import scodec.Decoder object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = - revealFailed - .compile - .count - .flatMap(IO.println) + decode.compile.count.flatMap(IO.println) private def byteStream: Stream[IO, Byte] = - Files[IO].readAll(Path("/Users/anikiforov/pcapng/http-ex.pcap")) + Files[IO].readAll(Path("/Users/anikiforov/pcapng/many_interfaces.pcapng")) private def revealFailed = byteStream From d9d24ed42c2a834d297c485fb7d1c2b66a30a329 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Tue, 7 Dec 2021 21:01:26 +0300 Subject: [PATCH 16/20] pcapng - granulate IDB --- .../scala/fs2/protocols/pcap/LinkType.scala | 9 +++++++-- .../pcapng/InterfaceDescriptionBlock.scala | 19 ++++++++++++++++--- .../scala/fs2/protocols/PcapNgExample.scala | 3 ++- .../fs2/protocols/pcapng/BlockTest.scala | 9 +++++---- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala index 04ecd741a6..b6a14a6aed 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala @@ -41,14 +41,17 @@ object LinkType { case object MPEG2TS extends LinkType case class Unknown(value: Long) extends LinkType - def toLong(lt: LinkType): Long = lt match { + def fromInt(l: Int): LinkType = + fromLong(l) + + def toInt(lt: LinkType): Int = lt match { case Null => 0 case Ethernet => 1 case Raw => 101 case IPv4 => 228 case IPv6 => 229 case MPEG2TS => 243 - case Unknown(value) => value + case Unknown(value) => value.toInt // discuss? } def fromLong(l: Long): LinkType = l match { @@ -61,6 +64,8 @@ object LinkType { case other => Unknown(other) } + def toLong(lt: LinkType): Long = toInt(lt) + implicit def codec(implicit bo: ByteOrdering): Codec[LinkType] = guint32.xmap[LinkType](fromLong, toLong) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 8ea113079a..2224082f60 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -22,17 +22,30 @@ package fs2.protocols package pcapng -import scodec.bits._ +import pcap._ import scodec.Codec +import scodec.bits._ import scodec.codecs._ -case class InterfaceDescriptionBlock(length: Length, bytes: ByteVector) extends BodyBlock +case class InterfaceDescriptionBlock( + length: Length, + linkType: LinkType, + snapLen: Long, + bytes: ByteVector +) extends BodyBlock object InterfaceDescriptionBlock { private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000001", hex"01000000") + // format: off def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | Block.ignoredCodec(hexConstant).as[InterfaceDescriptionBlock] + "IDB" | Block.codec(hexConstant) { length => + ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt)) :: + ("Reserved" | ignore(16)) :: + ("SnapLen" | guint32) :: + ("Block Bytes" | bytes(length.toLong.toInt - 20)) + }.dropUnits.as[InterfaceDescriptionBlock] + // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 2cbffaeb84..78279afb3d 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -27,11 +27,12 @@ import fs2.interop.scodec.StreamDecoder import fs2.io.file.{Files, Path} import fs2.protocols.pcapng.{BodyBlock, DummyBlock, SectionHeaderBlock} import scodec.Decoder +import cats.syntax.all._ object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = - decode.compile.count.flatMap(IO.println) + decode.compile.toList.flatMap(_.traverse_(IO.println)) private def byteStream: Stream[IO, Byte] = Files[IO].readAll(Path("/Users/anikiforov/pcapng/many_interfaces.pcapng")) diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 0b2c158935..a8d09c6bee 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -22,6 +22,7 @@ package fs2.protocols package pcapng +import fs2.protocols.pcap.LinkType.Ethernet import scodec.Attempt.Successful import scodec.DecodeResult import scodec.bits.ByteOrdering.LittleEndian @@ -92,10 +93,10 @@ private object BlockTest { object Interface { val header = hex"01000000" val length = hex"20000000" - val nonParsed = hex"01000000ffff0000090001000600000000000000" - val bytes = header ++ length ++ nonParsed ++ length - - val expected = InterfaceDescriptionBlock(Length(length), nonParsed) + val parsed = hex"01000000ffff0000" + val nonParsed = hex"090001000600000000000000" + val bytes = header ++ length ++ parsed ++ nonParsed ++ length + val expected = InterfaceDescriptionBlock(Length(length), Ethernet, 65535, nonParsed) } val enhancedHeader = hex"06000000" From 988ca77a84d00a8f51e6079c51ac0f3bccbb0b61 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Thu, 9 Dec 2021 23:43:07 +0300 Subject: [PATCH 17/20] pcapng - review and remove some code comments --- .gitignore | 1 - .scalafmt.conf | 5 -- .../scala/fs2/protocols/pcap/LinkType.scala | 2 +- .../fs2/protocols/pcapng/CaptureFile.scala | 61 +++++++++++++++++++ .../pcapng/InterfaceDescriptionBlock.scala | 19 ++++-- .../protocols/pcapng/SimplePacketBlock.scala | 37 +++++++++++ .../scala/fs2/protocols/PcapNgExample.scala | 29 +++++---- 7 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala diff --git a/.gitignore b/.gitignore index 5bad453f18..596ac4fd40 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,4 @@ project/metals.sbt project/project .vscode .bsp -.java-version node_modules diff --git a/.scalafmt.conf b/.scalafmt.conf index 90a6abaab8..f8ac3945a9 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -3,11 +3,6 @@ version = "3.5.8" style = default runner.dialect = scala213source3 -fileOverride { - "glob:**/src/main/scala-3/**" { - runner.dialect = scala3 - } -} project.excludeFilters = [ "scalafix/*" diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala index b6a14a6aed..9897e31307 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala @@ -51,7 +51,7 @@ object LinkType { case IPv4 => 228 case IPv6 => 229 case MPEG2TS => 243 - case Unknown(value) => value.toInt // discuss? + case Unknown(value) => value.toInt } def fromLong(l: Long): LinkType = l match { diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala new file mode 100644 index 0000000000..8a60d7ef37 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols.pcapng + +import cats.effect.MonadCancelThrow +import fs2.interop.scodec.StreamDecoder +import fs2.protocols.pcap.LinkType +import fs2.timeseries.TimeStamped +import fs2.{Pipe, Pull, Stream} +import scodec.bits.ByteVector + +object CaptureFile { + + val streamDecoder: StreamDecoder[BodyBlock] = + StreamDecoder.once(SectionHeaderBlock.codec).flatMap { shb => + StreamDecoder.many(BodyBlock.decoder(shb.ordering)) + } + + def parse[F[_]: MonadCancelThrow, A]( + f: (LinkType, ByteVector) => Option[A] + ): Pipe[F, Byte, TimeStamped[A]] = { bytes => + def go( + idbs: Vector[InterfaceDescriptionBlock], + blocks: Stream[F, BodyBlock] + ): Pull[F, TimeStamped[A], Unit] = + blocks.pull.uncons1.flatMap { + case Some((idb: InterfaceDescriptionBlock, tail)) => + go(idbs :+ idb, tail) + case Some((epb: EnhancedPacketBlock, tail)) => + val idb = idbs(epb.interfaceId.toInt) + val ts = ((epb.timestampHigh << 32) | epb.timestampLow) * idb.if_tsresol + val timeStamped = f(idb.linkType, epb.packetData).map(TimeStamped(ts, _)) + Pull.outputOption1(timeStamped) >> go(idbs, tail) + case Some((_, tail)) => go(idbs, tail) + case None => Pull.done + } + + bytes + .through(streamDecoder.toPipeByte) + .through(go(Vector.empty, _).stream) + } +} diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 2224082f60..c18f3c24d1 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -22,17 +22,24 @@ package fs2.protocols package pcapng -import pcap._ +import fs2.protocols.pcap._ import scodec.Codec import scodec.bits._ import scodec.codecs._ +import scala.concurrent.duration._ + case class InterfaceDescriptionBlock( length: Length, linkType: LinkType, snapLen: Long, bytes: ByteVector -) extends BodyBlock +) extends BodyBlock { + + // Should be extracted from options. Instead, we currently assume + // that required option wasn't found and fallback to a default value. + def if_tsresol: FiniteDuration = 1.microsecond +} object InterfaceDescriptionBlock { @@ -42,10 +49,10 @@ object InterfaceDescriptionBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = "IDB" | Block.codec(hexConstant) { length => - ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt)) :: - ("Reserved" | ignore(16)) :: - ("SnapLen" | guint32) :: - ("Block Bytes" | bytes(length.toLong.toInt - 20)) + ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt) ) :: + ("Reserved" | ignore(16) ) :: + ("SnapLen" | guint32 ) :: + ("Block Bytes" | bytes(length.toLong.toInt - 20) ) }.dropUnits.as[InterfaceDescriptionBlock] // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala new file mode 100644 index 0000000000..976d6fd2a6 --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols.pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +case class SimplePacketBlock(length: Length, bytes: ByteVector) extends BodyBlock + +object SimplePacketBlock { + + private def hexConstant(implicit ord: ByteOrdering) = + orderDependent(hex"00000003", hex"03000000") + + def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = + "SPB" | Block.ignoredCodec(hexConstant).as[ProcessInformationBlock] +} diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 78279afb3d..1a2b442546 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -23,23 +23,22 @@ package fs2 package protocols import cats.effect.{IO, IOApp} -import fs2.interop.scodec.StreamDecoder -import fs2.io.file.{Files, Path} -import fs2.protocols.pcapng.{BodyBlock, DummyBlock, SectionHeaderBlock} -import scodec.Decoder import cats.syntax.all._ +import fs2.io.file.{Files, Path} +import fs2.protocols.pcap.LinkType +import fs2.protocols.pcapng.{CaptureFile, DummyBlock} object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = - decode.compile.toList.flatMap(_.traverse_(IO.println)) + output.compile.toList.flatMap(x => IO.println(x.length)) // _.traverse_(IO.println) private def byteStream: Stream[IO, Byte] = Files[IO].readAll(Path("/Users/anikiforov/pcapng/many_interfaces.pcapng")) private def revealFailed = byteStream - .through(streamDecoder.toPipeByte) + .through(CaptureFile.streamDecoder.toPipeByte) .flatMap { case dummy: DummyBlock => Stream.emit(dummy) case _ => Stream.empty @@ -47,13 +46,13 @@ object PcapNgExample extends IOApp.Simple { .debug() private def decode = - byteStream.through(streamDecoder.toPipeByte) - - private val streamDecoder: StreamDecoder[BodyBlock] = - for { - hdr <- StreamDecoder.once(SectionHeaderBlock.codec) - fallback = DummyBlock.codec(hdr.ordering) - decoder = Decoder.choiceDecoder(BodyBlock.decoder(hdr.ordering), fallback) - block <- StreamDecoder.many(decoder) - } yield block + byteStream.through(CaptureFile.streamDecoder.toPipeByte) + + private def output = + byteStream.through( + CaptureFile.parse { + case (LinkType.Ethernet, bv) => bv.some + case _ => none + } + ) } From bbb70f910eec9495e0fb1d73db2d5292f70ca5c5 Mon Sep 17 00:00:00 2001 From: nikiforo Date: Tue, 23 Aug 2022 20:07:25 +0400 Subject: [PATCH 18/20] pcapng - add Unrecognized block --- .../scala-2/fs2/protocols/pcapng/Block.scala | 38 +++++++++++++++---- protocols/shared/src/main/scala-3/Block.scala | 35 +++++++++++++---- .../fs2/protocols/pcapng/BodyBlock.scala | 3 +- .../fs2/protocols/pcapng/CaptureFile.scala | 7 ++-- .../pcapng/EnhancedPacketBlock.scala | 34 ++++++++--------- .../pcapng/InterfaceDescriptionBlock.scala | 19 +++++----- .../pcapng/InterfaceStatisticsBlock.scala | 6 +-- .../pcapng/NameResolutionBlock.scala | 6 +-- .../pcapng/ProcessInformationBlock.scala | 6 +-- .../protocols/pcapng/SectionHeaderBlock.scala | 13 ++++--- .../protocols/pcapng/SimplePacketBlock.scala | 6 +-- .../protocols/pcapng/UnrecognizedBlock.scala | 13 +++++++ 12 files changed, 121 insertions(+), 65 deletions(-) create mode 100644 protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala diff --git a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala index b18a1b1df6..9a6bb2aaaf 100644 --- a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala @@ -33,20 +33,44 @@ trait Block object Block { // format: off - def codec[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( + private def codec[A, L <: HList, LB <: HList](blockType: Codec[A])(f: Length => Codec[L])( implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] - ): Codec[Unit :: Length :: LB] = - ("Block Type" | constant(hexConstant) ) :: + ): Codec[A :: Length :: LB] = + ("Block Type" | blockType ) :: ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ ("Block Total Length" | constant(length.bv) )} + // format: on - def ignoredCodec(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = - Block.codec(hexConstant) { length => - ("Block Bytes" | fixedSizeBytes(length.toLong - 12, bytes)) :: Codec.deriveHNil + def codecByHex[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( + implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit] + ): Codec[Unit :: Length :: LB] = codec(constant(hexConstant))(f) + + def codecByLength[L <: HList, LB <: HList](hexConstant: ByteVector, c: Codec[L])( + implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit], + ord: ByteOrdering, + ): Codec[Unit :: Length :: LB] = codecByHex(hexConstant)(length => fixedSizeBytes(length.toLong - 12, c)) + + def codecIgnored( + hexConstant: ByteVector + )(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = + codecByHex(hexConstant) { length => + fixedSizeBytes(length.toLong - 12, bytes) :: Codec.deriveHNil + }.dropUnits + + def codecUnrecognized( + implicit ord: ByteOrdering + ): Codec[ByteVector :: Length :: ByteVector :: HNil] = + codec(bytes(4)) { length => + fixedSizeBytes(length.toLong - 12, bytes) :: Codec.deriveHNil }.dropUnits - // format: on } diff --git a/protocols/shared/src/main/scala-3/Block.scala b/protocols/shared/src/main/scala-3/Block.scala index 739772ee60..c7a3a3309a 100644 --- a/protocols/shared/src/main/scala-3/Block.scala +++ b/protocols/shared/src/main/scala-3/Block.scala @@ -31,17 +31,36 @@ trait Block object Block { // format: off - inline def codec[L <: Tuple]( - hexConstant: ByteVector - )(f: Length => Codec[L]): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = - ("Block Type" | constant(hexConstant) ) :: + inline def codec[A, L <: Tuple]( + blockType: Codec[A] + )(f: Length => Codec[L]): Codec[Tuple.Concat[A *: Length *: L, Unit *: EmptyTuple]] = + ("Block Type" | blockType ) :: ("Block Total Length" | bytes(4).xmapc(Length(_))(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ ("Block Total Length" | constant(length.bv) )} + // format: on + + inline def codecByHex[L <: Tuple]( + hexConstant: ByteVector + )(f: Length => Codec[L]): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = + codec(constant(hexConstant))(f) - def ignoredCodec(hexConstant: ByteVector)(implicit ord: ByteOrdering): Codec[Length *: ByteVector *: EmptyTuple] = - codec(hexConstant) { length => - ("Block Bytes" | (fixedSizeBytes(length.toLong - 12, bytes))).tuple + inline def codecByLength[L <: Tuple](hexConstant: ByteVector, c: Codec[L])( + implicit ord: ByteOrdering, + ): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = + codecByHex(hexConstant)(length => fixedSizeBytes(length.toLong - 12, c)) + + def codecIgnored( + hexConstant: ByteVector + )(implicit ord: ByteOrdering): Codec[Length *: ByteVector *: EmptyTuple] = + codecByHex(hexConstant) { length => + fixedSizeBytes(length.toLong - 12, bytes).tuple + }.dropUnits + + def codecUnrecognized( + implicit ord: ByteOrdering + ): Codec[ByteVector *: Length *: ByteVector *: EmptyTuple] = + codec(bytes(4)) { length => + fixedSizeBytes(length.toLong - 12, bytes).tuple }.dropUnits - // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index 61fe256fa7..a6c321fc11 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -35,6 +35,7 @@ object BodyBlock { EnhancedPacketBlock.codec, NameResolutionBlock.codec, InterfaceStatisticsBlock.codec, - ProcessInformationBlock.codec + ProcessInformationBlock.codec, + UnrecognizedBlock.codec, ) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala index 8a60d7ef37..c69a0703e2 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala @@ -40,7 +40,7 @@ object CaptureFile { ): Pipe[F, Byte, TimeStamped[A]] = { bytes => def go( idbs: Vector[InterfaceDescriptionBlock], - blocks: Stream[F, BodyBlock] + blocks: Stream[F, BodyBlock], ): Pull[F, TimeStamped[A], Unit] = blocks.pull.uncons1.flatMap { case Some((idb: InterfaceDescriptionBlock, tail)) => @@ -54,8 +54,7 @@ object CaptureFile { case None => Pull.done } - bytes - .through(streamDecoder.toPipeByte) - .through(go(Vector.empty, _).stream) + val blocks = bytes.through(streamDecoder.toPipeByte) + blocks.through(go(Vector.empty, _).stream) } } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index d69fb389fa..d382cc44f7 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -40,29 +40,27 @@ case class EnhancedPacketBlock( object EnhancedPacketBlock { - def hexConstant(implicit ord: ByteOrdering): ByteVector = - orderDependent(hex"00000006", hex"06000000") + // format: off + def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = + "EPB" | Block.codecByLength(hexConstant, epbCodec).dropUnits.as[EnhancedPacketBlock] + + private def epbCodec(implicit ord: ByteOrdering) = + ("Interface ID" | guint32 ) :: + ("Timestamp (High)" | guint32 ) :: + ("Timestamp (Low)" | guint32 ) :: + ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => + ("Original Packet Length" | guint32 ) :: + ("Packet Data" | bytes(packetLength.toInt) ) :: + ("Packet padding" | ignore(padTo32Bits(packetLength.toInt) * 8) ) :: + ("Options" | bytes )} + // format: on - private def optionLength(length: Length, packetLength: Int)(implicit ord: ByteOrdering): Int = - length.toLong.toInt - 32 - packetLength - padTo32Bits(packetLength) + private def hexConstant(implicit ord: ByteOrdering): ByteVector = + orderDependent(hex"00000006", hex"06000000") private def padTo32Bits(length: Int) = { val rem = length % 4 if (rem == 0) 0 else 4 - rem } - - // format: off - def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = - "EPB" | Block.codec(hexConstant) { length => - ("Interface ID" | guint32 ) :: - ("Timestamp (High)" | guint32 ) :: - ("Timestamp (Low)" | guint32 ) :: - ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => - ("Original Packet Length" | guint32 ) :: - ("Packet Data" | bytes(packetLength.toInt) ) :: - ("Packet padding" | ignore(padTo32Bits(packetLength.toInt) * 8) ) :: - ("Options" | bytes(optionLength(length, packetLength.toInt)) )} - }.dropUnits.as[EnhancedPacketBlock] - // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index c18f3c24d1..443b7d0c47 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -43,16 +43,17 @@ case class InterfaceDescriptionBlock( object InterfaceDescriptionBlock { - private def hexConstant(implicit ord: ByteOrdering) = - orderDependent(hex"00000001", hex"01000000") - // format: off def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | Block.codec(hexConstant) { length => - ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt) ) :: - ("Reserved" | ignore(16) ) :: - ("SnapLen" | guint32 ) :: - ("Block Bytes" | bytes(length.toLong.toInt - 20) ) - }.dropUnits.as[InterfaceDescriptionBlock] + "IDB" | Block.codecByLength(hexConstant, idbCodec).dropUnits.as[InterfaceDescriptionBlock] + + private def idbCodec(implicit ord: ByteOrdering) = + ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt) ) :: + ("Reserved" | ignore(16) ) :: + ("SnapLen" | guint32 ) :: + ("Block Bytes" | bytes ) // format: on + + private def hexConstant(implicit ord: ByteOrdering) = + orderDependent(hex"00000001", hex"01000000") } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala index 135151bcaa..6d1393cb60 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -30,9 +30,9 @@ case class InterfaceStatisticsBlock(length: Length, bytes: ByteVector) extends B object InterfaceStatisticsBlock { + def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = + "ISB" | Block.codecIgnored(hexConstant).as[InterfaceStatisticsBlock] + private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000005", hex"05000000") - - def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = - "ISB" | Block.ignoredCodec(hexConstant).as[InterfaceStatisticsBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala index 75a2e59b13..42ccfd50e2 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -30,9 +30,9 @@ case class NameResolutionBlock(length: Length, bytes: ByteVector) extends BodyBl object NameResolutionBlock { + def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = + "NRB" | Block.codecIgnored(hexConstant).as[NameResolutionBlock] + private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000004", hex"04000000") - - def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = - "NRB" | Block.ignoredCodec(hexConstant).as[NameResolutionBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala index 37550b8b8e..f479e37f03 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala @@ -30,9 +30,9 @@ case class ProcessInformationBlock(length: Length, bytes: ByteVector) extends Bo object ProcessInformationBlock { + def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = + "PIB" | Block.codecIgnored(hexConstant).as[ProcessInformationBlock] + private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"80000001", hex"01000080") - - def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = - "PIB" | Block.ignoredCodec(hexConstant).as[ProcessInformationBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 7e4f5d9f33..2580954632 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -40,11 +40,12 @@ object SectionHeaderBlock { private val hexConstant = hex"0A0D0D0A" // format: off - val codec: Codec[SectionHeaderBlock] = "SHB" | Block.codec(hexConstant) { length => - ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit ord => - ("Major Version" | guint16 ) :: - ("Minor Version" | guint16 ) :: - ("Block Bytes" | bytes(length.toLong.toInt - 20) ) - }}.dropUnits.as[SectionHeaderBlock] + val codec: Codec[SectionHeaderBlock] = + "SHB" | Block.codecByHex(hexConstant) { length => + ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit ord => + ("Major Version" | guint16 ) :: + ("Minor Version" | guint16 ) :: + ("Block Bytes" | bytes(length.toLong.toInt - 20) ) + }}.dropUnits.as[SectionHeaderBlock] // format: on } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala index 976d6fd2a6..191708b506 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala @@ -29,9 +29,9 @@ case class SimplePacketBlock(length: Length, bytes: ByteVector) extends BodyBloc object SimplePacketBlock { + def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = + "SPB" | Block.codecIgnored(hexConstant).as[ProcessInformationBlock] + private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000003", hex"03000000") - - def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = - "SPB" | Block.ignoredCodec(hexConstant).as[ProcessInformationBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala new file mode 100644 index 0000000000..561b302b2e --- /dev/null +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala @@ -0,0 +1,13 @@ +package fs2.protocols.pcapng + +import scodec.Codec +import scodec.bits._ +import scodec.codecs._ + +case class UnrecognizedBlock(blockType: ByteVector, length: Length, bytes: ByteVector) extends BodyBlock + +object UnrecognizedBlock { + + def codec(implicit ord: ByteOrdering): Codec[UnrecognizedBlock] = + "UB" | Block.codecUnrecognized.as[UnrecognizedBlock] +} From f100073d8dda7b4d646e559ce9bcb0b1a50e750f Mon Sep 17 00:00:00 2001 From: nikiforo Date: Tue, 23 Aug 2022 20:11:10 +0400 Subject: [PATCH 19/20] pcapng - scalafmt and renaming --- .../pcapng/{Block.scala => BlockCodec.scala} | 47 ++++++++++--------- .../scala-3/{Block.scala => BlockCodec.scala} | 30 ++++++------ .../scala/fs2/protocols/pcap/LinkType.scala | 4 +- .../fs2/protocols/pcapng/BodyBlock.scala | 4 +- .../fs2/protocols/pcapng/CaptureFile.scala | 2 +- .../pcapng/EnhancedPacketBlock.scala | 10 ++-- .../pcapng/InterfaceDescriptionBlock.scala | 2 +- .../pcapng/InterfaceStatisticsBlock.scala | 2 +- .../scala/fs2/protocols/pcapng/Length.scala} | 22 ++------- .../pcapng/NameResolutionBlock.scala | 2 +- .../pcapng/ProcessInformationBlock.scala | 2 +- .../protocols/pcapng/SectionHeaderBlock.scala | 6 +-- .../protocols/pcapng/SimplePacketBlock.scala | 2 +- .../protocols/pcapng/UnrecognizedBlock.scala | 29 ++++++++++-- .../scala/fs2/protocols/pcapng/package.scala | 4 -- .../scala/fs2/protocols/PcapMpegExample.scala | 2 +- .../scala/fs2/protocols/PcapNgExample.scala | 21 ++------- .../fs2/protocols/pcapng/BlockTest.scala | 6 +-- 18 files changed, 94 insertions(+), 103 deletions(-) rename protocols/shared/src/main/scala-2/fs2/protocols/pcapng/{Block.scala => BlockCodec.scala} (67%) rename protocols/shared/src/main/scala-3/{Block.scala => BlockCodec.scala} (77%) rename protocols/shared/src/{test/scala/fs2/protocols/pcapng/DummyBlock.scala => main/scala/fs2/protocols/pcapng/Length.scala} (63%) diff --git a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala similarity index 67% rename from protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala rename to protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala index 9a6bb2aaaf..0b3e2d8793 100644 --- a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/Block.scala +++ b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala @@ -28,12 +28,10 @@ import scodec.codecs._ import shapeless.ops.hlist.{Init, Last, Prepend} import shapeless.{::, HList, HNil} -trait Block - -object Block { +object BlockCodec { // format: off - private def codec[A, L <: HList, LB <: HList](blockType: Codec[A])(f: Length => Codec[L])( + private def commonStructure[A, L <: HList, LB <: HList](blockType: Codec[A])(f: Length => Codec[L])( implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], @@ -45,32 +43,35 @@ object Block { ("Block Total Length" | constant(length.bv) )} // format: on - def codecByHex[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( - implicit - prepend: Prepend.Aux[L, Unit :: HNil, LB], - init: Init.Aux[LB, L], - last: Last.Aux[LB, Unit] - ): Codec[Unit :: Length :: LB] = codec(constant(hexConstant))(f) + def unknownByteOrder[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( + implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit] + ): Codec[Unit :: Length :: LB] = commonStructure(constant(hexConstant))(f) - def codecByLength[L <: HList, LB <: HList](hexConstant: ByteVector, c: Codec[L])( - implicit - prepend: Prepend.Aux[L, Unit :: HNil, LB], - init: Init.Aux[LB, L], - last: Last.Aux[LB, Unit], - ord: ByteOrdering, - ): Codec[Unit :: Length :: LB] = codecByHex(hexConstant)(length => fixedSizeBytes(length.toLong - 12, c)) + def byBlockBytesCodec[L <: HList, LB <: HList]( + hexConstant: ByteVector, + blockBytesCodec: Codec[L] + )(implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit], + ord: ByteOrdering + ): Codec[Unit :: Length :: LB] = + unknownByteOrder(hexConstant)(length => fixedSizeBytes(length.toLong - 12, blockBytesCodec)) - def codecIgnored( - hexConstant: ByteVector + def ignored( + hexConstant: ByteVector )(implicit ord: ByteOrdering): Codec[Length :: ByteVector :: HNil] = - codecByHex(hexConstant) { length => + unknownByteOrder(hexConstant) { length => fixedSizeBytes(length.toLong - 12, bytes) :: Codec.deriveHNil }.dropUnits - def codecUnrecognized( - implicit ord: ByteOrdering + def unrecognizedBlockType(implicit + ord: ByteOrdering ): Codec[ByteVector :: Length :: ByteVector :: HNil] = - codec(bytes(4)) { length => + commonStructure(bytes(4)) { length => fixedSizeBytes(length.toLong - 12, bytes) :: Codec.deriveHNil }.dropUnits } diff --git a/protocols/shared/src/main/scala-3/Block.scala b/protocols/shared/src/main/scala-3/BlockCodec.scala similarity index 77% rename from protocols/shared/src/main/scala-3/Block.scala rename to protocols/shared/src/main/scala-3/BlockCodec.scala index c7a3a3309a..80e636e18a 100644 --- a/protocols/shared/src/main/scala-3/Block.scala +++ b/protocols/shared/src/main/scala-3/BlockCodec.scala @@ -26,12 +26,10 @@ import scodec.Codec import scodec.bits._ import scodec.codecs._ -trait Block - -object Block { +object BlockCodec { // format: off - inline def codec[A, L <: Tuple]( + private inline def commonStructure[A, L <: Tuple]( blockType: Codec[A] )(f: Length => Codec[L]): Codec[Tuple.Concat[A *: Length *: L, Unit *: EmptyTuple]] = ("Block Type" | blockType ) :: @@ -40,27 +38,27 @@ object Block { ("Block Total Length" | constant(length.bv) )} // format: on - inline def codecByHex[L <: Tuple]( - hexConstant: ByteVector + inline def unknownByteOrder[L <: Tuple]( + hexConstant: ByteVector )(f: Length => Codec[L]): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = - codec(constant(hexConstant))(f) + commonStructure(constant(hexConstant))(f) - inline def codecByLength[L <: Tuple](hexConstant: ByteVector, c: Codec[L])( - implicit ord: ByteOrdering, + inline def byBlockBytesCodec[L <: Tuple](hexConstant: ByteVector, blockBytesCodec: Codec[L])( + implicit ord: ByteOrdering ): Codec[Tuple.Concat[Unit *: Length *: L, Unit *: EmptyTuple]] = - codecByHex(hexConstant)(length => fixedSizeBytes(length.toLong - 12, c)) + unknownByteOrder(hexConstant)(length => fixedSizeBytes(length.toLong - 12, blockBytesCodec)) - def codecIgnored( - hexConstant: ByteVector + def ignored( + hexConstant: ByteVector )(implicit ord: ByteOrdering): Codec[Length *: ByteVector *: EmptyTuple] = - codecByHex(hexConstant) { length => + unknownByteOrder(hexConstant) { length => fixedSizeBytes(length.toLong - 12, bytes).tuple }.dropUnits - def codecUnrecognized( - implicit ord: ByteOrdering + def unrecognizedBlockType(implicit + ord: ByteOrdering ): Codec[ByteVector *: Length *: ByteVector *: EmptyTuple] = - codec(bytes(4)) { length => + commonStructure(bytes(4)) { length => fixedSizeBytes(length.toLong - 12, bytes).tuple }.dropUnits } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala index 9897e31307..7fb4f9128a 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcap/LinkType.scala @@ -42,7 +42,7 @@ object LinkType { case class Unknown(value: Long) extends LinkType def fromInt(l: Int): LinkType = - fromLong(l) + fromLong(l.toLong) def toInt(lt: LinkType): Int = lt match { case Null => 0 @@ -64,7 +64,7 @@ object LinkType { case other => Unknown(other) } - def toLong(lt: LinkType): Long = toInt(lt) + def toLong(lt: LinkType): Long = toInt(lt).toLong implicit def codec(implicit bo: ByteOrdering): Codec[LinkType] = guint32.xmap[LinkType](fromLong, toLong) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala index a6c321fc11..939df85dc8 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/BodyBlock.scala @@ -25,7 +25,7 @@ package pcapng import scodec.bits._ import scodec.Decoder -trait BodyBlock extends Block +trait BodyBlock object BodyBlock { @@ -36,6 +36,6 @@ object BodyBlock { NameResolutionBlock.codec, InterfaceStatisticsBlock.codec, ProcessInformationBlock.codec, - UnrecognizedBlock.codec, + UnrecognizedBlock.codec ) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala index c69a0703e2..eb4b806c53 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala @@ -40,7 +40,7 @@ object CaptureFile { ): Pipe[F, Byte, TimeStamped[A]] = { bytes => def go( idbs: Vector[InterfaceDescriptionBlock], - blocks: Stream[F, BodyBlock], + blocks: Stream[F, BodyBlock] ): Pull[F, TimeStamped[A], Unit] = blocks.pull.uncons1.flatMap { case Some((idb: InterfaceDescriptionBlock, tail)) => diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index d382cc44f7..8f93940654 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -42,7 +42,7 @@ object EnhancedPacketBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = - "EPB" | Block.codecByLength(hexConstant, epbCodec).dropUnits.as[EnhancedPacketBlock] + "EPB" | BlockCodec.byBlockBytesCodec(hexConstant, epbCodec).dropUnits.as[EnhancedPacketBlock] private def epbCodec(implicit ord: ByteOrdering) = ("Interface ID" | guint32 ) :: @@ -51,16 +51,16 @@ object EnhancedPacketBlock { ("Captured Packet Length" | guint32 ).flatPrepend { packetLength => ("Original Packet Length" | guint32 ) :: ("Packet Data" | bytes(packetLength.toInt) ) :: - ("Packet padding" | ignore(padTo32Bits(packetLength.toInt) * 8) ) :: + ("Packet padding" | ignore(padTo32Bits(packetLength.toInt)) ) :: ("Options" | bytes )} // format: on private def hexConstant(implicit ord: ByteOrdering): ByteVector = orderDependent(hex"00000006", hex"06000000") - private def padTo32Bits(length: Int) = { + private def padTo32Bits(length: Int): Long = { val rem = length % 4 - if (rem == 0) 0 - else 4 - rem + val bytes = if (rem == 0) 0 else 4 - rem + bytes.toLong * 8 } } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 443b7d0c47..4ece0c2f8c 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -45,7 +45,7 @@ object InterfaceDescriptionBlock { // format: off def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | Block.codecByLength(hexConstant, idbCodec).dropUnits.as[InterfaceDescriptionBlock] + "IDB" | BlockCodec.byBlockBytesCodec(hexConstant, idbCodec).dropUnits.as[InterfaceDescriptionBlock] private def idbCodec(implicit ord: ByteOrdering) = ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt) ) :: diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala index 6d1393cb60..602db53108 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceStatisticsBlock.scala @@ -31,7 +31,7 @@ case class InterfaceStatisticsBlock(length: Length, bytes: ByteVector) extends B object InterfaceStatisticsBlock { def codec(implicit ord: ByteOrdering): Codec[InterfaceStatisticsBlock] = - "ISB" | Block.codecIgnored(hexConstant).as[InterfaceStatisticsBlock] + "ISB" | BlockCodec.ignored(hexConstant).as[InterfaceStatisticsBlock] private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000005", hex"05000000") diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Length.scala similarity index 63% rename from protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala rename to protocols/shared/src/main/scala/fs2/protocols/pcapng/Length.scala index ea387642f1..513bd42471 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/DummyBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/Length.scala @@ -22,24 +22,8 @@ package fs2.protocols package pcapng -import scodec.Codec -import scodec.bits._ -import scodec.codecs._ +import scodec.bits.{ByteOrdering, ByteVector} -case class DummyBlock( - blockType: ByteVector, - totalLength: Length, - body: ByteVector -) extends BodyBlock - -object DummyBlock { - - // format: off - def codec(implicit ord: ByteOrdering): Codec[DummyBlock] = "Dummy" | { - ("Block Type" | bytes(4) ) :: - ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => - ("Block Body" | bytes(length.toLong.toInt - 12) ) :: - ("Block Total Length" | constant(length.bv) )} - }.dropUnits.as[DummyBlock] - // format: on +case class Length(bv: ByteVector) extends AnyVal { + def toLong(implicit ord: ByteOrdering): Long = bv.toLong(signed = false, ord) } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala index 42ccfd50e2..1429f29774 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/NameResolutionBlock.scala @@ -31,7 +31,7 @@ case class NameResolutionBlock(length: Length, bytes: ByteVector) extends BodyBl object NameResolutionBlock { def codec(implicit ord: ByteOrdering): Codec[NameResolutionBlock] = - "NRB" | Block.codecIgnored(hexConstant).as[NameResolutionBlock] + "NRB" | BlockCodec.ignored(hexConstant).as[NameResolutionBlock] private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000004", hex"04000000") diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala index f479e37f03..2d009cfc8d 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ProcessInformationBlock.scala @@ -31,7 +31,7 @@ case class ProcessInformationBlock(length: Length, bytes: ByteVector) extends Bo object ProcessInformationBlock { def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = - "PIB" | Block.codecIgnored(hexConstant).as[ProcessInformationBlock] + "PIB" | BlockCodec.ignored(hexConstant).as[ProcessInformationBlock] private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"80000001", hex"01000080") diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 2580954632..61f25eb5a7 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -33,7 +33,7 @@ case class SectionHeaderBlock( majorVersion: Int, minorVersion: Int, bytes: ByteVector -) extends Block +) object SectionHeaderBlock { @@ -41,8 +41,8 @@ object SectionHeaderBlock { // format: off val codec: Codec[SectionHeaderBlock] = - "SHB" | Block.codecByHex(hexConstant) { length => - ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit ord => + "SHB" | BlockCodec.unknownByteOrder(hexConstant) { length => + ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit byteOrder => ("Major Version" | guint16 ) :: ("Minor Version" | guint16 ) :: ("Block Bytes" | bytes(length.toLong.toInt - 20) ) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala index 191708b506..80e46c503d 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SimplePacketBlock.scala @@ -30,7 +30,7 @@ case class SimplePacketBlock(length: Length, bytes: ByteVector) extends BodyBloc object SimplePacketBlock { def codec(implicit ord: ByteOrdering): Codec[ProcessInformationBlock] = - "SPB" | Block.codecIgnored(hexConstant).as[ProcessInformationBlock] + "SPB" | BlockCodec.ignored(hexConstant).as[ProcessInformationBlock] private def hexConstant(implicit ord: ByteOrdering) = orderDependent(hex"00000003", hex"03000000") diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala index 561b302b2e..64ccebb0b9 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/UnrecognizedBlock.scala @@ -1,13 +1,36 @@ -package fs2.protocols.pcapng +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.protocols +package pcapng import scodec.Codec import scodec.bits._ import scodec.codecs._ -case class UnrecognizedBlock(blockType: ByteVector, length: Length, bytes: ByteVector) extends BodyBlock +case class UnrecognizedBlock(blockType: ByteVector, length: Length, bytes: ByteVector) + extends BodyBlock object UnrecognizedBlock { def codec(implicit ord: ByteOrdering): Codec[UnrecognizedBlock] = - "UB" | Block.codecUnrecognized.as[UnrecognizedBlock] + "UB" | BlockCodec.unrecognizedBlockType.as[UnrecognizedBlock] } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala index cc3d97c0a3..f7b0e8ffe2 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/package.scala @@ -25,10 +25,6 @@ import scodec.bits.{ByteOrdering, ByteVector} package object pcapng { - case class Length(bv: ByteVector) extends AnyVal { - def toLong(implicit ord: ByteOrdering): Long = bv.toLong(signed = false, ord) - } - def orderDependent[T]( big: ByteVector, little: ByteVector diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala index d604f724fa..2aae1d4d61 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapMpegExample.scala @@ -38,7 +38,7 @@ import pcap.{CaptureFile, LinkType} * - of UDP datagrams * - containing MPEG transport stream packets */ -object PcapMpegExample { +object PcapMpegExample extends IOApp.Simple { case class CapturedPacket( source: SocketAddress[Ipv4Address], diff --git a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala index 1a2b442546..e11753205f 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/PcapNgExample.scala @@ -23,30 +23,19 @@ package fs2 package protocols import cats.effect.{IO, IOApp} -import cats.syntax.all._ +import cats.syntax.foldable._ +import cats.syntax.option._ import fs2.io.file.{Files, Path} import fs2.protocols.pcap.LinkType -import fs2.protocols.pcapng.{CaptureFile, DummyBlock} +import fs2.protocols.pcapng.CaptureFile object PcapNgExample extends IOApp.Simple { def run: IO[Unit] = - output.compile.toList.flatMap(x => IO.println(x.length)) // _.traverse_(IO.println) + output.compile.toList.flatMap(_.traverse_(IO.println)) private def byteStream: Stream[IO, Byte] = - Files[IO].readAll(Path("/Users/anikiforov/pcapng/many_interfaces.pcapng")) - - private def revealFailed = - byteStream - .through(CaptureFile.streamDecoder.toPipeByte) - .flatMap { - case dummy: DummyBlock => Stream.emit(dummy) - case _ => Stream.empty - } - .debug() - - private def decode = - byteStream.through(CaptureFile.streamDecoder.toPipeByte) + Files[IO].readAll(Path("/path/to/pcapng")) private def output = byteStream.through( diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index a8d09c6bee..461954fb6f 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -62,8 +62,8 @@ class BlockTest extends munit.FunSuite { assertEquals(actual, fullyDecoded(Enhanced4.expected)) } - test("real-data dummy") { - val actual = DummyBlock.codec(LittleEndian).decode(RD.bytes.bits) + test("real-data UnrecognizedBlock") { + val actual = UnrecognizedBlock.codec(LittleEndian).decode(RD.bytes.bits) assertEquals(actual, fullyDecoded(RD.expectedDummy)) } @@ -203,6 +203,6 @@ private object BlockTest { val bytes = header ++ length.bv ++ props ++ data ++ padding ++ opts ++ length.bv val expectedEPB = EnhancedPacketBlock(length, 0, 381443, 399317124, 66, 66, data, opts) - val expectedDummy = DummyBlock(header, length, props ++ data ++ padding ++ opts) + val expectedDummy = UnrecognizedBlock(header, length, props ++ data ++ padding ++ opts) } } From 18af9a3b42d52ce6462335a60aa5ba1fa02b034b Mon Sep 17 00:00:00 2001 From: nikiforo Date: Wed, 24 Aug 2022 11:58:28 +0400 Subject: [PATCH 20/20] pcapng - codestyle and scalafmt --- .../scala-2/fs2/protocols/pcapng/BlockCodec.scala | 13 +++++++------ .../shared/src/main/scala-3/BlockCodec.scala | 4 ++-- .../fs2/protocols/pcapng/ByteOrderMagic.scala | 2 ++ .../scala/fs2/protocols/pcapng/CaptureFile.scala | 3 ++- .../protocols/pcapng/EnhancedPacketBlock.scala | 8 ++++---- .../pcapng/InterfaceDescriptionBlock.scala | 14 ++++++++------ .../fs2/protocols/pcapng/SectionHeaderBlock.scala | 15 ++++++++------- .../scala/fs2/protocols/pcapng/BlockTest.scala | 3 ++- 8 files changed, 35 insertions(+), 27 deletions(-) diff --git a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala index 0b3e2d8793..3d8805b83d 100644 --- a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala +++ b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala @@ -30,13 +30,14 @@ import shapeless.{::, HList, HNil} object BlockCodec { - // format: off - private def commonStructure[A, L <: HList, LB <: HList](blockType: Codec[A])(f: Length => Codec[L])( - implicit - prepend: Prepend.Aux[L, Unit :: HNil, LB], - init: Init.Aux[LB, L], - last: Last.Aux[LB, Unit] + private def commonStructure[A, L <: HList, LB <: HList]( + blockType: Codec[A] + )(f: Length => Codec[L])(implicit + prepend: Prepend.Aux[L, Unit :: HNil, LB], + init: Init.Aux[LB, L], + last: Last.Aux[LB, Unit] ): Codec[A :: Length :: LB] = + // format: off ("Block Type" | blockType ) :: ("Block Total Length" | bytes(4).xmapc(Length)(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ diff --git a/protocols/shared/src/main/scala-3/BlockCodec.scala b/protocols/shared/src/main/scala-3/BlockCodec.scala index 80e636e18a..df45e7e5cc 100644 --- a/protocols/shared/src/main/scala-3/BlockCodec.scala +++ b/protocols/shared/src/main/scala-3/BlockCodec.scala @@ -28,10 +28,10 @@ import scodec.codecs._ object BlockCodec { - // format: off private inline def commonStructure[A, L <: Tuple]( - blockType: Codec[A] + blockType: Codec[A] )(f: Length => Codec[L]): Codec[Tuple.Concat[A *: Length *: L, Unit *: EmptyTuple]] = + // format: off ("Block Type" | blockType ) :: ("Block Total Length" | bytes(4).xmapc(Length(_))(_.bv) ).flatPrepend { length => ("Block Bytes" | f(length) ) :+ diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala index 0aa1365f9c..0168df9252 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/ByteOrderMagic.scala @@ -27,7 +27,9 @@ import scodec.bits._ import scodec.codecs._ object ByteOrderMagic extends Codec[ByteOrdering] { + private val BigEndian = hex"1a2b3c4d" + private val LittleEndian = hex"4d3c2b1a" def sizeBound = SizeBound.exact(32) diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala index eb4b806c53..3e708bd148 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/CaptureFile.scala @@ -19,7 +19,8 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package fs2.protocols.pcapng +package fs2.protocols +package pcapng import cats.effect.MonadCancelThrow import fs2.interop.scodec.StreamDecoder diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala index 8f93940654..da718b2ec3 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/EnhancedPacketBlock.scala @@ -40,11 +40,14 @@ case class EnhancedPacketBlock( object EnhancedPacketBlock { - // format: off def codec(implicit ord: ByteOrdering): Codec[EnhancedPacketBlock] = "EPB" | BlockCodec.byBlockBytesCodec(hexConstant, epbCodec).dropUnits.as[EnhancedPacketBlock] + private def hexConstant(implicit ord: ByteOrdering): ByteVector = + orderDependent(hex"00000006", hex"06000000") + private def epbCodec(implicit ord: ByteOrdering) = + // format: off ("Interface ID" | guint32 ) :: ("Timestamp (High)" | guint32 ) :: ("Timestamp (Low)" | guint32 ) :: @@ -55,9 +58,6 @@ object EnhancedPacketBlock { ("Options" | bytes )} // format: on - private def hexConstant(implicit ord: ByteOrdering): ByteVector = - orderDependent(hex"00000006", hex"06000000") - private def padTo32Bits(length: Int): Long = { val rem = length % 4 val bytes = if (rem == 0) 0 else 4 - rem diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala index 4ece0c2f8c..b49c6d786b 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/InterfaceDescriptionBlock.scala @@ -43,17 +43,19 @@ case class InterfaceDescriptionBlock( object InterfaceDescriptionBlock { - // format: off - def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = - "IDB" | BlockCodec.byBlockBytesCodec(hexConstant, idbCodec).dropUnits.as[InterfaceDescriptionBlock] + def codec(implicit ord: ByteOrdering): Codec[InterfaceDescriptionBlock] = { + val rawCodec = BlockCodec.byBlockBytesCodec(hexConstant, idbCodec) + "IDB" | rawCodec.dropUnits.as[InterfaceDescriptionBlock] + } + + private def hexConstant(implicit ord: ByteOrdering): ByteVector = + orderDependent(hex"00000001", hex"01000000") private def idbCodec(implicit ord: ByteOrdering) = + // format: off ("LinkType" | guint16.xmap[LinkType](LinkType.fromInt, LinkType.toInt) ) :: ("Reserved" | ignore(16) ) :: ("SnapLen" | guint32 ) :: ("Block Bytes" | bytes ) // format: on - - private def hexConstant(implicit ord: ByteOrdering) = - orderDependent(hex"00000001", hex"01000000") } diff --git a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala index 61f25eb5a7..d84f5fa959 100644 --- a/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala +++ b/protocols/shared/src/main/scala/fs2/protocols/pcapng/SectionHeaderBlock.scala @@ -39,13 +39,14 @@ object SectionHeaderBlock { private val hexConstant = hex"0A0D0D0A" - // format: off val codec: Codec[SectionHeaderBlock] = - "SHB" | BlockCodec.unknownByteOrder(hexConstant) { length => - ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit byteOrder => - ("Major Version" | guint16 ) :: - ("Minor Version" | guint16 ) :: - ("Block Bytes" | bytes(length.toLong.toInt - 20) ) - }}.dropUnits.as[SectionHeaderBlock] + "SHB" | BlockCodec.unknownByteOrder(hexConstant)(shbCodec).dropUnits.as[SectionHeaderBlock] + + private def shbCodec(length: Length) = + // format: off + ("Byte-Order Magic" | ByteOrderMagic ).flatPrepend { implicit byteOrder => + ("Major Version" | guint16 ) :: + ("Minor Version" | guint16 ) :: + ("Block Bytes" | bytes(length.toLong.toInt - 20) )} // format: on } diff --git a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala index 461954fb6f..6a962b607d 100644 --- a/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala +++ b/protocols/shared/src/test/scala/fs2/protocols/pcapng/BlockTest.scala @@ -78,7 +78,8 @@ class BlockTest extends munit.FunSuite { private object BlockTest { - // https://gitlab.com/wireshark/wireshark/-/wikis/Development/PcapNg dhcp.pcapng + // This data is taken from dhcp.pcapng file from + // https://gitlab.com/wireshark/wireshark/-/wikis/Development/PcapNg object DHCP { object SHB { val header = hex"0a0d0d0a"