diff --git a/experiment/tlstool/internal/dialer.go b/experiment/tlstool/internal/dialer.go new file mode 100644 index 00000000..d35ee8ef --- /dev/null +++ b/experiment/tlstool/internal/dialer.go @@ -0,0 +1,29 @@ +package internal + +import ( + "context" + "net" + "time" + + "github.com/ooni/probe-engine/netx" +) + +// Dialer creates net.Conn instances where (1) we delay writes if +// a delay is configured and (2) we split outgoing buffers if there +// is a configured splitter function. +type Dialer struct { + netx.Dialer + Delay time.Duration + Splitter func([]byte) [][]byte +} + +// DialContext implements netx.Dialer.DialContext. +func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + conn, err := d.Dialer.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + conn = SleeperWriter{Conn: conn, Delay: d.Delay} + conn = SplitterWriter{Conn: conn, Splitter: d.Splitter} + return conn, nil +} diff --git a/experiment/tlstool/internal/dialer_test.go b/experiment/tlstool/internal/dialer_test.go new file mode 100644 index 00000000..4741fde7 --- /dev/null +++ b/experiment/tlstool/internal/dialer_test.go @@ -0,0 +1,56 @@ +package internal_test + +import ( + "context" + "errors" + "testing" + + "github.com/ooni/probe-engine/experiment/tlstool/internal" +) + +func TestDialerFailure(t *testing.T) { + expected := errors.New("mocked error") + dialer := internal.Dialer{Dialer: internal.FakeDialer{ + Err: expected, + }} + conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853") + if !errors.Is(err, expected) { + t.Fatalf("not the error we expected: %+v", err) + } + if conn != nil { + t.Fatal("expected nil conn here") + } +} + +func TestDialerSuccess(t *testing.T) { + splitter := func([]byte) [][]byte { + return nil // any value is fine we just a need a splitter != nil here + } + innerconn := &internal.FakeConn{} + dialer := internal.Dialer{ + Delay: 12345, + Dialer: internal.FakeDialer{Conn: innerconn}, + Splitter: splitter, + } + conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853") + if err != nil { + t.Fatal(err) + } + sconn, ok := conn.(internal.SplitterWriter) + if !ok { + t.Fatal("the outer connection is not a splitter") + } + if sconn.Splitter == nil { + t.Fatal("not the splitter we expected") + } + dconn, ok := sconn.Conn.(internal.SleeperWriter) + if !ok { + t.Fatal("the inner connection is not a sleeper") + } + if dconn.Delay != 12345 { + t.Fatal("invalid delay") + } + if dconn.Conn != innerconn { + t.Fatal("invalid inner connection") + } +} diff --git a/experiment/tlstool/internal/segmenter/fake_test.go b/experiment/tlstool/internal/fake_test.go similarity index 98% rename from experiment/tlstool/internal/segmenter/fake_test.go rename to experiment/tlstool/internal/fake_test.go index 979375b4..06ddb16c 100644 --- a/experiment/tlstool/internal/segmenter/fake_test.go +++ b/experiment/tlstool/internal/fake_test.go @@ -1,4 +1,4 @@ -package segmenter +package internal import ( "context" diff --git a/experiment/tlstool/internal/internal.go b/experiment/tlstool/internal/internal.go new file mode 100644 index 00000000..957cde1f --- /dev/null +++ b/experiment/tlstool/internal/internal.go @@ -0,0 +1,57 @@ +// Package internal contains the implementation of tlstool. +package internal + +import ( + "time" + + "github.com/ooni/probe-engine/netx" +) + +// DialerConfig contains the config for creating a dialer +type DialerConfig struct { + Dialer netx.Dialer + Delay time.Duration + SNI string +} + +// NewSNISplitterDialer creates a new dialer that splits +// outgoing messages such that the SNI should end up being +// splitted into different TCP segments. +func NewSNISplitterDialer(config DialerConfig) Dialer { + return Dialer{ + Dialer: config.Dialer, + Delay: config.Delay, + Splitter: func(b []byte) [][]byte { + return SNISplitter(b, []byte(config.SNI)) + }, + } +} + +// NewThriceSplitterDialer creates a new dialer that splits +// outgoing messages in three parts according to the circumvention +// technique described by Kevin Boch in the Internet Measurement +// Village 2020 . +func NewThriceSplitterDialer(config DialerConfig) Dialer { + return Dialer{ + Dialer: config.Dialer, + Delay: config.Delay, + Splitter: Splitter84rest, + } +} + +// NewRandomSplitterDialer creates a new dialer that splits +// the SNI like the fixed splitting schema used by outline. See +// github.com/Jigsaw-Code/outline-go-tun2socks. +func NewRandomSplitterDialer(config DialerConfig) Dialer { + return Dialer{ + Dialer: config.Dialer, + Delay: config.Delay, + Splitter: Splitter3264rand, + } +} + +// NewVanillaDialer creates a new vanilla dialer that does +// nothing and is used to establish a baseline. +func NewVanillaDialer(config DialerConfig) Dialer { + return Dialer{Dialer: config.Dialer} +} diff --git a/experiment/tlstool/internal/internal_test.go b/experiment/tlstool/internal/internal_test.go new file mode 100644 index 00000000..f89dbe2f --- /dev/null +++ b/experiment/tlstool/internal/internal_test.go @@ -0,0 +1,40 @@ +package internal_test + +import ( + "context" + "testing" + + "github.com/ooni/probe-engine/experiment/tlstool/internal" + "github.com/ooni/probe-engine/netx" +) + +var config = internal.DialerConfig{ + Dialer: netx.NewDialer(netx.Config{}), + Delay: 10, + SNI: "dns.google", +} + +func dial(t *testing.T, d netx.Dialer) { + td := netx.NewTLSDialer(netx.Config{Dialer: d}) + conn, err := td.DialTLSContext(context.Background(), "tcp", "dns.google:853") + if err != nil { + t.Fatal(err) + } + conn.Close() +} + +func TestNewSNISplitterDialer(t *testing.T) { + dial(t, internal.NewSNISplitterDialer(config)) +} + +func TestNewThriceSplitterDialer(t *testing.T) { + dial(t, internal.NewThriceSplitterDialer(config)) +} + +func TestNewRandomSplitterDialer(t *testing.T) { + dial(t, internal.NewRandomSplitterDialer(config)) +} + +func TestNewVanillaDialer(t *testing.T) { + dial(t, internal.NewVanillaDialer(config)) +} diff --git a/experiment/tlstool/internal/patternsplitter/fake_test.go b/experiment/tlstool/internal/patternsplitter/fake_test.go deleted file mode 100644 index c65593b7..00000000 --- a/experiment/tlstool/internal/patternsplitter/fake_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package patternsplitter - -import ( - "context" - "io" - "net" - "time" -) - -type FakeDialer struct { - Conn net.Conn - Err error -} - -func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - time.Sleep(10 * time.Microsecond) - return d.Conn, d.Err -} - -type FakeConn struct { - ReadError error - ReadData []byte - SetDeadlineError error - SetReadDeadlineError error - SetWriteDeadlineError error - WriteData [][]byte - WriteError error -} - -func (c *FakeConn) Read(b []byte) (int, error) { - if len(c.ReadData) > 0 { - n := copy(b, c.ReadData) - c.ReadData = c.ReadData[n:] - return n, nil - } - if c.ReadError != nil { - return 0, c.ReadError - } - return 0, io.EOF -} - -func (c *FakeConn) Write(b []byte) (n int, err error) { - if c.WriteError != nil { - return 0, c.WriteError - } - c.WriteData = append(c.WriteData, b) - n = len(b) - return -} - -func (*FakeConn) Close() (err error) { - return -} - -func (*FakeConn) LocalAddr() net.Addr { - return &net.TCPAddr{} -} - -func (*FakeConn) RemoteAddr() net.Addr { - return &net.TCPAddr{} -} - -func (c *FakeConn) SetDeadline(t time.Time) (err error) { - return c.SetDeadlineError -} - -func (c *FakeConn) SetReadDeadline(t time.Time) (err error) { - return c.SetReadDeadlineError -} - -func (c *FakeConn) SetWriteDeadline(t time.Time) (err error) { - return c.SetWriteDeadlineError -} diff --git a/experiment/tlstool/internal/patternsplitter/patternsplitter.go b/experiment/tlstool/internal/patternsplitter/patternsplitter.go deleted file mode 100644 index 5ca75127..00000000 --- a/experiment/tlstool/internal/patternsplitter/patternsplitter.go +++ /dev/null @@ -1,63 +0,0 @@ -// Package patternsplitter contains code to split TCP segments in -// the middle of a specific byte pattern. -package patternsplitter - -import ( - "bytes" - "context" - "net" - "time" - - "github.com/ooni/probe-engine/netx" -) - -// Dialer is a dialer that splits TCP segments in the middle of -// a specific byte pattern and may optionally delay writing the second -// half of the segment by a specific number of milliseconds. -type Dialer struct { - netx.Dialer - Delay int64 - Pattern string -} - -// DialContext implements netx.Dialer.DialContext. -func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - conn, err := d.Dialer.DialContext(ctx, network, address) - if err != nil { - return nil, err - } - return Conn{Conn: conn, Delay: d.Delay, Pattern: []byte(d.Pattern)}, nil -} - -// Conn is the net.Conn generated by patternsplitter.Dialer. -// -// Caveat -// -// The connection will keep splitting segments when it sees the -// specified byte pattern. This behaviour is fine as long as we're -// just checking whether the TLS handshake works. -type Conn struct { - net.Conn - BeforeSecondWrite func() // for testing - Delay int64 - Pattern []byte -} - -// Write implements net.Conn.Write. -func (c Conn) Write(b []byte) (int, error) { - if idx := bytes.Index(b, c.Pattern); idx > -1 { - idx += len(c.Pattern) / 2 - if _, err := c.Conn.Write(b[:idx]); err != nil { - return 0, err - } - <-time.After(time.Duration(c.Delay) * time.Millisecond) - if c.BeforeSecondWrite != nil { - c.BeforeSecondWrite() - } - if _, err := c.Conn.Write(b[idx:]); err != nil { - return 0, err - } - return len(b), nil - } - return c.Conn.Write(b) -} diff --git a/experiment/tlstool/internal/patternsplitter/patternsplitter_test.go b/experiment/tlstool/internal/patternsplitter/patternsplitter_test.go deleted file mode 100644 index 6e7033cd..00000000 --- a/experiment/tlstool/internal/patternsplitter/patternsplitter_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package patternsplitter_test - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-engine/experiment/tlstool/internal/patternsplitter" -) - -func TestDialerFailure(t *testing.T) { - expected := errors.New("mocked error") - d := patternsplitter.Dialer{Dialer: patternsplitter.FakeDialer{Err: expected}} - conn, err := d.DialContext(context.Background(), "tcp", "1.1.1.1:853") - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if conn != nil { - t.Fatal("expected nil conn here") - } -} - -func TestDialerSuccess(t *testing.T) { - innerconn := &patternsplitter.FakeConn{} - d := patternsplitter.Dialer{ - Dialer: patternsplitter.FakeDialer{Conn: innerconn}, - Delay: 1234, - Pattern: "abcdef", - } - conn, err := d.DialContext(context.Background(), "tcp", "1.1.1.1:853") - if err != nil { - t.Fatal(err) - } - realconn, ok := conn.(patternsplitter.Conn) - if !ok { - t.Fatal("cannot cast conn to patternsplitter.SplitConn") - } - if realconn.Delay != 1234 { - t.Fatal("invalid Delay value") - } - if diff := cmp.Diff(realconn.Pattern, []byte("abcdef")); diff != "" { - t.Fatal(diff) - } - if realconn.Conn != innerconn { - t.Fatal("invalid Conn value") - } -} - -func TestWriteSuccessNoSplit(t *testing.T) { - const ( - pattern = "abc.def" - data = "deadbeefdeafbeef" - ) - innerconn := &patternsplitter.FakeConn{} - conn := patternsplitter.Conn{ - Conn: innerconn, - Pattern: []byte(pattern), - } - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != data { - t.Fatal("written invalid data") - } -} - -func TestWriteFailureNoSplit(t *testing.T) { - const ( - pattern = "abc.def" - data = "deadbeefdeafbeef" - ) - expected := errors.New("mocked error") - innerconn := &patternsplitter.FakeConn{ - WriteError: expected, - } - conn := patternsplitter.Conn{ - Conn: innerconn, - Pattern: []byte(pattern), - } - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } -} - -func TestWriteSuccessSplit(t *testing.T) { - const ( - pattern = "abc.def" - data = "deadbeefabc.defdeafbeef" - ) - innerconn := &patternsplitter.FakeConn{} - conn := patternsplitter.Conn{ - Conn: innerconn, - Pattern: []byte(pattern), - } - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 2 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "deadbeefabc" { - t.Fatal("written invalid data") - } - if string(innerconn.WriteData[1]) != ".defdeafbeef" { - t.Fatal("written invalid data") - } -} - -func TestWriteFailureSplitFirstWrite(t *testing.T) { - const ( - pattern = "abc.def" - data = "deadbeefabc.defdeafbeef" - ) - expected := errors.New("mocked error") - innerconn := &patternsplitter.FakeConn{ - WriteError: expected, - } - conn := patternsplitter.Conn{ - Conn: innerconn, - Pattern: []byte(pattern), - } - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 0 { - t.Fatal("some data has been written") - } -} - -func TestWriteFailureSplitSecondWrite(t *testing.T) { - const ( - pattern = "abc.def" - data = "deadbeefabc.defdeafbeef" - ) - expected := errors.New("mocked error") - innerconn := &patternsplitter.FakeConn{} - conn := patternsplitter.Conn{ - BeforeSecondWrite: func() { - innerconn.WriteError = expected // second write will then fail - }, - Conn: innerconn, - Pattern: []byte(pattern), - } - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("we expected to see just one write") - } - if string(innerconn.WriteData[0]) != "deadbeefabc" { - t.Fatal("written invalid data") - } -} diff --git a/experiment/tlstool/internal/segmenter/segmenter.go b/experiment/tlstool/internal/segmenter/segmenter.go deleted file mode 100644 index 8f0236af..00000000 --- a/experiment/tlstool/internal/segmenter/segmenter.go +++ /dev/null @@ -1,70 +0,0 @@ -// Package segmenter contains code that splits TCP data in three -// segments as described by Kevin Boch in https://youtu.be/ksojSRFLbBM?t=1140. -package segmenter - -import ( - "context" - "net" - "time" - - "github.com/ooni/probe-engine/netx" -) - -// Dialer creates connections where we split TCP data in three -// segments as mentioned above. We optionally also delay writing -// each segment by a specific number of milliseconds. -type Dialer struct { - netx.Dialer - Delay int64 -} - -// DialContext implements netx.Dialer.DialContext. -func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - conn, err := d.Dialer.DialContext(ctx, network, address) - if err != nil { - return nil, err - } - return Conn{Conn: conn, Delay: d.Delay}, nil -} - -// Conn is the net.Conn generated by segmenter.Dialer. -// -// Caveat -// -// The connection will keep splitting segments forever. This behaviour -// is fine as long as we're just checking whether the TLS handshake works. -type Conn struct { - net.Conn - BeforeSecondWrite func() // for testing - BeforeThirdWrite func() // for testing - Delay int64 -} - -// Write implements net.Conn.Write. -func (c Conn) Write(b []byte) (int, error) { - const ( - first = 8 - second = 12 - ) - if len(b) > second { - if _, err := c.Conn.Write(b[:first]); err != nil { - return 0, err - } - <-time.After(time.Duration(c.Delay) * time.Millisecond) - if c.BeforeSecondWrite != nil { - c.BeforeSecondWrite() - } - if _, err := c.Conn.Write(b[first:second]); err != nil { - return 0, err - } - <-time.After(time.Duration(c.Delay) * time.Millisecond) - if c.BeforeThirdWrite != nil { - c.BeforeThirdWrite() - } - if _, err := c.Conn.Write(b[second:]); err != nil { - return 0, err - } - return len(b), nil - } - return c.Conn.Write(b) -} diff --git a/experiment/tlstool/internal/segmenter/segmenter_test.go b/experiment/tlstool/internal/segmenter/segmenter_test.go deleted file mode 100644 index 2f9858a7..00000000 --- a/experiment/tlstool/internal/segmenter/segmenter_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package segmenter_test - -import ( - "context" - "errors" - "testing" - - "github.com/ooni/probe-engine/experiment/tlstool/internal/segmenter" -) - -func TestDialerFailure(t *testing.T) { - expected := errors.New("mocked error") - d := segmenter.Dialer{Dialer: segmenter.FakeDialer{Err: expected}} - conn, err := d.DialContext(context.Background(), "tcp", "1.1.1.1:853") - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if conn != nil { - t.Fatal("expected nil conn here") - } -} - -func TestDialerSuccess(t *testing.T) { - innerconn := &segmenter.FakeConn{} - d := segmenter.Dialer{ - Dialer: segmenter.FakeDialer{Conn: innerconn}, - Delay: 1234, - } - conn, err := d.DialContext(context.Background(), "tcp", "1.1.1.1:853") - if err != nil { - t.Fatal(err) - } - realconn, ok := conn.(segmenter.Conn) - if !ok { - t.Fatal("cannot cast conn to segmenter.SplitConn") - } - if realconn.Delay != 1234 { - t.Fatal("invalid Delay value") - } - if realconn.Conn != innerconn { - t.Fatal("invalid Conn value") - } -} - -func TestWriteEdgeCaseSmall(t *testing.T) { - const data = "1111111" - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "1111111" { - t.Fatal("first write is invalid") - } -} - -func TestWriteEdgeCaseMedium(t *testing.T) { - const data = "1111111122" - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "1111111122" { - t.Fatal("first write is invalid") - } -} - -func TestWriteEdgeCaseSmallByOne(t *testing.T) { - const data = "111111112222" - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "111111112222" { - t.Fatal("first write is invalid") - } -} - -func TestWriteEdgeCaseSmallByOneFailure(t *testing.T) { - expected := errors.New("mocked error") - const data = "111111112222" - innerconn := &segmenter.FakeConn{WriteError: expected} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 0 { - t.Fatal("invalid number of writes") - } -} - -func TestWriteSuccessMinimalCase(t *testing.T) { - const data = "1111111122223" - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 3 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "11111111" { - t.Fatal("first write is invalid") - } - if string(innerconn.WriteData[1]) != "2222" { - t.Fatal("first write is invalid") - } - if string(innerconn.WriteData[2]) != "3" { - t.Fatal("first write is invalid") - } -} - -func TestWriteSuccess(t *testing.T) { - const data = "111111112222333333333333" - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if err != nil { - t.Fatal(err) - } - if count != len(data) { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 3 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "11111111" { - t.Fatal("first write is invalid") - } - if string(innerconn.WriteData[1]) != "2222" { - t.Fatal("first write is invalid") - } - if string(innerconn.WriteData[2]) != "333333333333" { - t.Fatal("first write is invalid") - } -} - -func TestFirstWriteFailure(t *testing.T) { - const data = "111111112222333333333333" - expected := errors.New("mocked error") - innerconn := &segmenter.FakeConn{WriteError: expected} - conn := segmenter.Conn{Conn: innerconn} - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 0 { - t.Fatal("invalid number of writes") - } -} - -func TestSecondWriteFailure(t *testing.T) { - const data = "111111112222333333333333" - expected := errors.New("mocked error") - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{ - BeforeSecondWrite: func() { - innerconn.WriteError = expected - }, - Conn: innerconn, - } - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 1 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "11111111" { - t.Fatal("first write is invalid") - } -} - -func TestThirdWriteFailure(t *testing.T) { - const data = "111111112222333333333333" - expected := errors.New("mocked error") - innerconn := &segmenter.FakeConn{} - conn := segmenter.Conn{ - BeforeThirdWrite: func() { - innerconn.WriteError = expected - }, - Conn: innerconn, - } - count, err := conn.Write([]byte(data)) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if count != 0 { - t.Fatal("invalid count") - } - if len(innerconn.WriteData) != 2 { - t.Fatal("invalid number of writes") - } - if string(innerconn.WriteData[0]) != "11111111" { - t.Fatal("first write is invalid") - } - if string(innerconn.WriteData[1]) != "2222" { - t.Fatal("first write is invalid") - } -} diff --git a/experiment/tlstool/internal/splitter.go b/experiment/tlstool/internal/splitter.go new file mode 100644 index 00000000..795f9a65 --- /dev/null +++ b/experiment/tlstool/internal/splitter.go @@ -0,0 +1,67 @@ +package internal + +import ( + "bytes" + "math/rand" + "time" +) + +// SNISplitter splits input such that SNI is splitted across +// a bunch of different output buffers. +func SNISplitter(input []byte, sni []byte) (output [][]byte) { + idx := bytes.Index(input, sni) + if idx < 0 { + output = append(output, input) + return + } + output = append(output, input[:idx]) + // TODO(bassosimone): splitting every three bytes causes + // a bunch of Unicode chatacters (e.g., in Chinese) to be + // sent as part of the same segment. Is that OK? + const segmentsize = 3 + var buf []byte + for _, chr := range input[idx : idx+len(sni)] { + buf = append(buf, chr) + if len(buf) == segmentsize { + output = append(output, buf) + buf = nil + } + } + if len(buf) > 0 { + output = append(output, buf) + buf = nil + } + output = append(output, input[idx+len(sni):]) + return +} + +// Splitter84rest segments the specified buffer into three +// sub-buffers containing respectively 8 bytes, 4 bytes, and +// the rest of the buffer. This segment technique has been +// described by Kevin Bock during the Internet Measurements +// Village 2020: https://youtu.be/ksojSRFLbBM?t=1140. +func Splitter84rest(input []byte) (output [][]byte) { + if len(input) <= 12 { + output = append(output, input) + return + } + output = append(output, input[:8]) + output = append(output, input[8:12]) + output = append(output, input[12:]) + return +} + +// Splitter3264rand splits the specified buffer at a random +// offset between 32 and 64 bytes. This is the methodology used +// by github.com/Jigsaw-Code/outline-go-tun2socks. +func Splitter3264rand(input []byte) (output [][]byte) { + if len(input) <= 64 { + output = append(output, input) + return + } + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + offset := rnd.Intn(32) + 32 + output = append(output, input[:offset]) + output = append(output, input[offset:]) + return +} diff --git a/experiment/tlstool/internal/splitter_test.go b/experiment/tlstool/internal/splitter_test.go new file mode 100644 index 00000000..2af06769 --- /dev/null +++ b/experiment/tlstool/internal/splitter_test.go @@ -0,0 +1,143 @@ +package internal_test + +import ( + "testing" + + "github.com/ooni/probe-engine/experiment/tlstool/internal" + "github.com/ooni/probe-engine/internal/randx" +) + +func TestSplitter84restSmall(t *testing.T) { + input := []byte("1111222") + output := internal.Splitter84rest(input) + if len(output) != 1 { + t.Fatal("invalid output length") + } + if string(output[0]) != "1111222" { + t.Fatal("invalid output[0]") + } +} + +func TestSplitter84restGood(t *testing.T) { + input := []byte("1111222233334") + output := internal.Splitter84rest(input) + if len(output) != 3 { + t.Fatal("invalid output length") + } + if string(output[0]) != "11112222" { + t.Fatal("invalid output[0]") + } + if string(output[1]) != "3333" { + t.Fatal("invalid output[1]") + } + if string(output[2]) != "4" { + t.Fatal("invalid output[2]") + } +} + +func TestSplitter3264randSmall(t *testing.T) { + input := randx.Letters(64) + output := internal.Splitter3264rand([]byte(input)) + if len(output) != 1 { + t.Fatal("invalid output length") + } + if string(output[0]) != input { + t.Fatal("invalid output[0]") + } +} + +func TestSplitter3264Works(t *testing.T) { + input := randx.Letters(65) + output := internal.Splitter3264rand([]byte(input)) + for i := 0; i < 32; i++ { + if len(output) != 2 { + t.Fatal("invalid output length") + } + if len(output[0]) < 32 || len(output[0]) > 64 { + t.Fatal("invalid output[0] length") + } + } +} + +func TestSNISplitterEasyCase(t *testing.T) { + input := []byte("11112222334555foo.barbar.deadbeef.com6777778888") + sni := []byte("barbar.deadbeef.com") + output := internal.SNISplitter(input, sni) + if len(output) != 9 { + t.Fatal("invalid output length") + } + if string(output[0]) != "11112222334555foo." { + t.Fatal("invalid output[0]") + } + if string(output[1]) != "bar" { + t.Fatal("invalid output[1]") + } + if string(output[2]) != "bar" { + t.Fatal("invalid output[2]") + } + if string(output[3]) != ".de" { + t.Fatal("invalid output[3]") + } + if string(output[4]) != "adb" { + t.Fatal("invalid output[4]") + } + if string(output[5]) != "eef" { + t.Fatal("invalid output[5]") + } + if string(output[6]) != ".co" { + t.Fatal("invalid output[6]") + } + if string(output[7]) != "m" { + t.Fatal("invalid output[7]") + } + if string(output[8]) != "6777778888" { + t.Fatal("invalid output[8]") + } +} + +func TestSNISplitterNoMatch(t *testing.T) { + input := []byte("11112222334555foo.barbar.deadbeef.com6777778888") + sni := []byte("www.google.com") + output := internal.SNISplitter(input, sni) + if len(output) != 1 { + t.Fatal("invalid output length") + } + if string(output[0]) != string(input) { + t.Fatal("invalid output[0]") + } +} + +func TestSNISplitterWithUnicode(t *testing.T) { + input := []byte("11112222334555你好世界.com6777778888") + sni := []byte("你好世界.com") + output := internal.SNISplitter(input, sni) + t.Log(string(output[2])) + t.Log(output) + if len(output) != 8 { + t.Fatal("invalid output length") + } + if string(output[0]) != "11112222334555" { + t.Fatal("invalid output[0]") + } + if string(output[1]) != "你" { + t.Fatal("invalid output[1]") + } + if string(output[2]) != "好" { + t.Fatal("invalid output[2]") + } + if string(output[3]) != "世" { + t.Fatal("invalid output[3]") + } + if string(output[4]) != "界" { + t.Fatal("invalid output[4]") + } + if string(output[5]) != ".co" { + t.Fatal("invalid output[5]") + } + if string(output[6]) != "m" { + t.Fatal("invalid output[6]") + } + if string(output[7]) != "6777778888" { + t.Fatal("invalid output[7]") + } +} diff --git a/experiment/tlstool/internal/writer.go b/experiment/tlstool/internal/writer.go new file mode 100644 index 00000000..e7786319 --- /dev/null +++ b/experiment/tlstool/internal/writer.go @@ -0,0 +1,57 @@ +package internal + +import ( + "net" + "time" +) + +// SleeperWriter is a net.Conn that optionally sleeps for the +// specified delay before posting each write. +type SleeperWriter struct { + net.Conn + Delay time.Duration +} + +func (c SleeperWriter) Write(b []byte) (int, error) { + <-time.After(c.Delay) + return c.Conn.Write(b) +} + +// SplitterWriter is a writer that splits every outgoing buffer +// according to the rules specified by the Splitter. +// +// Caveat +// +// The TLS ClientHello may be retransmitted if the server is +// requesting us to restart the negotiation. Therefore, it is +// not safe to just run the splitting once. Since this code +// is meant to investigate TLS blocking, that's fine. +type SplitterWriter struct { + net.Conn + Splitter func([]byte) [][]byte +} + +// Write implements net.Conn.Write +func (c SplitterWriter) Write(b []byte) (int, error) { + if c.Splitter != nil { + return Writev(c.Conn, c.Splitter(b)) + } + return c.Conn.Write(b) +} + +// Writev writes all the vectors inside datalist using the specified +// conn. Returns either an error or the number of bytes sent. Note +// that this function skips any empty entry in datalist. +func Writev(conn net.Conn, datalist [][]byte) (int, error) { + var total int + for _, data := range datalist { + if len(data) > 0 { + count, err := conn.Write(data) + if err != nil { + return 0, err + } + total += count + } + } + return total, nil +} diff --git a/experiment/tlstool/internal/writer_test.go b/experiment/tlstool/internal/writer_test.go new file mode 100644 index 00000000..98348a59 --- /dev/null +++ b/experiment/tlstool/internal/writer_test.go @@ -0,0 +1,164 @@ +package internal_test + +import ( + "errors" + "testing" + "time" + + "github.com/ooni/probe-engine/experiment/tlstool/internal" +) + +func TestSleeperWriterWorksAsIntended(t *testing.T) { + origconn := &internal.FakeConn{} + const outdata = "deadbeefbadidea" + conn := internal.SleeperWriter{ + Conn: origconn, + Delay: 1 * time.Second, + } + before := time.Now() + count, err := conn.Write([]byte(outdata)) + elapsed := time.Since(before) + if err != nil { + t.Fatal(err) + } + if count != len(outdata) { + t.Fatal("unexpected count") + } + if len(origconn.WriteData) != 1 { + t.Fatal("wrong length of written data queue") + } + if string(origconn.WriteData[0]) != outdata { + t.Fatal("we did not write the right data") + } + if elapsed < 750*time.Millisecond { + t.Fatalf("unexpected elapsed time: %+v", elapsed) + } +} + +func TestSplitterWriterNoSplitSuccess(t *testing.T) { + innerconn := &internal.FakeConn{} + conn := internal.SplitterWriter{Conn: innerconn} + const data = "deadbeef" + count, err := conn.Write([]byte(data)) + if err != nil { + t.Fatal(err) + } + if count != len(data) { + t.Fatal("invalid count") + } + if len(innerconn.WriteData) != 1 { + t.Fatal("invalid data queue") + } + if string(innerconn.WriteData[0]) != data { + t.Fatal("invalid written data") + } +} + +func TestSplitterWriterNoSplitFailure(t *testing.T) { + expected := errors.New("mocked error") + innerconn := &internal.FakeConn{WriteError: expected} + conn := internal.SplitterWriter{Conn: innerconn} + const data = "deadbeef" + count, err := conn.Write([]byte(data)) + if !errors.Is(err, expected) { + t.Fatalf("not the error we expected: %+v", err) + } + if count != 0 { + t.Fatal("invalid count") + } + if len(innerconn.WriteData) != 0 { + t.Fatal("invalid data queue") + } +} + +func TestSplitterWriterSplitSuccess(t *testing.T) { + innerconn := &internal.FakeConn{} + conn := internal.SplitterWriter{ + Conn: innerconn, + Splitter: func(b []byte) [][]byte { + return [][]byte{ + b[:2], b[2:], + } + }, + } + const data = "deadbeef" + count, err := conn.Write([]byte(data)) + if err != nil { + t.Fatal(err) + } + if count != len(data) { + t.Fatal("invalid count") + } + if len(innerconn.WriteData) != 2 { + t.Fatal("invalid data queue") + } + if string(innerconn.WriteData[0]) != "de" { + t.Fatal("invalid written data[0]") + } + if string(innerconn.WriteData[1]) != "adbeef" { + t.Fatal("invalid written data[1]") + } +} + +func TestSplitterWriterSplitFailure(t *testing.T) { + expected := errors.New("mocked error") + innerconn := &internal.FakeConn{WriteError: expected} + conn := internal.SplitterWriter{ + Conn: innerconn, + Splitter: func(b []byte) [][]byte { + return [][]byte{ + b[:2], b[2:], + } + }, + } + const data = "deadbeef" + count, err := conn.Write([]byte(data)) + if !errors.Is(err, expected) { + t.Fatalf("not the error we expected: %+v", err) + } + if count != 0 { + t.Fatal("invalid count") + } + if len(innerconn.WriteData) != 0 { + t.Fatal("invalid data queue") + } +} + +func TestWritevWorksWithAlsoEmptyData(t *testing.T) { + conn := &internal.FakeConn{} + datalist := [][]byte{ + []byte("deadbeef"), + []byte(""), + []byte("dead"), + nil, + []byte("badidea"), + nil, + } + count, err := internal.Writev(conn, datalist) + if err != nil { + t.Fatal(err) + } + if count != 19 { + t.Fatal("invalid number of bytes written") + } +} + +func TestWritevFailsAsIntended(t *testing.T) { + expected := errors.New("mocked error") + conn := &internal.FakeConn{WriteError: expected} + datalist := [][]byte{ + []byte("deadbeef"), + []byte(""), + []byte("dead"), + nil, + []byte("badidea"), + nil, + } + count, err := internal.Writev(conn, datalist) + if !errors.Is(err, expected) { + t.Fatalf("not the error we expected: %+v", err) + } + if count != 0 { + t.Fatal("invalid number of bytes written") + } +} diff --git a/experiment/tlstool/tlstool.go b/experiment/tlstool/tlstool.go index 3dbd9e22..9cc5f145 100644 --- a/experiment/tlstool/tlstool.go +++ b/experiment/tlstool/tlstool.go @@ -13,9 +13,9 @@ import ( "crypto/tls" "fmt" "net" + "time" - "github.com/ooni/probe-engine/experiment/tlstool/internal/patternsplitter" - "github.com/ooni/probe-engine/experiment/tlstool/internal/segmenter" + "github.com/ooni/probe-engine/experiment/tlstool/internal" "github.com/ooni/probe-engine/internal/runtimex" "github.com/ooni/probe-engine/model" "github.com/ooni/probe-engine/netx" @@ -24,7 +24,7 @@ import ( const ( testName = "tlstool" - testVersion = "0.0.2" + testVersion = "0.0.3" ) // Config contains the experiment configuration. @@ -35,9 +35,12 @@ type Config struct { // TestKeys contains the experiment results. type TestKeys struct { - SegmenterFailure *string `json:"segmenter_failure"` - SNISplitFailure *string `json:"sni_split_failure"` - VanillaFailure *string `json:"vanilla_failure"` + Experiment map[string]*ExperimentKeys `json:"experiment"` +} + +// ExperimentKeys contains the specific experiment results. +type ExperimentKeys struct { + Failure *string `json:"failure"` } // Measurer performs the measurement. @@ -55,6 +58,25 @@ func (m Measurer) ExperimentVersion() string { return testVersion } +type method struct { + name string + newDialer func(internal.DialerConfig) internal.Dialer +} + +var allMethods = []method{{ + name: "vanilla", + newDialer: internal.NewVanillaDialer, +}, { + name: "snisplit", + newDialer: internal.NewSNISplitterDialer, +}, { + name: "random", + newDialer: internal.NewRandomSplitterDialer, +}, { + name: "thrice", + newDialer: internal.NewThriceSplitterDialer, +}} + // Run implements ExperimentMeasurer.Run. func (m Measurer) Run( ctx context.Context, @@ -62,22 +84,26 @@ func (m Measurer) Run( measurement *model.Measurement, callbacks model.ExperimentCallbacks, ) error { + // TODO(bassosimone): wondering whether this experiment should + // actually be merged with sniblocking instead? tk := new(TestKeys) + tk.Experiment = make(map[string]*ExperimentKeys) measurement.TestKeys = tk address := string(measurement.Input) - - err := m.segmenterRun(ctx, sess.Logger(), address) - callbacks.OnProgress(0.33, fmt.Sprintf("segmenter: %+v", err)) - tk.SegmenterFailure = archival.NewFailure(err) - - err = m.sniSplitRun(ctx, sess.Logger(), address) - callbacks.OnProgress(0.66, fmt.Sprintf("sni_split: %+v", err)) - tk.SNISplitFailure = archival.NewFailure(err) - - err = m.vanillaRun(ctx, sess.Logger(), address) - callbacks.OnProgress(0.99, fmt.Sprintf("vanilla: %+v", err)) - tk.VanillaFailure = archival.NewFailure(err) - + for idx, meth := range allMethods { + // TODO(bassosimone): here we actually want to use urlgetter + // if possible and collect standard test keys. + err := m.run(ctx, runConfig{ + address: address, + logger: sess.Logger(), + newDialer: meth.newDialer, + }) + percent := float64(idx) / float64(len(allMethods)) + callbacks.OnProgress(percent, fmt.Sprintf("%s: %+v", meth.name, err)) + tk.Experiment[meth.name] = &ExperimentKeys{ + Failure: archival.NewFailure(err), + } + } return nil } @@ -90,51 +116,24 @@ func (m Measurer) newDialer(logger model.Logger) netx.Dialer { return netx.NewDialer(netx.Config{FullResolver: resolver, Logger: logger}) } -func (m Measurer) vanillaRun(ctx context.Context, logger model.Logger, address string) error { - dialer := m.newDialer(logger) - tdialer := netx.NewTLSDialer(netx.Config{ - Dialer: dialer, - Logger: logger, - TLSConfig: m.tlsConfig(), - }) - conn, err := tdialer.DialTLSContext(ctx, "tcp", address) - if err != nil { - return err - } - conn.Close() - return nil +type runConfig struct { + address string + logger model.Logger + newDialer func(internal.DialerConfig) internal.Dialer } -func (m Measurer) sniSplitRun(ctx context.Context, logger model.Logger, address string) error { - dialer := &patternsplitter.Dialer{ - Dialer: m.newDialer(logger), - Delay: m.config.Delay, - Pattern: m.pattern(address), - } - tdialer := netx.NewTLSDialer(netx.Config{ - Dialer: dialer, - Logger: logger, - TLSConfig: m.tlsConfig(), +func (m Measurer) run(ctx context.Context, config runConfig) error { + dialer := config.newDialer(internal.DialerConfig{ + Dialer: m.newDialer(config.logger), + Delay: time.Duration(m.config.Delay) * time.Millisecond, + SNI: m.pattern(config.address), }) - conn, err := tdialer.DialTLSContext(ctx, "tcp", address) - if err != nil { - return err - } - conn.Close() - return nil -} - -func (m Measurer) segmenterRun(ctx context.Context, logger model.Logger, address string) error { - dialer := &segmenter.Dialer{ - Dialer: m.newDialer(logger), - Delay: m.config.Delay, - } tdialer := netx.NewTLSDialer(netx.Config{ Dialer: dialer, - Logger: logger, + Logger: config.logger, TLSConfig: m.tlsConfig(), }) - conn, err := tdialer.DialTLSContext(ctx, "tcp", address) + conn, err := tdialer.DialTLSContext(ctx, "tcp", config.address) if err != nil { return err } diff --git a/experiment/tlstool/tlstool_test.go b/experiment/tlstool/tlstool_test.go index 77f65578..8c2cd64e 100644 --- a/experiment/tlstool/tlstool_test.go +++ b/experiment/tlstool/tlstool_test.go @@ -15,7 +15,7 @@ func TestMeasurerExperimentNameVersion(t *testing.T) { if measurer.ExperimentName() != "tlstool" { t.Fatal("unexpected ExperimentName") } - if measurer.ExperimentVersion() != "0.0.2" { + if measurer.ExperimentVersion() != "0.0.3" { t.Fatal("unexpected ExperimentVersion") } }