Prod Patch 3 - Socket usage and Logging cleanup

Cargo.toml

@@ -30,7 +30,7 @@ errno = "0.2.3"
 radix = { git = "https://github.com/refraction-networking/radix" }
 tuntap = { git = "https://github.com/ewust/tuntap.rs" }
 ipnetwork = "^0.14.0"
-protobuf = "2.16.2"
+protobuf = "2.20.0"
 hkdf = "0.7"
 sha2 = "0.8.*"
 hex = "0.3.*"

application/config.toml

@@ -33,16 +33,19 @@ enable_v6 = false
 # be ignored and dropped. This is to prevent clients leveraging the outgoing
 # connections from the station to connect to sation infrastructure that would 
 # be othewise firewalled.
-covert_blocklist = [
+covert_blocklist_subnets = [
     "127.0.0.1/32",     # localhost ipv4
     "10.0.0.0/8",       # reserved ipv4 
     "172.16.0.0/12",    # reserved ipv4
     "192.168.0.0/16",   # reserved ipv4
     "fc00::/7 ",        # private network ipv6
     "fe80::0/16",       # link local ipv6
-    "::1/128"           # localhost ipv6
+    "::1/128",           # localhost ipv6
 ]
 
+covert_blocklist_domains = [
+    "localhost",
+]
 
 # List of addresses to filter out traffic from the detector. The primary functionality
 # of this is to prevent liveness testing from other stations in a conjure cluster from

application/lib/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"regexp"
 
 	"github.com/BurntSushi/toml"
 )
@@ -18,13 +19,17 @@ type Config struct {
 	// REST endpoint to share decoy registrations.
 	PreshareEndpoint string `toml:"preshare_endpoint"`
 
-	// isthe station capable of handling v4 / v6 with independent toggles. 
+	// isthe station capable of handling v4 / v6 with independent toggles.
 	EnableIPv4 bool `toml:"enable_v4"`
-	EnableIPv6 bool  `toml:"enable_v6"`
+	EnableIPv6 bool `toml:"enable_v6"`
 
-	// List of subnets with disallowed covert addresses.
-	CovertBlocklist []string `toml:"covert_blocklist"`
-	covertBlocklist []*net.IPNet
+	// List of disallowed subnets for covert addresses.
+	CovertBlocklistSubnets []string `toml:"covert_blocklist_subnets"`
+	covertBlocklistSubnets []*net.IPNet
+
+	// List of disallowed domain patterns for covert addresses.
+	CovertBlocklistDomains []string `toml:"covert_blocklist_domains"`
+	covertBlocklistDomains []*regexp.Regexp
 }
 
 func ParseConfig() (*Config, error) {
@@ -34,26 +39,50 @@ func ParseConfig() (*Config, error) {
 		return nil, fmt.Errorf("failed to load config: %v", err)
 	}
 
-	c.covertBlocklist = []*net.IPNet{}
-	for _, subnet := range c.CovertBlocklist {
+	c.parseBlocklists()
+
+	return &c, nil
+}
+
+func (c *Config) parseBlocklists() {
+	c.covertBlocklistSubnets = []*net.IPNet{}
+	for _, subnet := range c.CovertBlocklistSubnets {
 		_, ipNet, err := net.ParseCIDR(subnet)
-		if err != nil {
-			continue
+		if err == nil {
+			c.covertBlocklistSubnets = append(c.covertBlocklistSubnets, ipNet)
 		}
-
-		c.covertBlocklist = append(c.covertBlocklist, ipNet)
 	}
 
-	return &c, nil
+	c.covertBlocklistDomains = []*regexp.Regexp{}
+	for _, r := range c.CovertBlocklistDomains {
+		blockedDom := regexp.MustCompile(r)
+		if blockedDom != nil {
+			c.covertBlocklistDomains = append(c.covertBlocklistDomains, blockedDom)
+		}
+	}
 }
 
-func (c *Config) IsBlocklisted(addr net.IP) bool {
-	if addr == nil {
+func (c *Config) IsBlocklisted(urlStr string) bool {
+
+	host, _, err := net.SplitHostPort(urlStr)
+	if err != nil || host == "" {
+		// unable to parse host:port
 		return true
 	}
-	for _, subnet := range c.covertBlocklist {
-		if subnet.Contains(addr) {
-			return true
+
+	if addr := net.ParseIP(host); addr != nil {
+		for _, net := range c.covertBlocklistSubnets {
+			if net.Contains(addr) {
+				// blocked by IP address
+				return true
+			}
+		}
+	} else {
+		for _, pattern := range c.covertBlocklistDomains {
+			if pattern.MatchString(host) {
+				// blocked by Domain pattern
+				return true
+			}
 		}
 	}
 	return false

application/lib/config_test.go

@@ -1,7 +1,6 @@
 package lib
 
 import (
-	"net"
 	"os"
 	"testing"
 )
@@ -18,19 +17,75 @@ func TestConjureLibParseConfig(t *testing.T) {
 		t.Fatalf("No sockets provided to test parse")
 	}
 
-	if len(conf.covertBlocklist) == 0 {
+	if len(conf.covertBlocklistSubnets) == 0 {
 		t.Fatalf("failed to parse blocklist")
 	}
+}
+
+func TestConjureLibConfigBlocklist(t *testing.T) {
+
+	conf := &Config{
+		CovertBlocklistSubnets: []string{
+			"192.0.0.1/16",
+			"::1/128",
+		},
+		CovertBlocklistDomains: []string{
+			".*blocked\\.com$",
+			"blocked1\\.com",
+			"localhost",
+		},
+	}
+
+	conf.parseBlocklists()
+
+	// Addresses that pass Blocklisted check
+	goodURLs := []string{
+		"[::2]:443",
+		"blocked2.com:443",
+		"127.0.0.1:443",
+		"example.com:443",
+		"192.255.0.22:domain",
+
+		// These URLs will pass Blocklisted check, but fail at Dial("tcp", addr)
+		"https://127.0.0.1",
+		"https://example.com",
+	}
+
+	// Test Blocking
+	blockedURLs := []string{
+		"[::1]:443",
+		"blocked.com:443",
+		"abc.blocked.com:443",
+		"blocked1.com:443",
+		"192.0.2.1:http",
+		"localhost:443",
+	}
+
+	// These urls will fail Blocklisted check (and also fail Dial("tcp", addr)).
+	badURLs := []string{
+		"https://::1",
+		"https://[::1]:443",
+		"127.0.0.1",
+		"https://127.0.0.1:443",
+		"example.com",
+		"",
+	}
 
-	if !conf.IsBlocklisted(net.ParseIP("127.0.0.1")) {
-		t.Fatalf("Blocklist error - 127.0.0.1 should be blocked")
+	for _, s := range goodURLs {
+		if conf.IsBlocklisted(s) {
+			t.Fatalf("Blocklist error - %s should not be blocked", s)
+		}
 	}
 
-	if conf.IsBlocklisted(net.ParseIP("1.2.3.4")) {
-		t.Fatalf("Blocklist error - 1.2.3.4 should not be blocked")
+	for _, s := range blockedURLs {
+		if !conf.IsBlocklisted(s) {
+			t.Fatalf("Blocklist error - %s should be blocked", s)
+		}
 	}
 
-	if conf.IsBlocklisted(nil) {
-		t.Fatalf("Not sure what will happen.")
+	for _, s := range badURLs {
+		if !conf.IsBlocklisted(s) {
+			t.Fatalf("Blocklist error - %s should fail (malformed)", s)
+		}
 	}
 }

application/lib/detector_channel.go

@@ -0,0 +1,36 @@
+package lib
+
+import (
+	"log"
+	"os"
+	"sync"
+
+	"github.com/go-redis/redis"
+)
+
+var client *redis.Client
+var once sync.Once
+
+// Redis client is already multiplexed and long lived. It is threadsafe so it
+// should be able to be accessed by multiple registration threads concurrently
+// with no issues. PoolSize is tunable in case this ends up being an issue.
+func getRedisClient() *redis.Client {
+	once.Do(initRedisClient)
+	return client
+}
+
+func initRedisClient() {
+	client = redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+		PoolSize: 100,
+	})
+
+	// Ping to test redis connection
+	_, err := client.Ping().Result()
+	if err != nil {
+		logger := log.New(os.Stderr, "[REDIS] ", log.Ldate|log.Lmicroseconds)
+		logger.Printf("redis connection ping failed.")
+	}
+}

application/lib/registration.go

@@ -13,7 +13,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-redis/redis"
 	"github.com/golang/protobuf/proto"
 	pb "github.com/refraction-networking/gotapdance/protobuf"
 )
@@ -155,6 +154,47 @@ func (regManager *RegistrationManager) NewRegistration(c2s *pb.ClientToStation,
 	return &reg, nil
 }
 
+// NewRegistrationC2SWrapper creates a new registration from details provided. Adds the registration
+// to tracking map, But marks it as not valid.
+func (regManager *RegistrationManager) NewRegistrationC2SWrapper(c2sw *pb.C2SWrapper, includeV6 bool) (*DecoyRegistration, error) {
+	c2s := c2sw.GetRegistrationPayload()
+
+	// Generate keys from shared secret using HKDF
+	conjureKeys, err := GenSharedKeys(c2sw.GetSharedSecret())
+
+	phantomAddr, err := regManager.PhantomSelector.Select(
+		conjureKeys.DarkDecoySeed, uint(c2s.GetDecoyListGeneration()), includeV6)
+
+	if err != nil {
+		return nil, fmt.Errorf("Failed to select phantom IP address: %v", err)
+	}
+
+	clientAddr := net.IP(c2sw.GetRegistrationAddress())
+
+	if phantomAddr.To4 != nil && clientAddr.To4 == nil {
+		// This can happen if the client chooses from a set that contains no
+		// ipv6 options even if include ipv6 is enabled they will get ipv4.
+		return nil, fmt.Errorf("Failed because IPv6 client chose IPv4 phantom")
+	}
+
+	regSrc := c2sw.GetRegistrationSource()
+	reg := DecoyRegistration{
+		DarkDecoy:          phantomAddr,
+		registrationAddr:   net.IP(c2sw.GetRegistrationAddress()),
+		Keys:               &conjureKeys,
+		Covert:             c2s.GetCovertAddress(),
+		Mask:               c2s.GetMaskedDecoyServerName(),
+		Flags:              c2s.Flags,
+		Transport:          c2s.GetTransport(),
+		DecoyListVersion:   c2s.GetDecoyListGeneration(),
+		RegistrationTime:   time.Now(),
+		RegistrationSource: &regSrc,
+		regCount:           0,
+	}
+
+	return &reg, nil
+}
+
 // TrackRegistration adds the registration to the map WITHOUT marking it valid.
 func (regManager *RegistrationManager) TrackRegistration(d *DecoyRegistration) error {
 	err := regManager.registeredDecoys.Track(d)
@@ -200,6 +240,7 @@ func (regManager *RegistrationManager) RemoveOldRegistrations() {
 // DecoyRegistration is a struct for tracking individual sessions that are expecting or tracking connections.
 type DecoyRegistration struct {
 	DarkDecoy          net.IP
+	registrationAddr   net.IP
 	Keys               *ConjureSharedKeys
 	Covert, Mask       string
 	Flags              *pb.RegistrationFlags
@@ -315,6 +356,7 @@ func (reg *DecoyRegistration) GenerateC2SWrapper() *pb.C2SWrapper {
 		SharedSecret:        reg.Keys.SharedSecret,
 		RegistrationPayload: c2s,
 		RegistrationSource:  &source,
+		RegistrationAddress: []byte(reg.registrationAddr),
 	}
 	return protoPayload
 }
@@ -632,33 +674,29 @@ func (r *RegisteredDecoys) removeOldRegistrations(logger *log.Logger) {
 	}
 }
 
+// **NOTE**: If you mess with this function make sure the
+// session tracking tests on the detector side do what you expect
+// them to do. (conjure/src/session.rs)
 func registerForDetector(reg *DecoyRegistration) {
-	client, err := getRedisClient()
-	if err != nil {
+	client := getRedisClient()
+	if client == nil {
 		fmt.Printf("couldn't connect to redis")
-	} else {
-		if reg.DarkDecoy.To4() != nil {
-			client.Publish(DETECTOR_REG_CHANNEL, string(reg.DarkDecoy.To4()))
-		} else {
-			client.Publish(DETECTOR_REG_CHANNEL, string(reg.DarkDecoy.To16()))
-		}
-		client.Close()
+		return
 	}
-}
 
-func getRedisClient() (*redis.Client, error) {
-	var client *redis.Client
-	client = redis.NewClient(&redis.Options{
-		Addr:     "localhost:6379",
-		Password: "",
-		DB:       0,
-		PoolSize: 10,
-	})
+	duration := uint64(3 * time.Minute.Nanoseconds())
+	src := reg.registrationAddr.String()
+	phantom := reg.DarkDecoy.String()
+	msg := &pb.StationToDetector{
+		PhantomIp: &phantom,
+		ClientIp:  &src,
+		TimeoutNs: &duration,
+	}
 
-	_, err := client.Ping().Result()
+	s2d, err := proto.Marshal(msg)
 	if err != nil {
-		return client, err
+		// throw(fit)
+		return
 	}
-
-	return client, err
+	client.Publish(DETECTOR_REG_CHANNEL, string(s2d))
 }