Merge branch 'main' into pub-ip-task

.github/workflows/bin-no-publish.yaml

@@ -20,12 +20,12 @@ jobs:
       - name: Setup basesystem
         run: |
           cd bins
-          sudo ./bins-extra.sh --package basesystem
+          sudo -E ./bins-extra.sh --package basesystem
       - name: Build package (${{ inputs.package }})
         id: package
         run: |
           cd bins
-          sudo ./bins-extra.sh --package ${{ inputs.package }}
+          sudo -E ./bins-extra.sh --package ${{ inputs.package }}
       - name: Publish flist (tf-autobuilder, ${{ steps.package.outputs.name }})
         if: success()
         uses: threefoldtech/publish-flist@master

.github/workflows/bin-package.yaml

@@ -40,12 +40,12 @@ jobs:
       - name: Setup basesystem
         run: |
           cd bins
-          sudo ./bins-extra.sh --package basesystem
+          sudo -E ./bins-extra.sh --package basesystem
       - name: Build package (${{ inputs.package }})
         id: package
         run: |
           cd bins
-          sudo ./bins-extra.sh --package ${{ inputs.package }}
+          sudo -E ./bins-extra.sh --package ${{ inputs.package }}
       - name: Publish flist (tf-autobuilder, ${{ steps.package.outputs.name }})
         if: success()
         uses: threefoldtech/publish-flist@master

bins/bins-extra.sh

@@ -269,7 +269,7 @@ exclude_libs() {
 
 github_name() {
     # force github print
-    echo "name=${1}" >> $env:GITHUB_OUTPUT
+    echo "name=${1}" >> $GITHUB_OUTPUT
     echo "[+] github exported name: ${1}"
 }
 

cmds/internet/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/vishvananda/netlink"
 
 	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/pkg/environment"
 	"github.com/threefoldtech/zos/pkg/network/bootstrap"
 	"github.com/threefoldtech/zos/pkg/network/bridge"
 	"github.com/threefoldtech/zos/pkg/network/dhcp"
@@ -112,12 +113,29 @@ func check() error {
 	return backoff.RetryNotify(f, backoff.NewExponentialBackOff(), errHandler)
 }
 
+/*
+*
+configureZOS bootstraps the private zos network (private subnet) it goes as follows:
+  - Find a physical interface that can get an IPv4 over DHCP
+  - Once interface is found, a bridge called `zos` is created, then the interface that was
+    found in previous step is attached to the zos bridge.
+  - Bridge and interface are brought UP then a DHCP daemon is started on the zos to get an IP.
+
+In case there is a priv vlan is configured (kernel param vlan:priv=<id>) it is basically the same as
+before but with the next twist:
+- During probing of the interface, probing done on that vlan
+- ZOS is added to vlan as `bridge vlan add vid <id> dev zos pvid self untagged`
+- link is added to vlan as `bridge vlan add vid <id> dev <link>`
+*/
 func configureZOS() error {
+
+	env := environment.MustGet()
+
 	f := func() error {
 		log.Info().Msg("Start network bootstrap")
 
 		ifaceConfigs, err := bootstrap.AnalyzeLinks(
-			bootstrap.RequiresIPv4,
+			bootstrap.RequiresIPv4.WithVlan(env.PrivVlan),
 			bootstrap.PhysicalFilter,
 			bootstrap.PluggedFilter)
 		if err != nil {
@@ -134,7 +152,7 @@ func configureZOS() error {
 		}
 
 		log.Info().Str("interface", zosChild).Msg("selecting interface")
-		br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge)
+		br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge, env.PrivVlan)
 		if err != nil {
 			return err
 		}
@@ -167,6 +185,29 @@ func configureZOS() error {
 			return errors.Wrapf(err, "could not bring %s up", zosChild)
 		}
 
+		if env.PrivVlan != nil && env.PubVlan != nil {
+			// if both priv and pub vlan are configured it means
+			// that we can remove the default tagging of vlan 1
+			// remove default
+			if err := netlink.BridgeVlanDel(link, 1, true, true, false, false); err != nil {
+				return errors.Wrapf(err, "failed to delete default vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
+		if env.PrivVlan != nil {
+			// add new vlan
+			if err := netlink.BridgeVlanAdd(link, *env.PrivVlan, false, false, false, false); err != nil {
+				return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
+		if env.PubVlan != nil {
+			// add new vlan
+			if err := netlink.BridgeVlanAdd(link, *env.PubVlan, false, false, false, false); err != nil {
+				return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
 		dhcpService := dhcp.NewService(types.DefaultBridge, "", zinit.Default())
 		if err := dhcpService.DestroyOlderService(); err != nil {
 			log.Error().Err(err).Msgf("failed to destory older %s service", dhcpService.Name)

cmds/modules/networkd/main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
+	"github.com/threefoldtech/zos/pkg/environment"
 	"github.com/threefoldtech/zos/pkg/network/dhcp"
 	"github.com/threefoldtech/zos/pkg/network/public"
 	"github.com/threefoldtech/zos/pkg/network/types"
@@ -98,8 +99,9 @@ func action(cli *cli.Context) error {
 	if err != nil && err != public.ErrNoPublicConfig {
 		return errors.Wrap(err, "failed to get node public_config")
 	}
+
 	// EnsurePublicSetup knows how to handle a nil pub (in case of ErrNoPublicConfig)
-	master, err := public.EnsurePublicSetup(nodeID, pub)
+	master, err := public.EnsurePublicSetup(nodeID, environment.MustGet().PubVlan, pub)
 	if err != nil {
 		return errors.Wrap(err, "failed to setup public bridge")
 	}

cmds/modules/noded/main.go

@@ -22,6 +22,8 @@ import (
 	"github.com/threefoldtech/zos/pkg/monitord"
 	"github.com/threefoldtech/zos/pkg/perf"
 	"github.com/threefoldtech/zos/pkg/perf/publicip"
+	"github.com/threefoldtech/zos/pkg/perf/cpubench"
+	"github.com/threefoldtech/zos/pkg/perf/iperf"
 	"github.com/threefoldtech/zos/pkg/registrar"
 	"github.com/threefoldtech/zos/pkg/stubs"
 	"github.com/threefoldtech/zos/pkg/utils"
@@ -203,10 +205,9 @@ func action(cli *cli.Context) error {
 		return errors.Wrap(err, "failed to create a new perfMon")
 	}
 
-	iperfTest := perf.NewIperfTest()
-	perfMon.AddTask(&iperfTest)
+	perfMon.AddTask(iperf.NewTask())
 
-	cpuBenchmarkTask := perf.NewCPUBenchmarkTask()
+	cpuBenchmarkTask := cpubench.NewCPUBenchmarkTask()
 	perfMon.AddTask(&cpuBenchmarkTask)
 
 	perfMon.AddTask(publicip.NewTask())

docs/internals/boot.md

@@ -15,3 +15,10 @@ both `node-ready` and `boot` are not actual services, but instead they are there
 - `zos-debug`: means zos is running in debug mode
 - `zos-debug-vm`: forces zos to think it's running on a virtual machine. used mainly for development
 - `disable-gpu`: if provided GPU feature will be disabled on that node
+- `vlan:pub`: set the vlan tag of the node private subnet.
+- `vlan:priv`: sets the vlan tag of the node public subnet.
+- `pub:mac`: this accepts two values `random` (default), and `swap`. This flag is only effective in case public-config is set (via the dashboard)
+  - `random`: means the public interface will have a random (driven from the node id) mac address. this works perfectly well for `home` nodes
+  - `swap`: this is useful in case the public ip used in the public-config of the node has to come from the mac address of the physical nic. this flag then will make sure the mac of the physical nic is used by the `public` namespace. This is useful in case you hosting the node in the cloud where the public ip is only allowed to work with the mac assigned to the node physical node
+
+For more details of `VLAN` support in zos please read more [here](network/vlans.md)

docs/internals/network/readme.md

@@ -5,4 +5,5 @@
 - [definitions of the vocabulary used in the documentation](definitions.md)
 - [Introduction to networkd, the network manager of 0-OS](introduction.md)
 - [Detail about the wireguard mesh used to interconnect 0-OS nodes](mesh.md)
-- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md)
+- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md)
+- [VLANS](vlans.md)

docs/internals/network/vlans.md

@@ -0,0 +1,81 @@
+# VLANS
+
+ZOS support vlans by allowing the farmer to setup vlan for both private and public subnets.
+
+By default zos uses untagged traffic for both priv and public subnets (for both single or dual nic nodes). In some data centers and cloud providers, they can only provide tagged subnets.
+
+ZOS can then become VLAN aware by providing optional vlan tags during booting.
+
+## Private VLAN
+
+Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:priv` kernel command line parameter
+
+> Example `vlan:priv=302` will tag all private traffic with VLAN id `302`
+
+During boot, zos tries to find the first interface that has ipv4 (over dhcp) normally all interfaces are probed until one of them actually get an IP. If a vlan ID is set, the probing also happen on the proper vlan, then the private default bridge (called `zos`) is then setup correctly with the proper vlan
+
+```
+                ┌────────────────────────────────────┐
+                │              NODE                  │
+                │                                    │
+  vlan 302 ┌────┴──┐                                 │
+───────────┤  Nic  ├──────────┐                      │
+   tagged  └────┬──┘          │                      │
+                │        ┌────┴─────┐                │
+                │        │          │                │
+                │        │   zos    │  pvid 302      │
+                │        │   bridge ├──untagged      │
+                │        │          │                │
+                │        │          │                │
+                │        └──────────┘                │
+                │                                    │
+                │                                    │
+                │                                    │
+                └────────────────────────────────────┘
+```
+
+## Public VLAN
+
+> NOTE: Public VLAN in ZOS is **only** supported in a single nic setup. There is no support in dual nic yet
+
+Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:pub` kernel command line parameter
+
+> Example `vlan:pub=304` will tag all private traffic with VLAN id `304`
+
+zos internally create a public bridge `br-pub` that can uses a detected ingress link (usually in dual nic setup) or shares
+the same link as `zos` bridge by connecting to `br-pub` via a veth pair.
+
+Single NIC setup
+
+```
+                ┌─────────────────────────────────────────────┐
+                │                                             │
+304 tagged ┌────┴─────┐                                       │
+───────────┤   NIC    ├────────────┐                          │
+           └────┬─────┘            │                          │
+                │                  │                          │
+                │          ┌───────┴─────┐                    │
+                │          │             │                    │
+                │          │      zos    │                    │
+                │          │      bridge │                    │
+                │          │             │                    │
+                │          │             │                    │
+                │          └───────┬─────┘                    │
+                │                  │  pvid 304 untagged       │
+                │                  │                          │
+                │                  │                          │
+                │           ┌──────▼─────┐                    │
+                │           │            │                    │
+                │           │    br-pub  │                    │
+                │           │    bridge  │                    │
+                │           │            │                    │
+                │           │            │                    │
+                │           │            │                    │
+                │           └────────────┘                    │
+                │                                             │
+                └─────────────────────────────────────────────┘
+```
+
+## Dual NIC setup
+
+Right now public vlans are not supported in case of dual nic setups. So in case public network is only available on the second nic then it will always be untagged traffic. This means the `vlan:pub` flag is silently ignored

go.mod

@@ -77,6 +77,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/ethereum/go-ethereum v1.11.6 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -85,6 +86,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect

go.sum

@@ -208,13 +208,17 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/environment/environment.go

@@ -5,6 +5,8 @@ import (
 	"strconv"
 	"sync"
 
+	"slices"
+
 	"github.com/pkg/errors"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zos/pkg"
@@ -16,6 +18,20 @@ const (
 	baseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/"
 )
 
+// PubMac specify how the mac address of the public nic
+// (in case of public-config) is calculated
+type PubMac string
+
+const (
+	// PubMacRandom means the mac of the public nic will be chosen by the system
+	// the value won't change across reboots, but is based on the node id
+	// (default)
+	PubMacRandom PubMac = "random"
+	// PubMacSwap means the value of the mac is swapped with the physical nic
+	// where the public traffic is eventually going through
+	PubMacSwap PubMac = "swap"
+)
+
 // Environment holds information about running environment of a node
 // it defines the different constant based on the running mode (dev, test, prod)
 type Environment struct {
@@ -37,6 +53,18 @@ type Environment struct {
 	GraphQL       string
 
 	ExtendedConfigURL string
+
+	// private vlan to join
+	// if set, zos will use this as its priv vlan
+	PrivVlan *uint16
+
+	// pub vlan to join
+	// if set, zos will use this as it's pub vlan
+	// only in a single nic setup
+	PubVlan *uint16
+
+	// PubMac value from environment
+	PubMac PubMac
 }
 
 // RunningMode type
@@ -250,6 +278,39 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env.FarmID = pkg.FarmID(id)
 	}
 
+	if vlan, found := params.GetOne("vlan:priv"); found {
+		if !slices.Contains([]string{"none", "untagged", "un"}, vlan) {
+			tag, err := strconv.ParseUint(vlan, 10, 16)
+			if err != nil {
+				return env, errors.Wrap(err, "failed to parse priv vlan value")
+			}
+			tagU16 := uint16(tag)
+			env.PrivVlan = &tagU16
+		}
+	}
+
+	if vlan, found := params.GetOne("vlan:pub"); found {
+		if !slices.Contains([]string{"none", "untagged", "un"}, vlan) {
+			tag, err := strconv.ParseUint(vlan, 10, 16)
+			if err != nil {
+				return env, errors.Wrap(err, "failed to parse pub vlan value")
+			}
+			tagU16 := uint16(tag)
+			env.PubVlan = &tagU16
+		}
+	}
+
+	if mac, found := params.GetOne("pub:mac"); found {
+		v := PubMac(mac)
+		if slices.Contains([]PubMac{PubMacRandom, PubMacSwap}, v) {
+			env.PubMac = v
+		} else {
+			env.PubMac = PubMacRandom
+		}
+	} else {
+		env.PubMac = PubMacRandom
+	}
+
 	// Checking if there environment variable
 	// override default settings