diff --git a/.github/workflows/bin-no-publish.yaml b/.github/workflows/bin-no-publish.yaml index 1ef13b1da..1d10f56f9 100644 --- a/.github/workflows/bin-no-publish.yaml +++ b/.github/workflows/bin-no-publish.yaml @@ -20,12 +20,12 @@ jobs: - name: Setup basesystem run: | cd bins - sudo ./bins-extra.sh --package basesystem + sudo -E ./bins-extra.sh --package basesystem - name: Build package (${{ inputs.package }}) id: package run: | cd bins - sudo ./bins-extra.sh --package ${{ inputs.package }} + sudo -E ./bins-extra.sh --package ${{ inputs.package }} - name: Publish flist (tf-autobuilder, ${{ steps.package.outputs.name }}) if: success() uses: threefoldtech/publish-flist@master diff --git a/.github/workflows/bin-package.yaml b/.github/workflows/bin-package.yaml index a32fe5a15..41101015d 100644 --- a/.github/workflows/bin-package.yaml +++ b/.github/workflows/bin-package.yaml @@ -40,12 +40,12 @@ jobs: - name: Setup basesystem run: | cd bins - sudo ./bins-extra.sh --package basesystem + sudo -E ./bins-extra.sh --package basesystem - name: Build package (${{ inputs.package }}) id: package run: | cd bins - sudo ./bins-extra.sh --package ${{ inputs.package }} + sudo -E ./bins-extra.sh --package ${{ inputs.package }} - name: Publish flist (tf-autobuilder, ${{ steps.package.outputs.name }}) if: success() uses: threefoldtech/publish-flist@master diff --git a/bins/bins-extra.sh b/bins/bins-extra.sh index ea257c044..6ec137f17 100755 --- a/bins/bins-extra.sh +++ b/bins/bins-extra.sh @@ -269,7 +269,7 @@ exclude_libs() { github_name() { # force github print - echo "name=${1}" >> $env:GITHUB_OUTPUT + echo "name=${1}" >> $GITHUB_OUTPUT echo "[+] github exported name: ${1}" } diff --git a/cmds/internet/main.go b/cmds/internet/main.go index 8c5161b0b..c42988f88 100644 --- a/cmds/internet/main.go +++ b/cmds/internet/main.go @@ -12,6 +12,7 @@ import ( "github.com/vishvananda/netlink" "github.com/threefoldtech/zos/pkg/app" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/network/bootstrap" "github.com/threefoldtech/zos/pkg/network/bridge" "github.com/threefoldtech/zos/pkg/network/dhcp" @@ -112,12 +113,29 @@ func check() error { return backoff.RetryNotify(f, backoff.NewExponentialBackOff(), errHandler) } +/* +* +configureZOS bootstraps the private zos network (private subnet) it goes as follows: + - Find a physical interface that can get an IPv4 over DHCP + - Once interface is found, a bridge called `zos` is created, then the interface that was + found in previous step is attached to the zos bridge. + - Bridge and interface are brought UP then a DHCP daemon is started on the zos to get an IP. + +In case there is a priv vlan is configured (kernel param vlan:priv=) it is basically the same as +before but with the next twist: +- During probing of the interface, probing done on that vlan +- ZOS is added to vlan as `bridge vlan add vid dev zos pvid self untagged` +- link is added to vlan as `bridge vlan add vid dev ` +*/ func configureZOS() error { + + env := environment.MustGet() + f := func() error { log.Info().Msg("Start network bootstrap") ifaceConfigs, err := bootstrap.AnalyzeLinks( - bootstrap.RequiresIPv4, + bootstrap.RequiresIPv4.WithVlan(env.PrivVlan), bootstrap.PhysicalFilter, bootstrap.PluggedFilter) if err != nil { @@ -134,7 +152,7 @@ func configureZOS() error { } log.Info().Str("interface", zosChild).Msg("selecting interface") - br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge) + br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge, env.PrivVlan) if err != nil { return err } @@ -167,6 +185,29 @@ func configureZOS() error { return errors.Wrapf(err, "could not bring %s up", zosChild) } + if env.PrivVlan != nil && env.PubVlan != nil { + // if both priv and pub vlan are configured it means + // that we can remove the default tagging of vlan 1 + // remove default + if err := netlink.BridgeVlanDel(link, 1, true, true, false, false); err != nil { + return errors.Wrapf(err, "failed to delete default vlan on device '%s'", link.Attrs().Name) + } + } + + if env.PrivVlan != nil { + // add new vlan + if err := netlink.BridgeVlanAdd(link, *env.PrivVlan, false, false, false, false); err != nil { + return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name) + } + } + + if env.PubVlan != nil { + // add new vlan + if err := netlink.BridgeVlanAdd(link, *env.PubVlan, false, false, false, false); err != nil { + return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name) + } + } + dhcpService := dhcp.NewService(types.DefaultBridge, "", zinit.Default()) if err := dhcpService.DestroyOlderService(); err != nil { log.Error().Err(err).Msgf("failed to destory older %s service", dhcpService.Name) diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go index 4d60e9630..d91c1976e 100644 --- a/cmds/modules/networkd/main.go +++ b/cmds/modules/networkd/main.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/network/dhcp" "github.com/threefoldtech/zos/pkg/network/public" "github.com/threefoldtech/zos/pkg/network/types" @@ -98,8 +99,9 @@ func action(cli *cli.Context) error { if err != nil && err != public.ErrNoPublicConfig { return errors.Wrap(err, "failed to get node public_config") } + // EnsurePublicSetup knows how to handle a nil pub (in case of ErrNoPublicConfig) - master, err := public.EnsurePublicSetup(nodeID, pub) + master, err := public.EnsurePublicSetup(nodeID, environment.MustGet().PubVlan, pub) if err != nil { return errors.Wrap(err, "failed to setup public bridge") } diff --git a/cmds/modules/noded/main.go b/cmds/modules/noded/main.go index 5b2ffc221..938142c3d 100644 --- a/cmds/modules/noded/main.go +++ b/cmds/modules/noded/main.go @@ -22,6 +22,8 @@ import ( "github.com/threefoldtech/zos/pkg/monitord" "github.com/threefoldtech/zos/pkg/perf" "github.com/threefoldtech/zos/pkg/perf/publicip" + "github.com/threefoldtech/zos/pkg/perf/cpubench" + "github.com/threefoldtech/zos/pkg/perf/iperf" "github.com/threefoldtech/zos/pkg/registrar" "github.com/threefoldtech/zos/pkg/stubs" "github.com/threefoldtech/zos/pkg/utils" @@ -203,10 +205,9 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "failed to create a new perfMon") } - iperfTest := perf.NewIperfTest() - perfMon.AddTask(&iperfTest) + perfMon.AddTask(iperf.NewTask()) - cpuBenchmarkTask := perf.NewCPUBenchmarkTask() + cpuBenchmarkTask := cpubench.NewCPUBenchmarkTask() perfMon.AddTask(&cpuBenchmarkTask) perfMon.AddTask(publicip.NewTask()) diff --git a/docs/internals/boot.md b/docs/internals/boot.md index 7f01c3fbf..6c49b21e9 100644 --- a/docs/internals/boot.md +++ b/docs/internals/boot.md @@ -15,3 +15,10 @@ both `node-ready` and `boot` are not actual services, but instead they are there - `zos-debug`: means zos is running in debug mode - `zos-debug-vm`: forces zos to think it's running on a virtual machine. used mainly for development - `disable-gpu`: if provided GPU feature will be disabled on that node +- `vlan:pub`: set the vlan tag of the node private subnet. +- `vlan:priv`: sets the vlan tag of the node public subnet. +- `pub:mac`: this accepts two values `random` (default), and `swap`. This flag is only effective in case public-config is set (via the dashboard) + - `random`: means the public interface will have a random (driven from the node id) mac address. this works perfectly well for `home` nodes + - `swap`: this is useful in case the public ip used in the public-config of the node has to come from the mac address of the physical nic. this flag then will make sure the mac of the physical nic is used by the `public` namespace. This is useful in case you hosting the node in the cloud where the public ip is only allowed to work with the mac assigned to the node physical node + +For more details of `VLAN` support in zos please read more [here](network/vlans.md) diff --git a/docs/internals/network/readme.md b/docs/internals/network/readme.md index 0d554dacb..a4e7427cf 100644 --- a/docs/internals/network/readme.md +++ b/docs/internals/network/readme.md @@ -5,4 +5,5 @@ - [definitions of the vocabulary used in the documentation](definitions.md) - [Introduction to networkd, the network manager of 0-OS](introduction.md) - [Detail about the wireguard mesh used to interconnect 0-OS nodes](mesh.md) -- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md) \ No newline at end of file +- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md) +- [VLANS](vlans.md) diff --git a/docs/internals/network/vlans.md b/docs/internals/network/vlans.md new file mode 100644 index 000000000..3b0828041 --- /dev/null +++ b/docs/internals/network/vlans.md @@ -0,0 +1,81 @@ +# VLANS + +ZOS support vlans by allowing the farmer to setup vlan for both private and public subnets. + +By default zos uses untagged traffic for both priv and public subnets (for both single or dual nic nodes). In some data centers and cloud providers, they can only provide tagged subnets. + +ZOS can then become VLAN aware by providing optional vlan tags during booting. + +## Private VLAN + +Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:priv` kernel command line parameter + +> Example `vlan:priv=302` will tag all private traffic with VLAN id `302` + +During boot, zos tries to find the first interface that has ipv4 (over dhcp) normally all interfaces are probed until one of them actually get an IP. If a vlan ID is set, the probing also happen on the proper vlan, then the private default bridge (called `zos`) is then setup correctly with the proper vlan + +``` + ┌────────────────────────────────────┐ + │ NODE │ + │ │ + vlan 302 ┌────┴──┐ │ +───────────┤ Nic ├──────────┐ │ + tagged └────┬──┘ │ │ + │ ┌────┴─────┐ │ + │ │ │ │ + │ │ zos │ pvid 302 │ + │ │ bridge ├──untagged │ + │ │ │ │ + │ │ │ │ + │ └──────────┘ │ + │ │ + │ │ + │ │ + └────────────────────────────────────┘ +``` + +## Public VLAN + +> NOTE: Public VLAN in ZOS is **only** supported in a single nic setup. There is no support in dual nic yet + +Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:pub` kernel command line parameter + +> Example `vlan:pub=304` will tag all private traffic with VLAN id `304` + +zos internally create a public bridge `br-pub` that can uses a detected ingress link (usually in dual nic setup) or shares +the same link as `zos` bridge by connecting to `br-pub` via a veth pair. + +Single NIC setup + +``` + ┌─────────────────────────────────────────────┐ + │ │ +304 tagged ┌────┴─────┐ │ +───────────┤ NIC ├────────────┐ │ + └────┬─────┘ │ │ + │ │ │ + │ ┌───────┴─────┐ │ + │ │ │ │ + │ │ zos │ │ + │ │ bridge │ │ + │ │ │ │ + │ │ │ │ + │ └───────┬─────┘ │ + │ │ pvid 304 untagged │ + │ │ │ + │ │ │ + │ ┌──────▼─────┐ │ + │ │ │ │ + │ │ br-pub │ │ + │ │ bridge │ │ + │ │ │ │ + │ │ │ │ + │ │ │ │ + │ └────────────┘ │ + │ │ + └─────────────────────────────────────────────┘ +``` + +## Dual NIC setup + +Right now public vlans are not supported in case of dual nic setups. So in case public network is only available on the second nic then it will always be untagged traffic. This means the `vlan:pub` flag is silently ignored diff --git a/go.mod b/go.mod index 1dca8315c..6c133ec77 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/ethereum/go-ethereum v1.11.6 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -85,6 +86,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect diff --git a/go.sum b/go.sum index 52db02261..5bcd347da 100644 --- a/go.sum +++ b/go.sum @@ -208,6 +208,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -215,6 +217,8 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index f18bf2e0f..d1c4eb236 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -5,6 +5,8 @@ import ( "strconv" "sync" + "slices" + "github.com/pkg/errors" substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go" "github.com/threefoldtech/zos/pkg" @@ -16,6 +18,20 @@ const ( baseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/" ) +// PubMac specify how the mac address of the public nic +// (in case of public-config) is calculated +type PubMac string + +const ( + // PubMacRandom means the mac of the public nic will be chosen by the system + // the value won't change across reboots, but is based on the node id + // (default) + PubMacRandom PubMac = "random" + // PubMacSwap means the value of the mac is swapped with the physical nic + // where the public traffic is eventually going through + PubMacSwap PubMac = "swap" +) + // Environment holds information about running environment of a node // it defines the different constant based on the running mode (dev, test, prod) type Environment struct { @@ -37,6 +53,18 @@ type Environment struct { GraphQL string ExtendedConfigURL string + + // private vlan to join + // if set, zos will use this as its priv vlan + PrivVlan *uint16 + + // pub vlan to join + // if set, zos will use this as it's pub vlan + // only in a single nic setup + PubVlan *uint16 + + // PubMac value from environment + PubMac PubMac } // RunningMode type @@ -250,6 +278,39 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) { env.FarmID = pkg.FarmID(id) } + if vlan, found := params.GetOne("vlan:priv"); found { + if !slices.Contains([]string{"none", "untagged", "un"}, vlan) { + tag, err := strconv.ParseUint(vlan, 10, 16) + if err != nil { + return env, errors.Wrap(err, "failed to parse priv vlan value") + } + tagU16 := uint16(tag) + env.PrivVlan = &tagU16 + } + } + + if vlan, found := params.GetOne("vlan:pub"); found { + if !slices.Contains([]string{"none", "untagged", "un"}, vlan) { + tag, err := strconv.ParseUint(vlan, 10, 16) + if err != nil { + return env, errors.Wrap(err, "failed to parse pub vlan value") + } + tagU16 := uint16(tag) + env.PubVlan = &tagU16 + } + } + + if mac, found := params.GetOne("pub:mac"); found { + v := PubMac(mac) + if slices.Contains([]PubMac{PubMacRandom, PubMacSwap}, v) { + env.PubMac = v + } else { + env.PubMac = PubMacRandom + } + } else { + env.PubMac = PubMacRandom + } + // Checking if there environment variable // override default settings diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go index 1351435ed..0d83feb96 100644 --- a/pkg/flist/flist.go +++ b/pkg/flist/flist.go @@ -207,25 +207,35 @@ func (f *flistModule) mountRO(url, storage string) (string, error) { } logPath := filepath.Join(f.log, hash) + ".log" - var args []string - - args = append(args, + flistExt := filepath.Ext(url) + args := []string{ "--cache", f.cache, "--meta", flistPath, - "--storage-url", storage, "--daemon", "--log", logPath, - // this is always read-only - "--ro", - mountpoint, - ) + } - sublog.Info().Strs("args", args).Msg("starting 0-fs daemon") - cmd := f.commander.Command("g8ufs", args...) + var cmd *exec.Cmd + if flistExt == ".flist" { + sublog.Info().Strs("args", args).Msg("starting g8ufs daemon") + args = append(args, + "--storage-url", storage, + // this is always read-only + "--ro", + mountpoint, + ) + cmd = f.commander.Command("g8ufs", args...) + } else if flistExt == ".fl" { + sublog.Info().Strs("args", args).Msg("starting rfs daemon") + args = append([]string{"mount"}, append(args, mountpoint)...) + cmd = f.commander.Command("rfs", args...) + } else { + return "", errors.Errorf("unknown extension: '%s'", flistExt) + } var out []byte if out, err = cmd.CombinedOutput(); err != nil { - sublog.Err(err).Str("out", string(out)).Msg("fail to start 0-fs daemon") + sublog.Err(err).Str("out", string(out)).Msg("failed to start 0-fs daemon") return "", err } diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go index 5d22bb527..dfb15af2f 100644 --- a/pkg/kernel/kernel.go +++ b/pkg/kernel/kernel.go @@ -38,6 +38,22 @@ func (k Params) Get(key string) ([]string, bool) { return v, ok } +// GetOne gets a single value for given key. If key is provided +// multiple times in the cmdline, the last one is used. If key does +// not exist, or has no associated value, a false is returned +func (k Params) GetOne(key string) (string, bool) { + all, found := k.Get(key) + if !found { + return "", false + } + + if len(all) == 0 { + return "", false + } + + return all[len(all)-1], true +} + // IsDebug checks if zos-debug is set func (k Params) IsDebug() bool { return k.Exists(Debug) diff --git a/pkg/network.go b/pkg/network.go index 75fa95058..74f3f4e70 100644 --- a/pkg/network.go +++ b/pkg/network.go @@ -14,14 +14,6 @@ import ( //go:generate mkdir -p stubs //go:generate zbusc -module network -version 0.0.1 -name network -package stubs github.com/threefoldtech/zos/pkg+Networker stubs/network_stub.go -// Member holds information about a the network namespace of a container -type Member struct { - Namespace string - IPv6 net.IP - IPv4 net.IP - YggdrasilIP net.IP -} - // ContainerNetworkConfig defines how to construct the network namespace of a container type ContainerNetworkConfig struct { IPs []string diff --git a/pkg/network/bootstrap/bootstrap.go b/pkg/network/bootstrap/bootstrap.go index b22728e42..d822d9704 100644 --- a/pkg/network/bootstrap/bootstrap.go +++ b/pkg/network/bootstrap/bootstrap.go @@ -31,16 +31,35 @@ type IfaceConfig struct { DefaultGW net.IP } -// Requires tesll the analyser to wait for ip type -type Requires int +// Requires tells the analyzer to wait for ip type +type Requires struct { + ipv4 bool + ipv6 bool + vlan *uint16 +} -const ( +var ( // RequiresIPv4 requires ipv4 - RequiresIPv4 Requires = 1 << iota + RequiresIPv4 = Requires{ipv4: true} // RequiresIPv6 requires ipv6 - RequiresIPv6 + RequiresIPv6 = Requires{ipv6: true} ) +func (r Requires) WithIPv4(b bool) Requires { + r.ipv4 = b + return r +} + +func (r Requires) WithIPv6(b bool) Requires { + r.ipv6 = b + return r +} + +func (r Requires) WithVlan(b *uint16) Requires { + r.vlan = b + return r +} + type byIP4 []IfaceConfig func (a byIP4) Len() int { return len(a) } @@ -118,7 +137,7 @@ filter: defer cancel() for _, link := range filtered { wg.Add(1) - analyzeLink(ctx, &wg, ch, requires, link) + analyzeLinkAsync(ctx, &wg, ch, requires, link) } go func() { @@ -138,8 +157,8 @@ filter: return configs, nil } -// AnalyzeLink gets information about link -func AnalyzeLink(ctx context.Context, requires Requires, link netlink.Link) (cfg IfaceConfig, err error) { +// analyzeLink gets information about link +func analyzeLink(ctx context.Context, requires Requires, link netlink.Link) (cfg IfaceConfig, err error) { cfg.Name = link.Attrs().Name if link.Attrs().MasterIndex != 0 { // this is to avoid breaking setups if a link is @@ -170,15 +189,51 @@ func AnalyzeLink(ctx context.Context, requires Requires, link netlink.Link) (cfg }() name := link.Attrs().Name + + if err := options.Set(name, options.IPv6Disable(false)); err != nil { + return errors.Wrapf(err, "failed to enable ip6 on %s", name) + } + cfg.Name = name + if requires.vlan != nil { + vlanID := *requires.vlan + log.Debug().Uint32("vlan", uint32(vlanID)).Msg("setting up vlan interface for probing") + name = fmt.Sprintf("%s.%d", name, vlanID) + vl := netlink.Vlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: name, + ParentIndex: link.Attrs().Index, + }, + VlanId: int(vlanID), + VlanProtocol: netlink.VLAN_PROTOCOL_8021Q, + } + + if err := netlink.LinkAdd(&vl); err != nil { + return errors.Wrap(err, "failed to create vlan link for probing") + } + + link, err = netlink.LinkByName(name) + if err != nil { + return errors.Wrap(err, "failed to get vlan link for probing") + } + + if err := netlink.LinkSetUp(link); err != nil { + return errors.Wrap(err, "failed to set up vlan link for probing") + } + + defer func() { + _ = netlink.LinkDel(link) + }() + } + if err := options.Set(name, options.IPv6Disable(false)); err != nil { return errors.Wrapf(err, "failed to enable ip6 on %s", name) } log.Info().Str("interface", name).Msg("start DHCP probe") - if requires&RequiresIPv4 != 0 { + if requires.ipv4 { // requires IPv4 probe, err := dhcp.Probe(ctx, name) if err != nil { @@ -213,7 +268,7 @@ func AnalyzeLink(ctx context.Context, requires Requires, link netlink.Link) (cfg } } - if (requires&RequiresIPv6 == 0) || addrs6.Len() != 0 { + if !requires.ipv6 || addrs6.Len() != 0 { break loop } @@ -233,10 +288,10 @@ type analyzeResult struct { err error } -func analyzeLink(ctx context.Context, wg *sync.WaitGroup, out chan<- analyzeResult, requires Requires, link netlink.Link) { +func analyzeLinkAsync(ctx context.Context, wg *sync.WaitGroup, out chan<- analyzeResult, requires Requires, link netlink.Link) { go func() { defer wg.Done() - cfg, err := AnalyzeLink(ctx, requires, link) + cfg, err := analyzeLink(ctx, requires, link) out <- analyzeResult{cfg, err} }() } diff --git a/pkg/network/bootstrap/bridge.go b/pkg/network/bootstrap/bridge.go index b3d65492c..249c7893a 100644 --- a/pkg/network/bootstrap/bridge.go +++ b/pkg/network/bootstrap/bridge.go @@ -52,7 +52,7 @@ func DefaultBridgeValid() error { // CreateDefaultBridge creates the default bridge of the node that will received // the management interface -func CreateDefaultBridge(name string) (*netlink.Bridge, error) { +func CreateDefaultBridge(name string, vlan *uint16) (*netlink.Bridge, error) { log.Info().Msg("Create default bridge") br, err := bridge.New(name) if err != nil { @@ -67,5 +67,17 @@ func CreateDefaultBridge(name string) (*netlink.Bridge, error) { return nil, errors.Wrapf(err, "failed to disable ipv6 forwarding") } + if vlan == nil { + return br, nil + } + + if err := netlink.BridgeVlanDel(br, 1, true, true, true, false); err != nil { + return nil, errors.Wrap(err, "failed to delete default vlan tag") + } + + if err := netlink.BridgeVlanAdd(br, *vlan, true, true, true, false); err != nil { + return nil, errors.Wrap(err, "failed to set vlan for priv network") + } + return br, nil } diff --git a/pkg/network/bridge/bridge.go b/pkg/network/bridge/bridge.go index b10184315..e7a3230f8 100644 --- a/pkg/network/bridge/bridge.go +++ b/pkg/network/bridge/bridge.go @@ -5,6 +5,7 @@ import ( "os" "github.com/pkg/errors" + "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg/network/ifaceutil" "github.com/threefoldtech/zos/pkg/network/options" "github.com/vishvananda/netlink" @@ -15,7 +16,11 @@ func New(name string) (*netlink.Bridge, error) { attrs := netlink.NewLinkAttrs() attrs.Name = name attrs.MTU = 1500 - bridge := &netlink.Bridge{LinkAttrs: attrs} + enable := true + bridge := &netlink.Bridge{ + LinkAttrs: attrs, + VlanFiltering: &enable, + } if err := netlink.LinkAdd(bridge); err != nil && !os.IsExist(err) { return bridge, err @@ -97,9 +102,13 @@ func vethName(from, to string) string { // can be directly plugged or crossed over with a veth pair // if name is provided, the name will be used in case of veth pair instead of // a generated name -func Attach(link netlink.Link, bridge *netlink.Bridge, name ...string) error { +func Attach(link netlink.Link, bridge *netlink.Bridge, vlan *uint16, name ...string) error { if link.Type() == "device" { - return AttachNic(link, bridge) + if vlan != nil { + log.Warn().Msg("vlan is not supported in dual nic setup") + } + + return attachNic(link, bridge, nil) } else if link.Type() == "bridge" { linkBr := link.(*netlink.Bridge) n := vethName(link.Attrs().Name, bridge.Name) @@ -112,14 +121,14 @@ func Attach(link netlink.Link, bridge *netlink.Bridge, name ...string) error { return err } - return AttachNic(veth, linkBr) + return attachNic(veth, linkBr, vlan) } return fmt.Errorf("unsupported link type '%s'", link.Type()) } -// AttachNic attaches an interface to a bridge -func AttachNic(link netlink.Link, bridge *netlink.Bridge) error { +// attachNic attaches an interface to a bridge +func attachNic(link netlink.Link, bridge *netlink.Bridge, vlan *uint16) error { // Jan said this was fine if err := netlink.LinkSetUp(link); err != nil { return errors.Wrap(err, "could not set veth peer up") @@ -129,7 +138,23 @@ func AttachNic(link netlink.Link, bridge *netlink.Bridge) error { if err := options.Set(link.Attrs().Name, options.IPv6Disable(true)); err != nil { return errors.Wrap(err, "failed to disable ipv6 on link interface") } - return netlink.LinkSetMaster(link, bridge) + if err := netlink.LinkSetMaster(link, bridge); err != nil { + return errors.Wrapf(err, "failed to attach link %s to bridge %s", link.Attrs().Name, bridge.Name) + } + + if vlan == nil { + return nil + } + + if err := netlink.BridgeVlanDel(link, 1, true, true, false, false); err != nil { + return errors.Wrapf(err, "failed to delete default vlan tag on device '%s'", link.Attrs().Name) + } + + if err := netlink.BridgeVlanAdd(link, *vlan, true, true, false, false); err != nil { + return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name) + } + + return nil } // List all nics attached to a bridge diff --git a/pkg/network/bridge/bridge_test.go b/pkg/network/bridge/bridge_test.go index 70515f84e..d45fa3eb7 100644 --- a/pkg/network/bridge/bridge_test.go +++ b/pkg/network/bridge/bridge_test.go @@ -67,7 +67,7 @@ func TestAttachBridge(t *testing.T) { _ = netlink.LinkDel(dummy) }() - err = AttachNic(dummy, br) + err = attachNic(dummy, br, nil) assert.NoError(t, err) } diff --git a/pkg/network/networker.go b/pkg/network/networker.go index 71f3a9b50..1963885f5 100644 --- a/pkg/network/networker.go +++ b/pkg/network/networker.go @@ -17,6 +17,7 @@ import ( "github.com/blang/semver" "github.com/threefoldtech/zos/pkg/cache" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/network/bootstrap" @@ -932,7 +933,7 @@ func (n *networker) Namespace(id zos.NetID) string { func (n *networker) UnsetPublicConfig() error { id := n.identity.NodeID(context.Background()) - _, err := public.EnsurePublicSetup(id, nil) + _, err := public.EnsurePublicSetup(id, environment.MustGet().PubVlan, nil) return err } @@ -953,7 +954,7 @@ func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error { } id := n.identity.NodeID(context.Background()) - _, err = public.EnsurePublicSetup(id, &cfg) + _, err = public.EnsurePublicSetup(id, environment.MustGet().PubVlan, &cfg) if err != nil { return errors.Wrap(err, "failed to apply public config") } diff --git a/pkg/network/nr/container.go b/pkg/network/nr/container.go deleted file mode 100644 index 93735b5f0..000000000 --- a/pkg/network/nr/container.go +++ /dev/null @@ -1,195 +0,0 @@ -package nr - -import ( - "fmt" - "net" - "os" - - "github.com/containernetworking/plugins/pkg/ip" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/vishvananda/netlink" -) - -// ContainerConfig is an object used to pass the required network configuration -// for a container -type ContainerConfig struct { - ContainerID string - IPs []net.IP - PublicIP6 bool //true if the container must have a public ipv6 - IPv4Only bool // describe the state of the node, true mean it runs in ipv4 only mode -} - -// Join make a network namespace of a container join a network resource network -func (nr *NetResource) Join(cfg ContainerConfig) (join pkg.Member, err error) { - name, err := nr.BridgeName() - if err != nil { - return join, err - } - - br, err := bridge.Get(name) - if err != nil { - return join, err - } - - join.Namespace = cfg.ContainerID - netspace, err := namespace.Create(cfg.ContainerID) - if err != nil { - return join, err - } - - slog := log.With(). - Str("namespace", cfg.ContainerID). - Str("container", cfg.ContainerID). - Logger() - - defer func() { - if err != nil { - _ = namespace.Delete(netspace) - } - }() - - var hostVethName string - err = netspace.Do(func(host ns.NetNS) error { - if err := ifaceutil.SetLoUp(); err != nil { - return err - } - - slog.Info(). - Str("veth", "eth0"). - Msg("Create veth pair in net namespace") - hostVeth, containerVeth, err := ip.SetupVeth("eth0", 1500, host) - if err != nil { - return errors.Wrapf(err, "failed to create veth pair in namespace (%s)", join.Namespace) - } - - hostVethName = hostVeth.Name - - eth0, err := netlink.LinkByName(containerVeth.Name) - if err != nil { - return err - } - - for _, addr := range cfg.IPs { - slog.Info(). - Str("ip", addr.String()). - Msgf("set ip to container") - - if err := netlink.AddrAdd(eth0, &netlink.Addr{IPNet: &net.IPNet{ - IP: addr, - Mask: net.CIDRMask(24, 32), - }}); err != nil && !os.IsExist(err) { - return err - } - join.IPv4 = addr - } - - // if the node is IPv6 enabled and the user do not requires a public IPv6 - // then we create derive one to allow IPv6 traffic to go out - // if the user ask for a public IPv6, then the all config comes from SLAAC so we don't have to do anything ourself - if !cfg.IPv4Only && !cfg.PublicIP6 { - ipv6 := Convert4to6(nr.ID(), cfg.IPs[0]) - slog.Info(). - Str("ip", ipv6.String()). - Msgf("set ip to container") - - if err := netlink.AddrAdd(eth0, &netlink.Addr{IPNet: &net.IPNet{ - IP: ipv6, - Mask: net.CIDRMask(64, 128), - }}); err != nil && !os.IsExist(err) { - return err - } - join.IPv6 = ipv6 - } - - ipnet := nr.resource.Subnet - //sanity check this should be already handle by validate. - //but in case something went wrong. - if len(ipnet.IP) == 0 { - return fmt.Errorf("invalid network resource (%s): empty subnet", nr.id) - } - - ipnet.IP[len(ipnet.IP)-1] = 0x01 - - routes := []*netlink.Route{ - { - Dst: &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - }, - Gw: ipnet.IP, - LinkIndex: eth0.Attrs().Index, - }, - } - - // same logic as before, we set ipv6 routes only if this is required - if !cfg.IPv4Only && !cfg.PublicIP6 { - routes = append(routes, - &netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("::"), - Mask: net.CIDRMask(0, 128), - }, - Gw: net.ParseIP("fe80::1"), - LinkIndex: eth0.Attrs().Index, - }) - } - - for _, r := range routes { - slog.Info(). - Str("route", r.String()). - Msgf("set route to container") - err = netlink.RouteAdd(r) - if err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to set route %s on eth0", r.String()) - } - } - - return nil - }) - - if err != nil { - return join, err - } - - hostVeth, err := netlink.LinkByName(hostVethName) - if err != nil { - return join, err - } - - if err := options.Set(hostVeth.Attrs().Name, options.IPv6Disable(true)); err != nil { - return join, errors.Wrapf(err, "failed to disable ip6 on bridge %s", hostVeth.Attrs().Name) - } - - return join, bridge.AttachNic(hostVeth, br) -} - -// Leave delete a container network namespace -func (nr *NetResource) Leave(containerID string) error { - log.Info(). - Str("namespace", containerID). - Str("container", containerID). - Msg("delete container network namespace") - - namespc, err := namespace.GetByName(containerID) - if _, ok := err.(ns.NSPathNotExistErr); ok { - return nil - } else if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer namespc.Close() - - err = namespace.Delete(namespc) - if err != nil { - return err - } - return nil -} diff --git a/pkg/network/options/interface.go b/pkg/network/options/interface.go index 6832248e9..be608fe9c 100644 --- a/pkg/network/options/interface.go +++ b/pkg/network/options/interface.go @@ -22,7 +22,7 @@ func (s *sysOption) apply(inf string) error { // IPv6Disable disabled Ipv6 on interface func IPv6Disable(f bool) Option { return &sysOption{ - key: "net.ipv6.conf.%s.disable_ipv6", + key: "net/ipv6/conf/%s/disable_ipv6", val: flag(f), } } @@ -30,7 +30,7 @@ func IPv6Disable(f bool) Option { // ProxyArp sets proxy arp on interface func ProxyArp(f bool) Option { return &sysOption{ - key: "net.ipv4.conf.%s.proxy_arp", + key: "net/ipv4/conf/%s/proxy_arp", val: flag(f), } } @@ -38,7 +38,7 @@ func ProxyArp(f bool) Option { // AcceptRA enables or disables forwarding for ipv6 func AcceptRA(f RouterAdvertisements) Option { return &sysOption{ - key: "net.ipv6.conf.%s.accept_ra", + key: "net/ipv6/conf/%s/accept_ra", val: fmt.Sprintf("%d", f), } } @@ -46,7 +46,7 @@ func AcceptRA(f RouterAdvertisements) Option { // LearnDefaultRouteInRA Learn default router in Router Advertisement. func LearnDefaultRouteInRA(f bool) Option { return &sysOption{ - key: "net.ipv6.conf.%s.accept_ra_defrtr", + key: "net/ipv6/conf/%s/accept_ra_defrtr", val: flag(f), } } diff --git a/pkg/network/public/public.go b/pkg/network/public/public.go index 193eadcc4..57345837b 100644 --- a/pkg/network/public/public.go +++ b/pkg/network/public/public.go @@ -1,6 +1,7 @@ package public import ( + "bytes" "fmt" "net" "os" @@ -10,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/network/bootstrap" "github.com/threefoldtech/zos/pkg/network/bridge" @@ -31,6 +33,7 @@ const ( // PublicBridge public bridge name, exists only after a call to EnsurePublicSetup PublicBridge = types.PublicBridge + DefaultBridge = types.DefaultBridge PublicNamespace = types.PublicNamespace defaultPublicResolveConf = `nameserver 8.8.8.8 @@ -97,7 +100,7 @@ func IPs() ([]net.IPNet, error) { return ips, err } -func setupPublicBridge(br *netlink.Bridge) error { +func setupPublicBridge(br *netlink.Bridge, vlan *uint16) error { exit, err := detectExitNic() if err != nil { return errors.Wrap(err, "failed to find possible exit") @@ -109,41 +112,29 @@ func setupPublicBridge(br *netlink.Bridge) error { return errors.Wrapf(err, "failed to get link '%s' by name", exit) } - return attachPublicToExit(br, exitLink) + return attachPublicToExit(br, exitLink, vlan) } -func attachPublicToExit(br *netlink.Bridge, exit netlink.Link) error { +func attachPublicToExit(br *netlink.Bridge, exit netlink.Link, vlan *uint16) error { if err := netlink.LinkSetUp(exit); err != nil { return errors.Wrapf(err, "failed to set link '%s' up", exit.Attrs().Name) } - if err := bridge.Attach(exit, br, toZosVeth); err != nil { + if err := bridge.Attach(exit, br, vlan, toZosVeth); err != nil { return errors.Wrap(err, "failed to attach exit nic to public bridge 'br-pub'") } return nil } -func GetCurrentPublicExitLink() (netlink.Link, error) { - // return the upstream (exit) link for br-pub - br, err := bridge.Get(PublicBridge) - if err != nil { - return nil, errors.Wrap(err, "no public bridge found") - } - +// finds a link that is connected to that bridge that matches specific filter criteria +// the first link that matches is returned +func getUplink(br *netlink.Bridge, matches ...bootstrap.Filter) (netlink.Link, error) { all, err := netlink.LinkList() if err != nil { return nil, errors.Wrap(err, "failed to list node nics") } - // public bridge can be wired to either - matches := []bootstrap.Filter{ - // a nic - bootstrap.PhysicalFilter, - // a veth pair to another bridge (zos always) - bootstrap.VEthFilter, - } - for _, link := range all { for _, match := range matches { if ok, _ := match(link); !ok { @@ -159,6 +150,48 @@ func GetCurrentPublicExitLink() (netlink.Link, error) { return nil, os.ErrNotExist } +// GetCurrentPublicExitLink basically find how the public bridge (br-pub) +// is wired to the outside world. This can be either directly to a physical nic device +// or over zos bridge via a veth pair. +// in either way, that link is returned +// if a veth link is returned this means that the node is running +// in a single nic setup, if a physical device is returned then it means +// the node is running in a multi-mode setup +func GetCurrentPublicExitLink() (netlink.Link, error) { + // return the upstream (exit) link for br-pub + br, err := bridge.Get(PublicBridge) + if err != nil { + return nil, errors.Wrap(err, "no public bridge found") + } + + // since public `br-pub` exit can either be directly to a + // physical nic or to zos via an Veth pair then + // this tries to match over these 2 kinds + + return getUplink( + br, + // a nic + bootstrap.PhysicalFilter, + // a veth pair to another bridge (zos always) + bootstrap.VEthFilter) +} + +// GetPrivateExitLink returns the physical link zos is wired to +func GetPrivateExitLink() (netlink.Link, error) { + // return the upstream (exit) link for br-pub + br, err := bridge.Get(DefaultBridge) + if err != nil { + return nil, errors.Wrap(err, "no default bridge found") + } + + // zos bridge can only be connected to the outside world + // over a physical nic. + + return getUplink( + br, + bootstrap.PhysicalFilter) +} + // SetPublicExitLink rewires the br-pub to a different exit (upstream) device. // this upstream device can either be a physical free device, or zos bridge. // the method is idempotent. @@ -214,7 +247,7 @@ func SetPublicExitLink(link netlink.Link) error { } } - return attachPublicToExit(br, link) + return attachPublicToExit(br, link, environment.MustGet().PubVlan) } func HasPublicSetup() bool { @@ -303,7 +336,7 @@ func GetPublicSetup() (pkg.PublicConfig, error) { // // if no nic is found zos is selected. // changes to the br-pub exit nic can then be done later with SetPublicExitLink -func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.Bridge, error) { +func EnsurePublicSetup(nodeID pkg.Identifier, vlan *uint16, inf *pkg.PublicConfig) (*netlink.Bridge, error) { log.Debug().Msg("ensure public setup") br, err := ensurePublicBridge() if err != nil { @@ -314,7 +347,7 @@ func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.B if os.IsNotExist(err) { // bridge is not initialized, wire it. log.Debug().Msg("no public bridge uplink found, setting up...") - if err := setupPublicBridge(br); err != nil { + if err := setupPublicBridge(br, vlan); err != nil { return nil, err } } else if err != nil { @@ -515,6 +548,52 @@ func setupPublicNS(nodeID pkg.Identifier, iface *pkg.PublicConfig) error { } mac := ifaceutil.HardwareAddrFromInputBytes([]byte(nodeID.Identity() + publicNsMACDerivationSuffix)) + + env, err := environment.Get() + if err != nil { + return errors.Wrap(err, "failed to get environment") + } + + if env.PubMac == environment.PubMacSwap { + // this logic can be tricky. the idea is we need to + // swap the mac address of the uplink (where public traffic is eventually going out) + // with thee pubIface calculated above! + // but this swapping of course need to happen once. + // so first: + // - find the nic + // - find if the nic has already the predicted mac above + // - if not we take that one here and use it and replace + // the one of the nic with `mac` + // - if it already matches, then the swap was done already + log.Info().Msg("public mac should be swapped with upstream nic") + up, err := getPublicUpstream() + if err != nil { + return errors.Wrap(err, "failed to detect the public upstream device") + } + + if bytes.Equal(up.Attrs().HardwareAddr, mac) { + //the swap was done! + // we set mac to nil so the macvlan.Install does not change + // it + log.Info().Msg("public mac already swapped") + mac = nil + } else { + // if not, we then need to set the mac of the nic to that mac + // and then use the nic mac for the public interface + newMac := up.Attrs().HardwareAddr + + log.Info(). + Str("nic", mac.String()). + Str("bridge", newMac.String()). + Msg("swapping public interface with nic mac") + + if err := netlink.LinkSetHardwareAddr(up, mac); err != nil { + return errors.Wrap(err, "failed to set public uplink mac address") + } + mac = newMac + } + } + if err := macvlan.Install(pubIface, mac, ips, routes, pubNS); err != nil { return err } @@ -550,3 +629,20 @@ func setupPublicNS(nodeID pkg.Identifier, iface *pkg.PublicConfig) error { return nil } + +func getPublicUpstream() (netlink.Link, error) { + link, err := GetCurrentPublicExitLink() + if err != nil { + return nil, errors.Wrap(err, "failed to get current public link") + } + + // this link above can be a physical (device) link + // then we return that + if physical, _ := bootstrap.PhysicalFilter(link); physical { + return link, nil + } + + // otherwise, it must be the veth to zos bridge! + // so we do + return GetPrivateExitLink() +} diff --git a/pkg/perf/cpubench_task.go b/pkg/perf/cpubench/cpubench_task.go similarity index 93% rename from pkg/perf/cpubench_task.go rename to pkg/perf/cpubench/cpubench_task.go index 2e387427f..7e37cb496 100644 --- a/pkg/perf/cpubench_task.go +++ b/pkg/perf/cpubench/cpubench_task.go @@ -1,4 +1,4 @@ -package perf +package cpubench import ( "context" @@ -6,6 +6,7 @@ import ( "fmt" "os/exec" + "github.com/threefoldtech/zos/pkg/perf" "github.com/threefoldtech/zos/pkg/stubs" ) @@ -28,7 +29,7 @@ type CPUBenchmarkResult struct { Workloads int `json:"workloads"` } -var _ Task = (*CPUBenchmarkTask)(nil) +var _ perf.Task = (*CPUBenchmarkTask)(nil) // NewCPUBenchmarkTask returns a new CPU benchmark task. func NewCPUBenchmarkTask() CPUBenchmarkTask { @@ -59,7 +60,7 @@ func (c *CPUBenchmarkTask) Run(ctx context.Context) (interface{}, error) { if err != nil { return nil, fmt.Errorf("failed to parse cpubench output: %w", err) } - client := GetZbusClient(ctx) + client := perf.GetZbusClient(ctx) statistics := stubs.NewStatisticsStub(client) workloads, err := statistics.Workloads(ctx) diff --git a/pkg/perf/graphql_nodes.go b/pkg/perf/iperf/graphql_nodes.go similarity index 99% rename from pkg/perf/graphql_nodes.go rename to pkg/perf/iperf/graphql_nodes.go index 65f2c4cd3..302f513e8 100644 --- a/pkg/perf/graphql_nodes.go +++ b/pkg/perf/iperf/graphql_nodes.go @@ -1,4 +1,4 @@ -package perf +package iperf import ( "context" diff --git a/pkg/perf/iperf_task.go b/pkg/perf/iperf/iperf_task.go similarity index 55% rename from pkg/perf/iperf_task.go rename to pkg/perf/iperf/iperf_task.go index ec419bfb8..1ed13f780 100644 --- a/pkg/perf/iperf_task.go +++ b/pkg/perf/iperf/iperf_task.go @@ -1,15 +1,19 @@ -package perf +package iperf import ( "context" + "encoding/json" + "fmt" "net" + "os" "os/exec" + "path/filepath" - goIperf "github.com/BGrewell/go-iperf" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/network/iperf" + "github.com/threefoldtech/zos/pkg/perf" ) // IperfTest for iperf tcp/udp tests @@ -20,18 +24,24 @@ type IperfTest struct { // IperfResult for iperf test results type IperfResult struct { - UploadSpeed float64 `json:"upload_speed"` // in bit/sec - DownloadSpeed float64 `json:"download_speed"` // in bit/sec - NodeID uint32 `json:"node_id"` - NodeIpv4 string `json:"node_ip"` - TestType string `json:"test_type"` - Error string `json:"error"` - CpuReport goIperf.CpuUtilizationReport `json:"cpu_report"` + UploadSpeed float64 `json:"upload_speed"` // in bit/sec + DownloadSpeed float64 `json:"download_speed"` // in bit/sec + NodeID uint32 `json:"node_id"` + NodeIpv4 string `json:"node_ip"` + TestType string `json:"test_type"` + Error string `json:"error"` + CpuReport CPUUtilizationPercent `json:"cpu_report"` } -// NewIperfTest creates a new iperf test -func NewIperfTest() IperfTest { - return IperfTest{taskID: "iperf", schedule: "0 0 */6 * * *"} +// NewTask creates a new iperf test +func NewTask() perf.Task { + // because go-iperf left tmp directories with perf binary in it each time + // the task had run + matches, _ := filepath.Glob("/tmp/goiperf*") + for _, match := range matches { + os.RemoveAll(match) + } + return &IperfTest{taskID: "iperf", schedule: "0 0 */6 * * *"} } // ID returns the ID of the tcp task @@ -104,35 +114,46 @@ func (t *IperfTest) Run(ctx context.Context) (interface{}, error) { } func (t *IperfTest) runIperfTest(ctx context.Context, clientIP string, tcp bool) IperfResult { - iperfClient := goIperf.NewClient(clientIP) - iperfClient.SetBandwidth("1M") - iperfClient.SetPort(iperf.IperfPort) - iperfClient.SetInterval(20) - iperfClient.SetJSON(true) + opts := make([]string, 0) + opts = append(opts, + "--client", clientIP, + "--bandwidth", "1M", + "--port", fmt.Sprint(iperf.IperfPort), + "--interval", "20", + "--json", + ) if !tcp { - iperfClient.SetLength("16B") - iperfClient.SetProto(goIperf.PROTO_UDP) + opts = append(opts, "--length", "16B", "--udp") } - - err := iperfClient.Start() - if err != nil { - log.Error().Err(err).Msgf("failed to start iperf client with ip '%s'", clientIP) + output, err := exec.CommandContext(ctx, "iperf", opts...).CombinedOutput() + exitErr := &exec.ExitError{} + if err != nil && !errors.As(err, &exitErr) { + log.Err(err).Msg("failed to run iperf") + return IperfResult{} } - <-iperfClient.Done + var report iperfCommandOutput + if err := json.Unmarshal(output, &report); err != nil { + log.Err(err).Msg("failed to parse iperf output") + return IperfResult{} + } + proto := "tcp" + if !tcp { + proto = "udp" + } iperfResult := IperfResult{ - UploadSpeed: iperfClient.Report().End.SumSent.BitsPerSecond, - DownloadSpeed: iperfClient.Report().End.SumReceived.BitsPerSecond, - CpuReport: iperfClient.Report().End.CpuReport, + UploadSpeed: report.End.SumSent.BitsPerSecond, + DownloadSpeed: report.End.SumReceived.BitsPerSecond, + CpuReport: report.End.CPUUtilizationPercent, NodeIpv4: clientIP, - TestType: string(iperfClient.Proto()), - Error: iperfClient.Report().Error, + TestType: proto, + Error: report.Error, } - if !tcp && len(iperfClient.Report().End.Streams) > 0 { - iperfResult.DownloadSpeed = iperfClient.Report().End.Streams[0].Udp.BitsPerSecond + if !tcp && len(report.End.Streams) > 0 { + iperfResult.DownloadSpeed = report.End.Streams[0].UDP.BitsPerSecond } return iperfResult diff --git a/pkg/perf/iperf/iperf_types.go b/pkg/perf/iperf/iperf_types.go new file mode 100644 index 000000000..d79ff361e --- /dev/null +++ b/pkg/perf/iperf/iperf_types.go @@ -0,0 +1,134 @@ +package iperf + +type iperfCommandOutput struct { + Start Start `json:"start"` + Intervals []Interval `json:"intervals"` + End End `json:"end"` + Error string `json:"error"` +} + +type End struct { + Streams []EndStream `json:"streams"` + SumSent Sum `json:"sum_sent"` + SumReceived Sum `json:"sum_received"` + CPUUtilizationPercent CPUUtilizationPercent `json:"cpu_utilization_percent"` + SenderTCPCongestion string `json:"sender_tcp_congestion"` + ReceiverTCPCongestion string `json:"receiver_tcp_congestion"` +} + +type CPUUtilizationPercent struct { + HostTotal float64 `json:"host_total"` + HostUser float64 `json:"host_user"` + HostSystem float64 `json:"host_system"` + RemoteTotal float64 `json:"remote_total"` + RemoteUser float64 `json:"remote_user"` + RemoteSystem float64 `json:"remote_system"` +} + +type EndStream struct { + Sender Sum `json:"sender"` + Receiver Sum `json:"receiver"` + UDP UDPSum `json:"udp"` +} + +type UDPSum struct { + Socket int64 `json:"socket"` + Start float64 `json:"start"` + End float64 `json:"end"` + Seconds float64 `json:"seconds"` + Bytes int64 `json:"bytes"` + BitsPerSecond float64 `json:"bits_per_second"` + JitterMS float64 `json:"jitter_ms"` + LostPackets int64 `json:"lost_packets"` + Packets int64 `json:"packets"` + LostPercent float64 `json:"lost_percent"` + OutOfOrder int64 `json:"out_of_order"` + Sender bool `json:"sender"` +} + +type Sum struct { + Socket int64 `json:"socket"` + Start float64 `json:"start"` + End float64 `json:"end"` + Seconds float64 `json:"seconds"` + Bytes int64 `json:"bytes"` + BitsPerSecond float64 `json:"bits_per_second"` + Retransmits int64 `json:"retransmits"` + MaxSndCwnd int64 `json:"max_snd_cwnd"` + MaxSndWnd int64 `json:"max_snd_wnd"` + MaxRtt int64 `json:"max_rtt"` + MinRtt int64 `json:"min_rtt"` + MeanRtt int64 `json:"mean_rtt"` + Sender bool `json:"sender"` +} + +type Interval struct { + Streams []IntervalStream `json:"streams"` + Sum Sum `json:"sum"` +} + +type IntervalStream struct { + Socket int64 `json:"socket"` + Start float64 `json:"start"` + End float64 `json:"end"` + Seconds float64 `json:"seconds"` + Bytes int64 `json:"bytes"` + BitsPerSecond float64 `json:"bits_per_second"` + Retransmits int64 `json:"retransmits"` + SndCwnd int64 `json:"snd_cwnd"` + SndWnd int64 `json:"snd_wnd"` + Rtt int64 `json:"rtt"` + Rttvar int64 `json:"rttvar"` + Pmtu int64 `json:"pmtu"` + Omitted bool `json:"omitted"` + Sender bool `json:"sender"` +} + +type Start struct { + Connected []Connected `json:"connected"` + Version string `json:"version"` + SystemInfo string `json:"system_info"` + Timestamp Timestamp `json:"timestamp"` + ConnectingTo ConnectingTo `json:"connecting_to"` + Cookie string `json:"cookie"` + TCPMssDefault int64 `json:"tcp_mss_default"` + TargetBitrate int64 `json:"target_bitrate"` + FqRate int64 `json:"fq_rate"` + SockBufsize int64 `json:"sock_bufsize"` + SndbufActual int64 `json:"sndbuf_actual"` + RcvbufActual int64 `json:"rcvbuf_actual"` + TestStart TestStart `json:"test_start"` +} + +type Connected struct { + Socket int64 `json:"socket"` + LocalHost string `json:"local_host"` + LocalPort int64 `json:"local_port"` + RemoteHost string `json:"remote_host"` + RemotePort int64 `json:"remote_port"` +} + +type ConnectingTo struct { + Host string `json:"host"` + Port int64 `json:"port"` +} + +type TestStart struct { + Protocol string `json:"protocol"` + NumStreams int64 `json:"num_streams"` + Blksize int64 `json:"blksize"` + Omit int64 `json:"omit"` + Duration int64 `json:"duration"` + Bytes int64 `json:"bytes"` + Blocks int64 `json:"blocks"` + Reverse int64 `json:"reverse"` + Tos int64 `json:"tos"` + TargetBitrate int64 `json:"target_bitrate"` + Bidir int64 `json:"bidir"` + Fqrate int64 `json:"fqrate"` +} + +type Timestamp struct { + Time string `json:"time"` + Timesecs int64 `json:"timesecs"` +}