fixing #52; netem qdisc should work now when IP filter is applied

alexei-led · Nov 14, 2017 · 3e33d99 · 3e33d99
1 parent 105bcb2
commit 3e33d99
Show file tree

Hide file tree

Showing 4 changed files with 131 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,115 +1,173 @@
 # Change Log
+
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.4.7] - 2017-11-14
+
+### Fixed
+
+- issue [52](https://github.com/gaia-adm/pumba/issues/52) fixed `netem` qdisc, when using with IP filter
+- documentation fix
+
+### Changed
+
+- multi-platform build is not using `gox` tool anymore
+
 ## [v0.4.5] - 2017-09-06
+
 ### Fixed
+
 - issue [39](https://github.com/gaia-adm/pumba/issues/39) fixed, when all traffic blocked after applying `netem` `qdisc` with IP `filter` 
 
 ## [v0.4.3] - 2017-07-06
+
 ### Changed
+
 - multistage Docker build 
 - migrate to Codefresh.io CI/CD from CircleCI
 - new: homebrew formula for pumba
 
 ## [v0.4.2] - 2017-03-16
+
 ### Changed
+
 - minor bug fixes
 
 ## [v0.4.1] - 2017-02-01
+
 ### Changed
+
 - adopt [dobi](https://dnephin.github.io/dobi/) for local and CircleCI builds
 
 ## [v0.4.0] - 2017-01-29
+
 ### Updated
+
 - Replace Docker samalba client with default Docker SDK for Go. Kudos to [Sławomir Nowak](https://github.com/slnowak) for contribution.
 - update glide dependency and versions
 
 ## [v0.3.1] - 2016-12-13
+
 ### Added
+
 - `netem rate` command, contributed by Ricardo Martins @meqif
 
 ## [v0.3.0] - 2016-11-6
+
 ### Added
+
 - build label is calculated based on major build (VERSION file), git branch and commit
 - `rocker` build support - Rockerfile added
 - speedup build time - improve cache management
 
 ## [v0.2.4] - 2016-09-07
+
 ### Added
+
 - `netem --tc-image` flag to specify external Docker image with `tc` tool (try `gaiadocker/iproute2` image)
 - integration tests, run with `bats ./tests/`
 
 ## [v0.2.4] - 2016-08-10
+
 ### Added
+
 - `netem loss` network emulation packer loss, based on independent (Bernoulli) probability model
 - `netem loss-state` network emulation packer loss, based on 4-Markov state probability model
 - `netem loss-gemodel` network emulation packer loss, according to the Gilbert-Elliot loss model
 
 ## [v0.2.3] - 2016-08-07
+
 ### Fixed
+
 - `pause` command now can be interrupted with `Ctrl-C`; all paused processes will be unpaused
 - BUG: when using single container name, pumba disturbs all containers
 
 ## [v0.2.2] - 2016-08-04
+
 ### Changed
+
 - `--interval` flag is optional now; if missing Pumba will do single chaos action and exit
 - `netem delay --distribution` new option to define optional delay distribution, can be: {uniform | normal | pareto |  paretonormal}
 - check added to verify `tc` tool existence in container (`tc` is required for network emulation; part of `iproute2` package)
 - Use `Ctrl-C` to abort Pumba execution at any time
 
 ## [v0.2.0] - 2016-07-20
+
 ### Added
+
 - Network emulation for egress container traffic, powered by [netem](http://www.linuxfoundation.org/collaborate/workgroups/networking/netem)
+
 ### Updated
+
 - **Breaking Change** command line simplification ...
 - `chaos` command had been replaced by multiple standalone commands: `kill`, `netem`, `pause`, `rm`, `stop`
 - now it's possible to run multiple Pumba Docker containers (do not prevent)
 - **Only ONE** command per single Pumba run is supported, but it's possible to run multiple Pumba processes and containers
 
 ## [v0.1.11] - 2016-06-27
+
 ### Added
+
 - pause container processes for specified interval
 
 ## [v0.1.10] - 2016-06-05
+
 ### Fixed
+
 - set proper release tag in GitHub
 
 ## [0.1.9] - 2016-05-22
+
 ### Fixed
+
 - speed up build
 
 ## [0.1.8] - 2016-05-22
+
 ### Fixed
+
 - Added CA ca-certificates to Docker image: required for HTTPS
 
 ## [0.1.7] - 2016-05-21
-### Added
+
 ### Fixed
+
 - Report Pumba events to Slack
 
 ## [0.1.6] - 2016-04-25
+
 ### Added
+
 - added `gosu` to Pumba Docker image
 - Use Docker Label `com.gaiaadm.pumba.skip` to make Pumba ignoring the container. Avoid abusing it though.
+
 ### Fixed
+
 - Pumba runs as `pumba:pumba` user, instead of `root`
 
 ## [0.1.5] - 2016-04-13
+
 ### Added
+
 - File: `pumba_kube.yml` Kubernetes (1.1.x) deployment manifest
 - File: `pumba_coreos.service` CoreOS `fleet` service file
 - Flag: `--json` flag. When specified log will be generated in JSON format (Logstash and Splunk friendly)
 - Flag: `--slackhook` Slack web hook URL. Now Pumba can report log events to specified Slack channel.
 - Flag: `--slackchannel` Slack channel to report Pumba events in.
 - Flag: `--dry` enable 'dry run' mode: do not 'kill' containers, just log intention
 ### Changed
+
 - by default produce colarful log to TTY
+
 ### Fixed
+
 - fix failure when container name is empty (all containers)
 
 ## [0.1.4] - 2016-04-08
+
 This is initial release of Pumba Docker Chaos Testing
 ### Added
+
 - `run` command
 - `--random` option: randomly select matching image to "kill"
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.4.6
+0.4.7
diff --git a/container/client.go b/container/client.go
@@ -244,24 +244,40 @@ func (client dockerClient) stopNetemContainer(ctx context.Context, c Container,
 	log.Infof("%sStop netem for container %s on '%s'", prefix, c.ID(), netInterface)
 	if !dryrun {
 		if ip != nil {
-			// delete qdisc 'parent 1:3 netem'
+			// delete qdisc 'parent 1:1 handle 10:'
 			// http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
-			netemCommand := []string{"qdisc", "del", "dev", netInterface, "parent", "1:3", "netem"}
-			log.Debugf("netem command '%s'", strings.Join(netemCommand, " "))
+			netemCommand := []string{"qdisc", "del", "dev", netInterface, "parent", "1:1", "handle", "10:"}
+			log.Debugf("qdisc delete '%s'", strings.Join(netemCommand, " "))
+			err = client.tcCommand(ctx, c, netemCommand, tcimage)
+			if err != nil {
+				return err
+			}
+			// delete qdisc 'parent 1:2 handle 20:'
+			// http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
+			netemCommand = []string{"qdisc", "del", "dev", netInterface, "parent", "1:2", "handle", "20:"}
+			log.Debugf("qdisc delete '%s'", strings.Join(netemCommand, " "))
+			err = client.tcCommand(ctx, c, netemCommand, tcimage)
+			if err != nil {
+				return err
+			}
+			// delete qdisc 'parent 1:3 handle 30:'
+			// http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
+			netemCommand = []string{"qdisc", "del", "dev", netInterface, "parent", "1:3", "handle", "30:"}
+			log.Debugf("qdisc delete '%s'", strings.Join(netemCommand, " "))
 			err = client.tcCommand(ctx, c, netemCommand, tcimage)
 			if err != nil {
 				return err
 			}
 			// delete qdisc 'root handle 1: prio'
 			// http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
 			netemCommand = []string{"qdisc", "del", "dev", netInterface, "root", "handle", "1:", "prio"}
-			log.Debugf("netem command '%s'", strings.Join(netemCommand, " "))
+			log.Debugf("qdisc delete '%s'", strings.Join(netemCommand, " "))
 			err = client.tcCommand(ctx, c, netemCommand, tcimage)
 		} else {
 			// stop netem command
 			// http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
 			netemCommand := []string{"qdisc", "del", "dev", netInterface, "root", "netem"}
-			log.Debugf("netem command '%s'", strings.Join(netemCommand, " "))
+			log.Debugf("qdisc delete '%s'", strings.Join(netemCommand, " "))
 			err = client.tcCommand(ctx, c, netemCommand, tcimage)
 		}
 	}
@@ -282,7 +298,17 @@ func (client dockerClient) startNetemContainerIPFilter(ctx context.Context, c Co
 		// queue, apply netem command on that queue only, then route IP traffic to the low priority queue
 		// See more: http://www.linuxfoundation.org/collaborate/workgroups/networking/netem
 
-		//  Create a priority-based queue.
+		//            1:   root qdisc
+		//           / | \
+		//          /  |  \
+		//         /   |   \
+		//       1:1  1:2  1:3    classes
+		//        |    |    |
+		//       10:  20:  30:    qdiscs    qdiscs
+		//      sfq  sfq  netem
+		// band  0    1     2
+
+		// Create a priority-based queue. This *instantly* creates classes 1:1, 1:2, 1:3
 		// 'tc qdisc add dev <netInterface> root handle 1: prio'
 		// See more: http://man7.org/linux/man-pages/man8/tc-netem.8.html
 		handleCommand := []string{"qdisc", "add", "dev", netInterface, "root", "handle", "1:", "prio"}
@@ -292,20 +318,40 @@ func (client dockerClient) startNetemContainerIPFilter(ctx context.Context, c Co
 			return err
 		}
 
-		//  Delay everything in band 3
-		// 'tc qdisc add dev <netInterface> parent 1:3 netem <netemCmd>'
+		// Create Stochastic Fairness Queueing (sfq) queueing discipline for 1:1 class.
+		// 'tc qdisc add dev <netInterface> parent 1:1 handle 10: sfq'
+		// See more: https://linux.die.net/man/8/tc-sfq
+		netemCommand := append([]string{"qdisc", "add", "dev", netInterface, "parent", "1:1", "handle", "10:", "sfq"})
+		log.Debugf("netemCommand %s", netemCommand)
+		err = client.tcCommand(ctx, c, netemCommand, tcimage)
+		if err != nil {
+			return err
+		}
+
+		// Create Stochastic Fairness Queueing (sfq) queueing discipline for 1:2 class
+		// 'tc qdisc add dev <netInterface> parent 1:2 handle 20: sfq'
+		// See more: https://linux.die.net/man/8/tc-sfq
+		netemCommand = append([]string{"qdisc", "add", "dev", netInterface, "parent", "1:2", "handle", "20:", "sfq"})
+		log.Debugf("netemCommand %s", netemCommand)
+		err = client.tcCommand(ctx, c, netemCommand, tcimage)
+		if err != nil {
+			return err
+		}
+
+		// Add queueing discipline for 1:3 class. No traffic is going through 1:3 yet
+		// 'tc qdisc add dev <netInterface> parent 1:3 handle 30: netem <netemCmd>'
 		// See more: http://man7.org/linux/man-pages/man8/tc-netem.8.html
-		netemCommand := append([]string{"qdisc", "add", "dev", netInterface, "parent", "1:3", "netem"}, netemCmd...)
+		netemCommand = append([]string{"qdisc", "add", "dev", netInterface, "parent", "1:3", "handle", "30:", "netem"}, netemCmd...)
 		log.Debugf("netemCommand %s", netemCommand)
 		err = client.tcCommand(ctx, c, netemCommand, tcimage)
 		if err != nil {
 			return err
 		}
 
-		// # say traffic to $PORT is band 3
-		// 'tc filter add dev <netInterface> protocol ip parent 1:0 prio 3 u32 match ip dst <targetIP> flowid 1:3'
+		// # redirect traffic to specific IP through band 3
+		// 'tc filter add dev <netInterface> protocol ip parent 1:0 prio 1 u32 match ip dst <targetIP> flowid 1:3'
 		// See more: http://man7.org/linux/man-pages/man8/tc-netem.8.html
-		filterCommand := []string{"filter", "add", "dev", netInterface, "protocol", "ip", "parent", "1:0", "prio", "3",
+		filterCommand := []string{"filter", "add", "dev", netInterface, "protocol", "ip", "parent", "1:0", "prio", "1",
 			"u32", "match", "ip", "dst", strings.ToLower(targetIP), "flowid", "1:3"}
 		log.Debugf("filterCommand %s", filterCommand)
 		return client.tcCommand(ctx, c, filterCommand, tcimage)

diff --git a/container/client_test.go b/container/client_test.go
@@ -446,17 +446,27 @@ func TestNetemContainerIPFilter_Success(t *testing.T) {
 	engineClient.On("ContainerExecStart", ctx, "cmd1", types.ExecStartCheck{}).Return(nil)
 	engineClient.On("ContainerExecInspect", ctx, "cmd1").Return(types.ContainerExecInspect{}, nil)
 
-	config2 := types.ExecConfig{Cmd: []string{"tc", "qdisc", "add", "dev", "eth0", "parent", "1:3", "netem", "delay", "500ms"}, Privileged: true}
+	config2 := types.ExecConfig{Cmd: []string{"tc", "qdisc", "add", "dev", "eth0", "parent", "1:1", "handle", "10:", "sfq"}, Privileged: true}
 	engineClient.On("ContainerExecCreate", ctx, "abc123", config2).Return(types.IDResponse{ID: "cmd2"}, nil)
 	engineClient.On("ContainerExecStart", ctx, "cmd2", types.ExecStartCheck{}).Return(nil)
 	engineClient.On("ContainerExecInspect", ctx, "cmd2").Return(types.ContainerExecInspect{}, nil)
 
-	config3 := types.ExecConfig{Cmd: []string{"tc", "filter", "add", "dev", "eth0", "protocol", "ip",
-		"parent", "1:0", "prio", "3", "u32", "match", "ip", "dst", "10.10.0.1", "flowid", "1:3"}, Privileged: true}
+	config3 := types.ExecConfig{Cmd: []string{"tc", "qdisc", "add", "dev", "eth0", "parent", "1:2", "handle", "20:", "sfq"}, Privileged: true}
 	engineClient.On("ContainerExecCreate", ctx, "abc123", config3).Return(types.IDResponse{ID: "cmd3"}, nil)
 	engineClient.On("ContainerExecStart", ctx, "cmd3", types.ExecStartCheck{}).Return(nil)
 	engineClient.On("ContainerExecInspect", ctx, "cmd3").Return(types.ContainerExecInspect{}, nil)
 
+	config4 := types.ExecConfig{Cmd: []string{"tc", "qdisc", "add", "dev", "eth0", "parent", "1:3", "handle", "30:", "netem", "delay", "500ms"}, Privileged: true}
+	engineClient.On("ContainerExecCreate", ctx, "abc123", config4).Return(types.IDResponse{ID: "cmd4"}, nil)
+	engineClient.On("ContainerExecStart", ctx, "cmd4", types.ExecStartCheck{}).Return(nil)
+	engineClient.On("ContainerExecInspect", ctx, "cmd4").Return(types.ContainerExecInspect{}, nil)
+
+	config5 := types.ExecConfig{Cmd: []string{"tc", "filter", "add", "dev", "eth0", "protocol", "ip",
+		"parent", "1:0", "prio", "1", "u32", "match", "ip", "dst", "10.10.0.1", "flowid", "1:3"}, Privileged: true}
+	engineClient.On("ContainerExecCreate", ctx, "abc123", config5).Return(types.IDResponse{ID: "cmd5"}, nil)
+	engineClient.On("ContainerExecStart", ctx, "cmd5", types.ExecStartCheck{}).Return(nil)
+	engineClient.On("ContainerExecInspect", ctx, "cmd5").Return(types.ContainerExecInspect{}, nil)
+
 	client := dockerClient{containerAPI: engineClient}
 	err := client.NetemContainer(context.TODO(), c, "eth0", []string{"delay", "500ms"}, net.ParseIP("10.10.0.1"), 1*time.Millisecond, "", false)