Fix intermittant docker exec already running problem. Move to docker api 17.03.02

test/docker/Gopkg.lock

@@ -1,27 +1,71 @@
 # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
 
 
+[[projects]]
+  branch = "master"
+  name = "github.com/Azure/go-ansiterm"
+  packages = [
+    ".",
+    "winterm"
+  ]
+  revision = "d6e3b3328b783f23731bc4d058875b0371ff8109"
+
 [[projects]]
   name = "github.com/Microsoft/go-winio"
   packages = ["."]
-  revision = "78439966b38d69bf38227fbf57ac8a6fee70f69a"
+  revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f"
-  version = "v0.4.5"
+  version = "v0.4.7"
+
+[[projects]]
+  name = "github.com/Sirupsen/logrus"
+  packages = ["."]
+  revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
+  version = "v1.0.5"
 
 [[projects]]
   name = "github.com/docker/distribution"
-  packages = ["digest","reference"]
+  packages = [
+    "digest",
+    "reference"
+  ]
   revision = "48294d928ced5dd9b378f7fd7c6f5da3ff3f2c89"
   version = "v2.6.2"
 
 [[projects]]
   name = "github.com/docker/docker"
-  packages = ["api/types","api/types/blkiodev","api/types/container","api/types/events","api/types/filters","api/types/mount","api/types/network","api/types/reference","api/types/registry","api/types/strslice","api/types/swarm","api/types/time","api/types/versions","api/types/volume","client","pkg/tlsconfig"]
+  packages = [
-  revision = "092cba3727bb9b4a2f0e922cd6c0f93ea270e363"
+    "api/types",
-  version = "v1.13.1"
+    "api/types/blkiodev",
+    "api/types/container",
+    "api/types/events",
+    "api/types/filters",
+    "api/types/mount",
+    "api/types/network",
+    "api/types/reference",
+    "api/types/registry",
+    "api/types/strslice",
+    "api/types/swarm",
+    "api/types/time",
+    "api/types/versions",
+    "api/types/volume",
+    "client",
+    "pkg/jsonlog",
+    "pkg/jsonmessage",
+    "pkg/stdcopy",
+    "pkg/term",
+    "pkg/term/windows",
+    "pkg/tlsconfig"
+  ]
+  revision = "f5ec1e2936dcbe7b5001c2b817188b095c700c27"
+  version = "v17.03.2-ce"
 
 [[projects]]
   name = "github.com/docker/go-connections"
-  packages = ["nat","sockets","tlsconfig"]
+  packages = [
+    "nat",
+    "sockets",
+    "tlsconfig"
+  ]
   revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d"
   version = "v0.3.0"
 
@@ -37,21 +81,34 @@
   revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
   version = "v0.8.0"
 
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["ssh/terminal"]
+  revision = "88942b9c40a4c9d203b82b3731787b672d6e809b"
+
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp","proxy"]
+  packages = [
-  revision = "66aacef3dd8a676686c7ae3716979581e8b03c47"
+    "context",
+    "context/ctxhttp",
+    "proxy"
+  ]
+  revision = "6078986fec03a1dcc236c34816c71b0e05018fda"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
-  packages = ["windows"]
+  packages = [
-  revision = "9aade4d3a3b7e6d876cd3823ad20ec45fc035402"
+    "unix",
+    "windows"
+  ]
+  revision = "13d03a9a82fba647c21a0ef8fba44a795d0f0835"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "bf6ada0adb63f691f18ca1b3b95f55be8ec360be22928ca9e63ba47846c5687d"
+  inputs-digest = "c792836365447209421d5dc68a75fa77063408b8a6a2f9325b976581a0d60107"
   solver-name = "gps-cdcl"
   solver-version = 1

test/docker/Gopkg.toml

@@ -14,7 +14,7 @@
 
 [[constraint]]
   name = "github.com/docker/docker"
-  version = "^1.12"
+  version = "=v17.03.2-ce"
 
 [[constraint]]
   name = "github.com/docker/go-connections"

test/docker/devconfig_test.go

@@ -122,7 +122,7 @@ func TestDevWebDisabled(t *testing.T) {
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
 	t.Run("Web", func(t *testing.T) {
-		dspmqweb := execContainerWithOutput(t, cli, id, "mqm", []string{"dspmqweb"})
+		_, dspmqweb := execContainer(t, cli, id, "mqm", []string{"dspmqweb"})
 		if !strings.Contains(dspmqweb, "Server mqweb is not running.") {
 			t.Errorf("Expected dspmqweb to say server is not running; got \"%v\"", dspmqweb)
 		}
@@ -152,7 +152,7 @@ func TestDevConfigDisabled(t *testing.T) {
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
 	waitForWebReady(t, cli, id, insecureTLSConfig)
-	rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'display qlocal(DEV*)' | runmqsc"})
+	rc, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'display qlocal(DEV*)' | runmqsc"})
 	if rc == 0 {
 		t.Errorf("Expected DEV queues to be missing")
 	}

test/docker/docker_api_test.go

@@ -126,7 +126,7 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri
 	id := runContainer(t, cli, &containerConfig)
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
-	out := execContainerWithOutput(t, cli, id, "mqm", []string{"dspmq"})
+	_, out := execContainer(t, cli, id, "mqm", []string{"dspmq"})
 	if !strings.Contains(out, search) {
 		t.Errorf("Expected result of running dspmq to contain name=%v, got name=%v", search, out)
 	}
@@ -285,16 +285,18 @@ func TestVolumeUnmount(t *testing.T) {
 	defer cleanContainer(t, cli, ctr.ID)
 	waitForReady(t, cli, ctr.ID)
 	// Unmount the volume as root
-	rc := execContainerWithExitCode(t, cli, ctr.ID, "root", []string{"umount", "-l", "-f", "/mnt/mqm"})
+	rc, _ := execContainer(t, cli, ctr.ID, "root", []string{"umount", "-l", "-f", "/mnt/mqm"})
 	if rc != 0 {
 		t.Fatalf("Expected umount to work with rc=0, got %v", rc)
 	}
 	time.Sleep(3 * time.Second)
-	rc = execContainerWithExitCode(t, cli, ctr.ID, "mqm", []string{"chkmqhealthy"})
+	rc, _ = execContainer(t, cli, ctr.ID, "mqm", []string{"chkmqhealthy"})
 	if rc == 0 {
 		t.Errorf("Expected chkmqhealthy to fail")
-		t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"df"}))
+		_, df := execContainer(t, cli, ctr.ID, "mqm", []string{"df"})
-		t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"}))
+		t.Logf(df)
+		_, ps :=execContainer(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"})
+		t.Logf(ps)
 	}
 }
 
@@ -318,12 +320,12 @@ func TestZombies(t *testing.T) {
 	waitForReady(t, cli, id)
 	// Kill an MQ process with children.  After it is killed, its children
 	// will be adopted by PID 1, and should then be reaped when they die.
-	out := execContainerWithOutput(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"})
+	_, out := execContainer(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"})
 	if out == "0" {
 		t.Fatalf("Expected pkill to kill a process, got %v", out)
 	}
 	time.Sleep(3 * time.Second)
-	out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z'"})
+	_, out = execContainer(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z'"})
 	if out != "" {
 		count := strings.Count(out, "\n") + 1
 		t.Errorf("Expected zombies=0, got %v", count)
@@ -356,7 +358,7 @@ func TestMQSC(t *testing.T) {
 	id := runContainer(t, cli, &containerConfig)
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
-	rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"})
+	rc, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"})
 	if rc != 0 {
 		t.Fatalf("Expected runmqsc to exit with rc=0, got %v", rc)
 	}
@@ -392,17 +394,19 @@ func TestReadiness(t *testing.T) {
 	id := runContainer(t, cli, &containerConfig)
 	defer cleanContainer(t, cli, id)
 	queueCheckCommand := fmt.Sprintf("echo 'DISPLAY QLOCAL(test%v)' | runmqsc", numQueues)
-	t.Log(execContainerWithOutput(t, cli, id, "root", []string{"cat", "/etc/mqm/test.mqsc"}))
+	_, mqsc := execContainer(t, cli, id, "root", []string{"cat", "/etc/mqm/test.mqsc"})
+	t.Log(mqsc)
 	for {
-		readyRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"chkmqready"})
+		readyRC, _ := execContainer(t, cli, id, "mqm", []string{"chkmqready"})
-		queueCheckRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand})
+		queueCheckRC, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand})
 		t.Logf("readyRC=%v,queueCheckRC=%v\n", readyRC, queueCheckRC)
 		if readyRC == 0 {
 			if queueCheckRC != 0 {
 				t.Fatalf("chkmqready returned %v when MQSC had not finished", readyRC)
 			} else {
 				// chkmqready says OK, and the last queue exists, so return
-				t.Log(execContainerWithOutput(t, cli, id, "root", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test1)' | runmqsc"}))
+				_, runmqsc := execContainer(t, cli, id, "root", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test1)' | runmqsc"})
+				t.Log(runmqsc)
 				return
 			}
 		}
@@ -463,20 +467,20 @@ func TestErrorLogRotation(t *testing.T) {
 	waitForReady(t, cli, id)
 	dir := "/var/mqm/qmgrs/" + qmName + "/errors"
 	// Generate some content for the error logs, by trying to put messages under an unauthorized user
-	// execContainerWithOutput(t, cli, id, "fred", []string{"bash", "-c", "for i in {1..30} ; do /opt/mqm/samp/bin/amqsput FAKE; done"})
+	// execContainer(t, cli, id, "fred", []string{"bash", "-c", "for i in {1..30} ; do /opt/mqm/samp/bin/amqsput FAKE; done"})
-	execContainerWithOutput(t, cli, id, "root", []string{"useradd", "fred"})
+	execContainer(t, cli, id, "root", []string{"useradd", "fred"})
 	for {
-		execContainerWithOutput(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"})
+		execContainer(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"})
-		amqerr02size, err := strconv.Atoi(execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")}))
+
-		if err != nil {
+		_, derpaderp := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")})
-			t.Fatal(err)
+		amqerr02size, _ := strconv.Atoi(derpaderp)
-		}
+
 		if amqerr02size > 0 {
 			// We've done enough to cause log rotation
 			break
 		}
 	}
-	out := execContainerWithOutput(t, cli, id, "root", []string{"ls", "-l", dir})
+	_, out := execContainer(t, cli, id, "root", []string{"ls", "-l", dir})
 	t.Log(out)
 	stopContainer(t, cli, id)
 	b := copyFromContainer(t, cli, id, filepath.Join(dir, "AMQERR01.json"))

test/docker/docker_api_test_util.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"regexp"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -261,10 +262,9 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64
 	return rc
 }
 
-// execContainerWithExitCode runs a command in a running container, and returns the exit code
+// execContainer runs a command in a running container, and returns the exit code and output
-// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the
+func execContainer(t *testing.T, cli *client.Client, ID string, user string, cmd []string) (int, string) {
-// container to get output.  This is why these are two separate commands.
+	rerun:
-func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user string, cmd []string) int {
 	config := types.ExecConfig{
 		User:        user,
 		Privileged:  false,
@@ -280,6 +280,10 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user
 	if err != nil {
 		t.Fatal(err)
 	}
+	hijack, err := cli.ContainerExecAttach(context.Background(), resp.ID, config)
+	if err != nil {
+		t.Fatal(err)
+	}
 	cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{
 		Detach: false,
 		Tty:    false,
@@ -287,49 +291,15 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user
 	if err != nil {
 		t.Fatal(err)
 	}
-	inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return inspect.ExitCode
-}
-
-// execContainerWithOutput runs a command in a running container, and returns the output from stdout/stderr
-// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the
-// container to get output.  This is why these are two separate commands.
-func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user string, cmd []string) string {
-	config := types.ExecConfig{
-		User:         user,
-		Privileged:   false,
-		Tty:          false,
-		AttachStdin:  false,
-		AttachStdout: true,
-		AttachStderr: true,
-		Detach:       false,
-		Cmd:          cmd,
-	}
-	resp, err := cli.ContainerExecCreate(context.Background(), ID, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	hijack, err := cli.ContainerExecAttach(context.Background(), resp.ID, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{
-		Detach: false,
-		Tty:    false,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Wait for the command to finish
+	var exitcode int
 	for {
 		inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !inspect.Running {
+			exitcode = inspect.ExitCode
 			break
 		}
 	}
@@ -339,12 +309,22 @@ func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user s
 	if err != nil {
 		t.Fatal(err)
 	}
-	return strings.TrimSpace(buf.String())
+
+	outputStr := strings.TrimSpace(buf.String())
+
+	// Before we go let's just double check it did actually run because sometimes we get a "Exec command already running error"
+	alreadyRunningErr := regexp.MustCompile("Error: Exec command .* is already running")
+	if alreadyRunningErr.MatchString(outputStr) {
+		time.Sleep(1 * time.Second)
+		goto rerun
+	}
+
+	return exitcode, outputStr
 }
 
 func waitForReady(t *testing.T, cli *client.Client, ID string) {
 	for {
-		rc := execContainerWithExitCode(t, cli, ID, "mqm", []string{"chkmqready"})
+		rc, _ := execContainer(t, cli, ID, "mqm", []string{"chkmqready"})
 		if rc == 0 {
 			t.Log("MQ is ready")
 			return