From b02d8d27b2e7f2261ae32a7f813537d595f6806b Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Wed, 28 Mar 2018 15:57:07 +0100 Subject: [PATCH] Fix intermittant docker exec already running problem. Move to docker api 17.03.02 --- test/docker/Gopkg.lock | 81 ++++++++++++++++++++++++----- test/docker/Gopkg.toml | 2 +- test/docker/devconfig_test.go | 4 +- test/docker/docker_api_test.go | 44 +++++++++------- test/docker/docker_api_test_util.go | 64 ++++++++--------------- 5 files changed, 118 insertions(+), 77 deletions(-) diff --git a/test/docker/Gopkg.lock b/test/docker/Gopkg.lock index 6b9fb90..64f47d3 100644 --- a/test/docker/Gopkg.lock +++ b/test/docker/Gopkg.lock @@ -1,27 +1,71 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + branch = "master" + name = "github.com/Azure/go-ansiterm" + packages = [ + ".", + "winterm" + ] + revision = "d6e3b3328b783f23731bc4d058875b0371ff8109" + [[projects]] name = "github.com/Microsoft/go-winio" packages = ["."] - revision = "78439966b38d69bf38227fbf57ac8a6fee70f69a" - version = "v0.4.5" + revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f" + version = "v0.4.7" + +[[projects]] + name = "github.com/Sirupsen/logrus" + packages = ["."] + revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" + version = "v1.0.5" [[projects]] name = "github.com/docker/distribution" - packages = ["digest","reference"] + packages = [ + "digest", + "reference" + ] revision = "48294d928ced5dd9b378f7fd7c6f5da3ff3f2c89" version = "v2.6.2" [[projects]] name = "github.com/docker/docker" - packages = ["api/types","api/types/blkiodev","api/types/container","api/types/events","api/types/filters","api/types/mount","api/types/network","api/types/reference","api/types/registry","api/types/strslice","api/types/swarm","api/types/time","api/types/versions","api/types/volume","client","pkg/tlsconfig"] - revision = "092cba3727bb9b4a2f0e922cd6c0f93ea270e363" - version = "v1.13.1" + packages = [ + "api/types", + "api/types/blkiodev", + "api/types/container", + "api/types/events", + "api/types/filters", + "api/types/mount", + "api/types/network", + "api/types/reference", + "api/types/registry", + "api/types/strslice", + "api/types/swarm", + "api/types/time", + "api/types/versions", + "api/types/volume", + "client", + "pkg/jsonlog", + "pkg/jsonmessage", + "pkg/stdcopy", + "pkg/term", + "pkg/term/windows", + "pkg/tlsconfig" + ] + revision = "f5ec1e2936dcbe7b5001c2b817188b095c700c27" + version = "v17.03.2-ce" [[projects]] name = "github.com/docker/go-connections" - packages = ["nat","sockets","tlsconfig"] + packages = [ + "nat", + "sockets", + "tlsconfig" + ] revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" version = "v0.3.0" @@ -37,21 +81,34 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" + [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context","context/ctxhttp","proxy"] - revision = "66aacef3dd8a676686c7ae3716979581e8b03c47" + packages = [ + "context", + "context/ctxhttp", + "proxy" + ] + revision = "6078986fec03a1dcc236c34816c71b0e05018fda" [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["windows"] - revision = "9aade4d3a3b7e6d876cd3823ad20ec45fc035402" + packages = [ + "unix", + "windows" + ] + revision = "13d03a9a82fba647c21a0ef8fba44a795d0f0835" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "bf6ada0adb63f691f18ca1b3b95f55be8ec360be22928ca9e63ba47846c5687d" + inputs-digest = "c792836365447209421d5dc68a75fa77063408b8a6a2f9325b976581a0d60107" solver-name = "gps-cdcl" solver-version = 1 diff --git a/test/docker/Gopkg.toml b/test/docker/Gopkg.toml index d13cde4..c29f50e 100644 --- a/test/docker/Gopkg.toml +++ b/test/docker/Gopkg.toml @@ -14,7 +14,7 @@ [[constraint]] name = "github.com/docker/docker" - version = "^1.12" + version = "=v17.03.2-ce" [[constraint]] name = "github.com/docker/go-connections" diff --git a/test/docker/devconfig_test.go b/test/docker/devconfig_test.go index 7d013ab..1501dbe 100644 --- a/test/docker/devconfig_test.go +++ b/test/docker/devconfig_test.go @@ -122,7 +122,7 @@ func TestDevWebDisabled(t *testing.T) { defer cleanContainer(t, cli, id) waitForReady(t, cli, id) t.Run("Web", func(t *testing.T) { - dspmqweb := execContainerWithOutput(t, cli, id, "mqm", []string{"dspmqweb"}) + _, dspmqweb := execContainer(t, cli, id, "mqm", []string{"dspmqweb"}) if !strings.Contains(dspmqweb, "Server mqweb is not running.") { t.Errorf("Expected dspmqweb to say server is not running; got \"%v\"", dspmqweb) } @@ -152,7 +152,7 @@ func TestDevConfigDisabled(t *testing.T) { defer cleanContainer(t, cli, id) waitForReady(t, cli, id) waitForWebReady(t, cli, id, insecureTLSConfig) - rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'display qlocal(DEV*)' | runmqsc"}) + rc, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'display qlocal(DEV*)' | runmqsc"}) if rc == 0 { t.Errorf("Expected DEV queues to be missing") } diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 08be306..d93edbb 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -126,7 +126,7 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) waitForReady(t, cli, id) - out := execContainerWithOutput(t, cli, id, "mqm", []string{"dspmq"}) + _, out := execContainer(t, cli, id, "mqm", []string{"dspmq"}) if !strings.Contains(out, search) { t.Errorf("Expected result of running dspmq to contain name=%v, got name=%v", search, out) } @@ -285,16 +285,18 @@ func TestVolumeUnmount(t *testing.T) { defer cleanContainer(t, cli, ctr.ID) waitForReady(t, cli, ctr.ID) // Unmount the volume as root - rc := execContainerWithExitCode(t, cli, ctr.ID, "root", []string{"umount", "-l", "-f", "/mnt/mqm"}) + rc, _ := execContainer(t, cli, ctr.ID, "root", []string{"umount", "-l", "-f", "/mnt/mqm"}) if rc != 0 { t.Fatalf("Expected umount to work with rc=0, got %v", rc) } time.Sleep(3 * time.Second) - rc = execContainerWithExitCode(t, cli, ctr.ID, "mqm", []string{"chkmqhealthy"}) + rc, _ = execContainer(t, cli, ctr.ID, "mqm", []string{"chkmqhealthy"}) if rc == 0 { t.Errorf("Expected chkmqhealthy to fail") - t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"df"})) - t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"})) + _, df := execContainer(t, cli, ctr.ID, "mqm", []string{"df"}) + t.Logf(df) + _, ps :=execContainer(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"}) + t.Logf(ps) } } @@ -318,12 +320,12 @@ func TestZombies(t *testing.T) { waitForReady(t, cli, id) // Kill an MQ process with children. After it is killed, its children // will be adopted by PID 1, and should then be reaped when they die. - out := execContainerWithOutput(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"}) + _, out := execContainer(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"}) if out == "0" { t.Fatalf("Expected pkill to kill a process, got %v", out) } time.Sleep(3 * time.Second) - out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z'"}) + _, out = execContainer(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z'"}) if out != "" { count := strings.Count(out, "\n") + 1 t.Errorf("Expected zombies=0, got %v", count) @@ -356,7 +358,7 @@ func TestMQSC(t *testing.T) { id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) waitForReady(t, cli, id) - rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"}) + rc, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"}) if rc != 0 { t.Fatalf("Expected runmqsc to exit with rc=0, got %v", rc) } @@ -392,17 +394,19 @@ func TestReadiness(t *testing.T) { id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) queueCheckCommand := fmt.Sprintf("echo 'DISPLAY QLOCAL(test%v)' | runmqsc", numQueues) - t.Log(execContainerWithOutput(t, cli, id, "root", []string{"cat", "/etc/mqm/test.mqsc"})) + _, mqsc := execContainer(t, cli, id, "root", []string{"cat", "/etc/mqm/test.mqsc"}) + t.Log(mqsc) for { - readyRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"chkmqready"}) - queueCheckRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand}) + readyRC, _ := execContainer(t, cli, id, "mqm", []string{"chkmqready"}) + queueCheckRC, _ := execContainer(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand}) t.Logf("readyRC=%v,queueCheckRC=%v\n", readyRC, queueCheckRC) if readyRC == 0 { if queueCheckRC != 0 { t.Fatalf("chkmqready returned %v when MQSC had not finished", readyRC) } else { // chkmqready says OK, and the last queue exists, so return - t.Log(execContainerWithOutput(t, cli, id, "root", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test1)' | runmqsc"})) + _, runmqsc := execContainer(t, cli, id, "root", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test1)' | runmqsc"}) + t.Log(runmqsc) return } } @@ -463,20 +467,20 @@ func TestErrorLogRotation(t *testing.T) { waitForReady(t, cli, id) dir := "/var/mqm/qmgrs/" + qmName + "/errors" // Generate some content for the error logs, by trying to put messages under an unauthorized user - // execContainerWithOutput(t, cli, id, "fred", []string{"bash", "-c", "for i in {1..30} ; do /opt/mqm/samp/bin/amqsput FAKE; done"}) - execContainerWithOutput(t, cli, id, "root", []string{"useradd", "fred"}) + // execContainer(t, cli, id, "fred", []string{"bash", "-c", "for i in {1..30} ; do /opt/mqm/samp/bin/amqsput FAKE; done"}) + execContainer(t, cli, id, "root", []string{"useradd", "fred"}) for { - execContainerWithOutput(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"}) - amqerr02size, err := strconv.Atoi(execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")})) - if err != nil { - t.Fatal(err) - } + execContainer(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"}) + + _, derpaderp := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")}) + amqerr02size, _ := strconv.Atoi(derpaderp) + if amqerr02size > 0 { // We've done enough to cause log rotation break } } - out := execContainerWithOutput(t, cli, id, "root", []string{"ls", "-l", dir}) + _, out := execContainer(t, cli, id, "root", []string{"ls", "-l", dir}) t.Log(out) stopContainer(t, cli, id) b := copyFromContainer(t, cli, id, filepath.Join(dir, "AMQERR01.json")) diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index f0d5c5f..9889d0a 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -30,6 +30,7 @@ import ( "strings" "testing" "time" + "regexp" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -261,10 +262,9 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64 return rc } -// execContainerWithExitCode runs a command in a running container, and returns the exit code -// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the -// container to get output. This is why these are two separate commands. -func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user string, cmd []string) int { +// execContainer runs a command in a running container, and returns the exit code and output +func execContainer(t *testing.T, cli *client.Client, ID string, user string, cmd []string) (int, string) { + rerun: config := types.ExecConfig{ User: user, Privileged: false, @@ -280,6 +280,10 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user if err != nil { t.Fatal(err) } + hijack, err := cli.ContainerExecAttach(context.Background(), resp.ID, config) + if err != nil { + t.Fatal(err) + } cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ Detach: false, Tty: false, @@ -287,49 +291,15 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user if err != nil { t.Fatal(err) } - inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) - if err != nil { - t.Fatal(err) - } - return inspect.ExitCode -} - -// execContainerWithOutput runs a command in a running container, and returns the output from stdout/stderr -// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the -// container to get output. This is why these are two separate commands. -func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user string, cmd []string) string { - config := types.ExecConfig{ - User: user, - Privileged: false, - Tty: false, - AttachStdin: false, - AttachStdout: true, - AttachStderr: true, - Detach: false, - Cmd: cmd, - } - resp, err := cli.ContainerExecCreate(context.Background(), ID, config) - if err != nil { - t.Fatal(err) - } - hijack, err := cli.ContainerExecAttach(context.Background(), resp.ID, config) - if err != nil { - t.Fatal(err) - } - err = cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ - Detach: false, - Tty: false, - }) - if err != nil { - t.Fatal(err) - } // Wait for the command to finish + var exitcode int for { inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) if err != nil { t.Fatal(err) } if !inspect.Running { + exitcode = inspect.ExitCode break } } @@ -339,12 +309,22 @@ func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user s if err != nil { t.Fatal(err) } - return strings.TrimSpace(buf.String()) + + outputStr := strings.TrimSpace(buf.String()) + + // Before we go let's just double check it did actually run because sometimes we get a "Exec command already running error" + alreadyRunningErr := regexp.MustCompile("Error: Exec command .* is already running") + if alreadyRunningErr.MatchString(outputStr) { + time.Sleep(1 * time.Second) + goto rerun + } + + return exitcode, outputStr } func waitForReady(t *testing.T, cli *client.Client, ID string) { for { - rc := execContainerWithExitCode(t, cli, ID, "mqm", []string{"chkmqready"}) + rc, _ := execContainer(t, cli, ID, "mqm", []string{"chkmqready"}) if rc == 0 { t.Log("MQ is ready") return