Add TestZombies and improve exec output handling

cmd/runmqserver/main.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -30,6 +31,20 @@ import (
 	"github.com/ibm-messaging/mq-container/internal/name"
 )
 
+var debug = false
+
+func logDebug(msg string) {
+	if debug {
+		log.Printf("DEBUG: %v", msg)
+	}
+}
+
+func logDebugf(format string, args ...interface{}) {
+	if debug {
+		log.Printf("DEBUG: %v", fmt.Sprintf(format, args...))
+	}
+}
+
 // createDirStructure creates the default MQ directory structure under /var/mqm
 func createDirStructure() error {
 	out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s")
@@ -127,6 +142,10 @@ func stopQueueManager(name string) error {
 }
 
 func doMain() error {
+	debugEnv, ok := os.LookupEnv("DEBUG")
+	if ok && (debugEnv == "true" || debugEnv == "1") {
+		debug = true
+	}
 	accepted, err := checkLicense()
 	if err != nil {
 		return err

cmd/runmqserver/signals.go

@@ -50,11 +50,13 @@ func signalHandler(qmgr string) chan int {
 				// End the goroutine
 				return
 			case <-reapSignals:
+				logDebug("Received SIGCHLD signal")
 				reapZombies()
 			case job := <-control:
 				switch {
 				case job == startReaping:
 					// Add SIGCHLD to the list of signals we're listening to
+					logDebug("Listening for SIGCHLD signals")
 					signal.Notify(reapSignals, syscall.SIGCHLD)
 				case job == reapNow:
 					reapZombies()
@@ -75,5 +77,6 @@ func reapZombies() {
 		if pid == 0 || err == unix.ECHILD {
 			return
 		}
+		logDebugf("Reaped PID %v", pid)
 	}
 }

test/docker/docker_api_test.go

@@ -17,6 +17,7 @@ package main
 
 import (
 	"context"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -270,3 +271,38 @@ func TestVolumeUnmount(t *testing.T) {
 		t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"}))
 	}
 }
+
+// TestZombies starts a queue manager, then causes a zombie process to be
+// created, then checks that no zombies exist (runmqserver should reap them)
+func TestZombies(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1", "DEBUG=true"},
+		//ExposedPorts: ports,
+		ExposedPorts: nat.PortSet{
+			"1414/tcp": struct{}{},
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	// Kill an MQ process with children.  After it is killed, its children
+	// will be adopted by PID 1, and should then be reaped when they die.
+	out := execContainerWithOutput(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"})
+	if out == "0" {
+		t.Fatalf("Expected pkill to kill a process, got %v", out)
+	}
+	time.Sleep(3 * time.Second)
+	// Create a zombie process for up to ten seconds
+	out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z' | wc -l"})
+	count, err := strconv.Atoi(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != 0 {
+		t.Fatalf("Expected zombies=0, got %v", count)
+	}
+}

test/docker/docker_api_test_util.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -32,6 +33,7 @@ import (
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
+	"github.com/moby/moby/pkg/stdcopy"
 )
 
 func imageName() string {
@@ -241,13 +243,13 @@ func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user s
 	if err != nil {
 		t.Fatal(err)
 	}
-	// TODO: For some reason, each line seems to start with an extra, random character
-	buf, err := ioutil.ReadAll(hijack.Reader)
+	buf := new(bytes.Buffer)
+	// Each output line has a header, which needs to be removed
+	_, err = stdcopy.StdCopy(buf, buf, hijack.Reader)
 	if err != nil {
-		t.Fatal(err)
+		log.Fatal(err)
 	}
-	hijack.Close()
-	return string(buf)
+	return strings.TrimSpace(buf.String())
 }
 
 func waitForReady(t *testing.T, cli *client.Client, ID string) {
@@ -320,8 +322,11 @@ func inspectLogs(t *testing.T, cli *client.Client, ID string) string {
 	if err != nil {
 		log.Fatal(err)
 	}
-
 	buf := new(bytes.Buffer)
-	buf.ReadFrom(reader)
+	// Each output line has a header, which needs to be removed
+	_, err = stdcopy.StdCopy(buf, buf, reader)
+	if err != nil {
+		log.Fatal(err)
+	}
 	return buf.String()
 }