add multi-instance Queue Managers (#307)

* Initial code to implement multi-instance queue manager

* alter default mqsc to prevent race between listeners on standby startup

* Updates to multi-instance queue manager code

* initial multi instance test

* Multi-instance code improvements

* Multi instance fixes and first test

* configure queue manager

* Add mirror log filtering for mult-instance QMs

* Add log message for multi-instance enabled

* Improvements to container runtime logging

* refactor test

* Test active standby switch

* Improve createQueueManager function

* Test multi instance race

* wait

* multi instance mount tests

* skip race test

* mount tests

* no mount test

* single instance split mount tests

* readiness check

* More updates for handling standby queue manager

* Improve standby checks

* Minor fixes to miqm

* Fix logging of JSON errors

* Update copyrights

* Fix log includes

test/docker/docker_api_test.go

@@ -235,7 +235,7 @@ func withVolume(t *testing.T, metric bool) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	vol := createVolume(t, cli)
+	vol := createVolume(t, cli, t.Name())
 	defer removeVolume(t, cli, vol.Name)
 	containerConfig := container.Config{
 		Image: imageName(),
@@ -273,6 +273,62 @@ func withVolume(t *testing.T, metric bool) {
 	waitForReady(t, cli, ctr2.ID)
 }
 
+// TestWithSplitVolumesLogsData starts a queue manager with separate log/data mounts
+func TestWithSplitVolumesLogsData(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmsharedlogs := createVolume(t, cli, "qmsharedlogs")
+	defer removeVolume(t, cli, qmsharedlogs.Name)
+	qmshareddata := createVolume(t, cli, "qmshareddata")
+	defer removeVolume(t, cli, qmshareddata.Name)
+
+	err, qmID, qmVol := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs.Name, qmshareddata.Name, []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"})
+
+	defer removeVolume(t, cli, qmVol)
+	defer cleanContainer(t, cli, qmID)
+
+	waitForReady(t, cli, qmID)
+}
+
+// TestWithSplitVolumesLogsOnly starts a queue manager with a separate log mount
+func TestWithSplitVolumesLogsOnly(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmsharedlogs := createVolume(t, cli, "qmsharedlogs")
+	defer removeVolume(t, cli, qmsharedlogs.Name)
+
+	err, qmID, qmVol := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs.Name, "", []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"})
+
+	defer removeVolume(t, cli, qmVol)
+	defer cleanContainer(t, cli, qmID)
+
+	waitForReady(t, cli, qmID)
+}
+
+// TestWithSplitVolumesDataOnly starts a queue manager with a separate data mount
+func TestWithSplitVolumesDataOnly(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmshareddata := createVolume(t, cli, "qmshareddata")
+	defer removeVolume(t, cli, qmshareddata.Name)
+
+	err, qmID, qmVol := startMultiVolumeQueueManager(t, cli, true, "", qmshareddata.Name, []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"})
+
+	defer removeVolume(t, cli, qmVol)
+	defer cleanContainer(t, cli, qmID)
+
+	waitForReady(t, cli, qmID)
+}
+
 // TestNoVolumeWithRestart ensures a queue manager container can be stopped
 // and restarted cleanly
 func TestNoVolumeWithRestart(t *testing.T) {
@@ -302,7 +358,7 @@ func TestVolumeRequiresRoot(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	vol := createVolume(t, cli)
+	vol := createVolume(t, cli, t.Name())
 	defer removeVolume(t, cli, vol.Name)
 
 	// Set permissions on the volume to only allow root to write it
@@ -438,7 +494,7 @@ func TestVolumeUnmount(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	vol := createVolume(t, cli)
+	vol := createVolume(t, cli, t.Name())
 	defer removeVolume(t, cli, vol.Name)
 	containerConfig := container.Config{
 		Image: imageName(),

test/docker/docker_api_test_util.go

@@ -398,6 +398,14 @@ func stopContainer(t *testing.T, cli *client.Client, ID string) {
 	}
 }
 
+func killContainer(t *testing.T, cli *client.Client, ID string, signal string) {
+	t.Logf("Killing container: %v", ID)
+	err := cli.ContainerKill(context.Background(), ID, signal)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func getExitCodeFilename(t *testing.T) string {
 	return t.Name() + "ExitCode"
 }
@@ -522,6 +530,9 @@ func waitForReady(t *testing.T, cli *client.Client, ID string) {
 			if rc == 0 {
 				t.Log("MQ is ready")
 				return
+			} else if rc == 10 {
+				t.Log("MQ Readiness: Queue Manager Running as Standby")
+				return
 			}
 		case <-ctx.Done():
 			t.Fatal("Timed out waiting for container to become ready")
@@ -557,17 +568,17 @@ func removeNetwork(t *testing.T, cli *client.Client, ID string) {
 	}
 }
 
-func createVolume(t *testing.T, cli *client.Client) types.Volume {
+func createVolume(t *testing.T, cli *client.Client, name string) types.Volume {
 	v, err := cli.VolumeCreate(context.Background(), volume.VolumesCreateBody{
 		Driver:     "local",
 		DriverOpts: map[string]string{},
 		Labels:     map[string]string{},
-		Name:       t.Name(),
+		Name:       name,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Logf("Created volume %v", t.Name())
+	t.Logf("Created volume %v", v.Name)
 	return v
 }
 

test/docker/mq_multi_instance_test.go

@@ -0,0 +1,234 @@
+/*
+© Copyright IBM Corporation 2019
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/client"
+)
+
+var miEnv = []string{
+	"LICENSE=accept",
+	"MQ_QMGR_NAME=QM1",
+	"MQ_MULTI_INSTANCE=true",
+}
+
+// TestMultiInstanceStartStop creates 2 containers in a multi instance queue manager configuration
+// and starts/stop them checking we always have an active and standby
+func TestMultiInstanceStartStop(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err, qm1aId, qm1bId, volumes := configureMultiInstance(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, volume := range volumes {
+		defer removeVolume(t, cli, volume)
+	}
+	defer cleanContainer(t, cli, qm1aId)
+	defer cleanContainer(t, cli, qm1bId)
+
+	waitForReady(t, cli, qm1aId)
+	waitForReady(t, cli, qm1bId)
+
+	err, active, standby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	killContainer(t, cli, active, "SIGTERM")
+	time.Sleep(2 * time.Second)
+
+	if status := getQueueManagerStatus(t, cli, standby, "QM1"); strings.Compare(status, "Running") != 0 {
+		t.Fatalf("Expected QM1 to be running as active queue manager, dspmq returned status of %v", status)
+	}
+
+	startContainer(t, cli, qm1aId)
+	waitForReady(t, cli, qm1aId)
+
+	err, _, _ = getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+// TestMultiInstanceContainerStop starts 2 containers in a multi instance queue manager configuration,
+// stops the active queue manager, then checks to ensure the backup queue manager becomes active
+func TestMultiInstanceContainerStop(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err, qm1aId, qm1bId, volumes := configureMultiInstance(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, volume := range volumes {
+		defer removeVolume(t, cli, volume)
+	}
+	defer cleanContainer(t, cli, qm1aId)
+	defer cleanContainer(t, cli, qm1bId)
+
+	waitForReady(t, cli, qm1aId)
+	waitForReady(t, cli, qm1bId)
+
+	err, active, standby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stopContainer(t, cli, active)
+
+	if status := getQueueManagerStatus(t, cli, standby, "QM1"); strings.Compare(status, "Running") != 0 {
+		t.Fatalf("Expected QM1 to be running as active queue manager, dspmq returned status of %v", status)
+	}
+}
+
+// TestMultiInstanceRace starts 2 containers in separate goroutines in a multi instance queue manager
+// configuration, then checks to ensure that both an active and standby queue manager have been started
+func TestMultiInstanceRace(t *testing.T) {
+	t.Skipf("Skipping %v until file lock is implemented", t.Name())
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmsharedlogs := createVolume(t, cli, "qmsharedlogs")
+	defer removeVolume(t, cli, qmsharedlogs.Name)
+	qmshareddata := createVolume(t, cli, "qmshareddata")
+	defer removeVolume(t, cli, qmshareddata.Name)
+
+	qmsChannel := make(chan QMChan)
+
+	go singleMultiInstanceQueueManager(t, cli, qmsharedlogs.Name, qmshareddata.Name, qmsChannel)
+	go singleMultiInstanceQueueManager(t, cli, qmsharedlogs.Name, qmshareddata.Name, qmsChannel)
+
+	qm1a := <-qmsChannel
+	if qm1a.Error != nil {
+		t.Fatal(qm1a.Error)
+	}
+
+	qm1b := <-qmsChannel
+	if qm1b.Error != nil {
+		t.Fatal(qm1b.Error)
+	}
+
+	qm1aId, qm1aData := qm1a.QMId, qm1a.QMData
+	qm1bId, qm1bData := qm1b.QMId, qm1b.QMData
+
+	defer removeVolume(t, cli, qm1aData)
+	defer removeVolume(t, cli, qm1bData)
+	defer cleanContainer(t, cli, qm1aId)
+	defer cleanContainer(t, cli, qm1bId)
+
+	waitForReady(t, cli, qm1aId)
+	waitForReady(t, cli, qm1bId)
+
+	err, _, _ = getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestMultiInstanceNoSharedMounts starts 2 multi instance queue managers without providing shared log/data
+// mounts, then checks to ensure that the container terminates with the expected message
+func TestMultiInstanceNoSharedMounts(t *testing.T) {
+	t.Parallel()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err, qm1aId, qm1aData := startMultiVolumeQueueManager(t, cli, true, "", "", miEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeVolume(t, cli, qm1aData)
+	defer cleanContainer(t, cli, qm1aId)
+
+	waitForTerminationMessage(t, cli, qm1aId, "Missing required mount '/mnt/mqm-log'", 30*time.Second)
+}
+
+// TestMultiInstanceNoSharedLogs starts 2 multi instance queue managers without providing a shared log
+// mount, then checks to ensure that the container terminates with the expected message
+func TestMultiInstanceNoSharedLogs(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmshareddata := createVolume(t, cli, "qmshareddata")
+	defer removeVolume(t, cli, qmshareddata.Name)
+
+	err, qm1aId, qm1aData := startMultiVolumeQueueManager(t, cli, true, "", qmshareddata.Name, miEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeVolume(t, cli, qm1aData)
+	defer cleanContainer(t, cli, qm1aId)
+
+	waitForTerminationMessage(t, cli, qm1aId, "Missing required mount '/mnt/mqm-log'", 30*time.Second)
+}
+
+// TestMultiInstanceNoSharedData starts 2 multi instance queue managers without providing a shared data
+// mount, then checks to ensure that the container terminates with the expected message
+func TestMultiInstanceNoSharedData(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qmsharedlogs := createVolume(t, cli, "qmsharedlogs")
+	defer removeVolume(t, cli, qmsharedlogs.Name)
+
+	err, qm1aId, qm1aData := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs.Name, "", miEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeVolume(t, cli, qm1aData)
+	defer cleanContainer(t, cli, qm1aId)
+
+	waitForTerminationMessage(t, cli, qm1aId, "Missing required mount '/mnt/mqm-data'", 30*time.Second)
+}
+
+// TestMultiInstanceNoMounts starts 2 multi instance queue managers without providing a shared data
+// mount, then checks to ensure that the container terminates with the expected message
+func TestMultiInstanceNoMounts(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err, qm1aId, qm1aData := startMultiVolumeQueueManager(t, cli, false, "", "", miEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeVolume(t, cli, qm1aData)
+	defer cleanContainer(t, cli, qm1aId)
+
+	waitForTerminationMessage(t, cli, qm1aId, "Missing required mount '/mnt/mqm'", 30*time.Second)
+}

test/docker/mq_multi_instance_test_util.go

@@ -0,0 +1,178 @@
+/*
+© Copyright IBM Corporation 2019
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+)
+
+type QMChan struct {
+	QMId   string
+	QMData string
+	Error  error
+}
+
+// configureMultiInstance creates the volumes and containers required for basic testing
+// of multi instance queue managers. Returns error, qm1a ID, qm1b ID, slice of volume names
+func configureMultiInstance(t *testing.T, cli *client.Client) (error, string, string, []string) {
+
+	qmsharedlogs := createVolume(t, cli, "qmsharedlogs")
+	qmshareddata := createVolume(t, cli, "qmshareddata")
+
+	err, qm1aId, qm1aData := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs.Name, qmshareddata.Name, miEnv)
+	if err != nil {
+		return err, "", "", []string{}
+	}
+	time.Sleep(10 * time.Second)
+	err, qm1bId, qm1bData := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs.Name, qmshareddata.Name, miEnv)
+	if err != nil {
+		return err, "", "", []string{}
+	}
+
+	volumes := []string{qmsharedlogs.Name, qmshareddata.Name, qm1aData, qm1bData}
+
+	return nil, qm1aId, qm1bId, volumes
+}
+
+func singleMultiInstanceQueueManager(t *testing.T, cli *client.Client, qmsharedlogs string, qmshareddata string, qmsChannel chan QMChan) {
+	err, qmId, qmData := startMultiVolumeQueueManager(t, cli, true, qmsharedlogs, qmshareddata, miEnv)
+	if err != nil {
+		qmsChannel <- QMChan{Error: err}
+	}
+	qmsChannel <- QMChan{QMId: qmId, QMData: qmData}
+}
+
+func getHostConfig(t *testing.T, mounts int, qmsharedlogs string, qmshareddata string, qmdata string) container.HostConfig {
+
+	var hostConfig container.HostConfig
+
+	switch mounts {
+	case 1:
+		hostConfig = container.HostConfig{
+			Binds: []string{
+				coverageBind(t),
+				qmdata + ":/mnt/mqm",
+			},
+		}
+	case 2:
+		hostConfig = container.HostConfig{
+			Binds: []string{
+				coverageBind(t),
+				qmdata + ":/mnt/mqm",
+				qmshareddata + ":/mnt/mqm-data",
+			},
+		}
+	case 3:
+		hostConfig = container.HostConfig{
+			Binds: []string{
+				coverageBind(t),
+				qmdata + ":/mnt/mqm",
+				qmsharedlogs + ":/mnt/mqm-log",
+			},
+		}
+	case 4:
+		hostConfig = container.HostConfig{
+			Binds: []string{
+				coverageBind(t),
+				qmdata + ":/mnt/mqm",
+				qmsharedlogs + ":/mnt/mqm-log",
+				qmshareddata + ":/mnt/mqm-data",
+			},
+		}
+	}
+
+	return hostConfig
+}
+
+func startMultiVolumeQueueManager(t *testing.T, cli *client.Client, dataVol bool, qmsharedlogs string, qmshareddata string, env []string) (error, string, string) {
+	id := strconv.FormatInt(time.Now().UnixNano(), 10)
+	qmdata := createVolume(t, cli, id)
+	containerConfig := container.Config{
+		Image: imageName(),
+		Env:   env,
+	}
+	var hostConfig container.HostConfig
+
+	if !dataVol {
+		hostConfig = container.HostConfig{}
+	} else if qmsharedlogs == "" && qmshareddata == "" {
+		hostConfig = getHostConfig(t, 1, "", "", qmdata.Name)
+	} else if qmsharedlogs == "" {
+		hostConfig = getHostConfig(t, 2, "", qmshareddata, qmdata.Name)
+	} else if qmshareddata == "" {
+		hostConfig = getHostConfig(t, 3, qmsharedlogs, "", qmdata.Name)
+	} else {
+		hostConfig = getHostConfig(t, 4, qmsharedlogs, qmshareddata, qmdata.Name)
+	}
+	networkingConfig := network.NetworkingConfig{}
+	qm, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()+id)
+	if err != nil {
+		return err, "", ""
+	}
+	startContainer(t, cli, qm.ID)
+
+	return nil, qm.ID, qmdata.Name
+}
+
+func getActiveStandbyQueueManager(t *testing.T, cli *client.Client, qm1aId string, qm1bId string) (error, string, string) {
+	qm1aStatus := getQueueManagerStatus(t, cli, qm1aId, "QM1")
+	qm1bStatus := getQueueManagerStatus(t, cli, qm1bId, "QM1")
+
+	if qm1aStatus == "Running" && qm1bStatus == "Running as standby" {
+		return nil, qm1aId, qm1bId
+	} else if qm1bStatus == "Running" && qm1aStatus == "Running as standby" {
+		return nil, qm1bId, qm1aId
+	}
+	err := fmt.Errorf("Expected to be running in multi instance configuration, got status 1) %v status 2) %v", qm1aStatus, qm1bStatus)
+	return err, "", ""
+}
+
+func getQueueManagerStatus(t *testing.T, cli *client.Client, containerID string, queueManagerName string) string {
+	_, dspmqOut := execContainer(t, cli, containerID, "mqm", []string{"bash", "-c", "dspmq", "-m", queueManagerName})
+	regex := regexp.MustCompile(`STATUS\(.*\)`)
+	status := regex.FindString(dspmqOut)
+	status = strings.TrimSuffix(strings.TrimPrefix(status, "STATUS("), ")")
+	return status
+}
+
+func waitForTerminationMessage(t *testing.T, cli *client.Client, qmId string, terminationString string, timeout time.Duration) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	for {
+		select {
+		case <-time.After(1 * time.Second):
+			m := terminationMessage(t, cli, qmId)
+			if m != "" {
+				if !strings.Contains(m, terminationString) {
+					t.Fatalf("Expected container to fail on missing required mount. Got termination message: %v", m)
+				}
+				return
+			}
+		case <-ctx.Done():
+			t.Fatal("Timed out waiting for container to terminate")
+		}
+	}
+}