More metric tests and fix reconnect

* more metric test and fix reconnect

* remove build-devjmstest as dependency

test/docker/docker_api_test.go

@@ -16,13 +16,11 @@ limitations under the License.
 package main
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -74,9 +72,20 @@ func TestLicenseView(t *testing.T) {
 	}
 }
 
-// TestGoldenPath starts a queue manager successfully
-func TestGoldenPath(t *testing.T) {
+// TestGoldenPath starts a queue manager successfully when metrics are enabled
+func TestGoldenPathWithMetrics(t *testing.T) {
 	t.Parallel()
+	goldenPath(t, true)
+}
+
+// TestGoldenPath starts a queue manager successfully when metrics are disabled
+func TestGoldenPathNoMetrics(t *testing.T) {
+	t.Parallel()
+	goldenPath(t, false)
+}
+
+// Actual test function for TestGoldenPathNoMetrics & TestGoldenPathWithMetrics
+func goldenPath(t *testing.T, metric bool) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		t.Fatal(err)
@@ -84,6 +93,10 @@ func TestGoldenPath(t *testing.T) {
 	containerConfig := container.Config{
 		Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"},
 	}
+	if metric {
+		containerConfig.Env = append(containerConfig.Env, "MQ_ENABLE_METRICS=true")
+	}
+
 	id := runContainer(t, cli, &containerConfig)
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
@@ -143,9 +156,21 @@ func TestNoQueueManagerNameInvalidHostname(t *testing.T) {
 }
 
 // TestWithVolume runs a container with a Docker volume, then removes that
-// container and starts a new one with same volume.
-func TestWithVolume(t *testing.T) {
+// container and starts a new one with same volume. With metrics enabled
+func TestWithVolumeAndMetrics(t *testing.T) {
 	t.Parallel()
+	withVolume(t, true)
+}
+
+// TestWithVolume runs a container with a Docker volume, then removes that
+// container and starts a new one with same volume. With metrics disabled
+func TestWithVolumeNoMetrics(t *testing.T) {
+	t.Parallel()
+	withVolume(t, false)
+}
+
+// Actual test function for TestWithVolumeNoMetrics & TestWithVolumeAndMetrics
+func withVolume(t *testing.T, metric bool) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		t.Fatal(err)
@@ -156,6 +181,10 @@ func TestWithVolume(t *testing.T) {
 		Image: imageName(),
 		Env:   []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"},
 	}
+	if metric {
+		containerConfig.Env = append(containerConfig.Env, "MQ_ENABLE_METRICS=true")
+	}
+
 	hostConfig := container.HostConfig{
 		Binds: []string{
 			coverageBind(t),
@@ -423,37 +452,6 @@ func TestReadiness(t *testing.T) {
 	}
 }
 
-func countLines(t *testing.T, r io.Reader) int {
-	scanner := bufio.NewScanner(r)
-	count := 0
-	for scanner.Scan() {
-		count++
-	}
-	err := scanner.Err()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return count
-}
-
-func countTarLines(t *testing.T, b []byte) int {
-	r := bytes.NewReader(b)
-	tr := tar.NewReader(r)
-	total := 0
-	for {
-		_, err := tr.Next()
-		if err == io.EOF {
-			// End of TAR
-			break
-		}
-		if err != nil {
-			t.Fatal(err)
-		}
-		total += countLines(t, tr)
-	}
-	return total
-}
-
 func TestErrorLogRotation(t *testing.T) {
 	t.Parallel()
 	cli, err := client.NewEnvClient()
@@ -518,8 +516,20 @@ func TestErrorLogRotation(t *testing.T) {
 	}
 }
 
-func TestJSONLogFormat(t *testing.T) {
+// Tests the log comes out in JSON format when JSON format is enabled. With metrics enabled
+func TestJSONLogFormatWithMetrics(t *testing.T) {
 	t.Parallel()
+	jsonLogFormat(t, true)
+}
+
+// Tests the log comes out in JSON format when JSON format is enabled. With metrics disabled
+func TestJSONLogFormatNoMetrics(t *testing.T) {
+	t.Parallel()
+	jsonLogFormat(t, false)
+}
+
+// Actual test function for TestJSONLogFormatWithMetrics & TestJSONLogFormatNoMetrics
+func jsonLogFormat(t *testing.T, metric bool) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		t.Fatal(err)
@@ -530,6 +540,10 @@ func TestJSONLogFormat(t *testing.T) {
 			"LOG_FORMAT=json",
 		},
 	}
+	if metric {
+		containerConfig.Env = append(containerConfig.Env, "MQ_ENABLE_METRICS=true")
+	}
+
 	id := runContainer(t, cli, &containerConfig)
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)

test/docker/docker_api_test_util.go

@@ -571,3 +571,34 @@ func getWebPort(t *testing.T, cli *client.Client, ID string) string {
 	}
 	return i.NetworkSettings.Ports["9443/tcp"][0].HostPort
 }
+
+func countLines(t *testing.T, r io.Reader) int {
+	scanner := bufio.NewScanner(r)
+	count := 0
+	for scanner.Scan() {
+		count++
+	}
+	err := scanner.Err()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return count
+}
+
+func countTarLines(t *testing.T, b []byte) int {
+	r := bytes.NewReader(b)
+	tr := tar.NewReader(r)
+	total := 0
+	for {
+		_, err := tr.Next()
+		if err == io.EOF {
+			// End of TAR
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		total += countLines(t, tr)
+	}
+	return total
+}

test/docker/mqmetric_test.go

@@ -35,6 +35,7 @@ func TestGoldenPathMetric(t *testing.T) {
 	port := getMetricPort(t, cli, id)
 	// Now the container is ready we prod the prometheus endpoint until it's up.
 	waitForMetricReady(t, port)
+
 	// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
 	getMetrics(t, port)
 	time.Sleep(15 * time.Second)
@@ -188,3 +189,115 @@ func TestSlowPrometheus(t *testing.T) {
 	// Stop the container cleanly
 	stopContainer(t, cli, id)
 }
+
+func TestContainerRestart(t *testing.T) {
+	t.Parallel()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id := runContainerWithPorts(t, cli, metricsContainerConfig(), []int{defaultMetricPort})
+	defer cleanContainer(t, cli, id)
+	port := getMetricPort(t, cli, id)
+
+	// Now the container is ready we prod the prometheus endpoint until it's up.
+	waitForMetricReady(t, port)
+
+	// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
+	getMetrics(t, port)
+
+	time.Sleep(15 * time.Second)
+	metrics := getMetrics(t, port)
+	if len(metrics) <= 0 {
+		t.Log("Expected some metrics to be returned before the restart but had none...")
+		t.FailNow()
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+	// Start the container cleanly
+	startContainer(t, cli, id)
+
+	port = getMetricPort(t, cli, id)
+
+	// Now the container is ready we prod the prometheus endpoint until it's up.
+	waitForMetricReady(t, port)
+
+	// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
+	getMetrics(t, port)
+
+	time.Sleep(15 * time.Second)
+	metrics = getMetrics(t, port)
+	if len(metrics) <= 0 {
+		t.Log("Expected some metrics to be returned before the restart but had none...")
+		t.Fail()
+	}
+}
+
+func TestQMRestart(t *testing.T) {
+	t.Parallel()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id := runContainerWithPorts(t, cli, metricsContainerConfig(), []int{defaultMetricPort})
+	defer cleanContainer(t, cli, id)
+
+	port := getMetricPort(t, cli, id)
+
+	// Now the container is ready we prod the prometheus endpoint until it's up.
+	waitForMetricReady(t, port)
+
+	// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
+	getMetrics(t, port)
+
+	time.Sleep(15 * time.Second)
+	metrics := getMetrics(t, port)
+	if len(metrics) <= 0 {
+		t.Log("Expected some metrics to be returned before the restart but had none...")
+		t.FailNow()
+	}
+
+	// Restart just the QM (to simulate a lost connection)
+	t.Log("Stopping queue manager\n")
+	rc, out := execContainer(t, cli, id, "mqm", []string{"endmqm", "-w", defaultMetricQMName})
+	if rc != 0 {
+		t.Logf("Failed to stop the queue manager. rc=%d, err=%s", rc, out)
+		t.FailNow()
+	}
+	t.Log("starting queue manager\n")
+	rc, out = execContainer(t, cli, id, "mqm", []string{"strmqm", defaultMetricQMName})
+	if rc != 0 {
+		t.Logf("Failed to start the queue manager. rc=%d, err=%s", rc, out)
+		t.FailNow()
+	}
+
+	// Wait for the queue manager to come back up
+	time.Sleep(10 * time.Second)
+
+	// Now the container is ready we prod the prometheus endpoint until it's up.
+	waitForMetricReady(t, port)
+
+	// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
+	getMetrics(t, port)
+
+	time.Sleep(15 * time.Second)
+	metrics = getMetrics(t, port)
+	if len(metrics) <= 0 {
+		t.Log("Expected some metrics to be returned before the restart but had none...")
+		t.FailNow()
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+func TestValidValues(t *testing.T) {
+
+}
+
+func TestChangingValues(t *testing.T) {
+
+}

test/docker/mqmetric_test_util.go

@@ -40,6 +40,7 @@ type mqmetric struct {
 const defaultMetricURL = "/metrics"
 const defaultMetricPort = 9157
 const defaultMQNamespace = "ibmmq"
+const defaultMetricQMName = "qm1"
 
 func getMetrics(t *testing.T, port int) []mqmetric {
 	returned := []mqmetric{}
@@ -144,7 +145,7 @@ func waitForMetricReady(t *testing.T, port int) {
 			return
 		}
 
-		time.Sleep(time.Second * 5)
+		time.Sleep(time.Second * 10)
 	}
 	t.Fatalf("Metric endpoint failed to startup in timely manner")
 }
@@ -167,7 +168,7 @@ func metricsContainerConfig() *container.Config {
 	return &container.Config{
 		Env: []string{
 			"LICENSE=accept",
-			"MQ_QMGR_NAME=qm1",
+			"MQ_QMGR_NAME=" + defaultMetricQMName,
 			"MQ_ENABLE_METRICS=true",
 		},
 	}