From 531f2710c75917e27d8e334e58196a388dc63ee1 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Fri, 25 May 2018 13:09:48 +0100 Subject: [PATCH] Add Docker FV tests for metric gathering (#90) * initial metric tests * Add FV tests for metrics * correct comment in test --- Dockerfile-server | 4 +- glide.lock | 4 +- test/docker/mqmetric_test.go | 315 ++++++++++++++++++ test/docker/mqmetric_test_util.go | 160 +++++++++ .../mq-golang/mqmetric/discover.go | 71 ++-- .../mq-golang/mqmetric/mqmetric_test.go | 55 ++- 6 files changed, 550 insertions(+), 59 deletions(-) create mode 100644 test/docker/mqmetric_test.go create mode 100644 test/docker/mqmetric_test_util.go diff --git a/Dockerfile-server b/Dockerfile-server index 6510e1c..0bb9567 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -64,8 +64,8 @@ RUN chmod ug+x /usr/local/bin/runmqserver \ && chown mqm:mqm /usr/local/bin/*mq* \ && chmod ug+xs /usr/local/bin/chkmq* -# Always use port 1414 -EXPOSE 1414 +# Always use port 1414 for MQ & 9157 for the metrics +EXPOSE 1414 9157 ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic diff --git a/glide.lock b/glide.lock index 4ee8c67..a62c063 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: f59997762f179f5e20cadd65e11fcaae47ca0c3f8dee580e77fa4a9ec9d2c7d1 -updated: 2018-05-21T16:41:39.646218+01:00 +updated: 2018-05-25T11:36:22.237379254+01:00 imports: - name: github.com/beorn7/perks version: 3a771d992973f24aa725d07868b467d1ddfceafb @@ -10,7 +10,7 @@ imports: subpackages: - proto - name: github.com/ibm-messaging/mq-golang - version: 294f0c9e21f4608cc4749c11d928a36d26c8ace7 + version: 3541d35ac44bbd4b161a6488a8f96c59ec4bd707 subpackages: - ibmmq - mqmetric diff --git a/test/docker/mqmetric_test.go b/test/docker/mqmetric_test.go new file mode 100644 index 0000000..73042c6 --- /dev/null +++ b/test/docker/mqmetric_test.go @@ -0,0 +1,315 @@ +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" +) + +func TestGoldenPathMetric(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + + containerConfig := container.Config{ + Env: []string{ + "LICENSE=accept", + "MQ_QMGR_NAME=qm1", + "MQ_ENABLE_METRICS=true", + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + + hostname := getIPAddress(t, cli, id) + port := DEFAULT_METRIC_PORT + + // Now the container is ready we prod the prometheus endpoint until it's up. + waitForMetricReady(hostname, port) + + // Call once as mq_prometheus 'ignores' the first call and will not return any metrics + _, err = getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + time.Sleep(15 * time.Second) + metrics, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + if len(metrics) <= 0 { + t.Log("Expected some metrics to be returned but had none...") + t.Fail() + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} + +func TestMetricNames(t *testing.T) { + t.Parallel() + + approvedSuffixes := []string{"bytes", "seconds", "percentage", "count", "total"} + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + + containerConfig := container.Config{ + Env: []string{ + "LICENSE=accept", + "MQ_QMGR_NAME=qm1", + "MQ_ENABLE_METRICS=true", + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + + hostname := getIPAddress(t, cli, id) + port := DEFAULT_METRIC_PORT + + // Now the container is ready we prod the prometheus endpoint until it's up. + waitForMetricReady(hostname, port) + + // Call once as mq_prometheus 'ignores' the first call + _, err = getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + time.Sleep(15 * time.Second) + metrics, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + if len(metrics) <= 0 { + t.Log("Expected some metrics to be returned but had none...") + t.Fail() + } + + okMetrics := []string{} + badMetrics := []string{} + + for _, metric := range metrics { + ok := false + for _, e := range approvedSuffixes { + if strings.HasSuffix(metric.Key, e) { + ok = true + break + } + } + + if !ok { + t.Logf("Metric '%s' does not have an approved suffix", metric.Key) + badMetrics = append(badMetrics, metric.Key) + t.Fail() + } else { + okMetrics = append(okMetrics, metric.Key) + } + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} + +func TestMetricLabels(t *testing.T) { + t.Parallel() + + requiredLabels := []string{"qmgr"} + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + + containerConfig := container.Config{ + Env: []string{ + "LICENSE=accept", + "MQ_QMGR_NAME=qm1", + "MQ_ENABLE_METRICS=true", + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + + hostname := getIPAddress(t, cli, id) + port := DEFAULT_METRIC_PORT + + // Now the container is ready we prod the prometheus endpoint until it's up. + waitForMetricReady(hostname, port) + + // Call once as mq_prometheus 'ignores' the first call + _, err = getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + time.Sleep(15 * time.Second) + metrics, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + if len(metrics) <= 0 { + t.Log("Expected some metrics to be returned but had none...") + t.Fail() + } + + for _, metric := range metrics { + found := false + for key := range metric.Labels { + for _, e := range requiredLabels { + if key == e { + found = true + break + } + } + if found { + break + } + } + + if !found { + t.Logf("Metric '%s' with labels %s does not have one or more required labels - %s", metric.Key, metric.Labels, requiredLabels) + t.Fail() + } + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} + +func TestRapidFirePrometheus(t *testing.T) { + t.Parallel() + + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + + containerConfig := container.Config{ + Env: []string{ + "LICENSE=accept", + "MQ_QMGR_NAME=qm1", + "MQ_ENABLE_METRICS=true", + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + + hostname := getIPAddress(t, cli, id) + port := DEFAULT_METRIC_PORT + + // Now the container is ready we prod the prometheus endpoint until it's up. + waitForMetricReady(hostname, port) + + // Call once as mq_prometheus 'ignores' the first call and will not return any metrics + _, err = getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + // Rapid fire it then check we're still happy + for i := 0; i < 30; i++ { + _, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + time.Sleep(1 * time.Second) + } + + time.Sleep(11 * time.Second) + + metrics, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + if len(metrics) <= 0 { + t.Log("Expected some metrics to be returned but had none...") + t.Fail() + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} + +func TestSlowPrometheus(t *testing.T) { + t.Parallel() + + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + + containerConfig := container.Config{ + Env: []string{ + "LICENSE=accept", + "MQ_QMGR_NAME=qm1", + "MQ_ENABLE_METRICS=true", + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + + hostname := getIPAddress(t, cli, id) + port := DEFAULT_METRIC_PORT + + // Now the container is ready we prod the prometheus endpoint until it's up. + waitForMetricReady(hostname, port) + + // Call once as mq_prometheus 'ignores' the first call and will not return any metrics + _, err = getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + + // Send a request twice over a long period and check we're still happy + for i := 0; i < 2; i++ { + time.Sleep(30 * time.Second) + metrics, err := getMetricsFromEndpoint(hostname, port) + if err != nil { + t.Logf("Failed to call metric endpoint - %v", err) + t.FailNow() + } + if len(metrics) <= 0 { + t.Log("Expected some metrics to be returned but had none...") + t.Fail() + } + + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} diff --git a/test/docker/mqmetric_test_util.go b/test/docker/mqmetric_test_util.go new file mode 100644 index 0000000..3ce09b7 --- /dev/null +++ b/test/docker/mqmetric_test_util.go @@ -0,0 +1,160 @@ +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" +) + +type MQMETRIC struct { + Key string + Value string + Labels map[string]string +} + +const DEFAULT_METRIC_URL = "/metrics" +const DEFAULT_METRIC_PORT = 9157 +const DEFAULT_MQ_NAMESPACE = "ibmmq" + +func getMetricsFromEndpoint(host string, port int) ([]MQMETRIC, error) { + returned := []MQMETRIC{} + if host == "" { + return returned, fmt.Errorf("Test Error - Host was nil") + } + if port <= 0 { + return returned, fmt.Errorf("Test Error - port was not above 0") + } + urlToUse := fmt.Sprintf("http://%s:%d%s", host, port, DEFAULT_METRIC_URL) + + resp, err := http.Get(urlToUse) + if err != nil { + return returned, err + } + defer resp.Body.Close() + metricsRaw, err := ioutil.ReadAll(resp.Body) + if err != nil { + return returned, err + } + + return convertRawMetricToMap(string(metricsRaw)) +} + +// Also filters out all non "ibmmq" metrics +func convertRawMetricToMap(input string) ([]MQMETRIC, error) { + returnList := []MQMETRIC{} + if input == "" { + return returnList, fmt.Errorf("Test Error - Raw metric output was nil") + } + + scanner := bufio.NewScanner(strings.NewReader(input)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") { + // Comment line of HELP or TYPE. Ignore + continue + } + if !strings.HasPrefix(line, DEFAULT_MQ_NAMESPACE) { + // Not an ibmmq_ metric. Ignore + continue + } + //It's an IBM MQ metric! + key, value, labelMap, err := convertMetricLineToMetric(line) + if err != nil { + return returnList, fmt.Errorf("ibmmq_ metric could not be deciphered - %v", err) + } + + toAdd := MQMETRIC{ + Key: key, + Value: value, + Labels: labelMap, + } + + returnList = append(returnList, toAdd) + } + + return returnList, nil +} + +func convertMetricLineToMetric(input string) (string, string, map[string]string, error) { + // Lines are in the form "{}" or " " + // Get the key and value while skipping the label + var key, value string + labelMap := make(map[string]string) + if strings.Contains(input, "{") { + // Get key + splitted := strings.Split(input, "{") + if len(splitted) != 2 { + return "", "", labelMap, fmt.Errorf("Could not split by { Expected 2 but got %d - %s", len(splitted), input) + } + key = strings.TrimSpace(splitted[0]) + + // Get value + splitted = strings.Split(splitted[1], "}") + if len(splitted) != 2 { + return "", "", labelMap, fmt.Errorf("Could not split by } Expected 2 but got %d - %s", len(splitted), input) + } + value = strings.TrimSpace(splitted[1]) + + // Get labels + allLabels := strings.Split(splitted[0], ",") + for _, e := range allLabels { + labelPair := strings.Split(e, "=") + if len(labelPair) != 2 { + return "", "", labelMap, fmt.Errorf("Could not split label by '=' Expected 2 but got %d - %s", len(labelPair), e) + } + lkey := strings.TrimSpace(labelPair[0]) + lvalue := strings.TrimSpace(labelPair[1]) + lvalue = strings.Trim(lvalue, "\"") + labelMap[lkey] = lvalue + } + + } else { + splitted := strings.Split(input, " ") + if len(splitted) != 2 { + return "", "", labelMap, fmt.Errorf("Could not split by ' ' Expected 2 but got %d - %s", len(splitted), input) + } + key = strings.TrimSpace(splitted[0]) + value = strings.TrimSpace(splitted[1]) + } + return key, value, labelMap, nil +} + +func waitForMetricReady(host string, port int) error { + if host == "" { + return fmt.Errorf("Test Error - Host was nil") + } + if port <= 0 { + return fmt.Errorf("Test Error - port was not above 0") + } + timeout := 12 // 12 * 5 = 1 minute + for i := 0; i < timeout; i++ { + urlToUse := fmt.Sprintf("http://%s:%d", host, port) + resp, err := http.Get(urlToUse) + if err == nil { + resp.Body.Close() + return nil + } + + time.Sleep(time.Second * 5) + } + + return fmt.Errorf("Metric endpoint failed to startup in timely manner") +} diff --git a/vendor/github.com/ibm-messaging/mq-golang/mqmetric/discover.go b/vendor/github.com/ibm-messaging/mq-golang/mqmetric/discover.go index 132b561..e06d106 100755 --- a/vendor/github.com/ibm-messaging/mq-golang/mqmetric/discover.go +++ b/vendor/github.com/ibm-messaging/mq-golang/mqmetric/discover.go @@ -35,6 +35,7 @@ import ( "bufio" "fmt" "os" + "regexp" "strings" "github.com/ibm-messaging/mq-golang/ibmmq" @@ -263,7 +264,7 @@ func discoverElements(ty *MonType) error { } } - elem.MetricName = formatDescriptionElem(elem) + elem.MetricName = formatDescription(elem) ty.Elements[elementIndex] = elem } } @@ -631,59 +632,49 @@ bytes etc), and organisation of the elements of the name (units last) While we can't change the MQ-generated descriptions for its statistics, we can reformat most of them heuristically here. */ -func formatDescriptionElem(elem *MonElement) string { - s := formatDescription(elem.Description) - - unit := "" - switch elem.Datatype { - case ibmmq.MQIAMO_MONITOR_MICROSEC: - // Although the qmgr captures in us, we convert when - // pushing out to the backend, so this label needs to match - unit = "_seconds" - } - s += unit - - return s -} - -func formatDescription(baseName string) string { - s := baseName +func formatDescription(elem *MonElement) string { + s := elem.Description s = strings.Replace(s, " ", "_", -1) s = strings.Replace(s, "/", "_", -1) s = strings.Replace(s, "-", "_", -1) - /* common pattern is "xxx - yyy" leading to 3 ugly adjacent underscores */ - s = strings.Replace(s, "___", "_", -1) - s = strings.Replace(s, "__", "_", -1) + /* Make sure we don't have multiple underscores */ + multiunder := regexp.MustCompile("__*") + s = multiunder.ReplaceAllLiteralString(s, "_") /* make it all lowercase. Not essential, but looks better */ s = strings.ToLower(s) - // Do not use _count + /* Remove all cases of bytes, seconds, count or percentage (we add them back in later) */ s = strings.Replace(s, "_count", "", -1) + s = strings.Replace(s, "_bytes", "", -1) + s = strings.Replace(s, "_byte", "", -1) + s = strings.Replace(s, "_seconds", "", -1) + s = strings.Replace(s, "_second", "", -1) + s = strings.Replace(s, "_percentage", "", -1) // Switch round a couple of specific names - s = strings.Replace(s, "bytes_written", "written_bytes", -1) - s = strings.Replace(s, "bytes_max", "max_bytes", -1) - s = strings.Replace(s, "bytes_in_use", "in_use_bytes", -1) s = strings.Replace(s, "messages_expired", "expired_messages", -1) - if strings.HasSuffix(s, "free_space") { + // Add the unit at end + switch elem.Datatype { + case ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS: s = s + "_percentage" - s = strings.Replace(s, "__", "_", -1) - } - - // Make "byte", "file" and "message" units plural - if strings.HasSuffix(s, "byte") || - strings.HasSuffix(s, "message") || - strings.HasSuffix(s, "file") { - s = s + "s" - } - - // Move % to the end - if strings.Contains(s, "_percentage_") { - s = strings.Replace(s, "_percentage_", "_", -1) - s += "_percentage" + case ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB: + s = s + "_bytes" + case ibmmq.MQIAMO_MONITOR_MICROSEC: + s = s + "_seconds" + default: + if strings.Contains(s, "_total") { + /* If we specify it is a total in description put that at the end */ + s = strings.Replace(s, "_total", "", -1) + s = s + "_total" + } else if strings.Contains(s, "log_") { + /* Weird case where the log datatype is not MB or GB but should be bytes */ + s = s + "_bytes" + } else { + s = s + "_count" + } } return s diff --git a/vendor/github.com/ibm-messaging/mq-golang/mqmetric/mqmetric_test.go b/vendor/github.com/ibm-messaging/mq-golang/mqmetric/mqmetric_test.go index e5be9ef..1b97fb7 100644 --- a/vendor/github.com/ibm-messaging/mq-golang/mqmetric/mqmetric_test.go +++ b/vendor/github.com/ibm-messaging/mq-golang/mqmetric/mqmetric_test.go @@ -90,36 +90,61 @@ func TestReadPatterns(t *testing.T) { } } func TestFormatDescription(t *testing.T) { - give := [...]string{"hello", "no space", "no/slash", "no-dash", "single___underscore", "single__underscore", "ALLLOWER", "no_count", "this_bytes_written_switch", "this_bytes_max_switch", "this_bytes_in_use_switch", "this messages_expired_switch", "add_free_space", "suffix_byte", "suffix_message", "suffix_file", "this_percentage_move"} - expected := [...]string{"hello", "no_space", "no_slash", "no_dash", "single_underscore", "single_underscore", "alllower", "no", "this_written_bytes_switch", "this_max_bytes_switch", "this_in_use_bytes_switch", "this_expired_messages_switch", "add_free_space_percentage", "suffix_bytes", "suffix_messages", "suffix_files", "this_move_percentage"} + give := [...]string{"hello", "no space", "no/slash", "no-dash", "single___underscore", "single__underscore__multiplace", "ALLLOWER", "this_bytes_written_switch", "this_byte_max_switch", "this_seconds_in_use_switch", "this messages_expired_switch", "this_seconds_max_switch", "this_count_max_switch", "this_percentage_max_switch"} + expected := [...]string{"hello_count", "no_space_count", "no_slash_count", "no_dash_count", "single_underscore_count", "single_underscore_multiplace_count", "alllower_count", "this_written_switch_count", "this_max_switch_count", "this_in_use_switch_count", "this_expired_messages_switch_count", "this_max_switch_count", "this_max_switch_count", "this_max_switch_count"} for i, e := range give { - back := formatDescription(e) + elem := MonElement{ + Description: e, + } + back := formatDescription(&elem) if back != expected[i] { t.Logf("Gave %s. Expected: %s, Got: %s", e, expected[i], back) t.Fail() } } } -func TestFormatDescriptionElem(t *testing.T) { - test := MonElement{} - test.Description = "THIS-should__be/formatted_count" - expected := "this_should_be_formatted" - back := formatDescriptionElem(&test) - if back != expected { - t.Logf("Gave %s. Expected: %s, Got: %s", test.Description, expected, back) +func TestSuffixes(t *testing.T) { + baseDescription := "test_suffix" + types := [...]int32{ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB, ibmmq.MQIAMO_MONITOR_MICROSEC, ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS, 0} + expected := [...]string{baseDescription + "_bytes", baseDescription + "_bytes", baseDescription + "_seconds", baseDescription + "_percentage", baseDescription + "_percentage", baseDescription + "_count"} + + for i, ty := range types { + elem := MonElement{ + Description: baseDescription, + Datatype: ty, + } + back := formatDescription(&elem) + if back != expected[i] { + t.Logf("Gave %s/%d Expected: %s, Got: %s", baseDescription, ty, expected[i], back) + t.Fail() + } + } + + // special case log_bytes + elem := MonElement{ + Description: "log_test_suffix", + Datatype: 0, + } + back := formatDescription(&elem) + if back != "log_test_suffix_bytes" { + t.Logf("Gave log_test_suffix/0 Expected: %s, Got: %s", "log_test_suffix_bytes", back) t.Fail() } - test.Datatype = ibmmq.MQIAMO_MONITOR_MICROSEC - expected = "this_should_be_formatted_seconds" - back = formatDescriptionElem(&test) - if back != expected { - t.Logf("Gave %s. Expected: %s, Got: %s", test.Description, expected, back) + // special case log_total + elem = MonElement{ + Description: "log_total_suffix", + Datatype: 0, + } + back = formatDescription(&elem) + if back != "log_suffix_total" { + t.Logf("Gave log_total_suffix/0 Expected: %s, Got: %s", "log_suffix_total", back) t.Fail() } } + func TestParsePCFResponse(t *testing.T) { cfh := ibmmq.NewMQCFH() cfh.Type = ibmmq.MQCFT_RESPONSE