Add Docker FV tests for metric gathering (#90)

* initial metric tests

* Add FV tests for metrics

* correct comment in test
This commit is contained in:
Rob Parker
2018-05-25 13:09:48 +01:00
committed by Stephen Marshall
parent 91e20f0f98
commit 531f2710c7
6 changed files with 550 additions and 59 deletions

View File

@@ -64,8 +64,8 @@ RUN chmod ug+x /usr/local/bin/runmqserver \
&& chown mqm:mqm /usr/local/bin/*mq* \
&& chmod ug+xs /usr/local/bin/chkmq*
# Always use port 1414
EXPOSE 1414
# Always use port 1414 for MQ & 9157 for the metrics
EXPOSE 1414 9157
ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic

4
glide.lock generated
View File

@@ -1,5 +1,5 @@
hash: f59997762f179f5e20cadd65e11fcaae47ca0c3f8dee580e77fa4a9ec9d2c7d1
updated: 2018-05-21T16:41:39.646218+01:00
updated: 2018-05-25T11:36:22.237379254+01:00
imports:
- name: github.com/beorn7/perks
version: 3a771d992973f24aa725d07868b467d1ddfceafb
@@ -10,7 +10,7 @@ imports:
subpackages:
- proto
- name: github.com/ibm-messaging/mq-golang
version: 294f0c9e21f4608cc4749c11d928a36d26c8ace7
version: 3541d35ac44bbd4b161a6488a8f96c59ec4bd707
subpackages:
- ibmmq
- mqmetric

View File

@@ -0,0 +1,315 @@
/*
© Copyright IBM Corporation 2018
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"strings"
"testing"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
)
func TestGoldenPathMetric(t *testing.T) {
t.Parallel()
cli, err := client.NewEnvClient()
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
Env: []string{
"LICENSE=accept",
"MQ_QMGR_NAME=qm1",
"MQ_ENABLE_METRICS=true",
},
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
hostname := getIPAddress(t, cli, id)
port := DEFAULT_METRIC_PORT
// Now the container is ready we prod the prometheus endpoint until it's up.
waitForMetricReady(hostname, port)
// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
_, err = getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
time.Sleep(15 * time.Second)
metrics, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
if len(metrics) <= 0 {
t.Log("Expected some metrics to be returned but had none...")
t.Fail()
}
// Stop the container cleanly
stopContainer(t, cli, id)
}
func TestMetricNames(t *testing.T) {
t.Parallel()
approvedSuffixes := []string{"bytes", "seconds", "percentage", "count", "total"}
cli, err := client.NewEnvClient()
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
Env: []string{
"LICENSE=accept",
"MQ_QMGR_NAME=qm1",
"MQ_ENABLE_METRICS=true",
},
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
hostname := getIPAddress(t, cli, id)
port := DEFAULT_METRIC_PORT
// Now the container is ready we prod the prometheus endpoint until it's up.
waitForMetricReady(hostname, port)
// Call once as mq_prometheus 'ignores' the first call
_, err = getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
time.Sleep(15 * time.Second)
metrics, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
if len(metrics) <= 0 {
t.Log("Expected some metrics to be returned but had none...")
t.Fail()
}
okMetrics := []string{}
badMetrics := []string{}
for _, metric := range metrics {
ok := false
for _, e := range approvedSuffixes {
if strings.HasSuffix(metric.Key, e) {
ok = true
break
}
}
if !ok {
t.Logf("Metric '%s' does not have an approved suffix", metric.Key)
badMetrics = append(badMetrics, metric.Key)
t.Fail()
} else {
okMetrics = append(okMetrics, metric.Key)
}
}
// Stop the container cleanly
stopContainer(t, cli, id)
}
func TestMetricLabels(t *testing.T) {
t.Parallel()
requiredLabels := []string{"qmgr"}
cli, err := client.NewEnvClient()
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
Env: []string{
"LICENSE=accept",
"MQ_QMGR_NAME=qm1",
"MQ_ENABLE_METRICS=true",
},
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
hostname := getIPAddress(t, cli, id)
port := DEFAULT_METRIC_PORT
// Now the container is ready we prod the prometheus endpoint until it's up.
waitForMetricReady(hostname, port)
// Call once as mq_prometheus 'ignores' the first call
_, err = getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
time.Sleep(15 * time.Second)
metrics, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
if len(metrics) <= 0 {
t.Log("Expected some metrics to be returned but had none...")
t.Fail()
}
for _, metric := range metrics {
found := false
for key := range metric.Labels {
for _, e := range requiredLabels {
if key == e {
found = true
break
}
}
if found {
break
}
}
if !found {
t.Logf("Metric '%s' with labels %s does not have one or more required labels - %s", metric.Key, metric.Labels, requiredLabels)
t.Fail()
}
}
// Stop the container cleanly
stopContainer(t, cli, id)
}
func TestRapidFirePrometheus(t *testing.T) {
t.Parallel()
cli, err := client.NewEnvClient()
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
Env: []string{
"LICENSE=accept",
"MQ_QMGR_NAME=qm1",
"MQ_ENABLE_METRICS=true",
},
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
hostname := getIPAddress(t, cli, id)
port := DEFAULT_METRIC_PORT
// Now the container is ready we prod the prometheus endpoint until it's up.
waitForMetricReady(hostname, port)
// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
_, err = getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
// Rapid fire it then check we're still happy
for i := 0; i < 30; i++ {
_, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
time.Sleep(1 * time.Second)
}
time.Sleep(11 * time.Second)
metrics, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
if len(metrics) <= 0 {
t.Log("Expected some metrics to be returned but had none...")
t.Fail()
}
// Stop the container cleanly
stopContainer(t, cli, id)
}
func TestSlowPrometheus(t *testing.T) {
t.Parallel()
cli, err := client.NewEnvClient()
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
Env: []string{
"LICENSE=accept",
"MQ_QMGR_NAME=qm1",
"MQ_ENABLE_METRICS=true",
},
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
hostname := getIPAddress(t, cli, id)
port := DEFAULT_METRIC_PORT
// Now the container is ready we prod the prometheus endpoint until it's up.
waitForMetricReady(hostname, port)
// Call once as mq_prometheus 'ignores' the first call and will not return any metrics
_, err = getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
// Send a request twice over a long period and check we're still happy
for i := 0; i < 2; i++ {
time.Sleep(30 * time.Second)
metrics, err := getMetricsFromEndpoint(hostname, port)
if err != nil {
t.Logf("Failed to call metric endpoint - %v", err)
t.FailNow()
}
if len(metrics) <= 0 {
t.Log("Expected some metrics to be returned but had none...")
t.Fail()
}
}
// Stop the container cleanly
stopContainer(t, cli, id)
}

View File

@@ -0,0 +1,160 @@
/*
© Copyright IBM Corporation 2018
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bufio"
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
)
type MQMETRIC struct {
Key string
Value string
Labels map[string]string
}
const DEFAULT_METRIC_URL = "/metrics"
const DEFAULT_METRIC_PORT = 9157
const DEFAULT_MQ_NAMESPACE = "ibmmq"
func getMetricsFromEndpoint(host string, port int) ([]MQMETRIC, error) {
returned := []MQMETRIC{}
if host == "" {
return returned, fmt.Errorf("Test Error - Host was nil")
}
if port <= 0 {
return returned, fmt.Errorf("Test Error - port was not above 0")
}
urlToUse := fmt.Sprintf("http://%s:%d%s", host, port, DEFAULT_METRIC_URL)
resp, err := http.Get(urlToUse)
if err != nil {
return returned, err
}
defer resp.Body.Close()
metricsRaw, err := ioutil.ReadAll(resp.Body)
if err != nil {
return returned, err
}
return convertRawMetricToMap(string(metricsRaw))
}
// Also filters out all non "ibmmq" metrics
func convertRawMetricToMap(input string) ([]MQMETRIC, error) {
returnList := []MQMETRIC{}
if input == "" {
return returnList, fmt.Errorf("Test Error - Raw metric output was nil")
}
scanner := bufio.NewScanner(strings.NewReader(input))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "#") {
// Comment line of HELP or TYPE. Ignore
continue
}
if !strings.HasPrefix(line, DEFAULT_MQ_NAMESPACE) {
// Not an ibmmq_ metric. Ignore
continue
}
//It's an IBM MQ metric!
key, value, labelMap, err := convertMetricLineToMetric(line)
if err != nil {
return returnList, fmt.Errorf("ibmmq_ metric could not be deciphered - %v", err)
}
toAdd := MQMETRIC{
Key: key,
Value: value,
Labels: labelMap,
}
returnList = append(returnList, toAdd)
}
return returnList, nil
}
func convertMetricLineToMetric(input string) (string, string, map[string]string, error) {
// Lines are in the form "<key>{<labels>}<value>" or "<key> <value>"
// Get the key and value while skipping the label
var key, value string
labelMap := make(map[string]string)
if strings.Contains(input, "{") {
// Get key
splitted := strings.Split(input, "{")
if len(splitted) != 2 {
return "", "", labelMap, fmt.Errorf("Could not split by { Expected 2 but got %d - %s", len(splitted), input)
}
key = strings.TrimSpace(splitted[0])
// Get value
splitted = strings.Split(splitted[1], "}")
if len(splitted) != 2 {
return "", "", labelMap, fmt.Errorf("Could not split by } Expected 2 but got %d - %s", len(splitted), input)
}
value = strings.TrimSpace(splitted[1])
// Get labels
allLabels := strings.Split(splitted[0], ",")
for _, e := range allLabels {
labelPair := strings.Split(e, "=")
if len(labelPair) != 2 {
return "", "", labelMap, fmt.Errorf("Could not split label by '=' Expected 2 but got %d - %s", len(labelPair), e)
}
lkey := strings.TrimSpace(labelPair[0])
lvalue := strings.TrimSpace(labelPair[1])
lvalue = strings.Trim(lvalue, "\"")
labelMap[lkey] = lvalue
}
} else {
splitted := strings.Split(input, " ")
if len(splitted) != 2 {
return "", "", labelMap, fmt.Errorf("Could not split by ' ' Expected 2 but got %d - %s", len(splitted), input)
}
key = strings.TrimSpace(splitted[0])
value = strings.TrimSpace(splitted[1])
}
return key, value, labelMap, nil
}
func waitForMetricReady(host string, port int) error {
if host == "" {
return fmt.Errorf("Test Error - Host was nil")
}
if port <= 0 {
return fmt.Errorf("Test Error - port was not above 0")
}
timeout := 12 // 12 * 5 = 1 minute
for i := 0; i < timeout; i++ {
urlToUse := fmt.Sprintf("http://%s:%d", host, port)
resp, err := http.Get(urlToUse)
if err == nil {
resp.Body.Close()
return nil
}
time.Sleep(time.Second * 5)
}
return fmt.Errorf("Metric endpoint failed to startup in timely manner")
}

View File

@@ -35,6 +35,7 @@ import (
"bufio"
"fmt"
"os"
"regexp"
"strings"
"github.com/ibm-messaging/mq-golang/ibmmq"
@@ -263,7 +264,7 @@ func discoverElements(ty *MonType) error {
}
}
elem.MetricName = formatDescriptionElem(elem)
elem.MetricName = formatDescription(elem)
ty.Elements[elementIndex] = elem
}
}
@@ -631,59 +632,49 @@ bytes etc), and organisation of the elements of the name (units last)
While we can't change the MQ-generated descriptions for its statistics,
we can reformat most of them heuristically here.
*/
func formatDescriptionElem(elem *MonElement) string {
s := formatDescription(elem.Description)
unit := ""
switch elem.Datatype {
case ibmmq.MQIAMO_MONITOR_MICROSEC:
// Although the qmgr captures in us, we convert when
// pushing out to the backend, so this label needs to match
unit = "_seconds"
}
s += unit
return s
}
func formatDescription(baseName string) string {
s := baseName
func formatDescription(elem *MonElement) string {
s := elem.Description
s = strings.Replace(s, " ", "_", -1)
s = strings.Replace(s, "/", "_", -1)
s = strings.Replace(s, "-", "_", -1)
/* common pattern is "xxx - yyy" leading to 3 ugly adjacent underscores */
s = strings.Replace(s, "___", "_", -1)
s = strings.Replace(s, "__", "_", -1)
/* Make sure we don't have multiple underscores */
multiunder := regexp.MustCompile("__*")
s = multiunder.ReplaceAllLiteralString(s, "_")
/* make it all lowercase. Not essential, but looks better */
s = strings.ToLower(s)
// Do not use _count
/* Remove all cases of bytes, seconds, count or percentage (we add them back in later) */
s = strings.Replace(s, "_count", "", -1)
s = strings.Replace(s, "_bytes", "", -1)
s = strings.Replace(s, "_byte", "", -1)
s = strings.Replace(s, "_seconds", "", -1)
s = strings.Replace(s, "_second", "", -1)
s = strings.Replace(s, "_percentage", "", -1)
// Switch round a couple of specific names
s = strings.Replace(s, "bytes_written", "written_bytes", -1)
s = strings.Replace(s, "bytes_max", "max_bytes", -1)
s = strings.Replace(s, "bytes_in_use", "in_use_bytes", -1)
s = strings.Replace(s, "messages_expired", "expired_messages", -1)
if strings.HasSuffix(s, "free_space") {
// Add the unit at end
switch elem.Datatype {
case ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS:
s = s + "_percentage"
s = strings.Replace(s, "__", "_", -1)
}
// Make "byte", "file" and "message" units plural
if strings.HasSuffix(s, "byte") ||
strings.HasSuffix(s, "message") ||
strings.HasSuffix(s, "file") {
s = s + "s"
}
// Move % to the end
if strings.Contains(s, "_percentage_") {
s = strings.Replace(s, "_percentage_", "_", -1)
s += "_percentage"
case ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB:
s = s + "_bytes"
case ibmmq.MQIAMO_MONITOR_MICROSEC:
s = s + "_seconds"
default:
if strings.Contains(s, "_total") {
/* If we specify it is a total in description put that at the end */
s = strings.Replace(s, "_total", "", -1)
s = s + "_total"
} else if strings.Contains(s, "log_") {
/* Weird case where the log datatype is not MB or GB but should be bytes */
s = s + "_bytes"
} else {
s = s + "_count"
}
}
return s

View File

@@ -90,36 +90,61 @@ func TestReadPatterns(t *testing.T) {
}
}
func TestFormatDescription(t *testing.T) {
give := [...]string{"hello", "no space", "no/slash", "no-dash", "single___underscore", "single__underscore", "ALLLOWER", "no_count", "this_bytes_written_switch", "this_bytes_max_switch", "this_bytes_in_use_switch", "this messages_expired_switch", "add_free_space", "suffix_byte", "suffix_message", "suffix_file", "this_percentage_move"}
expected := [...]string{"hello", "no_space", "no_slash", "no_dash", "single_underscore", "single_underscore", "alllower", "no", "this_written_bytes_switch", "this_max_bytes_switch", "this_in_use_bytes_switch", "this_expired_messages_switch", "add_free_space_percentage", "suffix_bytes", "suffix_messages", "suffix_files", "this_move_percentage"}
give := [...]string{"hello", "no space", "no/slash", "no-dash", "single___underscore", "single__underscore__multiplace", "ALLLOWER", "this_bytes_written_switch", "this_byte_max_switch", "this_seconds_in_use_switch", "this messages_expired_switch", "this_seconds_max_switch", "this_count_max_switch", "this_percentage_max_switch"}
expected := [...]string{"hello_count", "no_space_count", "no_slash_count", "no_dash_count", "single_underscore_count", "single_underscore_multiplace_count", "alllower_count", "this_written_switch_count", "this_max_switch_count", "this_in_use_switch_count", "this_expired_messages_switch_count", "this_max_switch_count", "this_max_switch_count", "this_max_switch_count"}
for i, e := range give {
back := formatDescription(e)
elem := MonElement{
Description: e,
}
back := formatDescription(&elem)
if back != expected[i] {
t.Logf("Gave %s. Expected: %s, Got: %s", e, expected[i], back)
t.Fail()
}
}
}
func TestFormatDescriptionElem(t *testing.T) {
test := MonElement{}
test.Description = "THIS-should__be/formatted_count"
expected := "this_should_be_formatted"
back := formatDescriptionElem(&test)
if back != expected {
t.Logf("Gave %s. Expected: %s, Got: %s", test.Description, expected, back)
func TestSuffixes(t *testing.T) {
baseDescription := "test_suffix"
types := [...]int32{ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB, ibmmq.MQIAMO_MONITOR_MICROSEC, ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS, 0}
expected := [...]string{baseDescription + "_bytes", baseDescription + "_bytes", baseDescription + "_seconds", baseDescription + "_percentage", baseDescription + "_percentage", baseDescription + "_count"}
for i, ty := range types {
elem := MonElement{
Description: baseDescription,
Datatype: ty,
}
back := formatDescription(&elem)
if back != expected[i] {
t.Logf("Gave %s/%d Expected: %s, Got: %s", baseDescription, ty, expected[i], back)
t.Fail()
}
}
// special case log_bytes
elem := MonElement{
Description: "log_test_suffix",
Datatype: 0,
}
back := formatDescription(&elem)
if back != "log_test_suffix_bytes" {
t.Logf("Gave log_test_suffix/0 Expected: %s, Got: %s", "log_test_suffix_bytes", back)
t.Fail()
}
test.Datatype = ibmmq.MQIAMO_MONITOR_MICROSEC
expected = "this_should_be_formatted_seconds"
back = formatDescriptionElem(&test)
if back != expected {
t.Logf("Gave %s. Expected: %s, Got: %s", test.Description, expected, back)
// special case log_total
elem = MonElement{
Description: "log_total_suffix",
Datatype: 0,
}
back = formatDescription(&elem)
if back != "log_suffix_total" {
t.Logf("Gave log_total_suffix/0 Expected: %s, Got: %s", "log_suffix_total", back)
t.Fail()
}
}
func TestParsePCFResponse(t *testing.T) {
cfh := ibmmq.NewMQCFH()
cfh.Type = ibmmq.MQCFT_RESPONSE