Initial delivery of metrics code (#81)

* Initial delivery of metrics code

* Fix build issues

* Fix build issue with go vet

internal/metrics/exporter.go

@@ -0,0 +1,107 @@
+/*
+© Copyright IBM Corporation 2018
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metrics contains code to provide metrics for the queue manager
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace    = "ibmmq"
+	qmgrPrefix   = "qmgr"
+	qmgrLabel    = "qmgr"
+	objectPrefix = "object"
+	objectLabel  = "object"
+)
+
+type exporter struct {
+	qmName   string
+	gaugeMap map[string]*prometheus.GaugeVec
+}
+
+func newExporter(qmName string) *exporter {
+	return &exporter{
+		qmName:   qmName,
+		gaugeMap: make(map[string]*prometheus.GaugeVec),
+	}
+}
+
+// Describe provides details of all available metrics
+func (e *exporter) Describe(ch chan<- *prometheus.Desc) {
+
+	requestChannel <- false
+	response := <-responseChannel
+
+	for key, metric := range response {
+
+		// Allocate a Prometheus Gauge for each available metric
+		gaugeVec := createGaugeVec(metric.name, metric.description, metric.objectType)
+		e.gaugeMap[key] = gaugeVec
+
+		// Describe metric
+		gaugeVec.Describe(ch)
+	}
+}
+
+// Collect is called at regular intervals to provide the current metric data
+func (e *exporter) Collect(ch chan<- prometheus.Metric) {
+
+	requestChannel <- true
+	response := <-responseChannel
+
+	for key, metric := range response {
+
+		// Reset Prometheus Gauge
+		gaugeVec := e.gaugeMap[key]
+		gaugeVec.Reset()
+
+		// Populate Prometheus Gauge with metric values
+		for label, value := range metric.values {
+			if label == qmgrLabelValue {
+				gaugeVec.WithLabelValues(e.qmName).Set(value)
+			} else {
+				gaugeVec.WithLabelValues(label, e.qmName).Set(value)
+			}
+		}
+
+		// Collect metric
+		gaugeVec.Collect(ch)
+	}
+}
+
+// createGaugeVec returns a Prometheus GaugeVec populated with metric details
+func createGaugeVec(name, description string, objectType bool) *prometheus.GaugeVec {
+
+	prefix := qmgrPrefix
+	labels := []string{qmgrLabel}
+
+	if objectType {
+		prefix = objectPrefix
+		labels = []string{objectLabel, qmgrLabel}
+	}
+
+	gaugeVec := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      prefix + "_" + name,
+			Help:      description,
+		},
+		labels,
+	)
+	return gaugeVec
+}

internal/metrics/exporter_test.go

@@ -0,0 +1,54 @@
+/*
+© Copyright IBM Corporation 2018
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestCreateGaugeVec(t *testing.T) {
+
+	ch := make(chan *prometheus.Desc)
+	gaugeVec := createGaugeVec("MetricName", "MetricDescription", false)
+	go func() {
+		gaugeVec.Describe(ch)
+	}()
+	description := <-ch
+
+	expected := "Desc{fqName: \"ibmmq_qmgr_MetricName\", help: \"MetricDescription\", constLabels: {}, variableLabels: [qmgr]}"
+	actual := description.String()
+	if actual != expected {
+		t.Errorf("Expected value=%s; actual %s", expected, actual)
+	}
+}
+
+func TestCreateGaugeVec_ObjectLabel(t *testing.T) {
+
+	ch := make(chan *prometheus.Desc)
+	gaugeVec := createGaugeVec("MetricName", "MetricDescription", true)
+	go func() {
+		gaugeVec.Describe(ch)
+	}()
+	description := <-ch
+
+	expected := "Desc{fqName: \"ibmmq_object_MetricName\", help: \"MetricDescription\", constLabels: {}, variableLabels: [object qmgr]}"
+	actual := description.String()
+	if actual != expected {
+		t.Errorf("Expected value=%s; actual %s", expected, actual)
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,97 @@
+/*
+© Copyright IBM Corporation 2018
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metrics contains code to provide metrics for the queue manager
+package metrics
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/ibm-messaging/mq-container/internal/logger"
+	"github.com/ibm-messaging/mq-golang/mqmetric"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	defaultPort = "9157"
+	retryCount  = 3
+	retryWait   = 5
+)
+
+// GatherMetrics gathers metrics for the queue manager
+func GatherMetrics(qmName string, log *logger.Logger) {
+
+	for i := 0; i <= retryCount; i++ {
+		err := startMetricsGathering(qmName, log)
+		if err != nil {
+			log.Errorf("Metrics Error: %s", err.Error())
+		}
+		if i != retryCount {
+			log.Printf("Waiting %d seconds before retrying metrics gathering", retryWait)
+			time.Sleep(retryWait * time.Second)
+		} else {
+			log.Println("Unable to gather metrics - metrics are now disabled")
+		}
+	}
+}
+
+// startMetricsGathering starts gathering metrics for the queue manager
+func startMetricsGathering(qmName string, log *logger.Logger) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("Metrics Error: %v", r)
+		}
+	}()
+
+	log.Println("Starting metrics gathering")
+
+	// Set connection configuration
+	var connConfig mqmetric.ConnectionConfig
+	connConfig.ClientMode = false
+	connConfig.UserId = ""
+	connConfig.Password = ""
+
+	// Connect to the queue manager - open the command and dynamic reply queues
+	err := mqmetric.InitConnectionStats(qmName, "SYSTEM.DEFAULT.MODEL.QUEUE", "", &connConfig)
+	if err != nil {
+		return fmt.Errorf("Failed to connect to queue manager %s: %v", qmName, err)
+	}
+	defer mqmetric.EndConnection()
+
+	// Discover available metrics for the queue manager and subscribe to them
+	err = mqmetric.DiscoverAndSubscribe("", true, "")
+	if err != nil {
+		return fmt.Errorf("Failed to discover and subscribe to metrics: %v", err)
+	}
+
+	// Start processing metrics
+	go processMetrics(log)
+
+	// Register metrics
+	prometheus.MustRegister(newExporter(qmName))
+
+	// Setup HTTP server to handle requests from Prometheus
+	http.Handle("/metrics", prometheus.Handler())
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte("Status: METRICS ACTIVE"))
+	})
+	err = http.ListenAndServe(":"+defaultPort, nil)
+	return fmt.Errorf("Failed to handle metrics request: %v", err)
+}

internal/metrics/update.go

@@ -0,0 +1,119 @@
+/*
+© Copyright IBM Corporation 2018
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metrics contains code to provide metrics for the queue manager
+package metrics
+
+import (
+	"strings"
+	"time"
+
+	"github.com/ibm-messaging/mq-container/internal/logger"
+	"github.com/ibm-messaging/mq-golang/mqmetric"
+)
+
+const (
+	qmgrLabelValue = mqmetric.QMgrMapKey
+	requestTimeout = 10
+)
+
+var (
+	requestChannel  = make(chan bool)
+	responseChannel = make(chan map[string]*metricData)
+)
+
+type metricData struct {
+	name        string
+	description string
+	objectType  bool
+	values      map[string]float64
+}
+
+// processMetrics processes publications of metric data and handles describe/collect requests
+func processMetrics(log *logger.Logger) {
+
+	// Initialise metrics
+	metrics := initialiseMetrics()
+
+	for {
+		// Process publications of metric data
+		mqmetric.ProcessPublications()
+
+		// Handle describe/collect requests
+		select {
+		case collect := <-requestChannel:
+			if collect {
+				updateMetrics(metrics)
+			}
+			responseChannel <- metrics
+		case <-time.After(requestTimeout * time.Second):
+			log.Debugf("Metrics: No requests received within timeout period (%d seconds)", requestTimeout)
+		}
+	}
+}
+
+// initialiseMetrics sets initial details for all available metrics
+func initialiseMetrics() map[string]*metricData {
+
+	metrics := make(map[string]*metricData)
+
+	for _, metricClass := range mqmetric.Metrics.Classes {
+		for _, metricType := range metricClass.Types {
+			if !strings.Contains(metricType.ObjectTopic, "%s") {
+				for _, metricElement := range metricType.Elements {
+					metric := metricData{
+						name:        metricElement.MetricName,
+						description: metricElement.Description,
+					}
+					key := makeKey(metricElement)
+					metrics[key] = &metric
+				}
+			}
+		}
+	}
+	return metrics
+}
+
+// updateMetrics updates values for all available metrics
+func updateMetrics(metrics map[string]*metricData) {
+
+	for _, metricClass := range mqmetric.Metrics.Classes {
+		for _, metricType := range metricClass.Types {
+			if !strings.Contains(metricType.ObjectTopic, "%s") {
+				for _, metricElement := range metricType.Elements {
+
+					// Clear existing metric values
+					metric := metrics[makeKey(metricElement)]
+					metric.values = make(map[string]float64)
+
+					// Update metric with cached values of publication data
+					for label, value := range metricElement.Values {
+						normalisedValue := mqmetric.Normalise(metricElement, label, value)
+						metric.values[label] = normalisedValue
+					}
+
+					// Reset cached values of publication data for this metric
+					metricElement.Values = make(map[string]int64)
+				}
+			}
+		}
+	}
+}
+
+// makeKey builds a unique key for each metric
+func makeKey(metricElement *mqmetric.MonElement) string {
+	return metricElement.Parent.Parent.Name + "/" + metricElement.Parent.Name + "/" + metricElement.MetricName
+}

internal/metrics/update_test.go

@@ -0,0 +1,112 @@
+/*
+© Copyright IBM Corporation 2018
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"testing"
+
+	"github.com/ibm-messaging/mq-golang/mqmetric"
+)
+
+func TestInitialiseMetrics(t *testing.T) {
+
+	teardownTestCase := setupTestCase()
+	defer teardownTestCase()
+
+	metrics := initialiseMetrics()
+	metric, ok := metrics["ClassName/Type1Name/Element1Name"]
+
+	if !ok {
+		t.Error("Expected metric not found in map")
+	} else {
+		if metric.name != "Element1Name" {
+			t.Errorf("Expected name=%s; actual %s", "Element1Name", metric.name)
+		}
+		if metric.description != "Element1Description" {
+			t.Errorf("Expected description=%s; actual %s", "Element1Description", metric.description)
+		}
+		if metric.objectType != false {
+			t.Errorf("Expected objectType=%v; actual %v", false, metric.objectType)
+		}
+		if len(metric.values) != 0 {
+			t.Errorf("Expected values-size=%d; actual %d", 0, len(metric.values))
+		}
+	}
+	_, ok = metrics["ClassName/Type2Name/Element2Name"]
+	if ok {
+		t.Errorf("Unexpected metric found in map, %%s object topics should be ignored")
+	}
+
+	if len(metrics) != 1 {
+		t.Errorf("Map contains unexpected metrics, map size=%d", len(metrics))
+	}
+}
+
+func TestMakeKey(t *testing.T) {
+
+	teardownTestCase := setupTestCase()
+	defer teardownTestCase()
+
+	expected := "ClassName/Type1Name/Element1Name"
+	actual := makeKey(mqmetric.Metrics.Classes[0].Types[0].Elements[0])
+	if actual != expected {
+		t.Errorf("Expected value=%s; actual %s", expected, actual)
+	}
+}
+
+func setupTestCase() func() {
+	populateTestMetrics()
+	return func() {
+		cleanTestMetrics()
+	}
+}
+
+func populateTestMetrics() {
+
+	metricClass := new(mqmetric.MonClass)
+	metricType1 := new(mqmetric.MonType)
+	metricType2 := new(mqmetric.MonType)
+	metricElement1 := new(mqmetric.MonElement)
+	metricElement2 := new(mqmetric.MonElement)
+
+	metricClass.Name = "ClassName"
+	metricType1.Name = "Type1Name"
+	metricType2.Name = "Type2Name"
+	metricElement1.MetricName = "Element1Name"
+	metricElement1.Description = "Element1Description"
+	metricElement2.MetricName = "Element2Name"
+	metricElement2.Description = "Element2Description"
+	metricType1.ObjectTopic = "ObjectTopic"
+	metricType2.ObjectTopic = "%s"
+	metricElement1.Parent = metricType1
+	metricElement2.Parent = metricType2
+	metricType1.Parent = metricClass
+	metricType2.Parent = metricClass
+
+	metricType1.Elements = make(map[int]*mqmetric.MonElement)
+	metricType2.Elements = make(map[int]*mqmetric.MonElement)
+	metricType1.Elements[0] = metricElement1
+	metricType2.Elements[0] = metricElement2
+	metricClass.Types = make(map[int]*mqmetric.MonType)
+	metricClass.Types[0] = metricType1
+	metricClass.Types[1] = metricType2
+	mqmetric.Metrics.Classes = make(map[int]*mqmetric.MonClass)
+	mqmetric.Metrics.Classes[0] = metricClass
+}
+
+func cleanTestMetrics() {
+	mqmetric.Metrics.Classes = make(map[int]*mqmetric.MonClass)
+}