Improve prometheus error handling (#108)

internal/metrics/exporter.go

@@ -18,6 +18,7 @@ limitations under the License.
 package metrics
 
 import (
+	"github.com/ibm-messaging/mq-container/internal/logger"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -33,13 +34,15 @@ type exporter struct {
 	qmName       string
 	gaugeMap     map[string]*prometheus.GaugeVec
 	firstCollect bool
+	log          *logger.Logger
 }
 
-func newExporter(qmName string) *exporter {
+func newExporter(qmName string, log *logger.Logger) *exporter {
 	return &exporter{
 		qmName:       qmName,
 		gaugeMap:     make(map[string]*prometheus.GaugeVec),
 		firstCollect: true,
+		log:          log,
 	}
 }
 
@@ -76,10 +79,18 @@ func (e *exporter) Collect(ch chan<- prometheus.Metric) {
 		// - Skip on first collect to avoid build-up of accumulated values
 		if !e.firstCollect {
 			for label, value := range metric.values {
+				var err error
+				var gauge prometheus.Gauge
+
 				if label == qmgrLabelValue {
-					gaugeVec.WithLabelValues(e.qmName).Set(value)
+					gauge, err = gaugeVec.GetMetricWithLabelValues(e.qmName)
 				} else {
-					gaugeVec.WithLabelValues(label, e.qmName).Set(value)
+					gauge, err = gaugeVec.GetMetricWithLabelValues(label, e.qmName)
+				}
+				if err == nil {
+					gauge.Set(value)
+				} else {
+					e.log.Errorf("Metrics Error: %s", err.Error())
 				}
 			}
 		}

internal/metrics/exporter_test.go

@@ -27,10 +27,11 @@ func TestDescribe(t *testing.T) {
 
 	teardownTestCase := setupTestCase(false)
 	defer teardownTestCase()
+	log := getTestLogger()
 
 	ch := make(chan *prometheus.Desc)
 	go func() {
-		exporter := newExporter("qmName")
+		exporter := newExporter("qmName", log)
 		exporter.Describe(ch)
 	}()
 
@@ -39,7 +40,7 @@ func TestDescribe(t *testing.T) {
 		t.Errorf("Received unexpected collect request")
 	}
 
-	metrics, _ := initialiseMetrics(getTestLogger())
+	metrics, _ := initialiseMetrics(log)
 	responseChannel <- metrics
 
 	select {
@@ -58,8 +59,9 @@ func TestCollect(t *testing.T) {
 
 	teardownTestCase := setupTestCase(false)
 	defer teardownTestCase()
+	log := getTestLogger()
 
-	exporter := newExporter("qmName")
+	exporter := newExporter("qmName", log)
 	exporter.gaugeMap["ClassName/Type1Name/Element1Name"] = createGaugeVec("Element1Name", "Element1Description", false)
 
 	for i := 1; i <= 3; i++ {
@@ -76,7 +78,7 @@ func TestCollect(t *testing.T) {
 		}
 
 		populateTestMetrics(i, false)
-		metrics, _ := initialiseMetrics(getTestLogger())
+		metrics, _ := initialiseMetrics(log)
 		updateMetrics(metrics)
 		responseChannel <- metrics
 

internal/metrics/metrics.go

@@ -69,7 +69,12 @@ func startMetricsGathering(qmName string, log *logger.Logger) error {
 	wg.Wait()
 
 	// Register metrics
-	prometheus.MustRegister(newExporter(qmName))
+	exporter := newExporter(qmName, log)
+	err := prometheus.Register(exporter)
+	if err != nil {
+		return fmt.Errorf("Failed to register metrics: %v", err)
+	}
+	defer prometheus.Unregister(exporter)
 
 	// Setup HTTP server to handle requests from Prometheus
 	http.Handle("/metrics", prometheus.Handler())
@@ -78,7 +83,7 @@ func startMetricsGathering(qmName string, log *logger.Logger) error {
 		w.Write([]byte("Status: METRICS ACTIVE"))
 	})
 
-	err := http.ListenAndServe(":"+defaultPort, nil)
+	err = http.ListenAndServe(":"+defaultPort, nil)
 	if err != nil {
 		return fmt.Errorf("Failed to handle metrics request: %v", err)
 	}