Enhanced console logging

Introduce new environment variables:
  * MQ_LOGGING_CONSOLE_SOURCE
  * MQ_LOGGING_CONSOLE_FORMAT
  * MQ_LOGGING_CONSOLE_EXCLUDE_ID

Authored-by: Avinash Ganesh <avinash.v.g@in.ibm.com> and BHAVYA K R <bhavkris@in.ibm.com>

Dockerfile-server

@@ -138,6 +138,8 @@ EXPOSE 1414 9157 9443
 ENV MQ_OVERRIDE_DATA_PATH=/mnt/mqm/data MQ_OVERRIDE_INSTALLATION_NAME=Installation1 MQ_USER_NAME="mqm" PATH="${PATH}:/opt/mqm/bin"
 ENV MQ_GRACE_PERIOD=30
 ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic
+ENV MQ_LOGGING_CONSOLE_EXCLUDE_ID=AMQ5041I,AMQ5052I,AMQ5051I,AMQ5037I,AMQ5975I
+ENV WLP_LOGGING_MESSAGE_FORMAT=json
 # We can run as any UID
 USER 1001
 ENV MQ_CONNAUTH_USE_HTP=false

cmd/runmqdevserver/main.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2018, 2021
+© Copyright IBM Corporation 2018, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 	"syscall"
 
 	"github.com/ibm-messaging/mq-container/internal/htpasswd"
@@ -30,7 +31,20 @@ import (
 var log *logger.Logger
 
 func getLogFormat() string {
-	return os.Getenv("LOG_FORMAT")
+	logFormat := strings.ToLower(strings.TrimSpace(os.Getenv("MQ_LOGGING_CONSOLE_FORMAT")))
+	//old-style env var is used.
+	if logFormat == "" {
+		logFormat = strings.ToLower(strings.TrimSpace(os.Getenv("LOG_FORMAT")))
+	}
+
+	if logFormat != "" && (logFormat == "basic" || logFormat == "json") {
+		return logFormat
+	} else {
+		//this is the case where value is either empty string or set to something other than "basic"/"json"
+		logFormat = "basic"
+	}
+
+	return logFormat
 }
 
 func getDebug() bool {

cmd/runmqserver/logging.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2022
+© Copyright IBM Corporation 2017, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -58,7 +58,20 @@ func logTermination(args ...interface{}) {
 }
 
 func getLogFormat() string {
-	return os.Getenv("LOG_FORMAT")
+	logFormat := strings.ToLower(strings.TrimSpace(os.Getenv("MQ_LOGGING_CONSOLE_FORMAT")))
+	//old-style env var is used.
+	if logFormat == "" {
+		logFormat = strings.ToLower(strings.TrimSpace(os.Getenv("LOG_FORMAT")))
+	}
+
+	if logFormat != "" && (logFormat == "basic" || logFormat == "json") {
+		return logFormat
+	} else {
+		//this is the case where value is either empty string or set to something other than "basic"/"json"
+		logFormat = "basic"
+	}
+
+	return logFormat
 }
 
 // formatBasic formats a log message parsed from JSON, as "basic" text
@@ -81,6 +94,92 @@ func formatBasic(obj map[string]interface{}) string {
 	}
 	// Convert time zone information from some logs (e.g. Liberty) for consistency
 	obj["ibm_datetime"] = strings.Replace(obj["ibm_datetime"].(string), "+0000", "Z", 1)
+
+	if obj["type"] != nil && (obj["type"] == "liberty_message" || obj["type"] == "liberty_trace") {
+		timeStamp := obj["ibm_datetime"]
+		threadID := ""
+		srtModuleName := ""
+		logLevel := ""
+		ibmClassName := ""
+		srtIbmClassName := ""
+		ibmMethodName := ""
+		message := ""
+
+		if obj["loglevel"] != nil {
+			//threadID is captured below
+			if obj["ibm_threadId"] != nil {
+				threadID = obj["ibm_threadId"].(string)
+			}
+
+			//logLevel character to be mirrored in console web server logging is decided below
+			logLevelTmp := obj["loglevel"].(string)
+			switch logLevelTmp {
+			case "AUDIT":
+				logLevel = "A"
+			case "INFO":
+				logLevel = "I"
+			case "EVENT":
+				logLevel = "1"
+			case "ENTRY":
+				logLevel = ">"
+			case "EXIT":
+				logLevel = "<"
+			case "FINE":
+				logLevel = "1"
+			case "FINER":
+				logLevel = "2"
+			case "FINEST":
+				logLevel = "3"
+			default:
+				logLevel = string(logLevelTmp[0])
+			}
+
+			//This is a 13 characters string present in extracted out of module node
+			if obj["module"] != nil {
+				srtModuleNameArr := strings.Split(obj["module"].(string), ".")
+				arrLen := len(srtModuleNameArr)
+				srtModuleName = srtModuleNameArr[arrLen-1]
+				if len(srtModuleName) > 13 {
+					srtModuleName = srtModuleName[0:13]
+				}
+			}
+			if obj["ibm_className"] != nil {
+				ibmClassName = obj["ibm_className"].(string)
+
+				//A 13 character string is extracted from class name. This is required for FINE, FINER & FINEST log lines
+				ibmClassNameArr := strings.Split(ibmClassName, ".")
+				arrLen := len(ibmClassNameArr)
+				srtIbmClassName = ibmClassNameArr[arrLen-1]
+				if len(srtModuleName) > 13 {
+					srtIbmClassName = srtIbmClassName[0:13]
+				}
+			}
+			if obj["ibm_methodName"] != nil {
+				ibmMethodName = obj["ibm_methodName"].(string)
+			}
+			if obj["message"] != nil {
+				message = obj["message"].(string)
+			}
+
+			//For AUDIT & INFO logging
+			if logLevel == "A" || logLevel == "I" {
+				return fmt.Sprintf("%s %s %-13s %s %s %s %s\n", timeStamp, threadID, srtModuleName, logLevel, ibmClassName, ibmMethodName, message)
+			}
+			//For EVENT logLevel
+			if logLevelTmp == "EVENT" {
+				return fmt.Sprintf("%s %s %-13s %s %s\n", timeStamp, threadID, srtModuleName, logLevel, message)
+			}
+			//For ENTRY & EXIT
+			if logLevel == ">" || logLevel == "<" {
+				return fmt.Sprintf("%s %s %-13s %s %s %s\n", timeStamp, threadID, srtModuleName, logLevel, ibmMethodName, message)
+			}
+			//For deeper log levels
+			if logLevelTmp == "FINE" || logLevel == "2" || logLevel == "3" {
+				return fmt.Sprintf("%s %s %-13s %s %s %s %s\n", timeStamp, threadID, srtIbmClassName, logLevel, ibmClassName, ibmMethodName, message)
+			}
+
+		}
+	}
 	return fmt.Sprintf("%s %s\n", obj["ibm_datetime"], obj["message"])
 }
 
@@ -131,6 +230,11 @@ func configureLogger(name string) (mirrorFunc, error) {
 			return nil, err
 		}
 		return func(msg string, isQMLog bool) bool {
+			arrLoggingConsoleExcludeIds := strings.Split(strings.ToUpper(os.Getenv("MQ_LOGGING_CONSOLE_EXCLUDE_ID")), ",")
+			if isExcludedMsgIdPresent(msg, arrLoggingConsoleExcludeIds) {
+				//If excluded id is present do not mirror it, return back
+				return false
+			}
 			// Check if the message is JSON
 			if len(msg) > 0 && msg[0] == '{' {
 				obj, err := processLogMessage(msg)
@@ -155,6 +259,11 @@ func configureLogger(name string) (mirrorFunc, error) {
 			return nil, err
 		}
 		return func(msg string, isQMLog bool) bool {
+			arrLoggingConsoleExcludeIds := strings.Split(strings.ToUpper(os.Getenv("MQ_LOGGING_CONSOLE_EXCLUDE_ID")), ",")
+			if isExcludedMsgIdPresent(msg, arrLoggingConsoleExcludeIds) {
+				//If excluded id is present do not mirror it, return back
+				return false
+			}
 			// Check if the message is JSON
 			if len(msg) > 0 && msg[0] == '{' {
 				// Parse the JSON message, and print a simplified version
@@ -197,6 +306,16 @@ func filterQMLogMessage(obj map[string]interface{}) bool {
 	return false
 }
 
+// Function to check if ids provided in MQ_LOGGING_CONSOLE_EXCLUDE_ID are present in given log line or not
+func isExcludedMsgIdPresent(msg string, envExcludeIds []string) bool {
+	for _, id := range envExcludeIds {
+		if id != "" && strings.Contains(msg, strings.TrimSpace(id)) {
+			return true
+		}
+	}
+	return false
+}
+
 func logDiagnostics() {
 	if getDebug() {
 		log.Debug("--- Start Diagnostics ---")
@@ -238,3 +357,68 @@ func logDiagnostics() {
 		log.Debug("---  End Diagnostics  ---")
 	}
 }
+
+// Returns the value of MQ_LOGGING_CONSOLE_SOURCE environment variable
+func getMQLogConsoleSource() string {
+	return strings.ToLower(strings.TrimSpace(os.Getenv("MQ_LOGGING_CONSOLE_SOURCE")))
+
+}
+
+// Function to check if valid values are provided for environment variable MQ_LOGGING_CONSOLE_SOURCE. If not valid, main program throws a warning to console
+func isLogConsoleSourceValid() bool {
+	mqLogSource := getMQLogConsoleSource()
+	retValue := false
+	//If nothing is set, we will mirror all, so valid
+	if mqLogSource == "" {
+		return true
+	}
+
+	logConsoleSource := strings.Split(mqLogSource, ",")
+	//This will find out if the environment variable contains permitted values and is comma separated
+	for _, src := range logConsoleSource {
+		switch strings.TrimSpace(src) {
+		//If it is a permitted value, it is valid. Keep it as true, but dont return it. We may encounter something junk soon
+		case "qmgr", "web", "":
+			retValue = true
+		//If invalid entry arrives in-between/anywhere, just return false, there is no turning back
+		default:
+			return false
+		}
+	}
+
+	return retValue
+}
+
+// To check which all logs have to be mirrored
+func checkLogSourceForMirroring(source string) bool {
+	logsrcs := getMQLogConsoleSource()
+
+	//Nothing set, this is when we mirror all
+	if logsrcs == "" {
+		return true
+	}
+
+	//Split the csv environment value so that we get an accurate comparison instead of a contains() check
+	logSrcArr := strings.Split(logsrcs, ",")
+
+	//Iterate through the array to decide on mirroring
+	for _, arr := range logSrcArr {
+		switch strings.TrimSpace(arr) {
+		case "qmgr":
+			//If value of source is qmgr and it exists in environment variable, mirror qmgr logs
+			if source == "qmgr" {
+				return true
+			}
+		case "web":
+			//If value of source is web and it exists in environment variable, and mirror web logs
+			if source == "web" {
+				//If older environment variable is set make sure to print appropriate message
+				if os.Getenv("MQ_ENABLE_EMBEDDED_WEB_SERVER_LOG") != "" {
+					log.Println("Environment variable MQ_ENABLE_EMBEDDED_WEB_SERVER_LOG has now been replaced. Use MQ_LOGGING_CONSOLE_SOURCE instead.")
+				}
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/runmqserver/logging_test.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2020
+© Copyright IBM Corporation 2020, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 )
@@ -53,3 +54,86 @@ func TestFormatBasic(t *testing.T) {
 		})
 	}
 }
+
+// This test covers for functions isLogConsoleSourceValid() & checkLogSourceForMirroring()
+var mqLogSourcesTests = []struct {
+	testNum     int
+	logsrc      string
+	exptValid   bool
+	exptQmgrSrc bool
+	exptWebSrc  bool
+}{
+	{1, "qmgr,web", true, true, true},
+	{2, "qmgr", true, true, false},
+	{3, "web,qmgr", true, true, true},
+	{4, "web", true, false, true},
+	{5, " ", true, true, true},
+	{6, "QMGR,WEB", true, true, true},
+	{7, "qmgr,     ", true, true, false},
+	{8, "qmgr   ,    web", true, true, true},
+	{9, "qmgr,dummy", false, true, false},
+	{10, "fake,dummy", false, false, false},
+	{11, "qmgr,fake,dummy", false, true, false},
+	{12, "fake,dummy,web", false, false, true},
+	{13, "true", false, false, false},
+	{14, "false", false, false, false},
+	{15, "", true, true, true},
+}
+
+func TestLoggingConsoleSourceInputs(t *testing.T) {
+	for _, mqlogsrctest := range mqLogSourcesTests {
+		err := os.Setenv("MQ_LOGGING_CONSOLE_SOURCE", mqlogsrctest.logsrc)
+		if err != nil {
+			t.Error(err)
+		}
+		isValid := isLogConsoleSourceValid()
+		if isValid != mqlogsrctest.exptValid {
+			t.Errorf("Expected return value from isLogConsoleSourceValid() is %v for MQ_LOGGING_CONSOLE_SOURCE='%v', got %v\n", mqlogsrctest.exptValid, mqlogsrctest.logsrc, isValid)
+		}
+		isLogSrcQmgr := checkLogSourceForMirroring("qmgr")
+		if isLogSrcQmgr != mqlogsrctest.exptQmgrSrc {
+			t.Errorf("Expected return value from checkLogSourceForMirroring() is %v for MQ_LOGGING_CONSOLE_SOURCE='%v', got %v\n", mqlogsrctest.exptQmgrSrc, mqlogsrctest.logsrc, isLogSrcQmgr)
+		}
+		isLogSrcWeb := checkLogSourceForMirroring("web")
+		if isLogSrcWeb != mqlogsrctest.exptWebSrc {
+			t.Errorf("Expected return value from checkLogSourceForMirroring() is %v for MQ_LOGGING_CONSOLE_SOURCE='%v', got %v\n", mqlogsrctest.exptWebSrc, mqlogsrctest.logsrc, isLogSrcWeb)
+		}
+	}
+}
+
+// This test covers for function isExcludedMsgIdPresent()
+var mqExcludeIDTests = []struct {
+	testNum        int
+	exculdeIDsArr  []string
+	expectedRetVal bool
+	logEntry       string
+}{
+	{
+		1,
+		[]string{"AMQ5051I", "AMQ5037I", "AMQ5975I"},
+		true,
+		"{\"ibm_messageId\":\"AMQ5051I\",\"ibm_arithInsert1\":0,\"ibm_arithInsert2\":1,\"message\":\"AMQ5051I: The queue manager task 'AUTOCONFIG' has started.\"}",
+	},
+	{
+		2,
+		[]string{"AMQ5975I", "AMQ5037I"},
+		false,
+		"{\"ibm_messageId\":\"AMQ5051I\",\"ibm_arithInsert1\":0,\"ibm_arithInsert2\":1,\"message\":\"AMQ5051I: The queue manager task 'AUTOCONFIG' has started.\"}",
+	},
+	{
+		3,
+		[]string{""},
+		false,
+		"{\"ibm_messageId\":\"AMQ5051I\",\"ibm_arithInsert1\":0,\"ibm_arithInsert2\":1,\"message\":\"AMQ5051I: The queue manager task 'AUTOCONFIG' has started.\"}",
+	},
+}
+
+func TestIsExcludedMsgIDPresent(t *testing.T) {
+	for _, excludeIDTest := range mqExcludeIDTests {
+		retVal := isExcludedMsgIdPresent(excludeIDTest.logEntry, excludeIDTest.exculdeIDsArr)
+		if retVal != excludeIDTest.expectedRetVal {
+			t.Errorf("%v. Expected return value from isExcludedMsgIdPresent() is %v for MQ_LOGGING_CONSOLE_EXCLUDE_ID='%v', got %v\n",
+				excludeIDTest.testNum, excludeIDTest.expectedRetVal, excludeIDTest.exculdeIDsArr, retVal)
+		}
+	}
+}

cmd/runmqserver/main.go

@@ -160,6 +160,11 @@ func doMain() error {
 		return err
 	}
 
+	//Validate MQ_LOG_CONSOLE_SOURCE variable
+	if !isLogConsoleSourceValid() {
+		log.Println("One or more invalid value is provided for MQ_LOGGING_CONSOLE_SOURCE. Allowed values are 'qmgr' & 'web' in csv format")
+	}
+
 	err = postInit(name, keyLabel, defaultP12Truststore)
 	if err != nil {
 		logTermination(err)
@@ -210,17 +215,24 @@ func doMain() error {
 		log.Debug("Cancel log mirroring")
 		cancelMirror()
 	}()
-	// TODO: Use the error channel
+
+	//For mirroring mq system logs and qm logs, if environment variable is set
+	if checkLogSourceForMirroring("qmgr") {
+		//Mirror MQ system logs
 		_, err = mirrorSystemErrorLogs(ctx, &wg, mf)
 		if err != nil {
 			logTermination(err)
 			return err
 		}
+
+		//Mirror queue manager logs
 		_, err = mirrorQueueManagerErrorLogs(ctx, &wg, name, newQM, mf)
 		if err != nil {
 			logTermination(err)
 			return err
 		}
+	}
+
 	if *devFlag {
 		_, err = mirrorHTPasswdLogs(ctx, &wg, name, newQM, mf)
 		if err != nil {
@@ -228,9 +240,9 @@ func doMain() error {
 			return err
 		}
 	}
-	// Recommended to use this option in conjunction with setting WLP_LOGGING_MESSAGE_FORMAT=JSON
+
-	mirrorWebLog := os.Getenv("MQ_ENABLE_EMBEDDED_WEB_SERVER_LOG")
+	//For mirroring web server logs if source variable is set
-	if mirrorWebLog == "true" || mirrorWebLog == "1" {
+	if checkLogSourceForMirroring("web") {
 		_, err = mirrorWebServerLogs(ctx, &wg, name, newQM, mf)
 		if err != nil {
 			logTermination(err)

pkg/containerruntimelogger/logruntime.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2020
+© Copyright IBM Corporation 2017, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -102,5 +102,10 @@ func LogContainerDetails(log *logger.Logger) error {
 			}
 		}
 	}
+
+	if os.Getenv("MQ_LOGGING_CONSOLE_FORMAT") == "" && os.Getenv("LOG_FORMAT") != "" {
+		log.Println("Environment variable LOG_FORMAT is deprecated. Use MQ_LOGGING_CONSOLE_FORMAT instead.")
+	}
+
 	return nil
 }

test/docker/docker_api_test.go

@@ -106,6 +106,19 @@ func goldenPath(t *testing.T, metrics bool) {
 	defer cleanContainer(t, cli, id)
 	waitForReady(t, cli, id)
 
+	//By default AMQ5041I,AMQ5052I,AMQ5051I,AMQ5037I,AMQ5975I are excluded
+	jsonLogs := inspectLogs(t, cli, id)
+
+	isMessageFound := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFound == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	if strings.Contains(jsonLogs, "CWWKF0011I") {
+		t.Errorf("Web Server is off....CWWKF0011I message id is not expected")
+	}
+
 	t.Run("Validate Default LogFilePages", func(t *testing.T) {
 		testLogFilePages(t, cli, id, "qm1", "4096")
 	})
@@ -1158,28 +1171,6 @@ func jsonLogFormat(t *testing.T, metric bool) {
 	}
 }
 
-func TestBadLogFormat(t *testing.T) {
-	t.Parallel()
-
-	cli, err := client.NewClientWithOpts(client.FromEnv)
-	if err != nil {
-		t.Fatal(err)
-	}
-	containerConfig := container.Config{
-		Env: []string{
-			"LICENSE=accept",
-			"LOG_FORMAT=fake",
-		},
-	}
-	id := runContainer(t, cli, &containerConfig)
-	defer cleanContainer(t, cli, id)
-	rc := waitForContainer(t, cli, id, 20*time.Second)
-	if rc != 1 {
-		t.Errorf("Expected rc=1, got rc=%v", rc)
-	}
-	expectTerminationMessage(t, cli, id)
-}
-
 // TestMQJSONDisabled tests the case where MQ's JSON logging feature is
 // specifically disabled (which will disable log mirroring)
 func TestMQJSONDisabled(t *testing.T) {
@@ -1503,3 +1494,290 @@ func TestCustomLogFilePages(t *testing.T) {
 
 	testLogFilePages(t, cli, id, "qmlfp", "8192")
 }
+
+// TestLoggingConsoleSource tests default behavior which is
+// MQ_LOGGING_CONSOLE_SOURCE set to qmgr,web
+func TestLoggingConsoleSource(t *testing.T) {
+
+	t.Parallel()
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=true",
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	jsonLogs := inspectLogs(t, cli, id)
+
+	isMessageFound := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFound == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	if !strings.Contains(jsonLogs, "AMQ6206I") && !strings.Contains(jsonLogs, "CWWKF0011I") {
+		t.Errorf("Expected  messageIDs AMQ6206I and CWWKF0011I are not present")
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestOldBehaviorWebConsole sets LOG_FORMAT to json and verify logs are indeed in json format
+func TestOldBehaviorWebConsole(t *testing.T) {
+
+	t.Parallel()
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"LOG_FORMAT=json",
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	jsonLogs := inspectLogs(t, cli, id)
+
+	isMessageFound := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFound == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	if !strings.Contains(jsonLogs, "Environment variable LOG_FORMAT is deprecated. Use MQ_LOGGING_CONSOLE_FORMAT instead.") {
+		t.Errorf("Expected Message stating LOG_FORMAT is deprecated is not in the log")
+	}
+
+	isValidJSON := checkLogForValidJSON(jsonLogs)
+
+	if !isValidJSON {
+		t.Fatalf("Expected all log lines to be valid JSON.  But got error %v ", err)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestLoggingConsoleWithContRestart restarts the container and checks
+// that setting of env variable persists
+func TestLoggingConsoleWithContRestart(t *testing.T) {
+
+	t.Parallel()
+
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+			"MQ_LOGGING_CONSOLE_SOURCE=qmgr",
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	jsonLogs := inspectLogs(t, cli, id)
+
+	if !strings.Contains(jsonLogs, "AMQ6206I") {
+		t.Errorf("Expected  messageID AMQ6206I is not present in the message")
+	}
+
+	isMessageFound := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFound == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	stopContainer(t, cli, id)
+	startContainer(t, cli, id)
+	waitForReady(t, cli, id)
+
+	jsonLogs = inspectLogs(t, cli, id)
+
+	if !strings.Contains(jsonLogs, "Stopped queue manager") || strings.Contains(jsonLogs, "CWWKF0011I") {
+		t.Errorf("CWWKF0011I which is not expected is present!!!!!")
+	}
+
+	isMessageFoundAfterRestart := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFoundAfterRestart == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestLoggingWithQmgrAndExcludeId tests MQ_LOGGING_CONSOLE_SOURCE set to qmgr
+// and  exclude ID set to amq7230I.
+func TestLoggingWithQmgrAndExcludeId(t *testing.T) {
+	qmgrName := "qm1"
+
+	t.Parallel()
+
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+			"MQ_LOGGING_CONSOLE_SOURCE=qmgr",
+			"MQ_LOGGING_CONSOLE_FORMAT=json",
+			"MQ_LOGGING_CONSOLE_EXCLUDE_ID=amq7230I",
+		},
+	}
+
+	dir := "/var/mqm/qmgrs/" + qmgrName + "/errors"
+
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	jsonLogs := inspectLogs(t, cli, id)
+
+	isValidJSON := checkLogForValidJSON(jsonLogs)
+
+	if !isValidJSON {
+		t.Fatalf("Expected all log lines to be valid JSON.  But got error %v ", err)
+	}
+
+	if strings.Contains(jsonLogs, "AMQ7230I") {
+		t.Errorf("Expected to exclude messageId by default; but messageId \"AMQ7230I\" is present")
+	}
+
+	stopContainer(t, cli, id)
+
+	//checking that message is only excluded from the console log, but not from the MQ error log
+	b := copyFromContainer(t, cli, id, filepath.Join(dir, "AMQERR01.json"))
+
+	foundInLog := 0
+	r := bytes.NewReader(b)
+	scannernew := bufio.NewScanner(r)
+	for scannernew.Scan() {
+		textData := scannernew.Text()
+
+		if strings.Contains(textData, "AMQ7230I") {
+			foundInLog = 1
+		}
+	}
+	if foundInLog == 0 {
+		t.Errorf("mesageID AMQ7230I is not present in MQ LOG!!!!")
+	}
+
+}
+
+// TestLoggingConsoleSetToWeb tests MQ_LOGGING_CONSOLE_SOURCE set to web
+func TestLoggingConsoleSetToWeb(t *testing.T) {
+
+	t.Parallel()
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=true",
+			"MQ_LOGGING_CONSOLE_SOURCE=web",
+			"MQ_LOGGING_CONSOLE_EXCLUDE_ID=CWWKG0028A,CWWKS4105I",
+			"MQ_LOGGING_CONSOLE_FORMAT=json",
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+
+	// At the most we will wait for 60 seconds for the log to appear. If it doesn't
+	// appear within the time, we'll fail.
+	currentTime := time.Now()
+	waitTime := currentTime.Add(time.Second * 60)
+	var jsonLogs string
+	msgFound := false
+
+	for time.Now().Unix() < waitTime.Unix() {
+
+		jsonLogs = inspectLogs(t, cli, id)
+
+		if !strings.Contains(jsonLogs, "CWWKF0011I") {
+			time.Sleep(1 * time.Second)
+			continue
+		} else {
+			msgFound = true
+			break
+		}
+	}
+	if !msgFound {
+		t.Errorf("Expected  messageID CWWKF0011I is not present in the message")
+	}
+
+	if strings.Contains(jsonLogs, "AMQ5041I") || strings.Contains(jsonLogs, "AMQ5052I") ||
+		strings.Contains(jsonLogs, "AMQ5051I") || strings.Contains(jsonLogs, "AMQ5037I") ||
+		strings.Contains(jsonLogs, "AMQ5975I") || strings.Contains(jsonLogs, "CWWKG0028A") ||
+		strings.Contains(jsonLogs, "CWWKS4105I") {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestLoggingConsoleSetToQmgr test sets LOG_FORMAT to BASIC and MQ_LOGGING_CONSOLE_FORMAT to
+// json and check that log is in json format
+func TestLoggingConsoleSetToQmgr(t *testing.T) {
+	t.Parallel()
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+			"MQ_LOGGING_CONSOLE_SOURCE=qmgr",
+			"LOG_FORMAT=BASIC",
+			"MQ_LOGGING_CONSOLE_FORMAT=json",
+		},
+	}
+	id := runContainer(t, cli, &containerConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	jsonLogs := inspectLogs(t, cli, id)
+
+	isMessageFound := scanForExcludedEntries(jsonLogs)
+
+	if isMessageFound == true {
+		t.Errorf("Expected  to exclude messageId by default; but messageId \"%v\" is present", jsonLogs)
+	}
+
+	if !strings.Contains(jsonLogs, "AMQ6206I") {
+		t.Errorf("Expected  messageID AMQ6206I is not present in the message")
+	}
+
+	isValidJSON := checkLogForValidJSON(jsonLogs)
+
+	if !isValidJSON {
+		t.Fatalf("Expected all log lines to be valid JSON.  But got error %v ", err)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}

test/docker/docker_api_test_util.go

@@ -745,6 +745,7 @@ func inspectLogs(t *testing.T, cli *client.Client, ID string) string {
 		t.Fatal(err)
 	}
 	buf := new(bytes.Buffer)
+
 	// Each output line has a header, which needs to be removed
 	_, err = stdcopy.StdCopy(buf, buf, reader)
 	if err != nil {
@@ -890,6 +891,30 @@ func getMQVersion(t *testing.T, cli *client.Client) (string, error) {
 	return version, nil
 }
 
+// scanForExcludedEntries scans for default excluded messages
+func scanForExcludedEntries(msg string) bool {
+	if strings.Contains(msg, "AMQ5041I") || strings.Contains(msg, "AMQ5052I") ||
+		strings.Contains(msg, "AMQ5051I") || strings.Contains(msg, "AMQ5037I") ||
+		strings.Contains(msg, "AMQ5975I") {
+		return true
+	}
+	return false
+}
+
+// checkLogForValidJSON checks if the message is in Json format
+func checkLogForValidJSON(jsonLogs string) bool {
+	scanner := bufio.NewScanner(strings.NewReader(jsonLogs))
+	for scanner.Scan() {
+		var obj map[string]interface{}
+		s := scanner.Text()
+		err := json.Unmarshal([]byte(s), &obj)
+		if err != nil {
+			return false
+		}
+	}
+	return true
+}
+
 // runContainerWithAllConfig creates and starts a container, using the supplied ContainerConfig, HostConfig,
 // NetworkingConfig, and container name (or the value of t.Name if containerName="").
 func runContainerWithAllConfigError(t *testing.T, cli *client.Client, containerConfig *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) {