From 33defc0fc96382fa6a7bef911009f571c700f5cd Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Thu, 4 Apr 2019 16:46:07 +0100 Subject: [PATCH 1/3] Handle large mqsc files (#295) * buffer mqsc input to runmqsc * imports * error handling on runmqsc --- cmd/runmqserver/qmgr.go | 42 ++++++++++++++------------------- test/docker/docker_api_test.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 25 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 49b73c1..821bd66 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -16,7 +16,7 @@ limitations under the License. package main import ( - "io" + "bytes" "io/ioutil" "os" "os/exec" @@ -86,43 +86,35 @@ func configureQueueManager() error { log.Println(err) return err } - for _, file := range files { if strings.HasSuffix(file.Name(), ".mqsc") { abs := filepath.Join(configDir, file.Name()) // #nosec G204 cmd := exec.Command("runmqsc") - stdin, err := cmd.StdinPipe() + // Read mqsc file into variable + mqsc, err := ioutil.ReadFile(abs) if err != nil { - log.Println(err) - return err + log.Printf("Error reading file %v: %v", abs, err) + continue } - // Open the MQSC file for reading - // #nosec G304 - f, err := os.Open(abs) + // Write mqsc to buffer + var buffer bytes.Buffer + _, err = buffer.Write(mqsc) if err != nil { - log.Printf("Error opening %v: %v", abs, err) + log.Printf("Error writing MQSC file %v to buffer: %v", abs, err) + continue } - // Copy the contents to stdin of the runmqsc process - _, err = io.Copy(stdin, f) - if err != nil { - log.Errorf("Error reading %v: %v", abs, err) - } - err = f.Close() - if err != nil { - log.Errorf("Failed to close MQSC file handle: %v", err) - } - err = stdin.Close() - if err != nil { - log.Errorf("Failed to close MQSC stdin: %v", err) - } - // Run the command and wait for completion + // Buffer mqsc to stdin of runmqsc + cmd.Stdin = &buffer + // Run runmqsc command out, err := cmd.CombinedOutput() if err != nil { log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) + continue + } else { + // Print the runmqsc output, adding tab characters to make it more readable as part of the log + log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) } - // Print the runmqsc output, adding tab characters to make it more readable as part of the log - log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) } } return nil diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 37759f2..f540639 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -554,6 +554,49 @@ func TestMQSC(t *testing.T) { } } +// TestLargeMQSC creates a new image with a large MQSC file in, starts a container based +// on that image, and checks that the MQSC has been applied correctly. +func TestLargeMQSC(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + const numQueues = 1000 + var buf bytes.Buffer + for i := 1; i <= numQueues; i++ { + fmt.Fprintf(&buf, "* Test processing of a large MQSC file, defining queue test%v\nDEFINE QLOCAL(test%v)\n", i, i) + } + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD test.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/test.mqsc + USER mqm`, imageName())}, + {"test.mqsc", buf.String()}, + } + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + + rc, mqscOutput := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test" + strconv.Itoa(numQueues) + ")' | runmqsc"}) + if rc != 0 { + r := regexp.MustCompile("AMQ[0-9][0-9][0-9][0-9]E") + t.Fatalf("Expected runmqsc to exit with rc=0, got %v with error %v", rc, r.FindString(mqscOutput)) + } +} + // TestInvalidMQSC creates a new image with an MQSC file containing invalid MQSC, // tries to start a container based on that image, and checks that container terminates // func TestInvalidMQSC(t *testing.T) { From 44d0e0a432cd86bcca4ba01d34a86049322d47f9 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Fri, 5 Apr 2019 14:11:55 +0100 Subject: [PATCH 2/3] Redact sensitive MQSC logs --- cmd/runmqserver/qmgr.go | 18 ++++++++++-- test/docker/docker_api_test.go | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 821bd66..236d36d 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -21,6 +21,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/ibm-messaging/mq-container/internal/command" @@ -109,11 +110,11 @@ func configureQueueManager() error { // Run runmqsc command out, err := cmd.CombinedOutput() if err != nil { - log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) + log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, formatMQSCOutput(string(out))) continue } else { // Print the runmqsc output, adding tab characters to make it more readable as part of the log - log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) + log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, formatMQSCOutput(string(out))) } } } @@ -130,3 +131,16 @@ func stopQueueManager(name string) error { log.Println("Stopped queue manager") return nil } + +func formatMQSCOutput(out string) string { + // redact sensitive information + pattern, _ := regexp.Compile("(?i)LDAPPWD\\s?\\((.*?)\\)") + out = pattern.ReplaceAllString(out, "LDAPPWD(*********)") + pattern, _ = regexp.Compile("(?i)PASSWORD\\s?\\((.*?)\\)") + out = pattern.ReplaceAllString(out, "PASSWORD(*********)") + pattern, _ = regexp.Compile("(?i)SSLCRYP\\s?\\((.*?)\\)") + out = pattern.ReplaceAllString(out, "SSLCRYP(*********)") + + // add tab characters to make it more readable as part of the log + return strings.Replace(string(out), "\n", "\n\t", -1) +} diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index f540639..cd662fa 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -558,6 +558,7 @@ func TestMQSC(t *testing.T) { // on that image, and checks that the MQSC has been applied correctly. func TestLargeMQSC(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -597,6 +598,57 @@ func TestLargeMQSC(t *testing.T) { } } +// TestRedactMQSC creates a new image with a MQSC file that contains sensitive information, starts a container based +// on that image, and checks that the MQSC has been redacted in the logs. +func TestRedactMQSC(t *testing.T) { + t.Parallel() + + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + sslcryp := "GSK_PKCS11=/usr/lib/pkcs11/PKCS11_API.so;token-label;token-password;SYMMETRIC_CIPHER_ON;" + fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LDAPPWD(abcdefgh) B(2) PASSWORD(abcdefgh) C(3) SSLCRYP(%v) D(4)\n", sslcryp) + fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) ldappwd(12345678) B(2) password(12345678) C(3) sslcryp(%v) D(4)\n", sslcryp) + fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LDAPPWD (12?@!$gh) B(2) PASSWORD (12?@!$gh) C(3) SSLCRYP (%v) D(4)", sslcryp) + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD test.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/test.mqsc + USER mqm`, imageName())}, + {"test.mqsc", buf.String()}, + } + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + stopContainer(t, cli, id) + scanner := bufio.NewScanner(strings.NewReader(inspectLogs(t, cli, id))) + expectedOutput := "*TEST-REDACT-MQSC: A(1) LDAPPWD(*********) B(2) PASSWORD(*********) C(3) SSLCRYP(*********) D(4)" + for scanner.Scan() { + s := scanner.Text() + if strings.Contains(s, "*TEST-REDACT-MQSC:") && !strings.Contains(s, expectedOutput) { + t.Fatalf("Expected redacted MQSC output, got: %v", s) + } + } + err = scanner.Err() + if err != nil { + t.Fatal(err) + } +} + // TestInvalidMQSC creates a new image with an MQSC file containing invalid MQSC, // tries to start a container based on that image, and checks that container terminates // func TestInvalidMQSC(t *testing.T) { From 723fe2b998b8ccb7a628f1042f7ab2ef3ed45d81 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Tue, 9 Apr 2019 15:06:25 +0100 Subject: [PATCH 3/3] Handle multiple-spaces when redacting MQSC output --- cmd/runmqserver/qmgr.go | 6 +++--- test/docker/docker_api_test.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 236d36d..0e2984e 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -134,11 +134,11 @@ func stopQueueManager(name string) error { func formatMQSCOutput(out string) string { // redact sensitive information - pattern, _ := regexp.Compile("(?i)LDAPPWD\\s?\\((.*?)\\)") + pattern, _ := regexp.Compile("(?i)LDAPPWD\\s*?\\((.*?)\\)") out = pattern.ReplaceAllString(out, "LDAPPWD(*********)") - pattern, _ = regexp.Compile("(?i)PASSWORD\\s?\\((.*?)\\)") + pattern, _ = regexp.Compile("(?i)PASSWORD\\s*?\\((.*?)\\)") out = pattern.ReplaceAllString(out, "PASSWORD(*********)") - pattern, _ = regexp.Compile("(?i)SSLCRYP\\s?\\((.*?)\\)") + pattern, _ = regexp.Compile("(?i)SSLCRYP\\s*?\\((.*?)\\)") out = pattern.ReplaceAllString(out, "SSLCRYP(*********)") // add tab characters to make it more readable as part of the log diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index cd662fa..4b4e5af 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -611,7 +611,8 @@ func TestRedactMQSC(t *testing.T) { sslcryp := "GSK_PKCS11=/usr/lib/pkcs11/PKCS11_API.so;token-label;token-password;SYMMETRIC_CIPHER_ON;" fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LDAPPWD(abcdefgh) B(2) PASSWORD(abcdefgh) C(3) SSLCRYP(%v) D(4)\n", sslcryp) fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) ldappwd(12345678) B(2) password(12345678) C(3) sslcryp(%v) D(4)\n", sslcryp) - fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LDAPPWD (12?@!$gh) B(2) PASSWORD (12?@!$gh) C(3) SSLCRYP (%v) D(4)", sslcryp) + fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LdapPwd('12?@!$Gh') B(2) Password('12?@!$Gh') C(3) SSLCryp(%v) D(4)\n", sslcryp) + fmt.Fprintf(&buf, "*TEST-REDACT-MQSC: A(1) LDAPPWD (abcdefgh) B(2) PASSWORD\t(abcdefgh) C(3) SSLCRYP \t (%v) D(4)", sslcryp) var files = []struct { Name, Body string }{