add multi-instance Queue Managers (#307)

* Initial code to implement multi-instance queue manager

* alter default mqsc to prevent race between listeners on standby startup

* Updates to multi-instance queue manager code

* initial multi instance test

* Multi-instance code improvements

* Multi instance fixes and first test

* configure queue manager

* Add mirror log filtering for mult-instance QMs

* Add log message for multi-instance enabled

* Improvements to container runtime logging

* refactor test

* Test active standby switch

* Improve createQueueManager function

* Test multi instance race

* wait

* multi instance mount tests

* skip race test

* mount tests

* no mount test

* single instance split mount tests

* readiness check

* More updates for handling standby queue manager

* Improve standby checks

* Minor fixes to miqm

* Fix logging of JSON errors

* Update copyrights

* Fix log includes
This commit is contained in:
Stephen Marshall
2019-05-01 14:42:25 +01:00
committed by Arthur Barr
parent 63af43f19d
commit 6c72c894f7
22 changed files with 773 additions and 144 deletions

View File

@@ -1,5 +1,5 @@
/*
© Copyright IBM Corporation 2017
© Copyright IBM Corporation 2017, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -41,7 +41,7 @@ func queueManagerHealthy() (bool, error) {
return false, err
}
fmt.Printf("%s", out)
if !strings.Contains(string(out), "(RUNNING)") {
if !strings.Contains(string(out), "(RUNNING)") && !strings.Contains(string(out), "(RUNNING AS STANDBY)") {
return false, nil
}
return true, nil

View File

@@ -1,5 +1,5 @@
/*
© Copyright IBM Corporation 2017, 2018
© Copyright IBM Corporation 2017, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import (
"net"
"os"
"github.com/ibm-messaging/mq-container/internal/name"
"github.com/ibm-messaging/mq-container/internal/ready"
)
@@ -31,14 +32,25 @@ func main() {
if !r || err != nil {
os.Exit(1)
}
// Check if the queue manager has a running listener
conn, err := net.Dial("tcp", "127.0.0.1:1414")
name, err := name.GetQueueManagerName()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = conn.Close()
if err != nil {
fmt.Println(err)
// Check if the queue manager has a running listener
if standby, _ := ready.IsRunningAsStandbyQM(name); !standby {
conn, err := net.Dial("tcp", "127.0.0.1:1414")
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = conn.Close()
if err != nil {
fmt.Println(err)
}
} else {
fmt.Printf("Detected queue manager running in standby mode")
os.Exit(10)
}
}

View File

@@ -1,68 +0,0 @@
/*
© Copyright IBM Corporation 2017, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"runtime"
"strings"
containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime"
"github.com/ibm-messaging/mq-container/internal/user"
)
func logContainerDetails() {
log.Printf("CPU architecture: %v", runtime.GOARCH)
kv, err := containerruntime.GetKernelVersion()
if err == nil {
log.Printf("Linux kernel version: %v", kv)
}
cr, err := containerruntime.GetContainerRuntime()
if err == nil {
log.Printf("Container runtime: %v", cr)
}
bi, err := containerruntime.GetBaseImage()
if err == nil {
log.Printf("Base image: %v", bi)
}
u, err := user.GetUser()
if err == nil {
if len(u.SupplementalGID) == 0 {
log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID)
} else {
log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ","))
}
}
caps, err := containerruntime.GetCapabilities()
capLogged := false
if err == nil {
for k, v := range caps {
if len(v) > 0 {
log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ","))
capLogged = true
}
}
if !capLogged {
log.Print("Capabilities: none")
}
} else {
log.Errorf("Error getting capabilities: %v", err)
}
sc, err := containerruntime.GetSeccomp()
if err == nil {
log.Printf("seccomp enforcing mode: %v", sc)
}
log.Printf("Process security attributes: %v", containerruntime.GetSecurityAttributes())
}

View File

@@ -23,6 +23,7 @@ import (
"syscall"
"github.com/ibm-messaging/mq-container/internal/command"
containerruntimelogger "github.com/ibm-messaging/mq-container/internal/containerruntimelogger"
"github.com/ibm-messaging/mq-container/internal/logger"
"github.com/ibm-messaging/mq-container/internal/name"
)
@@ -117,7 +118,11 @@ func doMain() error {
return err
}
logContainerDetails()
err = containerruntimelogger.LogContainerDetails(log)
if err != nil {
logTermination(err)
return err
}
adminPassword, set := os.LookupEnv("MQ_ADMIN_PASSWORD")
if set {

View File

@@ -1,5 +1,5 @@
/*
© Copyright IBM Corporation 2017, 2018
© Copyright IBM Corporation 2017, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -17,15 +17,13 @@ package main
import (
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/ibm-messaging/mq-container/internal/command"
)
func createVolume(path string) error {
dataPath := filepath.Join(path, "data")
func createVolume(dataPath string) error {
fi, err := os.Stat(dataPath)
if err != nil {
if os.IsNotExist(err) {

View File

@@ -23,6 +23,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"github.com/ibm-messaging/mq-container/internal/command"
@@ -66,7 +67,7 @@ func formatSimple(datetime string, message string) string {
// mirrorSystemErrorLogs starts a goroutine to mirror the contents of the MQ system error logs
func mirrorSystemErrorLogs(ctx context.Context, wg *sync.WaitGroup, mf mirrorFunc) (chan error, error) {
// Always use the JSON log as the source
return mirrorLog(ctx, wg, "/var/mqm/errors/AMQERR01.json", false, mf)
return mirrorLog(ctx, wg, "/var/mqm/errors/AMQERR01.json", false, mf, false)
}
// mirrorQueueManagerErrorLogs starts a goroutine to mirror the contents of the MQ queue manager error logs
@@ -78,7 +79,7 @@ func mirrorQueueManagerErrorLogs(ctx context.Context, wg *sync.WaitGroup, name s
return nil, err
}
f := filepath.Join(mqini.GetErrorLogDirectory(qm), "AMQERR01.json")
return mirrorLog(ctx, wg, f, fromStart, mf)
return mirrorLog(ctx, wg, f, fromStart, mf, true)
}
func getDebug() bool {
@@ -99,21 +100,35 @@ func configureLogger(name string) (mirrorFunc, error) {
if err != nil {
return nil, err
}
return log.LogDirect, nil
return func(msg string, isQMLog bool) bool {
obj, err := processLogMessage(msg)
if err == nil && isQMLog && filterQMLogMessage(obj) {
return false
}
if err != nil {
log.Printf("Failed to unmarshall JSON - %v", msg)
} else {
fmt.Println(msg)
}
return true
}, nil
case "basic":
log, err = logger.NewLogger(os.Stderr, d, false, name)
if err != nil {
return nil, err
}
return func(msg string) {
return func(msg string, isQMLog bool) bool {
// Parse the JSON message, and print a simplified version
var obj map[string]interface{}
err := json.Unmarshal([]byte(msg), &obj)
obj, err := processLogMessage(msg)
if err == nil && isQMLog && filterQMLogMessage(obj) {
return false
}
if err != nil {
fmt.Printf("Failed to Unmarshall JSON - %v", err)
log.Printf("Failed to unmarshall JSON - %v", err)
} else {
fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string)))
}
return true
}, nil
default:
log, err = logger.NewLogger(os.Stdout, d, false, name)
@@ -124,6 +139,20 @@ func configureLogger(name string) (mirrorFunc, error) {
}
}
func processLogMessage(msg string) (map[string]interface{}, error) {
var obj map[string]interface{}
err := json.Unmarshal([]byte(msg), &obj)
return obj, err
}
func filterQMLogMessage(obj map[string]interface{}) bool {
hostname, err := os.Hostname()
if os.Getenv("MQ_MULTI_INSTANCE") == "true" && err == nil && !strings.Contains(obj["host"].(string), hostname) {
return true
}
return false
}
func logDiagnostics() {
log.Debug("--- Start Diagnostics ---")

View File

@@ -1,86 +0,0 @@
/*
© Copyright IBM Corporation 2017, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"runtime"
"strings"
containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime"
"github.com/ibm-messaging/mq-container/internal/user"
)
func logContainerDetails() error {
if runtime.GOOS != "linux" {
return fmt.Errorf("Unsupported platform: %v", runtime.GOOS)
}
log.Printf("CPU architecture: %v", runtime.GOARCH)
kv, err := containerruntime.GetKernelVersion()
if err == nil {
log.Printf("Linux kernel version: %v", kv)
}
cr, err := containerruntime.GetContainerRuntime()
if err == nil {
log.Printf("Container runtime: %v", cr)
}
bi, err := containerruntime.GetBaseImage()
if err == nil {
log.Printf("Base image: %v", bi)
}
u, err := user.GetUser()
if err == nil {
if len(u.SupplementalGID) == 0 {
log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID)
} else {
log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ","))
}
}
caps, err := containerruntime.GetCapabilities()
capLogged := false
if err == nil {
for k, v := range caps {
if len(v) > 0 {
log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ","))
capLogged = true
}
}
if !capLogged {
log.Print("Capabilities: none")
}
} else {
log.Errorf("Error getting capabilities: %v", err)
}
sc, err := containerruntime.GetSeccomp()
if err == nil {
log.Printf("seccomp enforcing mode: %v", sc)
}
log.Printf("Process security attributes: %v", containerruntime.GetSecurityAttributes())
m, err := containerruntime.GetMounts()
if err == nil {
if len(m) == 0 {
log.Print("No volume detected. Persistent messages may be lost")
} else {
for mountPoint, fsType := range m {
log.Printf("Detected '%v' volume mounted to %v", fsType, mountPoint)
if !containerruntime.SupportedFilesystem(fsType) {
return fmt.Errorf("%v uses unsupported filesystem type: %v", mountPoint, fsType)
}
}
}
}
return nil
}

View File

@@ -24,6 +24,7 @@ import (
"os"
"sync"
containerruntimelogger "github.com/ibm-messaging/mq-container/internal/containerruntimelogger"
"github.com/ibm-messaging/mq-container/internal/metrics"
"github.com/ibm-messaging/mq-container/internal/name"
"github.com/ibm-messaging/mq-container/internal/ready"
@@ -45,7 +46,7 @@ func doMain() error {
// Check whether they only want debug info
if *infoFlag {
logVersionInfo()
err = logContainerDetails()
err = containerruntimelogger.LogContainerDetails(log)
if err != nil {
log.Printf("Error displaying container details: %v", err)
}
@@ -86,14 +87,24 @@ func doMain() error {
collectDiagOnFail = true
if *devFlag == false {
err = logContainerDetails()
err = containerruntimelogger.LogContainerDetails(log)
if err != nil {
logTermination(err)
return err
}
}
err = createVolume("/mnt/mqm")
err = createVolume("/mnt/mqm/data")
if err != nil {
logTermination(err)
return err
}
err = createVolume("/mnt/mqm-log/log")
if err != nil {
logTermination(err)
return err
}
err = createVolume("/mnt/mqm-data/qmgrs")
if err != nil {
logTermination(err)
return err
@@ -149,15 +160,17 @@ func doMain() error {
logTermination(err)
return err
}
err = startQueueManager()
err = startQueueManager(name)
if err != nil {
logTermination(err)
return err
}
err = configureQueueManager()
if err != nil {
logTermination(err)
return err
if standby, _ := ready.IsRunningAsStandbyQM(name); !standby {
err = configureQueueManager()
if err != nil {
logTermination(err)
return err
}
}
enableMetrics := os.Getenv("MQ_ENABLE_METRICS")

View File

@@ -1,5 +1,5 @@
/*
© Copyright IBM Corporation 2018
© Copyright IBM Corporation 2018, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -49,16 +49,17 @@ func waitForFile(ctx context.Context, path string) (os.FileInfo, error) {
}
}
type mirrorFunc func(msg string)
type mirrorFunc func(msg string, isQMLog bool) bool
// mirrorAvailableMessages prints lines from the file, until no more are available
func mirrorAvailableMessages(f *os.File, mf mirrorFunc) {
func mirrorAvailableMessages(f *os.File, mf mirrorFunc, isQMLog bool) {
scanner := bufio.NewScanner(f)
count := 0
for scanner.Scan() {
t := scanner.Text()
mf(t)
count++
if mf(t, isQMLog) {
count++
}
}
if count > 0 {
log.Debugf("Mirrored %v log entries from %v", count, f.Name())
@@ -73,7 +74,7 @@ func mirrorAvailableMessages(f *os.File, mf mirrorFunc) {
// mirrorLog tails the specified file, and logs each line to stdout.
// This is useful for usability, as the container console log can show
// messages from the MQ error logs.
func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart bool, mf mirrorFunc) (chan error, error) {
func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart bool, mf mirrorFunc, isQMLog bool) (chan error, error) {
errorChannel := make(chan error, 1)
var offset int64 = -1
var f *os.File
@@ -147,7 +148,7 @@ func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart b
closing := false
for {
// If there's already data there, mirror it now.
mirrorAvailableMessages(f, mf)
mirrorAvailableMessages(f, mf, isQMLog)
// Wait for the new log file (after rotation)
newFI, err := waitForFile(ctx, path)
if err != nil {
@@ -161,7 +162,7 @@ func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart b
// log rotation happens before we can open the new file, then we
// could skip all those messages. This could happen with a very small
// MQ error log size.
mirrorAvailableMessages(f, mf)
mirrorAvailableMessages(f, mf, isQMLog)
err = f.Close()
if err != nil {
log.Errorf("Unable to close mirror file handle: %v", err)
@@ -176,7 +177,7 @@ func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart b
}
fi = newFI
// Don't seek this time, because we know it's a new file
mirrorAvailableMessages(f, mf)
mirrorAvailableMessages(f, mf, isQMLog)
}
select {
case <-ctx.Done():

View File

@@ -1,5 +1,5 @@
/*
© Copyright IBM Corporation 2018
© Copyright IBM Corporation 2018, 2019
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -41,9 +41,10 @@ func TestMirrorLogWithoutRotation(t *testing.T) {
count := 0
ctx, cancel := context.WithCancel(context.Background())
var wg sync.WaitGroup
_, err = mirrorLog(ctx, &wg, tmp.Name(), true, func(msg string) {
_, err = mirrorLog(ctx, &wg, tmp.Name(), true, func(msg string, isQMLog bool) bool {
count++
})
return true
}, false)
if err != nil {
t.Fatal(err)
}
@@ -82,9 +83,10 @@ func TestMirrorLogWithRotation(t *testing.T) {
count := 0
ctx, cancel := context.WithCancel(context.Background())
var wg sync.WaitGroup
_, err = mirrorLog(ctx, &wg, tmp.Name(), true, func(msg string) {
_, err = mirrorLog(ctx, &wg, tmp.Name(), true, func(msg string, isQMLog bool) bool {
count++
})
return true
}, false)
if err != nil {
t.Fatal(err)
}
@@ -135,9 +137,10 @@ func testMirrorLogExistingFile(t *testing.T, newQM bool) int {
count := 0
ctx, cancel := context.WithCancel(context.Background())
var wg sync.WaitGroup
_, err = mirrorLog(ctx, &wg, tmp.Name(), newQM, func(msg string) {
_, err = mirrorLog(ctx, &wg, tmp.Name(), newQM, func(msg string, isQMLog bool) bool {
count++
})
return true
}, false)
if err != nil {
t.Fatal(err)
}
@@ -179,8 +182,9 @@ func TestMirrorLogCancelWhileWaiting(t *testing.T) {
cancel()
wg.Wait()
}()
_, err := mirrorLog(ctx, &wg, "fake.log", true, func(msg string) {
})
_, err := mirrorLog(ctx, &wg, "fake.log", true, func(msg string, isQMLog bool) bool {
return true
}, false)
if err != nil {
t.Error(err)
}

View File

@@ -25,7 +25,9 @@ import (
"strings"
"github.com/ibm-messaging/mq-container/internal/command"
containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime"
"github.com/ibm-messaging/mq-container/internal/mqscredact"
"github.com/ibm-messaging/mq-container/internal/ready"
)
// createDirStructure creates the default MQ directory structure under /var/mqm
@@ -40,20 +42,48 @@ func createDirStructure() error {
}
// createQueueManager creates a queue manager, if it doesn't already exist.
// It returns true if one was created, or false if one already existed
// It returns true if one was created (or a standby was created), or false if one already existed
func createQueueManager(name string) (bool, error) {
log.Printf("Creating queue manager %v", name)
out, rc, err := command.Run("crtmqm", "-q", "-p", "1414", name)
// Run 'dspmqinf' to check if 'mqs.ini' configuration file exists
// If command succeeds, the queue manager (or standby queue manager) has already been created
_, _, err := command.Run("dspmqinf", name)
if err == nil {
log.Printf("Detected existing queue manager %v", name)
return false, nil
}
mounts, err := containerruntime.GetMounts()
if err != nil {
// 8=Queue manager exists, which is fine
if rc == 8 {
log.Printf("Detected existing queue manager %v", name)
return false, nil
}
log.Printf("crtmqm returned %v", rc)
log.Println(string(out))
log.Printf("Error getting mounts for queue manager")
return false, err
}
// Check if 'qm.ini' configuration file exists for the queue manager
// TODO : handle possible race condition - use a file lock?
dataDir := getQueueManagerDataDir(mounts, name)
_, err = os.Stat(filepath.Join(dataDir, "qm.ini"))
if err != nil {
// If 'qm.ini' is not found - run 'crtmqm' to create a new queue manager
args := getCreateQueueManagerArgs(mounts, name)
out, rc, err := command.Run("crtmqm", args...)
if err != nil {
log.Printf("Error %v creating queue manager: %v", rc, string(out))
return false, err
}
} else {
// If 'qm.ini' is found - run 'addmqinf' to create a standby queue manager with existing configuration
args := getCreateStandbyQueueManagerArgs(name)
out, rc, err := command.Run("addmqinf", args...)
if err != nil {
log.Printf("Error %v creating standby queue manager: %v", rc, string(out))
return false, err
}
log.Println("Created standby queue manager")
return true, nil
}
log.Println("Created queue manager")
return true, nil
}
@@ -70,10 +100,15 @@ func updateCommandLevel() error {
return nil
}
func startQueueManager() error {
func startQueueManager(name string) error {
log.Println("Starting queue manager")
out, rc, err := command.Run("strmqm")
out, rc, err := command.Run("strmqm", "-x", name)
if err != nil {
// 30=standby queue manager started, which is fine
if rc == 30 {
log.Printf("Started standby queue manager")
return nil
}
log.Printf("Error %v starting queue manager: %v", rc, string(out))
return err
}
@@ -136,12 +171,29 @@ func configureQueueManager() error {
func stopQueueManager(name string) error {
log.Println("Stopping queue manager")
out, _, err := command.Run("endmqm", "-w", "-r", name)
isStandby, err := ready.IsRunningAsStandbyQM(name)
if err != nil {
log.Printf("Error stopping queue manager: %v", string(out))
log.Printf("Error getting status for queue manager %v: ", name, err.Error())
return err
}
log.Println("Stopped queue manager")
args := []string{"-w", "-r", name}
if os.Getenv("MQ_MULTI_INSTANCE") == "true" {
if isStandby {
args = []string{"-x", name}
} else {
args = []string{"-s", "-w", "-r", name}
}
}
out, rc, err := command.Run("endmqm", args...)
if err != nil {
log.Printf("Error %v stopping queue manager: %v", rc, string(out))
return err
}
if isStandby {
log.Printf("Stopped standby queue manager")
} else {
log.Println("Stopped queue manager")
}
return nil
}
@@ -152,3 +204,44 @@ func formatMQSCOutput(out string) string {
// add tab characters to make it more readable as part of the log
return strings.Replace(string(out), "\n", "\n\t", -1)
}
func isStandbyQueueManager(name string) (bool, error) {
out, rc, err := command.Run("dspmq", "-n", "-m", name)
if err != nil {
log.Printf("Error %v getting status for queue manager %v: %v", rc, name, string(out))
return false, err
}
if strings.Contains(string(out), "(RUNNING AS STANDBY)") {
return true, nil
}
return false, nil
}
func getQueueManagerDataDir(mounts map[string]string, name string) string {
dataDir := filepath.Join("/var/mqm/qmgrs", name)
if _, ok := mounts["/mnt/mqm-data"]; ok {
dataDir = filepath.Join("/mnt/mqm-data/qmgrs", name)
}
return dataDir
}
func getCreateQueueManagerArgs(mounts map[string]string, name string) []string {
args := []string{"-q", "-p", "1414"}
if _, ok := mounts["/mnt/mqm-log"]; ok {
args = append(args, "-ld", "/mnt/mqm-log/log")
}
if _, ok := mounts["/mnt/mqm-data"]; ok {
args = append(args, "-md", "/mnt/mqm-data/qmgrs")
}
args = append(args, name)
return args
}
func getCreateStandbyQueueManagerArgs(name string) []string {
args := []string{"-s", "QueueManager"}
args = append(args, "-v", fmt.Sprintf("Name=%v", name))
args = append(args, "-v", fmt.Sprintf("Directory=%v", name))
args = append(args, "-v", "Prefix=/var/mqm")
args = append(args, "-v", fmt.Sprintf("DataPath=/mnt/mqm-data/qmgrs/%v", name))
return args
}