From 3848c39147eb006044af78f1d8626d557ac5ab24 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 21 Nov 2017 10:48:19 +0000 Subject: [PATCH 01/11] Add build section to docs --- docs/developing.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/developing.md b/docs/developing.md index 17ba25a..048c522 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -8,9 +8,15 @@ You need to ensure you have the following tools installed: * [Glide](https://glide.sh/) * [dep](https://github.com/golang/dep) (official Go dependency management tool) * make +* [Helm](https://helm.sh) - only needed for running the Kubernetes tests For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). +## Building a production image + +1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4) in the `downloads` directory +2. Run `make build-advancedserver` + ## Running the tests There are three main sets of tests: From 099397442b3712f78ef85d9b9b4cb961a3b63435 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 21 Nov 2017 14:32:09 +0000 Subject: [PATCH 02/11] Restructure packages for reuse --- Dockerfile-server | 14 ++--- cmd/chkmqhealthy/main.go | 2 +- cmd/runmqserver/main.go | 63 +++---------------- cmd/runmqserver/mqconfig.go | 2 +- .../capabilities/capabilities.go | 0 .../capabilities/capabilities_test.go | 0 internal/command/command.go | 32 ++++++++++ {pkg => internal}/name/name.go | 2 - {pkg => internal}/name/name_test.go | 0 9 files changed, 49 insertions(+), 66 deletions(-) rename {pkg/linux => internal}/capabilities/capabilities.go (100%) rename {pkg/linux => internal}/capabilities/capabilities_test.go (100%) create mode 100644 internal/command/command.go rename {pkg => internal}/name/name.go (95%) rename {pkg => internal}/name/name_test.go (100%) diff --git a/Dockerfile-server b/Dockerfile-server index 7222dcd..3774d5a 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -12,23 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +############################################################################### # Build stage to build Go code +############################################################################### FROM golang:1.9 as builder WORKDIR /go/src/github.com/ibm-messaging/mq-container/ COPY cmd/ ./cmd -COPY pkg/ ./pkg +COPY internal/ ./internal COPY vendor/ ./vendor RUN go build ./cmd/runmqserver/ RUN go build ./cmd/chkmqready/ RUN go build ./cmd/chkmqhealthy/ +# Run all unit tests +RUN go test -v ./cmd/... ./internal/... -# Build stage to run Go unit tests -FROM golang:1.9 as tester -COPY pkg/ ./pkg -RUN cd pkg/name && go test -RUN cd pkg/linux/capabilities && go test - +############################################################################### # Main build stage, to build MQ image +############################################################################### FROM ubuntu:16.04 # The URL to download the MQ installer from in tar.gz format diff --git a/cmd/chkmqhealthy/main.go b/cmd/chkmqhealthy/main.go index 64ead43..87062f3 100644 --- a/cmd/chkmqhealthy/main.go +++ b/cmd/chkmqhealthy/main.go @@ -22,7 +22,7 @@ import ( "os/exec" "strings" - "github.com/ibm-messaging/mq-container/pkg/name" + "github.com/ibm-messaging/mq-container/internal/name" ) func queueManagerHealthy() (bool, error) { diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 06e0714..3e47ef9 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -25,11 +25,11 @@ import ( "os/exec" "os/signal" "path/filepath" - "regexp" - "runtime" "strings" "syscall" + "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/name" "golang.org/x/sys/unix" ) @@ -99,56 +99,9 @@ func checkLicense() { os.Exit(1) } -// sanitizeQueueManagerName removes any invalid characters from a queue manager name -func sanitizeQueueManagerName(name string) string { - var re = regexp.MustCompile("[^a-zA-Z0-9._%/]") - return re.ReplaceAllString(name, "") -} - -// GetQueueManagerName resolves the queue manager name to use. Resolved from -// either an environment variable, or the hostname. -func getQueueManagerName() (string, error) { - var name string - var err error - name, ok := os.LookupEnv("MQ_QMGR_NAME") - if !ok || name == "" { - name, err = os.Hostname() - if err != nil { - return "", err - } - name = sanitizeQueueManagerName(name) - } - // TODO: What if the specified env variable is an invalid name? - return name, nil -} - -// runCommand runs an OS command. On Linux it waits for the command to -// complete and returns the exit status (return code). -func runCommand(name string, arg ...string) (string, int, error) { - cmd := exec.Command(name, arg...) - // Run the command and wait for completion - out, err := cmd.CombinedOutput() - if err != nil { - var rc int - // Only works on Linux - if runtime.GOOS == "linux" { - var ws unix.WaitStatus - unix.Wait4(cmd.Process.Pid, &ws, 0, nil) - rc = ws.ExitStatus() - } else { - rc = -1 - } - if rc == 0 { - return string(out), rc, nil - } - return string(out), rc, err - } - return string(out), 0, nil -} - // createDirStructure creates the default MQ directory structure under /var/mqm func createDirStructure() { - out, _, err := runCommand("/opt/mqm/bin/crtmqdir", "-f", "-s") + out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s") if err != nil { log.Fatalf("Error creating directory structure: %v\n", string(out)) } @@ -157,7 +110,7 @@ func createDirStructure() { func createQueueManager(name string) { log.Printf("Creating queue manager %v", name) - out, rc, err := runCommand("crtmqm", "-q", "-p", "1414", name) + out, rc, err := command.Run("crtmqm", "-q", "-p", "1414", name) if err != nil { // 8=Queue manager exists, which is fine if rc != 8 { @@ -173,7 +126,7 @@ func createQueueManager(name string) { func updateCommandLevel() { level, ok := os.LookupEnv("MQ_CMDLEVEL") if ok && level != "" { - out, rc, err := runCommand("strmqm", "-e", "CMDLEVEL="+level) + out, rc, err := command.Run("strmqm", "-e", "CMDLEVEL="+level) if err != nil { log.Fatalf("Error %v setting CMDLEVEL: %v", rc, string(out)) } @@ -182,7 +135,7 @@ func updateCommandLevel() { func startQueueManager() { log.Println("Starting queue manager") - out, rc, err := runCommand("strmqm") + out, rc, err := command.Run("strmqm") if err != nil { log.Fatalf("Error %v starting queue manager: %v", rc, string(out)) } @@ -223,7 +176,7 @@ func configureQueueManager() { func stopQueueManager() { log.Println("Stopping queue manager") - out, _, err := runCommand("endmqm", "-w") + out, _, err := command.Run("endmqm", "-w") if err != nil { log.Fatalf("Error stopping queue manager: %v", string(out)) } @@ -275,7 +228,7 @@ func main() { // Start SIGTERM handler channel done := createTerminateChannel() - name, err := getQueueManagerName() + name, err := name.GetQueueManagerName() if err != nil { log.Fatalln(err) } diff --git a/cmd/runmqserver/mqconfig.go b/cmd/runmqserver/mqconfig.go index 7a90fd2..a5e0f51 100644 --- a/cmd/runmqserver/mqconfig.go +++ b/cmd/runmqserver/mqconfig.go @@ -22,7 +22,7 @@ import ( "runtime" "strings" - "github.com/ibm-messaging/mq-container/pkg/linux/capabilities" + "github.com/ibm-messaging/mq-container/internal/capabilities" "golang.org/x/sys/unix" ) diff --git a/pkg/linux/capabilities/capabilities.go b/internal/capabilities/capabilities.go similarity index 100% rename from pkg/linux/capabilities/capabilities.go rename to internal/capabilities/capabilities.go diff --git a/pkg/linux/capabilities/capabilities_test.go b/internal/capabilities/capabilities_test.go similarity index 100% rename from pkg/linux/capabilities/capabilities_test.go rename to internal/capabilities/capabilities_test.go diff --git a/internal/command/command.go b/internal/command/command.go new file mode 100644 index 0000000..65a3781 --- /dev/null +++ b/internal/command/command.go @@ -0,0 +1,32 @@ +package command + +import ( + "os/exec" + "runtime" + + "golang.org/x/sys/unix" +) + +// Run runs an OS command. On Linux it waits for the command to +// complete and returns the exit status (return code). +func Run(name string, arg ...string) (string, int, error) { + cmd := exec.Command(name, arg...) + // Run the command and wait for completion + out, err := cmd.CombinedOutput() + if err != nil { + var rc int + // Only works on Linux + if runtime.GOOS == "linux" { + var ws unix.WaitStatus + unix.Wait4(cmd.Process.Pid, &ws, 0, nil) + rc = ws.ExitStatus() + } else { + rc = -1 + } + if rc == 0 { + return string(out), rc, nil + } + return string(out), rc, err + } + return string(out), 0, nil +} diff --git a/pkg/name/name.go b/internal/name/name.go similarity index 95% rename from pkg/name/name.go rename to internal/name/name.go index 449f905..0a97fd2 100644 --- a/pkg/name/name.go +++ b/internal/name/name.go @@ -20,11 +20,9 @@ package name import ( "os" "regexp" - //log "github.com/sirupsen/logrus" ) // sanitizeQueueManagerName removes any invalid characters from a queue manager name -// TODO: This is duplicate code func sanitizeQueueManagerName(name string) string { var re = regexp.MustCompile("[^a-zA-Z0-9._%/]") return re.ReplaceAllString(name, "") diff --git a/pkg/name/name_test.go b/internal/name/name_test.go similarity index 100% rename from pkg/name/name_test.go rename to internal/name/name_test.go From 5e787ba4cf91f4ab86e547525cf5f2c8686298fe Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 21 Nov 2017 15:28:03 +0000 Subject: [PATCH 03/11] Clean up build network on failure --- Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f777d1a..1662bff 100644 --- a/Makefile +++ b/Makefile @@ -127,11 +127,7 @@ define docker-build-mq --label IBM_PRODUCT_NAME=$5 \ --label IBM_PRODUCT_VERSION=$6 \ --build-arg MQ_PACKAGES="$(MQ_PACKAGES)" \ - . - # Stop the web server (will also remove the container) - $(DOCKER) kill $(BUILD_SERVER_CONTAINER) - # Delete the temporary network - $(DOCKER) network rm build + . ; $(DOCKER) kill $(BUILD_SERVER_CONTAINER) && $(DOCKER) network rm build endef # .PHONY: build-advancedserver-903 From 46110436b8e6b6d06b976620caa809ad578a436f Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 21 Nov 2017 15:53:14 +0000 Subject: [PATCH 04/11] Change command exit status handling --- internal/command/command.go | 43 +++++++++++++++++++---------- internal/command/command_test.go | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 internal/command/command_test.go diff --git a/internal/command/command.go b/internal/command/command.go index 65a3781..14d0c9e 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -1,32 +1,47 @@ +/* +© Copyright IBM Corporation 2017 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package command contains code to run external commands package command import ( "os/exec" "runtime" - - "golang.org/x/sys/unix" + "syscall" ) // Run runs an OS command. On Linux it waits for the command to // complete and returns the exit status (return code). +// Do not use this function to run shell built-ins (like "cd"), because +// the error handling works differently func Run(name string, arg ...string) (string, int, error) { cmd := exec.Command(name, arg...) // Run the command and wait for completion out, err := cmd.CombinedOutput() if err != nil { - var rc int - // Only works on Linux - if runtime.GOOS == "linux" { - var ws unix.WaitStatus - unix.Wait4(cmd.Process.Pid, &ws, 0, nil) - rc = ws.ExitStatus() - } else { - rc = -1 + // Assert that this is an ExitError + exiterr, ok := err.(*exec.ExitError) + // If the type assertion was correct, and we're on Linux + if ok && runtime.GOOS == "linux" { + status, ok := exiterr.Sys().(syscall.WaitStatus) + if ok { + return string(out), status.ExitStatus(), err + } } - if rc == 0 { - return string(out), rc, nil - } - return string(out), rc, err + return string(out), -1, err } return string(out), 0, nil } diff --git a/internal/command/command_test.go b/internal/command/command_test.go new file mode 100644 index 0000000..ccd5c57 --- /dev/null +++ b/internal/command/command_test.go @@ -0,0 +1,47 @@ +/* +© Copyright IBM Corporation 2017 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package command + +import ( + "runtime" + "testing" +) + +var commandTests = []struct { + name string + arg []string + rc int +}{ + {"ls", []string{}, 0}, + {"ls", []string{"madeup"}, 2}, + {"bash", []string{"-c", "exit 99"}, 99}, +} + +func TestRun(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping tests for package which only works on Linux") + } + for _, table := range commandTests { + arg := table.arg + _, rc, err := Run(table.name, arg...) + if rc != table.rc { + t.Errorf("Run(%v,%v) - expected %v, got %v", table.name, table.arg, table.rc, rc) + } + if rc != 0 && err == nil { + t.Errorf("Run(%v,%v) - expected error for non-zero return code (rc=%v)", table.name, table.arg, rc) + } + } +} From fef9ab4f0634c5f0a72148e0e96b73f7de63e969 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 22 Nov 2017 16:13:30 +0000 Subject: [PATCH 05/11] Fix runmqserver error handling --- cmd/runmqserver/license.go | 91 ++++++++++++ cmd/runmqserver/main.go | 216 ++++++++++------------------ cmd/runmqserver/signals.go | 79 ++++++++++ test/docker/docker_api_test_util.go | 8 +- 4 files changed, 252 insertions(+), 142 deletions(-) create mode 100644 cmd/runmqserver/license.go create mode 100644 cmd/runmqserver/signals.go diff --git a/cmd/runmqserver/license.go b/cmd/runmqserver/license.go new file mode 100644 index 0000000..c275a9f --- /dev/null +++ b/cmd/runmqserver/license.go @@ -0,0 +1,91 @@ +/* +© Copyright IBM Corporation 2017 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "errors" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +// resolveLicenseFile returns the file name of the MQ license file, taking into +// account the language set by the LANG environment variable +func resolveLicenseFile() string { + lang, ok := os.LookupEnv("LANG") + if !ok { + return "English.txt" + } + switch { + case strings.HasPrefix(lang, "zh_TW"): + return "Chinese_TW.txt" + case strings.HasPrefix(lang, "zh"): + return "Chinese.txt" + case strings.HasPrefix(lang, "cs"): + return "Czech.txt" + case strings.HasPrefix(lang, "fr"): + return "French.txt" + case strings.HasPrefix(lang, "de"): + return "German.txt" + case strings.HasPrefix(lang, "el"): + return "Greek.txt" + case strings.HasPrefix(lang, "id"): + return "Indonesian.txt" + case strings.HasPrefix(lang, "it"): + return "Italian.txt" + case strings.HasPrefix(lang, "ja"): + return "Japanese.txt" + case strings.HasPrefix(lang, "ko"): + return "Korean.txt" + case strings.HasPrefix(lang, "lt"): + return "Lithuanian.txt" + case strings.HasPrefix(lang, "pl"): + return "Polish.txt" + case strings.HasPrefix(lang, "pt"): + return "Portugese.txt" + case strings.HasPrefix(lang, "ru"): + return "Russian.txt" + case strings.HasPrefix(lang, "sl"): + return "Slovenian.txt" + case strings.HasPrefix(lang, "es"): + return "Spanish.txt" + case strings.HasPrefix(lang, "tr"): + return "Turkish.txt" + } + return "English.txt" +} + +func checkLicense() (bool, error) { + lic, ok := os.LookupEnv("LICENSE") + switch { + case ok && lic == "accept": + return true, nil + case ok && lic == "view": + file := filepath.Join("/opt/mqm/licenses", resolveLicenseFile()) + buf, err := ioutil.ReadFile(file) + if err != nil { + log.Println(err) + return false, err + } + log.Println(string(buf)) + return false, nil + } + log.Println("Error: Set environment variable LICENSE=accept to indicate acceptance of license terms and conditions.") + log.Println("License agreements and information can be viewed by setting the environment variable LICENSE=view. You can also set the LANG environment variable to view the license in a different language.") + return false, errors.New("Set environment variable LICENSE=accept to indicate acceptance of license terms and conditions") +} diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 3e47ef9..173fbbc 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -18,135 +18,73 @@ limitations under the License. package main import ( - "fmt" + "errors" "io/ioutil" "log" "os" "os/exec" - "os/signal" "path/filepath" "strings" - "syscall" "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/name" - "golang.org/x/sys/unix" ) -// resolveLicenseFile returns the file name of the MQ license file, taking into -// account the language set by the LANG environment variable -func resolveLicenseFile() string { - lang, ok := os.LookupEnv("LANG") - if !ok { - return "English.txt" - } - switch { - case strings.HasPrefix(lang, "zh_TW"): - return "Chinese_TW.txt" - case strings.HasPrefix(lang, "zh"): - return "Chinese.txt" - case strings.HasPrefix(lang, "cs"): - return "Czech.txt" - case strings.HasPrefix(lang, "fr"): - return "French.txt" - case strings.HasPrefix(lang, "de"): - return "German.txt" - case strings.HasPrefix(lang, "el"): - return "Greek.txt" - case strings.HasPrefix(lang, "id"): - return "Indonesian.txt" - case strings.HasPrefix(lang, "it"): - return "Italian.txt" - case strings.HasPrefix(lang, "ja"): - return "Japanese.txt" - case strings.HasPrefix(lang, "ko"): - return "Korean.txt" - case strings.HasPrefix(lang, "lt"): - return "Lithuanian.txt" - case strings.HasPrefix(lang, "pl"): - return "Polish.txt" - case strings.HasPrefix(lang, "pt"): - return "Portugese.txt" - case strings.HasPrefix(lang, "ru"): - return "Russian.txt" - case strings.HasPrefix(lang, "sl"): - return "Slovenian.txt" - case strings.HasPrefix(lang, "es"): - return "Spanish.txt" - case strings.HasPrefix(lang, "tr"): - return "Turkish.txt" - } - return "English.txt" -} - -func checkLicense() { - lic, ok := os.LookupEnv("LICENSE") - switch { - case ok && lic == "accept": - return - case ok && lic == "view": - file := filepath.Join("/opt/mqm/licenses", resolveLicenseFile()) - buf, err := ioutil.ReadFile(file) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - fmt.Println(string(buf)) - os.Exit(1) - } - fmt.Println("Error: Set environment variable LICENSE=accept to indicate acceptance of license terms and conditions.") - fmt.Println("License agreements and information can be viewed by setting the environment variable LICENSE=view. You can also set the LANG environment variable to view the license in a different language.") - os.Exit(1) -} - // createDirStructure creates the default MQ directory structure under /var/mqm -func createDirStructure() { +func createDirStructure() error { out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s") if err != nil { - log.Fatalf("Error creating directory structure: %v\n", string(out)) + log.Printf("Error creating directory structure: %v\n", string(out)) + return err } log.Println("Created directory structure under /var/mqm") + return nil } -func createQueueManager(name string) { +func createQueueManager(name string) error { log.Printf("Creating queue manager %v", name) out, rc, err := command.Run("crtmqm", "-q", "-p", "1414", name) if err != nil { // 8=Queue manager exists, which is fine if rc != 8 { log.Printf("crtmqm returned %v", rc) - log.Fatalln(string(out)) - } else { - log.Printf("Detected existing queue manager %v", name) - return + log.Println(string(out)) + return err } + log.Printf("Detected existing queue manager %v", name) } + return nil } -func updateCommandLevel() { +func updateCommandLevel() error { level, ok := os.LookupEnv("MQ_CMDLEVEL") if ok && level != "" { out, rc, err := command.Run("strmqm", "-e", "CMDLEVEL="+level) if err != nil { - log.Fatalf("Error %v setting CMDLEVEL: %v", rc, string(out)) + log.Printf("Error %v setting CMDLEVEL: %v", rc, string(out)) + return err } } + return nil } -func startQueueManager() { +func startQueueManager() error { log.Println("Starting queue manager") out, rc, err := command.Run("strmqm") if err != nil { - log.Fatalf("Error %v starting queue manager: %v", rc, string(out)) + log.Printf("Error %v starting queue manager: %v", rc, string(out)) + return err } log.Println("Started queue manager") + return nil } -func configureQueueManager() { +func configureQueueManager() error { const configDir string = "/etc/mqm" files, err := ioutil.ReadDir(configDir) if err != nil { - log.Fatal(err) + log.Println(err) + return err } for _, file := range files { @@ -154,12 +92,14 @@ func configureQueueManager() { abs := filepath.Join(configDir, file.Name()) mqsc, err := ioutil.ReadFile(abs) if err != nil { - log.Fatal(err) + log.Println(err) + return err } cmd := exec.Command("runmqsc") stdin, err := cmd.StdinPipe() if err != nil { - log.Fatal(err) + log.Println(err) + return err } stdin.Write(mqsc) stdin.Close() @@ -172,78 +112,76 @@ func configureQueueManager() { log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) } } + return nil } -func stopQueueManager() { +func stopQueueManager(name string) error { log.Println("Stopping queue manager") - out, _, err := command.Run("endmqm", "-w") + out, _, err := command.Run("endmqm", "-w", name) if err != nil { - log.Fatalf("Error stopping queue manager: %v", string(out)) + log.Printf("Error stopping queue manager: %v", string(out)) + return err } log.Println("Stopped queue manager") + return nil } -// createTerminateChannel creates a channel which will be closed when SIGTERM -// is received. -func createTerminateChannel() chan struct{} { - done := make(chan struct{}) - // Handle SIGTERM - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGTERM, syscall.SIGINT) - go func() { - sig := <-c - log.Printf("Signal received: %v", sig) - stopQueueManager() - close(done) - }() - return done -} - -// createReaperChannel creates a channel which will be used to reap zombie -// (defunct) processes. This is a responsibility of processes running -// as PID 1. -func createReaper() { - // Handle SIGCHLD - c := make(chan os.Signal, 3) - signal.Notify(c, syscall.SIGCHLD) - go func() { - for { - <-c - for { - var ws unix.WaitStatus - _, err := unix.Wait4(-1, &ws, 0, nil) - // If err indicates "no child processes" left to reap, then - // wait for next SIGCHLD signal - if err == unix.ECHILD { - break - } - } - } - }() -} - -func main() { - createReaper() - checkLicense() - // Start SIGTERM handler channel - done := createTerminateChannel() +func doMain() error { + accepted, err := checkLicense() + if err != nil { + return err + } + if !accepted { + return errors.New("License not accepted") + } name, err := name.GetQueueManagerName() if err != nil { - log.Fatalln(err) + log.Println(err) + return err } log.Printf("Using queue manager name: %v", name) + // Start signal handler + signalControl := signalHandler(name) + logConfig() err = createVolume("/mnt/mqm") if err != nil { - log.Fatal(err) + log.Println(err) + return err + } + err = createDirStructure() + if err != nil { + return err + } + err = createQueueManager(name) + if err != nil { + return err + } + err = updateCommandLevel() + if err != nil { + return err + } + err = startQueueManager() + if err != nil { + return err } - createDirStructure() - createQueueManager(name) - updateCommandLevel() - startQueueManager() configureQueueManager() + // Start reaping zombies from now on. + // Start this here, so that we don't reap any sub-processes created + // by this process (e.g. for crtmqm or strmqm) + signalControl <- startReaping + // Reap zombies now, just in case we've already got some + signalControl <- reapNow // Wait for terminate signal - <-done + <-signalControl + return nil +} + +func main() { + err := doMain() + if err != nil { + os.Exit(1) + } } diff --git a/cmd/runmqserver/signals.go b/cmd/runmqserver/signals.go new file mode 100644 index 0000000..04c686d --- /dev/null +++ b/cmd/runmqserver/signals.go @@ -0,0 +1,79 @@ +/* +© Copyright IBM Corporation 2017 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "log" + "os" + "os/signal" + "syscall" + + "golang.org/x/sys/unix" +) + +const ( + startReaping = iota + reapNow = iota +) + +func signalHandler(qmgr string) chan int { + control := make(chan int) + // Use separate channels for the signals, to avoid SIGCHLD signals swamping + // the buffer, and preventing other signals. + stopSignals := make(chan os.Signal) + reapSignals := make(chan os.Signal) + signal.Notify(stopSignals, syscall.SIGTERM, syscall.SIGINT) + go func() { + for { + select { + case sig := <-stopSignals: + log.Printf("Signal received: %v", sig) + signal.Stop(reapSignals) + signal.Stop(stopSignals) + stopQueueManager(qmgr) + // One final reap + reapZombies() + close(control) + // End the goroutine + return + case <-reapSignals: + reapZombies() + case job := <-control: + switch { + case job == startReaping: + // Add SIGCHLD to the list of signals we're listening to + signal.Notify(reapSignals, syscall.SIGCHLD) + case job == reapNow: + reapZombies() + } + } + } + }() + return control +} + +// reapZombies reaps any zombie (terminated) processes now. +// This function should be called before exiting. +func reapZombies() { + for { + var ws unix.WaitStatus + pid, err := unix.Wait4(-1, &ws, unix.WNOHANG, nil) + // If err or pid indicate "no child processes" + if pid == 0 || err == unix.ECHILD { + return + } + } +} diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index d67111d..5af6716 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -61,13 +61,15 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { // Log the results and continue t.Logf("Inspected container %v: %#v", ID, i) } - t.Logf("Killing container: %v", ID) - // Kill the container. This allows the coverage output to be generated. - err = cli.ContainerKill(context.Background(), ID, "SIGTERM") + t.Logf("Stopping container: %v", ID) + timeout := 10 * time.Second + // Stop the container. This allows the coverage output to be generated. + err = cli.ContainerStop(context.Background(), ID, &timeout) if err != nil { // Just log the error and continue t.Log(err) } + t.Log("Container stopped") // If a code coverage file has been generated, then rename it to match the test name os.Rename(filepath.Join(coverageDir(t), "container.cov"), filepath.Join(coverageDir(t), t.Name()+".cov")) // Log the container output for any container we're about to delete From d67433457486fd4a2c27ba9df6d6dbfa50e62526 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 22 Nov 2017 16:14:16 +0000 Subject: [PATCH 06/11] Improvements to coverage reporting --- Dockerfile-server.cover | 4 ++-- Makefile | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Dockerfile-server.cover b/Dockerfile-server.cover index e27550a..621390b 100644 --- a/Dockerfile-server.cover +++ b/Dockerfile-server.cover @@ -16,9 +16,9 @@ FROM golang:1.9 as builder WORKDIR /go/src/github.com/ibm-messaging/mq-container/ COPY cmd/ ./cmd -COPY pkg/ ./pkg +COPY internal/ ./internal COPY vendor/ ./vendor -RUN go test -c -covermode=count ./cmd/runmqserver +RUN go test -c -covermode=count -coverpkg $(go list ./cmd/runmqserver ./internal/... | paste -s -d, -) ./cmd/runmqserver FROM mq-advancedserver:latest-x86_64 diff --git a/Makefile b/Makefile index 1662bff..69e62ef 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ clean: downloads/mqadv_dev903_ubuntu_x86-64.tar.gz: $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers"$(END))) mkdir -p downloads - cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev903_ubuntu_x86-64.tar.gz + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev904_ubuntu_x86-64.tar.gz .PHONY: downloads downloads: downloads/mqadv_dev903_ubuntu_x86-64.tar.gz @@ -72,18 +72,24 @@ build-cov: .PHONY: test-advancedserver test-advancedserver: - cd pkg/name && go test + $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test $(TEST_OPTS_DOCKER) .PHONY: test-devserver test-devserver: - $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER)"$(END))) - cd pkg/name && go test + $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test .PHONY: test-advancedserver-cover test-advancedserver-cover: - cd pkg/name && go test + $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_REPO_ADVANCEDSERVER) on Docker with code coverage"$(END))) + rm -f ./coverage/unit*.cov + # Run unit tests with coverage, for each package under 'internal' + go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{} + echo 'mode: count' > ./coverage/unit.cov + tail -q -n +2 ./coverage/unit-*.cov >> ./coverage/unit.cov + go tool cover -html=./coverage/unit.cov -o ./coverage/unit.html + rm -f ./test/docker/coverage/*.cov rm -f ./coverage/docker.* cd test/docker && TEST_IMAGE=$(DOCKER_REPO_ADVANCEDSERVER):cover go test $(TEST_OPTS_DOCKER) @@ -91,6 +97,10 @@ test-advancedserver-cover: tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html + echo 'mode: count' > ./coverage/combined.cov + tail -q -n +2 ./coverage/unit.cov ./coverage/docker.cov >> ./coverage/combined.cov + go tool cover -html=./coverage/combined.cov -o ./coverage/combined.html + .PHONY: test-kubernetes-devserver test-kubernetes-devserver: $(call test-kubernetes,$(DOCKER_REPO_DEVSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-dev") From f4921712763304202373bf53c4ee206d83e5d6e0 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 22 Nov 2017 16:14:34 +0000 Subject: [PATCH 07/11] Test extra error paths --- test/docker/docker_api_test.go | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index c4701fb..22e54dd 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -167,3 +167,55 @@ func TestNoVolumeWithRestart(t *testing.T) { startContainer(t, cli, id) waitForReady(t, cli, id) } + +// Test the case where `crtmqm` will fail +func TestCreateQueueManagerFail(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + img, _, err := cli.ImageInspectWithRaw(context.Background(), imageName()) + oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + //ExposedPorts: ports, + ExposedPorts: nat.PortSet{ + "1414/tcp": struct{}{}, + }, + // Override the entrypoint to create the queue manager directory, but leave it empty. + // This will cause `crtmqm` to return with an exit code of 2. + Entrypoint: []string{"bash", "-c", "mkdir -p /mnt/mqm/data && mkdir -p /var/mqm/qmgrs/qm1 && exec " + oldEntrypoint}, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + rc := waitForContainer(t, cli, id, 10) + if rc != 1 { + t.Errorf("Expected rc=1, got rc=%v", rc) + } +} + +// Test the case where `strmqm` will fail +func TestStartQueueManagerFail(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + img, _, err := cli.ImageInspectWithRaw(context.Background(), imageName()) + oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + //ExposedPorts: ports, + ExposedPorts: nat.PortSet{ + "1414/tcp": struct{}{}, + }, + // Override the entrypoint to replace `crtmqm` with a no-op script. + // This will cause `strmqm` to return with an exit code of 16. + Entrypoint: []string{"bash", "-c", "echo '#!/bin/bash\n' > /opt/mqm/bin/crtmqm && exec " + oldEntrypoint}, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + rc := waitForContainer(t, cli, id, 10) + if rc != 1 { + t.Errorf("Expected rc=1, got rc=%v", rc) + } +} From 4200d9df1bdd55952eac34386ec7b74fdd0852c9 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 22 Nov 2017 17:20:57 +0000 Subject: [PATCH 08/11] Enable coverage for failure cases --- cmd/runmqserver/main.go | 4 +++- cmd/runmqserver/main_test.go | 12 ++++++++++++ test/docker/docker_api_test_util.go | 30 +++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 173fbbc..5df9608 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -179,9 +179,11 @@ func doMain() error { return nil } +var osExit = os.Exit + func main() { err := doMain() if err != nil { - os.Exit(1) + osExit(1) } } diff --git a/cmd/runmqserver/main_test.go b/cmd/runmqserver/main_test.go index a60a5e2..2a78204 100644 --- a/cmd/runmqserver/main_test.go +++ b/cmd/runmqserver/main_test.go @@ -17,6 +17,9 @@ package main import ( "flag" + "io/ioutil" + "log" + "strconv" "testing" ) @@ -29,6 +32,15 @@ func init() { // Test started when the test binary is started. Only calls main. func TestSystem(t *testing.T) { if *test { + var oldExit = osExit + defer func() { + osExit = oldExit + }() + osExit = func(rc int) { + // Write the exit code to a file instead + log.Printf("Writing exit code %v to file", strconv.Itoa(rc)) + ioutil.WriteFile("/var/coverage/exitCode", []byte(strconv.Itoa(rc)), 0644) + } main() } } diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 5af6716..f9100ca 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -22,6 +22,7 @@ import ( "log" "os" "path/filepath" + "strconv" "testing" "time" @@ -136,11 +137,40 @@ func stopContainer(t *testing.T, cli *client.Client, ID string) { } } +func getCoverageExitCode(t *testing.T, orig int64) int64 { + f := filepath.Join(coverageDir(t), "exitCode") + _, err := os.Stat(f) + if err != nil { + t.Log(err) + return orig + } + // Remove the file, ready for the next test + //defer os.Remove(f) + buf, err := ioutil.ReadFile(f) + if err != nil { + t.Log(err) + return orig + } + rc, err := strconv.Atoi(string(buf)) + if err != nil { + t.Log(err) + return orig + } + t.Logf("Retrieved exit code %v from file", rc) + return int64(rc) +} + // waitForContainer waits until a container has exited func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64) int64 { //ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) //defer cancel() rc, err := cli.ContainerWait(context.Background(), ID) + + // COVERAGE: When running coverage, the exit code is written to a file, + // to allow the coverage to be generated (which doesn't happen for non-zero + // exit codes) + rc = getCoverageExitCode(t, rc) + // err := <-errC if err != nil { t.Fatal(err) From 4aa8918745a7c91ab94e08562153408ee700ac04 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 28 Nov 2017 12:05:47 +0000 Subject: [PATCH 09/11] MQ Advanced for Developers V9.0.4 --- Makefile | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 69e62ef..9fb5216 100644 --- a/Makefile +++ b/Makefile @@ -51,13 +51,13 @@ clean: rm -rf ./build rm -rf ./deps -downloads/mqadv_dev903_ubuntu_x86-64.tar.gz: +downloads/mqadv_dev904_ubuntu_x86-64.tar.gz: $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers"$(END))) mkdir -p downloads cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev904_ubuntu_x86-64.tar.gz .PHONY: downloads -downloads: downloads/mqadv_dev903_ubuntu_x86-64.tar.gz +downloads: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz .PHONY: deps deps: @@ -140,31 +140,20 @@ define docker-build-mq . ; $(DOCKER) kill $(BUILD_SERVER_CONTAINER) && $(DOCKER) network rm build endef -# .PHONY: build-advancedserver-903 -# build-advancedserver-903: build downloads/CNJR7ML.tar.gz -# $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) -# $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,CNJR7ML.tar.gz,"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced","9.0.3") -# $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):9.0.3-$(DOCKER_TAG_ARCH) - .PHONY: build-advancedserver-904 build-advancedserver-904: downloads/CNLE4ML.tar.gz $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,CNLE4ML.tar.gz,"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced","9.0.4") - $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):9.0.4-$(DOCKER_TAG_ARCH) + $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):9.0.4.0-$(DOCKER_TAG_ARCH) .PHONY: build-advancedserver build-advancedserver: build-advancedserver-904 .PHONY: build-devserver -build-devserver: downloads/mqadv_dev903_ubuntu_x86-64.tar.gz +build-devserver: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) - $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,mqadv_dev903_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.3") - $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):9.0.3-$(DOCKER_TAG_ARCH) - -# .PHONY: build-server -# build-server: build downloads/CNJR7ML.tar.gz -# $(call docker-build-mq,mq-server:latest-$(DOCKER_TAG_ARCH),Dockerfile-server,"79afd716d55b4f149a87bec52c9dc1aa","IBM MQ","9.0.3") -# $(DOCKER) tag mq-server:latest-$(DOCKER_TAG_ARCH) mq-server:9.0.3-$(DOCKER_TAG_ARCH) + $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,mqadv_dev904_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.4") + $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):9.0.4.0-$(DOCKER_TAG_ARCH) .PHONY: build-advancedserver-cover build-advancedserver-cover: @@ -175,7 +164,7 @@ build-advancedserver-cover: # $(call docker-build-mq,mq-web:latest-$(DOCKER_TAG_ARCH),Dockerfile-mq-web) .PHONY: build-explorer -build-explorer: downloads/mqadv_dev903_ubuntu_x86-64.tar.gz - $(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,mqadv_dev903_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.3") +build-explorer: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz + $(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,mqadv_dev904_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.4") include formatting.mk From 7ea6b4c610aabcfc98300b766223a85fa3fba9f1 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 28 Nov 2017 12:08:19 +0000 Subject: [PATCH 10/11] Detect arch for image tag --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9fb5216..ecbd85f 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,7 @@ # limitations under the License. BUILD_SERVER_CONTAINER=build-server -# Set architecture for Go code. Don't set GOOS globally, so that tests can be run locally -export GOARCH ?= amd64 -DOCKER_TAG_ARCH ?= x86_64 +DOCKER_TAG_ARCH ?= $(shell uname -m) # By default, all Docker client commands are run inside a Docker container. # This means that newer features of the client can be used, even with an older daemon. DOCKER ?= docker run --tty --interactive --rm --volume /var/run/docker.sock:/var/run/docker.sock --volume "$(CURDIR)":/var/src --workdir /var/src docker:stable docker @@ -151,9 +149,13 @@ build-advancedserver: build-advancedserver-904 .PHONY: build-devserver build-devserver: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz +ifneq "x86_64" "$(shell uname -m)" +    $(error MQ Advanced for Developers is only available for x86_64 architecture) +else $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,mqadv_dev904_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.4") $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):9.0.4.0-$(DOCKER_TAG_ARCH) +endif .PHONY: build-advancedserver-cover build-advancedserver-cover: From 5780ec2a2658e2208b5af166120deca65939fdd5 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 28 Nov 2017 14:52:41 +0000 Subject: [PATCH 11/11] Easier multi-arch or multi-version builds --- Makefile | 42 +++++++++++++++++++++++++++--------------- docs/developing.md | 23 ++++++++++++++++++----- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index ecbd85f..b962c4e 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,22 @@ DOCKER_REPO_DEVSERVER ?= mq-devserver DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver DOCKER_FULL_DEVSERVER = $(DOCKER_REPO_DEVSERVER):$(DOCKER_TAG) DOCKER_FULL_ADVANCEDSERVER = $(DOCKER_REPO_ADVANCEDSERVER):$(DOCKER_TAG) +# MQ_PACKAGES is the list of MQ packages to install MQ_PACKAGES ?=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams +# MQ_VERSION is the fully qualified MQ version number to build +MQ_VERSION ?= 9.0.4.0 +# Archive names for IBM MQ Continuous Delivery Release for Ubuntu +MQ_ARCHIVE_9.0.3.0_ppc64le=CNJR5ML.tar.gz +MQ_ARCHIVE_9.0.3.0_s390x=CNJR6ML.tar.gz +MQ_ARCHIVE_9.0.3.0_x86_64=CNJR7ML.tar.gz +MQ_ARCHIVE_9.0.4.0_ppc64le=CNLE2ML.tar.gz +MQ_ARCHIVE_9.0.4.0_s390x=CNLE3ML.tar.gz +MQ_ARCHIVE_9.0.4.0_x86_64=CNLE4ML.tar.gz +# Archive names for IBM MQ Advanced for Developers for Ubuntu +MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz +MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz +MQ_ARCHIVE ?= $(MQ_ARCHIVE_$(MQ_VERSION)_$(DOCKER_TAG_ARCH)) +MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # Options to `go test` for the Kubernetes tests @@ -49,13 +64,13 @@ clean: rm -rf ./build rm -rf ./deps -downloads/mqadv_dev904_ubuntu_x86-64.tar.gz: +downloads/$(MQ_ARCHIVE_DEV): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers"$(END))) mkdir -p downloads - cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev904_ubuntu_x86-64.tar.gz + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) .PHONY: downloads -downloads: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz +downloads: downloads/$(MQ_ARCHIVE_DEV) .PHONY: deps deps: @@ -138,23 +153,20 @@ define docker-build-mq . ; $(DOCKER) kill $(BUILD_SERVER_CONTAINER) && $(DOCKER) network rm build endef -.PHONY: build-advancedserver-904 -build-advancedserver-904: downloads/CNLE4ML.tar.gz - $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) - $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,CNLE4ML.tar.gz,"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced","9.0.4") - $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):9.0.4.0-$(DOCKER_TAG_ARCH) - .PHONY: build-advancedserver -build-advancedserver: build-advancedserver-904 +build-advancedserver: downloads/$(MQ_ARCHIVE) + $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) + $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) + $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) .PHONY: build-devserver -build-devserver: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz +build-devserver: downloads/$(MQ_ARCHIVE_DEV) ifneq "x86_64" "$(shell uname -m)"     $(error MQ Advanced for Developers is only available for x86_64 architecture) else $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) - $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,mqadv_dev904_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.4") - $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):9.0.4.0-$(DOCKER_TAG_ARCH) + $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) + $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) endif .PHONY: build-advancedserver-cover @@ -166,7 +178,7 @@ build-advancedserver-cover: # $(call docker-build-mq,mq-web:latest-$(DOCKER_TAG_ARCH),Dockerfile-mq-web) .PHONY: build-explorer -build-explorer: downloads/mqadv_dev904_ubuntu_x86-64.tar.gz - $(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,mqadv_dev904_ubuntu_x86-64.tar.gz,"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)","9.0.4") +build-explorer: downloads/$(MQ_ARCHIVE_DEV) + $(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) include formatting.mk diff --git a/docs/developing.md b/docs/developing.md index 048c522..20bd753 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -13,19 +13,32 @@ You need to ensure you have the following tools installed: For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). ## Building a production image +This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. -1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4) in the `downloads` directory +1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4 on x86_64 architecture) in the `downloads` directory 2. Run `make build-advancedserver` +You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example: + +```bash +MQ_VERSION=9.0.3.0 make build-advancedserver +``` + ## Running the tests There are three main sets of tests: -1. Unit tests +1. Unit tests, which are run during a build 2. Docker tests, which test a complete Docker image, using the Docker API 3. Kubernetes tests, which test the Helm charts (and the Docker image) via [Helm](https://helm.sh) -### Running the tests -The unit and Docker tests can be run locally. For example: +### Running the Docker tests +The Docker tests can be run locally. Before you run them for the first time, you need to download the test dependencies: + +``` +make deps +``` + +You can then run the tests, for example: ``` make test-devserver @@ -37,7 +50,7 @@ or: make test-advancedserver ``` -### Running the tests with code coverage +### Running the Docker tests with code coverage You can produce code coverage results from the Docker tests by running the following: ```