From 0823fd1ceacbc99f174329813c6c5aaaabadcfed Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 12:29:31 +0000 Subject: [PATCH 01/17] Add TestVolumeUnmount --- test/docker/docker_api_test.go | 46 +++++++++++++++++- test/docker/docker_api_test_util.go | 75 ++++++++++++++++------------- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 22e54dd..297e1e6 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -19,6 +19,7 @@ import ( "context" "strings" "testing" + "time" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" @@ -94,7 +95,7 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) waitForReady(t, cli, id) - _, out := execContainer(t, cli, id, []string{"dspmq"}) + out := execContainerWithOutput(t, cli, id, "mqm", []string{"dspmq"}) if !strings.Contains(out, search) { t.Errorf("Expected result of running dspmq to contain name=%v, got name=%v", search, out) } @@ -219,3 +220,46 @@ func TestStartQueueManagerFail(t *testing.T) { t.Errorf("Expected rc=1, got rc=%v", rc) } } + +func TestVolumeUnmount(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + vol := createVolume(t, cli) + defer removeVolume(t, cli, vol.Name) + containerConfig := container.Config{ + Image: imageName(), + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + } + hostConfig := container.HostConfig{ + // SYS_ADMIN capability is required to unmount file systems + CapAdd: []string{ + "SYS_ADMIN", + }, + Binds: []string{ + coverageBind(t), + vol.Name + ":/mnt/mqm", + }, + } + networkingConfig := network.NetworkingConfig{} + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()) + if err != nil { + t.Fatal(err) + } + startContainer(t, cli, ctr.ID) + defer cleanContainer(t, cli, ctr.ID) + waitForReady(t, cli, ctr.ID) + // Unmount the volume as root + rc := execContainerWithExitCode(t, cli, ctr.ID, "root", []string{"umount", "-l", "-f", "/mnt/mqm"}) + if rc != 0 { + t.Fatalf("Expected umount to work with rc=0, got %v", rc) + } + time.Sleep(3 * time.Second) + rc = execContainerWithExitCode(t, cli, ctr.ID, "mqm", []string{"chkmqhealthy"}) + if rc == 0 { + t.Errorf("Expected chkmqhealthy to fail") + t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"df"})) + t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"})) + } +} diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index f9100ca..d1a30da 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -179,11 +179,45 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64 return rc } -// execContainer runs the specified command inside the container, returning the -// exit code and the stdout/stderr string. -func execContainer(t *testing.T, cli *client.Client, ID string, cmd []string) (int, string) { +// execContainerWithExitCode runs a command in a running container, and returns the exit code +// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the +// container to get output. This is why these are two separate commands. +func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user string, cmd []string) int { config := types.ExecConfig{ - User: "mqm", + User: user, + Privileged: false, + Tty: false, + AttachStdin: false, + // Note that you still need to attach stdout/stderr, even though they're not wanted + AttachStdout: true, + AttachStderr: true, + Detach: false, + Cmd: cmd, + } + resp, err := cli.ContainerExecCreate(context.Background(), ID, config) + if err != nil { + t.Fatal(err) + } + cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ + Detach: false, + Tty: false, + }) + if err != nil { + t.Fatal(err) + } + inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) + if err != nil { + t.Fatal(err) + } + return inspect.ExitCode +} + +// execContainerWithOutput runs a command in a running container, and returns the output from stdout/stderr +// Note: due to a bug in Docker/Moby code, you always get an exit code of 0 if you attach to the +// container to get output. This is why these are two separate commands. +func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user string, cmd []string) string { + config := types.ExecConfig{ + User: user, Privileged: false, Tty: false, AttachStdin: false, @@ -207,46 +241,19 @@ func execContainer(t *testing.T, cli *client.Client, ID string, cmd []string) (i if err != nil { t.Fatal(err) } - inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) - if err != nil { - t.Fatal(err) - } // TODO: For some reason, each line seems to start with an extra, random character buf, err := ioutil.ReadAll(hijack.Reader) if err != nil { t.Fatal(err) } hijack.Close() - return inspect.ExitCode, string(buf) + return string(buf) } func waitForReady(t *testing.T, cli *client.Client, ID string) { for { - resp, err := cli.ContainerExecCreate(context.Background(), ID, types.ExecConfig{ - User: "mqm", - Privileged: false, - Tty: false, - AttachStdin: false, - AttachStdout: true, - AttachStderr: true, - Detach: false, - Cmd: []string{"chkmqready"}, - }) - if err != nil { - t.Fatal(err) - } - cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ - Detach: false, - Tty: false, - }) - if err != nil { - t.Fatal(err) - } - inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) - if err != nil { - t.Fatal(err) - } - if inspect.ExitCode == 0 { + rc := execContainerWithExitCode(t, cli, ID, "mqm", []string{"chkmqready"}) + if rc == 0 { t.Log("MQ is ready") return } From 0ad595dc995142af29bd77c134bd7235eaf95aff Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 12:29:59 +0000 Subject: [PATCH 02/17] Improve dependency handling --- .travis.yml | 5 +---- Makefile | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2973abe..49d86a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: - sudo apt-get update - sudo apt-get -y install docker-ce - curl https://glide.sh/get | sh - - curl -LO https://github.com/golang/dep/releases/download/v0.3.0/dep-linux-amd64.zip + - curl -LO https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64.zip - unzip dep-linux-amd64.zip - sudo mv dep /usr/local/bin - rm dep-linux-amd64.zip @@ -21,9 +21,6 @@ before_install: install: - echo nothing -before_script: - - make deps - script: - make build-devserver - make test-devserver diff --git a/Makefile b/Makefile index b962c4e..96fadc6 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,14 @@ downloads: downloads/$(MQ_ARCHIVE_DEV) .PHONY: deps deps: glide install --strip-vendor + +# Vendor Go dependencies for the Docker tests +test/docker/vendor: + cd test/docker && dep ensure -vendor-only + +# Vendor Go dependencies for the Kubernetes tests +test/kubernetes/vendor: cd test/docker && dep ensure -vendor-only - cd test/kubernetes && dep ensure -vendor-only .PHONY: build-cov build-cov: @@ -84,17 +90,17 @@ build-cov: cd build; go test -c -covermode=count ../cmd/runmqserver .PHONY: test-advancedserver -test-advancedserver: +test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test $(TEST_OPTS_DOCKER) .PHONY: test-devserver -test-devserver: +test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test .PHONY: test-advancedserver-cover -test-advancedserver-cover: +test-advancedserver-cover: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_REPO_ADVANCEDSERVER) on Docker with code coverage"$(END))) rm -f ./coverage/unit*.cov # Run unit tests with coverage, for each package under 'internal' @@ -115,11 +121,11 @@ test-advancedserver-cover: go tool cover -html=./coverage/combined.cov -o ./coverage/combined.html .PHONY: test-kubernetes-devserver -test-kubernetes-devserver: +test-kubernetes-devserver: test/kubernetes/vendor $(call test-kubernetes,$(DOCKER_REPO_DEVSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-dev") .PHONY: test-kubernetes-advancedserver -test-kubernetes-advancedserver: +test-kubernetes-advancedserver: test/kubernetes/vendor $(call test-kubernetes,$(DOCKER_REPO_ADVANCEDSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-prod") define test-kubernetes From 5b0259ec6eb02a9cd39c25d5e29bec00704ebe1a Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 12:36:48 +0000 Subject: [PATCH 03/17] Add test comments --- test/docker/docker_api_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 297e1e6..66bd7a4 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -62,6 +62,7 @@ func TestLicenseView(t *testing.T) { } } +// TestGoldenPath starts a queue manager successfully func TestGoldenPath(t *testing.T) { cli, err := client.NewEnvClient() if err != nil { @@ -149,6 +150,8 @@ func TestWithVolume(t *testing.T) { waitForReady(t, cli, ctr2.ID) } +// TestNoVolumeWithRestart ensures a queue manager container can be stopped +// and restarted cleanly func TestNoVolumeWithRestart(t *testing.T) { cli, err := client.NewEnvClient() if err != nil { @@ -169,7 +172,7 @@ func TestNoVolumeWithRestart(t *testing.T) { waitForReady(t, cli, id) } -// Test the case where `crtmqm` will fail +// TestCreateQueueManagerFail causes a failure of `crtmqm` func TestCreateQueueManagerFail(t *testing.T) { cli, err := client.NewEnvClient() if err != nil { @@ -195,7 +198,7 @@ func TestCreateQueueManagerFail(t *testing.T) { } } -// Test the case where `strmqm` will fail +// TestStartQueueManagerFail causes a failure of `strmqm` func TestStartQueueManagerFail(t *testing.T) { cli, err := client.NewEnvClient() if err != nil { @@ -221,6 +224,10 @@ func TestStartQueueManagerFail(t *testing.T) { } } +// TestVolumeUnmount runs a queue manager with a volume, and then forces an +// unmount of the volume. The health check should then fail. +// This simulates behaviour seen in some cloud environments, where network +// attached storage gets unmounted. func TestVolumeUnmount(t *testing.T) { cli, err := client.NewEnvClient() if err != nil { From a80b839c14e3f3a9dc8010e6234d4cc24d089174 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 15:50:10 +0000 Subject: [PATCH 04/17] Add TestZombies and improve exec output handling --- cmd/runmqserver/main.go | 19 +++++++++++++++ cmd/runmqserver/signals.go | 3 +++ test/docker/docker_api_test.go | 36 +++++++++++++++++++++++++++++ test/docker/docker_api_test_util.go | 19 +++++++++------ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 5df9608..3e2c5b9 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -19,6 +19,7 @@ package main import ( "errors" + "fmt" "io/ioutil" "log" "os" @@ -30,6 +31,20 @@ import ( "github.com/ibm-messaging/mq-container/internal/name" ) +var debug = false + +func logDebug(msg string) { + if debug { + log.Printf("DEBUG: %v", msg) + } +} + +func logDebugf(format string, args ...interface{}) { + if debug { + log.Printf("DEBUG: %v", fmt.Sprintf(format, args...)) + } +} + // createDirStructure creates the default MQ directory structure under /var/mqm func createDirStructure() error { out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s") @@ -127,6 +142,10 @@ func stopQueueManager(name string) error { } func doMain() error { + debugEnv, ok := os.LookupEnv("DEBUG") + if ok && (debugEnv == "true" || debugEnv == "1") { + debug = true + } accepted, err := checkLicense() if err != nil { return err diff --git a/cmd/runmqserver/signals.go b/cmd/runmqserver/signals.go index 04c686d..f3b42b9 100644 --- a/cmd/runmqserver/signals.go +++ b/cmd/runmqserver/signals.go @@ -50,11 +50,13 @@ func signalHandler(qmgr string) chan int { // End the goroutine return case <-reapSignals: + logDebug("Received SIGCHLD signal") reapZombies() case job := <-control: switch { case job == startReaping: // Add SIGCHLD to the list of signals we're listening to + logDebug("Listening for SIGCHLD signals") signal.Notify(reapSignals, syscall.SIGCHLD) case job == reapNow: reapZombies() @@ -75,5 +77,6 @@ func reapZombies() { if pid == 0 || err == unix.ECHILD { return } + logDebugf("Reaped PID %v", pid) } } diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 66bd7a4..73e6692 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -17,6 +17,7 @@ package main import ( "context" + "strconv" "strings" "testing" "time" @@ -270,3 +271,38 @@ func TestVolumeUnmount(t *testing.T) { t.Logf(execContainerWithOutput(t, cli, ctr.ID, "mqm", []string{"ps", "-ef"})) } } + +// TestZombies starts a queue manager, then causes a zombie process to be +// created, then checks that no zombies exist (runmqserver should reap them) +func TestZombies(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1", "DEBUG=true"}, + //ExposedPorts: ports, + ExposedPorts: nat.PortSet{ + "1414/tcp": struct{}{}, + }, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + // Kill an MQ process with children. After it is killed, its children + // will be adopted by PID 1, and should then be reaped when they die. + out := execContainerWithOutput(t, cli, id, "mqm", []string{"pkill", "--signal", "kill", "-c", "amqzxma0"}) + if out == "0" { + t.Fatalf("Expected pkill to kill a process, got %v", out) + } + time.Sleep(3 * time.Second) + // Create a zombie process for up to ten seconds + out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z' | wc -l"}) + count, err := strconv.Atoi(out) + if err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("Expected zombies=0, got %v", count) + } +} diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index d1a30da..7258b17 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "testing" "time" @@ -32,6 +33,7 @@ import ( "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" + "github.com/moby/moby/pkg/stdcopy" ) func imageName() string { @@ -241,13 +243,13 @@ func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user s if err != nil { t.Fatal(err) } - // TODO: For some reason, each line seems to start with an extra, random character - buf, err := ioutil.ReadAll(hijack.Reader) + buf := new(bytes.Buffer) + // Each output line has a header, which needs to be removed + _, err = stdcopy.StdCopy(buf, buf, hijack.Reader) if err != nil { - t.Fatal(err) + log.Fatal(err) } - hijack.Close() - return string(buf) + return strings.TrimSpace(buf.String()) } func waitForReady(t *testing.T, cli *client.Client, ID string) { @@ -320,8 +322,11 @@ func inspectLogs(t *testing.T, cli *client.Client, ID string) string { if err != nil { log.Fatal(err) } - buf := new(bytes.Buffer) - buf.ReadFrom(reader) + // Each output line has a header, which needs to be removed + _, err = stdcopy.StdCopy(buf, buf, reader) + if err != nil { + log.Fatal(err) + } return buf.String() } From 2f3ca70d7b3a30031bae7d37a8f8ccd8c1bbea9c Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 16:08:12 +0000 Subject: [PATCH 05/17] Document running a single test --- docs/developing.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/developing.md b/docs/developing.md index 20bd753..895d877 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -50,6 +50,12 @@ or: make test-advancedserver ``` +You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command:: + +``` +TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver +``` + ### Running the Docker tests with code coverage You can produce code coverage results from the Docker tests by running the following: From 81b5db4969aeab5371c68c126b7192b1ff450be1 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 29 Nov 2017 17:25:35 +0000 Subject: [PATCH 06/17] Add unit test for license resolution --- Makefile | 5 + cmd/runmqserver/license.go | 3 +- cmd/runmqserver/license_test.go | 280 ++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 cmd/runmqserver/license_test.go diff --git a/Makefile b/Makefile index 96fadc6..8b595de 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,11 @@ build-cov: mkdir -p build cd build; go test -c -covermode=count ../cmd/runmqserver +# Shortcut to just run the unit tests +.PHONY: test-unit +test-unit: + docker build --target builder --file Dockerfile-server . + .PHONY: test-advancedserver test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) diff --git a/cmd/runmqserver/license.go b/cmd/runmqserver/license.go index c275a9f..1bd6824 100644 --- a/cmd/runmqserver/license.go +++ b/cmd/runmqserver/license.go @@ -50,7 +50,8 @@ func resolveLicenseFile() string { return "Italian.txt" case strings.HasPrefix(lang, "ja"): return "Japanese.txt" - case strings.HasPrefix(lang, "ko"): + // Need to differentiate Korean (ko) from Konkani (kok) + case strings.HasPrefix(lang, "ko") && !strings.HasPrefix(lang, "kok"): return "Korean.txt" case strings.HasPrefix(lang, "lt"): return "Lithuanian.txt" diff --git a/cmd/runmqserver/license_test.go b/cmd/runmqserver/license_test.go new file mode 100644 index 0000000..aa36a28 --- /dev/null +++ b/cmd/runmqserver/license_test.go @@ -0,0 +1,280 @@ +/* +© Copyright IBM Corporation 2017 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE_2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "os" + "testing" +) + +var licenseTests = []struct { + in string + out string +}{ + {"en_US.UTF_8", "English.txt"}, + {"es_GB", "Spanish.txt"}, + {"el_ES.UTF_8", "Greek.txt"}, + // Cover a wide variety of valid values + {"af", "English.txt"}, + {"af_ZA", "English.txt"}, + {"ar", "English.txt"}, + {"ar_AE", "English.txt"}, + {"ar_BH", "English.txt"}, + {"ar_DZ", "English.txt"}, + {"ar_EG", "English.txt"}, + {"ar_IQ", "English.txt"}, + {"ar_JO", "English.txt"}, + {"ar_KW", "English.txt"}, + {"ar_LB", "English.txt"}, + {"ar_LY", "English.txt"}, + {"ar_MA", "English.txt"}, + {"ar_OM", "English.txt"}, + {"ar_QA", "English.txt"}, + {"ar_SA", "English.txt"}, + {"ar_SY", "English.txt"}, + {"ar_TN", "English.txt"}, + {"ar_YE", "English.txt"}, + {"az", "English.txt"}, + {"az_AZ", "English.txt"}, + {"az_AZ", "English.txt"}, + {"be", "English.txt"}, + {"be_BY", "English.txt"}, + {"bg", "English.txt"}, + {"bg_BG", "English.txt"}, + {"bs_BA", "English.txt"}, + {"ca", "English.txt"}, + {"ca_ES", "English.txt"}, + {"cs", "Czech.txt"}, + {"cs_CZ", "Czech.txt"}, + {"cy", "English.txt"}, + {"cy_GB", "English.txt"}, + {"da", "English.txt"}, + {"da_DK", "English.txt"}, + {"de", "German.txt"}, + {"de_AT", "German.txt"}, + {"de_CH", "German.txt"}, + {"de_DE", "German.txt"}, + {"de_LI", "German.txt"}, + {"de_LU", "German.txt"}, + {"dv", "English.txt"}, + {"dv_MV", "English.txt"}, + {"el", "Greek.txt"}, + {"el_GR", "Greek.txt"}, + {"en", "English.txt"}, + {"en_AU", "English.txt"}, + {"en_BZ", "English.txt"}, + {"en_CA", "English.txt"}, + {"en_CB", "English.txt"}, + {"en_GB", "English.txt"}, + {"en_IE", "English.txt"}, + {"en_JM", "English.txt"}, + {"en_NZ", "English.txt"}, + {"en_PH", "English.txt"}, + {"en_TT", "English.txt"}, + {"en_US", "English.txt"}, + {"en_ZA", "English.txt"}, + {"en_ZW", "English.txt"}, + {"eo", "English.txt"}, + {"es", "Spanish.txt"}, + {"es_AR", "Spanish.txt"}, + {"es_BO", "Spanish.txt"}, + {"es_CL", "Spanish.txt"}, + {"es_CO", "Spanish.txt"}, + {"es_CR", "Spanish.txt"}, + {"es_DO", "Spanish.txt"}, + {"es_EC", "Spanish.txt"}, + {"es_ES", "Spanish.txt"}, + {"es_ES", "Spanish.txt"}, + {"es_GT", "Spanish.txt"}, + {"es_HN", "Spanish.txt"}, + {"es_MX", "Spanish.txt"}, + {"es_NI", "Spanish.txt"}, + {"es_PA", "Spanish.txt"}, + {"es_PE", "Spanish.txt"}, + {"es_PR", "Spanish.txt"}, + {"es_PY", "Spanish.txt"}, + {"es_SV", "Spanish.txt"}, + {"es_UY", "Spanish.txt"}, + {"es_VE", "Spanish.txt"}, + {"et", "English.txt"}, + {"et_EE", "English.txt"}, + {"eu", "English.txt"}, + {"eu_ES", "English.txt"}, + {"fa", "English.txt"}, + {"fa_IR", "English.txt"}, + {"fi", "English.txt"}, + {"fi_FI", "English.txt"}, + {"fo", "English.txt"}, + {"fo_FO", "English.txt"}, + {"fr", "French.txt"}, + {"fr_BE", "French.txt"}, + {"fr_CA", "French.txt"}, + {"fr_CH", "French.txt"}, + {"fr_FR", "French.txt"}, + {"fr_LU", "French.txt"}, + {"fr_MC", "French.txt"}, + {"gl", "English.txt"}, + {"gl_ES", "English.txt"}, + {"gu", "English.txt"}, + {"gu_IN", "English.txt"}, + {"he", "English.txt"}, + {"he_IL", "English.txt"}, + {"hi", "English.txt"}, + {"hi_IN", "English.txt"}, + {"hr", "English.txt"}, + {"hr_BA", "English.txt"}, + {"hr_HR", "English.txt"}, + {"hu", "English.txt"}, + {"hu_HU", "English.txt"}, + {"hy", "English.txt"}, + {"hy_AM", "English.txt"}, + {"id", "Indonesian.txt"}, + {"id_ID", "Indonesian.txt"}, + {"is", "English.txt"}, + {"is_IS", "English.txt"}, + {"it", "Italian.txt"}, + {"it_CH", "Italian.txt"}, + {"it_IT", "Italian.txt"}, + {"ja", "Japanese.txt"}, + {"ja_JP", "Japanese.txt"}, + {"ka", "English.txt"}, + {"ka_GE", "English.txt"}, + {"kk", "English.txt"}, + {"kk_KZ", "English.txt"}, + {"kn", "English.txt"}, + {"kn_IN", "English.txt"}, + {"ko", "Korean.txt"}, + {"ko_KR", "Korean.txt"}, + {"kok", "English.txt"}, + {"kok_IN", "English.txt"}, + {"ky", "English.txt"}, + {"ky_KG", "English.txt"}, + {"lt", "Lithuanian.txt"}, + {"lt_LT", "Lithuanian.txt"}, + {"lv", "English.txt"}, + {"lv_LV", "English.txt"}, + {"mi", "English.txt"}, + {"mi_NZ", "English.txt"}, + {"mk", "English.txt"}, + {"mk_MK", "English.txt"}, + {"mn", "English.txt"}, + {"mn_MN", "English.txt"}, + {"mr", "English.txt"}, + {"mr_IN", "English.txt"}, + {"ms", "English.txt"}, + {"ms_BN", "English.txt"}, + {"ms_MY", "English.txt"}, + {"mt", "English.txt"}, + {"mt_MT", "English.txt"}, + {"nb", "English.txt"}, + {"nb_NO", "English.txt"}, + {"nl", "English.txt"}, + {"nl_BE", "English.txt"}, + {"nl_NL", "English.txt"}, + {"nn_NO", "English.txt"}, + {"ns", "English.txt"}, + {"ns_ZA", "English.txt"}, + {"pa", "English.txt"}, + {"pa_IN", "English.txt"}, + {"pl", "Polish.txt"}, + {"pl_PL", "Polish.txt"}, + {"ps", "English.txt"}, + {"ps_AR", "English.txt"}, + {"pt", "Portugese.txt"}, + {"pt_BR", "Portugese.txt"}, + {"pt_PT", "Portugese.txt"}, + {"qu", "English.txt"}, + {"qu_BO", "English.txt"}, + {"qu_EC", "English.txt"}, + {"qu_PE", "English.txt"}, + {"ro", "English.txt"}, + {"ro_RO", "English.txt"}, + {"ru", "Russian.txt"}, + {"ru_RU", "Russian.txt"}, + {"sa", "English.txt"}, + {"sa_IN", "English.txt"}, + {"se", "English.txt"}, + {"se_FI", "English.txt"}, + {"se_FI", "English.txt"}, + {"se_FI", "English.txt"}, + {"se_NO", "English.txt"}, + {"se_NO", "English.txt"}, + {"se_NO", "English.txt"}, + {"se_SE", "English.txt"}, + {"se_SE", "English.txt"}, + {"se_SE", "English.txt"}, + {"sk", "English.txt"}, + {"sk_SK", "English.txt"}, + {"sl", "Slovenian.txt"}, + {"sl_SI", "Slovenian.txt"}, + {"sq", "English.txt"}, + {"sq_AL", "English.txt"}, + {"sr_BA", "English.txt"}, + {"sr_BA", "English.txt"}, + {"sr_SP", "English.txt"}, + {"sr_SP", "English.txt"}, + {"sv", "English.txt"}, + {"sv_FI", "English.txt"}, + {"sv_SE", "English.txt"}, + {"sw", "English.txt"}, + {"sw_KE", "English.txt"}, + {"syr", "English.txt"}, + {"syr_SY", "English.txt"}, + {"ta", "English.txt"}, + {"ta_IN", "English.txt"}, + {"te", "English.txt"}, + {"te_IN", "English.txt"}, + {"th", "English.txt"}, + {"th_TH", "English.txt"}, + {"tl", "English.txt"}, + {"tl_PH", "English.txt"}, + {"tn", "English.txt"}, + {"tn_ZA", "English.txt"}, + {"tr", "Turkish.txt"}, + {"tr_TR", "Turkish.txt"}, + {"tt", "English.txt"}, + {"tt_RU", "English.txt"}, + {"ts", "English.txt"}, + {"uk", "English.txt"}, + {"uk_UA", "English.txt"}, + {"ur", "English.txt"}, + {"ur_PK", "English.txt"}, + {"uz", "English.txt"}, + {"uz_UZ", "English.txt"}, + {"uz_UZ", "English.txt"}, + {"vi", "English.txt"}, + {"vi_VN", "English.txt"}, + {"xh", "English.txt"}, + {"xh_ZA", "English.txt"}, + {"zh", "Chinese.txt"}, + {"zh_CN", "Chinese.txt"}, + {"zh_HK", "Chinese.txt"}, + {"zh_MO", "Chinese.txt"}, + {"zh_SG", "Chinese.txt"}, + {"zh_TW", "Chinese_TW.txt"}, + {"zu", "English.txt"}, + {"zu_ZA", "English.txt"}, +} + +func TestResolveLicenseFile(t *testing.T) { + for _, table := range licenseTests { + os.Setenv("LANG", table.in) + f := resolveLicenseFile() + if f != table.out { + t.Errorf("resolveLicenseFile() with LANG=%v - expected %v, got %v", table.in, table.out, f) + } + } +} From 70f1a43fd80ed3f65e9f9161db292b5b8ecd5bb1 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Thu, 30 Nov 2017 15:20:00 +0000 Subject: [PATCH 07/17] More license tweaks --- cmd/runmqserver/license.go | 5 +++-- cmd/runmqserver/license_test.go | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/runmqserver/license.go b/cmd/runmqserver/license.go index 1bd6824..308ffe1 100644 --- a/cmd/runmqserver/license.go +++ b/cmd/runmqserver/license.go @@ -36,7 +36,8 @@ func resolveLicenseFile() string { return "Chinese_TW.txt" case strings.HasPrefix(lang, "zh"): return "Chinese.txt" - case strings.HasPrefix(lang, "cs"): + // Differentiate Czech (cs) and Kashubian (csb) + case strings.HasPrefix(lang, "cs") && !strings.HasPrefix(lang, "csb"): return "Czech.txt" case strings.HasPrefix(lang, "fr"): return "French.txt" @@ -50,7 +51,7 @@ func resolveLicenseFile() string { return "Italian.txt" case strings.HasPrefix(lang, "ja"): return "Japanese.txt" - // Need to differentiate Korean (ko) from Konkani (kok) + // Differentiate Korean (ko) from Konkani (kok) case strings.HasPrefix(lang, "ko") && !strings.HasPrefix(lang, "kok"): return "Korean.txt" case strings.HasPrefix(lang, "lt"): diff --git a/cmd/runmqserver/license_test.go b/cmd/runmqserver/license_test.go index aa36a28..7f1817d 100644 --- a/cmd/runmqserver/license_test.go +++ b/cmd/runmqserver/license_test.go @@ -25,6 +25,7 @@ var licenseTests = []struct { out string }{ {"en_US.UTF_8", "English.txt"}, + {"en_US.ISO-8859-15", "English.txt"}, {"es_GB", "Spanish.txt"}, {"el_ES.UTF_8", "Greek.txt"}, // Cover a wide variety of valid values @@ -59,6 +60,7 @@ var licenseTests = []struct { {"ca_ES", "English.txt"}, {"cs", "Czech.txt"}, {"cs_CZ", "Czech.txt"}, + {"csb_PL", "English.txt"}, {"cy", "English.txt"}, {"cy_GB", "English.txt"}, {"da", "English.txt"}, From f5515d72a3be17c58fc6439e825e3b35400af6e0 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 10:17:31 +0000 Subject: [PATCH 08/17] Test for security vulnerabilities --- test/docker/docker_api_test.go | 25 +++++++++++++++++++++++++ test/docker/docker_api_test_util.go | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 73e6692..0a5a6ab 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -81,6 +81,31 @@ func TestGoldenPath(t *testing.T) { waitForReady(t, cli, id) } +// TestSecurityVulnerabilities checks for any vulnerabilities in the image, as reported +// by Ubuntu +func TestSecurityVulnerabilities(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + containerConfig := container.Config{ + // Override the entrypoint to make "apt" only receive security updates, then check for updates + Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"}, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + // rc is the return code from apt-get + rc := waitForContainer(t, cli, id, 10) + if rc != 0 { + t.Fatalf("Expected success, got %v", rc) + } + log := inspectLogs(t, cli, id) + lines := strings.Split(strings.TrimSpace(log), "\n") + if len(lines) > 0 && lines[0] != "" { + t.Errorf("Expected no vulnerabilities, found the following:\n%v", log) + } +} + func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName string) { search := "QMNAME(" + expectedName + ")" cli, err := client.NewEnvClient() diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 7258b17..528bc98 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -143,11 +143,11 @@ func getCoverageExitCode(t *testing.T, orig int64) int64 { f := filepath.Join(coverageDir(t), "exitCode") _, err := os.Stat(f) if err != nil { - t.Log(err) + //t.Log(err) return orig } // Remove the file, ready for the next test - //defer os.Remove(f) + defer os.Remove(f) buf, err := ioutil.ReadFile(f) if err != nil { t.Log(err) From 176543e10059cc92dbf7f797d1011ca1cee1d82f Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 10:27:36 +0000 Subject: [PATCH 09/17] Run Docker tests in parallel --- test/docker/docker_api_test.go | 34 +++++++++++++---------------- test/docker/docker_api_test_util.go | 17 +++++++-------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 0a5a6ab..f892606 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -29,6 +29,7 @@ import ( ) func TestLicenseNotSet(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -43,6 +44,7 @@ func TestLicenseNotSet(t *testing.T) { } func TestLicenseView(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -65,16 +67,16 @@ func TestLicenseView(t *testing.T) { // TestGoldenPath starts a queue manager successfully func TestGoldenPath(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) } containerConfig := container.Config{ Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - //ExposedPorts: ports, - ExposedPorts: nat.PortSet{ - "1414/tcp": struct{}{}, - }, + // ExposedPorts: nat.PortSet{ + // "1414/tcp": struct{}{}, + // }, } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) @@ -84,6 +86,7 @@ func TestGoldenPath(t *testing.T) { // TestSecurityVulnerabilities checks for any vulnerabilities in the image, as reported // by Ubuntu func TestSecurityVulnerabilities(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -115,9 +118,6 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri containerConfig := container.Config{ Env: []string{"LICENSE=accept"}, Hostname: hostName, - ExposedPorts: nat.PortSet{ - "1414/tcp": struct{}{}, - }, } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) @@ -128,16 +128,19 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri } } func TestNoQueueManagerName(t *testing.T) { + t.Parallel() utilTestNoQueueManagerName(t, "test", "test") } func TestNoQueueManagerNameInvalidHostname(t *testing.T) { + t.Parallel() utilTestNoQueueManagerName(t, "test-1", "test1") } // TestWithVolume runs a container with a Docker volume, then removes that // container and starts a new one with same volume. func TestWithVolume(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -179,16 +182,13 @@ func TestWithVolume(t *testing.T) { // TestNoVolumeWithRestart ensures a queue manager container can be stopped // and restarted cleanly func TestNoVolumeWithRestart(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) } containerConfig := container.Config{ Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - //ExposedPorts: ports, - ExposedPorts: nat.PortSet{ - "1414/tcp": struct{}{}, - }, } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) @@ -200,6 +200,7 @@ func TestNoVolumeWithRestart(t *testing.T) { // TestCreateQueueManagerFail causes a failure of `crtmqm` func TestCreateQueueManagerFail(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -208,10 +209,6 @@ func TestCreateQueueManagerFail(t *testing.T) { oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") containerConfig := container.Config{ Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - //ExposedPorts: ports, - ExposedPorts: nat.PortSet{ - "1414/tcp": struct{}{}, - }, // Override the entrypoint to create the queue manager directory, but leave it empty. // This will cause `crtmqm` to return with an exit code of 2. Entrypoint: []string{"bash", "-c", "mkdir -p /mnt/mqm/data && mkdir -p /var/mqm/qmgrs/qm1 && exec " + oldEntrypoint}, @@ -226,6 +223,7 @@ func TestCreateQueueManagerFail(t *testing.T) { // TestStartQueueManagerFail causes a failure of `strmqm` func TestStartQueueManagerFail(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -234,10 +232,6 @@ func TestStartQueueManagerFail(t *testing.T) { oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") containerConfig := container.Config{ Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - //ExposedPorts: ports, - ExposedPorts: nat.PortSet{ - "1414/tcp": struct{}{}, - }, // Override the entrypoint to replace `crtmqm` with a no-op script. // This will cause `strmqm` to return with an exit code of 16. Entrypoint: []string{"bash", "-c", "echo '#!/bin/bash\n' > /opt/mqm/bin/crtmqm && exec " + oldEntrypoint}, @@ -255,6 +249,7 @@ func TestStartQueueManagerFail(t *testing.T) { // This simulates behaviour seen in some cloud environments, where network // attached storage gets unmounted. func TestVolumeUnmount(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -300,6 +295,7 @@ func TestVolumeUnmount(t *testing.T) { // TestZombies starts a queue manager, then causes a zombie process to be // created, then checks that no zombies exist (runmqserver should reap them) func TestZombies(t *testing.T) { + t.Parallel() cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 528bc98..d7bca1b 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -32,7 +32,6 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" "github.com/moby/moby/pkg/stdcopy" ) @@ -99,14 +98,14 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C // if coverage containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov") hostConfig := container.HostConfig{ - PortBindings: nat.PortMap{ - "1414/tcp": []nat.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: "1414", - }, - }, - }, + // PortBindings: nat.PortMap{ + // "1414/tcp": []nat.PortBinding{ + // { + // HostIP: "0.0.0.0", + // HostPort: "1414", + // }, + // }, + // }, Binds: []string{ coverageBind(t), }, From abbc8ce85231a4c7daa251ca1b29e4ed0117ed53 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 10:57:08 +0000 Subject: [PATCH 10/17] Remove unnecessary TTY usage --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8b595de..98a0ba6 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ BUILD_SERVER_CONTAINER=build-server DOCKER_TAG_ARCH ?= $(shell uname -m) # By default, all Docker client commands are run inside a Docker container. # This means that newer features of the client can be used, even with an older daemon. -DOCKER ?= docker run --tty --interactive --rm --volume /var/run/docker.sock:/var/run/docker.sock --volume "$(CURDIR)":/var/src --workdir /var/src docker:stable docker +DOCKER ?= docker run --interactive --rm --volume /var/run/docker.sock:/var/run/docker.sock --volume "$(CURDIR)":/var/src --workdir /var/src docker:stable docker DOCKER_TAG ?= latest-$(DOCKER_TAG_ARCH) DOCKER_REPO_DEVSERVER ?= mq-devserver DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver From c297e1bf5d9a9733978af7e36f16619518b3bdbb Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 11:49:44 +0000 Subject: [PATCH 11/17] Improve docs --- docs/developing.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/developing.md b/docs/developing.md index 895d877..007314f 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -2,12 +2,13 @@ ## Prerequisites You need to ensure you have the following tools installed: - * [Docker](https://www.docker.com/) +* GNU make + +You might also need the following tools installed: * [Go](https://golang.org/) - only needed for running the tests -* [Glide](https://glide.sh/) -* [dep](https://github.com/golang/dep) (official Go dependency management tool) -* make +* [Glide](https://glide.sh/) - only needed if you update the main dependencies +* [dep](https://github.com/golang/dep) (official Go dependency management tool) - only needed to prepare for running the tests * [Helm](https://helm.sh) - only needed for running the Kubernetes tests For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). @@ -24,6 +25,12 @@ You can build a different version of MQ by setting the `MQ_VERSION` environment MQ_VERSION=9.0.3.0 make build-advancedserver ``` +If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example: + +```bash +MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver +``` + ## Running the tests There are three main sets of tests: From 584a4f2eb4abbbafb6630beab3014e8ccbf0fe72 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 13:22:36 +0000 Subject: [PATCH 12/17] Fixes for running on older Docker versions --- Makefile | 5 +++-- docs/developing.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 98a0ba6..d26ef8b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ BUILD_SERVER_CONTAINER=build-server DOCKER_TAG_ARCH ?= $(shell uname -m) # By default, all Docker client commands are run inside a Docker container. # This means that newer features of the client can be used, even with an older daemon. -DOCKER ?= docker run --interactive --rm --volume /var/run/docker.sock:/var/run/docker.sock --volume "$(CURDIR)":/var/src --workdir /var/src docker:stable docker +DOCKER ?= docker DOCKER_TAG ?= latest-$(DOCKER_TAG_ARCH) DOCKER_REPO_DEVSERVER ?= mq-devserver DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver @@ -150,9 +150,10 @@ define docker-build-mq --volume "$(realpath ./downloads/)":/usr/share/nginx/html:ro \ --detach \ nginx:alpine + # Make sure we have the latest base image + $(DOCKER) pull ubuntu:16.04 # Build the new image $(DOCKER) build \ - --pull \ --tag $1 \ --file $2 \ --network build \ diff --git a/docs/developing.md b/docs/developing.md index 007314f..6fa9682 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -2,8 +2,8 @@ ## Prerequisites You need to ensure you have the following tools installed: -* [Docker](https://www.docker.com/) -* GNU make +* [Docker](https://www.docker.com/) V17.05 or later +* [GNU make](https://www.gnu.org/software/make/) You might also need the following tools installed: * [Go](https://golang.org/) - only needed for running the tests From b66a799d7f54f695232244ea1f627b7700c8b8d1 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 15:27:07 +0000 Subject: [PATCH 13/17] Fix Makefile error checking --- Makefile | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index d26ef8b..76e4ba6 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,14 @@ TEST_OPTS_DOCKER ?= # Options to `go test` for the Kubernetes tests TEST_OPTS_KUBERNETES ?= TEST_IMAGE ?= $(DOCKER_FULL_ADVANCEDSERVER) +NUM_CPU=$(shell docker info --format "{{ .NCPU }}") + +.PHONY: vars +vars: + echo $(DOCKER_SERVER_VERSION_MAJOR) + echo $(DOCKER_SERVER_VERSION_MINOR) + echo $(DOCKER_CLIENT_VERSION_MAJOR) + echo $(DOCKER_CLIENT_VERSION_MINOR) .PHONY: default default: build-devserver test @@ -97,12 +105,12 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: test-devserver test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test + cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test -parallel $(NUM_CPU) .PHONY: test-advancedserver-cover test-advancedserver-cover: test/docker/vendor @@ -165,24 +173,30 @@ define docker-build-mq . ; $(DOCKER) kill $(BUILD_SERVER_CONTAINER) && $(DOCKER) network rm build endef +DOCKER_SERVER_VERSION=$(shell docker version --format "{{ .Server.Version }}") +DOCKER_CLIENT_VERSION=$(shell docker version --format "{{ .Client.Version }}") +.PHONY: docker-version +docker-version: + @test "$(word 1,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -ge "17" || (echo "Error: Docker client 17.05 or greater is required" && exit 1) + @test "$(word 2,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -ge "05" || (echo "Error: Docker client 17.05 or greater is required" && exit 1) + @test "$(word 1,$(subst ., ,$(DOCKER_SERVER_VERSION)))" -ge "17" || (echo "Error: Docker server 17.05 or greater is required" && exit 1) + @test "$(word 2,$(subst ., ,$(DOCKER_SERVER_VERSION)))" -ge "05" || (echo "Error: Docker server 17.05 or greater is required" && exit 1) + .PHONY: build-advancedserver -build-advancedserver: downloads/$(MQ_ARCHIVE) +build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) .PHONY: build-devserver -build-devserver: downloads/$(MQ_ARCHIVE_DEV) -ifneq "x86_64" "$(shell uname -m)" -    $(error MQ Advanced for Developers is only available for x86_64 architecture) -else +build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version + @test "$(uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) -endif .PHONY: build-advancedserver-cover -build-advancedserver-cover: +build-advancedserver-cover: docker-version $(DOCKER) build -t $(DOCKER_REPO_ADVANCEDSERVER):cover -f Dockerfile-server.cover . # .PHONY: build-web From 878442905d5eacdf46e3a178fd8c311998b493ef Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 15:39:32 +0000 Subject: [PATCH 14/17] Fix Travis download of dep --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49d86a8..19f2776 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,8 @@ before_install: - sudo apt-get update - sudo apt-get -y install docker-ce - curl https://glide.sh/get | sh - - curl -LO https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64.zip - - unzip dep-linux-amd64.zip - - sudo mv dep /usr/local/bin - - rm dep-linux-amd64.zip + - curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64 + - chmod +x /usr/local/bin/dep install: - echo nothing From 18635b704c2fa464eea9d782f321358196e1344c Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 16:08:35 +0000 Subject: [PATCH 15/17] Additional fix for Travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19f2776..87fc76c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,8 @@ before_install: - sudo apt-get update - sudo apt-get -y install docker-ce - curl https://glide.sh/get | sh - - curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64 - - chmod +x /usr/local/bin/dep + - sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64 + - sudo chmod +x /usr/local/bin/dep install: - echo nothing From 499c6d4b18cae8fc1d191b70fcf4c4df2cd56063 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Dec 2017 17:54:32 +0000 Subject: [PATCH 16/17] Correct architecture test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76e4ba6..5ad0677 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version .PHONY: build-devserver build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version - @test "$(uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) + @test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) From d6182bf2fc4ba1e448159bc9965dd8f21e96a7cf Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 6 Dec 2017 10:19:47 +0000 Subject: [PATCH 17/17] Fix build problems --- install-mq.sh | 8 +++----- test/docker/docker_api_test_util.go | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/install-mq.sh b/install-mq.sh index 47c7fc6..685bd50 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -87,12 +87,10 @@ find /opt/mqm -name '*.tar.gz' -delete rm -f /etc/apt/sources.list.d/IBM_MQ.list rm -rf ${DIR_EXTRACT} -# Apply any bug fixes not included in base Ubuntu or MQ image. +#### Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -apt-get upgrade -y libkrb5-26-heimdal -apt-get upgrade -y libexpat1 - -# End of bug fixes +apt-get upgrade -y libdb5.3 +#### End of bug fixes # Clean up cached apt files rm -rf /var/lib/apt/lists/* diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index d7bca1b..d6957d5 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -32,7 +32,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" - "github.com/moby/moby/pkg/stdcopy" + "github.com/docker/docker/pkg/stdcopy" ) func imageName() string {