From 96311df3661fa68df66ffde2d49df7088bfea7dc Mon Sep 17 00:00:00 2001 From: Date: Wed, 28 Mar 2018 11:15:51 +0100 Subject: [PATCH 1/4] Fix build and test in WSL --- .gitattributes | 3 ++ Makefile | 11 +++++- docs/testing.md | 13 +------ test/docker/docker_api_test_util.go | 58 +++++++++++++++++++++++++---- 4 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ad37a6a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Fore LF line endings to prevent errors in Bash on Windows +* text=auto +*.sh text eol=lf diff --git a/Makefile b/Makefile index c1b94cc..eaa79d5 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,13 @@ MQ_IMAGE_DEVSERVER_BASE=mqadvanced-server-dev-base:$(MQ_VERSION)-$(ARCH)-$(BASE_ # Docker image name to use for JMS tests DEV_JMS_IMAGE=mq-dev-jms-test + +ifneq (,$(findstring Microsoft,$(shell uname -r))) + DOWNLOADS_DIR=$(patsubst /mnt/c%,C:%,$(realpath ./downloads/)) +else + DOWNLOADS_DIR=$(realpath ./downloads/) +endif + # Try to figure out which archive to use from the BASE_IMAGE ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" MQ_ARCHIVE_TYPE=UBUNTU @@ -137,7 +144,7 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -timeout 20m -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: build-devjmstest build-devjmstest: @@ -181,7 +188,7 @@ define docker-build-mq --name $(BUILD_SERVER_CONTAINER) \ --network build \ --network-alias build \ - --volume "$(realpath ./downloads/)":/usr/share/nginx/html:ro \ + --volume $(DOWNLOADS_DIR):/usr/share/nginx/html:ro \ --detach \ nginx:alpine # Build the new image (use --pull to make sure we have the latest base image) diff --git a/docs/testing.md b/docs/testing.md index 4c2ce78..3f677de 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,4 +1,4 @@ -# Testing +# Testing ## Prerequisites You need to ensure you have the following tools installed: @@ -8,15 +8,6 @@ You need to ensure you have the following tools installed: * [dep](https://github.com/golang/dep) (official Go dependency management tool) - needed to prepare for running the tests * [Helm](https://helm.sh) - only needed for running the Kubernetes tests -For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). - -## Preparing to run the tests -The test dependencies are not included with the source code, so you need to download them before you can run them. This can be done with the following command, which uses the `dep` tool: - -``` -make deps -``` - ## Running the tests There are two main sets of tests: @@ -57,7 +48,7 @@ make build-advancedserver-cover make test-advancedserver-cover ``` -In order to generate code coverage metrics from the Docker tests, the build step creates a new Docker image with an instrumented version of the code. Each test is then run individually, producing a coverage report each under `test/docker/coverage/`. These individual reports are then combined. The combined report is written to the `coverage` directory. +In order to generate code coverage metrics from the Docker tests, the build step creates a new Docker image with an instrumented version of the code. Each test is then run individually, producing a coverage report each under `test/docker/coverage/`. These individual reports are then combined. The combined report is written to the `coverage` directory. ### Running the Kubernetes tests diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index f0d5c5f..b86da57 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -25,7 +25,9 @@ import ( "io" "io/ioutil" "os" + "os/exec" "path/filepath" + "runtime" "strconv" "strings" "testing" @@ -79,17 +81,56 @@ func coverageBind(t *testing.T) string { return coverageDir(t) + ":/var/coverage" } -// terminationLog returns the name of the file to use for the termination log message -func terminationLog(t *testing.T) string { +// detect whether we are running in the Windows Subsystem for Linux +func isWSL(t *testing.T) bool { + if (runtime.GOOS == "linux") { + var ( + uname []byte + err error + ) + + if uname, err = exec.Command("uname", "-r").Output(); err != nil { + t.Fatal(err) + } + + return strings.Contains(string(uname), "Microsoft") + + } else { + return false; + } +} + +// get the path of the tmp directory, in UNIX or OS-specific style +func getTempDir(t *testing.T, unixStylePath bool) string { + if (isWSL(t)) { + if (unixStylePath) { + return "/mnt/c/Temp/" + } else { + return "C:/Temp/" + } + } else { + return "/tmp/" + } +} + +// terminationLogUnix returns the name of the file to use for the termination log message, with a UNIX path +func terminationLogUnixPath(t *testing.T) string { // Warning: this directory must be accessible to the Docker daemon, // in order to enable the bind mount - return "/tmp/" + t.Name() + "-termination-log" + return getTempDir(t, true) + t.Name() + "-termination-log" +} + +// terminationLogWindows returns the name of the file to use for the termination log message, with an OS specific path +func terminationLogOSPath(t *testing.T) string { + // Warning: this directory must be accessible to the Docker daemon, + // in order to enable the bind mount + return getTempDir(t, false) + t.Name() + "-termination-log" } // terminationBind returns a string to use to bind-mount a termination log file. // This is done using a bind, because you can't copy files from /dev out of the container. func terminationBind(t *testing.T) string { - n := terminationLog(t) + n := terminationLogUnixPath(t) // Remove it if it already exists os.Remove(n) // Create the empty file @@ -98,12 +139,12 @@ func terminationBind(t *testing.T) string { t.Fatal(err) } f.Close() - return n + ":/dev/termination-log" + return terminationLogOSPath(t) + ":/dev/termination-log" } // Returns the termination message, or an empty string if not set func terminationMessage(t *testing.T) string { - b, err := ioutil.ReadFile(terminationLog(t)) + b, err := ioutil.ReadFile(terminationLogUnixPath(t)) if err != nil { t.Log(err) } @@ -147,7 +188,7 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { if m != "" { t.Logf("Termination message: %v", m) } - os.Remove(terminationLog(t)) + os.Remove(terminationLogUnixPath(t)) t.Logf("Removing container: %s", ID) opts := types.ContainerRemoveOptions{ @@ -287,6 +328,9 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user if err != nil { t.Fatal(err) } + + time.Sleep(4*time.Second) + inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) if err != nil { t.Fatal(err) From f5beed0befcc99fb944d105f315b04463abcfd7e Mon Sep 17 00:00:00 2001 From: Date: Wed, 28 Mar 2018 11:23:50 +0100 Subject: [PATCH 2/4] Update guide to reference instructions for docker in WSL --- docs/building.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/building.md b/docs/building.md index 5c06ee8..e76d0d5 100644 --- a/docs/building.md +++ b/docs/building.md @@ -1,10 +1,12 @@ -# Building a Docker image +# Building a Docker image ## Prerequisites You need to ensure you have the following tools installed: * [Docker](https://www.docker.com/) V17.06.1 or later * [GNU make](https://www.gnu.org/software/make/) +If you are working in the Linux Subsystem for Windows, follow [this guide by Microsoft to set up Docker](https://blogs.msdn.microsoft.com/commandline/2017/12/08/cross-post-wsl-interoperability-with-docker/) first. + ## Building a production image This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. @@ -46,4 +48,3 @@ Note that if you are using Red Hat Enterprise Linux, you will need to create you ## Installed components This image includes the core MQ server, Java, language packs, and GSKit. This can be configured by setting the `MQ_PACKAGES` argument to `make`, or directly as a [Docker build argument](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables-build-arg). - From 13e41aeb3792a8991aa1294dcac176f69dfa6b6f Mon Sep 17 00:00:00 2001 From: Date: Wed, 28 Mar 2018 11:47:59 +0100 Subject: [PATCH 3/4] Tidy up code --- docs/building.md | 2 +- test/docker/docker_api_test_util.go | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/docs/building.md b/docs/building.md index e76d0d5..873bc4c 100644 --- a/docs/building.md +++ b/docs/building.md @@ -5,7 +5,7 @@ You need to ensure you have the following tools installed: * [Docker](https://www.docker.com/) V17.06.1 or later * [GNU make](https://www.gnu.org/software/make/) -If you are working in the Linux Subsystem for Windows, follow [this guide by Microsoft to set up Docker](https://blogs.msdn.microsoft.com/commandline/2017/12/08/cross-post-wsl-interoperability-with-docker/) first. +If you are working in the Windows Subsystem for Linux, follow [this guide by Microsoft to set up Docker](https://blogs.msdn.microsoft.com/commandline/2017/12/08/cross-post-wsl-interoperability-with-docker/) first. ## Building a production image This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index b86da57..7eeabb7 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -81,29 +81,26 @@ func coverageBind(t *testing.T) string { return coverageDir(t) + ":/var/coverage" } -// detect whether we are running in the Windows Subsystem for Linux +// isWSL return whether we are running in the Windows Subsystem for Linux func isWSL(t *testing.T) bool { - if (runtime.GOOS == "linux") { - var ( - uname []byte - err error - ) + if runtime.GOOS == "linux" { - if uname, err = exec.Command("uname", "-r").Output(); err != nil { + uname, err := exec.Command("uname", "-r").Output() + if (err != nil) { t.Fatal(err) } return strings.Contains(string(uname), "Microsoft") } else { - return false; + return false } } -// get the path of the tmp directory, in UNIX or OS-specific style +// getTempDir get the path of the tmp directory, in UNIX or OS-specific style func getTempDir(t *testing.T, unixStylePath bool) string { - if (isWSL(t)) { - if (unixStylePath) { + if isWSL(t) { + if unixStylePath { return "/mnt/c/Temp/" } else { return "C:/Temp/" @@ -113,14 +110,14 @@ func getTempDir(t *testing.T, unixStylePath bool) string { } } -// terminationLogUnix returns the name of the file to use for the termination log message, with a UNIX path +// terminationLogUnixPath returns the name of the file to use for the termination log message, with a UNIX path func terminationLogUnixPath(t *testing.T) string { // Warning: this directory must be accessible to the Docker daemon, // in order to enable the bind mount return getTempDir(t, true) + t.Name() + "-termination-log" } -// terminationLogWindows returns the name of the file to use for the termination log message, with an OS specific path +// terminationLogOSPath returns the name of the file to use for the termination log message, with an OS specific path func terminationLogOSPath(t *testing.T) string { // Warning: this directory must be accessible to the Docker daemon, // in order to enable the bind mount @@ -142,7 +139,7 @@ func terminationBind(t *testing.T) string { return terminationLogOSPath(t) + ":/dev/termination-log" } -// Returns the termination message, or an empty string if not set +// terminationMessage return the termination message, or an empty string if not set func terminationMessage(t *testing.T) string { b, err := ioutil.ReadFile(terminationLogUnixPath(t)) if err != nil { @@ -329,7 +326,7 @@ func execContainerWithExitCode(t *testing.T, cli *client.Client, ID string, user t.Fatal(err) } - time.Sleep(4*time.Second) + time.Sleep(4 * time.Second) inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) if err != nil { From 6287a25156e6a58f24fbdb0345a5319c00ccacfc Mon Sep 17 00:00:00 2001 From: Riccardo Biraghi Date: Mon, 9 Apr 2018 11:41:28 +0100 Subject: [PATCH 4/4] Better error reporting for runmqsc --- test/docker/docker_api_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 08be306..f150a44 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "path/filepath" + "regexp" "strconv" "strings" "testing" @@ -356,9 +357,11 @@ func TestMQSC(t *testing.T) { id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) waitForReady(t, cli, id) - rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"}) - if rc != 0 { - t.Fatalf("Expected runmqsc to exit with rc=0, got %v", rc) + mqscOutput := execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"}) + r := regexp.MustCompile("AMQ[0-9][0-9][0-9][0-9]E") + amqerrCode := r.FindString(mqscOutput) + if amqerrCode != "" { + t.Fatalf("MQSC command failed with error %v", amqerrCode) } } @@ -395,11 +398,13 @@ func TestReadiness(t *testing.T) { t.Log(execContainerWithOutput(t, cli, id, "root", []string{"cat", "/etc/mqm/test.mqsc"})) for { readyRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"chkmqready"}) - queueCheckRC := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand}) - t.Logf("readyRC=%v,queueCheckRC=%v\n", readyRC, queueCheckRC) + queueCheckOut := execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", queueCheckCommand}) + t.Logf("readyRC=%v", readyRC) if readyRC == 0 { - if queueCheckRC != 0 { - t.Fatalf("chkmqready returned %v when MQSC had not finished", readyRC) + r := regexp.MustCompile("AMQ[0-9][0-9][0-9][0-9]E") + amqerrCode := r.FindString(queueCheckOut) + if (amqerrCode != "") { + t.Fatalf("MQSC failed with error %v. chkmqready returned %v when MQSC had not finished", amqerrCode, readyRC) } else { // chkmqready says OK, and the last queue exists, so return t.Log(execContainerWithOutput(t, cli, id, "root", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test1)' | runmqsc"}))