Merge pull request #199 from parrobe/master

Various fixes for the RHEL image and tests

Makefile-RHEL

@@ -39,8 +39,6 @@ MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TA
 MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
 # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools
 MQ_IMAGE_GOLANG_SDK ?=mq-golang-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
-# DOCKER is the Docker command to run
-DOCKER ?= podman
 # MQ_PACKAGES specifies the MQ packages to install.  Defaults vary on base image.
 MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm
 
@@ -49,8 +47,6 @@ MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSe
 ###############################################################################
 # ARCH is the platform architecture (e.g. x86_64, ppc64le or s390x)
 ARCH = $(shell uname -m)
-# BUILD_SERVER_CONTAINER is the name of the web server container used at build time
-BUILD_SERVER_CONTAINER=build-server
 # NUM_CPU is the number of CPUs available to Docker.  Used to control how many
 # test run in parallel
 NUM_CPU = $(or $(shell docker info --format "{{ .NCPU }}"),2)
@@ -58,7 +54,7 @@ NUM_CPU = $(or $(shell docker info --format "{{ .NCPU }}"),2)
 BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE)))
 MQ_IMAGE_DEVSERVER_BASE=mqadvanced-server-dev-base:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
 # Docker image name to use for JMS tests
-DEV_JMS_IMAGE=mq-dev-jms-test
+DEV_JMS_IMAGE=mq-dev-jms-test:latest
 # Variables for versioning
 IMAGE_REVISION=$(shell git rev-parse HEAD)
 IMAGE_SOURCE=$(shell git config --get remote.origin.url)
@@ -90,6 +86,10 @@ MQ_ARCHIVE_DEV_9.1.0.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz
 # Build targets
 ###############################################################################
 
+# Vendor Go dependencies for the Docker tests
+test/docker/vendor:
+	cd test/docker && dep ensure -vendor-only
+
 downloads/$(MQ_ARCHIVE_DEV):
 	$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END)))
 	mkdir -p downloads
@@ -105,24 +105,37 @@ check-prereqs:
 	which podman || (echo "Missing required program podman" && exit 1)
 	yum list | grep yum-utils || (echo "Missing required package yum-utils" && exit 1)
 
+.PHONY: check-test-prereqs
+check-prereqs:
+	$(info $(SPACER)$(shell printf $(TITLE)"Checking for prereqs"$(END)))
+	which buildah || (echo "Missing required program buildah" && exit 1)
+	which docker || (echo "Missing required program docker" && exit 1)
 
 .PHONY: test-advancedserver
-test-advancedserver: 
+test-advancedserver: check-test-prereqs test/docker/vendor
 	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on $(shell docker --version)"$(END)))
+	buildah push $(MQ_IMAGE_ADVANCEDSERVER) docker-daemon:$(MQ_IMAGE_ADVANCEDSERVER)
+	docker tag docker.io/$(MQ_IMAGE_ADVANCEDSERVER) $(MQ_IMAGE_ADVANCEDSERVER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
 
 
 .PHONY: test-devserver
-test-devserver: 
+test-devserver: check-test-prereqs test/docker/vendor
 	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER) on $(shell docker --version)"$(END)))
+	buildah push $(MQ_IMAGE_DEVSERVER) docker-daemon:$(MQ_IMAGE_DEVSERVER)
+	docker tag docker.io/$(MQ_IMAGE_DEVSERVER) $(MQ_IMAGE_DEVSERVER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
 
 
 .PHONY: build-advancedserver
+build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE)
 build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs
 	$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END)))
 	mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)"
 
 
 .PHONY: build-devserver
+build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV)
 build-devserver: MQDEV=TRUE
 build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm
 build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs
@@ -142,4 +155,11 @@ build-go-programs: check-prereqs build-mqgolang-sdk
 	$(info $(SPACER)$(shell printf $(TITLE)"Build go programs"$(END)))
 	IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)"
 
+.PHONY: build-devjmstest
+build-devjmstest: check-test-prereqs
+	$(info $(SPACER)$(shell printf $(TITLE)"Build JMS tests for developer config"$(END)))
+	cd test/messaging && ./buildah.sh $(DEV_JMS_IMAGE)
+	buildah push $(DEV_JMS_IMAGE) docker-daemon:$(DEV_JMS_IMAGE)
+	docker tag docker.io/$(DEV_JMS_IMAGE) $(DEV_JMS_IMAGE)
+
 include formatting.mk

Makefile-UBUNTU

@@ -154,7 +154,7 @@ build-devjmstest:
 .PHONY: test-devserver
 test-devserver: test/docker/vendor
 	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER) on $(shell docker --version)"$(END)))
-	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
 
 coverage:
 	mkdir coverage

mq-advanced-server-rhel/go-buildah.sh

@@ -30,5 +30,6 @@ podman run \
   --env IMAGE_REVISION="$IMAGE_REVISION" \
   --env IMAGE_SOURCE="$IMAGE_SOURCE" \
   --env MQDEV=${dev} \
+  --rm \
   ${tag} \
   bash -c "cd /go/src/github.com/ibm-messaging/mq-container/ && ./mq-advanced-server-rhel/go-build.sh"

mq-advanced-server-rhel/install-mq-rhel.sh

@@ -56,6 +56,18 @@ find $scratchmnt/opt/mqm -name '*.tar.gz' -delete
 # Recommended: Set the default MQ installation (makes the MQ commands available on the PATH)
 buildah run $ctr -- /opt/mqm/bin/setmqinst -p /opt/mqm -i
 
+mkdir -p $scratchmnt/run/runmqserver
+chown 888:888 $scratchmnt/run/runmqserver
+
+# Remove the directory structure under /var/mqm which was created by the installer
+rm -rf $scratchmnt/var/mqm
+
+# Create the mount point for volumes
+mkdir -p $scratchmnt/mnt/mqm
+
+# Create a symlink for /var/mqm -> /mnt/mqm/data
+buildah run $ctr -- ln -s /mnt/mqm/data /var/mqm
+
 # Optional: Set these values for the IBM Cloud Vulnerability Report
 sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' $scratchmnt/etc/login.defs
 sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' $scratchmnt/etc/login.defs

mq-advanced-server-rhel/mq-buildah.sh

@@ -28,7 +28,7 @@ set -e
 
 # Use a "scratch" container, so the resulting image has minimal files
 # Resulting image won't have yum, for example
-readonly ctr_mq=$(buildah from scratch)
+readonly ctr_mq=$(buildah from rhel7)
 readonly mnt_mq=$(buildah mount $ctr_mq)
 readonly archive=downloads/$1
 readonly packages=$2
@@ -36,18 +36,12 @@ readonly tag=$3
 readonly version=$4
 readonly mqdev=$5
 
-# Initialize yum for use with the scratch container
-rpm --root $mnt_mq --initdb
-
-yumdownloader --destdir=/tmp redhat-release-server
-rpm --root $mnt_mq -ihv /tmp/redhat-release-server*.rpm || true
-
 ###############################################################################
 # Install MQ server
 ###############################################################################
 
 # Install the packages required by MQ
-yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \
+buildah run $ctr_mq -- yum install -y --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \
   bash \
   bc \
   coreutils \
@@ -63,15 +57,12 @@ yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt
   util-linux
 
 # Clean up cached files
-yum clean all --installroot=${mnt_mq}
+buildah run $ctr_mq -- yum clean all
 rm -rf ${mnt_mq}/var/cache/yum/*
 
 # Install MQ server packages into the MQ builder image
 ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}"
 
-# Remove the directory structure under /var/mqm which was created by the installer
-rm -rf ${mnt_mq}/var/mqm
-
 # Create the directory for MQ configuration files
 mkdir -p ${mnt_mq}/etc/mqm
 chown 888:888 ${mnt_mq}/etc/mqm
@@ -113,3 +104,5 @@ buildah config \
   $ctr_mq
 buildah unmount $ctr_mq
 buildah commit $ctr_mq $tag
+
+buildah rm $ctr_mq

mq-advanced-server-rhel/mq-golang-sdk-buildah.sh

@@ -43,3 +43,5 @@ buildah config \
   --env CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" \
   ${ctr}
 buildah commit ${ctr} ${tag}
+
+buildah rm ${ctr}

mq-advanced-server-rhel/mqdev-buildah.sh

@@ -37,7 +37,7 @@ readonly version=$3
 
 useradd --root $mnt_mq --system --uid 889 --gid mqm admin
 groupadd --root $mnt_mq --system --gid 890 mqclient
-useradd --root $mnt_mq --system --uid 890 --gid mqclient mqclient
+useradd --root $mnt_mq --system --uid 890 --gid mqclient app
 
 buildah run $ctr -- echo admin:passw0rd | chpasswd
 
@@ -79,3 +79,4 @@ buildah config \
 buildah unmount $ctr_mq
 buildah commit $ctr_mq $tag
 
+buildah rm $ctr_mq

test/docker/devconfig_test.go

@@ -70,6 +70,7 @@ func TestDevSecure(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	const tlsPassPhrase string = "passw0rd"
 	qm := "qm1"
 	appPassword := "differentPassw0rd"

test/docker/devconfig_test_util.go

@@ -26,6 +26,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -86,6 +87,7 @@ func runJMSTests(t *testing.T, cli *client.Client, ID string, tls bool, user, pa
 			"MQ_PORT_1414_TCP_ADDR=" + getIPAddress(t, cli, ID),
 			"MQ_USERNAME=" + user,
 			"MQ_CHANNEL=DEV.APP.SVRCONN",
+			"IBMJRE=" + os.Getenv("IBMJRE"),
 		},
 		Image: imageNameDevJMS(),
 	}

test/docker/docker_api_test.go

@@ -684,10 +684,11 @@ func TestVersioning(t *testing.T) {
 			dataAr := strings.Split(line, " ")
 			data := dataAr[len(dataAr)-1]
 
-			// Verify created
+			// Verify created is in a known timestamp format
 			_, err := time.Parse(time.RFC3339, data)
-			if err != nil {
+			_, err2 := time.Parse("2006-01-02T15:04:05-0700", data)
-				t.Errorf("Failed to validate Image created (%v) - %v", data, err)
+			if err != nil && err2 != nil {
+				t.Errorf("Failed to validate Image created stamp (%v) - %v or %v", data, time.RFC3339, "2006-01-02T15:04:05-0700")
 			}
 		}
 

test/messaging/buildah.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# © Copyright IBM Corporation 2018
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+###############################################################################
+# Setup MQ JMS Test container
+###############################################################################
+
+# Use a "scratch" container, so the resulting image has minimal files
+# Resulting image won't have yum, for example
+readonly ctr_mq=$(buildah from rhel7)
+readonly mnt_mq=$(buildah mount $ctr_mq)
+readonly imagename=$1
+
+buildah run $ctr_mq -- yum install -y \
+    java-1.7.0-openjdk-devel \
+    java \
+    which \
+    wget
+
+buildah run $ctr_mq -- sh -c "cd /tmp && wget http://mirror.olnevhost.net/pub/apache/maven/binaries/apache-maven-3.2.2-bin.tar.gz"
+tar xvf $mnt_mq/tmp/apache-maven-3.2.2-bin.tar.gz -C $mnt_mq/tmp/
+
+mkdir -p $mnt_mq/usr/src/mymaven
+cp pom.xml $mnt_mq/usr/src/mymaven/
+cp -R src $mnt_mq/usr/src/mymaven/src
+
+buildah run $ctr_mq -- sh -c "cd /usr/src/mymaven && export M2_HOME=/tmp/apache-maven-3.2.2 && export M2=\$M2_HOME/bin && export PATH=\$M2:\$PATH && mvn --version && mvn dependency:go-offline install && mvn --offline install"
+
+mkdir -p $mnt_mq/opt/app
+
+cp $mnt_mq/usr/src/mymaven/target/*.jar $mnt_mq/opt/app/
+cp $mnt_mq/usr/src/mymaven/target/lib/*.jar $mnt_mq/opt/app/
+
+###############################################################################
+# Post install tidy up
+###############################################################################
+
+rm -rf $mnt_mq/tmp/*
+rm -rf $mnt_mq/usr/src/mymaven
+
+# We can't uninstall tar or gzip because they are required
+buildah run $ctr_mq -- yum remove -y \
+    wget
+
+# Clean up cached files
+buildah run $ctr_mq -- yum clean all
+rm -rf ${mnt_mq}/var/cache/yum/*
+
+###############################################################################
+# Contain image finalization
+###############################################################################
+
+buildah config \
+  --os linux \
+  --label architecture=x86_64 \
+  --label name="${imagename%:*}" \
+  --entrypoint '["java", "-classpath", "/opt/app/*", "org.junit.platform.console.ConsoleLauncher", "-p", "com.ibm.mqcontainer.test", "--details", "verbose"]' \
+  $ctr_mq
+buildah unmount $ctr_mq
+buildah commit $ctr_mq $imagename
+
+buildah rm $ctr_mq

test/messaging/src/main/java/com/ibm/mqcontainer/test/JMSTests.java

@@ -84,9 +84,15 @@ class JMSTests {
         else {
             LOGGER.info(String.format("Using TLS.  Trust store=%s", TRUSTSTORE));
             SSLSocketFactory ssl = createSSLSocketFactory();
-            factory.setSSLSocketFactory(ssl);
+            factory.setSSLSocketFactory(ssl); 
-            factory.setSSLCipherSuite("SSL_RSA_WITH_AES_128_CBC_SHA256");
+            boolean ibmjre = System.getenv("IBMJRE").equals("true");
-            // LOGGER.info(Arrays.toString(ssl.getSupportedCipherSuites()));
+            if (ibmjre){
+                System.setProperty("com.ibm.mq.cfg.useIBMCipherMappings", "true");
+                factory.setSSLCipherSuite("SSL_RSA_WITH_AES_128_CBC_SHA256");
+            } else {
+                 System.setProperty("com.ibm.mq.cfg.useIBMCipherMappings", "false");
+                 factory.setSSLCipherSuite("TLS_RSA_WITH_AES_128_CBC_SHA256");
+            }
         }
         // Give up if unable to reconnect for 10 minutes
         // factory.setClientReconnectTimeout(600);