academy/mq-container
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ability to use different base image

Browse Source
This commit is contained in:
Arthur Barr
2017-12-11 09:12:39 +00:00
parent d6182bf2fc
commit a2326dc0f6
8 changed files with 262 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile-server
View File

@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
ARG BASE_IMAGE=ubuntu:16.04
############################################################################### ###############################################################################
# Build stage to build Go code # Build stage to build Go code
############################################################################### ###############################################################################
@@ -29,14 +31,14 @@ RUN go test -v ./cmd/... ./internal/...
############################################################################### ###############################################################################
# Main build stage, to build MQ image # Main build stage, to build MQ image
############################################################################### ###############################################################################
FROM ubuntu:16.04 FROM $BASE_IMAGE
# The URL to download the MQ installer from in tar.gz format # The URL to download the MQ installer from in tar.gz format
# This assumes an archive containing the MQ Debian (.deb) install packages # This assumes an archive containing the MQ Debian (.deb) install packages
ARG MQ_URL ARG MQ_URL
# The MQ packages to install # The MQ packages to install - see install-mq.sh for default value
ARG MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" ARG MQ_PACKAGES
COPY install-mq.sh /usr/local/bin/ COPY install-mq.sh /usr/local/bin/

124
Makefile
View File

@@ -12,45 +12,72 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
BUILD_SERVER_CONTAINER=build-server ###############################################################################
DOCKER_TAG_ARCH ?= $(shell uname -m) # Conditional variables - you can override the values of these variables from
# By default, all Docker client commands are run inside a Docker container. # the command line
# This means that newer features of the client can be used, even with an older daemon. ###############################################################################
DOCKER ?= docker # BASE_IMAGE is the base image to use for MQ, for example "ubuntu" or "rhel"
DOCKER_TAG ?= latest-$(DOCKER_TAG_ARCH) BASE_IMAGE ?= ubuntu:16.04
DOCKER_REPO_DEVSERVER ?= mq-devserver
DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver
DOCKER_FULL_DEVSERVER = $(DOCKER_REPO_DEVSERVER):$(DOCKER_TAG)
DOCKER_FULL_ADVANCEDSERVER = $(DOCKER_REPO_ADVANCEDSERVER):$(DOCKER_TAG)
# MQ_PACKAGES is the list of MQ packages to install
MQ_PACKAGES ?=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams
# MQ_VERSION is the fully qualified MQ version number to build # MQ_VERSION is the fully qualified MQ version number to build
MQ_VERSION ?= 9.0.4.0 MQ_VERSION ?= 9.0.4.0
# Archive names for IBM MQ Continuous Delivery Release for Ubuntu # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can
MQ_ARCHIVE_9.0.3.0_ppc64le=CNJR5ML.tar.gz # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture
MQ_ARCHIVE_9.0.3.0_s390x=CNJR6ML.tar.gz # Does not apply to MQ Advanced for Developers.
MQ_ARCHIVE_9.0.3.0_x86_64=CNJR7ML.tar.gz MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz
MQ_ARCHIVE_9.0.4.0_ppc64le=CNLE2ML.tar.gz
MQ_ARCHIVE_9.0.4.0_s390x=CNLE3ML.tar.gz
MQ_ARCHIVE_9.0.4.0_x86_64=CNLE4ML.tar.gz
# Archive names for IBM MQ Advanced for Developers for Ubuntu
MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz
MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz
MQ_ARCHIVE ?= $(MQ_ARCHIVE_$(MQ_VERSION)_$(DOCKER_TAG_ARCH))
MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION))
# Options to `go test` for the Docker tests # Options to `go test` for the Docker tests
TEST_OPTS_DOCKER ?= TEST_OPTS_DOCKER ?=
# Options to `go test` for the Kubernetes tests # Options to `go test` for the Kubernetes tests
TEST_OPTS_KUBERNETES ?= TEST_OPTS_KUBERNETES ?=
TEST_IMAGE ?= $(DOCKER_FULL_ADVANCEDSERVER) # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image
NUM_CPU=$(shell docker info --format "{{ .NCPU }}") MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
# MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced for Developers image
MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
# DOCKER is the Docker command to run
DOCKER ?= docker
###############################################################################
# Other variables
###############################################################################
# ARCH is the platform architecture (e.g. x86_64, ppc64le or s390x)
ARCH = $(shell uname -m)
# BUILD_SERVER_CONTAINER is the name of the web server container used at build time
BUILD_SERVER_CONTAINER=build-server
# NUM_CPU is the number of CPUs available to Docker. Used to control how many
# test run in parallel
NUM_CPU=$(shell docker info --format "{{ .NCPU }}")
# BASE_IMAGE_TAG is a normalized version of BASE_IMAGE, suitable for use in a Docker tag
BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE)))
# Try to figure out which archive to use from the BASE_IMAGE
ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu"
MQ_ARCHIVE_TYPE=UBUNTU
else
MQ_ARCHIVE_TYPE=LINUX
endif
# Try to figure out which archive to use from the architecture
ifeq "$(ARCH)" "x86_64"
MQ_ARCHIVE_ARCH=X86-64
else ifeq "$(ARCH)" "ppc64le"
MQ_ARCHIVE_ARCH=LE_POWER
else ifeq "$(ARCH)" "s390x"
MQ_ARCHIVE_ARCH=SYSTEM_Z
endif
# Archive names for IBM MQ Advanced for Developers for Ubuntu
MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz
MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz
# MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced
# for Developers can be installed
MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION))
###############################################################################
# Build targets
###############################################################################
.PHONY: vars .PHONY: vars
vars: vars:
echo $(DOCKER_SERVER_VERSION_MAJOR) #ifeq "$(findstring ubuntu,$(BASE_IMAGE))","ubuntu"
echo $(DOCKER_SERVER_VERSION_MINOR) @echo $(MQ_ARCHIVE_ARCH)
echo $(DOCKER_CLIENT_VERSION_MAJOR) @echo $(MQ_ARCHIVE_TYPE)
echo $(DOCKER_CLIENT_VERSION_MINOR) @echo $(MQ_ARCHIVE)
.PHONY: default .PHONY: default
default: build-devserver test default: build-devserver test
@@ -105,16 +132,16 @@ test-unit:
.PHONY: test-advancedserver .PHONY: test-advancedserver
test-advancedserver: test/docker/vendor test-advancedserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END)))
cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
.PHONY: test-devserver .PHONY: test-devserver
test-devserver: test/docker/vendor test-devserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END)))
cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test -parallel $(NUM_CPU) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU)
.PHONY: test-advancedserver-cover .PHONY: test-advancedserver-cover
test-advancedserver-cover: test/docker/vendor test-advancedserver-cover: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_REPO_ADVANCEDSERVER) on Docker with code coverage"$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on Docker with code coverage"$(END)))
rm -f ./coverage/unit*.cov rm -f ./coverage/unit*.cov
# Run unit tests with coverage, for each package under 'internal' # Run unit tests with coverage, for each package under 'internal'
go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{} go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{}
@@ -124,7 +151,7 @@ test-advancedserver-cover: test/docker/vendor
rm -f ./test/docker/coverage/*.cov rm -f ./test/docker/coverage/*.cov
rm -f ./coverage/docker.* rm -f ./coverage/docker.*
cd test/docker && TEST_IMAGE=$(DOCKER_REPO_ADVANCEDSERVER):cover go test $(TEST_OPTS_DOCKER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER)-cover go test $(TEST_OPTS_DOCKER)
echo 'mode: count' > ./coverage/docker.cov echo 'mode: count' > ./coverage/docker.cov
tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov
go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html
@@ -135,15 +162,15 @@ test-advancedserver-cover: test/docker/vendor
.PHONY: test-kubernetes-devserver .PHONY: test-kubernetes-devserver
test-kubernetes-devserver: test/kubernetes/vendor test-kubernetes-devserver: test/kubernetes/vendor
$(call test-kubernetes,$(DOCKER_REPO_DEVSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-dev") $(call test-kubernetes,$(MQ_IMAGE_DEVSERVER),"../../charts/ibm-mqadvanced-server-dev")
.PHONY: test-kubernetes-advancedserver .PHONY: test-kubernetes-advancedserver
test-kubernetes-advancedserver: test/kubernetes/vendor test-kubernetes-advancedserver: test/kubernetes/vendor
$(call test-kubernetes,$(DOCKER_REPO_ADVANCEDSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-prod") $(call test-kubernetes,$(MQ_IMAGE_ADVANCEDSERVER),"../../charts/ibm-mqadvanced-server-prod")
define test-kubernetes define test-kubernetes
$(info $(SPACER)$(shell printf $(TITLE)"Test $1:$2 on Kubernetes"$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Test $1 on Kubernetes"$(END)))
cd test/kubernetes && TEST_REPO=$1 TEST_TAG=$2 TEST_CHART=$3 go test $(TEST_OPTS_KUBERNETES) cd test/kubernetes && TEST_IMAGE=$1 TEST_CHART=$2 go test $(TEST_OPTS_KUBERNETES)
endef endef
define docker-build-mq define docker-build-mq
@@ -159,13 +186,14 @@ define docker-build-mq
--detach \ --detach \
nginx:alpine nginx:alpine
# Make sure we have the latest base image # Make sure we have the latest base image
$(DOCKER) pull ubuntu:16.04 $(DOCKER) pull $(BASE_IMAGE)
# Build the new image # Build the new image
$(DOCKER) build \ $(DOCKER) build \
--tag $1 \ --tag $1 \
--file $2 \ --file $2 \
--network build \ --network build \
--build-arg MQ_URL=http://build:80/$3 \ --build-arg MQ_URL=http://build:80/$3 \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--label IBM_PRODUCT_ID=$4 \ --label IBM_PRODUCT_ID=$4 \
--label IBM_PRODUCT_NAME=$5 \ --label IBM_PRODUCT_NAME=$5 \
--label IBM_PRODUCT_VERSION=$6 \ --label IBM_PRODUCT_VERSION=$6 \
@@ -184,27 +212,27 @@ docker-version:
.PHONY: build-advancedserver .PHONY: build-advancedserver
build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version
$(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END)))
$(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION))
$(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) # $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(ARCH)-$(subst :,-,$(BASE_IMAGE))
.PHONY: build-devserver .PHONY: build-devserver
build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version
@test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) @test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1)
$(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END)))
$(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
$(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) # $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(ARCH)
.PHONY: build-advancedserver-cover .PHONY: build-advancedserver-cover
build-advancedserver-cover: docker-version build-advancedserver-cover: docker-version
$(DOCKER) build -t $(DOCKER_REPO_ADVANCEDSERVER):cover -f Dockerfile-server.cover . $(DOCKER) build -t $(MQ_IMAGE_ADVANCEDSERVER)-cover -f Dockerfile-server.cover .
# .PHONY: build-web # .PHONY: build-web
# build-web: build downloads/CNJR7ML.tar.gz # build-web: build downloads/CNJR7ML.tar.gz
# $(call docker-build-mq,mq-web:latest-$(DOCKER_TAG_ARCH),Dockerfile-mq-web) # $(call docker-build-mq,mq-web:latest-$(ARCH),Dockerfile-mq-web)
.PHONY: build-explorer .PHONY: build-explorer
build-explorer: downloads/$(MQ_ARCHIVE_DEV) build-explorer: downloads/$(MQ_ARCHIVE_DEV)
$(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
include formatting.mk include formatting.mk

35
docs/building.md Normal file
View File

@@ -0,0 +1,35 @@
# Building a Docker image
## Prerequisites
You need to ensure you have the following tools installed:
* [Docker](https://www.docker.com/) V17.05 or later
* [GNU make](https://www.gnu.org/software/make/)
## Building a production image
This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures.
1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `IBM_MQ_9.0.4.0_UBUNTU_X86-64.tar.gz` for MQ V9.0.4 for Ubuntu on x86_64 architecture) in the `downloads` directory
2. Run `make build-advancedserver`
You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example:
```bash
MQ_VERSION=9.0.4.0 make build-advancedserver
```
If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
```bash
MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver
```
## Building on a different base image
By default, the MQ images use Ubuntu as the base layer. You can build using a Red Hat Enterprise Linux compatible base layer by setting the `BASE_IMAGE` environment variable. For example:
```
BASE_IMAGE=centos:7 make build-advancedserver
```
The `make` tool will try and locate the right archive file under the `downloads` directory, based on your platform architecture and your `MQ_VERSION` environment variable, for example `IBM_MQ_9.0.4.0_LINUX_X86_64.tar.gz` for MQ V9.0.4.0 on x86_64. You can also set the `MQ_ARCHIVE` environment variable to set the specific file name.
Note that if you are using Red Hat Enterprise Linux, you will need to create your own base image layer, with your subscription enabled, as described [here](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/getting_started_with_containers/get_started_with_docker_formatted_container_images). The MQ image build needs to install some additional packages, and a subscription is required to access the Red Hat repositories.

53
docs/developing.md → docs/testing.md
View File

@@ -1,34 +1,20 @@
# Developing # Testing
## Prerequisites ## Prerequisites
You need to ensure you have the following tools installed: You need to ensure you have the following tools installed:
* [Docker](https://www.docker.com/) V17.05 or later * [Docker](https://www.docker.com/)
* [GNU make](https://www.gnu.org/software/make/) * [GNU make](https://www.gnu.org/software/make/)
You might also need the following tools installed:
* [Go](https://golang.org/) - only needed for running the tests * [Go](https://golang.org/) - only needed for running the tests
* [Glide](https://glide.sh/) - only needed if you update the main dependencies * [dep](https://github.com/golang/dep) (official Go dependency management tool) - needed to prepare for running the tests
* [dep](https://github.com/golang/dep) (official Go dependency management tool) - only needed to prepare for running the tests
* [Helm](https://helm.sh) - only needed for running the Kubernetes tests * [Helm](https://helm.sh) - only needed for running the Kubernetes tests
For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/).
## Building a production image ## Preparing to run the tests
This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. The test dependencies are not included with the source code, so you need to download them before you can run them. This can be done with the following command, which uses the `dep` tool:
1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4 on x86_64 architecture) in the `downloads` directory
2. Run `make build-advancedserver`
You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example:
```bash
MQ_VERSION=9.0.3.0 make build-advancedserver
``` ```
make deps
If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
```bash
MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver
``` ```
## Running the tests ## Running the tests
@@ -39,30 +25,31 @@ There are three main sets of tests:
3. Kubernetes tests, which test the Helm charts (and the Docker image) via [Helm](https://helm.sh) 3. Kubernetes tests, which test the Helm charts (and the Docker image) via [Helm](https://helm.sh)
### Running the Docker tests ### Running the Docker tests
The Docker tests can be run locally. Before you run them for the first time, you need to download the test dependencies: The Docker tests can be run locally on a machine with Docker. For example:
```
make deps
```
You can then run the tests, for example:
``` ```
make test-devserver make test-devserver
```
or:
```
make test-advancedserver make test-advancedserver
``` ```
You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example:
```
MQ_IMAGE_ADVANCEDSERVER=mqadvanced-server9.0.4.0-x86_64-ubuntu-16.04 make test-advancedserver
```
You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command:: You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command::
``` ```
TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver
``` ```
You can also use the same environment variables you specified when [building](./building), for example, the following will try and test an image called `mqadvanced-server9.0.3.0-x86_64-ubuntu-16.04`:
```
MQ_VERSION=9.0.3.0 make test-advancedserver
```
### Running the Docker tests with code coverage ### Running the Docker tests with code coverage
You can produce code coverage results from the Docker tests by running the following: You can produce code coverage results from the Docker tests by running the following:
@@ -79,5 +66,5 @@ In order to generate code coverage metrics from the Docker tests, the build step
For the Kubernetes tests, you need to have built the Docker image, and pushed it to the registry used by your Kubernetes cluster. Most of the configuration used by the tests is picked up from your `kubectl` configuration, but you will typically need to specify the image details. For example: For the Kubernetes tests, you need to have built the Docker image, and pushed it to the registry used by your Kubernetes cluster. Most of the configuration used by the tests is picked up from your `kubectl` configuration, but you will typically need to specify the image details. For example:
```bash ```bash
DOCKER_REPO_DEVSERVER=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver MQ_IMAGE=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver
``` ```

86
install-mq.sh
View File

@@ -18,25 +18,53 @@
# Fail on any non-zero return code # Fail on any non-zero return code
set -ex set -ex
export DEBIAN_FRONTEND=noninteractive test -f /usr/bin/yum && RHEL=true || RHEL=false
test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false
# If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults
if [ -z $MQ_PACKAGES ]; then
$UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams"
$RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm"
fi
if ($UBUNTU); then
export DEBIAN_FRONTEND=noninteractive
# Install additional packages required by MQ, this install process and the runtime scripts
apt-get update
apt-get install -y --no-install-recommends \
bash \
bc \
ca-certificates \
coreutils \
curl \
debianutils \
file \
findutils \
gawk \
grep \
libc-bin \
mount \
passwd \
procps \
sed \
tar \
util-linux
fi
# Install additional packages required by MQ, this install process and the runtime scripts # Install additional packages required by MQ, this install process and the runtime scripts
apt-get update $RHEL && yum -y install \
apt-get install -y --no-install-recommends \
bash \ bash \
bc \ bc \
ca-certificates \ ca-certificates \
coreutils \ coreutils \
curl \ curl \
debianutils \
file \ file \
findutils \ findutils \
gawk \ gawk \
glibc-common \
grep \ grep \
libc-bin \
mount \
passwd \ passwd \
procps \ procps-ng \
sed \ sed \
tar \ tar \
util-linux util-linux
@@ -49,30 +77,37 @@ curl -LO $MQ_URL
tar -zxvf ./*.tar.gz tar -zxvf ./*.tar.gz
# Remove packages only needed by this script # Remove packages only needed by this script
apt-get purge -y \ $UBUNTU && apt-get purge -y \
ca-certificates \ ca-certificates \
curl curl
# Note: ca-certificates and curl are installed by default in RHEL
# Remove any orphaned packages # Remove any orphaned packages
apt-get autoremove -y $UBUNTU && apt-get autoremove -y
# Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images # Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images
groupadd --system --gid 999 mqm $UBUNTU && groupadd --system --gid 999 mqm
useradd --system --uid 999 --gid mqm mqm $UBUNTU && useradd --system --uid 999 --gid mqm mqm
$RHEL && groupadd --system --gid 888 mqm
$RHEL && useradd --system --uid 888 --gid mqm mqm
usermod -G mqm root usermod -G mqm root
# Find directory containing .deb files # Find directory containing .deb files
DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1) $UBUNTU && DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1)
$RHEL && DIR_RPM=$(find ${DIR_EXTRACT} -name "*.rpm" -printf "%h\n" | sort -u | head -1)
# Find location of mqlicense.sh # Find location of mqlicense.sh
MQLICENSE=$(find ${DIR_EXTRACT} -name "mqlicense.sh") MQLICENSE=$(find ${DIR_EXTRACT} -name "mqlicense.sh")
# Accept the MQ license # Accept the MQ license
${MQLICENSE} -text_only -accept ${MQLICENSE} -text_only -accept
echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list $UBUNTU && echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list
# Install MQ using the DEB packages # Install MQ using the DEB packages
apt-get update $UBUNTU && apt-get update
apt-get install -y $MQ_PACKAGES $UBUNTU && apt-get install -y $MQ_PACKAGES
$RHEL && cd $DIR_RPM && rpm -ivh $MQ_PACKAGES
# Remove 32-bit libraries from 64-bit container # Remove 32-bit libraries from 64-bit container
find /opt/mqm /var/mqm -type f -exec file {} \; | awk -F: '/ELF 32-bit/{print $1}' | xargs --no-run-if-empty rm -f find /opt/mqm /var/mqm -type f -exec file {} \; | awk -F: '/ELF 32-bit/{print $1}' | xargs --no-run-if-empty rm -f
@@ -84,19 +119,21 @@ find /opt/mqm -name '*.tar.gz' -delete
/opt/mqm/bin/setmqinst -p /opt/mqm -i /opt/mqm/bin/setmqinst -p /opt/mqm -i
# Clean up all the downloaded files # Clean up all the downloaded files
rm -f /etc/apt/sources.list.d/IBM_MQ.list $UBUNTU && rm -f /etc/apt/sources.list.d/IBM_MQ.list
rm -rf ${DIR_EXTRACT} rm -rf ${DIR_EXTRACT}
#### Apply any bug fixes not included in base Ubuntu or MQ image. # Apply any bug fixes not included in base Ubuntu or MQ image.
# Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run
apt-get upgrade -y libdb5.3
#### End of bug fixes
# Clean up cached apt files # End of bug fixes
rm -rf /var/lib/apt/lists/*
# Clean up cached files
$UBUNTU && rm -rf /var/lib/apt/lists/*
$RHEL && yum -y clean all
$RHEL && rm -rf /var/cache/yum/*
# Optional: Update the command prompt with the MQ version # Optional: Update the command prompt with the MQ version
echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot $UBUNTU && echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot
# Remove the directory structure under /var/mqm which was created by the installer # Remove the directory structure under /var/mqm which was created by the installer
rm -rf /var/mqm rm -rf /var/mqm
@@ -113,4 +150,7 @@ ln -s /mnt/mqm/data /var/mqm
# Optional: Set these values for the Bluemix Vulnerability Report # Optional: Set these values for the Bluemix Vulnerability Report
sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' /etc/login.defs sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' /etc/login.defs
sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' /etc/login.defs sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' /etc/login.defs
sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' /etc/pam.d/common-password
$UBUNTU && PAM_FILE=/etc/pam.d/common-password
$RHEL && PAM_FILE=/etc/pam.d/password-auth
sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' $PAM_FILE

22
test/docker/docker_api_test.go
View File

@@ -91,18 +91,24 @@ func TestSecurityVulnerabilities(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
containerConfig := container.Config{ // containerConfig := container.Config{
// Override the entrypoint to make "apt" only receive security updates, then check for updates // // Override the entrypoint to make "apt" only receive security updates, then check for updates
Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"}, // Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"},
// }
// id := runContainer(t, cli, &containerConfig)
// defer cleanContainer(t, cli, id)
// // rc is the return code from apt-get
// rc := waitForContainer(t, cli, id, 10)
rc, _ := runContainerOneShot(t, cli, "bash", "-c", "test -d /etc/apt")
if rc != 0 {
t.Skip("Skipping test because container is not Ubuntu-based")
} }
id := runContainer(t, cli, &containerConfig) // Override the entrypoint to make "apt" only receive security updates, then check for updates
defer cleanContainer(t, cli, id) rc, log := runContainerOneShot(t, cli, "bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade")
// rc is the return code from apt-get
rc := waitForContainer(t, cli, id, 10)
if rc != 0 { if rc != 0 {
t.Fatalf("Expected success, got %v", rc) t.Fatalf("Expected success, got %v", rc)
} }
log := inspectLogs(t, cli, id)
lines := strings.Split(strings.TrimSpace(log), "\n") lines := strings.Split(strings.TrimSpace(log), "\n")
if len(lines) > 0 && lines[0] != "" { if len(lines) > 0 && lines[0] != "" {
t.Errorf("Expected no vulnerabilities, found the following:\n%v", log) t.Errorf("Expected no vulnerabilities, found the following:\n%v", log)

9
test/docker/docker_api_test_util.go
View File

@@ -120,6 +120,15 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C
return ctr.ID return ctr.ID
} }
func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (int64, string) {
containerConfig := container.Config{
Entrypoint: command,
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
return waitForContainer(t, cli, id, 10), inspectLogs(t, cli, id)
}
func startContainer(t *testing.T, cli *client.Client, ID string) { func startContainer(t *testing.T, cli *client.Client, ID string) {
t.Logf("Starting container: %v", ID) t.Logf("Starting container: %v", ID)
startOptions := types.ContainerStartOptions{} startOptions := types.ContainerStartOptions{}

90
test/kubernetes/helm_test_util.go
View File

@@ -17,36 +17,34 @@ package main
import ( import (
"bytes" "bytes"
"fmt"
"os" "os"
"os/exec"
"runtime"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
"unicode"
"golang.org/x/sys/unix" "github.com/ibm-messaging/mq-container/internal/command"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd"
) )
func imageName() string { func image(t *testing.T) string {
v, ok := os.LookupEnv("TEST_REPO") v, ok := os.LookupEnv("TEST_IMAGE")
if !ok { if !ok {
v = "ibmcom/mq" t.Fatal("TEST_IMAGE environment variable not set")
} }
return v return v
} }
func imageTag() string { func imageName(t *testing.T) string {
v, ok := os.LookupEnv("TEST_TAG") return strings.Fields(strings.Replace(image(t), ":", " ", -1))[0]
if !ok { }
v = "latest"
} func imageTag(t *testing.T) string {
return v return strings.Fields(strings.Replace(image(t), ":", " ", -1))[1]
} }
func chartName() string { func chartName() string {
@@ -57,32 +55,6 @@ func chartName() string {
return v return v
} }
// runCommand runs an OS command. On Linux it waits for the command to
// complete and returns the exit status (return code).
// TODO: duplicated from cmd/runmqserver/main.go
func runCommand(t *testing.T, name string, arg ...string) (string, int, error) {
t.Logf("Running command: %v %v", name, strings.Trim(fmt.Sprintf("%v", arg), "[]"))
cmd := exec.Command(name, arg...)
// Run the command and wait for completion
out, err := cmd.CombinedOutput()
if err != nil {
var rc int
// Only works on Linux
if runtime.GOOS == "linux" {
var ws unix.WaitStatus
unix.Wait4(cmd.Process.Pid, &ws, 0, nil)
rc = ws.ExitStatus()
} else {
rc = -1
}
if rc == 0 {
return string(out), rc, nil
}
return string(out), rc, err
}
return string(out), 0, nil
}
func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string { func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string {
pods := getPodsForHelmRelease(t, cs, release) pods := getPodsForHelmRelease(t, cs, release)
opt := v1.PodLogOptions{} opt := v1.PodLogOptions{}
@@ -98,7 +70,7 @@ func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string
func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values ...string) { func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values ...string) {
chart := chartName() chart := chartName()
tag := "latest" tag := imageTag(t)
arg := []string{ arg := []string{
"install", "install",
"--debug", "--debug",
@@ -106,7 +78,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
"--name", "--name",
release, release,
"--set", "--set",
"image.repository=" + imageName(), "image.repository=" + imageName(t),
"--set", "--set",
"image.tag=" + tag, "image.tag=" + tag,
"--set", "--set",
@@ -116,7 +88,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
for _, value := range values { for _, value := range values {
arg = append(arg, "--set", value) arg = append(arg, "--set", value)
} }
out, _, err := runCommand(t, "helm", arg...) out, _, err := command.Run("helm", arg...)
t.Log(out) t.Log(out)
if err != nil { if err != nil {
t.Error(out) t.Error(out)
@@ -127,7 +99,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
func helmDelete(t *testing.T, cs *kubernetes.Clientset, release string) { func helmDelete(t *testing.T, cs *kubernetes.Clientset, release string) {
t.Log("Deleting Helm release") t.Log("Deleting Helm release")
t.Log(inspectLogs(t, cs, release)) t.Log(inspectLogs(t, cs, release))
out, _, err := runCommand(t, "helm", "delete", "--purge", release) out, _, err := command.Run("helm", "delete", "--purge", release)
if err != nil { if err != nil {
t.Error(out) t.Error(out)
t.Fatal(err) t.Fatal(err)
@@ -151,7 +123,10 @@ func helmDeletePVC(t *testing.T, cs *kubernetes.Clientset, release string) {
} }
func kubeLogin(t *testing.T) *kubernetes.Clientset { func kubeLogin(t *testing.T) *kubernetes.Clientset {
kc := os.Getenv("HOME") + "/.kube/config" kc := os.Getenv("KUBECONFIG")
if kc == "" {
kc = os.Getenv("HOME") + "/.kube/config"
}
c, err := clientcmd.BuildConfigFromFlags("", kc) c, err := clientcmd.BuildConfigFromFlags("", kc)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -167,7 +142,7 @@ func kubeExec(t *testing.T, podName string, name string, arg ...string) (string,
// Current version of Kubernetes Go client doesn't support "exec", so run this via the command line // Current version of Kubernetes Go client doesn't support "exec", so run this via the command line
param := []string{"exec", podName, "--", name} param := []string{"exec", podName, "--", name}
param = append(param, arg...) param = append(param, arg...)
return runCommand(t, "kubectl", param...) return command.Run("kubectl", param...)
} }
func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) { func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) {
@@ -199,12 +174,12 @@ func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) {
for { for {
// Current version of Kubernetes Go client doesn't support "exec", so run this via the command line // Current version of Kubernetes Go client doesn't support "exec", so run this via the command line
// TODO: If we run "chkmqready" here, it doesn't seem to work // TODO: If we run "chkmqready" here, it doesn't seem to work
//out, _, err := runCommand(t, "kubectl", "exec", podName, "--", "dspmq") //out, _, err := command.Run(t, "kubectl", "exec", podName, "--", "dspmq")
out, _, err := kubeExec(t, podName, "dspmq") out, _, err := kubeExec(t, podName, "dspmq")
//out, rc, err := runCommand(t, "kubectl", "exec", podName, "--", "chkmqready") //out, rc, err := command.Run(t, "kubectl", "exec", podName, "--", "chkmqready")
if err != nil { if err != nil {
t.Error(out) t.Error(out)
out2, _, err2 := runCommand(t, "kubectl", "describe", "pod", podName) out2, _, err2 := command.Run("kubectl", "describe", "pod", podName)
if err2 == nil { if err2 == nil {
t.Log(out2) t.Log(out2)
} }
@@ -258,17 +233,32 @@ func volumesAvailable(t *testing.T, cs *kubernetes.Clientset) bool {
return false return false
} }
// digitsOnly returns only the digits from the supplied string
func digitsOnly(s string) string {
return strings.Map(
func(r rune) rune {
if unicode.IsDigit(r) {
return r
}
return -1
},
s,
)
}
// assertKubeVersion is used to assert that a test requires a specific version of Kubernetes // assertKubeVersion is used to assert that a test requires a specific version of Kubernetes
func assertKubeVersion(t *testing.T, cs *kubernetes.Clientset, major int, minor int) { func assertKubeVersion(t *testing.T, cs *kubernetes.Clientset, major int, minor int) {
v, err := cs.Discovery().ServerVersion() v, err := cs.Discovery().ServerVersion()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
maj, err := strconv.Atoi(v.Major) // Make sure we use only the digits from the version, to account for things like "1.8+"
maj, err := strconv.Atoi(digitsOnly(v.Major))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
min, err := strconv.Atoi(v.Minor) // Make sure we use only the digits from the version, to account for things like "1.8+"
min, err := strconv.Atoi(digitsOnly(v.Minor))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }