academy/mq-container
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ability to use different base image

Browse Source
This commit is contained in:
Arthur Barr
2017-12-11 09:12:39 +00:00
parent d6182bf2fc
commit a2326dc0f6
8 changed files with 262 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile-server
View File

@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARG BASE_IMAGE=ubuntu:16.04
###############################################################################
# Build stage to build Go code
###############################################################################
@@ -29,14 +31,14 @@ RUN go test -v ./cmd/... ./internal/...
###############################################################################
# Main build stage, to build MQ image
###############################################################################
FROM ubuntu:16.04
FROM $BASE_IMAGE
# The URL to download the MQ installer from in tar.gz format
# This assumes an archive containing the MQ Debian (.deb) install packages
ARG MQ_URL
# The MQ packages to install
ARG MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams"
# The MQ packages to install - see install-mq.sh for default value
ARG MQ_PACKAGES
COPY install-mq.sh /usr/local/bin/

124
Makefile
View File

@@ -12,45 +12,72 @@
# See the License for the specific language governing permissions and
# limitations under the License.
BUILD_SERVER_CONTAINER=build-server
DOCKER_TAG_ARCH ?= $(shell uname -m)
# By default, all Docker client commands are run inside a Docker container.
# This means that newer features of the client can be used, even with an older daemon.
DOCKER ?= docker
DOCKER_TAG ?= latest-$(DOCKER_TAG_ARCH)
DOCKER_REPO_DEVSERVER ?= mq-devserver
DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver
DOCKER_FULL_DEVSERVER = $(DOCKER_REPO_DEVSERVER):$(DOCKER_TAG)
DOCKER_FULL_ADVANCEDSERVER = $(DOCKER_REPO_ADVANCEDSERVER):$(DOCKER_TAG)
# MQ_PACKAGES is the list of MQ packages to install
MQ_PACKAGES ?=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams
###############################################################################
# Conditional variables - you can override the values of these variables from
# the command line
###############################################################################
# BASE_IMAGE is the base image to use for MQ, for example "ubuntu" or "rhel"
BASE_IMAGE ?= ubuntu:16.04
# MQ_VERSION is the fully qualified MQ version number to build
MQ_VERSION ?= 9.0.4.0
# Archive names for IBM MQ Continuous Delivery Release for Ubuntu
MQ_ARCHIVE_9.0.3.0_ppc64le=CNJR5ML.tar.gz
MQ_ARCHIVE_9.0.3.0_s390x=CNJR6ML.tar.gz
MQ_ARCHIVE_9.0.3.0_x86_64=CNJR7ML.tar.gz
MQ_ARCHIVE_9.0.4.0_ppc64le=CNLE2ML.tar.gz
MQ_ARCHIVE_9.0.4.0_s390x=CNLE3ML.tar.gz
MQ_ARCHIVE_9.0.4.0_x86_64=CNLE4ML.tar.gz
# Archive names for IBM MQ Advanced for Developers for Ubuntu
MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz
MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz
MQ_ARCHIVE ?= $(MQ_ARCHIVE_$(MQ_VERSION)_$(DOCKER_TAG_ARCH))
MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION))
# MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can
# be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture
# Does not apply to MQ Advanced for Developers.
MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz
# Options to `go test` for the Docker tests
TEST_OPTS_DOCKER ?=
# Options to `go test` for the Kubernetes tests
TEST_OPTS_KUBERNETES ?=
TEST_IMAGE ?= $(DOCKER_FULL_ADVANCEDSERVER)
NUM_CPU=$(shell docker info --format "{{ .NCPU }}")
# MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image
MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
# MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced for Developers image
MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG)
# DOCKER is the Docker command to run
DOCKER ?= docker
###############################################################################
# Other variables
###############################################################################
# ARCH is the platform architecture (e.g. x86_64, ppc64le or s390x)
ARCH = $(shell uname -m)
# BUILD_SERVER_CONTAINER is the name of the web server container used at build time
BUILD_SERVER_CONTAINER=build-server
# NUM_CPU is the number of CPUs available to Docker. Used to control how many
# test run in parallel
NUM_CPU=$(shell docker info --format "{{ .NCPU }}")
# BASE_IMAGE_TAG is a normalized version of BASE_IMAGE, suitable for use in a Docker tag
BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE)))
# Try to figure out which archive to use from the BASE_IMAGE
ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu"
MQ_ARCHIVE_TYPE=UBUNTU
else
MQ_ARCHIVE_TYPE=LINUX
endif
# Try to figure out which archive to use from the architecture
ifeq "$(ARCH)" "x86_64"
MQ_ARCHIVE_ARCH=X86-64
else ifeq "$(ARCH)" "ppc64le"
MQ_ARCHIVE_ARCH=LE_POWER
else ifeq "$(ARCH)" "s390x"
MQ_ARCHIVE_ARCH=SYSTEM_Z
endif
# Archive names for IBM MQ Advanced for Developers for Ubuntu
MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz
MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz
# MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced
# for Developers can be installed
MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION))
###############################################################################
# Build targets
###############################################################################
.PHONY: vars
vars:
echo $(DOCKER_SERVER_VERSION_MAJOR)
echo $(DOCKER_SERVER_VERSION_MINOR)
echo $(DOCKER_CLIENT_VERSION_MAJOR)
echo $(DOCKER_CLIENT_VERSION_MINOR)
#ifeq "$(findstring ubuntu,$(BASE_IMAGE))","ubuntu"
@echo $(MQ_ARCHIVE_ARCH)
@echo $(MQ_ARCHIVE_TYPE)
@echo $(MQ_ARCHIVE)
.PHONY: default
default: build-devserver test
@@ -105,16 +132,16 @@ test-unit:
.PHONY: test-advancedserver
test-advancedserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END)))
cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
.PHONY: test-devserver
test-devserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END)))
cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test -parallel $(NUM_CPU)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU)
.PHONY: test-advancedserver-cover
test-advancedserver-cover: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_REPO_ADVANCEDSERVER) on Docker with code coverage"$(END)))
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on Docker with code coverage"$(END)))
rm -f ./coverage/unit*.cov
# Run unit tests with coverage, for each package under 'internal'
go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{}
@@ -124,7 +151,7 @@ test-advancedserver-cover: test/docker/vendor
rm -f ./test/docker/coverage/*.cov
rm -f ./coverage/docker.*
cd test/docker && TEST_IMAGE=$(DOCKER_REPO_ADVANCEDSERVER):cover go test $(TEST_OPTS_DOCKER)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER)-cover go test $(TEST_OPTS_DOCKER)
echo 'mode: count' > ./coverage/docker.cov
tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov
go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html
@@ -135,15 +162,15 @@ test-advancedserver-cover: test/docker/vendor
.PHONY: test-kubernetes-devserver
test-kubernetes-devserver: test/kubernetes/vendor
$(call test-kubernetes,$(DOCKER_REPO_DEVSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-dev")
$(call test-kubernetes,$(MQ_IMAGE_DEVSERVER),"../../charts/ibm-mqadvanced-server-dev")
.PHONY: test-kubernetes-advancedserver
test-kubernetes-advancedserver: test/kubernetes/vendor
$(call test-kubernetes,$(DOCKER_REPO_ADVANCEDSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-prod")
$(call test-kubernetes,$(MQ_IMAGE_ADVANCEDSERVER),"../../charts/ibm-mqadvanced-server-prod")
define test-kubernetes
$(info $(SPACER)$(shell printf $(TITLE)"Test $1:$2 on Kubernetes"$(END)))
cd test/kubernetes && TEST_REPO=$1 TEST_TAG=$2 TEST_CHART=$3 go test $(TEST_OPTS_KUBERNETES)
$(info $(SPACER)$(shell printf $(TITLE)"Test $1 on Kubernetes"$(END)))
cd test/kubernetes && TEST_IMAGE=$1 TEST_CHART=$2 go test $(TEST_OPTS_KUBERNETES)
endef
define docker-build-mq
@@ -159,13 +186,14 @@ define docker-build-mq
--detach \
nginx:alpine
# Make sure we have the latest base image
$(DOCKER) pull ubuntu:16.04
$(DOCKER) pull $(BASE_IMAGE)
# Build the new image
$(DOCKER) build \
--tag $1 \
--file $2 \
--network build \
--build-arg MQ_URL=http://build:80/$3 \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--label IBM_PRODUCT_ID=$4 \
--label IBM_PRODUCT_NAME=$5 \
--label IBM_PRODUCT_VERSION=$6 \
@@ -184,27 +212,27 @@ docker-version:
.PHONY: build-advancedserver
build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version
$(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END)))
$(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION))
$(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH)
$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END)))
$(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION))
# $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(ARCH)-$(subst :,-,$(BASE_IMAGE))
.PHONY: build-devserver
build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version
@test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1)
$(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END)))
$(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
$(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH)
$(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END)))
$(call docker-build-mq,$(MQ_IMAGE_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
# $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(ARCH)
.PHONY: build-advancedserver-cover
build-advancedserver-cover: docker-version
$(DOCKER) build -t $(DOCKER_REPO_ADVANCEDSERVER):cover -f Dockerfile-server.cover .
$(DOCKER) build -t $(MQ_IMAGE_ADVANCEDSERVER)-cover -f Dockerfile-server.cover .
# .PHONY: build-web
# build-web: build downloads/CNJR7ML.tar.gz
# $(call docker-build-mq,mq-web:latest-$(DOCKER_TAG_ARCH),Dockerfile-mq-web)
# $(call docker-build-mq,mq-web:latest-$(ARCH),Dockerfile-mq-web)
.PHONY: build-explorer
build-explorer: downloads/$(MQ_ARCHIVE_DEV)
$(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
$(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
include formatting.mk

35
docs/building.md Normal file
View File

@@ -0,0 +1,35 @@
# Building a Docker image
## Prerequisites
You need to ensure you have the following tools installed:
* [Docker](https://www.docker.com/) V17.05 or later
* [GNU make](https://www.gnu.org/software/make/)
## Building a production image
This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures.
1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `IBM_MQ_9.0.4.0_UBUNTU_X86-64.tar.gz` for MQ V9.0.4 for Ubuntu on x86_64 architecture) in the `downloads` directory
2. Run `make build-advancedserver`
You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example:
```bash
MQ_VERSION=9.0.4.0 make build-advancedserver
```
If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
```bash
MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver
```
## Building on a different base image
By default, the MQ images use Ubuntu as the base layer. You can build using a Red Hat Enterprise Linux compatible base layer by setting the `BASE_IMAGE` environment variable. For example:
```
BASE_IMAGE=centos:7 make build-advancedserver
```
The `make` tool will try and locate the right archive file under the `downloads` directory, based on your platform architecture and your `MQ_VERSION` environment variable, for example `IBM_MQ_9.0.4.0_LINUX_X86_64.tar.gz` for MQ V9.0.4.0 on x86_64. You can also set the `MQ_ARCHIVE` environment variable to set the specific file name.
Note that if you are using Red Hat Enterprise Linux, you will need to create your own base image layer, with your subscription enabled, as described [here](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/getting_started_with_containers/get_started_with_docker_formatted_container_images). The MQ image build needs to install some additional packages, and a subscription is required to access the Red Hat repositories.

53
docs/developing.md → docs/testing.md
View File

@@ -1,34 +1,20 @@
# Developing
# Testing
## Prerequisites
You need to ensure you have the following tools installed:
* [Docker](https://www.docker.com/) V17.05 or later
* [Docker](https://www.docker.com/)
* [GNU make](https://www.gnu.org/software/make/)
You might also need the following tools installed:
* [Go](https://golang.org/) - only needed for running the tests
* [Glide](https://glide.sh/) - only needed if you update the main dependencies
* [dep](https://github.com/golang/dep) (official Go dependency management tool) - only needed to prepare for running the tests
* [dep](https://github.com/golang/dep) (official Go dependency management tool) - needed to prepare for running the tests
* [Helm](https://helm.sh) - only needed for running the Kubernetes tests
For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/).
## Building a production image
This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures.
## Preparing to run the tests
The test dependencies are not included with the source code, so you need to download them before you can run them. This can be done with the following command, which uses the `dep` tool:
1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4 on x86_64 architecture) in the `downloads` directory
2. Run `make build-advancedserver`
You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example:
```bash
MQ_VERSION=9.0.3.0 make build-advancedserver
```
If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
```bash
MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver
make deps
```
## Running the tests
@@ -39,30 +25,31 @@ There are three main sets of tests:
3. Kubernetes tests, which test the Helm charts (and the Docker image) via [Helm](https://helm.sh)
### Running the Docker tests
The Docker tests can be run locally. Before you run them for the first time, you need to download the test dependencies:
```
make deps
```
You can then run the tests, for example:
The Docker tests can be run locally on a machine with Docker. For example:
```
make test-devserver
```
or:
```
make test-advancedserver
```
You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example:
```
MQ_IMAGE_ADVANCEDSERVER=mqadvanced-server9.0.4.0-x86_64-ubuntu-16.04 make test-advancedserver
```
You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command::
```
TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver
```
You can also use the same environment variables you specified when [building](./building), for example, the following will try and test an image called `mqadvanced-server9.0.3.0-x86_64-ubuntu-16.04`:
```
MQ_VERSION=9.0.3.0 make test-advancedserver
```
### Running the Docker tests with code coverage
You can produce code coverage results from the Docker tests by running the following:
@@ -79,5 +66,5 @@ In order to generate code coverage metrics from the Docker tests, the build step
For the Kubernetes tests, you need to have built the Docker image, and pushed it to the registry used by your Kubernetes cluster. Most of the configuration used by the tests is picked up from your `kubectl` configuration, but you will typically need to specify the image details. For example:
```bash
DOCKER_REPO_DEVSERVER=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver
MQ_IMAGE=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver
```

80
install-mq.sh
View File

@@ -18,11 +18,20 @@
# Fail on any non-zero return code
set -ex
export DEBIAN_FRONTEND=noninteractive
test -f /usr/bin/yum && RHEL=true || RHEL=false
test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false
# Install additional packages required by MQ, this install process and the runtime scripts
apt-get update
apt-get install -y --no-install-recommends \
# If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults
if [ -z $MQ_PACKAGES ]; then
$UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams"
$RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm"
fi
if ($UBUNTU); then
export DEBIAN_FRONTEND=noninteractive
# Install additional packages required by MQ, this install process and the runtime scripts
apt-get update
apt-get install -y --no-install-recommends \
bash \
bc \
ca-certificates \
@@ -40,6 +49,25 @@ apt-get install -y --no-install-recommends \
sed \
tar \
util-linux
fi
# Install additional packages required by MQ, this install process and the runtime scripts
$RHEL && yum -y install \
bash \
bc \
ca-certificates \
coreutils \
curl \
file \
findutils \
gawk \
glibc-common \
grep \
passwd \
procps-ng \
sed \
tar \
util-linux
# Download and extract the MQ installation files
DIR_EXTRACT=/tmp/mq
@@ -49,30 +77,37 @@ curl -LO $MQ_URL
tar -zxvf ./*.tar.gz
# Remove packages only needed by this script
apt-get purge -y \
$UBUNTU && apt-get purge -y \
ca-certificates \
curl
# Note: ca-certificates and curl are installed by default in RHEL
# Remove any orphaned packages
apt-get autoremove -y
$UBUNTU && apt-get autoremove -y
# Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images
groupadd --system --gid 999 mqm
useradd --system --uid 999 --gid mqm mqm
$UBUNTU && groupadd --system --gid 999 mqm
$UBUNTU && useradd --system --uid 999 --gid mqm mqm
$RHEL && groupadd --system --gid 888 mqm
$RHEL && useradd --system --uid 888 --gid mqm mqm
usermod -G mqm root
# Find directory containing .deb files
DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1)
$UBUNTU && DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1)
$RHEL && DIR_RPM=$(find ${DIR_EXTRACT} -name "*.rpm" -printf "%h\n" | sort -u | head -1)
# Find location of mqlicense.sh
MQLICENSE=$(find ${DIR_EXTRACT} -name "mqlicense.sh")
# Accept the MQ license
${MQLICENSE} -text_only -accept
echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list
$UBUNTU && echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list
# Install MQ using the DEB packages
apt-get update
apt-get install -y $MQ_PACKAGES
$UBUNTU && apt-get update
$UBUNTU && apt-get install -y $MQ_PACKAGES
$RHEL && cd $DIR_RPM && rpm -ivh $MQ_PACKAGES
# Remove 32-bit libraries from 64-bit container
find /opt/mqm /var/mqm -type f -exec file {} \; | awk -F: '/ELF 32-bit/{print $1}' | xargs --no-run-if-empty rm -f
@@ -84,19 +119,21 @@ find /opt/mqm -name '*.tar.gz' -delete
/opt/mqm/bin/setmqinst -p /opt/mqm -i
# Clean up all the downloaded files
rm -f /etc/apt/sources.list.d/IBM_MQ.list
$UBUNTU && rm -f /etc/apt/sources.list.d/IBM_MQ.list
rm -rf ${DIR_EXTRACT}
#### Apply any bug fixes not included in base Ubuntu or MQ image.
# Apply any bug fixes not included in base Ubuntu or MQ image.
# Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run
apt-get upgrade -y libdb5.3
#### End of bug fixes
# Clean up cached apt files
rm -rf /var/lib/apt/lists/*
# End of bug fixes
# Clean up cached files
$UBUNTU && rm -rf /var/lib/apt/lists/*
$RHEL && yum -y clean all
$RHEL && rm -rf /var/cache/yum/*
# Optional: Update the command prompt with the MQ version
echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot
$UBUNTU && echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot
# Remove the directory structure under /var/mqm which was created by the installer
rm -rf /var/mqm
@@ -113,4 +150,7 @@ ln -s /mnt/mqm/data /var/mqm
# Optional: Set these values for the Bluemix Vulnerability Report
sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' /etc/login.defs
sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' /etc/login.defs
sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' /etc/pam.d/common-password
$UBUNTU && PAM_FILE=/etc/pam.d/common-password
$RHEL && PAM_FILE=/etc/pam.d/password-auth
sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' $PAM_FILE

22
test/docker/docker_api_test.go
View File

@@ -91,18 +91,24 @@ func TestSecurityVulnerabilities(t *testing.T) {
if err != nil {
t.Fatal(err)
}
containerConfig := container.Config{
// Override the entrypoint to make "apt" only receive security updates, then check for updates
Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"},
// containerConfig := container.Config{
// // Override the entrypoint to make "apt" only receive security updates, then check for updates
// Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"},
// }
// id := runContainer(t, cli, &containerConfig)
// defer cleanContainer(t, cli, id)
// // rc is the return code from apt-get
// rc := waitForContainer(t, cli, id, 10)
rc, _ := runContainerOneShot(t, cli, "bash", "-c", "test -d /etc/apt")
if rc != 0 {
t.Skip("Skipping test because container is not Ubuntu-based")
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
// rc is the return code from apt-get
rc := waitForContainer(t, cli, id, 10)
// Override the entrypoint to make "apt" only receive security updates, then check for updates
rc, log := runContainerOneShot(t, cli, "bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade")
if rc != 0 {
t.Fatalf("Expected success, got %v", rc)
}
log := inspectLogs(t, cli, id)
lines := strings.Split(strings.TrimSpace(log), "\n")
if len(lines) > 0 && lines[0] != "" {
t.Errorf("Expected no vulnerabilities, found the following:\n%v", log)

9
test/docker/docker_api_test_util.go
View File

@@ -120,6 +120,15 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C
return ctr.ID
}
func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (int64, string) {
containerConfig := container.Config{
Entrypoint: command,
}
id := runContainer(t, cli, &containerConfig)
defer cleanContainer(t, cli, id)
return waitForContainer(t, cli, id, 10), inspectLogs(t, cli, id)
}
func startContainer(t *testing.T, cli *client.Client, ID string) {
t.Logf("Starting container: %v", ID)
startOptions := types.ContainerStartOptions{}

90
test/kubernetes/helm_test_util.go
View File

@@ -17,36 +17,34 @@ package main
import (
"bytes"
"fmt"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"testing"
"time"
"unicode"
"golang.org/x/sys/unix"
"github.com/ibm-messaging/mq-container/internal/command"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/tools/clientcmd"
)
func imageName() string {
v, ok := os.LookupEnv("TEST_REPO")
func image(t *testing.T) string {
v, ok := os.LookupEnv("TEST_IMAGE")
if !ok {
v = "ibmcom/mq"
t.Fatal("TEST_IMAGE environment variable not set")
}
return v
}
func imageTag() string {
v, ok := os.LookupEnv("TEST_TAG")
if !ok {
v = "latest"
}
return v
func imageName(t *testing.T) string {
return strings.Fields(strings.Replace(image(t), ":", " ", -1))[0]
}
func imageTag(t *testing.T) string {
return strings.Fields(strings.Replace(image(t), ":", " ", -1))[1]
}
func chartName() string {
@@ -57,32 +55,6 @@ func chartName() string {
return v
}
// runCommand runs an OS command. On Linux it waits for the command to
// complete and returns the exit status (return code).
// TODO: duplicated from cmd/runmqserver/main.go
func runCommand(t *testing.T, name string, arg ...string) (string, int, error) {
t.Logf("Running command: %v %v", name, strings.Trim(fmt.Sprintf("%v", arg), "[]"))
cmd := exec.Command(name, arg...)
// Run the command and wait for completion
out, err := cmd.CombinedOutput()
if err != nil {
var rc int
// Only works on Linux
if runtime.GOOS == "linux" {
var ws unix.WaitStatus
unix.Wait4(cmd.Process.Pid, &ws, 0, nil)
rc = ws.ExitStatus()
} else {
rc = -1
}
if rc == 0 {
return string(out), rc, nil
}
return string(out), rc, err
}
return string(out), 0, nil
}
func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string {
pods := getPodsForHelmRelease(t, cs, release)
opt := v1.PodLogOptions{}
@@ -98,7 +70,7 @@ func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string
func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values ...string) {
chart := chartName()
tag := "latest"
tag := imageTag(t)
arg := []string{
"install",
"--debug",
@@ -106,7 +78,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
"--name",
release,
"--set",
"image.repository=" + imageName(),
"image.repository=" + imageName(t),
"--set",
"image.tag=" + tag,
"--set",
@@ -116,7 +88,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
for _, value := range values {
arg = append(arg, "--set", value)
}
out, _, err := runCommand(t, "helm", arg...)
out, _, err := command.Run("helm", arg...)
t.Log(out)
if err != nil {
t.Error(out)
@@ -127,7 +99,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values
func helmDelete(t *testing.T, cs *kubernetes.Clientset, release string) {
t.Log("Deleting Helm release")
t.Log(inspectLogs(t, cs, release))
out, _, err := runCommand(t, "helm", "delete", "--purge", release)
out, _, err := command.Run("helm", "delete", "--purge", release)
if err != nil {
t.Error(out)
t.Fatal(err)
@@ -151,7 +123,10 @@ func helmDeletePVC(t *testing.T, cs *kubernetes.Clientset, release string) {
}
func kubeLogin(t *testing.T) *kubernetes.Clientset {
kc := os.Getenv("HOME") + "/.kube/config"
kc := os.Getenv("KUBECONFIG")
if kc == "" {
kc = os.Getenv("HOME") + "/.kube/config"
}
c, err := clientcmd.BuildConfigFromFlags("", kc)
if err != nil {
t.Fatal(err)
@@ -167,7 +142,7 @@ func kubeExec(t *testing.T, podName string, name string, arg ...string) (string,
// Current version of Kubernetes Go client doesn't support "exec", so run this via the command line
param := []string{"exec", podName, "--", name}
param = append(param, arg...)
return runCommand(t, "kubectl", param...)
return command.Run("kubectl", param...)
}
func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) {
@@ -199,12 +174,12 @@ func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) {
for {
// Current version of Kubernetes Go client doesn't support "exec", so run this via the command line
// TODO: If we run "chkmqready" here, it doesn't seem to work
//out, _, err := runCommand(t, "kubectl", "exec", podName, "--", "dspmq")
//out, _, err := command.Run(t, "kubectl", "exec", podName, "--", "dspmq")
out, _, err := kubeExec(t, podName, "dspmq")
//out, rc, err := runCommand(t, "kubectl", "exec", podName, "--", "chkmqready")
//out, rc, err := command.Run(t, "kubectl", "exec", podName, "--", "chkmqready")
if err != nil {
t.Error(out)
out2, _, err2 := runCommand(t, "kubectl", "describe", "pod", podName)
out2, _, err2 := command.Run("kubectl", "describe", "pod", podName)
if err2 == nil {
t.Log(out2)
}
@@ -258,17 +233,32 @@ func volumesAvailable(t *testing.T, cs *kubernetes.Clientset) bool {
return false
}
// digitsOnly returns only the digits from the supplied string
func digitsOnly(s string) string {
return strings.Map(
func(r rune) rune {
if unicode.IsDigit(r) {
return r
}
return -1
},
s,
)
}
// assertKubeVersion is used to assert that a test requires a specific version of Kubernetes
func assertKubeVersion(t *testing.T, cs *kubernetes.Clientset, major int, minor int) {
v, err := cs.Discovery().ServerVersion()
if err != nil {
t.Fatal(err)
}
maj, err := strconv.Atoi(v.Major)
// Make sure we use only the digits from the version, to account for things like "1.8+"
maj, err := strconv.Atoi(digitsOnly(v.Major))
if err != nil {
t.Fatal(err)
}
min, err := strconv.Atoi(v.Minor)
// Make sure we use only the digits from the version, to account for things like "1.8+"
min, err := strconv.Atoi(digitsOnly(v.Minor))
if err != nil {
t.Fatal(err)
}