diff --git a/Dockerfile-server b/Dockerfile-server index 3774d5a..224774f 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +ARG BASE_IMAGE=ubuntu:16.04 + ############################################################################### # Build stage to build Go code ############################################################################### @@ -29,14 +31,14 @@ RUN go test -v ./cmd/... ./internal/... ############################################################################### # Main build stage, to build MQ image ############################################################################### -FROM ubuntu:16.04 +FROM $BASE_IMAGE # The URL to download the MQ installer from in tar.gz format # This assumes an archive containing the MQ Debian (.deb) install packages ARG MQ_URL -# The MQ packages to install -ARG MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" +# The MQ packages to install - see install-mq.sh for default value +ARG MQ_PACKAGES COPY install-mq.sh /usr/local/bin/ diff --git a/Dockerfile-server.cover b/Dockerfile-server.cover index 621390b..0122224 100644 --- a/Dockerfile-server.cover +++ b/Dockerfile-server.cover @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +ARG BASE_IMAGE + # Build stage to build Go code FROM golang:1.9 as builder WORKDIR /go/src/github.com/ibm-messaging/mq-container/ @@ -20,7 +22,7 @@ COPY internal/ ./internal COPY vendor/ ./vendor RUN go test -c -covermode=count -coverpkg $(go list ./cmd/runmqserver ./internal/... | paste -s -d, -) ./cmd/runmqserver -FROM mq-advancedserver:latest-x86_64 +FROM $BASE_IMAGE # Copy in the version of the code instrumented for code coverage COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqserver.test /usr/local/bin/ diff --git a/Makefile b/Makefile index 5ad0677..1dbfe6a 100644 --- a/Makefile +++ b/Makefile @@ -12,45 +12,74 @@ # See the License for the specific language governing permissions and # limitations under the License. -BUILD_SERVER_CONTAINER=build-server -DOCKER_TAG_ARCH ?= $(shell uname -m) -# By default, all Docker client commands are run inside a Docker container. -# This means that newer features of the client can be used, even with an older daemon. -DOCKER ?= docker -DOCKER_TAG ?= latest-$(DOCKER_TAG_ARCH) -DOCKER_REPO_DEVSERVER ?= mq-devserver -DOCKER_REPO_ADVANCEDSERVER ?= mq-advancedserver -DOCKER_FULL_DEVSERVER = $(DOCKER_REPO_DEVSERVER):$(DOCKER_TAG) -DOCKER_FULL_ADVANCEDSERVER = $(DOCKER_REPO_ADVANCEDSERVER):$(DOCKER_TAG) -# MQ_PACKAGES is the list of MQ packages to install -MQ_PACKAGES ?=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams +############################################################################### +# Conditional variables - you can override the values of these variables from +# the command line +############################################################################### +# BASE_IMAGE is the base image to use for MQ, for example "ubuntu" or "rhel" +BASE_IMAGE ?= ubuntu:16.04 # MQ_VERSION is the fully qualified MQ version number to build MQ_VERSION ?= 9.0.4.0 -# Archive names for IBM MQ Continuous Delivery Release for Ubuntu -MQ_ARCHIVE_9.0.3.0_ppc64le=CNJR5ML.tar.gz -MQ_ARCHIVE_9.0.3.0_s390x=CNJR6ML.tar.gz -MQ_ARCHIVE_9.0.3.0_x86_64=CNJR7ML.tar.gz -MQ_ARCHIVE_9.0.4.0_ppc64le=CNLE2ML.tar.gz -MQ_ARCHIVE_9.0.4.0_s390x=CNLE3ML.tar.gz -MQ_ARCHIVE_9.0.4.0_x86_64=CNLE4ML.tar.gz -# Archive names for IBM MQ Advanced for Developers for Ubuntu -MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz -MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz -MQ_ARCHIVE ?= $(MQ_ARCHIVE_$(MQ_VERSION)_$(DOCKER_TAG_ARCH)) -MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION)) +# MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can +# be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture +# Does not apply to MQ Advanced for Developers. +MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # Options to `go test` for the Kubernetes tests TEST_OPTS_KUBERNETES ?= -TEST_IMAGE ?= $(DOCKER_FULL_ADVANCEDSERVER) -NUM_CPU=$(shell docker info --format "{{ .NCPU }}") +# MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +# MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +# DOCKER is the Docker command to run +DOCKER ?= docker +# MQ_PACKAGES specifies the MQ packages (.deb or .rpm) to install. Defaults vary on base image. +MQ_PACKAGES ?= +############################################################################### +# Other variables +############################################################################### +# ARCH is the platform architecture (e.g. x86_64, ppc64le or s390x) +ARCH = $(shell uname -m) +# BUILD_SERVER_CONTAINER is the name of the web server container used at build time +BUILD_SERVER_CONTAINER=build-server +# NUM_CPU is the number of CPUs available to Docker. Used to control how many +# test run in parallel +NUM_CPU=$(shell docker info --format "{{ .NCPU }}") +# BASE_IMAGE_TAG is a normalized version of BASE_IMAGE, suitable for use in a Docker tag +BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE))) + +# Try to figure out which archive to use from the BASE_IMAGE +ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" + MQ_ARCHIVE_TYPE=UBUNTU +else + MQ_ARCHIVE_TYPE=LINUX +endif +# Try to figure out which archive to use from the architecture +ifeq "$(ARCH)" "x86_64" + MQ_ARCHIVE_ARCH=X86-64 +else ifeq "$(ARCH)" "ppc64le" + MQ_ARCHIVE_ARCH=LE_POWER +else ifeq "$(ARCH)" "s390x" + MQ_ARCHIVE_ARCH=SYSTEM_Z +endif +# Archive names for IBM MQ Advanced for Developers for Ubuntu +MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz +MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz +# MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced +# for Developers can be installed +MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION)) + +############################################################################### +# Build targets +############################################################################### .PHONY: vars vars: - echo $(DOCKER_SERVER_VERSION_MAJOR) - echo $(DOCKER_SERVER_VERSION_MINOR) - echo $(DOCKER_CLIENT_VERSION_MAJOR) - echo $(DOCKER_CLIENT_VERSION_MINOR) +#ifeq "$(findstring ubuntu,$(BASE_IMAGE))","ubuntu" + @echo $(MQ_ARCHIVE_ARCH) + @echo $(MQ_ARCHIVE_TYPE) + @echo $(MQ_ARCHIVE) .PHONY: default default: build-devserver test @@ -105,26 +134,28 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(DOCKER_FULL_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: test-devserver test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(DOCKER_FULL_DEVSERVER) go test -parallel $(NUM_CPU) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU) .PHONY: test-advancedserver-cover test-advancedserver-cover: test/docker/vendor - $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_REPO_ADVANCEDSERVER) on Docker with code coverage"$(END))) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on Docker with code coverage"$(END))) rm -f ./coverage/unit*.cov # Run unit tests with coverage, for each package under 'internal' go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{} +# ls -1 ./cmd | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./cmd/{}/... echo 'mode: count' > ./coverage/unit.cov tail -q -n +2 ./coverage/unit-*.cov >> ./coverage/unit.cov go tool cover -html=./coverage/unit.cov -o ./coverage/unit.html rm -f ./test/docker/coverage/*.cov rm -f ./coverage/docker.* - cd test/docker && TEST_IMAGE=$(DOCKER_REPO_ADVANCEDSERVER):cover go test $(TEST_OPTS_DOCKER) + mkdir -p ./test/docker/coverage/ + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER)-cover TEST_COVER=true go test $(TEST_OPTS_DOCKER) echo 'mode: count' > ./coverage/docker.cov tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html @@ -135,15 +166,15 @@ test-advancedserver-cover: test/docker/vendor .PHONY: test-kubernetes-devserver test-kubernetes-devserver: test/kubernetes/vendor - $(call test-kubernetes,$(DOCKER_REPO_DEVSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-dev") + $(call test-kubernetes,$(MQ_IMAGE_DEVSERVER),"../../charts/ibm-mqadvanced-server-dev") .PHONY: test-kubernetes-advancedserver test-kubernetes-advancedserver: test/kubernetes/vendor - $(call test-kubernetes,$(DOCKER_REPO_ADVANCEDSERVER),$(DOCKER_TAG),"../../charts/ibm-mqadvanced-server-prod") + $(call test-kubernetes,$(MQ_IMAGE_ADVANCEDSERVER),"../../charts/ibm-mqadvanced-server-prod") define test-kubernetes - $(info $(SPACER)$(shell printf $(TITLE)"Test $1:$2 on Kubernetes"$(END))) - cd test/kubernetes && TEST_REPO=$1 TEST_TAG=$2 TEST_CHART=$3 go test $(TEST_OPTS_KUBERNETES) + $(info $(SPACER)$(shell printf $(TITLE)"Test $1 on Kubernetes"$(END))) + cd test/kubernetes && TEST_IMAGE=$1 TEST_CHART=$2 go test $(TEST_OPTS_KUBERNETES) endef define docker-build-mq @@ -159,13 +190,14 @@ define docker-build-mq --detach \ nginx:alpine # Make sure we have the latest base image - $(DOCKER) pull ubuntu:16.04 + $(DOCKER) pull $(BASE_IMAGE) # Build the new image $(DOCKER) build \ --tag $1 \ --file $2 \ --network build \ --build-arg MQ_URL=http://build:80/$3 \ + --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --label IBM_PRODUCT_ID=$4 \ --label IBM_PRODUCT_NAME=$5 \ --label IBM_PRODUCT_VERSION=$6 \ @@ -184,27 +216,21 @@ docker-version: .PHONY: build-advancedserver build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version - $(info $(SPACER)$(shell printf $(TITLE)"Build $(DOCKER_FULL_ADVANCEDSERVER)"$(END))) - $(call docker-build-mq,$(DOCKER_FULL_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) - $(DOCKER) tag $(DOCKER_FULL_ADVANCEDSERVER) $(DOCKER_REPO_ADVANCEDSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) + $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) + $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) .PHONY: build-devserver build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version @test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) - $(info $(shell printf $(TITLE)"Build $(DOCKER_FULL_DEVSERVER)"$(END))) - $(call docker-build-mq,$(DOCKER_FULL_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) - $(DOCKER) tag $(DOCKER_FULL_DEVSERVER) $(DOCKER_REPO_DEVSERVER):$(MQ_VERSION)-$(DOCKER_TAG_ARCH) + $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END))) + $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-advancedserver-cover build-advancedserver-cover: docker-version - $(DOCKER) build -t $(DOCKER_REPO_ADVANCEDSERVER):cover -f Dockerfile-server.cover . - -# .PHONY: build-web -# build-web: build downloads/CNJR7ML.tar.gz -# $(call docker-build-mq,mq-web:latest-$(DOCKER_TAG_ARCH),Dockerfile-mq-web) + $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_IMAGE_ADVANCEDSERVER)-cover -f Dockerfile-server.cover . .PHONY: build-explorer build-explorer: downloads/$(MQ_ARCHIVE_DEV) - $(call docker-build-mq,mq-explorer:latest-$(DOCKER_TAG_ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) + $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) include formatting.mk diff --git a/charts/ibm-mqadvanced-server-dev/Chart.yaml b/charts/ibm-mqadvanced-server-dev/Chart.yaml index dcfb384..2ca4bf6 100644 --- a/charts/ibm-mqadvanced-server-dev/Chart.yaml +++ b/charts/ibm-mqadvanced-server-dev/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 description: IBM MQ queue manager name: ibm-mqadvanced-server-dev -version: 1.0.2 -icon: https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/07/IBM-MQ-Square-200.png +version: 1.1.0 +icon: https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/12/ibm_mq_200.png tillerVersion: ">=2.4.0" diff --git a/charts/ibm-mqadvanced-server-dev/README.md b/charts/ibm-mqadvanced-server-dev/README.md index 4ad0405..4351190 100644 --- a/charts/ibm-mqadvanced-server-dev/README.md +++ b/charts/ibm-mqadvanced-server-dev/README.md @@ -1,5 +1,3 @@ -![IBM MQ logo](https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/07/IBM-MQ-Square-200.png) - # IBM MQ IBM® MQ is messaging middleware that simplifies and accelerates the integration of diverse applications and business data across multiple platforms. It uses message queues to facilitate the exchanges of information and offers a single messaging solution for cloud, mobile, Internet of Things (IoT) and on-premises environments. @@ -12,7 +10,7 @@ This chart deploys a single IBM MQ Advanced for Developers server (queue manager - Kubernetes 1.6 or greater, with beta APIs enabled - If persistence is enabled (see [configuration](#configuration)), then you either need to create a PersistentVolume, or specify a Storage Class if classes are defined in your cluster. - +¸ ## Installing the Chart To install the chart with the release name `foo`: @@ -63,7 +61,7 @@ The following table lists the configurable parameters of the `ibm-mqadvanced-ser | `queueManager.name` | MQ Queue Manager name | Helm release name | | `queueManager.dev.adminPassword` | Developer defaults - administrator password | Random generated string. See the notes that appear when you install for how to retrieve this. | | `queueManager.dev.appPassword` | Developer defaults - app password | `nil` (no password required to connect an MQ client) | -| `nameOverride` | Set to partially override the resource names used in this chart | `nil` | +| `nameOverride` | Set to partially override the resource names used in this chart | `ibm-mq` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. diff --git a/charts/ibm-mqadvanced-server-dev/templates/stateful-set.yaml b/charts/ibm-mqadvanced-server-dev/templates/stateful-set.yaml index 1d567aa..3788453 100644 --- a/charts/ibm-mqadvanced-server-dev/templates/stateful-set.yaml +++ b/charts/ibm-mqadvanced-server-dev/templates/stateful-set.yaml @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Capabilities.APIVersions.Has "apps/v1beta2" }} +apiVersion: apps/v1beta2 +{{- else }} apiVersion: apps/v1beta1 +{{- end }} kind: StatefulSet metadata: name: {{ template "fullname" . }} @@ -22,9 +26,14 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" spec: + selector: + matchLabels: + app: {{ template "fullname" . }} serviceName: {{ .Values.service.name }} replicas: 1 - {{- if and (ge (.Capabilities.KubeVersion.Major | int) 1) (ge (.Capabilities.KubeVersion.Minor | int) 7) }} + {{- if and (.Capabilities.KubeVersion.Major | hasPrefix "1") (.Capabilities.KubeVersion.Minor | hasPrefix "7") }} + # Set updateStrategy to "RollingUpdate", if we're on Kubernetes 1.7. + # It's already the default for apps/v1beta2 (Kubernetes 1.8 onwards) updateStrategy: type: RollingUpdate {{- end }} @@ -37,6 +46,19 @@ spec: heritage: "{{ .Release.Service }}" QM_IDENTIFIER: "{{ .Release.Name }}" spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - key: beta.kubernetes.io/os + operator: In + values: + - linux {{- if .Values.image.pullSecret }} imagePullSecrets: - name: {{ .Values.image.pullSecret }} diff --git a/charts/ibm-mqadvanced-server-dev/templates/test.yaml b/charts/ibm-mqadvanced-server-dev/templates/tests/test.yaml similarity index 100% rename from charts/ibm-mqadvanced-server-dev/templates/test.yaml rename to charts/ibm-mqadvanced-server-dev/templates/tests/test.yaml diff --git a/charts/ibm-mqadvanced-server-dev/values-metadata.yaml b/charts/ibm-mqadvanced-server-dev/values-metadata.yaml new file mode 100644 index 0000000..6a20a79 --- /dev/null +++ b/charts/ibm-mqadvanced-server-dev/values-metadata.yaml @@ -0,0 +1,176 @@ +license: + __metadata: + label: "License" + type: "string" + required: true + options: + - label: "Accepted" + value: "accept" + - label: "Not accepted" + value: "not accepted" + +image: + __metadata: + label: "Image" + repository: + __metadata: + label: "Image repository" + description: "Docker image location" + type: "string" + immutable: false + required: true + tag: + __metadata: + label: "Image tag" + description: "Docker image tag" + type: "string" + immutable: false + required: true + pullPolicy: + __metadata: + name: "pullPolicy" + label: "Image pull policy" + description: "Always, Never, or IfNotPresent. Defaults to Always" + type: "string" + immutable: false + required: true + options: + - label: "Always" + value: "Always" + - label: "Never" + value: "Never" + - label: "IfNotPresent" + value: "IfNotPresent" + pullSecret: + __metadata: + label: "Image pull secret" + description: "Secret to use when pulling the image. Set this when using an image from a private registry" + type: "string" + required: false + +## global persistence parameters +persistence: + __metadata: + label: "Persistence" + enabled: + __metadata: + label: "Enable persistence" + description: "Whether or not to store MQ messages and configuration on a Persistent Volume" + type: "boolean" + required: true + useDynamicProvisioning: + __metadata: + label: "Use dynamic provisioning" + description: "Whether or not to use Storage Classes to provision a Persisent Volume automatically" + type: "boolean" + required: true + +dataPVC: + __metadata: + label: "Data PVC" + name: + __metadata: + label: "Name" + description: "Name of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: true + storageClassName: + __metadata: + label: "Storage Class name" + description: "Storage class of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: false + size: + __metadata: + label: "Size" + description: "Size of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: true + +service: + __metadata: + label: "Service" + name: + __metadata: + label: "Service name" + description: "Service name" + type: "string" + required: true + type: + __metadata: + label: "Service type" + description: "Type of service" + type: "string" + required: true + options: + - label: "ClusterIP" + value: "ClusterIP" + - label: "NodePort" + value: "NodePort" + - label: "LoadBalancer" + value: "LoadBalancer" + - label: "ExternalName" + value: "ExternalName" + +resources: + __metadata: + label: "Resources" + requests: + cpu: + __metadata: + label: "CPU request" + description: "The requested CPU" + type: "string" + required: true + memory: + __metadata: + label: "Memory request" + description: "The requested memory" + type: "string" + required: true + limits: + cpu: + __metadata: + label: "CPU limit" + description: "The CPU limit" + type: "string" + required: true + memory: + __metadata: + label: "Memory limit" + description: "The memory limit" + type: "string" + required: true + +queueManager: + __metadata: + label: "Queue manager" + name: + __metadata: + label: "Queue manager name" + description: "MQ queue manager name, which defaults to the Helm release name" + type: "string" + required: false + dev: + __metadata: + label: "Default Developer Configuration" + description: "Configure some queue manager basics, suitable for developer use" + adminPassword: + __metadata: + label: "Admin password" + description: "Password for 'admin' user" + type: "password" + required: false + appPassword: + __metadata: + label: "App password" + description: "Password for 'app' user to use for messaging" + type: "password" + required: false + +nameOverride: + __metadata: + label: "Name override" + description: "This can be set to partially override the name of the resources created by this chart" + type: "string" + required: false diff --git a/charts/ibm-mqadvanced-server-dev/values.yaml b/charts/ibm-mqadvanced-server-dev/values.yaml index 75786e0..f3e4dbb 100644 --- a/charts/ibm-mqadvanced-server-dev/values.yaml +++ b/charts/ibm-mqadvanced-server-dev/values.yaml @@ -65,4 +65,4 @@ queueManager: appPassword: # nameOverride can be set to partially override the name of the resources created by this chart -nameOverride: +nameOverride: ibm-mq diff --git a/charts/ibm-mqadvanced-server-prod/Chart.yaml b/charts/ibm-mqadvanced-server-prod/Chart.yaml index f6a303e..bc7bcbe 100644 --- a/charts/ibm-mqadvanced-server-prod/Chart.yaml +++ b/charts/ibm-mqadvanced-server-prod/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 description: IBM MQ queue manager name: ibm-mqadvanced-server-prod -version: 1.0.2 -icon: https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/07/IBM-MQ-Square-200.png +version: 1.1.0 +icon: https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/12/ibm_mq_200.png tillerVersion: ">=2.4.0" diff --git a/charts/ibm-mqadvanced-server-prod/README.md b/charts/ibm-mqadvanced-server-prod/README.md index ee506c0..d5e03b1 100644 --- a/charts/ibm-mqadvanced-server-prod/README.md +++ b/charts/ibm-mqadvanced-server-prod/README.md @@ -1,5 +1,3 @@ -![IBM MQ logo](https://developer.ibm.com/messaging/wp-content/uploads/sites/18/2017/07/IBM-MQ-Square-200.png) - # IBM MQ IBM® MQ is messaging middleware that simplifies and accelerates the integration of diverse applications and business data across multiple platforms. It uses message queues to facilitate the exchanges of information and offers a single messaging solution for cloud, mobile, Internet of Things (IoT) and on-premises environments. @@ -45,8 +43,8 @@ The following table lists the configurable parameters of the `ibm-mqadvanced-ser | Parameter | Description | Default | | ------------------------------- | --------------------------------------------------------------- | ------------------------------------------ | | `license` | Set to `accept` to accept the terms of the IBM license | `"not accepted"` | -| `image.repository` | Image full name including repository | `nil` | -| `image.tag` | Image tag | `nil` | +| `image.repository` | Image full name including repository | `MQ image in your registry` | +| `image.tag` | Image tag | `Tag of MQ image in your registry` | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | | `image.pullSecret` | Image pull secret, if you are using a private Docker registry | `nil` | | `persistence.enabled` | Use persistent volumes for all defined volumes | `true` | @@ -61,7 +59,7 @@ The following table lists the configurable parameters of the `ibm-mqadvanced-ser | `resources.requests.cpu` | Kubernetes CPU request for the Queue Manager container | `1` | | `resources.requests.memory` | Kubernetes memory request for the Queue Manager container | `1Gi` | | `queueManager.name` | MQ Queue Manager name | Helm release name | -| `nameOverride` | Set to partially override the resource names used in this chart | `nil` | +| `nameOverride` | Set to partially override the resource names used in this chart | `ibm-mq` | | `livenessProbe.initialDelaySeconds` | The initial delay before starting the liveness probe. Useful for slower systems that take longer to start the Queue Manager. | 60 | | `livenessProbe.periodSeconds` | How often to run the probe | 10 | | `livenessProbe.timeoutSeconds` | Number of seconds after which the probe times out | 5 | diff --git a/charts/ibm-mqadvanced-server-prod/templates/stateful-set.yaml b/charts/ibm-mqadvanced-server-prod/templates/stateful-set.yaml index fd65853..5fb435c 100644 --- a/charts/ibm-mqadvanced-server-prod/templates/stateful-set.yaml +++ b/charts/ibm-mqadvanced-server-prod/templates/stateful-set.yaml @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Capabilities.APIVersions.Has "apps/v1beta2" }} +apiVersion: apps/v1beta2 +{{- else }} apiVersion: apps/v1beta1 +{{- end }} kind: StatefulSet metadata: name: {{ template "fullname" . }} @@ -22,9 +26,14 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" spec: + selector: + matchLabels: + app: {{ template "fullname" . }} serviceName: {{ .Values.service.name }} replicas: 1 - {{- if and (ge (.Capabilities.KubeVersion.Major | int) 1) (ge (.Capabilities.KubeVersion.Minor | int) 7) }} + {{- if and (.Capabilities.KubeVersion.Major | hasPrefix "1") (.Capabilities.KubeVersion.Minor | hasPrefix "7") }} + # Set updateStrategy to "RollingUpdate", if we're on Kubernetes 1.7. + # It's already the default for apps/v1beta2 (Kubernetes 1.8 onwards) updateStrategy: type: RollingUpdate {{- end }} @@ -37,6 +46,19 @@ spec: heritage: "{{ .Release.Service }}" QM_IDENTIFIER: "{{ .Release.Name }}" spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - key: beta.kubernetes.io/os + operator: In + values: + - linux {{- if .Values.image.pullSecret }} imagePullSecrets: - name: {{ .Values.image.pullSecret }} diff --git a/charts/ibm-mqadvanced-server-prod/templates/test.yaml b/charts/ibm-mqadvanced-server-prod/templates/tests/test.yaml similarity index 100% rename from charts/ibm-mqadvanced-server-prod/templates/test.yaml rename to charts/ibm-mqadvanced-server-prod/templates/tests/test.yaml diff --git a/charts/ibm-mqadvanced-server-prod/values-metadata.yaml b/charts/ibm-mqadvanced-server-prod/values-metadata.yaml new file mode 100644 index 0000000..38a7df2 --- /dev/null +++ b/charts/ibm-mqadvanced-server-prod/values-metadata.yaml @@ -0,0 +1,219 @@ +license: + __metadata: + label: "License" + type: "string" + required: true + options: + - label: "Accepted" + value: "accept" + - label: "Not accepted" + value: "not accepted" + +image: + __metadata: + label: "Image" + repository: + __metadata: + label: "Image repository" + description: "Docker image location" + type: "string" + immutable: false + required: true + tag: + __metadata: + label: "Image tag" + description: "Docker image tag" + type: "string" + immutable: false + required: true + pullPolicy: + __metadata: + name: "pullPolicy" + label: "Image pull policy" + description: "Always, Never, or IfNotPresent. Defaults to Always" + type: "string" + immutable: false + required: true + options: + - label: "Always" + value: "Always" + - label: "Never" + value: "Never" + - label: "IfNotPresent" + value: "IfNotPresent" + pullSecret: + __metadata: + label: "Image pull secret" + description: "Secret to use when pulling the image. Set this when using an image from a private registry" + type: "string" + required: false + +## global persistence parameters +persistence: + __metadata: + label: "Persistence" + enabled: + __metadata: + label: "Enable persistence" + description: "Whether or not to store MQ messages and configuration on a Persistent Volume" + type: "boolean" + required: true + useDynamicProvisioning: + __metadata: + label: "Use dynamic provisioning" + description: "Whether or not to use Storage Classes to provision a Persisent Volume automatically" + type: "boolean" + required: true + +dataPVC: + __metadata: + label: "Data PVC" + name: + __metadata: + label: "Name" + description: "Name of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: true + storageClassName: + __metadata: + label: "Storage Class name" + description: "Storage class of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: false + size: + __metadata: + label: "Size" + description: "Size of Persistent Volume Claim, used for MQ objects and messages" + type: "string" + required: true + +service: + __metadata: + label: "Service" + name: + __metadata: + label: "Service name" + description: "Service name" + type: "string" + required: true + type: + __metadata: + label: "Service type" + description: "Type of service" + type: "string" + required: true + options: + - label: "ClusterIP" + value: "ClusterIP" + - label: "NodePort" + value: "NodePort" + - label: "LoadBalancer" + value: "LoadBalancer" + - label: "ExternalName" + value: "ExternalName" + +resources: + __metadata: + label: "Resources" + requests: + cpu: + __metadata: + label: "CPU request" + description: "The requested CPU" + type: "string" + required: true + memory: + __metadata: + label: "Memory request" + description: "The requested memory" + type: "string" + required: true + limits: + cpu: + __metadata: + label: "CPU limit" + description: "The CPU limit" + type: "string" + required: true + memory: + __metadata: + label: "Memory limit" + description: "The memory limit" + type: "string" + required: true + +queueManager: + __metadata: + label: "Queue manager" + name: + __metadata: + label: "Queue manager name" + description: "MQ queue manager name, which defaults to the Helm release name" + type: "string" + required: false + +nameOverride: + __metadata: + label: "Name override" + description: "This can be set to partially override the name of the resources created by this chart" + type: "string" + required: false + +# livenessProbe section specifies setting for the MQ liveness probe, which checks for a running Queue Manager +livenessProbe: + __metadata: + label: "Liveness probe" + # initialDelaySeconds should be raised if your system cannot start the Queue Manager in 60 seconds + initialDelaySeconds: + __metadata: + label: "Initial delay (seconds)" + description: "How long to wait before starting the probe. Raise this delay if your system cannot start the Queue Manager in the default time period" + type: "number" + required: false + periodSeconds: + __metadata: + label: "Period (seconds)" + description: "How often to perform the probe" + type: "number" + required: false + timeoutSeconds: + __metadata: + label: "Timeout (seconds)" + description: "How long before a probe times out" + type: "number" + required: false + failureThreshold: + __metadata: + label: "Failure threshold" + description: "Number of times the probe can fail before taking action" + type: "number" + required: false + +# readinessProbe section specifies setting for the MQ readiness probe, which checks when the MQ listener is running +readinessProbe: + __metadata: + label: "Readiness probe" + initialDelaySeconds: + __metadata: + label: "Initial delay (seconds)" + description: "How long to wait before starting the probe" + type: "number" + required: false + periodSeconds: + __metadata: + label: "Period (seconds)" + description: "How often to perform the probe" + type: "number" + required: false + timeoutSeconds: + __metadata: + label: "Timeout (seconds)" + description: "How long before a probe times out" + type: "number" + required: false + failureThreshold: + __metadata: + label: "Failure threshold" + description: "Number of times the probe can fail before taking action" + type: "number" + required: false diff --git a/charts/ibm-mqadvanced-server-prod/values.yaml b/charts/ibm-mqadvanced-server-prod/values.yaml index d55bd22..3b34048 100644 --- a/charts/ibm-mqadvanced-server-prod/values.yaml +++ b/charts/ibm-mqadvanced-server-prod/values.yaml @@ -59,7 +59,7 @@ queueManager: name: # nameOverride can be set to partially override the name of the resources created by this chart -nameOverride: +nameOverride: ibm-mq # livenessProbe section specifies setting for the MQ liveness probe, which checks for a running Queue Manager livenessProbe: diff --git a/cmd/runmqserver/main_test.go b/cmd/runmqserver/main_test.go index 2a78204..0643f10 100644 --- a/cmd/runmqserver/main_test.go +++ b/cmd/runmqserver/main_test.go @@ -25,6 +25,8 @@ import ( var test *bool +const filename = "/var/coverage/exitCode" + func init() { test = flag.Bool("test", false, "Set to true when running tests for coverage") } @@ -38,8 +40,11 @@ func TestSystem(t *testing.T) { }() osExit = func(rc int) { // Write the exit code to a file instead - log.Printf("Writing exit code %v to file", strconv.Itoa(rc)) - ioutil.WriteFile("/var/coverage/exitCode", []byte(strconv.Itoa(rc)), 0644) + log.Printf("Writing exit code %v to file %v", strconv.Itoa(rc), filename) + err := ioutil.WriteFile(filename, []byte(strconv.Itoa(rc)), 0644) + if err != nil { + log.Print(err) + } } main() } diff --git a/docs/building.md b/docs/building.md new file mode 100644 index 0000000..89dd8fc --- /dev/null +++ b/docs/building.md @@ -0,0 +1,35 @@ +# Building a Docker image + +## Prerequisites +You need to ensure you have the following tools installed: +* [Docker](https://www.docker.com/) V17.05 or later +* [GNU make](https://www.gnu.org/software/make/) + +## Building a production image +This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. + +1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `IBM_MQ_9.0.4.0_UBUNTU_X86-64.tar.gz` for MQ V9.0.4 for Ubuntu on x86_64 architecture) in the `downloads` directory +2. Run `make build-advancedserver` + +You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example: + +```bash +MQ_VERSION=9.0.4.0 make build-advancedserver +``` + +If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example: + +```bash +MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver +``` + +## Building on a different base image +By default, the MQ images use Ubuntu as the base layer. You can build using a Red Hat Enterprise Linux compatible base layer by setting the `BASE_IMAGE` environment variable. For example: + +``` +BASE_IMAGE=centos:7 make build-advancedserver +``` + +The `make` tool will try and locate the right archive file under the `downloads` directory, based on your platform architecture and your `MQ_VERSION` environment variable, for example `IBM_MQ_9.0.4.0_LINUX_X86_64.tar.gz` for MQ V9.0.4.0 on x86_64. You can also set the `MQ_ARCHIVE` environment variable to set the specific file name. + +Note that if you are using Red Hat Enterprise Linux, you will need to create your own base image layer, with your subscription enabled, as described [here](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/getting_started_with_containers/get_started_with_docker_formatted_container_images). The MQ image build needs to install some additional packages, and a subscription is required to access the Red Hat repositories. \ No newline at end of file diff --git a/docs/developing.md b/docs/testing.md similarity index 60% rename from docs/developing.md rename to docs/testing.md index 6fa9682..12c16c1 100644 --- a/docs/developing.md +++ b/docs/testing.md @@ -1,34 +1,20 @@ -# Developing +# Testing ## Prerequisites You need to ensure you have the following tools installed: -* [Docker](https://www.docker.com/) V17.05 or later +* [Docker](https://www.docker.com/) * [GNU make](https://www.gnu.org/software/make/) - -You might also need the following tools installed: * [Go](https://golang.org/) - only needed for running the tests -* [Glide](https://glide.sh/) - only needed if you update the main dependencies -* [dep](https://github.com/golang/dep) (official Go dependency management tool) - only needed to prepare for running the tests +* [dep](https://github.com/golang/dep) (official Go dependency management tool) - needed to prepare for running the tests * [Helm](https://helm.sh) - only needed for running the Kubernetes tests For running the Kubernetes tests, a Kubernetes environment is needed, for example [Minikube](https://github.com/kubernetes/minikube) or [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/). -## Building a production image -This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. +## Preparing to run the tests +The test dependencies are not included with the source code, so you need to download them before you can run them. This can be done with the following command, which uses the `dep` tool: -1. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `CNLE4ML.tar.gz` for MQ V9.0.4 on x86_64 architecture) in the `downloads` directory -2. Run `make build-advancedserver` - -You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example: - -```bash -MQ_VERSION=9.0.3.0 make build-advancedserver ``` - -If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example: - -```bash -MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 build-advancedserver +make deps ``` ## Running the tests @@ -39,30 +25,31 @@ There are three main sets of tests: 3. Kubernetes tests, which test the Helm charts (and the Docker image) via [Helm](https://helm.sh) ### Running the Docker tests -The Docker tests can be run locally. Before you run them for the first time, you need to download the test dependencies: - -``` -make deps -``` - -You can then run the tests, for example: +The Docker tests can be run locally on a machine with Docker. For example: ``` make test-devserver -``` - -or: - -``` make test-advancedserver ``` +You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example: + +``` +MQ_IMAGE_ADVANCEDSERVER=mqadvanced-server9.0.4.0-x86_64-ubuntu-16.04 make test-advancedserver +``` + You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command:: ``` TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver ``` +You can also use the same environment variables you specified when [building](./building), for example, the following will try and test an image called `mqadvanced-server9.0.3.0-x86_64-ubuntu-16.04`: + +``` +MQ_VERSION=9.0.3.0 make test-advancedserver +``` + ### Running the Docker tests with code coverage You can produce code coverage results from the Docker tests by running the following: @@ -79,5 +66,5 @@ In order to generate code coverage metrics from the Docker tests, the build step For the Kubernetes tests, you need to have built the Docker image, and pushed it to the registry used by your Kubernetes cluster. Most of the configuration used by the tests is picked up from your `kubectl` configuration, but you will typically need to specify the image details. For example: ```bash -DOCKER_REPO_DEVSERVER=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver +MQ_IMAGE=mycluster.icp:8500/default/mq-devserver make test-kubernetes-devserver ``` diff --git a/install-mq.sh b/install-mq.sh index 685bd50..c79b19e 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -18,25 +18,53 @@ # Fail on any non-zero return code set -ex -export DEBIAN_FRONTEND=noninteractive +test -f /usr/bin/yum && RHEL=true || RHEL=false +test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false + +# If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults +if [ -z $MQ_PACKAGES ]; then + $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" + $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm" +fi + +if ($UBUNTU); then + export DEBIAN_FRONTEND=noninteractive + # Install additional packages required by MQ, this install process and the runtime scripts + apt-get update + apt-get install -y --no-install-recommends \ + bash \ + bc \ + ca-certificates \ + coreutils \ + curl \ + debianutils \ + file \ + findutils \ + gawk \ + grep \ + libc-bin \ + mount \ + passwd \ + procps \ + sed \ + tar \ + util-linux +fi # Install additional packages required by MQ, this install process and the runtime scripts -apt-get update -apt-get install -y --no-install-recommends \ +$RHEL && yum -y install \ bash \ bc \ ca-certificates \ coreutils \ curl \ - debianutils \ file \ findutils \ gawk \ + glibc-common \ grep \ - libc-bin \ - mount \ passwd \ - procps \ + procps-ng \ sed \ tar \ util-linux @@ -49,30 +77,37 @@ curl -LO $MQ_URL tar -zxvf ./*.tar.gz # Remove packages only needed by this script -apt-get purge -y \ +$UBUNTU && apt-get purge -y \ ca-certificates \ curl +# Note: ca-certificates and curl are installed by default in RHEL + # Remove any orphaned packages -apt-get autoremove -y +$UBUNTU && apt-get autoremove -y # Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images -groupadd --system --gid 999 mqm -useradd --system --uid 999 --gid mqm mqm +$UBUNTU && groupadd --system --gid 999 mqm +$UBUNTU && useradd --system --uid 999 --gid mqm mqm +$RHEL && groupadd --system --gid 888 mqm +$RHEL && useradd --system --uid 888 --gid mqm mqm usermod -G mqm root # Find directory containing .deb files -DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1) +$UBUNTU && DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1) +$RHEL && DIR_RPM=$(find ${DIR_EXTRACT} -name "*.rpm" -printf "%h\n" | sort -u | head -1) # Find location of mqlicense.sh MQLICENSE=$(find ${DIR_EXTRACT} -name "mqlicense.sh") # Accept the MQ license ${MQLICENSE} -text_only -accept -echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list +$UBUNTU && echo "deb [trusted=yes] file:${DIR_DEB} ./" > /etc/apt/sources.list.d/IBM_MQ.list # Install MQ using the DEB packages -apt-get update -apt-get install -y $MQ_PACKAGES +$UBUNTU && apt-get update +$UBUNTU && apt-get install -y $MQ_PACKAGES + +$RHEL && cd $DIR_RPM && rpm -ivh $MQ_PACKAGES # Remove 32-bit libraries from 64-bit container find /opt/mqm /var/mqm -type f -exec file {} \; | awk -F: '/ELF 32-bit/{print $1}' | xargs --no-run-if-empty rm -f @@ -84,19 +119,21 @@ find /opt/mqm -name '*.tar.gz' -delete /opt/mqm/bin/setmqinst -p /opt/mqm -i # Clean up all the downloaded files -rm -f /etc/apt/sources.list.d/IBM_MQ.list +$UBUNTU && rm -f /etc/apt/sources.list.d/IBM_MQ.list rm -rf ${DIR_EXTRACT} -#### Apply any bug fixes not included in base Ubuntu or MQ image. +# Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -apt-get upgrade -y libdb5.3 -#### End of bug fixes -# Clean up cached apt files -rm -rf /var/lib/apt/lists/* +# End of bug fixes + +# Clean up cached files +$UBUNTU && rm -rf /var/lib/apt/lists/* +$RHEL && yum -y clean all +$RHEL && rm -rf /var/cache/yum/* # Optional: Update the command prompt with the MQ version -echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot +$UBUNTU && echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot # Remove the directory structure under /var/mqm which was created by the installer rm -rf /var/mqm @@ -113,4 +150,7 @@ ln -s /mnt/mqm/data /var/mqm # Optional: Set these values for the Bluemix Vulnerability Report sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' /etc/login.defs sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' /etc/login.defs -sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' /etc/pam.d/common-password + +$UBUNTU && PAM_FILE=/etc/pam.d/common-password +$RHEL && PAM_FILE=/etc/pam.d/password-auth +sed -i 's/password\t\[success=1 default=ignore\]\tpam_unix\.so obscure sha512/password\t[success=1 default=ignore]\tpam_unix.so obscure sha512 minlen=8/' $PAM_FILE diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index f892606..15aad29 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -17,7 +17,7 @@ package main import ( "context" - "strconv" + "fmt" "strings" "testing" "time" @@ -91,18 +91,24 @@ func TestSecurityVulnerabilities(t *testing.T) { if err != nil { t.Fatal(err) } - containerConfig := container.Config{ - // Override the entrypoint to make "apt" only receive security updates, then check for updates - Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"}, + // containerConfig := container.Config{ + // // Override the entrypoint to make "apt" only receive security updates, then check for updates + // Entrypoint: []string{"bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade"}, + // } + // id := runContainer(t, cli, &containerConfig) + // defer cleanContainer(t, cli, id) + // // rc is the return code from apt-get + // rc := waitForContainer(t, cli, id, 10) + + rc, _ := runContainerOneShot(t, cli, "bash", "-c", "test -d /etc/apt") + if rc != 0 { + t.Skip("Skipping test because container is not Ubuntu-based") } - id := runContainer(t, cli, &containerConfig) - defer cleanContainer(t, cli, id) - // rc is the return code from apt-get - rc := waitForContainer(t, cli, id, 10) + // Override the entrypoint to make "apt" only receive security updates, then check for updates + rc, log := runContainerOneShot(t, cli, "bash", "-c", "source /etc/os-release && echo \"deb http://security.ubuntu.com/ubuntu/ ${VERSION_CODENAME}-security main restricted\" > /etc/apt/sources.list && apt-get update 2>&1 >/dev/null && apt-get --simulate -qq upgrade") if rc != 0 { t.Fatalf("Expected success, got %v", rc) } - log := inspectLogs(t, cli, id) lines := strings.Split(strings.TrimSpace(log), "\n") if len(lines) > 0 && lines[0] != "" { t.Errorf("Expected no vulnerabilities, found the following:\n%v", log) @@ -317,13 +323,41 @@ func TestZombies(t *testing.T) { t.Fatalf("Expected pkill to kill a process, got %v", out) } time.Sleep(3 * time.Second) - // Create a zombie process for up to ten seconds - out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z' | wc -l"}) - count, err := strconv.Atoi(out) + out = execContainerWithOutput(t, cli, id, "mqm", []string{"bash", "-c", "ps -lA | grep '^. Z'"}) + if out != "" { + count := strings.Count(out, "\n") + 1 + t.Errorf("Expected zombies=0, got %v", count) + t.Error(out) + t.Fail() + } +} + +// TestMQSC creates a new image with an MQSC file in, starts a container based +// on that image, and checks that the MQSC has been applied correctly. +func TestMQSC(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) } - if count != 0 { - t.Fatalf("Expected zombies=0, got %v", count) + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf("FROM %v\nADD test.mqsc /etc/mqm/", imageName())}, + {"test.mqsc", "DEFINE QLOCAL(test)"}, + } + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + rc := execContainerWithExitCode(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test)' | runmqsc"}) + if rc != 0 { + t.Fatalf("Expected runmqsc to exit with rc=0, got %v", rc) } } diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index d6957d5..9558d37 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -16,8 +16,11 @@ limitations under the License. package main import ( + "archive/tar" "bytes" "context" + "encoding/json" + "io" "io/ioutil" "log" "os" @@ -32,6 +35,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" ) @@ -43,6 +47,14 @@ func imageName() string { return image } +func coverage() bool { + cover := os.Getenv("TEST_COVER") + if cover == "true" || cover == "1" { + return true + } + return false +} + // coverageDir returns the host directory to use for code coverage data func coverageDir(t *testing.T) string { dir, err := os.Getwd() @@ -62,6 +74,11 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { if err == nil { // Log the results and continue t.Logf("Inspected container %v: %#v", ID, i) + s, err := json.MarshalIndent(i, "", " ") + if err != nil { + t.Fatal(err) + } + t.Logf("Inspected container %v: %v", ID, string(s)) } t.Logf("Stopping container: %v", ID) timeout := 10 * time.Second @@ -120,6 +137,15 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C return ctr.ID } +func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (int64, string) { + containerConfig := container.Config{ + Entrypoint: command, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + return waitForContainer(t, cli, id, 10), inspectLogs(t, cli, id) +} + func startContainer(t *testing.T, cli *client.Client, ID string) { t.Logf("Starting container: %v", ID) startOptions := types.ContainerStartOptions{} @@ -142,7 +168,7 @@ func getCoverageExitCode(t *testing.T, orig int64) int64 { f := filepath.Join(coverageDir(t), "exitCode") _, err := os.Stat(f) if err != nil { - //t.Log(err) + t.Log(err) return orig } // Remove the file, ready for the next test @@ -167,10 +193,12 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64 //defer cancel() rc, err := cli.ContainerWait(context.Background(), ID) - // COVERAGE: When running coverage, the exit code is written to a file, - // to allow the coverage to be generated (which doesn't happen for non-zero - // exit codes) - rc = getCoverageExitCode(t, rc) + if coverage() { + // COVERAGE: When running coverage, the exit code is written to a file, + // to allow the coverage to be generated (which doesn't happen for non-zero + // exit codes) + rc = getCoverageExitCode(t, rc) + } // err := <-errC if err != nil { @@ -329,3 +357,67 @@ func inspectLogs(t *testing.T, cli *client.Client, ID string) string { } return buf.String() } + +// generateTAR creates a TAR-formatted []byte, with the specified files included. +func generateTAR(t *testing.T, files []struct{ Name, Body string }) []byte { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + err := tw.WriteHeader(hdr) + if err != nil { + t.Fatal(err) + } + _, err = tw.Write([]byte(file.Body)) + if err != nil { + t.Fatal(err) + } + } + err := tw.Close() + if err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// createImage creates a new Docker image with the specified files included. +func createImage(t *testing.T, cli *client.Client, files []struct{ Name, Body string }) string { + r := bytes.NewReader(generateTAR(t, files)) + tag := strings.ToLower(t.Name()) + buildOptions := types.ImageBuildOptions{ + Context: r, + Tags: []string{tag}, + } + resp, err := cli.ImageBuild(context.Background(), r, buildOptions) + if err != nil { + t.Fatal(err) + } + // resp (ImageBuildResponse) contains a series of JSON messages + dec := json.NewDecoder(resp.Body) + for { + m := jsonmessage.JSONMessage{} + err := dec.Decode(&m) + if m.Error != nil { + t.Fatal(m.ErrorMessage) + } + t.Log(strings.TrimSpace(m.Stream)) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + } + return tag +} + +// deleteImage deletes a Docker image +func deleteImage(t *testing.T, cli *client.Client, id string) { + cli.ImageRemove(context.Background(), id, types.ImageRemoveOptions{ + Force: true, + }) +} diff --git a/test/kubernetes/helm_test_util.go b/test/kubernetes/helm_test_util.go index 2b5a118..b80964c 100644 --- a/test/kubernetes/helm_test_util.go +++ b/test/kubernetes/helm_test_util.go @@ -17,36 +17,34 @@ package main import ( "bytes" - "fmt" "os" - "os/exec" - "runtime" "strconv" "strings" "testing" "time" + "unicode" - "golang.org/x/sys/unix" + "github.com/ibm-messaging/mq-container/internal/command" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/clientcmd" ) -func imageName() string { - v, ok := os.LookupEnv("TEST_REPO") +func image(t *testing.T) string { + v, ok := os.LookupEnv("TEST_IMAGE") if !ok { - v = "ibmcom/mq" + t.Fatal("TEST_IMAGE environment variable not set") } return v } -func imageTag() string { - v, ok := os.LookupEnv("TEST_TAG") - if !ok { - v = "latest" - } - return v +func imageName(t *testing.T) string { + return strings.Fields(strings.Replace(image(t), ":", " ", -1))[0] +} + +func imageTag(t *testing.T) string { + return strings.Fields(strings.Replace(image(t), ":", " ", -1))[1] } func chartName() string { @@ -57,32 +55,6 @@ func chartName() string { return v } -// runCommand runs an OS command. On Linux it waits for the command to -// complete and returns the exit status (return code). -// TODO: duplicated from cmd/runmqserver/main.go -func runCommand(t *testing.T, name string, arg ...string) (string, int, error) { - t.Logf("Running command: %v %v", name, strings.Trim(fmt.Sprintf("%v", arg), "[]")) - cmd := exec.Command(name, arg...) - // Run the command and wait for completion - out, err := cmd.CombinedOutput() - if err != nil { - var rc int - // Only works on Linux - if runtime.GOOS == "linux" { - var ws unix.WaitStatus - unix.Wait4(cmd.Process.Pid, &ws, 0, nil) - rc = ws.ExitStatus() - } else { - rc = -1 - } - if rc == 0 { - return string(out), rc, nil - } - return string(out), rc, err - } - return string(out), 0, nil -} - func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string { pods := getPodsForHelmRelease(t, cs, release) opt := v1.PodLogOptions{} @@ -98,7 +70,7 @@ func inspectLogs(t *testing.T, cs *kubernetes.Clientset, release string) string func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values ...string) { chart := chartName() - tag := "latest" + tag := imageTag(t) arg := []string{ "install", "--debug", @@ -106,7 +78,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values "--name", release, "--set", - "image.repository=" + imageName(), + "image.repository=" + imageName(t), "--set", "image.tag=" + tag, "--set", @@ -116,7 +88,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values for _, value := range values { arg = append(arg, "--set", value) } - out, _, err := runCommand(t, "helm", arg...) + out, _, err := command.Run("helm", arg...) t.Log(out) if err != nil { t.Error(out) @@ -127,7 +99,7 @@ func helmInstall(t *testing.T, cs *kubernetes.Clientset, release string, values func helmDelete(t *testing.T, cs *kubernetes.Clientset, release string) { t.Log("Deleting Helm release") t.Log(inspectLogs(t, cs, release)) - out, _, err := runCommand(t, "helm", "delete", "--purge", release) + out, _, err := command.Run("helm", "delete", "--purge", release) if err != nil { t.Error(out) t.Fatal(err) @@ -151,7 +123,10 @@ func helmDeletePVC(t *testing.T, cs *kubernetes.Clientset, release string) { } func kubeLogin(t *testing.T) *kubernetes.Clientset { - kc := os.Getenv("HOME") + "/.kube/config" + kc := os.Getenv("KUBECONFIG") + if kc == "" { + kc = os.Getenv("HOME") + "/.kube/config" + } c, err := clientcmd.BuildConfigFromFlags("", kc) if err != nil { t.Fatal(err) @@ -167,7 +142,7 @@ func kubeExec(t *testing.T, podName string, name string, arg ...string) (string, // Current version of Kubernetes Go client doesn't support "exec", so run this via the command line param := []string{"exec", podName, "--", name} param = append(param, arg...) - return runCommand(t, "kubectl", param...) + return command.Run("kubectl", param...) } func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) { @@ -199,12 +174,12 @@ func waitForReady(t *testing.T, cs *kubernetes.Clientset, release string) { for { // Current version of Kubernetes Go client doesn't support "exec", so run this via the command line // TODO: If we run "chkmqready" here, it doesn't seem to work - //out, _, err := runCommand(t, "kubectl", "exec", podName, "--", "dspmq") + //out, _, err := command.Run(t, "kubectl", "exec", podName, "--", "dspmq") out, _, err := kubeExec(t, podName, "dspmq") - //out, rc, err := runCommand(t, "kubectl", "exec", podName, "--", "chkmqready") + //out, rc, err := command.Run(t, "kubectl", "exec", podName, "--", "chkmqready") if err != nil { t.Error(out) - out2, _, err2 := runCommand(t, "kubectl", "describe", "pod", podName) + out2, _, err2 := command.Run("kubectl", "describe", "pod", podName) if err2 == nil { t.Log(out2) } @@ -258,17 +233,32 @@ func volumesAvailable(t *testing.T, cs *kubernetes.Clientset) bool { return false } +// digitsOnly returns only the digits from the supplied string +func digitsOnly(s string) string { + return strings.Map( + func(r rune) rune { + if unicode.IsDigit(r) { + return r + } + return -1 + }, + s, + ) +} + // assertKubeVersion is used to assert that a test requires a specific version of Kubernetes func assertKubeVersion(t *testing.T, cs *kubernetes.Clientset, major int, minor int) { v, err := cs.Discovery().ServerVersion() if err != nil { t.Fatal(err) } - maj, err := strconv.Atoi(v.Major) + // Make sure we use only the digits from the version, to account for things like "1.8+" + maj, err := strconv.Atoi(digitsOnly(v.Major)) if err != nil { t.Fatal(err) } - min, err := strconv.Atoi(v.Minor) + // Make sure we use only the digits from the version, to account for things like "1.8+" + min, err := strconv.Atoi(digitsOnly(v.Minor)) if err != nil { t.Fatal(err) }