From 80e7707deb7d92fb0d14d18cf9ce062f375e4bb1 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 1 Oct 2018 10:17:38 +0100 Subject: [PATCH 01/12] Replace master with singularity --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95f631e..2666037 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ [![Build Status](https://travis-ci.org/ibm-messaging/mq-container.svg?branch=master)](https://travis-ci.org/ibm-messaging/mq-container) -**Note**: The `master` branch may be in an *unstable or even broken state* during development. -To get a stable version, please use the correct [branch](https://github.com/ibm-messaging/mq-container/branches) for your MQ version, instead of the `master` branch. +**Note**: The `singularity` branch may be in an *unstable or even broken state* during development. +To get a stable version, please use the correct [branch](https://github.com/ibm-messaging/mq-container/branches) for your MQ version, instead of the `singularity` branch. IBM MQ logo From 0e567ccea7aba5ef36d0b0754a10b6dbf15051ca Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 1 Oct 2018 10:18:49 +0100 Subject: [PATCH 02/12] Remove dynamic Prometheus files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 8d8334b..8ca3f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ build coverage downloads incubating/mqipt/ms81* +vendor/github.com/prometheus/client_model/bin/ +vendor/github.com/prometheus/client_model/.classpath +vendor/github.com/prometheus/client_model/.project +vendor/github.com/prometheus/client_model/.settings* From 6abbbb039444151de2471fa5d63a9ad5152f2509 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 26 Sep 2018 16:17:15 +0100 Subject: [PATCH 03/12] Enable web console for mqadvanced-server --- Dockerfile-server | 7 ++++-- Makefile-RHEL | 3 +-- Makefile-UBUNTU | 12 +++------ .../{post_init_dev.go => post_init.go} | 2 -- cmd/runmqserver/post_init_other.go | 22 ---------------- cmd/runmqserver/webserver.go | 2 -- incubating/mqadvanced-server-dev/Dockerfile | 2 -- install-mq.sh | 4 +-- mq-advanced-server-rhel/mq-buildah.sh | 4 +++ mq-advanced-server-rhel/mqdev-buildah.sh | 2 +- .../Installation1/servers/mqweb/mqwebuser.xml | 25 +++++++++++++++++++ .../Installation1/servers/mqweb/tls.xml | 0 12 files changed, 41 insertions(+), 44 deletions(-) rename cmd/runmqserver/{post_init_dev.go => post_init.go} (98%) delete mode 100644 cmd/runmqserver/post_init_other.go create mode 100644 web/installations/Installation1/servers/mqweb/mqwebuser.xml rename {incubating/mqadvanced-server-dev/web => web}/installations/Installation1/servers/mqweb/tls.xml (100%) diff --git a/Dockerfile-server b/Dockerfile-server index 4324555..c1c9970 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -66,8 +66,11 @@ RUN chmod ug+x /usr/local/bin/runmqserver \ && chown mqm:mqm /usr/local/bin/*mq* \ && chmod ug+xs /usr/local/bin/chkmq* -# Always use port 1414 for MQ & 9157 for the metrics -EXPOSE 1414 9157 +# Always use port 1414 for MQ, 9157 for the metrics & 9443 for the web console +EXPOSE 1414 9157 9443 + +# Copy web XML files +COPY web /etc/mqm/web ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic diff --git a/Makefile-RHEL b/Makefile-RHEL index 3a8f0de..8dc2409 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -40,7 +40,7 @@ MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools MQ_IMAGE_GOLANG_SDK ?=mq-golang-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_PACKAGES specifies the MQ packages to install. Defaults vary on base image. -MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm +MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm ############################################################################### # Other variables @@ -134,7 +134,6 @@ build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs .PHONY: build-devserver build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: MQDEV=TRUE -build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END))) mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)" diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 4ff4c65..cf84362 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -127,7 +127,7 @@ downloads/$(MQ_ARCHIVE_DEV): downloads/$(MQ_SDK_ARCHIVE): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) mkdir -p downloads - cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) .PHONY: downloads downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) @@ -225,12 +225,6 @@ build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk-ex $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) .PHONY: build-devserver -# Target-specific variable to add web server into devserver image -ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" -build-devserver: MQ_PACKAGES=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams ibmmq-web -else -build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm -endif build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) @@ -241,7 +235,7 @@ build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex build-advancedserver-cover: docker-version $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_IMAGE_ADVANCEDSERVER)-cover -f Dockerfile-server.cover . -.PHONY: build-explorer +.PHONY: build-explorer build-explorer: downloads/$(MQ_ARCHIVE_DEV) docker-pull $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) @@ -251,7 +245,7 @@ build-sdk: downloads/$(MQ_SDK_ARCHIVE) build-sdk-ex .PHONY: build-sdk-ex ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" build-sdk-ex: MQ_PACKAGES=ibmmq-sdk ibmmq-samples build-essential -else +else build-sdk-ex: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm endif build-sdk-ex: docker-version docker-pull diff --git a/cmd/runmqserver/post_init_dev.go b/cmd/runmqserver/post_init.go similarity index 98% rename from cmd/runmqserver/post_init_dev.go rename to cmd/runmqserver/post_init.go index 861c566..59d6157 100644 --- a/cmd/runmqserver/post_init_dev.go +++ b/cmd/runmqserver/post_init.go @@ -1,5 +1,3 @@ -// +build mqdev - /* © Copyright IBM Corporation 2018 diff --git a/cmd/runmqserver/post_init_other.go b/cmd/runmqserver/post_init_other.go deleted file mode 100644 index 71f3458..0000000 --- a/cmd/runmqserver/post_init_other.go +++ /dev/null @@ -1,22 +0,0 @@ -// +build !mqdev - -/* -© Copyright IBM Corporation 2018 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -func postInit(name string) error { - return nil -} diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index a88786a..9318b09 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -1,5 +1,3 @@ -// +build mqdev - /* © Copyright IBM Corporation 2018 diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index f022258..e7cc68c 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -60,6 +60,4 @@ COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ COPY incubating/mqadvanced-server-dev/web /etc/mqm/web RUN chmod +x /usr/local/bin/runmq* -EXPOSE 9443 - ENTRYPOINT ["runmqdevserver"] diff --git a/install-mq.sh b/install-mq.sh index 6273628..ac57978 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -23,8 +23,8 @@ test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false # If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults if [ -z "$MQ_PACKAGES" ]; then - $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" - $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm" + $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams ibmmq-web" + $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm" fi if ($UBUNTU); then diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 12c803e..7b6bfeb 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -75,6 +75,9 @@ install --mode 0750 --owner 888 --group 888 ./build/runmqserver ${mnt_mq}/usr/lo install --mode 6750 --owner 888 --group 888 ./build/chk* ${mnt_mq}/usr/local/bin/ install --mode 0750 --owner 888 --group 888 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt +# Copy web XML files +cp -R web ${mnt_mq}/etc/mqm/web + ############################################################################### # Final Buildah commands ############################################################################### @@ -90,6 +93,7 @@ fi buildah config \ --port 1414/tcp \ --port 9157/tcp \ + --port 9443/tcp \ --os linux \ --label architecture=x86_64 \ --label io.openshift.tags="$OSTAG" \ diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index 4c9f7e8..1d29fbe 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -52,7 +52,7 @@ install --mode 0750 --owner 888 --group 888 ./build/runmqdevserver ${mnt_mq}/usr cp incubating/mqadvanced-server-dev/*.tpl ${mnt_mq}/etc/mqm/ # Copy web XML files for default developer configuration -cp -R incubating/mqadvanced-server-dev/web ${mnt_mq}/etc/mqm/web +cp -R incubating/mqadvanced-server-dev/web/ ${mnt_mq}/etc/mqm/web ############################################################################### # Final Buildah commands diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml b/web/installations/Installation1/servers/mqweb/mqwebuser.xml new file mode 100644 index 0000000..7bb6ae1 --- /dev/null +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml @@ -0,0 +1,25 @@ + + + + appSecurity-2.0 + + + + + + + + + + + + + + + + + + + + + diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml b/web/installations/Installation1/servers/mqweb/tls.xml similarity index 100% rename from incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml rename to web/installations/Installation1/servers/mqweb/tls.xml From 77eb7381e7d43051bf138672906889bd99f3a36c Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 3 Oct 2018 11:06:17 +0100 Subject: [PATCH 04/12] Move template and keystore functions to internal packages --- cmd/runmqdevserver/main.go | 3 ++- cmd/runmqdevserver/mqsc.go | 4 +++- cmd/runmqdevserver/tls.go | 22 ++++++++++--------- .../keystore}/keystore.go | 9 +++++--- .../mqtemplate/mqtemplate.go | 10 +++++---- 5 files changed, 29 insertions(+), 19 deletions(-) rename {cmd/runmqdevserver => internal/keystore}/keystore.go (95%) rename cmd/runmqdevserver/template.go => internal/mqtemplate/mqtemplate.go (83%) diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go index 8546bc8..f1d8cd9 100644 --- a/cmd/runmqdevserver/main.go +++ b/cmd/runmqdevserver/main.go @@ -24,6 +24,7 @@ import ( "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/logger" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" "github.com/ibm-messaging/mq-container/internal/name" ) @@ -85,7 +86,7 @@ func configureLogger() error { func configureWeb(qmName string) error { out := "/etc/mqm/web/installations/Installation1/angular.persistence/admin.json" - return processTemplateFile("/etc/mqm/admin.json.tpl", out, map[string]string{"QueueManagerName": qmName}) + return mqtemplate.ProcessTemplateFile("/etc/mqm/admin.json.tpl", out, map[string]string{"QueueManagerName": qmName}, log) } func logTerminationf(format string, args ...interface{}) { diff --git a/cmd/runmqdevserver/mqsc.go b/cmd/runmqdevserver/mqsc.go index 77e3dfa..d2d111b 100644 --- a/cmd/runmqdevserver/mqsc.go +++ b/cmd/runmqdevserver/mqsc.go @@ -17,6 +17,8 @@ package main import ( "os" + + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) func updateMQSC(appPasswordRequired bool) error { @@ -30,7 +32,7 @@ func updateMQSC(appPasswordRequired bool) error { if os.Getenv("MQ_DEV") == "true" { const mqscTemplate string = mqsc + ".tpl" // Re-configure channel if app password not set - err := processTemplateFile(mqsc+".tpl", mqsc, map[string]string{"ChckClnt": checkClient}) + err := mqtemplate.ProcessTemplateFile(mqsc+".tpl", mqsc, map[string]string{"ChckClnt": checkClient}, log) if err != nil { return err } diff --git a/cmd/runmqdevserver/tls.go b/cmd/runmqdevserver/tls.go index 7e6ed2c..55681f5 100644 --- a/cmd/runmqdevserver/tls.go +++ b/cmd/runmqdevserver/tls.go @@ -21,20 +21,22 @@ import ( "path/filepath" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/keystore" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) -func configureWebTLS(cms *KeyStore) error { +func configureWebTLS(cms *keystore.KeyStore) error { dir := "/run/runmqdevserver/tls" - ks := NewJKSKeyStore(filepath.Join(dir, "key.jks"), cms.Password) - ts := NewJKSKeyStore(filepath.Join(dir, "trust.jks"), cms.Password) + ks := keystore.NewJKSKeyStore(filepath.Join(dir, "key.jks"), cms.Password) + ts := keystore.NewJKSKeyStore(filepath.Join(dir, "trust.jks"), cms.Password) log.Debug("Creating key store") - err := ks.Create() + err := ks.Create(log) if err != nil { return err } log.Debug("Creating trust store") - err = ts.Create() + err = ts.Create(log) if err != nil { return err } @@ -104,14 +106,14 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { } } - cms := NewCMSKeyStore(keyFile, passPhrase) + cms := keystore.NewCMSKeyStore(keyFile, passPhrase) - err = cms.Create() + err = cms.Create(log) if err != nil { return err } - err = cms.CreateStash() + err = cms.CreateStash(log) if err != nil { return err } @@ -145,11 +147,11 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { const mqsc string = "/etc/mqm/20-dev-tls.mqsc" const mqscTemplate string = mqsc + ".tpl" - err = processTemplateFile(mqscTemplate, mqsc, map[string]string{ + err = mqtemplate.ProcessTemplateFile(mqscTemplate, mqsc, map[string]string{ "SSLKeyR": filepath.Join(dir, "key"), "CertificateLabel": newLabel, "SSLCipherSpec": sslCipherSpec, - }) + }, log) if err != nil { return err } diff --git a/cmd/runmqdevserver/keystore.go b/internal/keystore/keystore.go similarity index 95% rename from cmd/runmqdevserver/keystore.go rename to internal/keystore/keystore.go index 87eac87..c48bd45 100644 --- a/cmd/runmqdevserver/keystore.go +++ b/internal/keystore/keystore.go @@ -13,7 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main + +// Package keystore contains code to create and update keystores +package keystore import ( "bufio" @@ -23,6 +25,7 @@ import ( "strings" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/logger" ) // KeyStore describes information about a keystore file @@ -54,7 +57,7 @@ func NewCMSKeyStore(filename, password string) *KeyStore { } // Create a key store, if it doesn't already exist -func (ks *KeyStore) Create() error { +func (ks *KeyStore) Create(log *logger.Logger) error { _, err := os.Stat(ks.Filename) if err == nil { // Keystore already exists so we should refresh it by deleting it. @@ -95,7 +98,7 @@ func (ks *KeyStore) Create() error { } // CreateStash creates a key stash, if it doesn't already exist -func (ks *KeyStore) CreateStash() error { +func (ks *KeyStore) CreateStash(log *logger.Logger) error { extension := filepath.Ext(ks.Filename) stashFile := ks.Filename[0:len(ks.Filename)-len(extension)] + ".sth" log.Debugf("TLS stash file: %v", stashFile) diff --git a/cmd/runmqdevserver/template.go b/internal/mqtemplate/mqtemplate.go similarity index 83% rename from cmd/runmqdevserver/template.go rename to internal/mqtemplate/mqtemplate.go index 7941633..28d80fc 100644 --- a/cmd/runmqdevserver/template.go +++ b/internal/mqtemplate/mqtemplate.go @@ -13,7 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main + +// Package mqtemplate contains code to process template files +package mqtemplate import ( "os" @@ -21,12 +23,12 @@ import ( "text/template" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/logger" ) -// processTemplateFile takes a Go templateFile, and processes it with the +// ProcessTemplateFile takes a Go templateFile, and processes it with the // supplied data, writing to destFile -func processTemplateFile(templateFile, destFile string, data interface{}) error { - // Re-configure channel if app password not set +func ProcessTemplateFile(templateFile, destFile string, data interface{}, log *logger.Logger) error { t, err := template.ParseFiles(templateFile) if err != nil { log.Error(err) From 149915d587690854642e415f91755153a757e8b7 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 3 Oct 2018 13:19:09 +0100 Subject: [PATCH 05/12] Configure Single-Sign-On for the web server --- cmd/runmqserver/post_init.go | 14 ++- cmd/runmqserver/webserver.go | 98 +++++++++++++++++++ internal/keystore/keystore.go | 18 ++++ .../servers/mqweb/mqwebuser.xml.tpl | 44 +++++++++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl diff --git a/cmd/runmqserver/post_init.go b/cmd/runmqserver/post_init.go index 59d6157..e2bae03 100644 --- a/cmd/runmqserver/post_init.go +++ b/cmd/runmqserver/post_init.go @@ -20,18 +20,26 @@ import ( ) // postInit is run after /var/mqm is set up -// This version of postInit is only included as part of the MQ Advanced for Developers build func postInit(name string) error { disable := os.Getenv("MQ_DISABLE_WEB_CONSOLE") if disable != "true" && disable != "1" { + + // Configure Single-Sign-On for the web server (if enabled) + enableSSO := os.Getenv("MQ_ENABLE_SSO") + if enableSSO == "true" || enableSSO == "1" { + err := configureSSO() + if err != nil { + return err + } + } + // Configure the web server (if installed) err := configureWebServer() if err != nil { return err } // Start the web server, in the background (if installed) - // WARNING: No error handling or health checking available for the web server, - // which is why it's limited to use with MQ Advanced for Developers only + // WARNING: No error handling or health checking available for the web server go func() { startWebServer() }() diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index 9318b09..6ff5c63 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -21,9 +21,12 @@ import ( "os" "os/exec" "path/filepath" + "strings" "syscall" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/keystore" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) func startWebServer() error { @@ -75,6 +78,101 @@ func CopyFile(src, dest string) error { return err } +func configureSSO() error { + + // Ensure all required environment variables are set for SSO + requiredEnvVars := []string{ + "MQ_WEB_ADMIN_USERS", + "MQ_OIDC_CLIENT_ID", + "MQ_OIDC_CLIENT_SECRET", + "MQ_OIDC_AUTHORIZATION_ENDPOINT", + "MQ_OIDC_TOKEN_ENDPOINT", + "MQ_OIDC_JWK_ENDPOINT", + "MQ_OIDC_ISSUER_IDENTIFIER", + "MQ_OIDC_CERTIFICATE", + } + for _, envVar := range requiredEnvVars { + if len(os.Getenv(envVar)) == 0 { + return fmt.Errorf("%v must be set when MQ_ENABLE_SSO=true", envVar) + } + } + + // Check mqweb directory exists + const mqwebDir string = "/etc/mqm/web/installations/Installation1/servers/mqweb" + _, err := os.Stat(mqwebDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + // Process SSO template for generating file mqwebuser.xml + adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), ",") + err = mqtemplate.ProcessTemplateFile(mqwebDir+"/mqwebuser.xml.tpl", mqwebDir+"/mqwebuser.xml", map[string][]string{"AdminUser": adminUsers}, log) + if err != nil { + return err + } + + // Configure SSO TLS + return configureSSO_TLS() +} + +func configureSSO_TLS() error { + + // Create tls directory + dir := "/run/tls" + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, 0770) + if err != nil { + return err + } + mqmUID, mqmGID, err := command.LookupMQM() + if err != nil { + log.Error(err) + return err + } + err = os.Chown(dir, mqmUID, mqmGID) + if err != nil { + log.Error(err) + return err + } + } else { + return err + } + } + + // Setup key store & trust store + ks := keystore.NewJKSKeyStore(filepath.Join(dir, "key.jks"), "password") + ts := keystore.NewJKSKeyStore(filepath.Join(dir, "trust.jks"), "password") + + log.Debug("Creating key store") + err = ks.Create(log) + if err != nil { + return err + } + log.Debug("Creating trust store") + err = ts.Create(log) + if err != nil { + return err + } + log.Debug("Creating self-signed certificate in key store") + err = ks.CreateSelfSignedCertificate("default", "CN=IBMMQWeb,O=IBM,OU=Platform,C=GB") + if err != nil { + return err + } + log.Debug("Importing self-signed certificate into trust store") + err = ts.Import(ks.Filename, ks.Password) + if err != nil { + return err + } + log.Debug("Adding OIDC CA certificate to trust store") + err = ts.Add(os.Getenv("MQ_OIDC_CERTIFICATE"), "OIDC") + return err +} + func configureWebServer() error { _, err := os.Stat("/opt/mqm/bin/strmqweb") if err != nil { diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index c48bd45..78d582b 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -134,6 +134,24 @@ func (ks *KeyStore) Import(inputFile, password string) error { return nil } +// CreateSelfSignedCertificate creates a self-signed certificate in the keystore +func (ks *KeyStore) CreateSelfSignedCertificate(label, dn string) error { + out, _, err := command.Run(ks.command, "-cert", "-create", "-db", ks.Filename, "-pw", ks.Password, "-label", label, "-dn", dn) + if err != nil { + return fmt.Errorf("error running \"%v -cert -create\": %v %s", ks.command, err, out) + } + return nil +} + +// Add adds a CA certificate to the keystore +func (ks *KeyStore) Add(inputFile, label string) error { + out, _, err := command.Run(ks.command, "-cert", "-add", "-db", ks.Filename, "-type", ks.keyStoreType, "-pw", ks.Password, "-file", inputFile, "-label", label) + if err != nil { + return fmt.Errorf("error running \"%v -cert -add\": %v %s", ks.command, err, out) + } + return nil +} + // GetCertificateLabels returns the labels of all certificates in the key store func (ks *KeyStore) GetCertificateLabels() ([]string, error) { out, _, err := command.Run(ks.command, "-cert", "-list", "-type", ks.keyStoreType, "-db", ks.Filename, "-pw", ks.Password) diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl new file mode 100644 index 0000000..65a4b60 --- /dev/null +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -0,0 +1,44 @@ + + + + openidConnectClient-1.0 + ssl-1.0 + + + + + + {{- range $index, $element := .AdminUser}} + + {{- end}} + + + + + + + + + + + + + + + + + + + + + + From 5ba73c1d2a20b583869aa138d002fb3baafb2589 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 8 Oct 2018 16:51:37 +0100 Subject: [PATCH 06/12] update apparmor --- install-mq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-mq.sh b/install-mq.sh index ac57978..71f5a19 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -139,7 +139,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y gcc-5-base gnupg gpgv libgcrypt20 libstdc++6 perl-base --only-upgrade +$UBUNTU && apt-get install -y libapparmor1 --only-upgrade # End of bug fixes # Clean up cached files From 574386fe82304814ae4a3412ecbe497fdd7171cc Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Fri, 12 Oct 2018 14:13:20 +0100 Subject: [PATCH 07/12] Split SSO admin user list on newlines (#229) --- cmd/runmqserver/webserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index 6ff5c63..d3146a4 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -108,7 +108,7 @@ func configureSSO() error { } // Process SSO template for generating file mqwebuser.xml - adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), ",") + adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), "\n") err = mqtemplate.ProcessTemplateFile(mqwebDir+"/mqwebuser.xml.tpl", mqwebDir+"/mqwebuser.xml", map[string][]string{"AdminUser": adminUsers}, log) if err != nil { return err From e6049ecb93679f432989553f534506e107b6eb84 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Thu, 18 Oct 2018 13:43:48 +0100 Subject: [PATCH 08/12] add integration into the docker tag (#233) --- Makefile-RHEL | 4 ++-- Makefile-UBUNTU | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index 8dc2409..7ddd69e 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -32,9 +32,9 @@ MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image -MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image -MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_SDK is the name and tag of the built MQ Advanced for Developers SDK image MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index cf84362..5a6e87d 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -32,9 +32,9 @@ MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image -MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image -MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_SDK is the name and tag of the built MQ Advanced for Developers SDK image MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools From c88329d77982717a8ca0414c9c8c3811e12aaf96 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Thu, 25 Oct 2018 10:17:02 +0100 Subject: [PATCH 09/12] Fix for iframe issue with web console (#238) --- web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl index 65a4b60..3bb4b8c 100644 --- a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -36,6 +36,8 @@ issuerIdentifier="${env.MQ_OIDC_ISSUER_IDENTIFIER}"> + + From b8227abf7f95f4537d02dd6040407ad0a4e09324 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Fri, 26 Oct 2018 14:47:49 +0100 Subject: [PATCH 10/12] Add uniqueUserIdentifier --- cmd/runmqserver/webserver.go | 1 + web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index d3146a4..9b218ab 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -85,6 +85,7 @@ func configureSSO() error { "MQ_WEB_ADMIN_USERS", "MQ_OIDC_CLIENT_ID", "MQ_OIDC_CLIENT_SECRET", + "MQ_OIDC_UNIQUE_USER_IDENTIFIER", "MQ_OIDC_AUTHORIZATION_ENDPOINT", "MQ_OIDC_TOKEN_ENDPOINT", "MQ_OIDC_JWK_ENDPOINT", diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl index 3bb4b8c..5cebb71 100644 --- a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -27,6 +27,7 @@ Date: Wed, 7 Nov 2018 11:47:41 +0000 Subject: [PATCH 11/12] Add TLS support (#243) * Add TLS support * Security fix for libsystemd0 systemd systemd-sysv libudev1 --- cmd/runmqserver/webserver.go | 10 +++++----- install-mq.sh | 8 +++++--- internal/keystore/keystore.go | 9 +++++++++ mq-advanced-server-rhel/mq-buildah.sh | 3 ++- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index 9b218ab..9c82a1d 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -159,17 +159,17 @@ func configureSSO_TLS() error { if err != nil { return err } - log.Debug("Creating self-signed certificate in key store") - err = ks.CreateSelfSignedCertificate("default", "CN=IBMMQWeb,O=IBM,OU=Platform,C=GB") + log.Debug("Generating PKCS12 file") + err = ks.GeneratePKCS12("/mnt/tls/tls.key", "/mnt/tls/tls.crt", "/run/tls/tls.p12", "default", "password") if err != nil { return err } - log.Debug("Importing self-signed certificate into trust store") - err = ts.Import(ks.Filename, ks.Password) + log.Debug("Importing certificate into key store") + err = ks.Import("/run/tls/tls.p12", "password") if err != nil { return err } - log.Debug("Adding OIDC CA certificate to trust store") + log.Debug("Adding OIDC certificate to trust store") err = ts.Add(os.Getenv("MQ_OIDC_CERTIFICATE"), "OIDC") return err } diff --git a/install-mq.sh b/install-mq.sh index 71f5a19..cd8bdd2 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -63,7 +63,8 @@ if ($UBUNTU); then procps \ sed \ tar \ - util-linux + util-linux \ + openssl fi # Install additional packages required by MQ, this install process and the runtime scripts @@ -82,7 +83,8 @@ $RHEL && yum -y install \ procps-ng \ sed \ tar \ - util-linux + util-linux \ + openssl # Download and extract the MQ installation files DIR_EXTRACT=/tmp/mq @@ -139,7 +141,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y libapparmor1 --only-upgrade +$UBUNTU && apt-get install -y libsystemd0 systemd systemd-sysv libudev1 --only-upgrade # End of bug fixes # Clean up cached files diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 78d582b..2f4b759 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -125,6 +125,15 @@ func (ks *KeyStore) CreateStash(log *logger.Logger) error { return nil } +// GeneratePKCS12 generates a PKCS12 file +func (ks *KeyStore) GeneratePKCS12(keyFile, crtFile, pkcs12File, label, password string) error { + out, _, err := command.Run("openssl", "pkcs12", "-export", "-inkey", keyFile, "-in", crtFile, "-out", pkcs12File, "-name", label, "-passout", "pass:"+password) + if err != nil { + return fmt.Errorf("error running \"openssl pkcs12 -export\": %v %s", err, out) + } + return nil +} + // Import imports a certificate file in the keystore func (ks *KeyStore) Import(inputFile, password string) error { out, _, err := command.Run(ks.command, "-cert", "-import", "-file", inputFile, "-pw", password, "-target", ks.Filename, "-target_pw", ks.Password, "-target_type", ks.keyStoreType) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 7b6bfeb..d16d351 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -57,7 +57,8 @@ buildah run $ctr_mq -- yum install -y --setopt install_weak_deps=false --setopt= procps-ng \ sed \ tar \ - util-linux + util-linux \ + openssl # Clean up cached files buildah run $ctr_mq -- yum clean all From e74ba3fd756fdce01af7bcafdff007e3ff12ab8e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 5 Dec 2018 13:34:58 +0000 Subject: [PATCH 12/12] update perl-base to fix security vulnerability --- install-mq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-mq.sh b/install-mq.sh index cd8bdd2..141dc9d 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -141,7 +141,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y libsystemd0 systemd systemd-sysv libudev1 --only-upgrade +$UBUNTU && apt-get install -y libsystemd0 systemd systemd-sysv libudev1 perl-base --only-upgrade # End of bug fixes # Clean up cached files