Further changes with htpasswd provider

authservice/mqhtpass/src/htpass.c

@@ -25,7 +25,44 @@ limitations under the License.
 #include <apr_errno.h>
 #include <apr_md5.h>
 
-char *find_hash(char *, char *);
+bool htpass_valid_file(char *filename)
+{
+  bool valid = true;
+  FILE *fp;
+  char *huser;
+
+  fp = fopen(filename, "r");
+  if (fp == NULL)
+  {
+    log_errorf("Error %d opening htpasswd file '%s'", errno, filename);
+  }
+  if (fp)
+  {
+    const size_t line_size = 1024;
+    char *line = malloc(line_size);
+    while (fgets(line, line_size, fp) != NULL)
+    {
+      char *saveptr;
+      // Need to use strtok_r to be safe for multiple threads
+      huser = strtok_r(line, ":", &saveptr);
+      if (strlen(huser) >= 12)
+      {
+        log_errorf("Invalid htpasswd file for use with IBM MQ.  User '%s' is longer than twelve characters", huser);
+        valid = false;
+        break;
+      }
+      else {
+
+      }
+    }
+    fclose(fp);
+    if (line)
+    {
+      free(line);
+    }
+  }
+  return valid;
+}
 
 char *find_hash(char *filename, char *user)
 {
@@ -58,19 +95,27 @@ char *find_hash(char *filename, char *user)
     }
     fclose(fp);
     if (line)
+    {
       free(line);
     }
+  }
   if (!found)
   {
     hash = NULL;
   }
-  return (hash);
+  return hash;
 }
 
 bool htpass_authenticate_user(char *filename, char *user, char *password)
 {
   char *hash = find_hash(filename, user);
   bool result = false;
+  if (hash == NULL)
+  {
+    log_debugf("User does not exist. user=%s", user);
+  }
+  else
+  {
     // Use the Apache Portable Runtime utilities to validate the password against the hash.
     // Supports multiple hashing algorithms, but we should only be using bcrypt
     apr_status_t status = apr_password_validate(password, hash);
@@ -84,7 +129,8 @@ bool htpass_authenticate_user(char *filename, char *user, char *password)
     {
       log_debugf("Incorrect password supplied. user=%s", user);
     }
-  return (result);
+  }
+  return result;
 }
 
 bool htpass_valid_user(char *filename, char *user)
@@ -95,5 +141,5 @@ bool htpass_valid_user(char *filename, char *user)
   {
     valid = true;
   }
-  return (valid);
+  return valid;
 }

authservice/mqhtpass/src/htpass.h

@@ -17,7 +17,28 @@ limitations under the License.
 #ifndef _HTPASS_H
 #define _HTPASS_H
 
-_Bool htpass_authenticate_user(char *, char *, char *);
+/**
-_Bool htpass_valid_user(char *, char *);
+ * Validate an HTPasswd file for use with IBM MQ.
+ * 
+ * @param filename the HTPasswd file
+ */
+_Bool htpass_valid_file(char *filename);
+
+/**
+ * Authenticate a user, based on the supplied file name.
+ * 
+ * @param filename the HTPasswd file
+ * @param user the user name to authenticate
+ * @param password the password of the user
+ */
+_Bool htpass_authenticate_user(char *filename, char *user, char *password);
+
+/**
+ * Validate that a user exists in the password file.
+ * 
+ * @param filename the HTPasswd file
+ * @param user the user name to validate
+ */
+_Bool htpass_valid_user(char *filename, char *user);
 
 #endif

authservice/mqhtpass/src/htpass_test.c

@@ -36,9 +36,30 @@ void test_fail(const char *test_name)
   exit(1);
 }
 
+// ----------------------------------------------------------------------------
+// Simple tests for file validation
+// ----------------------------------------------------------------------------
+
+void test_htpass_valid_file_ok()
+{
+  test_start();
+  int ok = htpass_valid_file("./src/htpass_test.htpasswd");
+  if (!ok)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_valid_file_too_long()
+{
+  test_start();
+  int ok = htpass_valid_file("./src/htpass_test_invalid.htpasswd");
+  if (ok)
+    test_fail(__func__);
+  test_pass();
+}
 
 // ----------------------------------------------------------------------------
-// Simple tests
+// Simple tests for authentication
 // ----------------------------------------------------------------------------
 
 void test_htpass_authenticate_user_fred_valid()
@@ -136,6 +157,10 @@ void check_log_file_valid(char *filename)
       errors++;
     }
   }
+  if (line)
+  {
+    free(line);
+  }
   fclose(log);
 }
 
@@ -173,6 +198,8 @@ int main()
   // Turn on debugging for the tests
   setenv("DEBUG", "true", true);
   log_init("htpass_test.log");
+  test_htpass_valid_file_ok();
+  test_htpass_valid_file_too_long();
   test_htpass_authenticate_user_fred_valid();
   test_htpass_authenticate_user_fred_invalid1();
   test_htpass_authenticate_user_fred_invalid2();

authservice/mqhtpass/src/htpass_test_invalid.htpasswd

@@ -0,0 +1,3 @@
+fred:$2y$05$3Fp9epsqEwWOHdyj9Ngf9.qfX34kzc9zNrdQ7kac0GmcCvQjIkAwy
+barney:$2y$05$l8EoyCQ9y2PyfUzIDDfTyu7SSaJEYB1TuHy07xZvN7xt/pR3SIw0a
+namewhichisfartoolongformq:$2y$05$l8EoyCQ9y2PyfUzIDDfTyu7SSaJEYB1TuHy07xZvN7xt/pR3SIw0a

authservice/mqhtpass/src/log.c

@@ -42,7 +42,10 @@ void init_debug(){
   }
 }
 
-int log_init(char *filename)
+/**
+ * Internal function to initialize the log with the given file mode.
+ */
+int log_init_internal(char *filename, const char *mode)
 {
   int result = 0;
   pid = getpid();
@@ -51,6 +54,7 @@ int log_init(char *filename)
     fp = fopen(filename, "a");
     if (fp)
     {
+      // Disable buffering for this file
       setbuf(fp, NULL);
     }
     else
@@ -62,6 +66,18 @@ int log_init(char *filename)
   return result;
 }
 
+int log_init_reset(char *filename)
+{
+  // Open the log file for writing (overwrite if it already exists)
+  return log_init_internal(filename, "w");
+}
+
+int log_init(char *filename)
+{
+  // Open the log file file for appending
+  return log_init_internal(filename, "a");
+}
+
 void log_init_file(FILE *f)
 {
   fp = f;
@@ -105,12 +121,18 @@ void log_printf(const char *source_file, int source_line, const char *level, con
     if (strftime(date_buf, sizeof date_buf, "%FT%T", utc))
     {
        // Round microseconds down to milliseconds, for consistency
-       cur += snprintf(cur, end-cur, ", \"ibm_datetime\":\"%s.%03ldZ", date_buf, now.tv_usec / 1000);
+       cur += snprintf(cur, end-cur, ", \"ibm_datetime\":\"%s.%03ldZ\"", date_buf, now.tv_usec / 1000);
     }
     cur += snprintf(cur, end-cur, ", \"ibm_processId\":\"%d\"", pid);
     cur += snprintf(cur, end-cur, ", \"module\":\"%s:%d\"", source_file, source_line);
     cur += snprintf(cur, end-cur, ", \"message\":\"");
 
+    if (strncmp(level, "DEBUG", 5) == 0)
+    {
+      // Add a prefix on any debug messages
+      cur += snprintf(cur, end-cur, "mqhtpass: ");
+    }
+
     // Print log message, using varargs
     va_list args;
     va_start(args, format);

authservice/mqhtpass/src/log.h

@@ -17,20 +17,30 @@ limitations under the License.
 #ifndef _LOG_H
 #define _LOG_H
 
+/**
+ * Initialize the log to use the given file name, wiping any existing contents.
+ */
+int log_init_reset(char *filename);
+
 /**
  * Initialize the log to use the given file name.
  */
-int log_init(char *);
+int log_init(char *filename);
 
 /**
  * Initialize the log with an existing file handle.
  */
-void log_init_file(FILE *);
+void log_init_file(FILE *f);
 
 /**
  * Write a message to the log file, based on a printf format string.
+ * 
+ * @param source_file the name of the source code file submitting this log message
+ * @param source_line the line of code in the source file
+ * @param level the log level, one of "DEBUG", "INFO" or "ERROR"
+ * @param format the printf format string for the message
  */
-void log_printf(const char*, int, const char*, const char*, ...);
+void log_printf(const char *source_file, int source_line, const char *level, const char *format, ...);
 
 void log_close();
 

authservice/mqhtpass/src/mqhtpass.c

@@ -24,16 +24,15 @@ limitations under the License.
 #include "log.h"
 #include "htpass.h"
 
-/****************************************************************************/
+// Declare the internal functions that implement the interface
-/* Declare the internal functions that implement the interface              */
-/****************************************************************************/
 MQZ_INIT_AUTHORITY MQStart;
 static MQZ_AUTHENTICATE_USER mqhtpass_authenticate_user;
 static MQZ_FREE_USER mqhtpass_free_user;
-static MQZ_TERM_AUTHORITY mqhtpass_term_auth;
+static MQZ_TERM_AUTHORITY mqhtpass_terminate;
 
-#define LOG_FILE "/var/mqm/errors/mqhtpass.log"
+#define LOG_FILE "/var/mqm/errors/mqhtpass.json"
 #define HTPASSWD_FILE "/etc/mqm/mq.htpasswd"
+#define NAME "MQ Advanced for Developers custom authentication service"
 
 static char *trim(char *s);
 
@@ -44,13 +43,7 @@ static char *trim(char *s);
  *
  * This function is called whenever the module is loaded.  The Options
  * field will show whether it's a PRIMARY (i.e. during qmgr startup) or
- * SECONDARY (any other time - normally during the start of an agent
+ * SECONDARY.
- * process which is not necessarily the same as during MQCONN, especially
- * when running multi-threaded agents) initialization, but there's
- * nothing different that we'd want to do here based on that flag.
- *
- * Because of when the init function is called, there is no need to
- * worry about multi-threaded stuff in this particular function.
  */
 void MQENTRY MQStart(
     MQHCONFIG hc,
@@ -66,31 +59,52 @@ void MQENTRY MQStart(
   MQLONG Reason = MQRC_NONE;
   int log_rc = 0;
 
+  if (Options == MQZIO_PRIMARY)
+  {
+    // Reset the log file.  The file could still get large if debug is turned on,
+    // but this is a simpler solution for now.
+    log_rc = log_init_reset(LOG_FILE);
+  }
+  else
+  {
     log_rc = log_init(LOG_FILE);
+  }
+
   if (log_rc != 0)
   {
     CC = MQCC_FAILED;
     Reason = MQRC_INITIALIZATION_FAILED;
   }
 
-  log_infof("MQStart options=%s qmgr=%s", ((Options == MQZIO_SECONDARY) ? "Secondary" : "Primary"), trim(QMgrName));
+  if (Options == MQZIO_PRIMARY)
-  /************************************************************************/
+  {
-  /* Initialize the entry point vectors.  This is performed for both      */
+    log_infof("Initializing %s", NAME);
-  /* global and process initialisation, i.e whatever the value of the     */
+  }
-  /* Options field.                                                       */
+  log_debugf("MQStart options=%s qmgr=%s", ((Options == MQZIO_SECONDARY) ? "Secondary" : "Primary"), trim(QMgrName));
-  /************************************************************************/
+
+  if (!htpass_valid_file(HTPASSWD_FILE))
+  {
+    CC = MQCC_FAILED;
+    Reason = MQRC_INITIALIZATION_FAILED;
+  }
+
+  // Initialize the functions to use for each entry point
   if (CC == MQCC_OK)
+  {
     hc->MQZEP_Call(hc, MQZID_INIT_AUTHORITY, (PMQFUNC)MQStart, &CC, &Reason);
-
+  }
   if (CC == MQCC_OK)
-    hc->MQZEP_Call(hc, MQZID_TERM_AUTHORITY, (PMQFUNC)mqhtpass_term_auth, &CC, &Reason);
+  {
-
+    hc->MQZEP_Call(hc, MQZID_TERM_AUTHORITY, (PMQFUNC)mqhtpass_terminate, &CC, &Reason);
+  }
   if (CC == MQCC_OK)
+  {
     hc->MQZEP_Call(hc, MQZID_AUTHENTICATE_USER, (PMQFUNC)mqhtpass_authenticate_user, &CC, &Reason);
-
+  }
   if (CC == MQCC_OK)
+  {
     hc->MQZEP_Call(hc, MQZID_FREE_USER, (PMQFUNC)mqhtpass_free_user, &CC, &Reason);
-
+  }
   *Version = MQZAS_VERSION_5;
   *pCompCode = CC;
   *pReason = Reason;
@@ -98,24 +112,7 @@ void MQENTRY MQStart(
 }
 
 /**
- * Called during the connection of any application. This allows the OAM
+ * Called during the connection of any application.
- * to change the userid associated with the connection, regardless of the
- * operating system user ID. One reason you might want to do that is to
- * deal with non-standard user IDs, which perhaps are longer than 12
- * characters. The CorrelationPtr can be assigned in this function to
- * point to some OAM-managed storage, and is available as part of the
- * MQZED structure for all subsequent functions. Note that there is only
- * one CorrelPtr stored for the user's hconn, so if two OAMs are chained
- * and both want to manage storage for the connection, there would be
- * difficulties as there is no reverse call that would allow the second
- * to reset the first's pointer (or vice versa). I'd suggest instead
- * using something like thread-specific storage as each thread is tied
- * to the hconn.
- *
- * When a clntconn/svrconn channel connects to the queue manager, the
- * authentication is supposed to take two stages. First as the
- * channel program connects, and then as the MCAUSER is set. You will
- * see this as "initial" and "change" context values in the parameters.
  */
 static void MQENTRY mqhtpass_authenticate_user(
     PMQCHAR pQMgrName,
@@ -146,17 +143,19 @@ static void MQENTRY mqhtpass_authenticate_user(
     spuser = malloc(pSecurityParms->CSPUserIdLength + 1);
     if (!spuser)
     {
-      log_errorf("Unable to allocate memory");
+      log_errorf("%s is unable to allocate memory for a user", NAME);
       return;
     }
     strncpy(spuser, pSecurityParms->CSPUserIdPtr, pSecurityParms->CSPUserIdLength);
     spuser[pSecurityParms->CSPUserIdLength] = 0;
-    sppass = malloc(pSecurityParms->CSPPasswordLength + 1);
+    sppass = malloc((pSecurityParms->CSPPasswordLength + 1));
     if (!sppass)
     {
-      log_errorf("Unable to allocate memory");
+      log_errorf("%s is unable to allocate memory for a password", NAME);
       if (spuser)
+      {
         free(spuser);
+      }
       return;
     }
     strncpy(sppass, pSecurityParms->CSPPasswordPtr, pSecurityParms->CSPPasswordLength);
@@ -175,34 +174,49 @@ static void MQENTRY mqhtpass_authenticate_user(
     else
     {
       log_debugf(
-          "Failed to authenticate user=%s effuser=%s applname=%s cspuser=%s cc=%d reason=%d",
+          "User authentication failed user=%s effuser=%s applname=%s cspuser=%s cc=%d reason=%d",
-          pIdentityContext->UserIdentifier,
+          trim(pIdentityContext->UserIdentifier),
-          pApplicationContext->EffectiveUserID,
+          trim(pApplicationContext->EffectiveUserID),
-          pApplicationContext->ApplName,
+          trim(pApplicationContext->ApplName),
-          spuser,
+          trim(spuser),
           *pCompCode,
           *pReason);
     }
     if (spuser)
+    {
       free(spuser);
+    }
     if (sppass)
+    {
       free(sppass);
     }
+  }
   else
   {
     // Password not supplied, so just check that the user ID is valid
     spuser = malloc(sizeof(PMQCHAR12) + 1);
-    if (!sppass)
+    if (!spuser)
     {
-      log_errorf("Unable to allocate memory");
+      log_errorf("%s is unable to allocate memory to check a user", NAME);
       return;
     }
     strncpy(spuser, pApplicationContext->EffectiveUserID, strlen(pApplicationContext->EffectiveUserID));
     spuser[sizeof(PMQCHAR12)] = 0;
-    log_debugf("%s without CSP user set.  effectiveuid=%s", __func__, spuser);
+    log_debugf("%s without CSP user set.  effectiveuid=%s env=%d, callertype=%d, type=%d, accttoken=%d applidentitydata=%d", __func__, spuser, pApplicationContext->Environment, pApplicationContext->CallerType, pApplicationContext->AuthenticationType, pIdentityContext->AccountingToken, pIdentityContext->ApplIdentityData);
+    if (strncmp(spuser, "mqm", 3) == 0)
+    {
+      // Special case: pass the "mqm" user on for validation up the chain
+      // A warning in the completion code means MQ will pass this to other authorization services
+      *pCompCode = MQCC_WARNING;
+      *pReason = MQRC_NONE;
+      *pContinuation = MQZCI_CONTINUE;
+    }
+    else
+    {
       bool valid_user = htpass_valid_user(HTPASSWD_FILE, spuser);
       if (valid_user)
       {
+        // An OK completion code means MQ will accept this user is authenticated
         *pCompCode = MQCC_OK;
         *pReason = MQRC_NONE;
         *pContinuation = MQZCI_CONTINUE;
@@ -211,25 +225,25 @@ static void MQENTRY mqhtpass_authenticate_user(
       else
       {
         log_debugf(
-          "Invalid user=%s effuser=%s applname=%s cspuser=%s cc=%d reason=%d",
+            "User authentication failed user=%s effuser=%s applname=%s cspuser=%s cc=%d reason=%d",
-          pIdentityContext->UserIdentifier,
+            trim(pIdentityContext->UserIdentifier),
-          pApplicationContext->EffectiveUserID,
+            trim(pApplicationContext->EffectiveUserID),
-          pApplicationContext->ApplName,
+            trim(pApplicationContext->ApplName),
-          spuser,
+            trim(spuser),
             *pCompCode,
             *pReason);
       }
       if (spuser)
+      {
         free(spuser);
       }
+    }
+  }
   return;
 }
 
 /**
- * Called during MQDISC, as the inverse of the Authenticate. If the authorization
+ * Called during MQDISC, as the inverse of the call to authenticate.
- * service has allocated private storage to hold additional information about
- * the user, then this is the time to free it. No more calls will be made
- * to the authorization service for this connection instance of this user.
  */
 static void MQENTRY mqhtpass_free_user(
     PMQCHAR pQMgrName,
@@ -247,12 +261,9 @@ static void MQENTRY mqhtpass_free_user(
 }
 
 /**
- * Called during MQDISC, as the inverse of the Authenticate. If the OAM
+ * Called when the authorization service is terminated.
- * has allocated private storage to hold additional information about
- * the user, then this is the time to free it. No more calls will be made
- * to the authorization service for this connection instance of this user.
  */
-static void MQENTRY mqhtpass_term_auth(
+static void MQENTRY mqhtpass_terminate(
     MQHCONFIG hc,
     MQLONG Options,
     PMQCHAR pQMgrName,
@@ -260,7 +271,7 @@ static void MQENTRY mqhtpass_term_auth(
     PMQLONG pCompCode,
     PMQLONG pReason)
 {
-  log_debugf("mqhtpass_term_auth()");
+  log_infof("Terminating %s", NAME);
   if (Options == MQZTO_PRIMARY)
   {
     log_close();

cmd/runmqserver/logging.go

@@ -100,6 +100,11 @@ func mirrorQueueManagerErrorLogs(ctx context.Context, wg *sync.WaitGroup, name s
 	return mirrorLog(ctx, wg, f, fromStart, mf, true)
 }
 
+// mirrorHTPasswdLogs starts a goroutine to mirror the contents of the MQ HTPasswd authorization service's log
+func mirrorHTPasswdLogs(ctx context.Context, wg *sync.WaitGroup, name string, fromStart bool, mf mirrorFunc) (chan error, error) {
+	return mirrorLog(ctx, wg, "/var/mqm/errors/mqhtpass.json", false, mf, true)
+}
+
 func getDebug() bool {
 	debug := os.Getenv("DEBUG")
 	if debug == "true" || debug == "1" {
@@ -124,7 +129,7 @@ func configureLogger(name string) (mirrorFunc, error) {
 				return false
 			}
 			if err != nil {
-				log.Printf("Failed to unmarshall JSON - %v", msg)
+				log.Printf("Failed to unmarshall JSON in log message - %v", msg)
 			} else {
 				fmt.Println(msg)
 			}
@@ -142,7 +147,7 @@ func configureLogger(name string) (mirrorFunc, error) {
 				return false
 			}
 			if err != nil {
-				log.Printf("Failed to unmarshall JSON - %v", err)
+				log.Printf("Failed to unmarshall JSON in log message - %v", err)
 			} else {
 				fmt.Printf(formatBasic(obj))
 				// fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string)))

cmd/runmqserver/main.go

@@ -188,6 +188,13 @@ func doMain() error {
 		logTermination(err)
 		return err
 	}
+	if *devFlag {
+		_, err = mirrorHTPasswdLogs(ctx, &wg, name, newQM, mf)
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
 	err = updateCommandLevel()
 	if err != nil {
 		logTermination(err)

docs/pluggable-connauth.md

@@ -24,6 +24,6 @@ Use an administrative tool or your application to connect to queue manager using
 
 #### Troubleshooting
 
-A log file named `amqpasdev.log` is generated under `/var/mqm/errors` directory path of the container.  This file will contain all the failed connection authentication requests.
+A log file named `mqhtpass.log` is generated under `/var/mqm/errors` directory path of the container.  This file will contain all the failed connection authentication requests.  Additional information is logged to this file if the environment variable `DEBUG` is set to `true`.
 
-**Please note**: This log file is based on circular logging and the maximum size is restricted to 1MB.
+**Please note**: This log file will be wiped when the queue manager is next started.

docs/security.md

@@ -18,18 +18,3 @@ docker run \
   --detach \
   ibm-mqadvanced-server:9.2.1.0-amd64
 ```
-
-The MQ Advanced for Developers image does require the "chown", "setuid", "setgid" and "audit_write" capabilities (plus "dac_override" if you're using an image based on Red Hat Enterprise Linux).  This is because it uses the "sudo" command to change passwords inside the container.  For example, in Docker, you could do the following:
-
-```sh
-docker run \
-  --cap-drop=ALL \
-  --cap-add=CHOWN \
-  --cap-add=SETUID \
-  --cap-add=SETGID \
-  --cap-add=AUDIT_WRITE \
-  --env LICENSE=accept \
-  --env MQ_QMGR_NAME=QM1 \
-  --detach \
-  ibm-mqadvanced-server-dev:9.2.1.0-amd64
-```

incubating/mqadvanced-server-dev/install-extra-packages.sh

@@ -36,6 +36,6 @@ if ($YUM); then
 fi
 
 if ($MICRODNF); then
-    microdnf install apr-util-openssl
+    microdnf --disableplugin=subscription-manager install apr-util-openssl
-    microdnf clean all
+    microdnf --disableplugin=subscription-manager clean all
 fi

install-mq-server-prereqs.sh

@@ -66,7 +66,7 @@ if ($RPM); then
   EXTRA_RPMS="bash bc ca-certificates file findutils gawk glibc-common grep ncurses-compat-libs passwd procps-ng sed shadow-utils tar util-linux which"
   # Install additional packages required by MQ, this install process and the runtime scripts
   $YUM && yum -y install --setopt install_weak_deps=false ${EXTRA_RPMS}
-  $MICRODNF && microdnf install ${EXTRA_RPMS}
+  $MICRODNF && microdnf --disableplugin=subscription-manager install ${EXTRA_RPMS}
 fi
 
 # Apply any bug fixes not included in base Ubuntu or MQ image.
@@ -78,4 +78,4 @@ $UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libu
 $UBUNTU && rm -rf /var/lib/apt/lists/*
 $YUM && yum -y clean all
 $YUM && rm -rf /var/cache/yum/*
-$MICRODNF && microdnf clean all
+$MICRODNF && microdnf --disableplugin=subscription-manager clean all