commit 88a9e485e3709a77867542f5e9a5f7ac60238f49
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Nov 29 22:18:18 2017 +0000

    TLS: Fix excessive calling of smtp_auth_acl under AUTH_TLS.  Bug 2203

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 0aabc535..76c72de9 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -187,7 +187,7 @@ static smtp_cmd_list cmd_list[] = {
   { "auth",       sizeof("auth")-1,       AUTH_CMD, TRUE,  TRUE  },
   #ifdef SUPPORT_TLS
   { "starttls",   sizeof("starttls")-1,   STARTTLS_CMD, FALSE, FALSE },
-  { "tls_auth",   0,                      TLS_AUTH_CMD, FALSE, TRUE },
+  { "tls_auth",   0,                      TLS_AUTH_CMD, FALSE, FALSE },
   #endif
 
 /* If you change anything above here, also fix the definitions below. */
@@ -2826,8 +2826,12 @@ if (check_proxy_protocol_host())
   smtps port for use with older style SSL MTAs. */
 
 #ifdef SUPPORT_TLS
-  if (tls_in.on_connect && tls_server_start(tls_require_ciphers, &user_msg) != OK)
-    return smtp_log_tls_fail(user_msg);
+  if (tls_in.on_connect)
+    {
+    if (tls_server_start(tls_require_ciphers, &user_msg) != OK)
+      return smtp_log_tls_fail(user_msg);
+    cmd_list[CMD_LIST_TLS_AUTH].is_mail_cmd = TRUE;
+    }
 #endif
 
 /* Run the connect ACL if it exists */
@@ -3800,7 +3804,6 @@ cmd_list[CMD_LIST_HELO].is_mail_cmd = TRUE;
 cmd_list[CMD_LIST_EHLO].is_mail_cmd = TRUE;
 #ifdef SUPPORT_TLS
 cmd_list[CMD_LIST_STARTTLS].is_mail_cmd = TRUE;
-cmd_list[CMD_LIST_TLS_AUTH].is_mail_cmd = TRUE;
 #endif
 
 /* Set the local signal handler for SIGTERM - it tries to end off tidily */

commit 2d9c1e31d619a61ca09b30a3abfe73ddfed67ce5
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Nov 29 23:22:34 2017 +0000

    TLS: avoid calling smtp_auth_acl on client cert when no tls authenticator is configured

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 76c72de9..00e9d41a 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -3849,24 +3849,24 @@ while (done <= 0)
      )
     {
     cmd_list[CMD_LIST_TLS_AUTH].is_mail_cmd = FALSE;
-    if (  acl_smtp_auth
-       && (rc = acl_check(ACL_WHERE_AUTH, NULL, acl_smtp_auth,
-		  &user_msg, &log_msg)) != OK
-       )
-      {
-      done = smtp_handle_acl_fail(ACL_WHERE_AUTH, rc, user_msg, log_msg);
-      continue;
-      }
 
     for (au = auths; au; au = au->next)
       if (strcmpic(US"tls", au->driver_name) == 0)
 	{
-	smtp_cmd_data = NULL;
-
-	if (smtp_in_auth(au, &s, &ss) == OK)
-	  { DEBUG(D_auth) debug_printf("tls auth succeeded\n"); }
+	if (  acl_smtp_auth
+	   && (rc = acl_check(ACL_WHERE_AUTH, NULL, acl_smtp_auth,
+		      &user_msg, &log_msg)) != OK
+	   )
+	  done = smtp_handle_acl_fail(ACL_WHERE_AUTH, rc, user_msg, log_msg);
 	else
-	  { DEBUG(D_auth) debug_printf("tls auth not succeeded\n"); }
+	  {
+	  smtp_cmd_data = NULL;
+
+	  if (smtp_in_auth(au, &s, &ss) == OK)
+	    { DEBUG(D_auth) debug_printf("tls auth succeeded\n"); }
+	  else
+	    { DEBUG(D_auth) debug_printf("tls auth not succeeded\n"); }
+	  }
 	break;
 	}
     }

commit f9d9829cee89e77043113209b0fb619d66274bed
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Fri Dec 1 22:43:19 2017 +0000

    Debug: fix coding in dnssec reporting.  Bug 2205

diff --git a/src/src/host.c b/src/src/host.c
index 05bde3fb..1f0d9195 100644
--- a/src/src/host.c
+++ b/src/src/host.c
@@ -2612,8 +2612,8 @@ if ((whichrrs & HOST_FIND_BY_SRV) != 0)
 
   DEBUG(D_dns)
     if ((dnssec_request || dnssec_require)
-	& !dns_is_secure(&dnsa)
-	& dns_is_aa(&dnsa))
+	&& !dns_is_secure(&dnsa)
+	&& dns_is_aa(&dnsa))
       debug_printf("DNS lookup of %.256s (SRV) requested AD, but got AA\n", host->name);
 
   if (dnssec_request)

commit c73a4d073e195c11bb5b03b91c61478ab6935593
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Sun Dec 3 18:17:43 2017 +0100

    DKIM: Ignore non-DKIM TXT records in DNS response. Bug 2207

diff --git a/src/src/dkim.c b/src/src/dkim.c
index 5e97c1b7..9731a63d 100644
--- a/src/src/dkim.c
+++ b/src/src/dkim.c
@@ -73,6 +73,9 @@ for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS);
       if (answer_offset >= PDKIM_DNS_TXT_MAX_RECLEN)
 	return PDKIM_FAIL;	/*XXX better error detail?  logging? */
       }
+
+    /* check if this looks like a DKIM record */
+    if (strncasecmp(answer, "v=dkim", 6) != 0) continue;
     return PDKIM_OK;
     }
 
@@ -148,7 +151,7 @@ if (!(s = sig->domain)) s = US"<UNSET>";
 logmsg = string_append(logmsg, 2, "d=", s);
 if (!(s = sig->selector)) s = US"<UNSET>";
 logmsg = string_append(logmsg, 2, " s=", s);
-logmsg = string_append(logmsg, 7, 
+logmsg = string_append(logmsg, 7,
 " c=", sig->canon_headers == PDKIM_CANON_SIMPLE ? "simple" : "relaxed",
 "/",   sig->canon_body    == PDKIM_CANON_SIMPLE ? "simple" : "relaxed",
 " a=", dkim_sig_to_a_tag(sig),
@@ -371,7 +374,7 @@ for (sig = dkim_signatures; sig; sig = sig->next)
 
     dkim_verify_status = dkim_exim_expand_query(DKIM_VERIFY_STATUS);
     dkim_verify_reason = dkim_exim_expand_query(DKIM_VERIFY_REASON);
-    
+
     if ((rc = dkim_acl_call(id, res_ptr, user_msgptr, log_msgptr)) != OK)
       return rc;
     }

commit c1f0ee1044f2d39648ffdd96194566020d3d164e
Author: Andreas Piesk <a.piesk@mailbox.org>
Date:   Fri Dec 22 10:05:02 2017 +0000

    Fix build of nisplus lookup

diff --git a/src/src/lookups/nisplus.c b/src/src/lookups/nisplus.c
index e4184bbd..7cd00eed 100644
--- a/src/src/lookups/nisplus.c
+++ b/src/src/lookups/nisplus.c
@@ -54,7 +54,6 @@ nis_object *tno, *eno;
 struct entry_obj *eo;
 struct table_obj *ta;
 uschar *p = query + length;
-uschar *yield = NULL;
 gstring * yield = NULL;
 
 do_cache = do_cache;   /* Placate picky compilers */
@@ -173,7 +172,7 @@ for (i = 0; i < eo->en_cols.en_cols_len; i++)
       yield = string_catn(yield, US"\"", 1);
       }
     else
-    eyield = string_catn(yield, value, len);
+      yield = string_catn(yield, value, len);
 
     yield = string_catn(yield, US" ", 1);
     }

commit 032e3aa3d12b3c939c8602ff801847a8305bd849
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Fri Dec 22 10:25:56 2017 +0000

    Fix const issue in nisplus lookup

diff --git a/src/src/lookups/nisplus.c b/src/src/lookups/nisplus.c
index 7cd00eed..1f226d52 100644
--- a/src/src/lookups/nisplus.c
+++ b/src/src/lookups/nisplus.c
@@ -42,18 +42,18 @@ yield is the concatenation of all the fields, preceded by their names and an
 equals sign. */
 
 static int
-nisplus_find(void *handle, uschar *filename, uschar *query, int length,
+nisplus_find(void *handle, uschar *filename, const uschar *query, int length,
   uschar **result, uschar **errmsg, uint *do_cache)
 {
 int i;
 int error_error = FAIL;
-uschar *field_name = NULL;
+const uschar * field_name = NULL;
 nis_result *nrt = NULL;
 nis_result *nre = NULL;
 nis_object *tno, *eno;
 struct entry_obj *eo;
 struct table_obj *ta;
-uschar *p = query + length;
+const uschar * p = query + length;
 gstring * yield = NULL;
 
 do_cache = do_cache;   /* Placate picky compilers */
@@ -63,12 +63,15 @@ has been given. */
 
 while (p > query && p[-1] != ':') p--;
 
-if (p > query)
+if (p > query)		/* get the query without the result-field */
   {
+  uint len = p-1 - query;
   field_name = p;
-  p[-1] = 0;
+  query = string_copyn(query, len);
+  p = query + len;
   }
-else p = query + length;
+else
+  p = query + length;
 
 /* Now search backwards to find the comma that starts the
 table name. */
@@ -100,7 +103,7 @@ if (tno->zo_data.zo_type != TABLE_OBJ)
   *errmsg = string_sprintf("NIS+ error: %s is not a table", p);
   goto NISPLUS_EXIT;
   }
-ta = &(tno->zo_data.objdata_u.ta_data);
+ta = &tno->zo_data.objdata_u.ta_data;
 
 /* Now look up the entry in the table, check that we got precisely one
 object and that it is a table entry. */
@@ -152,7 +155,7 @@ for (i = 0; i < eo->en_cols.en_cols_len; i++)
 
   /* Concatenate all fields if no specific one selected */
 
-  if (field_name == NULL)
+  if (!field_name)
     {
     yield = string_cat (yield, tc->tc_name);
     yield = string_catn(yield, US"=", 1);
@@ -195,11 +198,9 @@ if (field_name)
 else
   store_reset(yield->s + yield->ptr + 1);
 
-/* Restore the colon in the query, and free result store before
-finishing. */
+/* Free result store before finishing. */
 
 NISPLUS_EXIT:
-if (field_name) field_name[-1] = ':';
 if (nrt) nis_freeresult(nrt);
 if (nre) nis_freeresult(nre);
 

commit 8143ff45b2b9edd3532fe4056d5736ecc4e0a766
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Dec 24 21:30:20 2017 +0000

    Lookups: fix pgsql multiple-row, single-column return
    
    Report & fix from James <list@xdrv.co.uk>; additional tidying and testcase by JGH
    
    Broken-by: acec9514b1

diff --git a/src/src/lookups/pgsql.c b/src/src/lookups/pgsql.c
index 0b771f59..cece43b6 100644
--- a/src/src/lookups/pgsql.c
+++ b/src/src/lookups/pgsql.c
@@ -140,7 +140,7 @@ has the password removed. This copy is also used for debugging output. */
 for (i = 2; i >= 0; i--)
   {
   uschar *pp = Ustrrchr(server, '/');
-  if (pp == NULL)
+  if (!pp)
     {
     *errmsg = string_sprintf("incomplete pgSQL server data: %s",
       (i == 2)? server : server_copy);
@@ -156,18 +156,16 @@ for (i = 2; i >= 0; i--)
 start is the identification of the server (host or path). See if we have a
 cached connection to the server. */
 
-for (cn = pgsql_connections; cn != NULL; cn = cn->next)
-  {
+for (cn = pgsql_connections; cn; cn = cn->next)
   if (Ustrcmp(cn->server, server_copy) == 0)
     {
     pg_conn = cn->handle;
     break;
     }
-  }
 
 /* If there is no cached connection, we must set one up. */
 
-if (cn == NULL)
+if (!cn)
   {
   uschar *port = US"";
 
@@ -178,7 +176,7 @@ if (cn == NULL)
     uschar *last_slash, *last_dot, *p;
 
     p = ++server;
-    while (*p != 0 && *p != ')') p++;
+    while (*p && *p != ')') p++;
     *p = 0;
 
     last_slash = Ustrrchr(server, '/');
@@ -191,10 +189,9 @@ if (cn == NULL)
     We have to call PQsetdbLogin with '/var/run/postgresql' as the hostname
     argument and put '5432' into the port variable. */
 
-    if (last_slash == NULL || last_dot == NULL)
+    if (!last_slash || !last_dot)
       {
-      *errmsg = string_sprintf("PGSQL invalid filename for socket: %s",
-        server);
+      *errmsg = string_sprintf("PGSQL invalid filename for socket: %s", server);
       *defer_break = TRUE;
       return DEFER;
       }
@@ -211,13 +208,13 @@ if (cn == NULL)
   else
     {
     uschar *p;
-    if ((p = Ustrchr(server, ':')) != NULL)
+    if ((p = Ustrchr(server, ':')))
       {
       *p++ = 0;
       port = p;
       }
 
-    if (Ustrchr(server, '/') != NULL)
+    if (Ustrchr(server, '/'))
       {
       *errmsg = string_sprintf("unexpected slash in pgSQL server hostname: %s",
         server);
@@ -280,37 +277,37 @@ else
 
 /* Run the query */
 
-  pg_result = PQexec(pg_conn, CS query);
-  switch(PQresultStatus(pg_result))
-    {
-    case PGRES_EMPTY_QUERY:
-    case PGRES_COMMAND_OK:
-      /* The command was successful but did not return any data since it was
-      not SELECT but either an INSERT, UPDATE or DELETE statement. Tell the
-      high level code to not cache this query, and clean the current cache for
-      this handle by setting *do_cache zero. */
-
-      result = string_cat(result, US PQcmdTuples(pg_result));
-      *do_cache = 0;
-      DEBUG(D_lookup) debug_printf("PGSQL: command does not return any data "
-	"but was successful. Rows affected: %s\n", result->s);
-      break;
+pg_result = PQexec(pg_conn, CS query);
+switch(PQresultStatus(pg_result))
+  {
+  case PGRES_EMPTY_QUERY:
+  case PGRES_COMMAND_OK:
+    /* The command was successful but did not return any data since it was
+    not SELECT but either an INSERT, UPDATE or DELETE statement. Tell the
+    high level code to not cache this query, and clean the current cache for
+    this handle by setting *do_cache zero. */
+
+    result = string_cat(result, US PQcmdTuples(pg_result));
+    *do_cache = 0;
+    DEBUG(D_lookup) debug_printf("PGSQL: command does not return any data "
+      "but was successful. Rows affected: %s\n", string_from_gstring(result));
+    break;
 
-    case PGRES_TUPLES_OK:
-      break;
+  case PGRES_TUPLES_OK:
+    break;
 
-    default:
-      /* This was the original code:
-      *errmsg = string_sprintf("PGSQL: query failed: %s\n",
-			       PQresultErrorMessage(pg_result));
-      This was suggested by a user:
-      */
-
-      *errmsg = string_sprintf("PGSQL: query failed: %s (%s) (%s)\n",
-                             PQresultErrorMessage(pg_result),
-                             PQresStatus(PQresultStatus(pg_result)), query);
-      goto PGSQL_EXIT;
-    }
+  default:
+    /* This was the original code:
+    *errmsg = string_sprintf("PGSQL: query failed: %s\n",
+			     PQresultErrorMessage(pg_result));
+    This was suggested by a user:
+    */
+
+    *errmsg = string_sprintf("PGSQL: query failed: %s (%s) (%s)\n",
+			   PQresultErrorMessage(pg_result),
+			   PQresStatus(PQresultStatus(pg_result)), query);
+    goto PGSQL_EXIT;
+  }
 
 /* Result is in pg_result. Find the number of fields returned. If this is one,
 we don't add field names to the data. Otherwise we do. If the query did not
@@ -329,7 +326,7 @@ for (i = 0; i < num_tuples; i++)
     result = string_catn(result, US"\n", 1);
 
   if (num_fields == 1)
-    result = string_catn(NULL,
+    result = string_catn(result,
 	US PQgetvalue(pg_result, i, 0), PQgetlength(pg_result, i, 0));
   else
     {
@@ -342,17 +339,13 @@ for (i = 0; i < num_tuples; i++)
     }
   }
 
-/* If result is NULL then no data has been found and so we return FAIL.
-Otherwise, we must terminate the string which has been built; string_cat()
-always leaves enough room for a terminating zero. */
+/* If result is NULL then no data has been found and so we return FAIL. */
 
 if (!result)
   {
   yield = FAIL;
   *errmsg = US"PGSQL: no data found";
   }
-else
-  store_reset(result->s + result->ptr + 1);
 
 /* Get here by goto from various error checks. */
 
@@ -367,6 +360,7 @@ if (pg_result) PQclear(pg_result);
 
 if (result)
   {
+  store_reset(result->s + result->ptr + 1);
   *resultptr = string_from_gstring(result);
   return OK;
   }

commit db98649c6f4fc9ae797effb07302cb4f2dd48690
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Dec 28 20:51:28 2017 +0000

    DKIM: tighter checking while parsing signature headers.  Bug 2217

diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c
index 20366a46..b884671d 100644
--- a/src/src/pdkim/pdkim.c
+++ b/src/src/pdkim/pdkim.c
@@ -490,7 +490,12 @@ for (p = raw_hdr; ; p++)
 
     if (c == ';' || c == '\0')
       {
-      if (cur_tag && cur_val)
+      /* We must have both tag and value, and tags must be one char except
+      for the possibility of "bh". */
+
+      if (  cur_tag && cur_val
+	 && (cur_tag->ptr == 1 || *cur_tag->s == 'b')
+	 )
         {
 	(void) string_from_gstring(cur_val);
 	pdkim_strtrim(cur_val);
@@ -500,8 +505,14 @@ for (p = raw_hdr; ; p++)
 	switch (*cur_tag->s)
 	  {
 	  case 'b':
-	    pdkim_decode_base64(cur_val->s,
-			    cur_tag->s[1] == 'h' ? &sig->bodyhash : &sig->sighash);
+	    switch (cur_tag->s[1])
+	      {
+	      case '\0': pdkim_decode_base64(cur_val->s, &sig->sighash); break;
+	      case 'h':  if (cur_tag->ptr == 2)
+			   pdkim_decode_base64(cur_val->s, &sig->bodyhash);
+			 break;
+	      default:   break;
+	      }
 	    break;
 	  case 'v':
 	      /* We only support version 1, and that is currently the

commit 4c35563b96a3f37ea7d215c47f7725d24c1e4a1f
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Dec 28 20:09:05 2017 +0000

    Fix crash associated with dnsdb lookup done from DKIM ACL.  Bug 2215
    
    Broken-by: cc55f4208e

diff --git a/src/src/expand.c b/src/src/expand.c
index e754fbc8..3a63a7c7 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -7546,7 +7546,7 @@ DEBUG(D_expand)
   if (expand_string_forcedfail)
     debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n");
   }
-if (resetok_p) *resetok_p = resetok;
+if (resetok_p && !resetok) *resetok_p = FALSE;
 expand_level--;
 return NULL;
 }
@@ -7560,28 +7560,35 @@ Returns:  the expanded string, or NULL if expansion failed; if failure was
           due to a lookup deferring, search_find_defer will be TRUE
 */
 
-uschar *
-expand_string(uschar *string)
+const uschar *
+expand_cstring(const uschar * string)
 {
-search_find_defer = FALSE;
-malformed_header = FALSE;
-return (Ustrpbrk(string, "$\\") == NULL)? string :
-  expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
+if (Ustrpbrk(string, "$\\") != NULL)
+  {
+  int old_pool = store_pool;
+  uschar * s;
+
+  search_find_defer = FALSE;
+  malformed_header = FALSE;
+  store_pool = POOL_MAIN;
+    s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
+  store_pool = old_pool;
+  return s;
+  }
+return string;
 }
 
 
-
-const uschar *
-expand_cstring(const uschar *string)
+uschar *
+expand_string(uschar * string)
 {
-search_find_defer = FALSE;
-malformed_header = FALSE;
-return (Ustrpbrk(string, "$\\") == NULL)? string :
-  expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
+return US expand_cstring(CUS string);
 }
 
 
 
+
+
 /*************************************************
 *              Expand and copy                   *
 *************************************************/
@@ -7812,8 +7819,6 @@ return (  (  Ustrstr(s, "failed to expand") != NULL
 * Error-checking for testsuite                   *
 *************************************************/
 typedef struct {
-  const char *	filename;
-  int		linenumber;
   uschar * 	region_start;
   uschar *	region_end;
   const uschar *var_name;
@@ -7834,7 +7839,8 @@ if (var_data >= e->region_start  &&  var_data < e->region_end)
 void
 assert_no_variables(void * ptr, int len, const char * filename, int linenumber)
 {
-err_ctx e = {filename, linenumber, ptr, US ptr + len, NULL };
+err_ctx e = { .region_start = ptr, .region_end = US ptr + len,
+	      .var_name = NULL, .var_data = NULL };
 int i;
 var_entry * v;
 
@@ -7855,10 +7861,16 @@ for (v = var_table; v < var_table + var_table_size; v++)
   if (v->type == vtype_stringptr)
     assert_variable_notin(US v->name, *(USS v->value), &e);
 
+/* check dns and address trees */
+tree_walk(tree_dns_fails,     assert_variable_notin, &e);
+tree_walk(tree_duplicates,    assert_variable_notin, &e);
+tree_walk(tree_nonrecipients, assert_variable_notin, &e);
+tree_walk(tree_unusable,      assert_variable_notin, &e);
+
 if (e.var_name)
   log_write(0, LOG_MAIN|LOG_PANIC_DIE,
     "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'",
-    e.var_name, e.filename, e.linenumber, e.var_data);
+    e.var_name, filename, linenumber, e.var_data);
 }
 
 
diff --git a/src/src/queue.c b/src/src/queue.c
index f9468119..09a149e9 100644
--- a/src/src/queue.c
+++ b/src/src/queue.c
@@ -699,6 +699,7 @@ for (i = queue_run_in_order ? -1 : 0;
       }
     }                                  /* End loop for list of messages */
 
+  tree_nonrecipients = NULL;
   store_reset(reset_point1);           /* Scavenge list of messages */
 
   /* If this was the first time through for random order processing, and

commit 70ba304da5933bcf3a9afb040228b2a37b3beefe
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Dec 27 23:32:02 2017 +0000

    Fix issue with continued-connections when the DNS shifts unreliably

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 61e8d8a4..5d351dd7 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -3924,7 +3924,9 @@ for (cutoff_retry = 0;
   {
   host_item *nexthost = NULL;
   int unexpired_hosts_tried = 0;
+  BOOL continue_host_tried = FALSE;
 
+retry_non_continued:
   for (host = hostlist;
           host
        && unexpired_hosts_tried < ob->hosts_max_try
@@ -4057,14 +4059,16 @@ for (cutoff_retry = 0;
     result of the lookup. Set expired FALSE, to save the outer loop executing
     twice. */
 
-    if (  continue_hostname
-       && (  Ustrcmp(continue_hostname, host->name) != 0
-          || Ustrcmp(continue_host_address, host->address) != 0
-       )  )
-      {
-      expired = FALSE;
-      continue;      /* With next host */
-      }
+    if (continue_hostname)
+      if (  Ustrcmp(continue_hostname, host->name) != 0
+         || Ustrcmp(continue_host_address, host->address) != 0
+	 )
+	{
+	expired = FALSE;
+	continue;      /* With next host */
+	}
+      else
+	continue_host_tried = TRUE;
 
     /* Reset the default next host in case a multihomed host whose addresses
     are not looked up till just above added to the host list. */
@@ -4522,6 +4526,32 @@ for (cutoff_retry = 0;
       }
     }   /* End of loop for trying multiple hosts. */
 
+  /* If we failed to find a matching host in the list, for an already-open
+  connection, just close it and start over with the list.  This can happen
+  for routing that changes from run to run, or big multi-IP sites with
+  round-robin DNS. */
+
+  if (continue_hostname && !continue_host_tried)
+    {
+    int fd = cutthrough.fd >= 0 ? cutthrough.fd : 0;
+
+    DEBUG(D_transport) debug_printf("no hosts match already-open connection\n");
+#ifdef SUPPORT_TLS
+    if (tls_out.active == fd)
+      {
+      (void) tls_write(FALSE, US"QUIT\r\n", 6, FALSE);
+      tls_close(FALSE, TRUE);
+      }
+    else
+#else
+      (void) write(fd, US"QUIT\r\n", 6);
+#endif
+    (void) close(fd);
+    cutthrough.fd = -1;
+    continue_hostname = NULL;
+    goto retry_non_continued;
+    }
+
   /* This is the end of the loop that repeats iff expired is TRUE and
   ob->delay_after_cutoff is FALSE. The second time round we will
   try those hosts that haven't been tried since the message arrived. */

commit 13ca5f2522f4165a0aa6f1b7143a1a1cfa37bf5a
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Dec 30 13:55:54 2017 +0000

    MIME ACL: fix SMTP response for non-accept result of the ACL.  Bug 2214.
    
    As far as I can see this was broken back in 2013, f4c1088 for 4.82

diff --git a/src/src/receive.c b/src/src/receive.c
index 1f1954c5..3a550cfe 100644
--- a/src/src/receive.c
+++ b/src/src/receive.c
@@ -1348,7 +1348,7 @@ run_mime_acl(uschar *acl, BOOL *smtp_yield_ptr, uschar **smtp_reply_ptr,
   uschar **blackholed_by_ptr)
 {
 FILE *mbox_file;
-uschar rfc822_file_path[2048];
+uschar * rfc822_file_path = NULL;
 unsigned long mbox_size;
 header_line *my_headerlist;
 uschar *user_msg, *log_msg;
@@ -1356,8 +1356,6 @@ int mime_part_count_buffer = -1;
 uschar * mbox_filename;
 int rc = OK;
 
-memset(CS rfc822_file_path,0,2048);
-
 /* check if it is a MIME message */
 
 for (my_headerlist = header_list; my_headerlist; my_headerlist = my_headerlist->next)
@@ -1397,7 +1395,7 @@ mime_part_count = -1;
 rc = mime_acl_check(acl, mbox_file, NULL, &user_msg, &log_msg);
 (void)fclose(mbox_file);
 
-if (Ustrlen(rfc822_file_path) > 0)
+if (rfc822_file_path)
   {
   mime_part_count = mime_part_count_buffer;
 
@@ -1405,36 +1403,31 @@ if (Ustrlen(rfc822_file_path) > 0)
     {
     log_write(0, LOG_PANIC,
          "acl_smtp_mime: can't unlink RFC822 spool file, skipping.");
-      goto END_MIME_ACL;
+    goto END_MIME_ACL;
     }
+  rfc822_file_path = NULL;
   }
 
 /* check if we must check any message/rfc822 attachments */
 if (rc == OK)
   {
-  uschar * scandir;
+  uschar * scandir = string_copyn(mbox_filename,
+	      Ustrrchr(mbox_filename, '/') - mbox_filename);
   struct dirent * entry;
   DIR * tempdir;
 
-  scandir = string_copyn(mbox_filename, Ustrrchr(mbox_filename, '/') - mbox_filename);
-
-  tempdir = opendir(CS scandir);
-  for (;;)
-    {
-    if (!(entry = readdir(tempdir)))
-      break;
+  for (tempdir = opendir(CS scandir); entry = readdir(tempdir); )
     if (strncmpic(US entry->d_name, US"__rfc822_", 9) == 0)
       {
-      (void) string_format(rfc822_file_path, sizeof(rfc822_file_path),
-	"%s/%s", scandir, entry->d_name);
-      DEBUG(D_receive) debug_printf("RFC822 attachment detected: running MIME ACL for '%s'\n",
-	rfc822_file_path);
+      rfc822_file_path = string_sprintf("%s/%s", scandir, entry->d_name);
+      DEBUG(D_receive)
+	debug_printf("RFC822 attachment detected: running MIME ACL for '%s'\n",
+	  rfc822_file_path);
       break;
       }
-    }
   closedir(tempdir);
 
-  if (entry)
+  if (rfc822_file_path)
     {
     if ((mbox_file = Ufopen(rfc822_file_path, "rb")))
       {
@@ -1463,10 +1456,10 @@ else if (rc != OK)
 #ifdef EXPERIMENTAL_DCC
   dcc_ok = 0;
 #endif
-  if (  smtp_input
-     && smtp_handle_acl_fail(ACL_WHERE_MIME, rc, user_msg, log_msg) != 0)
+  if (smtp_input)
     {
-    *smtp_yield_ptr = FALSE;    /* No more messages after dropped connection */
+    if (smtp_handle_acl_fail(ACL_WHERE_MIME, rc, user_msg, log_msg) != 0)
+      *smtp_yield_ptr = FALSE;  /* No more messages after dropped connection */
     *smtp_reply_ptr = US"";     /* Indicate reply already sent */
     }
   message_id[0] = 0;            /* Indicate no message accepted */
@@ -3481,9 +3474,10 @@ else
 #endif /* DISABLE_DKIM */
 
 #ifdef WITH_CONTENT_SCAN
-    if (recipients_count > 0 &&
-        acl_smtp_mime != NULL &&
-        !run_mime_acl(acl_smtp_mime, &smtp_yield, &smtp_reply, &blackholed_by))
+    if (  recipients_count > 0
+       && acl_smtp_mime
+       && !run_mime_acl(acl_smtp_mime, &smtp_yield, &smtp_reply, &blackholed_by)
+       )
       goto TIDYUP;
 #endif /* WITH_CONTENT_SCAN */
 
@@ -3603,9 +3597,10 @@ else
     {
 
 #ifdef WITH_CONTENT_SCAN
-    if (acl_not_smtp_mime != NULL &&
-        !run_mime_acl(acl_not_smtp_mime, &smtp_yield, &smtp_reply,
-          &blackholed_by))
+    if (  acl_not_smtp_mime
+       && !run_mime_acl(acl_not_smtp_mime, &smtp_yield, &smtp_reply,
+          &blackholed_by)
+       )
       goto TIDYUP;
 #endif /* WITH_CONTENT_SCAN */
 
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 00e9d41a..2603da25 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -3142,7 +3142,7 @@ return;
 
 /* This function is called when acl_check() fails. As well as calls from within
 this module, it is called from receive.c for an ACL after DATA. It sorts out
-logging the incident, and sets up the error response. A message containing
+logging the incident, and sends the error response. A message containing
 newlines is turned into a multiline SMTP response, but for logging, only the
 first line is used.
 

commit 20913a313f33fd8ae2dea9f379552975867c6394
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Jan 7 15:03:25 2018 +0000

    DKIM: permit dkim_private_key to override dkim_strict on signing.  Bug 2220

diff --git a/src/src/dkim.c b/src/src/dkim.c
index 9731a63d..18427fe9 100644
--- a/src/src/dkim.c
+++ b/src/src/dkim.c
@@ -531,8 +531,12 @@ switch (what)
 }
 
 
-/* Generate signatures for the given file, returning a string.
+/* Generate signatures for the given file.
 If a prefix is given, prepend it to the file for the calculations.
+
+Return:
+  NULL:		error; error string written
+  string: 	signature header(s), or a zero-length string (not an error)
 */
 
 gstring *
@@ -702,9 +706,15 @@ while ((dkim_signing_domain = string_nextinlist(&dkim_domain, &sep, NULL, 0)))
       }
     }
   }
+if (!ctx.sig)
+  {
+  DEBUG(D_transport) debug_printf("DKIM: no viable signatures to use\n");
+  sigbuf = string_get(1);	/* return a zero-len string */
+  goto CLEANUP;
+  }
 
-if (prefix)
-  pdkim_feed(&ctx, prefix, Ustrlen(prefix));
+if (prefix && (pdkim_feed(&ctx, prefix, Ustrlen(prefix))) != PDKIM_OK)
+  goto pk_bad;
 
 if (lseek(fd, off, SEEK_SET) < 0)
   sread = -1;
@@ -729,9 +739,8 @@ if ((pdkim_rc = pdkim_feed_finish(&ctx, &sig, errstr)) != PDKIM_OK)
 for (sigbuf = NULL; sig; sig = sig->next)
   sigbuf = string_append(sigbuf, 2, US sig->signature_header, US"\r\n");
 
-(void) string_from_gstring(sigbuf);
-
 CLEANUP:
+  (void) string_from_gstring(sigbuf);
   store_pool = old_pool;
   errno = save_errno;
   return sigbuf;

commit 0a4484f261a51cfe0fc6ece78f47f9507a34e2ba
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Jan 13 18:11:21 2018 +0000

    Lookups: fix mysql lookup returns for no-data "queries",
    when the number of rows affected is returned.  Bug 2223
    
    Broken-by: acec9514b1
    
    Also enhance the testsuite mysql testcase to be standalone and move to standard-run set
    and add a specific testcase for this bug.
    
    Testcase working on Fedora at least - we'll see what happens on other platforms
    where executable locaation may vary.

diff --git a/src/src/lookups/mysql.c b/src/src/lookups/mysql.c
index ff1ef832..8d101b27 100644
--- a/src/src/lookups/mysql.c
+++ b/src/src/lookups/mysql.c
@@ -281,7 +281,7 @@ we return the number of rows affected by the command. In this event, we do NOT
 want to cache the result; also the whole cache for the handle must be cleaned
 up. Setting do_cache zero requests this. */
 
-if ((mysql_result = mysql_use_result(mysql_handle)) == NULL)
+if (!(mysql_result = mysql_use_result(mysql_handle)))
   {
   if ( mysql_field_count(mysql_handle) == 0 )
     {
@@ -314,34 +314,32 @@ while ((mysql_row_data = mysql_fetch_row(mysql_result)))
   if (result)
     result = string_catn(result, US"\n", 1);
 
-  if (num_fields == 1)
-    {
-    if (mysql_row_data[0] != NULL)    /* NULL value yields nothing */
-      {
-      result = string_catn(result, US mysql_row_data[0],
-        lengths[0]);
-      (void) string_from_gstring(result);
-      }
-    }
+  if (num_fields != 1)
+    for (i = 0; i < num_fields; i++)
+      result = lf_quote(US fields[i].name, US mysql_row_data[i], lengths[i],
+			result);
 
-  else for (i = 0; i < num_fields; i++)
-    result = lf_quote(US fields[i].name, US mysql_row_data[i], lengths[i], result);
+  else if (mysql_row_data[0] != NULL)    /* NULL value yields nothing */
+      result = string_catn(result, US mysql_row_data[0], lengths[0]);
   }
 
 /* more results? -1 = no, >0 = error, 0 = yes (keep looping)
    This is needed because of the CLIENT_MULTI_RESULTS on mysql_real_connect(),
    we don't expect any more results. */
 
-while((i = mysql_next_result(mysql_handle)) >= 0) {
-   if(i == 0) {   /* Just ignore more results */
-     DEBUG(D_lookup) debug_printf("MYSQL: got unexpected more results\n");
-     continue;
-   }
+while((i = mysql_next_result(mysql_handle)) >= 0)
+  {
+  if(i == 0)	/* Just ignore more results */
+    {
+    DEBUG(D_lookup) debug_printf("MYSQL: got unexpected more results\n");
+    continue;
+    }
 
-   *errmsg = string_sprintf("MYSQL: lookup result error when checking for more results: %s\n",
-       mysql_error(mysql_handle));
-   goto MYSQL_EXIT;
-}
+  *errmsg = string_sprintf(
+	"MYSQL: lookup result error when checking for more results: %s\n",
+	mysql_error(mysql_handle));
+  goto MYSQL_EXIT;
+  }
 
 /* If result is NULL then no data has been found and so we return FAIL.
 Otherwise, we must terminate the string which has been built; string_cat()
@@ -352,11 +350,6 @@ if (!result)
   yield = FAIL;
   *errmsg = US"MYSQL: no data found";
   }
-else
-  {
-  (void) string_from_gstring(result);
-  store_reset(result->s + result->ptr + 1);
-  }
 
 /* Get here by goto from various error checks and from the case where no data
 was read (e.g. an update query). */
@@ -372,7 +365,8 @@ if (mysql_result) mysql_free_result(mysql_result);
 
 if (result)
   {
-  *resultptr = result->s;
+  *resultptr = string_from_gstring(result);
+  store_reset(result->s + (result->size = result->ptr + 1));
   return OK;
   }
 else

commit d2fe8622a815e36bf66b04eb772d5ec0ba8e13af
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Tue Jan 16 16:06:24 2018 +0100

    Fix %D string expansion to not use millisec
    
    log_selector +millisec should not change the expansion of %D
    (used in log_file_path and maybe other places)

diff --git a/src/src/tod.c b/src/src/tod.c
index 55933dc8..290dc6dd 100644
--- a/src/src/tod.c
+++ b/src/src/tod.c
@@ -54,7 +54,6 @@ tod_stamp(int type)
 {
 struct timeval now;
 struct tm * t;
-int off = 0;
 
 gettimeofday(&now, NULL);
 
@@ -90,9 +89,17 @@ t = timestamps_utc ? gmtime(&now.tv_sec) : localtime(&now.tv_sec);
 switch(type)
   {
   case tod_log_bare:          /* Format used in logging without timezone */
-    off = sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d",
+#ifndef COMPILE_UTILITY
+    if (LOGGING(millisec))
+      sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d.%03d",
+      1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday,
+      t->tm_hour, t->tm_min, t->tm_sec, (int)(now.tv_usec/1000));
+    else
+#endif
+      sprintf(CS timebuf, "%04d-%02d-%02d %02d:%02d:%02d",
       1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday,
       t->tm_hour, t->tm_min, t->tm_sec);
+
     break;
 
     /* Format used as suffix of log file name when 'log_datestamp' is active. For
@@ -101,19 +108,19 @@ switch(type)
 #ifdef TESTING_LOG_DATESTAMP
   case tod_log_datestamp_daily:
   case tod_log_datestamp_monthly:
-    off = sprintf(CS timebuf, "%04d%02d%02d%02d%02d",
+    sprintf(CS timebuf, "%04d%02d%02d%02d%02d",
       1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday, t->tm_hour, t->tm_min);
     break;
 
 #else
   case tod_log_datestamp_daily:
-    off = sprintf(CS timebuf, "%04d%02d%02d",
+    sprintf(CS timebuf, "%04d%02d%02d",
       1900 + t->tm_year, 1 + t->tm_mon, t->tm_mday);
     break;
 
   case tod_log_datestamp_monthly:
 #ifndef COMPILE_UTILITY
-    off = sprintf(CS timebuf, "%04d%02d",
+    sprintf(CS timebuf, "%04d%02d",
       1900 + t->tm_year, 1 + t->tm_mon);
 #endif
     break;
@@ -212,13 +219,6 @@ switch(type)
     break;
   }
 
-#ifndef COMPILE_UTILITY
-if (LOGGING(millisec) && off > 0)
-  (void) sprintf(CS timebuf + off, ".%03d", (int)(now.tv_usec/1000));
-#else
-off = off;	/* Compiler quietening */
-#endif
-
 return timebuf;
 }
 

commit 0b187e3ccb921c482601ee4a9e13c0caa2710794
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Jan 14 18:40:50 2018 +0000

    DKIM: DNS records having no v= tag are acceptable.  Bug 2207
    
    Broken-by c73a4d073e

diff --git a/src/src/dkim.c b/src/src/dkim.c
index 18427fe9..b4aecbc3 100644
--- a/src/src/dkim.c
+++ b/src/src/dkim.c
@@ -75,7 +75,8 @@ for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS);
       }
 
     /* check if this looks like a DKIM record */
-    if (strncasecmp(answer, "v=dkim", 6) != 0) continue;
+    if (strncmp(answer, "v=", 2) == 0 && strncasecmp(answer, "v=dkim", 6) != 0)
+      continue;
     return PDKIM_OK;
     }
 
diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c
index b884671d..186258a6 100644
--- a/src/src/pdkim/pdkim.c
+++ b/src/src/pdkim/pdkim.c
@@ -654,7 +654,8 @@ while ((ele = string_nextinlist(&raw_record, &sep, NULL, 0)))
   }
 
 /* Set fallback defaults */
-if (!pub->version    ) pub->version     = string_copy(PDKIM_PUB_RECORD_VERSION);
+if (!pub->version)
+  pub->version = string_copy(PDKIM_PUB_RECORD_VERSION);
 else if (Ustrcmp(pub->version, PDKIM_PUB_RECORD_VERSION) != 0)
   {
   DEBUG(D_acl) debug_printf(" Bad v= field\n");

commit 4360153635f43c3fffee39f6767ff17b11f38c55
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Jan 25 21:27:00 2018 +0000

    Cutthrough: fix multi-message initiating connections.  Bug 2230

diff --git a/src/src/acl.c b/src/src/acl.c
index 739cd91a..750fd2da 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -3293,6 +3293,8 @@ for (; cb != NULL; cb = cb->next)
 		p = pp;
 		}
 	      }
+	    else
+	      DEBUG(D_acl) debug_printf(" cutthrough request ignored for nonfirst rcpt\n");
 	    break;
 	    }
 	  *log_msgptr = string_sprintf("\"control=%s\" on %s item",
@@ -4422,22 +4424,29 @@ switch (where)
     else if (  rc == OK
 	    && cutthrough.delivery
 	    && rcpt_count > cutthrough.nrcpt
-	    && (rc = open_cutthrough_connection(addr)) == DEFER
 	    )
-      if (cutthrough.defer_pass)
-	{
-	uschar * s = addr->message;
-	/* Horrid kludge to recover target's SMTP message */
-	while (*s) s++;
-	do --s; while (!isdigit(*s));
-	if (*--s && isdigit(*s) && *--s && isdigit(*s)) *user_msgptr = s;
-	acl_temp_details = TRUE;
-	}
-      else
-	{
-	HDEBUG(D_acl) debug_printf_indent("cutthrough defer; will spool\n");
-	rc = OK;
-	}
+      {
+      if ((rc = open_cutthrough_connection(addr)) == DEFER)
+	if (cutthrough.defer_pass)
+	  {
+	  uschar * s = addr->message;
+	  /* Horrid kludge to recover target's SMTP message */
+	  while (*s) s++;
+	  do --s; while (!isdigit(*s));
+	  if (*--s && isdigit(*s) && *--s && isdigit(*s)) *user_msgptr = s;
+	  acl_temp_details = TRUE;
+	  }
+	else
+	  {
+	  HDEBUG(D_acl) debug_printf_indent("cutthrough defer; will spool\n");
+	  rc = OK;
+	  }
+      }
+    else HDEBUG(D_acl) if (cutthrough.delivery)
+      if (rcpt_count <= cutthrough.nrcpt)
+	debug_printf_indent("ignore cutthrough request; nonfirst message\n");
+      else if (rc != OK)
+	debug_printf_indent("ignore cutthrough request; ACL did not accept\n");
     break;
 
   case ACL_WHERE_PREDATA:
diff --git a/src/src/verify.c b/src/src/verify.c
index b957c709..0f0430b5 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -1380,6 +1380,7 @@ if(fd >= 0)
   _cutthrough_puts(US"QUIT\r\n", 6);	/* avoid recursion */
   _cutthrough_flush_send();
   cutthrough.fd = -1;			/* avoid recursion via read timeout */
+  cutthrough.nrcpt = 0;			/* permit re-cutthrough on subsequent message */
 
   /* Wait a short time for response, and discard it */
   cutthrough_response(fd, '2', NULL, 1);

commit baabef80f200a5e028a6f1487276a441caa40255
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Fri Jan 26 18:40:41 2018 +0000

    Cutthrough: fix for port-number defined by router.  Bug 2229

diff --git a/src/src/verify.c b/src/src/verify.c
index 0f0430b5..5486ca23 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -388,18 +388,21 @@ if (addr->transport == cutthrough.addr.transport)
 
       host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
 
-      if (!smtp_get_interface(tf->interface, host_af, addr, &interface,
-	      US"callout") ||
-	  !smtp_get_port(tf->port, addr, &port, US"callout"))
+      if (  !smtp_get_interface(tf->interface, host_af, addr, &interface,
+	      US"callout")
+	 || !smtp_get_port(tf->port, addr, &port, US"callout")
+	 )
 	log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
 	  addr->message);
 
+      smtp_port_for_connect(host, port);
+
       if (  (  interface == cutthrough.interface
 	    || (  interface
 	       && cutthrough.interface
 	       && Ustrcmp(interface, cutthrough.interface) == 0
 	    )  )
-	 && port == cutthrough.host.port
+	 && host->port == cutthrough.host.port
 	 )
 	{
 	uschar * resp = NULL;

commit 4b8ff36089f93f3315b94da2bdb68285340790a7
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Jan 27 15:03:01 2018 +0000

    GnuTLS: fix to ignore timeout on unrelated callout connection.  Bug 2174

diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 527ad28b..fc3aba59 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2105,10 +2105,8 @@ DEBUG(D_tls) debug_printf("about to gnutls_handshake\n");
 sigalrm_seen = FALSE;
 alarm(ob->command_timeout);
 do
-  {
   rc = gnutls_handshake(state->session);
-  } while ((rc == GNUTLS_E_AGAIN) ||
-      (rc == GNUTLS_E_INTERRUPTED && !sigalrm_seen));
+while (rc == GNUTLS_E_AGAIN || rc == GNUTLS_E_INTERRUPTED && !sigalrm_seen);
 alarm(0);
 
 if (rc != GNUTLS_E_SUCCESS)
@@ -2225,6 +2223,7 @@ ssize_t inbytes;
 DEBUG(D_tls) debug_printf("Calling gnutls_record_recv(%p, %p, %u)\n",
   state->session, state->xfer_buffer, ssl_xfer_buffer_size);
 
+sigalrm_seen = FALSE;
 if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout);
 inbytes = gnutls_record_recv(state->session, state->xfer_buffer,
   MIN(ssl_xfer_buffer_size, lim));

commit 2dfd20fe244da439b1f6becb4e29c8cb83e2d399
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Sat Jan 27 22:58:03 2018 +0100

    Build: .git may be a file when this repo is a submodule

diff --git a/src/scripts/reversion b/src/scripts/reversion
index c82d9c0a..00fc028b 100755
--- a/src/scripts/reversion
+++ b/src/scripts/reversion
@@ -37,7 +37,7 @@ fi
 
 # If this tree is a git working directory, use that to get version information.
 
-if [ -d ../../.git ] || [ "$1" = "release" ]
+if [ -d ../../.git ] || [ -f ../../.git ] || [ "$1" = "release" ]
 then
 	# Modify the output of git describe into separate parts for
 	# the name "exim" and the release and variant versions.

commit 20532ba65b0a952e5e471a502be79767efcae8d0
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Jan 28 20:32:28 2018 +0000

    Debugging: fix potential null-derefs in DSN debug_printfs

diff --git a/src/src/deliver.c b/src/src/deliver.c
index de552f4c..9f9990be 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -6152,7 +6152,7 @@ if (process_recipients != RECIP_IGNORE)
       new->dsn_flags = r->dsn_flags & rf_dsnflags;
       new->dsn_orcpt = r->orcpt;
       DEBUG(D_deliver) debug_printf("DSN: set orcpt: %s  flags: %d\n",
-	new->dsn_orcpt, new->dsn_flags);
+	new->dsn_orcpt ? new->dsn_orcpt : US"", new->dsn_flags);
 
       switch (process_recipients)
         {
@@ -7221,11 +7221,12 @@ for (addr_dsntmp = addr_succeed; addr_dsntmp; addr_dsntmp = addr_dsntmp->next)
       "DSN: envid: %s  ret: %d\n"
       "DSN: Final recipient: %s\n"
       "DSN: Remote SMTP server supports DSN: %d\n",
-      addr_dsntmp->router->name,
+      addr_dsntmp->router ? addr_dsntmp->router->name : US"(unknown)",
       addr_dsntmp->address,
       sender_address,
-      addr_dsntmp->dsn_orcpt, addr_dsntmp->dsn_flags,
-      dsn_envid, dsn_ret,
+      addr_dsntmp->dsn_orcpt ? addr_dsntmp->dsn_orcpt : US"NULL",
+      addr_dsntmp->dsn_flags,
+      dsn_envid ? dsn_envid : US"NULL", dsn_ret,
       addr_dsntmp->address,
       addr_dsntmp->dsn_aware
       );

commit d3c2dcc991b7f267ad56f0294827febff3d6688d
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Thu Feb 1 22:41:13 2018 +0100

    Add release-process/micro-release script

diff --git a/release-process/micro-release b/release-process/micro-release
new file mode 100755
index 00000000..f0975130
--- /dev/null
+++ b/release-process/micro-release
@@ -0,0 +1,146 @@
+#!/usr/bin/perl
+
+use v5.10.1;
+use strict;
+use warnings;
+use feature 'say';
+use Getopt::Long;
+use File::Temp qw(tempfile);
+use Pod::Usage;
+
+my $BRANCH_PATTERN = qr/^
+    (?<base>(?<name>exim)-
+    (?<version>
+    (?<major>\d+)
+    _(?<minor>\d+)
+    )(?:_(?<security>\d+))?
+    )\+fixes/x;
+
+sub trim { return join '', shift =~ /^\s*(.*?)\s*$/; }
+
+my %o = (edit => 1, dotted => 1, force => 0, auto => 0);
+
+GetOptions(
+    \%o,
+    'edit!',
+    'dotted!',
+    'force!', 'auto!',
+    'help|h' => sub { pod2usage(-verbose => 1, -exit => 0) },
+    'man|m'  => sub {
+        pod2usage(
+            -verbose   => 2,
+            -exit      => 0,
+            -noperldoc => system('perldoc -V >/dev/null 2>&1')
+        );
+    },
+) or pod2usage;
+
+my $branch = trim `git rev-parse --abbrev-ref HEAD`;    # exim-4_90+fixes
+die "$0: Branch `$branch` does not match `$BRANCH_PATTERN`"
+  unless "$branch" =~ /$BRANCH_PATTERN/;
+
+my $name = $+{name};
+my $base = $+{base};    # exim-4_90 (branch base)
+
+my $release_legacy = $+{version} . ($+{security} // '_0');    # 4.90_0
+my $release_dotted = join '.', $+{major}, $+{minor}, $+{security} // 0; # 4.90.0
+
+my $distance = trim `git rev-list --count $base..`;
+
+my $micro_legacy = "${release_legacy}_$distance";    # 4_90_0_20
+my $micro_dotted = "${release_dotted}.$distance";    # 4.90.0.20
+
+my $tag = "$name-" . ($o{dotted} ? $micro_dotted : $micro_legacy);
+
+# my $commit = trim `git show --no-patch --pretty=format:%h`;
+
+my ($fh, $filename) = tempfile();
+print $fh <<XXX;
+Exim fixup release $micro_dotted
+# Enter a one line tag description. The above is the sensible default.
+XXX
+close $fh;
+
+system(
+      defined $ENV{VISUAL} ? $ENV{VISUAL}
+    : defined $ENV{EDITOR} ? $ENV{EDITOR}
+    :                        'vi' => $filename
+  ) == 0
+  or exit $?
+  if $o{edit};
+
+if (not $o{auto}) {
+    print "Commit tag `$tag` to local repository? [Y/n] ";
+    exit 0 unless <STDIN> =~ /^y(?:es)?|$/i;
+}
+
+system(
+    git => 'tag',
+    '--sign', ($o{force} ? '--force' : ()),
+    -a => $tag,
+    -F => $filename,
+  ) == 0
+  or exit $?;
+
+exec git => 'tag',
+  '--verify'    => $tag;
+
+__END__
+
+=head1 NAME
+
+ micro-release - tag the current commit on a +fixes branch
+
+=head1 SYNOPSIS
+
+ micro-release [--[no]-edit] [--[no]-dotted] [--force] [TAG]
+
+=head1 DESCRIPTION
+
+This tools is for internal use only. It tags the current commit
+of a F<+fixes> branch as a "micro release".
+
+The tag is taken from the command line parameter I<TAG> or is calculated from the
+branch name and the distance to the release tag. The branch name needs
+to match the branch tag.
+
+    branch tag:    exim-4_90
+    branch name:   exim-4_90+fixes
+    resulting tag: exim-4.90.0.<distance>   (dotted version)
+                   exim-4_90_0_<distance>   (classic version)
+
+    branch tag:    exim-4_90_1   (security release)
+    branch name:   exim-4_90_1+fixes
+    resulting tag: exim-4.90.1.<distance>   (dotted version)
+                   exim-4_90_1_<distance>   (classic version)
+
+The tag will be GPG signed.
+
+=head1 OPTIONS
+
+=over
+
+=item B<--[no]-edit>
+
+Spawn or do not spawn an editor. The default message will be "Exim
+<tagname> fixup release".
+
+=item B<--dotted>
+
+Generate a dotted tag name. (default: on)
+
+=item B<--force>
+
+(Re)Move an existing tag name. Use with care!
+
+=item B<-h>|B<--help>
+
+Short help.
+
+=item B<-m>|B<--man>
+
+The manual page.
+
+=back
+
+=cut

commit 9227485d83bbff9bcb4cb54d8f1d3a225b46f06c
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Sat Feb 3 17:05:09 2018 +0100

    Small fixups for micro-release (now: tag-fixes-release) script

diff --git a/release-process/micro-release b/release-process/scripts/tag-fixes-release
similarity index 77%
rename from release-process/micro-release
rename to release-process/scripts/tag-fixes-release
index f0975130..8d0beee0 100755
--- a/release-process/micro-release
+++ b/release-process/scripts/tag-fixes-release
@@ -52,7 +52,11 @@ my $micro_dotted = "${release_dotted}.$distance";    # 4.90.0.20
 
 my $tag = "$name-" . ($o{dotted} ? $micro_dotted : $micro_legacy);
 
-# my $commit = trim `git show --no-patch --pretty=format:%h`;
+# check if the tag already exists
+if (not $o{force}) {
+    `git rev-parse --quiet --verify '$tag'`;         # ignore stdout
+    die "$0: tag '$tag' exists already\n" if $? == 0;
+}
 
 my ($fh, $filename) = tempfile();
 print $fh <<XXX;
@@ -66,12 +70,12 @@ system(
     : defined $ENV{EDITOR} ? $ENV{EDITOR}
     :                        'vi' => $filename
   ) == 0
-  or exit $?
+  or exit $? >> 8
   if $o{edit};
 
 if (not $o{auto}) {
     print "Commit tag `$tag` to local repository? [Y/n] ";
-    exit 0 unless <STDIN> =~ /^y(?:es)?|$/i;
+    exit 0 unless <STDIN> =~ /^(:?y(?:es)?|)$/i;
 }
 
 system(
@@ -80,29 +84,33 @@ system(
     -a => $tag,
     -F => $filename,
   ) == 0
-  or exit $?;
+  or exit $? >> 8;
+
+system(
+    git        => 'tag',
+    '--verify' => $tag
+  ) == 0
+  or exit $? >> 8;
 
-exec git => 'tag',
-  '--verify'    => $tag;
+say "\nDo not forget to `git push --follow-tags ...` now."
 
 __END__
 
 =head1 NAME
 
- micro-release - tag the current commit on a +fixes branch
+ tag-fixes-release - tag the current commit on a +fixes branch
 
 =head1 SYNOPSIS
 
- micro-release [--[no]-edit] [--[no]-dotted] [--force] [TAG]
+ tag-fixes-release [--[no]-edit] [--[no]-dotted] [--force]
 
 =head1 DESCRIPTION
 
-This tools is for internal use only. It tags the current commit
-of a F<+fixes> branch as a "micro release".
+This B<tag-fixes-release> tools is for internal use only. It tags the
+current commit of a F<+fixes> branch as a "micro release".
 
-The tag is taken from the command line parameter I<TAG> or is calculated from the
-branch name and the distance to the release tag. The branch name needs
-to match the branch tag.
+The tag is calculated from the branch name and the distance to the
+branch tag. The branch name needs to match the branch tag.
 
     branch tag:    exim-4_90
     branch name:   exim-4_90+fixes

commit e5fc5d4ba779be4c57bd08ad2da70b6e1a85a549
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Feb 7 23:09:55 2018 +0000

    DKIM: fix buffer overflow in verify
    
    Caused crash in free() by corrupting malloc metadata.
    
    Reported-by: University of Cambridge
    Broken-by: 80a47a2c96

diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c
index 186258a6..62934927 100644
--- a/src/src/pdkim/pdkim.c
+++ b/src/src/pdkim/pdkim.c
@@ -701,7 +701,7 @@ if (sig->canon_body == PDKIM_CANON_RELAXED)
   if (!relaxed_data)
     {
     BOOL seen_wsp = FALSE;
-    const uschar * p;
+    const uschar * p, * r;
     int q = 0;
 
     /* We want to be able to free this else we allocate
@@ -712,7 +712,7 @@ if (sig->canon_body == PDKIM_CANON_RELAXED)
     relaxed_data = store_malloc(sizeof(blob) + orig_data->len+1);
     relaxed_data->data = US (relaxed_data+1);
 
-    for (p = orig_data->data; *p; p++)
+    for (p = orig_data->data, r = p + orig_data->len; p < r; p++)
       {
       char c = *p;
       if (c == '\r')
@@ -838,6 +838,7 @@ ctx->linebuf_offset = 0;
 
 /* -------------------------------------------------------------------------- */
 /* Call from pdkim_feed below for processing complete body lines */
+/* NOTE: the line is not NUL-terminated; but we have a count */
 
 static void
 pdkim_bodyline_complete(pdkim_ctx * ctx)

commit 1d602879523c878458f81b9d9d00df49fdcea9ef
Author: Phil Pennock <pdp@exim.org>
Date:   Wed Feb 7 22:59:03 2018 -0500

    Repair Heimdal GSSAPI authenticator init
    
    Broken-by: f2ed27cf5f
    (cherry picked from commit 7be145827de6464f18601325df0091b7c7ab908e)

diff --git a/src/src/drtables.c b/src/src/drtables.c
index 1d815579..5ed8d01c 100644
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -119,7 +119,7 @@ auth_info auths_available[] = {
   {
   .driver_name =	US"heimdal_gssapi",
   .options =		auth_heimdal_gssapi_options,
-  .options_count 	 &auth_heimdal_gssapi_options_count,
+  .options_count =	&auth_heimdal_gssapi_options_count,
   .options_block =	&auth_heimdal_gssapi_option_defaults,
   .options_len =	sizeof(auth_heimdal_gssapi_options_block),
   .init =		auth_heimdal_gssapi_init,

commit 00cddd57ad37a7eb9543c3ea3d34714c7425f720
Author: Phil Pennock <pdp@exim.org>
Date:   Wed Feb 7 23:15:37 2018 -0500

    Repair Heimdal GSSAPI authenticator init part 2
    
    Broken-by: d185889f4
    (cherry picked from commit aab9a84358906493bde0efc6aa13b00e59096396)

diff --git a/src/src/auths/heimdal_gssapi.c b/src/src/auths/heimdal_gssapi.c
index b583a73f..631b9790 100644
--- a/src/src/auths/heimdal_gssapi.c
+++ b/src/src/auths/heimdal_gssapi.c
@@ -80,9 +80,9 @@ auth_heimdal_gssapi_options_block auth_heimdal_gssapi_option_defaults = {
 #ifdef MACRO_PREDEF
 
 /* Dummy values */
-void auth_heimdal_init(auth_instance *ablock) {}
-int auth_heimdal_server(auth_instance *ablock, uschar *data) {return 0;}
-int auth_heimdal_client(auth_instance *ablock, smtp_inblock *inblock,
+void auth_heimdal_gssapi_init(auth_instance *ablock) {}
+int auth_heimdal_gssapi_server(auth_instance *ablock, uschar *data) {return 0;}
+int auth_heimdal_gssapi_client(auth_instance *ablock, smtp_inblock *inblock,
   smtp_outblock *outblock, int timeout, uschar *buffer, int buffsize) {return 0;}
 void auth_heimdal_gssapi_version_report(FILE *f) {}
 

commit 062990cc1b2f9e5d82a413b53c8f0569075de700
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Mon Feb 5 22:23:32 2018 +0100

    Fix base64d() buffer size (CVE-2018-6789)
    
    Credits for discovering this bug: Meh Chang <meh@devco.re>

diff --git a/src/src/base64.c b/src/src/base64.c
index f6f187f0..e58ca6c7 100644
--- a/src/src/base64.c
+++ b/src/src/base64.c
@@ -152,10 +152,14 @@ static uschar dec64table[] = {
 int
 b64decode(const uschar *code, uschar **ptr)
 {
+
 int x, y;
-uschar *result = store_get(3*(Ustrlen(code)/4) + 1);
+uschar *result;
 
-*ptr = result;
+{
+  int l = Ustrlen(code);
+  *ptr = result = store_get(1 + l/4 * 3 + l%4);
+}
 
 /* Each cycle of the loop handles a quantum of 4 input bytes. For the last
 quantum this may decode to 1, 2, or 3 output bytes. */

commit bd5cdd5f2ddf4e9c26ff9353d15324dd0bbda37c
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Mon Feb 12 17:29:22 2018 +0000

    ACL: Enforce non-usability of control=utf8_downconvert in MAIL ACL.  Bug 2239

diff --git a/src/src/acl.c b/src/src/acl.c
index 750fd2da..95ee2667 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -533,7 +533,9 @@ static control_def controls_list[] = {
   },
 #ifdef SUPPORT_I18N
 [CONTROL_UTF8_DOWNCONVERT] =
-  { US"utf8_downconvert",        TRUE, 0 }
+  { US"utf8_downconvert",        TRUE,
+    (unsigned) ~((1<<ACL_WHERE_RCPT) | (1<<ACL_WHERE_VRFY))
+  }
 #endif
 };
 

commit 695a7097d5f737350c6bbb007f9f8a3b7cad5c24
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Feb 17 16:53:27 2018 +0000

    Fix memory leak during multi-message reception using STARTTLS
    
    Reported-by: Wolfgang Breyha

diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index fc3aba59..58f50510 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2202,9 +2202,10 @@ gnutls_certificate_free_credentials(state->x509_cred);
 
 
 state->tlsp->active = -1;
+if (state->xfer_buffer) store_free(state->xfer_buffer);
 memcpy(state, &exim_gnutls_state_init, sizeof(exim_gnutls_state_init));
 
-if ((state_server.session == NULL) && (state_client.session == NULL))
+if (!state_server.session && !state_client.session)
   {
   gnutls_global_deinit();
   exim_gnutls_base_init_done = FALSE;
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index b225eb76..03b9023d 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -2073,7 +2073,7 @@ DEBUG(D_tls)
    smtp_read_response()/ip_recv().
    Hence no need to duplicate for _in and _out.
  */
-ssl_xfer_buffer = store_malloc(ssl_xfer_buffer_size);
+if (!ssl_xfer_buffer) ssl_xfer_buffer = store_malloc(ssl_xfer_buffer_size);
 ssl_xfer_buffer_lwm = ssl_xfer_buffer_hwm = 0;
 ssl_xfer_eof = ssl_xfer_error = 0;
 

commit 2a8bf6ec7eb5ef9760bae9c8b104966d25cfb510
Author: Wolfgang Breyha <wbreyha@gmx.net>
Date:   Mon Feb 19 18:27:55 2018 +0000

    OpenSSL: Fix memory leak during multi-message connections using STARTTLS
    
    Reported-by: Wolfgang Breyha
    Fix-by: Wolfgang Breyha, with additions from Jeremy Harris

diff --git a/src/src/daemon.c b/src/src/daemon.c
index 37fefd4b..f7a27477 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -653,7 +653,7 @@ if (pid == 0)
         the data structures if necessary. */
 
 #ifdef SUPPORT_TLS
-        tls_close(TRUE, FALSE);
+        tls_close(TRUE, TLS_NO_SHUTDOWN);
 #endif
 
         /* Reset SIGHUP and SIGCHLD in the child in both cases. */
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 9f9990be..2dfa9e38 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -4990,7 +4990,7 @@ all pipes, so I do not see a reason to use non-blocking IO here
   if (cutthrough.fd >= 0 && cutthrough.callout_hold_only)
     {
 #ifdef SUPPORT_TLS
-    tls_close(FALSE, FALSE);
+    tls_close(FALSE, TLS_NO_SHUTDOWN);
 #endif
     (void) close(cutthrough.fd);
     release_cutthrough_connection(US"passed to transport proc");
diff --git a/src/src/exim.c b/src/src/exim.c
index d71af0ac..b18d3688 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -543,7 +543,7 @@ close_unwanted(void)
 if (smtp_input)
   {
   #ifdef SUPPORT_TLS
-  tls_close(TRUE, FALSE);      /* Shut down the TLS library */
+  tls_close(TRUE, TLS_NO_SHUTDOWN);      /* Shut down the TLS library */
   #endif
   (void)close(fileno(smtp_in));
   (void)close(fileno(smtp_out));
diff --git a/src/src/functions.h b/src/src/functions.h
index e50fa6f9..53891162 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -50,7 +50,7 @@ extern int     tls_client_start(int, host_item *, address_item *,
 		dns_answer *,
 # endif
 		uschar **);
-extern void    tls_close(BOOL, BOOL);
+extern void    tls_close(BOOL, int);
 extern BOOL    tls_could_read(void);
 extern int     tls_export_cert(uschar *, size_t, void *);
 extern int     tls_feof(void);
diff --git a/src/src/macros.h b/src/src/macros.h
index 764c65b8..5f9c7422 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -991,5 +991,10 @@ enum { FILTER_UNSET, FILTER_FORWARD, FILTER_EXIM, FILTER_SIEVE };
 #define UTF8_VERT_2DASH		"\xE2\x95\x8E"
 
 
+/* Options on tls_close */
+#define TLS_NO_SHUTDOWN		0
+#define TLS_SHUTDOWN_NOWAIT	1
+#define TLS_SHUTDOWN_WAIT	2
+
 
 /* End of macros.h */
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 2603da25..3339f9a1 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -3732,7 +3732,7 @@ else
   smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname);
 
 #ifdef SUPPORT_TLS
-tls_close(TRUE, TRUE);
+tls_close(TRUE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 log_write(L_smtp_connection, LOG_MAIN, "%s closed by QUIT",
@@ -5417,7 +5417,7 @@ while (done <= 0)
 	smtp_printf("554 Security failure\r\n", FALSE);
 	break;
       }
-    tls_close(TRUE, TRUE);
+    tls_close(TRUE, TLS_SHUTDOWN_NOWAIT);
     break;
     #endif
 
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 58f50510..ffadd96b 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2180,12 +2180,15 @@ return OK;
 daemon, to shut down the TLS library, without actually doing a shutdown (which
 would tamper with the TLS session in the parent process).
 
-Arguments:   TRUE if gnutls_bye is to be called
+Arguments:
+  shutdown	1 if TLS close-alert is to be sent,
+ 		2 if also response to be waited for
+
 Returns:     nothing
 */
 
 void
-tls_close(BOOL is_server, BOOL shutdown)
+tls_close(BOOL is_server, int shutdown)
 {
 exim_gnutls_state_st *state = is_server ? &state_server : &state_client;
 
@@ -2193,8 +2196,12 @@ if (!state->tlsp || state->tlsp->active < 0) return;  /* TLS was not active */
 
 if (shutdown)
   {
-  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS\n");
-  gnutls_bye(state->session, GNUTLS_SHUT_WR);
+  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
+    shutdown > 1 ? " (with response-wait)" : "");
+
+  alarm(2);
+  gnutls_bye(state->session, shutdown > 1 ? GNUTLS_SHUT_RDWR : GNUTLS_SHUT_WR);
+  alarm(0);
   }
 
 gnutls_deinit(state->session);
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index 03b9023d..da7eb8a4 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -152,6 +152,7 @@ typedef struct tls_ext_ctx_cb {
   uschar *certificate;
   uschar *privatekey;
   BOOL is_server;
+  STACK_OF(X509_NAME) * acceptable_certnames;
 #ifndef DISABLE_OCSP
   STACK_OF(X509) *verify_stack;		/* chain for verifying the proof */
   union {
@@ -1488,6 +1489,7 @@ cbinfo = store_malloc(sizeof(tls_ext_ctx_cb));
 cbinfo->certificate = certificate;
 cbinfo->privatekey = privatekey;
 cbinfo->is_server = host==NULL;
+cbinfo->acceptable_certnames = NULL;
 #ifndef DISABLE_OCSP
 cbinfo->verify_stack = NULL;
 if (!host)
@@ -1732,6 +1734,9 @@ chain_from_pem_file(const uschar * file, STACK_OF(X509) * verify_stack)
 BIO * bp;
 X509 * x;
 
+while (sk_X509_num(verify_stack) > 0)
+  X509_free(sk_X509_pop(verify_stack));
+
 if (!(bp = BIO_new_file(CS file, "r"))) return FALSE;
 while ((x = PEM_read_bio_X509(bp, NULL, 0, NULL)))
   sk_X509_push(verify_stack, x);
@@ -1741,7 +1746,8 @@ return TRUE;
 
 
 
-/* Called by both client and server startup
+/* Called by both client and server startup; on the server possibly
+repeated after a Server Name Indication.
 
 Arguments:
   sctx          SSL_CTX* to initialise
@@ -1791,6 +1797,7 @@ if (expcerts && *expcerts)
 	{ file = NULL; dir = expcerts; }
       else
 	{
+	/*XXX somewhere down here we leak memory per-STARTTLS, on a multi-message conn, server-side */
 	file = expcerts; dir = NULL;
 #ifndef DISABLE_OCSP
 	/* In the server if we will be offering an OCSP proof, load chain from
@@ -1831,11 +1838,21 @@ if (expcerts && *expcerts)
       */
       if (file)
 	{
-	STACK_OF(X509_NAME) * names = SSL_load_client_CA_file(CS file);
+	tls_ext_ctx_cb * cbinfo = host
+	  ? client_static_cbinfo : server_static_cbinfo;
+	STACK_OF(X509_NAME) * names;
 
+	if ((names = cbinfo->acceptable_certnames))
+	  {
+	  sk_X509_NAME_pop_free(names, X509_NAME_free);
+	  cbinfo->acceptable_certnames = NULL;
+	  }
+	names = SSL_load_client_CA_file(CS file);
+
+	SSL_CTX_set_client_CA_list(sctx, names);
 	DEBUG(D_tls) debug_printf("Added %d certificate authorities.\n",
 				    sk_X509_NAME_num(names));
-	SSL_CTX_set_client_CA_list(sctx, names);
+	cbinfo->acceptable_certnames = names;
 	}
       }
     }
@@ -2446,7 +2463,16 @@ if (error == SSL_ERROR_ZERO_RETURN)
   receive_ferror = smtp_ferror;
   receive_smtp_buffered = smtp_buffered;
 
+  if (SSL_get_shutdown(server_ssl) == SSL_RECEIVED_SHUTDOWN)
+ 	SSL_shutdown(server_ssl);
+
+  sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
+  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames, X509_NAME_free);
   SSL_free(server_ssl);
+  SSL_CTX_free(server_ctx);
+  server_static_cbinfo->verify_stack = NULL;
+  server_static_cbinfo->acceptable_certnames = NULL;
+  server_ctx = NULL;
   server_ssl = NULL;
   tls_in.active = -1;
   tls_in.bits = 0;
@@ -2680,15 +2706,19 @@ return len;
 daemon, to shut down the TLS library, without actually doing a shutdown (which
 would tamper with the SSL session in the parent process).
 
-Arguments:   TRUE if SSL_shutdown is to be called
+Arguments:
+  shutdown	1 if TLS close-alert is to be sent,
+ 		2 if also response to be waited for
+
 Returns:     nothing
 
 Used by both server-side and client-side TLS.
 */
 
 void
-tls_close(BOOL is_server, BOOL shutdown)
+tls_close(BOOL is_server, int shutdown)
 {
+SSL_CTX **ctxp = is_server ? &server_ctx : &client_ctx;
 SSL **sslp = is_server ? &server_ssl : &client_ssl;
 int *fdp = is_server ? &tls_in.active : &tls_out.active;
 
@@ -2696,13 +2726,38 @@ if (*fdp < 0) return;  /* TLS was not active */
 
 if (shutdown)
   {
-  DEBUG(D_tls) debug_printf("tls_close(): shutting down SSL\n");
-  SSL_shutdown(*sslp);
+  int rc;
+  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
+    shutdown > 1 ? " (with response-wait)" : "");
+
+  if (  (rc = SSL_shutdown(*sslp)) == 0	/* send "close notify" alert */
+     && shutdown > 1)
+    {
+    alarm(2);
+    rc = SSL_shutdown(*sslp);		/* wait for response */
+    alarm(0);
+    }
+
+  if (rc < 0) DEBUG(D_tls)
+    {
+    ERR_error_string(ERR_get_error(), ssl_errstring);
+    debug_printf("SSL_shutdown: %s\n", ssl_errstring);
+    }
+  }
+
+if (is_server)
+  {
+  sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
+  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames,
+    X509_NAME_free);
+  server_static_cbinfo->verify_stack = NULL;
+  server_static_cbinfo->acceptable_certnames = NULL;
   }
 
+SSL_CTX_free(*ctxp);
 SSL_free(*sslp);
+*ctxp = NULL;
 *sslp = NULL;
-
 *fdp = -1;
 }
 
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 5d351dd7..acfec506 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2233,7 +2233,7 @@ if (sx->send_quit)
   (void)smtp_write_command(&sx->outblock, SCMD_FLUSH, "QUIT\r\n");
 
 #ifdef SUPPORT_TLS
-tls_close(FALSE, TRUE);
+tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 /* Close the socket, and return the appropriate value, first setting
@@ -2667,7 +2667,7 @@ for (fd_bits = 3; fd_bits; )
     if ((rc = read(pfd[0], buf, bsize)) <= 0)
       {
       fd_bits = 0;
-      tls_close(FALSE, TRUE);
+      tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
       }
     else
       {
@@ -3457,7 +3457,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit)
 	  a new EHLO. If we don't get a good response, we don't attempt to pass
 	  the socket on. */
 
-	  tls_close(FALSE, TRUE);
+	  tls_close(FALSE, TLS_SHUTDOWN_WAIT);
 	  smtp_peer_options = smtp_peer_options_wrap;
 	  sx.ok = !sx.smtps
 	    && smtp_write_command(&sx.outblock, SCMD_FLUSH,
@@ -3522,7 +3522,7 @@ propagate it from the initial
 	    close(pfd[0]);
 	    /* tidy the inter-proc to disconn the proxy proc */
 	    waitpid(pid, NULL, 0);
-	    tls_close(FALSE, FALSE);
+	    tls_close(FALSE, TLS_NO_SHUTDOWN);
 	    (void)close(sx.inblock.sock);
 	    continue_transport = NULL;
 	    continue_hostname = NULL;
@@ -3568,7 +3568,7 @@ if (sx.send_quit) (void)smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n")
 END_OFF:
 
 #ifdef SUPPORT_TLS
-tls_close(FALSE, TRUE);
+tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 /* Close the socket, and return the appropriate value, first setting
@@ -4540,7 +4540,7 @@ retry_non_continued:
     if (tls_out.active == fd)
       {
       (void) tls_write(FALSE, US"QUIT\r\n", 6, FALSE);
-      tls_close(FALSE, TRUE);
+      tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
       }
     else
 #else
diff --git a/src/src/verify.c b/src/src/verify.c
index 5486ca23..544069bb 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -821,7 +821,7 @@ tls_retry_connection:
 	      debug_printf_indent("problem after random/rset/mfrom; reopen conn\n");
 	    random_local_part = NULL;
 #ifdef SUPPORT_TLS
-	    tls_close(FALSE, TRUE);
+	    tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 	    HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 	    (void)close(sx.inblock.sock);
@@ -1088,7 +1088,7 @@ no_conn:
       if (sx.inblock.sock >= 0)
 	{
 #ifdef SUPPORT_TLS
-	tls_close(FALSE, TRUE);
+	tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 	HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 	(void)close(sx.inblock.sock);
@@ -1389,7 +1389,7 @@ if(fd >= 0)
   cutthrough_response(fd, '2', NULL, 1);
 
 #ifdef SUPPORT_TLS
-  tls_close(FALSE, TRUE);
+  tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
   HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
   (void)close(fd);

commit 8b17525025ccd7af1299c9fee2bd43595ab3de09
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Feb 22 11:26:34 2018 +0000

    Fix exim_dbmbuild to permit directoryless filenames.  Bug 2242
    
    Broken-by: 0a6c178c6c

diff --git a/src/src/exim_dbmbuild.c b/src/src/exim_dbmbuild.c
index 2dcc40f3..fb5317b8 100644
--- a/src/src/exim_dbmbuild.c
+++ b/src/src/exim_dbmbuild.c
@@ -210,6 +210,8 @@ Ustrcat(temp_dbmname, ".dbmbuild_temp");
 Ustrcpy(dirname, temp_dbmname);
 if ((bptr = Ustrrchr(dirname, '/')))
   *bptr = '\0';
+else
+  Ustrcpy(dirname, ".");
 
 /* It is apparently necessary to open with O_RDWR for this to work
 with gdbm-1.7.3, though no reading is actually going to be done. */

commit 4cdbbcada0318fdc8db463fbf203a4afa1914d92
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Feb 22 23:52:17 2018 +0000

    OpenSSL: revert needless free of certificate list.  The library handles it internally.
    
    Reported-by: Torsten Tributh

diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index da7eb8a4..addda713 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -152,7 +152,6 @@ typedef struct tls_ext_ctx_cb {
   uschar *certificate;
   uschar *privatekey;
   BOOL is_server;
-  STACK_OF(X509_NAME) * acceptable_certnames;
 #ifndef DISABLE_OCSP
   STACK_OF(X509) *verify_stack;		/* chain for verifying the proof */
   union {
@@ -1489,7 +1488,6 @@ cbinfo = store_malloc(sizeof(tls_ext_ctx_cb));
 cbinfo->certificate = certificate;
 cbinfo->privatekey = privatekey;
 cbinfo->is_server = host==NULL;
-cbinfo->acceptable_certnames = NULL;
 #ifndef DISABLE_OCSP
 cbinfo->verify_stack = NULL;
 if (!host)
@@ -1840,19 +1838,11 @@ if (expcerts && *expcerts)
 	{
 	tls_ext_ctx_cb * cbinfo = host
 	  ? client_static_cbinfo : server_static_cbinfo;
-	STACK_OF(X509_NAME) * names;
-
-	if ((names = cbinfo->acceptable_certnames))
-	  {
-	  sk_X509_NAME_pop_free(names, X509_NAME_free);
-	  cbinfo->acceptable_certnames = NULL;
-	  }
-	names = SSL_load_client_CA_file(CS file);
+	STACK_OF(X509_NAME) * names = SSL_load_client_CA_file(CS file);
 
 	SSL_CTX_set_client_CA_list(sctx, names);
 	DEBUG(D_tls) debug_printf("Added %d certificate authorities.\n",
 				    sk_X509_NAME_num(names));
-	cbinfo->acceptable_certnames = names;
 	}
       }
     }
@@ -2467,11 +2457,9 @@ if (error == SSL_ERROR_ZERO_RETURN)
  	SSL_shutdown(server_ssl);
 
   sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
-  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames, X509_NAME_free);
   SSL_free(server_ssl);
   SSL_CTX_free(server_ctx);
   server_static_cbinfo->verify_stack = NULL;
-  server_static_cbinfo->acceptable_certnames = NULL;
   server_ctx = NULL;
   server_ssl = NULL;
   tls_in.active = -1;
@@ -2748,10 +2736,7 @@ if (shutdown)
 if (is_server)
   {
   sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
-  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames,
-    X509_NAME_free);
   server_static_cbinfo->verify_stack = NULL;
-  server_static_cbinfo->acceptable_certnames = NULL;
   }
 
 SSL_CTX_free(*ctxp);

commit ae06ddc47aa43bb5e2dcbe2643e41649d1947a9d
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Feb 24 20:04:25 2018 +0000

    I18N: Fix utf8_downconvert propagation through a redirect router

diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c
index c823f023..0f9839a3 100644
--- a/src/src/routers/redirect.c
+++ b/src/src/routers/redirect.c
@@ -465,8 +465,9 @@ while (generated)
     }
 
 #ifdef SUPPORT_I18N
-    next->prop.utf8_msg = string_is_utf8(next->address)
-      || (sender_address && string_is_utf8(sender_address));
+    if (!next->prop.utf8_msg)
+      next->prop.utf8_msg = string_is_utf8(next->address)
+        || (sender_address && string_is_utf8(sender_address));
 #endif
 
   DEBUG(D_route)
@@ -567,9 +568,9 @@ addr_prop.remove_headers = NULL;
 addr_prop.srs_sender = NULL;
 #endif
 #ifdef SUPPORT_I18N
-addr_prop.utf8_msg = FALSE;	/*XXX should we not copy this from the parent? */
-addr_prop.utf8_downcvt = FALSE;
-addr_prop.utf8_downcvt_maybe = FALSE;
+addr_prop.utf8_msg = addr->prop.utf8_msg;
+addr_prop.utf8_downcvt = addr->prop.utf8_downcvt;
+addr_prop.utf8_downcvt_maybe = addr->prop.utf8_downcvt_maybe;
 #endif
 
 

commit d4ddcd11d9158cef132253f33b76f28a0bdb3bba
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Mar 11 19:09:19 2018 +0000

    Fix ldap lookups for zero-length attribute value.  Bug 2251
    Broken-by: acec9514b1

diff --git a/src/src/lookups/ldap.c b/src/src/lookups/ldap.c
index 235af0f9..06db734c 100644
--- a/src/src/lookups/ldap.c
+++ b/src/src/lookups/ldap.c
@@ -884,13 +884,13 @@ while ((rc = ldap_result(lcp->ld, msgid, 0, timeoutptr, &result)) ==
   result = NULL;
   }            /* End "while" loop for multiple results */
 
-/* Terminate the dynamic string that we have built and reclaim unused store */
+/* Terminate the dynamic string that we have built and reclaim unused store.
+In the odd case of a single attribute with zero-length value, allocate
+an empty string. */
 
-if (data)
-  {
-  (void) string_from_gstring(data);
-  store_reset(data->s + data->ptr + 1);
-  }
+if (!data) data = string_get(1);
+(void) string_from_gstring(data);
+store_reset(data->s + data->ptr + 1);
 
 /* Copy the last dn into eldap_dn */
 

commit fc35a505f8b412be5cf09bf587c237f3316a2bfe
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 13 13:52:26 2018 +0000

    Mark variables unused before release of store in the daemon loop

diff --git a/src/src/daemon.c b/src/src/daemon.c
index f7a27477..259c21fe 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -563,18 +563,13 @@ if (pid == 0)
 
     /* Reclaim up the store used in accepting this message */
 
-    return_path = sender_address = NULL;
-    authenticated_sender = NULL;
-    sending_ip_address = NULL;
-    deliver_host_address = deliver_host =
-    deliver_domain_orig = deliver_localpart_orig = NULL;
-    dnslist_domain = dnslist_matched = NULL;
-    callout_address = NULL;
-#ifndef DISABLE_DKIM
-    dkim_cur_signer = NULL;
-#endif
-    acl_var_m = NULL;
-    store_reset(reset_point);
+      {
+      int r = receive_messagecount;
+      BOOL q = queue_only_policy;
+      smtp_reset(reset_point);
+      queue_only_policy = q;
+      receive_messagecount = r;
+      }
 
     /* If queue_only is set or if there are too many incoming connections in
     existence, session_local_queue_only will be TRUE. If it is not, check
diff --git a/src/src/functions.h b/src/src/functions.h
index 53891162..ccd3d581 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -416,6 +416,7 @@ extern void    smtp_log_no_mail(void);
 extern void    smtp_message_code(uschar **, int *, uschar **, uschar **, BOOL);
 extern void    smtp_proxy_tls(uschar *, size_t, int *, int);
 extern BOOL    smtp_read_response(smtp_inblock *, uschar *, int, int, int);
+extern void    smtp_reset(void *);
 extern void    smtp_respond(uschar *, int, BOOL, uschar *);
 extern void    smtp_notquit_exit(uschar *, uschar *, uschar *, ...);
 extern void    smtp_port_for_connect(host_item *, int);
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 3339f9a1..8bbcafb1 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -1954,13 +1954,13 @@ return TRUE;
 *************************************************/
 
 /* This function is called whenever the SMTP session is reset from
-within either of the setup functions.
+within either of the setup functions; also from the daemon loop.
 
 Argument:   the stacking pool storage reset point
 Returns:    nothing
 */
 
-static void
+void
 smtp_reset(void *reset_point)
 {
 recipients_list = NULL;
@@ -2004,9 +2004,8 @@ bmi_verdicts = NULL;
 #endif
 dnslist_domain = dnslist_matched = NULL;
 #ifndef DISABLE_DKIM
-dkim_signers = NULL;
-dkim_disable_verify = FALSE;
-dkim_collect_input = FALSE;
+dkim_cur_signer = dkim_signers = NULL;
+dkim_disable_verify = dkim_collect_input = FALSE;
 #endif
 dsn_ret = 0;
 dsn_envid = NULL;
@@ -2015,10 +2014,7 @@ deliver_host = deliver_host_address = NULL;	/* Can be set by ACL */
 prdr_requested = FALSE;
 #endif
 #ifdef EXPERIMENTAL_SPF
-spf_header_comment = NULL;
-spf_received = NULL;
-spf_result = NULL;
-spf_smtp_comment = NULL;
+spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 #endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;

commit 1f3a53b0af67aac909b2ba6ce1244a8a64d9a718
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 13 16:27:54 2018 +0000

    Mark variables unused before release of store in the queue-runner loop

diff --git a/src/src/functions.h b/src/src/functions.h
index ccd3d581..dfe8ff57 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -430,13 +430,14 @@ extern int     smtp_write_command(smtp_outblock *, int, const char *, ...) PRINT
 extern int     spam(const uschar **);
 extern FILE   *spool_mbox(unsigned long *, const uschar *, uschar **);
 #endif
-extern BOOL    spool_move_message(uschar *, uschar *, uschar *, uschar *);
+extern void    spool_clear_header_globals(void);
 extern uschar *spool_dname(const uschar *, uschar *);
 extern uschar *spool_fname(const uschar *, const uschar *, const uschar *, const uschar *);
-extern uschar *spool_sname(const uschar *, uschar *);
+extern BOOL    spool_move_message(uschar *, uschar *, uschar *, uschar *);
 extern int     spool_open_datafile(uschar *);
 extern int     spool_open_temp(uschar *);
 extern int     spool_read_header(uschar *, BOOL, BOOL);
+extern uschar *spool_sname(const uschar *, uschar *);
 extern int     spool_write_header(uschar *, int, uschar **);
 extern int     stdin_getc(unsigned);
 extern int     stdin_feof(void);
diff --git a/src/src/queue.c b/src/src/queue.c
index 09a149e9..e613bfac 100644
--- a/src/src/queue.c
+++ b/src/src/queue.c
@@ -601,9 +601,7 @@ for (i = queue_run_in_order ? -1 : 0;
 
       /* Recover store used when reading the header */
 
-      received_protocol = NULL;
-      sender_address = sender_ident = NULL;
-      authenticated_id = authenticated_sender = NULL;
+      spool_clear_header_globals();
       store_reset(reset_point2);
       if (!wanted) continue;      /* With next message */
       }
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 2a99c63d..d4db37de 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -206,49 +206,13 @@ return TRUE;
 
 
 
-/*************************************************
-*             Read spool header file             *
-*************************************************/
-
-/* This function reads a spool header file and places the data into the
-appropriate global variables. The header portion is always read, but header
-structures are built only if read_headers is set true. It isn't, for example,
-while generating -bp output.
-
-It may be possible for blocks of nulls (binary zeroes) to get written on the
-end of a file if there is a system crash during writing. It was observed on an
-earlier version of Exim that omitted to fsync() the files - this is thought to
-have been the cause of that incident, but in any case, this code must be robust
-against such an event, and if such a file is encountered, it must be treated as
-malformed.
-
-As called from deliver_message() (at least) we are running as root.
-
-Arguments:
-  name          name of the header file, including the -H
-  read_headers  TRUE if in-store header structures are to be built
-  subdir_set    TRUE is message_subdir is already set
-
-Returns:        spool_read_OK        success
-                spool_read_notopen   open failed
-                spool_read_enverror  error in the envelope portion
-                spool_read_hdrerror  error in the header portion
-*/
-
-int
-spool_read_header(uschar *name, BOOL read_headers, BOOL subdir_set)
-{
-FILE *f = NULL;
-int n;
-int rcount = 0;
-long int uid, gid;
-BOOL inheader = FALSE;
-uschar *p;
-
 /* Reset all the global variables to their default values. However, there is
 one exception. DO NOT change the default value of dont_deliver, because it may
 be forced by an external setting. */
 
+void
+spool_clear_header_globals(void)
+{
 acl_var_c = acl_var_m = NULL;
 authenticated_id = NULL;
 authenticated_sender = NULL;
@@ -328,6 +292,53 @@ message_utf8_downconvert = 0;
 
 dsn_ret = 0;
 dsn_envid = NULL;
+}
+
+
+/*************************************************
+*             Read spool header file             *
+*************************************************/
+
+/* This function reads a spool header file and places the data into the
+appropriate global variables. The header portion is always read, but header
+structures are built only if read_headers is set true. It isn't, for example,
+while generating -bp output.
+
+It may be possible for blocks of nulls (binary zeroes) to get written on the
+end of a file if there is a system crash during writing. It was observed on an
+earlier version of Exim that omitted to fsync() the files - this is thought to
+have been the cause of that incident, but in any case, this code must be robust
+against such an event, and if such a file is encountered, it must be treated as
+malformed.
+
+As called from deliver_message() (at least) we are running as root.
+
+Arguments:
+  name          name of the header file, including the -H
+  read_headers  TRUE if in-store header structures are to be built
+  subdir_set    TRUE is message_subdir is already set
+
+Returns:        spool_read_OK        success
+                spool_read_notopen   open failed
+                spool_read_enverror  error in the envelope portion
+                spool_read_hdrerror  error in the header portion
+*/
+
+int
+spool_read_header(uschar *name, BOOL read_headers, BOOL subdir_set)
+{
+FILE *f = NULL;
+int n;
+int rcount = 0;
+long int uid, gid;
+BOOL inheader = FALSE;
+uschar *p;
+
+/* Reset all the global variables to their default values. However, there is
+one exception. DO NOT change the default value of dont_deliver, because it may
+be forced by an external setting. */
+
+spool_clear_header_globals();
 
 /* Generate the full name and open the file. If message_subdir is already
 set, just look in the given directory. Otherwise, look in both the split

commit b9aa0a847979a8b7e917c25a734c4e176c52be59
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Mar 15 14:23:04 2018 +0000

    Mark variables that are unused before release of store in the receive message loop

diff --git a/src/src/acl.c b/src/src/acl.c
index 95ee2667..23c28c73 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -4475,7 +4475,7 @@ switch (where)
   }
 
 deliver_domain = deliver_localpart = deliver_address_data =
-  sender_address_data = NULL;
+  deliver_domain_data = sender_address_data = NULL;
 
 /* A DISCARD response is permitted only for message ACLs, excluding the PREDATA
 ACL, which is really in the middle of an SMTP command. */
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 8bbcafb1..a96b4e96 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -1987,8 +1987,8 @@ active_local_sender_retain = local_sender_retain;    /* Can be set by ACL */
 sending_ip_address = NULL;
 return_path = sender_address = NULL;
 sender_data = NULL;				     /* Can be set by ACL */
-deliver_localpart_orig = NULL;
-deliver_domain_orig = NULL;
+deliver_localpart_parent = deliver_localpart_orig = NULL;
+deliver_domain_parent = deliver_domain_orig = NULL;
 callout_address = NULL;
 submission_name = NULL;                              /* Can be set by ACL */
 raw_sender = NULL;                  /* After SMTP rewrite, before qualifying */

commit aea78c24dfc83958336dc3cb3418cea5ca8d221b
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Mar 17 15:19:08 2018 +0000

    DMARC: add variables to list of those now-unused at the tail of the SMTP per-message loop

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index a96b4e96..527d3706 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2016,6 +2016,12 @@ prdr_requested = FALSE;
 #ifdef EXPERIMENTAL_SPF
 spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 #endif
+#ifdef EXPERIMENTAL_DMARC
+dmarc_has_been_checked = dmarc_disable_verify = dmarc_enable_forensic = FALSE;
+dmarc_ar_header = dmarc_domain_policy = dmarc_forensic_sender =
+dmarc_history_file = dmarc_status = dmarc_status_text =
+dmarc_tld_file = dmarc_used_domain = NULL;
+#endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;
 #endif

commit e06f773b7024ad7025fbb3dab2a7c073746d431f
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Mar 17 23:55:45 2018 +0000

    Fix heavy-pipeline SMTP command input corruption.  Bug 2250

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 527d3706..88bd9391 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -359,9 +359,6 @@ rc = smtp_getc(GETC_BUFFER_UNLIMITED);
 if (rc < 0) return TRUE;      /* End of file or error */
 
 smtp_ungetc(rc);
-rc = smtp_inend - smtp_inptr;
-if (rc > 150) rc = 150;
-smtp_inptr[rc] = 0;
 return FALSE;
 }
 
@@ -439,9 +436,10 @@ if (!smtp_out) return FALSE;
 fflush(smtp_out);
 if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout);
 
-/* Limit amount read, so non-message data is not fed to DKIM */
+/* Limit amount read, so non-message data is not fed to DKIM.
+Take care to not touch the safety NUL at the end of the buffer. */
 
-rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE, lim));
+rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE-1, lim));
 save_errno = errno;
 alarm(0);
 if (rc <= 0)
@@ -2429,10 +2427,12 @@ else
     (sender_host_address ? protocols : protocols_local) [pnormal];
 
 /* Set up the buffer for inputting using direct read() calls, and arrange to
-call the local functions instead of the standard C ones. */
+call the local functions instead of the standard C ones.  Place a NUL at the
+end of the buffer to safety-stop C-string reads from it. */
 
 if (!(smtp_inbuffer = US malloc(IN_BUFFER_SIZE)))
   log_write(0, LOG_MAIN|LOG_PANIC_DIE, "malloc() failed for SMTP input buffer");
+smtp_inbuffer[IN_BUFFER_SIZE-1] = '\0';
 
 receive_getc = smtp_getc;
 receive_getbuf = smtp_getbuf;
@@ -5684,7 +5684,7 @@ while (done <= 0)
     if (smtp_inend >= smtp_inbuffer + IN_BUFFER_SIZE)
       smtp_inend = smtp_inbuffer + IN_BUFFER_SIZE - 1;
     c = smtp_inend - smtp_inptr;
-    if (c > 150) c = 150;
+    if (c > 150) c = 150;	/* limit logged amount */
     smtp_inptr[c] = 0;
     incomplete_transaction_log(US"sync failure");
     log_write(0, LOG_MAIN|LOG_REJECT, "SMTP protocol synchronization error "

commit e863ac56f091041d8bf86acbb1ebb682440712ee
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Mar 18 18:48:13 2018 +0000

    Unbreak DMARC
    
    Broken-by: aea78c24df

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 88bd9391..d866886d 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2018,7 +2018,7 @@ spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 dmarc_has_been_checked = dmarc_disable_verify = dmarc_enable_forensic = FALSE;
 dmarc_ar_header = dmarc_domain_policy = dmarc_forensic_sender =
 dmarc_history_file = dmarc_status = dmarc_status_text =
-dmarc_tld_file = dmarc_used_domain = NULL;
+dmarc_used_domain = NULL;
 #endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;

commit 88fc7b4382a3e98948e085dd7581dd80b801bca4
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 20 17:54:47 2018 +0000

    Fix pipe transport to not use a socket-only syscall.  Bug 2257
    
    Broken-by: 42055a3385

diff --git a/src/src/macros.h b/src/src/macros.h
index 5f9c7422..242ef37f 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -853,6 +853,7 @@ enum {
 #define topt_use_bdat		0x100  /* prepend chunks with RFC3030 BDAT header */
 #define topt_output_string	0x200  /* create string rather than write to fd */
 #define topt_continuation	0x400  /* do not reset buffer */
+#define topt_not_socket		0x800  /* cannot do socket-only syscalls */
 
 /* Options for smtp_write_command */
 
diff --git a/src/src/transport.c b/src/src/transport.c
index 47da45fd..74103ef4 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -245,7 +245,8 @@ for (i = 0; i < 100; i++)
 	tls_out.active == fd ? tls_write(FALSE, block, len, more) :
 #endif
 #ifdef MSG_MORE
-	more ? send(fd, block, len, MSG_MORE) :
+	more && !(tctx->options & topt_not_socket)
+	  ? send(fd, block, len, MSG_MORE) :
 #endif
 	write(fd, block, len);
     save_errno = errno;
diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c
index ece1da51..6350445c 100644
--- a/src/src/transports/appendfile.c
+++ b/src/src/transports/appendfile.c
@@ -945,9 +945,7 @@ copy_mbx_message(int to_fd, int from_fd, off_t saved_size)
 int used;
 off_t size;
 struct stat statbuf;
-transport_ctx tctx = {{0}};
-
-tctx.u.fd = to_fd;
+transport_ctx tctx = { .u={.fd = to_fd}, .options = topt_not_socket };
 
 /* If the current mailbox size is zero, write a header block */
 
@@ -2921,12 +2919,12 @@ at initialization time. */
 if (yield == OK)
   {
   transport_ctx tctx = {
-    {fd},
-    tblock,
-    addr,
-    ob->check_string,
-    ob->escape_string,
-    ob->options
+    .u = {.fd=fd},
+    .tblock = tblock,
+    .addr = addr,
+    .check_string = ob->check_string,
+    .escape_string = ob->escape_string,
+    .options =  ob->options | topt_not_socket
   };
   if (!transport_write_message(&tctx, 0))
     yield = DEFER;
diff --git a/src/src/transports/autoreply.c b/src/src/transports/autoreply.c
index bf31951f..2ac06caa 100644
--- a/src/src/transports/autoreply.c
+++ b/src/src/transports/autoreply.c
@@ -694,15 +694,17 @@ if (return_message)
     :
     US"------ This is a copy of the message, including all the headers.\n";
   transport_ctx tctx = {
-    {fileno(f)},
-    tblock,
-    addr,
-    NULL, NULL,
-    (tblock->body_only ? topt_no_headers : 0) |
-    (tblock->headers_only ? topt_no_body : 0) |
-    (tblock->return_path_add ? topt_add_return_path : 0) |
-    (tblock->delivery_date_add ? topt_add_delivery_date : 0) |
-    (tblock->envelope_to_add ? topt_add_envelope_to : 0)
+    .u = {.fd = fileno(f)},
+    .tblock = tblock,
+    .addr = addr,
+    .check_string = NULL,
+    .escape_string =  NULL,
+    .options = (tblock->body_only ? topt_no_headers : 0)
+    	| (tblock->headers_only ? topt_no_body : 0)
+    	| (tblock->return_path_add ? topt_add_return_path : 0)
+    	| (tblock->delivery_date_add ? topt_add_delivery_date : 0)
+    	| (tblock->envelope_to_add ? topt_add_envelope_to : 0)
+	| topt_not_socket
   };
 
   if (bounce_return_size_limit > 0 && !tblock->headers_only)
diff --git a/src/src/transports/pipe.c b/src/src/transports/pipe.c
index 0361cc81..81327c6d 100644
--- a/src/src/transports/pipe.c
+++ b/src/src/transports/pipe.c
@@ -566,12 +566,11 @@ const uschar *envlist = ob->environment;
 uschar *cmd, *ss;
 uschar *eol = ob->use_crlf ? US"\r\n" : US"\n";
 transport_ctx tctx = {
-  {0},
-  tblock,
-  addr,
-  ob->check_string,
-  ob->escape_string,
-  ob->options /* set at initialization time */
+  .tblock = tblock,
+  .addr = addr,
+  .check_string = ob->check_string,
+  .escape_string = ob->escape_string,
+  ob->options | topt_not_socket /* set at initialization time */
 };
 
 DEBUG(D_transport) debug_printf("%s transport entered\n", tblock->name);

commit f81d6431fae4191b8851fba9345c4b0ed22d5650
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Mar 21 11:34:22 2018 +0000

    Pipe transport, part two.  Bug 2257

diff --git a/src/src/transport.c b/src/src/transport.c
index 74103ef4..d7670e0c 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -263,7 +263,8 @@ for (i = 0; i < 100; i++)
 	tls_out.active == fd ? tls_write(FALSE, block, len, more) :
 #endif
 #ifdef MSG_MORE
-	more ? send(fd, block, len, MSG_MORE) :
+	more && !(tctx->options & topt_not_socket)
+	  ? send(fd, block, len, MSG_MORE) :
 #endif
 	write(fd, block, len);
 

commit e85aad604e8dba48dd9f2dbe9644ca8a46f3137c
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Fri Mar 23 12:18:53 2018 +0000

    Fix spool_wireformat final-dot on LMTP transport.  Bug 2258
    
    Broken-by: 328c5688db

diff --git a/src/src/transport.c b/src/src/transport.c
index d7670e0c..d35d25f6 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -1132,9 +1132,10 @@ if (!(tctx->options & topt_no_body))
   if (len != 0) return FALSE;
   }
 
-/* Finished with the check string */
+/* Finished with the check string, and spool-format consideration */
 
 nl_check_length = nl_escape_length = 0;
+spool_file_wireformat = FALSE;
 
 /* If requested, add a terminating "." line (SMTP output). */
 
@@ -1401,6 +1402,7 @@ filter was not NL, insert a NL to make the SMTP protocol work. */
 if (yield)
   {
   nl_check_length = nl_escape_length = 0;
+  spool_file_wireformat = FALSE;
   if (  tctx->options & topt_end_dot
      && ( last_filter_was_NL
         ? !write_chunk(tctx, US".\n", 2)

commit aabb09d27607fcda9ec49ad28994ceaf3bc65977
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Mon Mar 26 15:53:49 2018 +0100

    Cutthrough: enforce non-use in combination with DKIM signing or transport filter
    
    Broken-by: 02b41d7106

diff --git a/src/src/verify.c b/src/src/verify.c
index 544069bb..6e010528 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -1026,6 +1026,20 @@ no_conn:
     here is where we want to leave the conn open.  Ditto for a lazy-close
     verify. */
 
+    if (cutthrough.delivery)
+      {
+      if (addr->transport->filter_command)
+        {
+        cutthrough.delivery= FALSE;
+        HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n");
+        }
+      if (ob->dkim.dkim_domain)
+        {
+        cutthrough.delivery= FALSE;
+        HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n");
+        }
+      }
+
     if (  (cutthrough.delivery || options & vopt_callout_hold)
        && rcpt_count == 1
        && done
