/[pcre]/code/trunk/pcre_compile.c

Diff of /code/trunk/pcre_compile.c

Parent Directory | Revision Log | View Patch Patch

-revision 150 by ph10,
Tue Apr 17 08:22:40 2007 UTC
+revision 240 by ph10,
Tue Sep 11 15:47:20 2007 UTC
 Line 42 
 POSSIBILITY OF SUCH DAMAGE.
  supporting internal functions that are not used by other modules. */
+ #ifdef HAVE_CONFIG_H
+ #include "config.h"
+ #endif
  #define NLBLOCK cd             /* Block containing newline information */
  #define PSSTART start_pattern  /* Field containing processed string start */
  #define PSEND   end_pattern    /* Field containing processed string end */
  #include "pcre_internal.h"
-Line 58 
 used by pcretest. DEBUG is not defined w
+Line 61 
 used by pcretest. DEBUG is not defined w
  #endif
+ /* Macro for setting individual bits in class bitmaps. */
+ #define SETBIT(a,b) a[b/8] |= (1 << (b%8))
+ /* Maximum length value to check against when making sure that the integer that
+ holds the compiled pattern length does not overflow. We make it a bit less than
+ INT_MAX to allow for adding in group terminating bytes, so that we don't have
+ to check them every time. */
+ #define OFLOW_MAX (INT_MAX - 20)
  /*************************************************
  *      Code parameters and static tables         *
  *************************************************/
-Line 87 
 static const short int escapes[] = {
+Line 102 
 static const short int escapes[] = {
 ,      0,      0,      0,      0,      0,      0,      0,   /* 0 - 7 */
 ,      0,    ':',    ';',    '<',    '=',    '>',    '?',   /* 8 - ? */
     '@', -ESC_A, -ESC_B, -ESC_C, -ESC_D, -ESC_E,      0, -ESC_G,   /* @ - G */
-,      0,      0,      0,      0,      0,      0,      0,   /* H - O */
+ -ESC_H,      0,      0, -ESC_K,      0,      0,      0,      0,   /* H - O */
- -ESC_P, -ESC_Q, -ESC_R, -ESC_S,      0,      0,      0, -ESC_W,   /* P - W */
+ -ESC_P, -ESC_Q, -ESC_R, -ESC_S,      0,      0, -ESC_V, -ESC_W,   /* P - W */
  -ESC_X,      0, -ESC_Z,    '[',   '\\',    ']',    '^',    '_',   /* X - _ */
     '`',      7, -ESC_b,      0, -ESC_d,  ESC_e,  ESC_f,      0,   /* ` - g */
-,      0,      0, -ESC_k,      0,      0,  ESC_n,      0,   /* h - o */
+ -ESC_h,      0,      0, -ESC_k,      0,      0,  ESC_n,      0,   /* h - o */
- -ESC_p,      0,  ESC_r, -ESC_s,  ESC_tee,    0,      0, -ESC_w,   /* p - w */
+ -ESC_p,      0,  ESC_r, -ESC_s,  ESC_tee,    0, -ESC_v, -ESC_w,   /* p - w */
 ,      0, -ESC_z                                            /* x - z */
  };
-Line 106 
 static const short int escapes[] = {
+Line 121 
 static const short int escapes[] = {
  /*  70 */     0,     0,      0,       0,      0,     0,      0,      0,
  /*  78 */     0,   '`',    ':',     '#',    '@',  '\'',    '=',    '"',
  /*  80 */     0,     7, -ESC_b,       0, -ESC_d, ESC_e,  ESC_f,      0,
- /*  88 */     0,     0,      0,     '{',      0,     0,      0,      0,
+ /*  88 */-ESC_h,     0,      0,     '{',      0,     0,      0,      0,
  /*  90 */     0,     0, -ESC_k,     'l',      0, ESC_n,      0, -ESC_p,
  /*  98 */     0, ESC_r,      0,     '}',      0,     0,      0,      0,
- /*  A0 */     0,   '~', -ESC_s, ESC_tee,      0,     0, -ESC_w,      0,
+ /*  A0 */     0,   '~', -ESC_s, ESC_tee,      0,-ESC_v, -ESC_w,      0,
  /*  A8 */     0,-ESC_z,      0,       0,      0,   '[',      0,      0,
  /*  B0 */     0,     0,      0,       0,      0,     0,      0,      0,
  /*  B8 */     0,     0,      0,       0,      0,   ']',    '=',    '-',
  /*  C0 */   '{',-ESC_A, -ESC_B,  -ESC_C, -ESC_D,-ESC_E,      0, -ESC_G,
- /*  C8 */     0,     0,      0,       0,      0,     0,      0,      0,
+ /*  C8 */-ESC_H,     0,      0,       0,      0,     0,      0,      0,
- /*  D0 */   '}',     0,      0,       0,      0,     0,      0, -ESC_P,
+ /*  D0 */   '}',     0, -ESC_K,       0,      0,     0,      0, -ESC_P,
  /*  D8 */-ESC_Q,-ESC_R,      0,       0,      0,     0,      0,      0,
- /*  E0 */  '\\',     0, -ESC_S,       0,      0,     0, -ESC_W, -ESC_X,
+ /*  E0 */  '\\',     0, -ESC_S,       0,      0,-ESC_V, -ESC_W, -ESC_X,
  /*  E8 */     0,-ESC_Z,      0,       0,      0,     0,      0,      0,
  /*  F0 */     0,     0,      0,       0,      0,     0,      0,      0,
  /*  F8 */     0,     0,      0,       0,      0,     0,      0,      0
-Line 125 
 static const short int escapes[] = {
+Line 140 
 static const short int escapes[] = {
  #endif
- /* Tables of names of POSIX character classes and their lengths. The list is
+ /* Table of special "verbs" like (*PRUNE). This is a short table, so it is
- terminated by a zero length entry. The first three must be alpha, lower, upper,
+ searched linearly. Put all the names into a single string, in order to reduce
- as this is assumed for handling case independence. */
+ the number of relocations when a shared library is dynamically linked. */
- static const char *const posix_names[] = {
+ typedef struct verbitem {
-   "alpha", "lower", "upper",
+   int   len;
-   "alnum", "ascii", "blank", "cntrl", "digit", "graph",
+   int   op;
-   "print", "punct", "space", "word",  "xdigit" };
+ } verbitem;
+ static const char verbnames[] =
+   "ACCEPT\0"
+   "COMMIT\0"
+   "F\0"
+   "FAIL\0"
+   "PRUNE\0"
+   "SKIP\0"
+   "THEN";
+ static verbitem verbs[] = {
+   { 6, OP_ACCEPT },
+   { 6, OP_COMMIT },
+   { 1, OP_FAIL },
+   { 4, OP_FAIL },
+   { 5, OP_PRUNE },
+   { 4, OP_SKIP  },
+   { 4, OP_THEN  }
+ };
+ static int verbcount = sizeof(verbs)/sizeof(verbitem);
+ /* Tables of names of POSIX character classes and their lengths. The names are
+ now all in a single string, to reduce the number of relocations when a shared
+ library is dynamically loaded. The list of lengths is terminated by a zero
+ length entry. The first three must be alpha, lower, upper, as this is assumed
+ for handling case independence. */
+ static const char posix_names[] =
+   "alpha\0"  "lower\0"  "upper\0"  "alnum\0"  "ascii\0"  "blank\0"
+   "cntrl\0"  "digit\0"  "graph\0"  "print\0"  "punct\0"  "space\0"
+   "word\0"   "xdigit";
  static const uschar posix_name_lengths[] = {
 , 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 };
-Line 171 
 static const int posix_class_maps[] = {
+Line 219 
 static const int posix_class_maps[] = {
  /* The texts of compile-time error messages. These are "char *" because they
  are passed to the outside world. Do not ever re-use any error number, because
  they are documented. Always add a new error instead. Messages marked DEAD below
- are no longer used. */
+ are no longer used. This used to be a table of strings, but in order to reduce
+ the number of relocations needed when a shared library is loaded dynamically,
- static const char *error_texts[] = {
+ it is now one long string. We cannot use a table of offsets, because the
-   "no error",
+ lengths of inserts such as XSTRING(MAX_NAME_SIZE) are not known. Instead, we
-   "\\ at end of pattern",
+ simply count through to the one we want - this isn't a performance issue
-   "\\c at end of pattern",
+ because these strings are used only when there is a compilation error. */
-   "unrecognized character follows \\",
-   "numbers out of order in {} quantifier",
+ static const char error_texts[] =
+   "no error\0"
+   "\\ at end of pattern\0"
+   "\\c at end of pattern\0"
+   "unrecognized character follows \\\0"
+   "numbers out of order in {} quantifier\0"
    /* 5 */
-   "number too big in {} quantifier",
+   "number too big in {} quantifier\0"
-   "missing terminating ] for character class",
+   "missing terminating ] for character class\0"
-   "invalid escape sequence in character class",
+   "invalid escape sequence in character class\0"
-   "range out of order in character class",
+   "range out of order in character class\0"
-   "nothing to repeat",
+   "nothing to repeat\0"
    /* 10 */
-   "operand of unlimited repeat could match the empty string",  /** DEAD **/
+   "operand of unlimited repeat could match the empty string\0"  /** DEAD **/
-   "internal error: unexpected repeat",
+   "internal error: unexpected repeat\0"
-   "unrecognized character after (?",
+   "unrecognized character after (?\0"
-   "POSIX named classes are supported only within a class",
+   "POSIX named classes are supported only within a class\0"
-   "missing )",
+   "missing )\0"
    /* 15 */
-   "reference to non-existent subpattern",
+   "reference to non-existent subpattern\0"
-   "erroffset passed as NULL",
+   "erroffset passed as NULL\0"
-   "unknown option bit(s) set",
+   "unknown option bit(s) set\0"
-   "missing ) after comment",
+   "missing ) after comment\0"
-   "parentheses nested too deeply",  /** DEAD **/
+   "parentheses nested too deeply\0"  /** DEAD **/
    /* 20 */
-   "regular expression too large",
+   "regular expression is too large\0"
-   "failed to get memory",
+   "failed to get memory\0"
-   "unmatched parentheses",
+   "unmatched parentheses\0"
-   "internal error: code overflow",
+   "internal error: code overflow\0"
-   "unrecognized character after (?<",
+   "unrecognized character after (?<\0"
    /* 25 */
-   "lookbehind assertion is not fixed length",
+   "lookbehind assertion is not fixed length\0"
-   "malformed number or name after (?(",
+   "malformed number or name after (?(\0"
-   "conditional group contains more than two branches",
+   "conditional group contains more than two branches\0"
-   "assertion expected after (?(",
+   "assertion expected after (?(\0"
-   "(?R or (?digits must be followed by )",
+   "(?R or (?[+-]digits must be followed by )\0"
    /* 30 */
-   "unknown POSIX class name",
+   "unknown POSIX class name\0"
-   "POSIX collating elements are not supported",
+   "POSIX collating elements are not supported\0"
-   "this version of PCRE is not compiled with PCRE_UTF8 support",
+   "this version of PCRE is not compiled with PCRE_UTF8 support\0"
-   "spare error",  /** DEAD **/
+   "spare error\0"  /** DEAD **/
-   "character value in \\x{...} sequence is too large",
+   "character value in \\x{...} sequence is too large\0"
    /* 35 */
-   "invalid condition (?(0)",
+   "invalid condition (?(0)\0"
-   "\\C not allowed in lookbehind assertion",
+   "\\C not allowed in lookbehind assertion\0"
-   "PCRE does not support \\L, \\l, \\N, \\U, or \\u",
+   "PCRE does not support \\L, \\l, \\N, \\U, or \\u\0"
-   "number after (?C is > 255",
+   "number after (?C is > 255\0"
-   "closing ) for (?C expected",
+   "closing ) for (?C expected\0"
    /* 40 */
-   "recursive call could loop indefinitely",
+   "recursive call could loop indefinitely\0"
-   "unrecognized character after (?P",
+   "unrecognized character after (?P\0"
-   "syntax error in subpattern name (missing terminator)",
+   "syntax error in subpattern name (missing terminator)\0"
-   "two named subpatterns have the same name",
+   "two named subpatterns have the same name\0"
-   "invalid UTF-8 string",
+   "invalid UTF-8 string\0"
    /* 45 */
-   "support for \\P, \\p, and \\X has not been compiled",
+   "support for \\P, \\p, and \\X has not been compiled\0"
-   "malformed \\P or \\p sequence",
+   "malformed \\P or \\p sequence\0"
-   "unknown property name after \\P or \\p",
+   "unknown property name after \\P or \\p\0"
-   "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " characters)",
+   "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " characters)\0"
-   "too many named subpatterns (maximum " XSTRING(MAX_NAME_COUNT) ")",
+   "too many named subpatterns (maximum " XSTRING(MAX_NAME_COUNT) ")\0"
    /* 50 */
-   "repeated subpattern is too long",
+   "repeated subpattern is too long\0"    /** DEAD **/
-   "octal value is greater than \\377 (not in UTF-8 mode)",
+   "octal value is greater than \\377 (not in UTF-8 mode)\0"
-   "internal error: overran compiling workspace",
+   "internal error: overran compiling workspace\0"
-   "internal error: previously-checked referenced subpattern not found",
+   "internal error: previously-checked referenced subpattern not found\0"
-   "DEFINE group contains more than one branch",
+   "DEFINE group contains more than one branch\0"
    /* 55 */
-   "repeating a DEFINE group is not allowed",
+   "repeating a DEFINE group is not allowed\0"
-   "inconsistent NEWLINE options",
+   "inconsistent NEWLINE options\0"
-   "\\g is not followed by an (optionally braced) non-zero number"
+   "\\g is not followed by a braced name or an optionally braced non-zero number\0"
- };
+   "(?+ or (?- or (?(+ or (?(- must be followed by a non-zero number\0"
+   "(*VERB) with an argument is not supported\0"
+   /* 60 */
+   "(*VERB) not recognized\0"
+   "number is too big";
  /* Table to identify digits and hex digits. This is used when compiling
-Line 373 
 static const unsigned char ebcdic_charta
+Line 430 
 static const unsigned char ebcdic_charta
  /* Definition to allow mutual recursion */
  static BOOL
-   compile_regex(int, int, uschar **, const uschar **, int *, BOOL, int, int *,
+   compile_regex(int, int, uschar **, const uschar **, int *, BOOL, BOOL, int,
-     int *, branch_chain *, compile_data *, int *);
+     int *, int *, branch_chain *, compile_data *, int *);
  /*************************************************
+ *            Find an error text                  *
+ *************************************************/
+ /* The error texts are now all in one long string, to save on relocations. As
+ some of the text is of unknown length, we can't use a table of offsets.
+ Instead, just count through the strings. This is not a performance issue
+ because it happens only when there has been a compilation error.
+ Argument:   the error number
+ Returns:    pointer to the error string
+ */
+ static const char *
+ find_error_text(int n)
+ {
+ const char *s = error_texts;
+ for (; n > 0; n--) while (*s++ != 0);
+ return s;
+ }
+ /*************************************************
  *            Handle escapes                      *
  *************************************************/
-Line 399 
 Arguments:
+Line 478 
 Arguments:
  Returns:         zero or positive => a data character
                   negative => a special escape sequence
-                  on error, errorptr is set
+                  on error, errorcodeptr is set
  */
  static int
-Line 452 
 else
+Line 531 
 else
      /* \g must be followed by a number, either plain or braced. If positive, it
      is an absolute backreference. If negative, it is a relative backreference.
-     This is a Perl 5.10 feature. */
+     This is a Perl 5.10 feature. Perl 5.10 also supports \g{name} as a
+     reference to a named group. This is part of Perl's movement towards a
+     unified syntax for back references. As this is synonymous with \k{name}, we
+     fudge it up by pretending it really was \k. */
      case 'g':
      if (ptr[1] == '{')
        {
+       const uschar *p;
+       for (p = ptr+2; *p != 0 && *p != '}'; p++)
+         if (*p != '-' && (digitab[*p] & ctype_digit) == 0) break;
+       if (*p != 0 && *p != '}')
+         {
+         c = -ESC_k;
+         break;
+         }
        braced = TRUE;
        ptr++;
        }
-Line 473 
 else
+Line 563 
 else
      while ((digitab[ptr[1]] & ctype_digit) != 0)
        c = c * 10 + *(++ptr) - '0';
+     if (c < 0)
+       {
+       *errorcodeptr = ERR61;
+       break;
+       }
      if (c == 0 || (braced && *(++ptr) != '}'))
        {
        *errorcodeptr = ERR57;
-       return 0;
+       break;
        }
      if (negated)
-Line 484 
 else
+Line 580 
 else
        if (c > bracount)
          {
          *errorcodeptr = ERR15;
-         return 0;
+         break;
          }
        c = bracount - (c - 1);
        }
-Line 513 
 else
+Line 609 
 else
        c -= '0';
        while ((digitab[ptr[1]] & ctype_digit) != 0)
          c = c * 10 + *(++ptr) - '0';
+       if (c < 0)
+         {
+         *errorcodeptr = ERR61;
+         break;
+         }
        if (c < 10 || c <= bracount)
          {
          c = -(ESC_REF + c);
-Line 608 
 else
+Line 709 
 else
      if (c == 0)
        {
        *errorcodeptr = ERR2;
-       return 0;
+       break;
        }
  #ifndef EBCDIC  /* ASCII coding */
-Line 684 
 if (c == '{')
+Line 785 
 if (c == '{')
      *negptr = TRUE;
      ptr++;
      }
-   for (i = 0; i < sizeof(name) - 1; i++)
+   for (i = 0; i < (int)sizeof(name) - 1; i++)
      {
      c = *(++ptr);
      if (c == 0) goto ERROR_RETURN;
-Line 713 
 top = _pcre_utt_size;
+Line 814 
 top = _pcre_utt_size;
  while (bot < top)
    {
    i = (bot + top) >> 1;
-   c = strcmp(name, _pcre_utt[i].name);
+   c = strcmp(name, _pcre_utt_names + _pcre_utt[i].name_offset);
    if (c == 0)
      {
      *dptr = _pcre_utt[i].value;
-Line 887 
 for (; *ptr != 0; ptr++)
+Line 988 
 for (; *ptr != 0; ptr++)
      {
      while (*(++ptr) != ']')
        {
+       if (*ptr == 0) return -1;
        if (*ptr == '\\')
          {
          if (*(++ptr) == 0) return -1;
-Line 914 
 for (; *ptr != 0; ptr++)
+Line 1016 
 for (; *ptr != 0; ptr++)
    /* An opening parens must now be a real metacharacter */
    if (*ptr != '(') continue;
-   if (ptr[1] != '?')
+   if (ptr[1] != '?' && ptr[1] != '*')
      {
      count++;
      if (name == NULL && count == lorn) return count;
-Line 1042 
 for (;;)
+Line 1144 
 for (;;)
    {
    int d;
    register int op = *cc;
    switch (op)
      {
      case OP_CBRA:
-Line 1131 
 for (;;)
+Line 1232 
 for (;;)
      case OP_TYPEEXACT:
      branchlength += GET2(cc,1);
+     if (cc[3] == OP_PROP || cc[3] == OP_NOTPROP) cc += 2;
      cc += 4;
      break;
-Line 1239 
 for (;;)
+Line 1341 
 for (;;)
      code += _pcre_OP_lengths[c];
      }
-   /* In UTF-8 mode, opcodes that are followed by a character may be followed by
+   /* Otherwise, we can get the item's length from the table, except that for
-   a multi-byte character. The length in the table is a minimum, so we have to
+   repeated character types, we have to test for \p and \P, which have an extra
-   arrange to skip the extra bytes. */
+   two bytes of parameters. */
    else
      {
+     switch(c)
+       {
+       case OP_TYPESTAR:
+       case OP_TYPEMINSTAR:
+       case OP_TYPEPLUS:
+       case OP_TYPEMINPLUS:
+       case OP_TYPEQUERY:
+       case OP_TYPEMINQUERY:
+       case OP_TYPEPOSSTAR:
+       case OP_TYPEPOSPLUS:
+       case OP_TYPEPOSQUERY:
+       if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2;
+       break;
+       case OP_TYPEUPTO:
+       case OP_TYPEMINUPTO:
+       case OP_TYPEEXACT:
+       case OP_TYPEPOSUPTO:
+       if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;
+       break;
+       }
+     /* Add in the fixed length from the table */
      code += _pcre_OP_lengths[c];
+   /* In UTF-8 mode, opcodes that are followed by a character may be followed by
+   a multi-byte character. The length in the table is a minimum, so we have to
+   arrange to skip the extra bytes. */
  #ifdef SUPPORT_UTF8
      if (utf8) switch(c)
        {
-Line 1303 
 for (;;)
+Line 1434 
 for (;;)
    if (c == OP_XCLASS) code += GET(code, 1);
-   /* Otherwise, we get the item's length from the table. In UTF-8 mode, opcodes
+   /* Otherwise, we can get the item's length from the table, except that for
-   that are followed by a character may be followed by a multi-byte character.
+   repeated character types, we have to test for \p and \P, which have an extra
-   The length in the table is a minimum, so we have to arrange to skip the extra
+   two bytes of parameters. */
-   bytes. */
    else
      {
+     switch(c)
+       {
+       case OP_TYPESTAR:
+       case OP_TYPEMINSTAR:
+       case OP_TYPEPLUS:
+       case OP_TYPEMINPLUS:
+       case OP_TYPEQUERY:
+       case OP_TYPEMINQUERY:
+       case OP_TYPEPOSSTAR:
+       case OP_TYPEPOSPLUS:
+       case OP_TYPEPOSQUERY:
+       if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2;
+       break;
+       case OP_TYPEPOSUPTO:
+       case OP_TYPEUPTO:
+       case OP_TYPEMINUPTO:
+       case OP_TYPEEXACT:
+       if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;
+       break;
+       }
+     /* Add in the fixed length from the table */
      code += _pcre_OP_lengths[c];
+     /* In UTF-8 mode, opcodes that are followed by a character may be followed
+     by a multi-byte character. The length in the table is a minimum, so we have
+     to arrange to skip the extra bytes. */
  #ifdef SUPPORT_UTF8
      if (utf8) switch(c)
        {
-Line 1370 
 for (code = first_significant_code(code
+Line 1529 
 for (code = first_significant_code(code
    c = *code;
-   if (c == OP_BRA || c == OP_CBRA || c == OP_ONCE)
+   /* Groups with zero repeats can of course be empty; skip them. */
+   if (c == OP_BRAZERO || c == OP_BRAMINZERO)
+     {
+     code += _pcre_OP_lengths[c];
+     do code += GET(code, 1); while (*code == OP_ALT);
+     c = *code;
+     continue;
+     }
+   /* For other groups, scan the branches. */
+   if (c == OP_BRA || c == OP_CBRA || c == OP_ONCE || c == OP_COND)
      {
      BOOL empty_branch;
      if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */
-Line 1386 
 for (code = first_significant_code(code
+Line 1557 
 for (code = first_significant_code(code
        }
      while (*code == OP_ALT);
      if (!empty_branch) return FALSE;   /* All branches are non-empty */
+     c = *code;
-     /* Move past the KET and fudge things so that the increment in the "for"
-     above has no effect. */
-     c = OP_END;
-     code += 1 + LINK_SIZE - _pcre_OP_lengths[c];
      continue;
      }
-Line 1399 
 for (code = first_significant_code(code
+Line 1565 
 for (code = first_significant_code(code
    switch (c)
      {
-     /* Check for quantifiers after a class */
+     /* Check for quantifiers after a class. XCLASS is used for classes that
+     cannot be represented just by a bit map. This includes negated single
+     high-valued characters. The length in _pcre_OP_lengths[] is zero; the
+     actual length is stored in the compiled code, so we must update "code"
+     here. */
  #ifdef SUPPORT_UTF8
      case OP_XCLASS:
-     ccode = code + GET(code, 1);
+     ccode = code += GET(code, 1);
      goto CHECK_CLASS_REPEAT;
  #endif
-Line 1465 
 for (code = first_significant_code(code
+Line 1635 
 for (code = first_significant_code(code
      case OP_TYPEEXACT:
      return FALSE;
+     /* These are going to continue, as they may be empty, but we have to
+     fudge the length for the \p and \P cases. */
+     case OP_TYPESTAR:
+     case OP_TYPEMINSTAR:
+     case OP_TYPEPOSSTAR:
+     case OP_TYPEQUERY:
+     case OP_TYPEMINQUERY:
+     case OP_TYPEPOSQUERY:
+     if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2;
+     break;
+     /* Same for these */
+     case OP_TYPEUPTO:
+     case OP_TYPEMINUPTO:
+     case OP_TYPEPOSUPTO:
+     if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;
+     break;
      /* End of branch */
      case OP_KET:
-Line 1581 
 Returns:     a value representing the na
+Line 1771 
 Returns:     a value representing the na
  static int
  check_posix_name(const uschar *ptr, int len)
  {
+ const char *pn = posix_names;
  register int yield = 0;
  while (posix_name_lengths[yield] != 0)
    {
    if (len == posix_name_lengths[yield] &&
-     strncmp((const char *)ptr, posix_names[yield], len) == 0) return yield;
+     strncmp((const char *)ptr, pn, len) == 0) return yield;
+   pn += posix_name_lengths[yield] + 1;
    yield++;
    }
  return -1;
-Line 1627 
 adjust_recurse(uschar *group, int adjust
+Line 1819 
 adjust_recurse(uschar *group, int adjust
    uschar *save_hwm)
  {
  uschar *ptr = group;
  while ((ptr = (uschar *)find_recurse(ptr, utf8)) != NULL)
    {
    int offset;
-Line 1925 
 if (next >= 0) switch(op_code)
+Line 2118 
 if (next >= 0) switch(op_code)
    case OP_NOT_WORDCHAR:
    return next <= 127 && (cd->ctypes[next] & ctype_word) != 0;
+   case OP_HSPACE:
+   case OP_NOT_HSPACE:
+   switch(next)
+     {
+     case 0x09:
+     case 0x20:
+     case 0xa0:
+     case 0x1680:
+     case 0x180e:
+     case 0x2000:
+     case 0x2001:
+     case 0x2002:
+     case 0x2003:
+     case 0x2004:
+     case 0x2005:
+     case 0x2006:
+     case 0x2007:
+     case 0x2008:
+     case 0x2009:
+     case 0x200A:
+     case 0x202f:
+     case 0x205f:
+     case 0x3000:
+     return op_code != OP_HSPACE;
+     default:
+     return op_code == OP_HSPACE;
+     }
+   case OP_VSPACE:
+   case OP_NOT_VSPACE:
+   switch(next)
+     {
+     case 0x0a:
+     case 0x0b:
+     case 0x0c:
+     case 0x0d:
+     case 0x85:
+     case 0x2028:
+     case 0x2029:
+     return op_code != OP_VSPACE;
+     default:
+     return op_code == OP_VSPACE;
+     }
    default:
    return FALSE;
    }
-Line 1959 
 switch(op_code)
+Line 2196 
 switch(op_code)
      case ESC_W:
      return item <= 127 && (cd->ctypes[item] & ctype_word) != 0;
+     case ESC_h:
+     case ESC_H:
+     switch(item)
+       {
+       case 0x09:
+       case 0x20:
+       case 0xa0:
+       case 0x1680:
+       case 0x180e:
+       case 0x2000:
+       case 0x2001:
+       case 0x2002:
+       case 0x2003:
+       case 0x2004:
+       case 0x2005:
+       case 0x2006:
+       case 0x2007:
+       case 0x2008:
+       case 0x2009:
+       case 0x200A:
+       case 0x202f:
+       case 0x205f:
+       case 0x3000:
+       return -next != ESC_h;
+       default:
+       return -next == ESC_h;
+       }
+     case ESC_v:
+     case ESC_V:
+     switch(item)
+       {
+       case 0x0a:
+       case 0x0b:
+       case 0x0c:
+       case 0x0d:
+       case 0x85:
+       case 0x2028:
+       case 0x2029:
+       return -next != ESC_v;
+       default:
+       return -next == ESC_v;
+       }
      default:
      return FALSE;
      }
    case OP_DIGIT:
-   return next == -ESC_D || next == -ESC_s || next == -ESC_W;
+   return next == -ESC_D || next == -ESC_s || next == -ESC_W ||
+          next == -ESC_h || next == -ESC_v;
    case OP_NOT_DIGIT:
    return next == -ESC_d;
-Line 1973 
 switch(op_code)
+Line 2255 
 switch(op_code)
    return next == -ESC_S || next == -ESC_d || next == -ESC_w;
    case OP_NOT_WHITESPACE:
-   return next == -ESC_s;
+   return next == -ESC_s || next == -ESC_h || next == -ESC_v;
+   case OP_HSPACE:
+   return next == -ESC_S || next == -ESC_H || next == -ESC_d || next == -ESC_w;
+   case OP_NOT_HSPACE:
+   return next == -ESC_h;
+   /* Can't have \S in here because VT matches \S (Perl anomaly) */
+   case OP_VSPACE:
+   return next == -ESC_V || next == -ESC_d || next == -ESC_w;
+   case OP_NOT_VSPACE:
+   return next == -ESC_v;
    case OP_WORDCHAR:
-   return next == -ESC_W || next == -ESC_s;
+   return next == -ESC_W || next == -ESC_s || next == -ESC_h || next == -ESC_v;
    case OP_NOT_WORDCHAR:
    return next == -ESC_w || next == -ESC_d;
-Line 2091 
 for (;; ptr++)
+Line 2386 
 for (;; ptr++)
    BOOL possessive_quantifier;
    BOOL is_quantifier;
    BOOL is_recurse;
+   BOOL reset_bracount;
    int class_charcount;
    int class_lastchar;
    int newoptions;
    int recno;
+   int refsign;
    int skipbytes;
    int subreqbyte;
    int subfirstbyte;
-Line 2127 
 for (;; ptr++)
+Line 2424 
 for (;; ptr++)
      */
      if (code < last_code) code = last_code;
+     /* Paranoid check for integer overflow */
+     if (OFLOW_MAX - *lengthptr < code - last_code)
+       {
+       *errorcodeptr = ERR20;
+       goto FAILED;
+       }
      *lengthptr += code - last_code;
      DPRINTF(("length=%d added %d c=%c\n", *lengthptr, code - last_code, c));
-Line 2239 
 for (;; ptr++)
+Line 2545 
 for (;; ptr++)
      *ptrptr = ptr;
      if (lengthptr != NULL)
        {
+       if (OFLOW_MAX - *lengthptr < code - last_code)
+         {
+         *errorcodeptr = ERR20;
+         goto FAILED;
+         }
        *lengthptr += code - last_code;   /* To include callout length */
        DPRINTF((">> end branch\n"));
        }
-Line 2301 
 for (;; ptr++)
+Line 2612 
 for (;; ptr++)
        goto FAILED;
        }
-     /* If the first character is '^', set the negation flag and skip it. */
+     /* If the first character is '^', set the negation flag and skip it. Also,
+     if the first few characters (either before or after ^) are \Q\E or \E we
+     skip them too. This makes for compatibility with Perl. */
-     if ((c = *(++ptr)) == '^')
+     negate_class = FALSE;
+     for (;;)
        {
-       negate_class = TRUE;
        c = *(++ptr);
-       }
+       if (c == '\\')
-     else
+         {
-       {
+         if (ptr[1] == 'E') ptr++;
-       negate_class = FALSE;
+           else if (strncmp((const char *)ptr+1, "Q\\E", 3) == 0) ptr += 3;
+             else break;
+         }
+       else if (!negate_class && c == '^')
+         negate_class = TRUE;
+       else break;
        }
      /* Keep a count of chars with values < 256 so that we can optimize the case
-Line 2451 
 for (;; ptr++)
+Line 2769 
 for (;; ptr++)
        of the specials, which just set a flag. The sequence \b is a special
        case. Inside a class (and only there) it is treated as backspace.
        Elsewhere it marks a word boundary. Other escapes have preset maps ready
-       to or into the one we are building. We assume they have more than one
+       to 'or' into the one we are building. We assume they have more than one
        character in them, so set class_charcount bigger than one. */
        if (c == '\\')
-Line 2471 
 for (;; ptr++)
+Line 2789 
 for (;; ptr++)
            else inescq = TRUE;
            continue;
            }
+         else if (-c == ESC_E) continue;  /* Ignore orphan \E */
          if (c < 0)
            {
-Line 2519 
 for (;; ptr++)
+Line 2838 
 for (;; ptr++)
            else if (c == -ESC_d || c == -ESC_D || c == -ESC_w ||
                     c == -ESC_W || c == -ESC_s || c == -ESC_S) continue;
+           /* We need to deal with \H, \h, \V, and \v in both phases because
+           they use extra memory. */
+           if (-c == ESC_h)
+             {
+             SETBIT(classbits, 0x09); /* VT */
+             SETBIT(classbits, 0x20); /* SPACE */
+             SETBIT(classbits, 0xa0); /* NSBP */
+ #ifdef SUPPORT_UTF8
+             if (utf8)
+               {
+               class_utf8 = TRUE;
+               *class_utf8data++ = XCL_SINGLE;
+               class_utf8data += _pcre_ord2utf8(0x1680, class_utf8data);
+               *class_utf8data++ = XCL_SINGLE;
+               class_utf8data += _pcre_ord2utf8(0x180e, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x2000, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x200A, class_utf8data);
+               *class_utf8data++ = XCL_SINGLE;
+               class_utf8data += _pcre_ord2utf8(0x202f, class_utf8data);
+               *class_utf8data++ = XCL_SINGLE;
+               class_utf8data += _pcre_ord2utf8(0x205f, class_utf8data);
+               *class_utf8data++ = XCL_SINGLE;
+               class_utf8data += _pcre_ord2utf8(0x3000, class_utf8data);
+               }
+ #endif
+             continue;
+             }
+           if (-c == ESC_H)
+             {
+             for (c = 0; c < 32; c++)
+               {
+               int x = 0xff;
+               switch (c)
+                 {
+                 case 0x09/8: x ^= 1 << (0x09%8); break;
+                 case 0x20/8: x ^= 1 << (0x20%8); break;
+                 case 0xa0/8: x ^= 1 << (0xa0%8); break;
+                 default: break;
+                 }
+               classbits[c] |= x;
+               }
+ #ifdef SUPPORT_UTF8
+             if (utf8)
+               {
+               class_utf8 = TRUE;
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x0100, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x167f, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x1681, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x180d, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x180f, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x1fff, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x200B, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x202e, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x2030, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x205e, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x2060, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x2fff, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x3001, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x7fffffff, class_utf8data);
+               }
+ #endif
+             continue;
+             }
+           if (-c == ESC_v)
+             {
+             SETBIT(classbits, 0x0a); /* LF */
+             SETBIT(classbits, 0x0b); /* VT */
+             SETBIT(classbits, 0x0c); /* FF */
+             SETBIT(classbits, 0x0d); /* CR */
+             SETBIT(classbits, 0x85); /* NEL */
+ #ifdef SUPPORT_UTF8
+             if (utf8)
+               {
+               class_utf8 = TRUE;
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x2028, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x2029, class_utf8data);
+               }
+ #endif
+             continue;
+             }
+           if (-c == ESC_V)
+             {
+             for (c = 0; c < 32; c++)
+               {
+               int x = 0xff;
+               switch (c)
+                 {
+                 case 0x0a/8: x ^= 1 << (0x0a%8);
+                              x ^= 1 << (0x0b%8);
+                              x ^= 1 << (0x0c%8);
+                              x ^= 1 << (0x0d%8);
+                              break;
+                 case 0x85/8: x ^= 1 << (0x85%8); break;
+                 default: break;
+                 }
+               classbits[c] |= x;
+               }
+ #ifdef SUPPORT_UTF8
+             if (utf8)
+               {
+               class_utf8 = TRUE;
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x0100, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x2027, class_utf8data);
+               *class_utf8data++ = XCL_RANGE;
+               class_utf8data += _pcre_ord2utf8(0x2029, class_utf8data);
+               class_utf8data += _pcre_ord2utf8(0x7fffffff, class_utf8data);
+               }
+ #endif
+             continue;
+             }
            /* We need to deal with \P and \p in both phases. */
  #ifdef SUPPORT_UCP
-Line 2570 
 for (;; ptr++)
+Line 3016 
 for (;; ptr++)
        oldptr = ptr;
+       /* Remember \r or \n */
+       if (c == '\r' || c == '\n') cd->external_flags |= PCRE_HASCRORLF;
+       /* Check for range */
        if (!inescq && ptr[1] == '-')
          {
          int d;
-Line 2637 
 for (;; ptr++)
+Line 3089 
 for (;; ptr++)
          if (d == c) goto LONE_SINGLE_CHARACTER;  /* A few lines below */
+         /* Remember \r or \n */
+         if (d == '\r' || d == '\n') cd->external_flags |= PCRE_HASCRORLF;
          /* In UTF-8 mode, if the upper limit is > 255, or > 127 for caseless
          matching, we have to use an XCLASS with extra data items. Caseless
          matching for characters > 127 is available only if UCP support is
-Line 2659 
 for (;; ptr++)
+Line 3115 
 for (;; ptr++)
              unsigned int origd = d;
              while (get_othercase_range(&cc, origd, &occ, &ocd))
                {
-               if (occ >= c && ocd <= d) continue;  /* Skip embedded ranges */
+               if (occ >= (unsigned int)c &&
+                   ocd <= (unsigned int)d)
+                 continue;                          /* Skip embedded ranges */
-               if (occ < c  && ocd >= c - 1)        /* Extend the basic range */
+               if (occ < (unsigned int)c  &&
+                   ocd >= (unsigned int)c - 1)      /* Extend the basic range */
                  {                                  /* if there is overlap,   */
                  c = occ;                           /* noting that if occ < c */
                  continue;                          /* we can't have ocd > d  */
                  }                                  /* because a subrange is  */
-               if (ocd > d && occ <= d + 1)         /* always shorter than    */
+               if (ocd > (unsigned int)d &&
+                   occ <= (unsigned int)d + 1)      /* always shorter than    */
                  {                                  /* the basic range.       */
                  d = ocd;
                  continue;
-Line 2786 
 for (;; ptr++)
+Line 3246 
 for (;; ptr++)
        goto FAILED;
        }
+ /* This code has been disabled because it would mean that \s counts as
+ an explicit \r or \n reference, and that's not really what is wanted. Now
+ we set the flag only if there is a literal "\r" or "\n" in the class. */
+ #if 0
+     /* Remember whether \r or \n are in this class */
+     if (negate_class)
+       {
+       if ((classbits[1] & 0x24) != 0x24) cd->external_flags |= PCRE_HASCRORLF;
+       }
+     else
+       {
+       if ((classbits[1] & 0x24) != 0) cd->external_flags |= PCRE_HASCRORLF;
+       }
+ #endif
      /* If class_charcount is 1, we saw precisely one character whose value is
-     less than 256. In non-UTF-8 mode we can always optimize. In UTF-8 mode, we
+     less than 256. As long as there were no characters >= 128 and there was no
-     can optimize the negative case only if there were no characters >= 128
+     use of \p or \P, in other words, no use of any XCLASS features, we can
-     because OP_NOT and the related opcodes like OP_NOTSTAR operate on
+     optimize.
-     single-bytes only. This is an historical hangover. Maybe one day we can
-     tidy these opcodes to handle multi-byte characters.
+     In UTF-8 mode, we can optimize the negative case only if there were no
+     characters >= 128 because OP_NOT and the related opcodes like OP_NOTSTAR
+     operate on single-bytes only. This is an historical hangover. Maybe one day
+     we can tidy these opcodes to handle multi-byte characters.
      The optimization throws away the bit map. We turn the item into a
 -character OP_CHAR[NC] if it's positive, or OP_NOT if it's negative. Note
-Line 2801 
 for (;; ptr++)
+Line 3283 
 for (;; ptr++)
      reqbyte, save the previous value for reinstating. */
  #ifdef SUPPORT_UTF8
-     if (class_charcount == 1 &&
+     if (class_charcount == 1 && !class_utf8 &&
-           (!utf8 ||
+       (!utf8 || !negate_class || class_lastchar < 128))
-           (!class_utf8 && (!negate_class || class_lastchar < 128))))
  #else
      if (class_charcount == 1)
  #endif
-Line 3076 
 for (;; ptr++)
+Line 3556 
 for (;; ptr++)
        /* All real repeats make it impossible to handle partial matching (maybe
        one day we will be able to remove this restriction). */
-       if (repeat_max != 1) cd->nopartial = TRUE;
+       if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL;
        /* Combine the op_type with the repeat_type */
-Line 3226 
 for (;; ptr++)
+Line 3706 
 for (;; ptr++)
        /* All real repeats make it impossible to handle partial matching (maybe
        one day we will be able to remove this restriction). */
-       if (repeat_max != 1) cd->nopartial = TRUE;
+       if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL;
        if (repeat_min == 0 && repeat_max == -1)
          *code++ = OP_CRSTAR + repeat_type;
-Line 3262 
 for (;; ptr++)
+Line 3742 
 for (;; ptr++)
          goto FAILED;
          }
-       /* This is a paranoid check to stop integer overflow later on */
-       if (len > MAX_DUPLENGTH)
-         {
-         *errorcodeptr = ERR50;
-         goto FAILED;
-         }
        /* If the maximum repeat count is unlimited, find the end of the bracket
        by scanning through from the start, and compute the offset back to it
        from the current code pointer. There may be an OP_OPT setting following
-Line 3358 
 for (;; ptr++)
+Line 3830 
 for (;; ptr++)
          if (repeat_min > 1)
            {
            /* In the pre-compile phase, we don't actually do the replication. We
-           just adjust the length as if we had. */
+           just adjust the length as if we had. Do some paranoid checks for
+           potential integer overflow. */
            if (lengthptr != NULL)
-             *lengthptr += (repeat_min - 1)*length_prevgroup;
+             {
+             int delta = (repeat_min - 1)*length_prevgroup;
+             if ((double)(repeat_min - 1)*(double)length_prevgroup >
+                                                             (double)INT_MAX ||
+                 OFLOW_MAX - *lengthptr < delta)
+               {
+               *errorcodeptr = ERR20;
+               goto FAILED;
+               }
+             *lengthptr += delta;
+             }
            /* This is compiling for real */
-Line 3399 
 for (;; ptr++)
+Line 3882 
 for (;; ptr++)
          /* In the pre-compile phase, we don't actually do the replication. We
          just adjust the length as if we had. For each repetition we must add 1
          to the length for BRAZERO and for all but the last repetition we must
-         add 2 + 2*LINKSIZE to allow for the nesting that occurs. */
+         add 2 + 2*LINKSIZE to allow for the nesting that occurs. Do some
+         paranoid checks to avoid integer overflow. */
          if (lengthptr != NULL && repeat_max > 0)
-           *lengthptr += repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) -
+           {
-- 2*LINK_SIZE;  /* Last one doesn't nest */
+           int delta = repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) -
+- 2*LINK_SIZE;   /* Last one doesn't nest */
+           if ((double)repeat_max *
+                 (double)(length_prevgroup + 1 + 2 + 2*LINK_SIZE)
+                   > (double)INT_MAX ||
+               OFLOW_MAX - *lengthptr < delta)
+             {
+             *errorcodeptr = ERR20;
+             goto FAILED;
+             }
+           *lengthptr += delta;
+           }
          /* This is compiling for real */
-Line 3555 
 for (;; ptr++)
+Line 4050 
 for (;; ptr++)
      /* ===================================================================*/
      /* Start of nested parenthesized sub-expression, or comment or lookahead or
      lookbehind or option setting or condition or all the other extended
-     parenthesis forms. First deal with the specials; all are introduced by ?,
+     parenthesis forms.  */
-     and the appearance of any of them means that this is not a capturing
-     group. */
      case '(':
      newoptions = options;
      skipbytes = 0;
      bravalue = OP_CBRA;
      save_hwm = cd->hwm;
+     reset_bracount = FALSE;
+     /* First deal with various "verbs" that can be introduced by '*'. */
+     if (*(++ptr) == '*' && (cd->ctypes[ptr[1]] & ctype_letter) != 0)
+       {
+       int i, namelen;
+       const char *vn = verbnames;
+       const uschar *name = ++ptr;
+       previous = NULL;
+       while ((cd->ctypes[*++ptr] & ctype_letter) != 0);
+       if (*ptr == ':')
+         {
+         *errorcodeptr = ERR59;   /* Not supported */
+         goto FAILED;
+         }
+       if (*ptr != ')')
+         {
+         *errorcodeptr = ERR60;
+         goto FAILED;
+         }
+       namelen = ptr - name;
+       for (i = 0; i < verbcount; i++)
+         {
+         if (namelen == verbs[i].len &&
+             strncmp((char *)name, vn, namelen) == 0)
+           {
+           *code = verbs[i].op;
+           if (*code++ == OP_ACCEPT) cd->had_accept = TRUE;
+           break;
+           }
+         vn += verbs[i].len + 1;
+         }
+       if (i < verbcount) continue;
+       *errorcodeptr = ERR60;
+       goto FAILED;
+       }
-     if (*(++ptr) == '?')
+     /* Deal with the extended parentheses; all are introduced by '?', and the
+     appearance of any of them means that this is not a capturing group. */
+     else if (*ptr == '?')
        {
        int i, set, unset, namelen;
        int *optset;
-Line 3586 
 for (;; ptr++)
+Line 4119 
 for (;; ptr++)
          /* ------------------------------------------------------------ */
+         case '|':                 /* Reset capture count for each branch */
+         reset_bracount = TRUE;
+         /* Fall through */
+         /* ------------------------------------------------------------ */
          case ':':                 /* Non-capturing bracket */
          bravalue = OP_BRA;
          ptr++;
-Line 3621 
 for (;; ptr++)
+Line 4159 
 for (;; ptr++)
          code[1+LINK_SIZE] = OP_CREF;
          skipbytes = 3;
+         refsign = -1;
          /* Check for a test for recursion in a named group. */
-Line 3644 
 for (;; ptr++)
+Line 4183 
 for (;; ptr++)
            terminator = '\'';
            ptr++;
            }
-         else terminator = 0;
+         else
+           {
+           terminator = 0;
+           if (ptr[1] == '-' || ptr[1] == '+') refsign = *(++ptr);
+           }
          /* We now expect to read a name; any thing else is an error */
-Line 3680 
 for (;; ptr++)
+Line 4223 
 for (;; ptr++)
          if (lengthptr != NULL) break;
          /* In the real compile we do the work of looking for the actual
-         reference. */
+         reference. If the string started with "+" or "-" we require the rest to
+         be digits, in which case recno will be set. */
+         if (refsign > 0)
+           {
+           if (recno <= 0)
+             {
+             *errorcodeptr = ERR58;
+             goto FAILED;
+             }
+           if (refsign == '-')
+             {
+             recno = cd->bracount - recno + 1;
+             if (recno <= 0)
+               {
+               *errorcodeptr = ERR15;
+               goto FAILED;
+               }
+             }
+           else recno += cd->bracount;
+           PUT2(code, 2+LINK_SIZE, recno);
+           break;
+           }
+         /* Otherwise (did not start with "+" or "-"), start by looking for the
+         name. */
          slot = cd->name_table;
          for (i = 0; i < cd->names_found; i++)
-Line 3772 
 for (;; ptr++)
+Line 4340 
 for (;; ptr++)
          /* ------------------------------------------------------------ */
          case '!':                 /* Negative lookahead */
-         bravalue = OP_ASSERT_NOT;
          ptr++;
+         if (*ptr == ')')          /* Optimize (?!) */
+           {
+           *code++ = OP_FAIL;
+           previous = NULL;
+           continue;
+           }
+         bravalue = OP_ASSERT_NOT;
          break;
-Line 3999 
 for (;; ptr++)
+Line 4573 
 for (;; ptr++)
          /* ------------------------------------------------------------ */
+         case '-': case '+':
          case '0': case '1': case '2': case '3': case '4':   /* Recursion or */
          case '5': case '6': case '7': case '8': case '9':   /* subroutine */
            {
            const uschar *called;
+           if ((refsign = *ptr) == '+') ptr++;
+           else if (refsign == '-')
+             {
+             if ((digitab[ptr[1]] & ctype_digit) == 0)
+               goto OTHER_CHAR_AFTER_QUERY;
+             ptr++;
+             }
            recno = 0;
            while((digitab[*ptr] & ctype_digit) != 0)
              recno = recno * 10 + *ptr++ - '0';
            if (*ptr != ')')
              {
              *errorcodeptr = ERR29;
              goto FAILED;
              }
+           if (refsign == '-')
+             {
+             if (recno == 0)
+               {
+               *errorcodeptr = ERR58;
+               goto FAILED;
+               }
+             recno = cd->bracount - recno + 1;
+             if (recno <= 0)
+               {
+               *errorcodeptr = ERR15;
+               goto FAILED;
+               }
+             }
+           else if (refsign == '+')
+             {
+             if (recno == 0)
+               {
+               *errorcodeptr = ERR58;
+               goto FAILED;
+               }
+             recno += cd->bracount;
+             }
            /* Come here from code above that handles a named recursion */
            HANDLE_RECURSION:
-Line 4084 
 for (;; ptr++)
+Line 4693 
 for (;; ptr++)
          /* ------------------------------------------------------------ */
          default:              /* Other characters: check option setting */
+         OTHER_CHAR_AFTER_QUERY:
          set = unset = 0;
          optset = &set;
-Line 4095 
 for (;; ptr++)
+Line 4705 
 for (;; ptr++)
              case 'J':    /* Record that it changed in the external options */
              *optset |= PCRE_DUPNAMES;
-             cd->external_options |= PCRE_JCHANGED;
+             cd->external_flags |= PCRE_JCHANGED;
              break;
              case 'i': *optset |= PCRE_CASELESS; break;
-Line 4218 
 for (;; ptr++)
+Line 4828 
 for (;; ptr++)
           errorcodeptr,                 /* Where to put an error message */
           (bravalue == OP_ASSERTBACK ||
            bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */
+          reset_bracount,               /* True if (?| group */
           skipbytes,                    /* Skip over bracket number */
           &subfirstbyte,                /* For possible first char */
           &subreqbyte,                  /* For possible last char */
-Line 4234 
 for (;; ptr++)
+Line 4845 
 for (;; ptr++)
      is on the bracket. */
      /* If this is a conditional bracket, check that there are no more than
-     two branches in the group, or just one if it's a DEFINE group. */
+     two branches in the group, or just one if it's a DEFINE group. We do this
+     in the real compile phase, not in the pre-pass, where the whole group may
+     not be available. */
-     if (bravalue == OP_COND)
+     if (bravalue == OP_COND && lengthptr == NULL)
        {
        uschar *tc = code;
        int condcount = 0;
-Line 4283 
 for (;; ptr++)
+Line 4896 
 for (;; ptr++)
        goto FAILED;
        }
-     /* In the pre-compile phase, update the length by the length of the nested
+     /* In the pre-compile phase, update the length by the length of the group,
-     group, less the brackets at either end. Then reduce the compiled code to
+     less the brackets at either end. Then reduce the compiled code to just a
-     just the brackets so that it doesn't use much memory if it is duplicated by
+     set of non-capturing brackets so that it doesn't use much memory if it is
-     a quantifier. */
+     duplicated by a quantifier.*/
      if (lengthptr != NULL)
        {
+       if (OFLOW_MAX - *lengthptr < length_prevgroup - 2 - 2*LINK_SIZE)
+         {
+         *errorcodeptr = ERR20;
+         goto FAILED;
+         }
        *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE;
-       code++;
+       *code++ = OP_BRA;
        PUTINC(code, 0, 1 + LINK_SIZE);
        *code++ = OP_KET;
        PUTINC(code, 0, 1 + LINK_SIZE);
+       break;    /* No need to waste time with special character handling */
        }
      /* Otherwise update the main code pointer to the end of the group. */
-     else code = tempcode;
+     code = tempcode;
      /* For a DEFINE group, required and first character settings are not
      relevant. */
-Line 4396 
 for (;; ptr++)
+Line 5015 
 for (;; ptr++)
        zerofirstbyte = firstbyte;
        zeroreqbyte = reqbyte;
-       /* \k<name> or \k'name' is a back reference by name (Perl syntax) */
+       /* \k<name> or \k'name' is a back reference by name (Perl syntax).
+       We also support \k{name} (.NET syntax) */
-       if (-c == ESC_k && (ptr[1] == '<' || ptr[1] == '\''))
+       if (-c == ESC_k && (ptr[1] == '<' || ptr[1] == '\'' || ptr[1] == '{'))
          {
          is_recurse = FALSE;
-         terminator = (*(++ptr) == '<')? '>' : '\'';
+         terminator = (*(++ptr) == '<')? '>' : (*ptr == '\'')? '\'' : '}';
          goto NAMED_REF_OR_RECURSE;
          }
-Line 4502 
 for (;; ptr++)
+Line 5122 
 for (;; ptr++)
      *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARNC : OP_CHAR;
      for (c = 0; c < mclength; c++) *code++ = mcbuffer[c];
+     /* Remember if \r or \n were seen */
+     if (mcbuffer[0] == '\r' || mcbuffer[0] == '\n')
+       cd->external_flags |= PCRE_HASCRORLF;
      /* Set the first and required bytes appropriately. If no previous first
      byte, set it from this character, but revert to none on a zero repeat.
      Otherwise, leave the firstbyte value alone, and don't change it on a zero
-Line 4567 
 This function is used during the pre-com
+Line 5192 
 This function is used during the pre-com
  out the amount of memory needed, as well as during the real compile phase. The
  value of lengthptr distinguishes the two phases.
- Argument:
+ Arguments:
    options        option bits, including any changes for this subpattern
    oldims         previous settings of ims option bits
    codeptr        -> the address of the current code pointer
    ptrptr         -> the address of the current pattern pointer
    errorcodeptr   -> pointer to error code variable
    lookbehind     TRUE if this is a lookbehind assertion
+   reset_bracount TRUE to reset the count for each branch
    skipbytes      skip this many bytes at start (for brackets and OP_COND)
    firstbyteptr   place to put the first required character, or a negative number
    reqbyteptr     place to put the last required character, or a negative number
-Line 4587 
 Returns:         TRUE on success
+Line 5213 
 Returns:         TRUE on success
  static BOOL
  compile_regex(int options, int oldims, uschar **codeptr, const uschar **ptrptr,
-   int *errorcodeptr, BOOL lookbehind, int skipbytes, int *firstbyteptr,
+   int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes,
-   int *reqbyteptr, branch_chain *bcptr, compile_data *cd, int *lengthptr)
+   int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd,
+   int *lengthptr)
  {
  const uschar *ptr = *ptrptr;
  uschar *code = *codeptr;
-Line 4598 
 uschar *reverse_count = NULL;
+Line 5225 
 uschar *reverse_count = NULL;
  int firstbyte, reqbyte;
  int branchfirstbyte, branchreqbyte;
  int length;
+ int orig_bracount;
+ int max_bracount;
  branch_chain bc;
  bc.outer = bcptr;
-Line 4626 
 code += 1 + LINK_SIZE + skipbytes;
+Line 5255 
 code += 1 + LINK_SIZE + skipbytes;
  /* Loop for each alternative branch */
+ orig_bracount = max_bracount = cd->bracount;
  for (;;)
    {
+   /* For a (?| group, reset the capturing bracket count so that each branch
+   uses the same numbers. */
+   if (reset_bracount) cd->bracount = orig_bracount;
    /* Handle a change of ims options at the start of the branch */
    if ((options & PCRE_IMS) != oldims)
-Line 4657 
 for (;;)
+Line 5292 
 for (;;)
      return FALSE;
      }
+   /* Keep the highest bracket count in case (?| was used and some branch
+   has fewer than the rest. */
+   if (cd->bracount > max_bracount) max_bracount = cd->bracount;
    /* In the real compile phase, there is some post-processing to be done. */
    if (lengthptr == NULL)
-Line 4720 
 for (;;)
+Line 5360 
 for (;;)
        }
      }
-   /* Reached end of expression, either ')' or end of pattern. Go back through
+   /* Reached end of expression, either ')' or end of pattern. In the real
-   the alternative branches and reverse the chain of offsets, with the field in
+   compile phase, go back through the alternative branches and reverse the chain
-   the BRA item now becoming an offset to the first alternative. If there are
+   of offsets, with the field in the BRA item now becoming an offset to the
-   no alternatives, it points to the end of the group. The length in the
+   first alternative. If there are no alternatives, it points to the end of the
-   terminating ket is always the length of the whole bracketed item. If any of
+   group. The length in the terminating ket is always the length of the whole
-   the ims options were changed inside the group, compile a resetting op-code
+   bracketed item. If any of the ims options were changed inside the group,
-   following, except at the very end of the pattern. Return leaving the pointer
+   compile a resetting op-code following, except at the very end of the pattern.
-   at the terminating char. */
+   Return leaving the pointer at the terminating char. */
    if (*ptr != '|')
      {
-     int branch_length = code - last_branch;
+     if (lengthptr == NULL)
-     do
        {
-       int prev_length = GET(last_branch, 1);
+       int branch_length = code - last_branch;
-       PUT(last_branch, 1, branch_length);
+       do
-       branch_length = prev_length;
+         {
-       last_branch -= branch_length;
+         int prev_length = GET(last_branch, 1);
+         PUT(last_branch, 1, branch_length);
+         branch_length = prev_length;
+         last_branch -= branch_length;
+         }
+       while (branch_length > 0);
        }
-     while (branch_length > 0);
      /* Fill in the ket */
-Line 4756 
 for (;;)
+Line 5399 
 for (;;)
        length += 2;
        }
+     /* Retain the highest bracket number, in case resetting was used. */
+     cd->bracount = max_bracount;
      /* Set values to pass back */
      *codeptr = code;
      *ptrptr = ptr;
      *firstbyteptr = firstbyte;
      *reqbyteptr = reqbyte;
-     if (lengthptr != NULL) *lengthptr += length;
+     if (lengthptr != NULL)
+       {
+       if (OFLOW_MAX - *lengthptr < length)
+         {
+         *errorcodeptr = ERR20;
+         return FALSE;
+         }
+       *lengthptr += length;
+       }
      return TRUE;
      }
-   /* Another branch follows; insert an "or" node. Its length field points back
+   /* Another branch follows. In the pre-compile phase, we can move the code
+   pointer back to where it was for the start of the first branch. (That is,
+   pretend that each branch is the only one.)
+   In the real compile phase, insert an ALT node. Its length field points back
    to the previous branch while the bracket remains open. At the end the chain
    is reversed. It's done like this so that the start of the bracket has a
    zero offset until it is closed, making it possible to detect recursion. */
-   *code = OP_ALT;
+   if (lengthptr != NULL)
-   PUT(code, 1, code - last_branch);
+     {
-   bc.current = last_branch = code;
+     code = *codeptr + 1 + LINK_SIZE + skipbytes;
-   code += 1 + LINK_SIZE;
+     length += 1 + LINK_SIZE;
+     }
+   else
+     {
+     *code = OP_ALT;
+     PUT(code, 1, code - last_branch);
+     bc.current = last_branch = code;
+     code += 1 + LINK_SIZE;
+     }
    ptr++;
-   length += 1 + LINK_SIZE;
    }
  /* Control never reaches here */
  }
-Line 5059 
 real_pcre *re;
+Line 5726 
 real_pcre *re;
  int length = 1;  /* For final END opcode */
  int firstbyte, reqbyte, newline;
  int errorcode = 0;
+ int skipatstart = 0;
  #ifdef SUPPORT_UTF8
  BOOL utf8;
  #endif
-Line 5137 
 cd->fcc = tables + fcc_offset;
+Line 5805 
 cd->fcc = tables + fcc_offset;
  cd->cbits = tables + cbits_offset;
  cd->ctypes = tables + ctypes_offset;
+ /* Check for global one-time settings at the start of the pattern, and remember
+ the offset for later. */
+ while (ptr[skipatstart] == '(' && ptr[skipatstart+1] == '*')
+   {
+   int newnl = 0;
+   int newbsr = 0;
+   if (strncmp((char *)(ptr+skipatstart+2), "CR)", 3) == 0)
+     { skipatstart += 5; newnl = PCRE_NEWLINE_CR; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "LF)", 3)  == 0)
+     { skipatstart += 5; newnl = PCRE_NEWLINE_LF; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "CRLF)", 5)  == 0)
+     { skipatstart += 7; newnl = PCRE_NEWLINE_CR + PCRE_NEWLINE_LF; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "ANY)", 4) == 0)
+     { skipatstart += 6; newnl = PCRE_NEWLINE_ANY; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "ANYCRLF)", 8)  == 0)
+     { skipatstart += 10; newnl = PCRE_NEWLINE_ANYCRLF; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "BSR_ANYCRLF)", 12) == 0)
+     { skipatstart += 14; newbsr = PCRE_BSR_ANYCRLF; }
+   else if (strncmp((char *)(ptr+skipatstart+2), "BSR_UNICODE)", 12) == 0)
+     { skipatstart += 14; newbsr = PCRE_BSR_UNICODE; }
+   if (newnl != 0)
+     options = (options & ~PCRE_NEWLINE_BITS) | newnl;
+   else if (newbsr != 0)
+     options = (options & ~(PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) | newbsr;
+   else break;
+   }
+ /* Check validity of \R options. */
+ switch (options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE))
+   {
+   case 0:
+   case PCRE_BSR_ANYCRLF:
+   case PCRE_BSR_UNICODE:
+   break;
+   default: errorcode = ERR56; goto PCRE_EARLY_ERROR_RETURN;
+   }
  /* Handle different types of newline. The three bits give seven cases. The
  current code allows for fixed one- or two-byte sequences, plus "any" and
  "anycrlf". */
- switch (options & (PCRE_NEWLINE_CRLF | PCRE_NEWLINE_ANY))
+ switch (options & PCRE_NEWLINE_BITS)
    {
-   case 0: newline = NEWLINE; break;   /* Compile-time default */
+   case 0: newline = NEWLINE; break;   /* Build-time default */
    case PCRE_NEWLINE_CR: newline = '\r'; break;
    case PCRE_NEWLINE_LF: newline = '\n'; break;
    case PCRE_NEWLINE_CR+
-Line 5206 
 cd->hwm = cworkspace;
+Line 5916 
 cd->hwm = cworkspace;
  cd->start_pattern = (const uschar *)pattern;
  cd->end_pattern = (const uschar *)(pattern + strlen(pattern));
  cd->req_varyopt = 0;
- cd->nopartial = FALSE;
  cd->external_options = options;
+ cd->external_flags = 0;
  /* Now do the pre-compile. On error, errorcode will be set non-zero, so we
  don't need to look at the result of the function here. The initial options have
-Line 5215 
 been put into the cd block so that they
+Line 5925 
 been put into the cd block so that they
  found within the regex right at the beginning. Bringing initial option settings
  outside can help speed up starting point checks. */
+ ptr += skipatstart;
  code = cworkspace;
  *code = OP_BRA;
  (void)compile_regex(cd->external_options, cd->external_options & PCRE_IMS,
-   &code, &ptr, &errorcode, FALSE, 0, &firstbyte, &reqbyte, NULL, cd, &length);
+   &code, &ptr, &errorcode, FALSE, FALSE, 0, &firstbyte, &reqbyte, NULL, cd,
+   &length);
  if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN;
  DPRINTF(("end pre-compile: length=%d workspace=%d\n", length,
-Line 5244 
 if (re == NULL)
+Line 5956 
 if (re == NULL)
    goto PCRE_EARLY_ERROR_RETURN;
    }
- /* Put in the magic number, and save the sizes, initial options, and character
+ /* Put in the magic number, and save the sizes, initial options, internal
- table pointer. NULL is used for the default character tables. The nullpad field
+ flags, and character table pointer. NULL is used for the default character
- is at the end; it's there to help in the case when a regex compiled on a system
+ tables. The nullpad field is at the end; it's there to help in the case when a
- with 4-byte pointers is run on another with 8-byte pointers. */
+ regex compiled on a system with 4-byte pointers is run on another with 8-byte
+ pointers. */
  re->magic_number = MAGIC_NUMBER;
  re->size = size;
  re->options = cd->external_options;
+ re->flags = cd->external_flags;
  re->dummy1 = 0;
  re->first_byte = 0;
  re->req_byte = 0;
-Line 5276 
 codestart = cd->name_table + re->name_en
+Line 5990 
 codestart = cd->name_table + re->name_en
  cd->start_code = codestart;
  cd->hwm = cworkspace;
  cd->req_varyopt = 0;
- cd->nopartial = FALSE;
+ cd->had_accept = FALSE;
  /* Set up a starting, non-extracting bracket, then compile the expression. On
  error, errorcode will be set non-zero, so we don't need to look at the result
  of the function here. */
- ptr = (const uschar *)pattern;
+ ptr = (const uschar *)pattern + skipatstart;
  code = (uschar *)codestart;
  *code = OP_BRA;
  (void)compile_regex(re->options, re->options & PCRE_IMS, &code, &ptr,
-   &errorcode, FALSE, 0, &firstbyte, &reqbyte, NULL, cd, NULL);
+   &errorcode, FALSE, FALSE, 0, &firstbyte, &reqbyte, NULL, cd, NULL);
  re->top_bracket = cd->bracount;
  re->top_backref = cd->top_backref;
+ re->flags = cd->external_flags;
- if (cd->nopartial) re->options |= PCRE_NOPARTIAL;
+ if (cd->had_accept) reqbyte = -1;   /* Must disable after (*ACCEPT) */
  /* If not reached end of pattern on success, there's an excess bracket. */
-Line 5332 
 if (errorcode != 0)
+Line 6047 
 if (errorcode != 0)
    PCRE_EARLY_ERROR_RETURN:
    *erroroffset = ptr - (const uschar *)pattern;
    PCRE_EARLY_ERROR_RETURN2:
-   *errorptr = error_texts[errorcode];
+   *errorptr = find_error_text(errorcode);
    if (errorcodeptr != NULL) *errorcodeptr = errorcode;
    return NULL;
    }
-Line 5361 
 if ((re->options & PCRE_ANCHORED) == 0)
+Line 6076 
 if ((re->options & PCRE_ANCHORED) == 0)
        int ch = firstbyte & 255;
        re->first_byte = ((firstbyte & REQ_CASELESS) != 0 &&
           cd->fcc[ch] == ch)? ch : firstbyte;
-       re->options |= PCRE_FIRSTSET;
+       re->flags |= PCRE_FIRSTSET;
        }
      else if (is_startline(codestart, 0, cd->backref_map))
-       re->options |= PCRE_STARTLINE;
+       re->flags |= PCRE_STARTLINE;
      }
    }
-Line 5378 
 if (reqbyte >= 0 &&
+Line 6093 
 if (reqbyte >= 0 &&
    int ch = reqbyte & 255;
    re->req_byte = ((reqbyte & REQ_CASELESS) != 0 &&
      cd->fcc[ch] == ch)? (reqbyte & ~REQ_CASELESS) : reqbyte;
-   re->options |= PCRE_REQCHSET;
+   re->flags |= PCRE_REQCHSET;
    }
  /* Print out the compiled data if debugging is enabled. This is never the
-Line 5389 
 case when building a production library.
+Line 6104 
 case when building a production library.
  printf("Length = %d top_bracket = %d top_backref = %d\n",
    length, re->top_bracket, re->top_backref);
- if (re->options != 0)
+ printf("Options=%08x\n", re->options);
-   {
-   printf("%s%s%s%s%s%s%s%s%s\n",
-     ((re->options & PCRE_NOPARTIAL) != 0)? "nopartial " : "",
-     ((re->options & PCRE_ANCHORED) != 0)? "anchored " : "",
-     ((re->options & PCRE_CASELESS) != 0)? "caseless " : "",
-     ((re->options & PCRE_EXTENDED) != 0)? "extended " : "",
-     ((re->options & PCRE_MULTILINE) != 0)? "multiline " : "",
-     ((re->options & PCRE_DOTALL) != 0)? "dotall " : "",
-     ((re->options & PCRE_DOLLAR_ENDONLY) != 0)? "endonly " : "",
-     ((re->options & PCRE_EXTRA) != 0)? "extra " : "",
-     ((re->options & PCRE_UNGREEDY) != 0)? "ungreedy " : "");
-   }
- if ((re->options & PCRE_FIRSTSET) != 0)
+ if ((re->flags & PCRE_FIRSTSET) != 0)
    {
    int ch = re->first_byte & 255;
    const char *caseless = ((re->first_byte & REQ_CASELESS) == 0)?
-Line 5412 
 if ((re->options & PCRE_FIRSTSET) != 0)
+Line 6115 
 if ((re->options & PCRE_FIRSTSET) != 0)
      else printf("First char = \\x%02x%s\n", ch, caseless);
    }
- if ((re->options & PCRE_REQCHSET) != 0)
+ if ((re->flags & PCRE_REQCHSET) != 0)
    {
    int ch = re->req_byte & 255;
    const char *caseless = ((re->req_byte & REQ_CASELESS) == 0)?
-Line 5429 
 was compiled can be seen. */
+Line 6132 
 was compiled can be seen. */
  if (code - codestart > length)
    {
    (pcre_free)(re);
-   *errorptr = error_texts[ERR23];
+   *errorptr = find_error_text(ERR23);
    *erroroffset = ptr - (uschar *)pattern;
    if (errorcodeptr != NULL) *errorcodeptr = ERR23;
    return NULL;

 Legend:



Removed from v.150
 


changed lines


 
Added in v.240
 Legend:



Removed from v.150
 


changed lines


 
Added in v.240
-Removed from v.150
+Added in v.240

	ViewVC Help
Powered by ViewVC 1.1.5