/[pcre]/code/branches/pcre16/pcre_compile.c
ViewVC logotype

Diff of /code/branches/pcre16/pcre_compile.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 406 by ph10, Mon Mar 23 12:05:43 2009 UTC revision 744 by zherczeg, Sun Nov 13 16:31:38 2011 UTC
# Line 6  Line 6 
6  and semantics are as close as possible to those of the Perl 5 language.  and semantics are as close as possible to those of the Perl 5 language.
7    
8                         Written by Philip Hazel                         Written by Philip Hazel
9             Copyright (c) 1997-2009 University of Cambridge             Copyright (c) 1997-2011 University of Cambridge
10    
11  -----------------------------------------------------------------------------  -----------------------------------------------------------------------------
12  Redistribution and use in source and binary forms, with or without  Redistribution and use in source and binary forms, with or without
# Line 53  supporting internal functions that are n Line 53  supporting internal functions that are n
53  #include "pcre_internal.h"  #include "pcre_internal.h"
54    
55    
56  /* When DEBUG is defined, we need the pcre_printint() function, which is also  /* When PCRE_DEBUG is defined, we need the pcre_printint() function, which is
57  used by pcretest. DEBUG is not defined when building a production library. */  also used by pcretest. PCRE_DEBUG is not defined when building a production
58    library. */
59    
60  #ifdef DEBUG  #ifdef PCRE_DEBUG
61  #include "pcre_printint.src"  #include "pcre_printint.src"
62  #endif  #endif
63    
# Line 91  is 4 there is plenty of room. */ Line 92  is 4 there is plenty of room. */
92    
93  #define COMPILE_WORK_SIZE (4096)  #define COMPILE_WORK_SIZE (4096)
94    
95    /* The overrun tests check for a slightly smaller size so that they detect the
96    overrun before it actually does run off the end of the data block. */
97    
98    #define WORK_SIZE_CHECK (COMPILE_WORK_SIZE - 100)
99    
100    
101  /* Table for handling escaped characters in the range '0'-'z'. Positive returns  /* Table for handling escaped characters in the range '0'-'z'. Positive returns
102  are simple data values; negative values are for special things like \d and so  are simple data values; negative values are for special things like \d and so
# Line 118  static const short int escapes[] = { Line 124  static const short int escapes[] = {
124       -ESC_H,                  0,       -ESC_H,                  0,
125       0,                       -ESC_K,       0,                       -ESC_K,
126       0,                       0,       0,                       0,
127       0,                       0,       -ESC_N,                  0,
128       -ESC_P,                  -ESC_Q,       -ESC_P,                  -ESC_Q,
129       -ESC_R,                  -ESC_S,       -ESC_R,                  -ESC_S,
130       0,                       0,       0,                       0,
# Line 165  static const short int escapes[] = { Line 171  static const short int escapes[] = {
171  /*  B8 */     0,     0,      0,       0,      0,   ']',    '=',    '-',  /*  B8 */     0,     0,      0,       0,      0,   ']',    '=',    '-',
172  /*  C0 */   '{',-ESC_A, -ESC_B,  -ESC_C, -ESC_D,-ESC_E,      0, -ESC_G,  /*  C0 */   '{',-ESC_A, -ESC_B,  -ESC_C, -ESC_D,-ESC_E,      0, -ESC_G,
173  /*  C8 */-ESC_H,     0,      0,       0,      0,     0,      0,      0,  /*  C8 */-ESC_H,     0,      0,       0,      0,     0,      0,      0,
174  /*  D0 */   '}',     0, -ESC_K,       0,      0,     0,      0, -ESC_P,  /*  D0 */   '}',     0, -ESC_K,       0,      0,-ESC_N,      0, -ESC_P,
175  /*  D8 */-ESC_Q,-ESC_R,      0,       0,      0,     0,      0,      0,  /*  D8 */-ESC_Q,-ESC_R,      0,       0,      0,     0,      0,      0,
176  /*  E0 */  '\\',     0, -ESC_S,       0,      0,-ESC_V, -ESC_W, -ESC_X,  /*  E0 */  '\\',     0, -ESC_S,       0,      0,-ESC_V, -ESC_W, -ESC_X,
177  /*  E8 */     0,-ESC_Z,      0,       0,      0,     0,      0,      0,  /*  E8 */     0,-ESC_Z,      0,       0,      0,     0,      0,      0,
# Line 182  string is built from string macros so th Line 188  string is built from string macros so th
188  platforms. */  platforms. */
189    
190  typedef struct verbitem {  typedef struct verbitem {
191    int   len;    int   len;                 /* Length of verb name */
192    int   op;    int   op;                  /* Op when no arg, or -1 if arg mandatory */
193      int   op_arg;              /* Op when arg present, or -1 if not allowed */
194  } verbitem;  } verbitem;
195    
196  static const char verbnames[] =  static const char verbnames[] =
197      "\0"                       /* Empty name is a shorthand for MARK */
198      STRING_MARK0
199    STRING_ACCEPT0    STRING_ACCEPT0
200    STRING_COMMIT0    STRING_COMMIT0
201    STRING_F0    STRING_F0
# Line 196  static const char verbnames[] = Line 205  static const char verbnames[] =
205    STRING_THEN;    STRING_THEN;
206    
207  static const verbitem verbs[] = {  static const verbitem verbs[] = {
208    { 6, OP_ACCEPT },    { 0, -1,        OP_MARK },
209    { 6, OP_COMMIT },    { 4, -1,        OP_MARK },
210    { 1, OP_FAIL },    { 6, OP_ACCEPT, -1 },
211    { 4, OP_FAIL },    { 6, OP_COMMIT, -1 },
212    { 5, OP_PRUNE },    { 1, OP_FAIL,   -1 },
213    { 4, OP_SKIP  },    { 4, OP_FAIL,   -1 },
214    { 4, OP_THEN  }    { 5, OP_PRUNE,  OP_PRUNE_ARG },
215      { 4, OP_SKIP,   OP_SKIP_ARG  },
216      { 4, OP_THEN,   OP_THEN_ARG  }
217  };  };
218    
219  static const int verbcount = sizeof(verbs)/sizeof(verbitem);  static const int verbcount = sizeof(verbs)/sizeof(verbitem);
# Line 250  static const int posix_class_maps[] = { Line 261  static const int posix_class_maps[] = {
261    cbit_xdigit,-1,          0              /* xdigit */    cbit_xdigit,-1,          0              /* xdigit */
262  };  };
263    
264    /* Table of substitutes for \d etc when PCRE_UCP is set. The POSIX class
265    substitutes must be in the order of the names, defined above, and there are
266    both positive and negative cases. NULL means no substitute. */
267    
268    #ifdef SUPPORT_UCP
269    static const uschar *substitutes[] = {
270      (uschar *)"\\P{Nd}",    /* \D */
271      (uschar *)"\\p{Nd}",    /* \d */
272      (uschar *)"\\P{Xsp}",   /* \S */       /* NOTE: Xsp is Perl space */
273      (uschar *)"\\p{Xsp}",   /* \s */
274      (uschar *)"\\P{Xwd}",   /* \W */
275      (uschar *)"\\p{Xwd}"    /* \w */
276    };
277    
278    static const uschar *posix_substitutes[] = {
279      (uschar *)"\\p{L}",     /* alpha */
280      (uschar *)"\\p{Ll}",    /* lower */
281      (uschar *)"\\p{Lu}",    /* upper */
282      (uschar *)"\\p{Xan}",   /* alnum */
283      NULL,                   /* ascii */
284      (uschar *)"\\h",        /* blank */
285      NULL,                   /* cntrl */
286      (uschar *)"\\p{Nd}",    /* digit */
287      NULL,                   /* graph */
288      NULL,                   /* print */
289      NULL,                   /* punct */
290      (uschar *)"\\p{Xps}",   /* space */    /* NOTE: Xps is POSIX space */
291      (uschar *)"\\p{Xwd}",   /* word */
292      NULL,                   /* xdigit */
293      /* Negated cases */
294      (uschar *)"\\P{L}",     /* ^alpha */
295      (uschar *)"\\P{Ll}",    /* ^lower */
296      (uschar *)"\\P{Lu}",    /* ^upper */
297      (uschar *)"\\P{Xan}",   /* ^alnum */
298      NULL,                   /* ^ascii */
299      (uschar *)"\\H",        /* ^blank */
300      NULL,                   /* ^cntrl */
301      (uschar *)"\\P{Nd}",    /* ^digit */
302      NULL,                   /* ^graph */
303      NULL,                   /* ^print */
304      NULL,                   /* ^punct */
305      (uschar *)"\\P{Xps}",   /* ^space */   /* NOTE: Xps is POSIX space */
306      (uschar *)"\\P{Xwd}",   /* ^word */
307      NULL                    /* ^xdigit */
308    };
309    #define POSIX_SUBSIZE (sizeof(posix_substitutes)/sizeof(uschar *))
310    #endif
311    
312  #define STRING(a)  # a  #define STRING(a)  # a
313  #define XSTRING(s) STRING(s)  #define XSTRING(s) STRING(s)
# Line 262  the number of relocations needed when a Line 320  the number of relocations needed when a
320  it is now one long string. We cannot use a table of offsets, because the  it is now one long string. We cannot use a table of offsets, because the
321  lengths of inserts such as XSTRING(MAX_NAME_SIZE) are not known. Instead, we  lengths of inserts such as XSTRING(MAX_NAME_SIZE) are not known. Instead, we
322  simply count through to the one we want - this isn't a performance issue  simply count through to the one we want - this isn't a performance issue
323  because these strings are used only when there is a compilation error. */  because these strings are used only when there is a compilation error.
324    
325    Each substring ends with \0 to insert a null character. This includes the final
326    substring, so that the whole string ends with \0\0, which can be detected when
327    counting through. */
328    
329  static const char error_texts[] =  static const char error_texts[] =
330    "no error\0"    "no error\0"
# Line 309  static const char error_texts[] = Line 371  static const char error_texts[] =
371    /* 35 */    /* 35 */
372    "invalid condition (?(0)\0"    "invalid condition (?(0)\0"
373    "\\C not allowed in lookbehind assertion\0"    "\\C not allowed in lookbehind assertion\0"
374    "PCRE does not support \\L, \\l, \\N, \\U, or \\u\0"    "PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u\0"
375    "number after (?C is > 255\0"    "number after (?C is > 255\0"
376    "closing ) for (?C expected\0"    "closing ) for (?C expected\0"
377    /* 40 */    /* 40 */
# Line 331  static const char error_texts[] = Line 393  static const char error_texts[] =
393    "internal error: previously-checked referenced subpattern not found\0"    "internal error: previously-checked referenced subpattern not found\0"
394    "DEFINE group contains more than one branch\0"    "DEFINE group contains more than one branch\0"
395    /* 55 */    /* 55 */
396    "repeating a DEFINE group is not allowed\0"    "repeating a DEFINE group is not allowed\0"  /** DEAD **/
397    "inconsistent NEWLINE options\0"    "inconsistent NEWLINE options\0"
398    "\\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number\0"    "\\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number\0"
399    "a numbered reference must not be zero\0"    "a numbered reference must not be zero\0"
400    "(*VERB) with an argument is not supported\0"    "an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT)\0"
401    /* 60 */    /* 60 */
402    "(*VERB) not recognized\0"    "(*VERB) not recognized\0"
403    "number is too big\0"    "number is too big\0"
404    "subpattern name expected\0"    "subpattern name expected\0"
405    "digit expected after (?+\0"    "digit expected after (?+\0"
406    "] is an invalid data character in JavaScript compatibility mode";    "] is an invalid data character in JavaScript compatibility mode\0"
407      /* 65 */
408      "different names for subpatterns of the same number are not allowed\0"
409      "(*MARK) must have an argument\0"
410      "this version of PCRE is not compiled with PCRE_UCP support\0"
411      "\\c must be followed by an ASCII character\0"
412      "\\k is not followed by a braced, angle-bracketed, or quoted name\0"
413      ;
414    
415  /* Table to identify digits and hex digits. This is used when compiling  /* Table to identify digits and hex digits. This is used when compiling
416  patterns. Note that the tables in chartables are dependent on the locale, and  patterns. Note that the tables in chartables are dependent on the locale, and
# Line 478  static const unsigned char ebcdic_charta Line 546  static const unsigned char ebcdic_charta
546  /* Definition to allow mutual recursion */  /* Definition to allow mutual recursion */
547    
548  static BOOL  static BOOL
549    compile_regex(int, int, uschar **, const uschar **, int *, BOOL, BOOL, int,    compile_regex(int, uschar **, const uschar **, int *, BOOL, BOOL, int, int,
550      int *, int *, branch_chain *, compile_data *, int *);      int *, int *, branch_chain *, compile_data *, int *);
551    
552    
# Line 500  static const char * Line 568  static const char *
568  find_error_text(int n)  find_error_text(int n)
569  {  {
570  const char *s = error_texts;  const char *s = error_texts;
571  for (; n > 0; n--) while (*s++ != 0) {};  for (; n > 0; n--)
572      {
573      while (*s++ != 0) {};
574      if (*s == 0) return "Error text not found (please report)";
575      }
576  return s;  return s;
577  }  }
578    
579    
580  /*************************************************  /*************************************************
581    *            Check for counted repeat            *
582    *************************************************/
583    
584    /* This function is called when a '{' is encountered in a place where it might
585    start a quantifier. It looks ahead to see if it really is a quantifier or not.
586    It is only a quantifier if it is one of the forms {ddd} {ddd,} or {ddd,ddd}
587    where the ddds are digits.
588    
589    Arguments:
590      p         pointer to the first char after '{'
591    
592    Returns:    TRUE or FALSE
593    */
594    
595    static BOOL
596    is_counted_repeat(const uschar *p)
597    {
598    if ((digitab[*p++] & ctype_digit) == 0) return FALSE;
599    while ((digitab[*p] & ctype_digit) != 0) p++;
600    if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE;
601    
602    if (*p++ != CHAR_COMMA) return FALSE;
603    if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE;
604    
605    if ((digitab[*p++] & ctype_digit) == 0) return FALSE;
606    while ((digitab[*p] & ctype_digit) != 0) p++;
607    
608    return (*p == CHAR_RIGHT_CURLY_BRACKET);
609    }
610    
611    
612    
613    /*************************************************
614  *            Handle escapes                      *  *            Handle escapes                      *
615  *************************************************/  *************************************************/
616    
# Line 571  else Line 676  else
676    
677      case CHAR_l:      case CHAR_l:
678      case CHAR_L:      case CHAR_L:
679      case CHAR_N:      *errorcodeptr = ERR37;
680        break;
681    
682      case CHAR_u:      case CHAR_u:
683        if ((options & PCRE_JAVASCRIPT_COMPAT) != 0)
684          {
685          /* In JavaScript, \u must be followed by four hexadecimal numbers.
686          Otherwise it is a lowercase u letter. */
687          if ((digitab[ptr[1]] & ctype_xdigit) != 0 && (digitab[ptr[2]] & ctype_xdigit) != 0
688               && (digitab[ptr[3]] & ctype_xdigit) != 0 && (digitab[ptr[4]] & ctype_xdigit) != 0)
689            {
690            int i;
691            c = 0;
692            for (i = 0; i < 4; ++i)
693              {
694              register int cc = *(++ptr);
695    #ifndef EBCDIC  /* ASCII/UTF-8 coding */
696              if (cc >= CHAR_a) cc -= 32;               /* Convert to upper case */
697              c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
698    #else           /* EBCDIC coding */
699              if (cc >= CHAR_a && cc <= CHAR_z) cc += 64;  /* Convert to upper case */
700              c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
701    #endif
702              }
703            }
704          }
705        else
706          *errorcodeptr = ERR37;
707        break;
708    
709      case CHAR_U:      case CHAR_U:
710      *errorcodeptr = ERR37;      /* In JavaScript, \U is an uppercase U letter. */
711        if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) *errorcodeptr = ERR37;
712      break;      break;
713    
714      /* \g must be followed by one of a number of specific things:      /* In a character class, \g is just a literal "g". Outside a character
715        class, \g must be followed by one of a number of specific things:
716    
717      (1) A number, either plain or braced. If positive, it is an absolute      (1) A number, either plain or braced. If positive, it is an absolute
718      backreference. If negative, it is a relative backreference. This is a Perl      backreference. If negative, it is a relative backreference. This is a Perl
# Line 594  else Line 729  else
729      the -ESC_g code (cf \k). */      the -ESC_g code (cf \k). */
730    
731      case CHAR_g:      case CHAR_g:
732        if (isclass) break;
733      if (ptr[1] == CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_APOSTROPHE)      if (ptr[1] == CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_APOSTROPHE)
734        {        {
735        c = -ESC_g;        c = -ESC_g;
# Line 722  else Line 858  else
858      treated as a data character. */      treated as a data character. */
859    
860      case CHAR_x:      case CHAR_x:
861        if ((options & PCRE_JAVASCRIPT_COMPAT) != 0)
862          {
863          /* In JavaScript, \x must be followed by two hexadecimal numbers.
864          Otherwise it is a lowercase x letter. */
865          if ((digitab[ptr[1]] & ctype_xdigit) != 0 && (digitab[ptr[2]] & ctype_xdigit) != 0)
866            {
867            int i;
868            c = 0;
869            for (i = 0; i < 2; ++i)
870              {
871              register int cc = *(++ptr);
872    #ifndef EBCDIC  /* ASCII/UTF-8 coding */
873              if (cc >= CHAR_a) cc -= 32;               /* Convert to upper case */
874              c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
875    #else           /* EBCDIC coding */
876              if (cc >= CHAR_a && cc <= CHAR_z) cc += 64;  /* Convert to upper case */
877              c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
878    #endif
879              }
880            }
881          break;
882          }
883    
884      if (ptr[1] == CHAR_LEFT_CURLY_BRACKET)      if (ptr[1] == CHAR_LEFT_CURLY_BRACKET)
885        {        {
886        const uschar *pt = ptr + 2;        const uschar *pt = ptr + 2;
# Line 772  else Line 931  else
931      break;      break;
932    
933      /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped.      /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped.
934      This coding is ASCII-specific, but then the whole concept of \cx is      An error is given if the byte following \c is not an ASCII character. This
935        coding is ASCII-specific, but then the whole concept of \cx is
936      ASCII-specific. (However, an EBCDIC equivalent has now been added.) */      ASCII-specific. (However, an EBCDIC equivalent has now been added.) */
937    
938      case CHAR_c:      case CHAR_c:
# Line 782  else Line 942  else
942        *errorcodeptr = ERR2;        *errorcodeptr = ERR2;
943        break;        break;
944        }        }
945    #ifndef EBCDIC    /* ASCII/UTF-8 coding */
946  #ifndef EBCDIC  /* ASCII/UTF-8 coding */      if (c > 127)  /* Excludes all non-ASCII in either mode */
947          {
948          *errorcodeptr = ERR68;
949          break;
950          }
951      if (c >= CHAR_a && c <= CHAR_z) c -= 32;      if (c >= CHAR_a && c <= CHAR_z) c -= 32;
952      c ^= 0x40;      c ^= 0x40;
953  #else           /* EBCDIC coding */  #else             /* EBCDIC coding */
954      if (c >= CHAR_a && c <= CHAR_z) c += 64;      if (c >= CHAR_a && c <= CHAR_z) c += 64;
955      c ^= 0xC0;      c ^= 0xC0;
956  #endif  #endif
# Line 809  else Line 973  else
973      }      }
974    }    }
975    
976    /* Perl supports \N{name} for character names, as well as plain \N for "not
977    newline". PCRE does not support \N{name}. However, it does support
978    quantification such as \N{2,3}. */
979    
980    if (c == -ESC_N && ptr[1] == CHAR_LEFT_CURLY_BRACKET &&
981         !is_counted_repeat(ptr+2))
982      *errorcodeptr = ERR37;
983    
984    /* If PCRE_UCP is set, we change the values for \d etc. */
985    
986    if ((options & PCRE_UCP) != 0 && c <= -ESC_D && c >= -ESC_w)
987      c -= (ESC_DU - ESC_D);
988    
989    /* Set the pointer to the final character before returning. */
990    
991  *ptrptr = ptr;  *ptrptr = ptr;
992  return c;  return c;
993  }  }
# Line 909  return -1; Line 1088  return -1;
1088    
1089    
1090  /*************************************************  /*************************************************
 *            Check for counted repeat            *  
 *************************************************/  
   
 /* This function is called when a '{' is encountered in a place where it might  
 start a quantifier. It looks ahead to see if it really is a quantifier or not.  
 It is only a quantifier if it is one of the forms {ddd} {ddd,} or {ddd,ddd}  
 where the ddds are digits.  
   
 Arguments:  
   p         pointer to the first char after '{'  
   
 Returns:    TRUE or FALSE  
 */  
   
 static BOOL  
 is_counted_repeat(const uschar *p)  
 {  
 if ((digitab[*p++] & ctype_digit) == 0) return FALSE;  
 while ((digitab[*p] & ctype_digit) != 0) p++;  
 if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE;  
   
 if (*p++ != CHAR_COMMA) return FALSE;  
 if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE;  
   
 if ((digitab[*p++] & ctype_digit) == 0) return FALSE;  
 while ((digitab[*p] & ctype_digit) != 0) p++;  
   
 return (*p == CHAR_RIGHT_CURLY_BRACKET);  
 }  
   
   
   
 /*************************************************  
1091  *         Read repeat counts                     *  *         Read repeat counts                     *
1092  *************************************************/  *************************************************/
1093    
# Line 1009  return p; Line 1155  return p;
1155    
1156    
1157  /*************************************************  /*************************************************
1158  *       Find forward referenced subpattern       *  *  Subroutine for finding forward reference      *
1159  *************************************************/  *************************************************/
1160    
1161  /* This function scans along a pattern's text looking for capturing  /* This recursive function is called only from find_parens() below. The
1162    top-level call starts at the beginning of the pattern. All other calls must
1163    start at a parenthesis. It scans along a pattern's text looking for capturing
1164  subpatterns, and counting them. If it finds a named pattern that matches the  subpatterns, and counting them. If it finds a named pattern that matches the
1165  name it is given, it returns its number. Alternatively, if the name is NULL, it  name it is given, it returns its number. Alternatively, if the name is NULL, it
1166  returns when it reaches a given numbered subpattern. This is used for forward  returns when it reaches a given numbered subpattern. Recursion is used to keep
1167  references to subpatterns. We know that if (?P< is encountered, the name will  track of subpatterns that reset the capturing group numbers - the (?| feature.
1168  be terminated by '>' because that is checked in the first pass.  
1169    This function was originally called only from the second pass, in which we know
1170    that if (?< or (?' or (?P< is encountered, the name will be correctly
1171    terminated because that is checked in the first pass. There is now one call to
1172    this function in the first pass, to check for a recursive back reference by
1173    name (so that we can make the whole group atomic). In this case, we need check
1174    only up to the current position in the pattern, and that is still OK because
1175    and previous occurrences will have been checked. To make this work, the test
1176    for "end of pattern" is a check against cd->end_pattern in the main loop,
1177    instead of looking for a binary zero. This means that the special first-pass
1178    call can adjust cd->end_pattern temporarily. (Checks for binary zero while
1179    processing items within the loop are OK, because afterwards the main loop will
1180    terminate.)
1181    
1182  Arguments:  Arguments:
1183    ptr          current position in the pattern    ptrptr       address of the current character pointer (updated)
1184    cd           compile background data    cd           compile background data
1185    name         name to seek, or NULL if seeking a numbered subpattern    name         name to seek, or NULL if seeking a numbered subpattern
1186    lorn         name length, or subpattern number if name is NULL    lorn         name length, or subpattern number if name is NULL
1187    xmode        TRUE if we are in /x mode    xmode        TRUE if we are in /x mode
1188      utf8         TRUE if we are in UTF-8 mode
1189      count        pointer to the current capturing subpattern number (updated)
1190    
1191  Returns:       the number of the named subpattern, or -1 if not found  Returns:       the number of the named subpattern, or -1 if not found
1192  */  */
1193    
1194  static int  static int
1195  find_parens(const uschar *ptr, compile_data *cd, const uschar *name, int lorn,  find_parens_sub(uschar **ptrptr, compile_data *cd, const uschar *name, int lorn,
1196    BOOL xmode)    BOOL xmode, BOOL utf8, int *count)
1197  {  {
1198  const uschar *thisname;  uschar *ptr = *ptrptr;
1199  int count = cd->bracount;  int start_count = *count;
1200    int hwm_count = start_count;
1201    BOOL dup_parens = FALSE;
1202    
1203    /* If the first character is a parenthesis, check on the type of group we are
1204    dealing with. The very first call may not start with a parenthesis. */
1205    
1206  for (; *ptr != 0; ptr++)  if (ptr[0] == CHAR_LEFT_PARENTHESIS)
1207    {    {
1208    int term;    /* Handle specials such as (*SKIP) or (*UTF8) etc. */
1209    
1210      if (ptr[1] == CHAR_ASTERISK) ptr += 2;
1211    
1212      /* Handle a normal, unnamed capturing parenthesis. */
1213    
1214      else if (ptr[1] != CHAR_QUESTION_MARK)
1215        {
1216        *count += 1;
1217        if (name == NULL && *count == lorn) return *count;
1218        ptr++;
1219        }
1220    
1221      /* All cases now have (? at the start. Remember when we are in a group
1222      where the parenthesis numbers are duplicated. */
1223    
1224      else if (ptr[2] == CHAR_VERTICAL_LINE)
1225        {
1226        ptr += 3;
1227        dup_parens = TRUE;
1228        }
1229    
1230      /* Handle comments; all characters are allowed until a ket is reached. */
1231    
1232      else if (ptr[2] == CHAR_NUMBER_SIGN)
1233        {
1234        for (ptr += 3; *ptr != 0; ptr++) if (*ptr == CHAR_RIGHT_PARENTHESIS) break;
1235        goto FAIL_EXIT;
1236        }
1237    
1238      /* Handle a condition. If it is an assertion, just carry on so that it
1239      is processed as normal. If not, skip to the closing parenthesis of the
1240      condition (there can't be any nested parens). */
1241    
1242      else if (ptr[2] == CHAR_LEFT_PARENTHESIS)
1243        {
1244        ptr += 2;
1245        if (ptr[1] != CHAR_QUESTION_MARK)
1246          {
1247          while (*ptr != 0 && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
1248          if (*ptr != 0) ptr++;
1249          }
1250        }
1251    
1252      /* Start with (? but not a condition. */
1253    
1254      else
1255        {
1256        ptr += 2;
1257        if (*ptr == CHAR_P) ptr++;                      /* Allow optional P */
1258    
1259        /* We have to disambiguate (?<! and (?<= from (?<name> for named groups */
1260    
1261        if ((*ptr == CHAR_LESS_THAN_SIGN && ptr[1] != CHAR_EXCLAMATION_MARK &&
1262            ptr[1] != CHAR_EQUALS_SIGN) || *ptr == CHAR_APOSTROPHE)
1263          {
1264          int term;
1265          const uschar *thisname;
1266          *count += 1;
1267          if (name == NULL && *count == lorn) return *count;
1268          term = *ptr++;
1269          if (term == CHAR_LESS_THAN_SIGN) term = CHAR_GREATER_THAN_SIGN;
1270          thisname = ptr;
1271          while (*ptr != term) ptr++;
1272          if (name != NULL && lorn == ptr - thisname &&
1273              strncmp((const char *)name, (const char *)thisname, lorn) == 0)
1274            return *count;
1275          term++;
1276          }
1277        }
1278      }
1279    
1280    /* Past any initial parenthesis handling, scan for parentheses or vertical
1281    bars. Stop if we get to cd->end_pattern. Note that this is important for the
1282    first-pass call when this value is temporarily adjusted to stop at the current
1283    position. So DO NOT change this to a test for binary zero. */
1284    
1285    for (; ptr < cd->end_pattern; ptr++)
1286      {
1287    /* Skip over backslashed characters and also entire \Q...\E */    /* Skip over backslashed characters and also entire \Q...\E */
1288    
1289    if (*ptr == CHAR_BACKSLASH)    if (*ptr == CHAR_BACKSLASH)
1290      {      {
1291      if (*(++ptr) == 0) return -1;      if (*(++ptr) == 0) goto FAIL_EXIT;
1292      if (*ptr == CHAR_Q) for (;;)      if (*ptr == CHAR_Q) for (;;)
1293        {        {
1294        while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {};        while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {};
1295        if (*ptr == 0) return -1;        if (*ptr == 0) goto FAIL_EXIT;
1296        if (*(++ptr) == CHAR_E) break;        if (*(++ptr) == CHAR_E) break;
1297        }        }
1298      continue;      continue;
# Line 1065  for (; *ptr != 0; ptr++) Line 1309  for (; *ptr != 0; ptr++)
1309      BOOL negate_class = FALSE;      BOOL negate_class = FALSE;
1310      for (;;)      for (;;)
1311        {        {
1312        int c = *(++ptr);        if (ptr[1] == CHAR_BACKSLASH)
       if (c == CHAR_BACKSLASH)  
1313          {          {
1314          if (ptr[1] == CHAR_E)          if (ptr[2] == CHAR_E)
1315            ptr++;            ptr+= 2;
1316          else if (strncmp((const char *)ptr+1,          else if (strncmp((const char *)ptr+2,
1317                   STR_Q STR_BACKSLASH STR_E, 3) == 0)                   STR_Q STR_BACKSLASH STR_E, 3) == 0)
1318            ptr += 3;            ptr += 4;
1319          else          else
1320            break;            break;
1321          }          }
1322        else if (!negate_class && c == CHAR_CIRCUMFLEX_ACCENT)        else if (!negate_class && ptr[1] == CHAR_CIRCUMFLEX_ACCENT)
1323            {
1324          negate_class = TRUE;          negate_class = TRUE;
1325            ptr++;
1326            }
1327        else break;        else break;
1328        }        }
1329    
# Line 1093  for (; *ptr != 0; ptr++) Line 1339  for (; *ptr != 0; ptr++)
1339        if (*ptr == 0) return -1;        if (*ptr == 0) return -1;
1340        if (*ptr == CHAR_BACKSLASH)        if (*ptr == CHAR_BACKSLASH)
1341          {          {
1342          if (*(++ptr) == 0) return -1;          if (*(++ptr) == 0) goto FAIL_EXIT;
1343          if (*ptr == CHAR_Q) for (;;)          if (*ptr == CHAR_Q) for (;;)
1344            {            {
1345            while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {};            while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {};
1346            if (*ptr == 0) return -1;            if (*ptr == 0) goto FAIL_EXIT;
1347            if (*(++ptr) == CHAR_E) break;            if (*(++ptr) == CHAR_E) break;
1348            }            }
1349          continue;          continue;
# Line 1110  for (; *ptr != 0; ptr++) Line 1356  for (; *ptr != 0; ptr++)
1356    
1357    if (xmode && *ptr == CHAR_NUMBER_SIGN)    if (xmode && *ptr == CHAR_NUMBER_SIGN)
1358      {      {
1359      while (*(++ptr) != 0 && *ptr != CHAR_NL) {};      ptr++;
1360      if (*ptr == 0) return -1;      while (*ptr != 0)
1361          {
1362          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
1363          ptr++;
1364    #ifdef SUPPORT_UTF8
1365          if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
1366    #endif
1367          }
1368        if (*ptr == 0) goto FAIL_EXIT;
1369      continue;      continue;
1370      }      }
1371    
1372    /* An opening parens must now be a real metacharacter */    /* Check for the special metacharacters */
1373    
1374    if (*ptr != CHAR_LEFT_PARENTHESIS) continue;    if (*ptr == CHAR_LEFT_PARENTHESIS)
   if (ptr[1] != CHAR_QUESTION_MARK && ptr[1] != CHAR_ASTERISK)  
1375      {      {
1376      count++;      int rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf8, count);
1377      if (name == NULL && count == lorn) return count;      if (rc > 0) return rc;
1378      continue;      if (*ptr == 0) goto FAIL_EXIT;
1379      }      }
1380    
1381    ptr += 2;    else if (*ptr == CHAR_RIGHT_PARENTHESIS)
1382    if (*ptr == CHAR_P) ptr++;                      /* Allow optional P */      {
1383        if (dup_parens && *count < hwm_count) *count = hwm_count;
1384        goto FAIL_EXIT;
1385        }
1386    
1387    /* We have to disambiguate (?<! and (?<= from (?<name> */    else if (*ptr == CHAR_VERTICAL_LINE && dup_parens)
1388        {
1389        if (*count > hwm_count) hwm_count = *count;
1390        *count = start_count;
1391        }
1392      }
1393    
1394    if ((*ptr != CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_EXCLAMATION_MARK ||  FAIL_EXIT:
1395        ptr[1] == CHAR_EQUALS_SIGN) && *ptr != CHAR_APOSTROPHE)  *ptrptr = ptr;
1396      continue;  return -1;
1397    }
1398    
   count++;  
1399    
1400    if (name == NULL && count == lorn) return count;  
1401    term = *ptr++;  
1402    if (term == CHAR_LESS_THAN_SIGN) term = CHAR_GREATER_THAN_SIGN;  /*************************************************
1403    thisname = ptr;  *       Find forward referenced subpattern       *
1404    while (*ptr != term) ptr++;  *************************************************/
1405    if (name != NULL && lorn == ptr - thisname &&  
1406        strncmp((const char *)name, (const char *)thisname, lorn) == 0)  /* This function scans along a pattern's text looking for capturing
1407      return count;  subpatterns, and counting them. If it finds a named pattern that matches the
1408    name it is given, it returns its number. Alternatively, if the name is NULL, it
1409    returns when it reaches a given numbered subpattern. This is used for forward
1410    references to subpatterns. We used to be able to start this scan from the
1411    current compiling point, using the current count value from cd->bracount, and
1412    do it all in a single loop, but the addition of the possibility of duplicate
1413    subpattern numbers means that we have to scan from the very start, in order to
1414    take account of such duplicates, and to use a recursive function to keep track
1415    of the different types of group.
1416    
1417    Arguments:
1418      cd           compile background data
1419      name         name to seek, or NULL if seeking a numbered subpattern
1420      lorn         name length, or subpattern number if name is NULL
1421      xmode        TRUE if we are in /x mode
1422      utf8         TRUE if we are in UTF-8 mode
1423    
1424    Returns:       the number of the found subpattern, or -1 if not found
1425    */
1426    
1427    static int
1428    find_parens(compile_data *cd, const uschar *name, int lorn, BOOL xmode,
1429      BOOL utf8)
1430    {
1431    uschar *ptr = (uschar *)cd->start_pattern;
1432    int count = 0;
1433    int rc;
1434    
1435    /* If the pattern does not start with an opening parenthesis, the first call
1436    to find_parens_sub() will scan right to the end (if necessary). However, if it
1437    does start with a parenthesis, find_parens_sub() will return when it hits the
1438    matching closing parens. That is why we have to have a loop. */
1439    
1440    for (;;)
1441      {
1442      rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf8, &count);
1443      if (rc > 0 || *ptr++ == 0) break;
1444    }    }
1445    
1446  return -1;  return rc;
1447  }  }
1448    
1449    
1450    
1451    
1452  /*************************************************  /*************************************************
1453  *      Find first significant op code            *  *      Find first significant op code            *
1454  *************************************************/  *************************************************/
1455    
1456  /* This is called by several functions that scan a compiled expression looking  /* This is called by several functions that scan a compiled expression looking
1457  for a fixed first character, or an anchoring op code etc. It skips over things  for a fixed first character, or an anchoring op code etc. It skips over things
1458  that do not influence this. For some calls, a change of option is important.  that do not influence this. For some calls, it makes sense to skip negative
1459  For some calls, it makes sense to skip negative forward and all backward  forward and all backward assertions, and also the \b assertion; for others it
1460  assertions, and also the \b assertion; for others it does not.  does not.
1461    
1462  Arguments:  Arguments:
1463    code         pointer to the start of the group    code         pointer to the start of the group
   options      pointer to external options  
   optbit       the option bit whose changing is significant, or  
                  zero if none are  
1464    skipassert   TRUE if certain assertions are to be skipped    skipassert   TRUE if certain assertions are to be skipped
1465    
1466  Returns:       pointer to the first significant opcode  Returns:       pointer to the first significant opcode
1467  */  */
1468    
1469  static const uschar*  static const uschar*
1470  first_significant_code(const uschar *code, int *options, int optbit,  first_significant_code(const uschar *code, BOOL skipassert)
   BOOL skipassert)  
1471  {  {
1472  for (;;)  for (;;)
1473    {    {
1474    switch ((int)*code)    switch ((int)*code)
1475      {      {
     case OP_OPT:  
     if (optbit > 0 && ((int)code[1] & optbit) != (*options & optbit))  
       *options = (int)code[1];  
     code += 2;  
     break;  
   
1476      case OP_ASSERT_NOT:      case OP_ASSERT_NOT:
1477      case OP_ASSERTBACK:      case OP_ASSERTBACK:
1478      case OP_ASSERTBACK_NOT:      case OP_ASSERTBACK_NOT:
# Line 1200  for (;;) Line 1488  for (;;)
1488    
1489      case OP_CALLOUT:      case OP_CALLOUT:
1490      case OP_CREF:      case OP_CREF:
1491        case OP_NCREF:
1492      case OP_RREF:      case OP_RREF:
1493        case OP_NRREF:
1494      case OP_DEF:      case OP_DEF:
1495      code += _pcre_OP_lengths[*code];      code += _pcre_OP_lengths[*code];
1496      break;      break;
# Line 1216  for (;;) Line 1506  for (;;)
1506    
1507    
1508  /*************************************************  /*************************************************
1509  *        Find the fixed length of a pattern      *  *        Find the fixed length of a branch       *
1510  *************************************************/  *************************************************/
1511    
1512  /* Scan a pattern and compute the fixed length of subject that will match it,  /* Scan a branch and compute the fixed length of subject that will match it,
1513  if the length is fixed. This is needed for dealing with backward assertions.  if the length is fixed. This is needed for dealing with backward assertions.
1514  In UTF8 mode, the result is in characters rather than bytes.  In UTF8 mode, the result is in characters rather than bytes. The branch is
1515    temporarily terminated with OP_END when this function is called.
1516    
1517    This function is called when a backward assertion is encountered, so that if it
1518    fails, the error message can point to the correct place in the pattern.
1519    However, we cannot do this when the assertion contains subroutine calls,
1520    because they can be forward references. We solve this by remembering this case
1521    and doing the check at the end; a flag specifies which mode we are running in.
1522    
1523  Arguments:  Arguments:
1524    code     points to the start of the pattern (the bracket)    code     points to the start of the pattern (the bracket)
1525    options  the compiling options    utf8     TRUE in UTF-8 mode
1526      atend    TRUE if called when the pattern is complete
1527      cd       the "compile data" structure
1528    
1529  Returns:   the fixed length, or -1 if there is no fixed length,  Returns:   the fixed length,
1530                 or -1 if there is no fixed length,
1531               or -2 if \C was encountered               or -2 if \C was encountered
1532                 or -3 if an OP_RECURSE item was encountered and atend is FALSE
1533  */  */
1534    
1535  static int  static int
1536  find_fixedlength(uschar *code, int options)  find_fixedlength(uschar *code, BOOL utf8, BOOL atend, compile_data *cd)
1537  {  {
1538  int length = -1;  int length = -1;
1539    
# Line 1245  branch, check the length against that of Line 1546  branch, check the length against that of
1546  for (;;)  for (;;)
1547    {    {
1548    int d;    int d;
1549      uschar *ce, *cs;
1550    register int op = *cc;    register int op = *cc;
1551    switch (op)    switch (op)
1552      {      {
1553        /* We only need to continue for OP_CBRA (normal capturing bracket) and
1554        OP_BRA (normal non-capturing bracket) because the other variants of these
1555        opcodes are all concerned with unlimited repeated groups, which of course
1556        are not of fixed length. They will cause a -1 response from the default
1557        case of this switch. */
1558    
1559      case OP_CBRA:      case OP_CBRA:
1560      case OP_BRA:      case OP_BRA:
1561      case OP_ONCE:      case OP_ONCE:
1562        case OP_ONCE_NC:
1563      case OP_COND:      case OP_COND:
1564      d = find_fixedlength(cc + ((op == OP_CBRA)? 2:0), options);      d = find_fixedlength(cc + ((op == OP_CBRA)? 2:0), utf8, atend, cd);
1565      if (d < 0) return d;      if (d < 0) return d;
1566      branchlength += d;      branchlength += d;
1567      do cc += GET(cc, 1); while (*cc == OP_ALT);      do cc += GET(cc, 1); while (*cc == OP_ALT);
# Line 1261  for (;;) Line 1570  for (;;)
1570    
1571      /* Reached end of a branch; if it's a ket it is the end of a nested      /* Reached end of a branch; if it's a ket it is the end of a nested
1572      call. If it's ALT it is an alternation in a nested call. If it is      call. If it's ALT it is an alternation in a nested call. If it is
1573      END it's the end of the outer call. All can be handled by the same code. */      END it's the end of the outer call. All can be handled by the same code.
1574        Note that we must not include the OP_KETRxxx opcodes here, because they
1575        all imply an unlimited repeat. */
1576    
1577      case OP_ALT:      case OP_ALT:
1578      case OP_KET:      case OP_KET:
     case OP_KETRMAX:  
     case OP_KETRMIN:  
1579      case OP_END:      case OP_END:
1580      if (length < 0) length = branchlength;      if (length < 0) length = branchlength;
1581        else if (length != branchlength) return -1;        else if (length != branchlength) return -1;
# Line 1275  for (;;) Line 1584  for (;;)
1584      branchlength = 0;      branchlength = 0;
1585      break;      break;
1586    
1587        /* A true recursion implies not fixed length, but a subroutine call may
1588        be OK. If the subroutine is a forward reference, we can't deal with
1589        it until the end of the pattern, so return -3. */
1590    
1591        case OP_RECURSE:
1592        if (!atend) return -3;
1593        cs = ce = (uschar *)cd->start_code + GET(cc, 1);  /* Start subpattern */
1594        do ce += GET(ce, 1); while (*ce == OP_ALT);       /* End subpattern */
1595        if (cc > cs && cc < ce) return -1;                /* Recursion */
1596        d = find_fixedlength(cs + 2, utf8, atend, cd);
1597        if (d < 0) return d;
1598        branchlength += d;
1599        cc += 1 + LINK_SIZE;
1600        break;
1601    
1602      /* Skip over assertive subpatterns */      /* Skip over assertive subpatterns */
1603    
1604      case OP_ASSERT:      case OP_ASSERT:
# Line 1288  for (;;) Line 1612  for (;;)
1612    
1613      case OP_REVERSE:      case OP_REVERSE:
1614      case OP_CREF:      case OP_CREF:
1615        case OP_NCREF:
1616      case OP_RREF:      case OP_RREF:
1617        case OP_NRREF:
1618      case OP_DEF:      case OP_DEF:
     case OP_OPT:  
1619      case OP_CALLOUT:      case OP_CALLOUT:
1620      case OP_SOD:      case OP_SOD:
1621      case OP_SOM:      case OP_SOM:
1622        case OP_SET_SOM:
1623      case OP_EOD:      case OP_EOD:
1624      case OP_EODN:      case OP_EODN:
1625      case OP_CIRC:      case OP_CIRC:
1626        case OP_CIRCM:
1627      case OP_DOLL:      case OP_DOLL:
1628        case OP_DOLLM:
1629      case OP_NOT_WORD_BOUNDARY:      case OP_NOT_WORD_BOUNDARY:
1630      case OP_WORD_BOUNDARY:      case OP_WORD_BOUNDARY:
1631      cc += _pcre_OP_lengths[*cc];      cc += _pcre_OP_lengths[*cc];
# Line 1306  for (;;) Line 1634  for (;;)
1634      /* Handle literal characters */      /* Handle literal characters */
1635    
1636      case OP_CHAR:      case OP_CHAR:
1637      case OP_CHARNC:      case OP_CHARI:
1638      case OP_NOT:      case OP_NOT:
1639        case OP_NOTI:
1640      branchlength++;      branchlength++;
1641      cc += 2;      cc += 2;
1642  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
1643      if ((options & PCRE_UTF8) != 0)      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
       {  
       while ((*cc & 0xc0) == 0x80) cc++;  
       }  
1644  #endif  #endif
1645      break;      break;
1646    
# Line 1325  for (;;) Line 1651  for (;;)
1651      branchlength += GET2(cc,1);      branchlength += GET2(cc,1);
1652      cc += 4;      cc += 4;
1653  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
1654      if ((options & PCRE_UTF8) != 0)      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
       {  
       while((*cc & 0x80) == 0x80) cc++;  
       }  
1655  #endif  #endif
1656      break;      break;
1657    
# Line 1407  for (;;) Line 1730  for (;;)
1730    
1731    
1732  /*************************************************  /*************************************************
1733  *    Scan compiled regex for numbered bracket    *  *    Scan compiled regex for specific bracket    *
1734  *************************************************/  *************************************************/
1735    
1736  /* This little function scans through a compiled pattern until it finds a  /* This little function scans through a compiled pattern until it finds a
1737  capturing bracket with the given number.  capturing bracket with the given number, or, if the number is negative, an
1738    instance of OP_REVERSE for a lookbehind. The function is global in the C sense
1739    so that it can be called from pcre_study() when finding the minimum matching
1740    length.
1741    
1742  Arguments:  Arguments:
1743    code        points to start of expression    code        points to start of expression
1744    utf8        TRUE in UTF-8 mode    utf8        TRUE in UTF-8 mode
1745    number      the required bracket number    number      the required bracket number or negative to find a lookbehind
1746    
1747  Returns:      pointer to the opcode for the bracket, or NULL if not found  Returns:      pointer to the opcode for the bracket, or NULL if not found
1748  */  */
1749    
1750  static const uschar *  const uschar *
1751  find_bracket(const uschar *code, BOOL utf8, int number)  _pcre_find_bracket(const uschar *code, BOOL utf8, int number)
1752  {  {
1753  for (;;)  for (;;)
1754    {    {
1755    register int c = *code;    register int c = *code;
1756    
1757    if (c == OP_END) return NULL;    if (c == OP_END) return NULL;
1758    
1759    /* XCLASS is used for classes that cannot be represented just by a bit    /* XCLASS is used for classes that cannot be represented just by a bit
# Line 1435  for (;;) Line 1762  for (;;)
1762    
1763    if (c == OP_XCLASS) code += GET(code, 1);    if (c == OP_XCLASS) code += GET(code, 1);
1764    
1765      /* Handle recursion */
1766    
1767      else if (c == OP_REVERSE)
1768        {
1769        if (number < 0) return (uschar *)code;
1770        code += _pcre_OP_lengths[c];
1771        }
1772    
1773    /* Handle capturing bracket */    /* Handle capturing bracket */
1774    
1775    else if (c == OP_CBRA)    else if (c == OP_CBRA || c == OP_SCBRA ||
1776               c == OP_CBRAPOS || c == OP_SCBRAPOS)
1777      {      {
1778      int n = GET2(code, 1+LINK_SIZE);      int n = GET2(code, 1+LINK_SIZE);
1779      if (n == number) return (uschar *)code;      if (n == number) return (uschar *)code;
# Line 1446  for (;;) Line 1782  for (;;)
1782    
1783    /* Otherwise, we can get the item's length from the table, except that for    /* Otherwise, we can get the item's length from the table, except that for
1784    repeated character types, we have to test for \p and \P, which have an extra    repeated character types, we have to test for \p and \P, which have an extra
1785    two bytes of parameters. */    two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we
1786      must add in its length. */
1787    
1788    else    else
1789      {      {
# Line 1470  for (;;) Line 1807  for (;;)
1807        case OP_TYPEPOSUPTO:        case OP_TYPEPOSUPTO:
1808        if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;        if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;
1809        break;        break;
1810    
1811          case OP_MARK:
1812          case OP_PRUNE_ARG:
1813          case OP_SKIP_ARG:
1814          code += code[1];
1815          break;
1816    
1817          case OP_THEN_ARG:
1818          code += code[1];
1819          break;
1820        }        }
1821    
1822      /* Add in the fixed length from the table */      /* Add in the fixed length from the table */
# Line 1484  for (;;) Line 1831  for (;;)
1831      if (utf8) switch(c)      if (utf8) switch(c)
1832        {        {
1833        case OP_CHAR:        case OP_CHAR:
1834        case OP_CHARNC:        case OP_CHARI:
1835        case OP_EXACT:        case OP_EXACT:
1836          case OP_EXACTI:
1837        case OP_UPTO:        case OP_UPTO:
1838          case OP_UPTOI:
1839        case OP_MINUPTO:        case OP_MINUPTO:
1840          case OP_MINUPTOI:
1841        case OP_POSUPTO:        case OP_POSUPTO:
1842          case OP_POSUPTOI:
1843        case OP_STAR:        case OP_STAR:
1844          case OP_STARI:
1845        case OP_MINSTAR:        case OP_MINSTAR:
1846          case OP_MINSTARI:
1847        case OP_POSSTAR:        case OP_POSSTAR:
1848          case OP_POSSTARI:
1849        case OP_PLUS:        case OP_PLUS:
1850          case OP_PLUSI:
1851        case OP_MINPLUS:        case OP_MINPLUS:
1852          case OP_MINPLUSI:
1853        case OP_POSPLUS:        case OP_POSPLUS:
1854          case OP_POSPLUSI:
1855        case OP_QUERY:        case OP_QUERY:
1856          case OP_QUERYI:
1857        case OP_MINQUERY:        case OP_MINQUERY:
1858          case OP_MINQUERYI:
1859        case OP_POSQUERY:        case OP_POSQUERY:
1860          case OP_POSQUERYI:
1861        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];
1862        break;        break;
1863        }        }
# Line 1541  for (;;) Line 1901  for (;;)
1901    
1902    /* Otherwise, we can get the item's length from the table, except that for    /* Otherwise, we can get the item's length from the table, except that for
1903    repeated character types, we have to test for \p and \P, which have an extra    repeated character types, we have to test for \p and \P, which have an extra
1904    two bytes of parameters. */    two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we
1905      must add in its length. */
1906    
1907    else    else
1908      {      {
# Line 1565  for (;;) Line 1926  for (;;)
1926        case OP_TYPEEXACT:        case OP_TYPEEXACT:
1927        if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;        if (code[3] == OP_PROP || code[3] == OP_NOTPROP) code += 2;
1928        break;        break;
1929    
1930          case OP_MARK:
1931          case OP_PRUNE_ARG:
1932          case OP_SKIP_ARG:
1933          code += code[1];
1934          break;
1935    
1936          case OP_THEN_ARG:
1937          code += code[1];
1938          break;
1939        }        }
1940    
1941      /* Add in the fixed length from the table */      /* Add in the fixed length from the table */
# Line 1579  for (;;) Line 1950  for (;;)
1950      if (utf8) switch(c)      if (utf8) switch(c)
1951        {        {
1952        case OP_CHAR:        case OP_CHAR:
1953        case OP_CHARNC:        case OP_CHARI:
1954        case OP_EXACT:        case OP_EXACT:
1955          case OP_EXACTI:
1956        case OP_UPTO:        case OP_UPTO:
1957          case OP_UPTOI:
1958        case OP_MINUPTO:        case OP_MINUPTO:
1959          case OP_MINUPTOI:
1960        case OP_POSUPTO:        case OP_POSUPTO:
1961          case OP_POSUPTOI:
1962        case OP_STAR:        case OP_STAR:
1963          case OP_STARI:
1964        case OP_MINSTAR:        case OP_MINSTAR:
1965          case OP_MINSTARI:
1966        case OP_POSSTAR:        case OP_POSSTAR:
1967          case OP_POSSTARI:
1968        case OP_PLUS:        case OP_PLUS:
1969          case OP_PLUSI:
1970        case OP_MINPLUS:        case OP_MINPLUS:
1971          case OP_MINPLUSI:
1972        case OP_POSPLUS:        case OP_POSPLUS:
1973          case OP_POSPLUSI:
1974        case OP_QUERY:        case OP_QUERY:
1975          case OP_QUERYI:
1976        case OP_MINQUERY:        case OP_MINQUERY:
1977          case OP_MINQUERYI:
1978        case OP_POSQUERY:        case OP_POSQUERY:
1979          case OP_POSQUERYI:
1980        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];
1981        break;        break;
1982        }        }
# Line 1621  Arguments: Line 2005  Arguments:
2005    code        points to start of search    code        points to start of search
2006    endcode     points to where to stop    endcode     points to where to stop
2007    utf8        TRUE if in UTF8 mode    utf8        TRUE if in UTF8 mode
2008      cd          contains pointers to tables etc.
2009    
2010  Returns:      TRUE if what is matched could be empty  Returns:      TRUE if what is matched could be empty
2011  */  */
2012    
2013  static BOOL  static BOOL
2014  could_be_empty_branch(const uschar *code, const uschar *endcode, BOOL utf8)  could_be_empty_branch(const uschar *code, const uschar *endcode, BOOL utf8,
2015      compile_data *cd)
2016  {  {
2017  register int c;  register int c;
2018  for (code = first_significant_code(code + _pcre_OP_lengths[*code], NULL, 0, TRUE);  for (code = first_significant_code(code + _pcre_OP_lengths[*code], TRUE);
2019       code < endcode;       code < endcode;
2020       code = first_significant_code(code + _pcre_OP_lengths[c], NULL, 0, TRUE))       code = first_significant_code(code + _pcre_OP_lengths[c], TRUE))
2021    {    {
2022    const uschar *ccode;    const uschar *ccode;
2023    
# Line 1647  for (code = first_significant_code(code Line 2033  for (code = first_significant_code(code
2033      continue;      continue;
2034      }      }
2035    
2036      /* For a recursion/subroutine call, if its end has been reached, which
2037      implies a backward reference subroutine call, we can scan it. If it's a
2038      forward reference subroutine call, we can't. To detect forward reference
2039      we have to scan up the list that is kept in the workspace. This function is
2040      called only when doing the real compile, not during the pre-compile that
2041      measures the size of the compiled pattern. */
2042    
2043      if (c == OP_RECURSE)
2044        {
2045        const uschar *scode;
2046        BOOL empty_branch;
2047    
2048        /* Test for forward reference */
2049    
2050        for (scode = cd->start_workspace; scode < cd->hwm; scode += LINK_SIZE)
2051          if (GET(scode, 0) == code + 1 - cd->start_code) return TRUE;
2052    
2053        /* Not a forward reference, test for completed backward reference */
2054    
2055        empty_branch = FALSE;
2056        scode = cd->start_code + GET(code, 1);
2057        if (GET(scode, 1) == 0) return TRUE;    /* Unclosed */
2058    
2059        /* Completed backwards reference */
2060    
2061        do
2062          {
2063          if (could_be_empty_branch(scode, endcode, utf8, cd))
2064            {
2065            empty_branch = TRUE;
2066            break;
2067            }
2068          scode += GET(scode, 1);
2069          }
2070        while (*scode == OP_ALT);
2071    
2072        if (!empty_branch) return FALSE;  /* All branches are non-empty */
2073        continue;
2074        }
2075    
2076    /* Groups with zero repeats can of course be empty; skip them. */    /* Groups with zero repeats can of course be empty; skip them. */
2077    
2078    if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO)    if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO ||
2079          c == OP_BRAPOSZERO)
2080      {      {
2081      code += _pcre_OP_lengths[c];      code += _pcre_OP_lengths[c];
2082      do code += GET(code, 1); while (*code == OP_ALT);      do code += GET(code, 1); while (*code == OP_ALT);
# Line 1657  for (code = first_significant_code(code Line 2084  for (code = first_significant_code(code
2084      continue;      continue;
2085      }      }
2086    
2087      /* A nested group that is already marked as "could be empty" can just be
2088      skipped. */
2089    
2090      if (c == OP_SBRA  || c == OP_SBRAPOS ||
2091          c == OP_SCBRA || c == OP_SCBRAPOS)
2092        {
2093        do code += GET(code, 1); while (*code == OP_ALT);
2094        c = *code;
2095        continue;
2096        }
2097    
2098    /* For other groups, scan the branches. */    /* For other groups, scan the branches. */
2099    
2100    if (c == OP_BRA || c == OP_CBRA || c == OP_ONCE || c == OP_COND)    if (c == OP_BRA  || c == OP_BRAPOS ||
2101          c == OP_CBRA || c == OP_CBRAPOS ||
2102          c == OP_ONCE || c == OP_ONCE_NC ||
2103          c == OP_COND)
2104      {      {
2105      BOOL empty_branch;      BOOL empty_branch;
2106      if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */      if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */
# Line 1675  for (code = first_significant_code(code Line 2116  for (code = first_significant_code(code
2116        empty_branch = FALSE;        empty_branch = FALSE;
2117        do        do
2118          {          {
2119          if (!empty_branch && could_be_empty_branch(code, endcode, utf8))          if (!empty_branch && could_be_empty_branch(code, endcode, utf8, cd))
2120            empty_branch = TRUE;            empty_branch = TRUE;
2121          code += GET(code, 1);          code += GET(code, 1);
2122          }          }
# Line 1746  for (code = first_significant_code(code Line 2187  for (code = first_significant_code(code
2187      case OP_ALLANY:      case OP_ALLANY:
2188      case OP_ANYBYTE:      case OP_ANYBYTE:
2189      case OP_CHAR:      case OP_CHAR:
2190      case OP_CHARNC:      case OP_CHARI:
2191      case OP_NOT:      case OP_NOT:
2192        case OP_NOTI:
2193      case OP_PLUS:      case OP_PLUS:
2194      case OP_MINPLUS:      case OP_MINPLUS:
2195      case OP_POSPLUS:      case OP_POSPLUS:
# Line 1787  for (code = first_significant_code(code Line 2229  for (code = first_significant_code(code
2229      case OP_KET:      case OP_KET:
2230      case OP_KETRMAX:      case OP_KETRMAX:
2231      case OP_KETRMIN:      case OP_KETRMIN:
2232        case OP_KETRPOS:
2233      case OP_ALT:      case OP_ALT:
2234      return TRUE;      return TRUE;
2235    
# Line 1795  for (code = first_significant_code(code Line 2238  for (code = first_significant_code(code
2238    
2239  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2240      case OP_STAR:      case OP_STAR:
2241        case OP_STARI:
2242      case OP_MINSTAR:      case OP_MINSTAR:
2243        case OP_MINSTARI:
2244      case OP_POSSTAR:      case OP_POSSTAR:
2245        case OP_POSSTARI:
2246      case OP_QUERY:      case OP_QUERY:
2247        case OP_QUERYI:
2248      case OP_MINQUERY:      case OP_MINQUERY:
2249        case OP_MINQUERYI:
2250      case OP_POSQUERY:      case OP_POSQUERY:
2251        case OP_POSQUERYI:
2252        if (utf8 && code[1] >= 0xc0) code += _pcre_utf8_table4[code[1] & 0x3f];
2253        break;
2254    
2255      case OP_UPTO:      case OP_UPTO:
2256        case OP_UPTOI:
2257      case OP_MINUPTO:      case OP_MINUPTO:
2258        case OP_MINUPTOI:
2259      case OP_POSUPTO:      case OP_POSUPTO:
2260      if (utf8) while ((code[2] & 0xc0) == 0x80) code++;      case OP_POSUPTOI:
2261        if (utf8 && code[3] >= 0xc0) code += _pcre_utf8_table4[code[3] & 0x3f];
2262      break;      break;
2263  #endif  #endif
2264    
2265        /* MARK, and PRUNE/SKIP/THEN with an argument must skip over the argument
2266        string. */
2267    
2268        case OP_MARK:
2269        case OP_PRUNE_ARG:
2270        case OP_SKIP_ARG:
2271        code += code[1];
2272        break;
2273    
2274        case OP_THEN_ARG:
2275        code += code[1];
2276        break;
2277    
2278        /* None of the remaining opcodes are required to match a character. */
2279    
2280        default:
2281        break;
2282      }      }
2283    }    }
2284    
# Line 1822  return TRUE; Line 2295  return TRUE;
2295  the current branch of the current pattern to see if it could match the empty  the current branch of the current pattern to see if it could match the empty
2296  string. If it could, we must look outwards for branches at other levels,  string. If it could, we must look outwards for branches at other levels,
2297  stopping when we pass beyond the bracket which is the subject of the recursion.  stopping when we pass beyond the bracket which is the subject of the recursion.
2298    This function is called only during the real compile, not during the
2299    pre-compile.
2300    
2301  Arguments:  Arguments:
2302    code        points to start of the recursion    code        points to start of the recursion
2303    endcode     points to where to stop (current RECURSE item)    endcode     points to where to stop (current RECURSE item)
2304    bcptr       points to the chain of current (unclosed) branch starts    bcptr       points to the chain of current (unclosed) branch starts
2305    utf8        TRUE if in UTF-8 mode    utf8        TRUE if in UTF-8 mode
2306      cd          pointers to tables etc
2307    
2308  Returns:      TRUE if what is matched could be empty  Returns:      TRUE if what is matched could be empty
2309  */  */
2310    
2311  static BOOL  static BOOL
2312  could_be_empty(const uschar *code, const uschar *endcode, branch_chain *bcptr,  could_be_empty(const uschar *code, const uschar *endcode, branch_chain *bcptr,
2313    BOOL utf8)    BOOL utf8, compile_data *cd)
2314  {  {
2315  while (bcptr != NULL && bcptr->current >= code)  while (bcptr != NULL && bcptr->current_branch >= code)
2316    {    {
2317    if (!could_be_empty_branch(bcptr->current, endcode, utf8)) return FALSE;    if (!could_be_empty_branch(bcptr->current_branch, endcode, utf8, cd))
2318        return FALSE;
2319    bcptr = bcptr->outer;    bcptr = bcptr->outer;
2320    }    }
2321  return TRUE;  return TRUE;
# Line 1870  where Perl recognizes it as the POSIX cl Line 2347  where Perl recognizes it as the POSIX cl
2347  "l\ower". This is a lesser evil that not diagnosing bad classes when Perl does,  "l\ower". This is a lesser evil that not diagnosing bad classes when Perl does,
2348  I think.  I think.
2349    
2350    A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not.
2351    It seems that the appearance of a nested POSIX class supersedes an apparent
2352    external class. For example, [:a[:digit:]b:] matches "a", "b", ":", or
2353    a digit.
2354    
2355    In Perl, unescaped square brackets may also appear as part of class names. For
2356    example, [:a[:abc]b:] gives unknown POSIX class "[:abc]b:]". However, for
2357    [:a[:abc]b][b:] it gives unknown POSIX class "[:abc]b][b:]", which does not
2358    seem right at all. PCRE does not allow closing square brackets in POSIX class
2359    names.
2360    
2361  Arguments:  Arguments:
2362    ptr      pointer to the initial [    ptr      pointer to the initial [
2363    endptr   where to return the end pointer    endptr   where to return the end pointer
# Line 1884  int terminator;          /* Don't combin Line 2372  int terminator;          /* Don't combin
2372  terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */  terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */
2373  for (++ptr; *ptr != 0; ptr++)  for (++ptr; *ptr != 0; ptr++)
2374    {    {
2375    if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) ptr++; else    if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
2376        ptr++;
2377      else if (*ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE;
2378      else
2379      {      {
     if (*ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE;  
2380      if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)      if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
2381        {        {
2382        *endptr = ptr;        *endptr = ptr;
2383        return TRUE;        return TRUE;
2384        }        }
2385        if (*ptr == CHAR_LEFT_SQUARE_BRACKET &&
2386             (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT ||
2387              ptr[1] == CHAR_EQUALS_SIGN) &&
2388            check_posix_syntax(ptr, endptr))
2389          return FALSE;
2390      }      }
2391    }    }
2392  return FALSE;  return FALSE;
# Line 2020  auto_callout(uschar *code, const uschar Line 2515  auto_callout(uschar *code, const uschar
2515  {  {
2516  *code++ = OP_CALLOUT;  *code++ = OP_CALLOUT;
2517  *code++ = 255;  *code++ = 255;
2518  PUT(code, 0, ptr - cd->start_pattern);  /* Pattern offset */  PUT(code, 0, (int)(ptr - cd->start_pattern));  /* Pattern offset */
2519  PUT(code, LINK_SIZE, 0);                /* Default length */  PUT(code, LINK_SIZE, 0);                       /* Default length */
2520  return code + 2*LINK_SIZE;  return code + 2*LINK_SIZE;
2521  }  }
2522    
# Line 2046  Returns:             nothing Line 2541  Returns:             nothing
2541  static void  static void
2542  complete_callout(uschar *previous_callout, const uschar *ptr, compile_data *cd)  complete_callout(uschar *previous_callout, const uschar *ptr, compile_data *cd)
2543  {  {
2544  int length = ptr - cd->start_pattern - GET(previous_callout, 2);  int length = (int)(ptr - cd->start_pattern - GET(previous_callout, 2));
2545  PUT(previous_callout, 2 + LINK_SIZE, length);  PUT(previous_callout, 2 + LINK_SIZE, length);
2546  }  }
2547    
# Line 2096  for (++c; c <= d; c++) Line 2591  for (++c; c <= d; c++)
2591    
2592  return TRUE;  return TRUE;
2593  }  }
2594    
2595    
2596    
2597    /*************************************************
2598    *        Check a character and a property        *
2599    *************************************************/
2600    
2601    /* This function is called by check_auto_possessive() when a property item
2602    is adjacent to a fixed character.
2603    
2604    Arguments:
2605      c            the character
2606      ptype        the property type
2607      pdata        the data for the type
2608      negated      TRUE if it's a negated property (\P or \p{^)
2609    
2610    Returns:       TRUE if auto-possessifying is OK
2611    */
2612    
2613    static BOOL
2614    check_char_prop(int c, int ptype, int pdata, BOOL negated)
2615    {
2616    const ucd_record *prop = GET_UCD(c);
2617    switch(ptype)
2618      {
2619      case PT_LAMP:
2620      return (prop->chartype == ucp_Lu ||
2621              prop->chartype == ucp_Ll ||
2622              prop->chartype == ucp_Lt) == negated;
2623    
2624      case PT_GC:
2625      return (pdata == _pcre_ucp_gentype[prop->chartype]) == negated;
2626    
2627      case PT_PC:
2628      return (pdata == prop->chartype) == negated;
2629    
2630      case PT_SC:
2631      return (pdata == prop->script) == negated;
2632    
2633      /* These are specials */
2634    
2635      case PT_ALNUM:
2636      return (_pcre_ucp_gentype[prop->chartype] == ucp_L ||
2637              _pcre_ucp_gentype[prop->chartype] == ucp_N) == negated;
2638    
2639      case PT_SPACE:    /* Perl space */
2640      return (_pcre_ucp_gentype[prop->chartype] == ucp_Z ||
2641              c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR)
2642              == negated;
2643    
2644      case PT_PXSPACE:  /* POSIX space */
2645      return (_pcre_ucp_gentype[prop->chartype] == ucp_Z ||
2646              c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
2647              c == CHAR_FF || c == CHAR_CR)
2648              == negated;
2649    
2650      case PT_WORD:
2651      return (_pcre_ucp_gentype[prop->chartype] == ucp_L ||
2652              _pcre_ucp_gentype[prop->chartype] == ucp_N ||
2653              c == CHAR_UNDERSCORE) == negated;
2654      }
2655    return FALSE;
2656    }
2657  #endif  /* SUPPORT_UCP */  #endif  /* SUPPORT_UCP */
2658    
2659    
# Line 2109  whether the next thing could possibly ma Line 2667  whether the next thing could possibly ma
2667  sense to automatically possessify the repeated item.  sense to automatically possessify the repeated item.
2668    
2669  Arguments:  Arguments:
2670    op_code       the repeated op code    previous      pointer to the repeated opcode
   this          data for this item, depends on the opcode  
2671    utf8          TRUE in UTF-8 mode    utf8          TRUE in UTF-8 mode
   utf8_char     used for utf8 character bytes, NULL if not relevant  
2672    ptr           next character in pattern    ptr           next character in pattern
2673    options       options bits    options       options bits
2674    cd            contains pointers to tables etc.    cd            contains pointers to tables etc.
# Line 2121  Returns:        TRUE if possessifying is Line 2677  Returns:        TRUE if possessifying is
2677  */  */
2678    
2679  static BOOL  static BOOL
2680  check_auto_possessive(int op_code, int item, BOOL utf8, uschar *utf8_char,  check_auto_possessive(const uschar *previous, BOOL utf8, const uschar *ptr,
2681    const uschar *ptr, int options, compile_data *cd)    int options, compile_data *cd)
2682  {  {
2683  int next;  int c, next;
2684    int op_code = *previous++;
2685    
2686  /* Skip whitespace and comments in extended mode */  /* Skip whitespace and comments in extended mode */
2687    
# Line 2135  if ((options & PCRE_EXTENDED) != 0) Line 2692  if ((options & PCRE_EXTENDED) != 0)
2692      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
2693      if (*ptr == CHAR_NUMBER_SIGN)      if (*ptr == CHAR_NUMBER_SIGN)
2694        {        {
2695        while (*(++ptr) != 0)        ptr++;
2696          while (*ptr != 0)
2697            {
2698          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
2699            ptr++;
2700    #ifdef SUPPORT_UTF8
2701            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
2702    #endif
2703            }
2704        }        }
2705      else break;      else break;
2706      }      }
# Line 2172  if ((options & PCRE_EXTENDED) != 0) Line 2736  if ((options & PCRE_EXTENDED) != 0)
2736      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
2737      if (*ptr == CHAR_NUMBER_SIGN)      if (*ptr == CHAR_NUMBER_SIGN)
2738        {        {
2739        while (*(++ptr) != 0)        ptr++;
2740          while (*ptr != 0)
2741            {
2742          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
2743            ptr++;
2744    #ifdef SUPPORT_UTF8
2745            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
2746    #endif
2747            }
2748        }        }
2749      else break;      else break;
2750      }      }
# Line 2185  if (*ptr == CHAR_ASTERISK || *ptr == CHA Line 2756  if (*ptr == CHAR_ASTERISK || *ptr == CHA
2756    strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)    strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
2757      return FALSE;      return FALSE;
2758    
2759  /* Now compare the next item with the previous opcode. If the previous is a  /* Now compare the next item with the previous opcode. First, handle cases when
2760  positive single character match, "item" either contains the character or, if  the next item is a character. */
 "item" is greater than 127 in utf8 mode, the character's bytes are in  
 utf8_char. */  
   
   
 /* Handle cases when the next item is a character. */  
2761    
2762  if (next >= 0) switch(op_code)  if (next >= 0) switch(op_code)
2763    {    {
2764    case OP_CHAR:    case OP_CHAR:
2765  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2766    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2767  #else  #else
2768    (void)(utf8_char);  /* Keep compiler happy by referencing function argument */    c = *previous;
2769  #endif  #endif
2770    return item != next;    return c != next;
2771    
2772    /* For CHARNC (caseless character) we must check the other case. If we have    /* For CHARI (caseless character) we must check the other case. If we have
2773    Unicode property support, we can use it to test the other case of    Unicode property support, we can use it to test the other case of
2774    high-valued characters. */    high-valued characters. */
2775    
2776    case OP_CHARNC:    case OP_CHARI:
2777  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2778    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2779    #else
2780      c = *previous;
2781  #endif  #endif
2782    if (item == next) return FALSE;    if (c == next) return FALSE;
2783  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2784    if (utf8)    if (utf8)
2785      {      {
# Line 2222  if (next >= 0) switch(op_code) Line 2790  if (next >= 0) switch(op_code)
2790  #else  #else
2791      othercase = NOTACHAR;      othercase = NOTACHAR;
2792  #endif  #endif
2793      return (unsigned int)item != othercase;      return (unsigned int)c != othercase;
2794      }      }
2795    else    else
2796  #endif  /* SUPPORT_UTF8 */  #endif  /* SUPPORT_UTF8 */
2797    return (item != cd->fcc[next]);  /* Non-UTF-8 mode */    return (c != cd->fcc[next]);  /* Non-UTF-8 mode */
2798    
2799    /* For OP_NOT, "item" must be a single-byte character. */    /* For OP_NOT and OP_NOTI, the data is always a single-byte character. These
2800      opcodes are not used for multi-byte characters, because they are coded using
2801      an XCLASS instead. */
2802    
2803    case OP_NOT:    case OP_NOT:
2804    if (item == next) return TRUE;    return (c = *previous) == next;
2805    if ((options & PCRE_CASELESS) == 0) return FALSE;  
2806      case OP_NOTI:
2807      if ((c = *previous) == next) return TRUE;
2808  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2809    if (utf8)    if (utf8)
2810      {      {
# Line 2243  if (next >= 0) switch(op_code) Line 2815  if (next >= 0) switch(op_code)
2815  #else  #else
2816      othercase = NOTACHAR;      othercase = NOTACHAR;
2817  #endif  #endif
2818      return (unsigned int)item == othercase;      return (unsigned int)c == othercase;
2819      }      }
2820    else    else
2821  #endif  /* SUPPORT_UTF8 */  #endif  /* SUPPORT_UTF8 */
2822    return (item == cd->fcc[next]);  /* Non-UTF-8 mode */    return (c == cd->fcc[next]);  /* Non-UTF-8 mode */
2823    
2824      /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* set.
2825      When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */
2826    
2827    case OP_DIGIT:    case OP_DIGIT:
2828    return next > 127 || (cd->ctypes[next] & ctype_digit) == 0;    return next > 127 || (cd->ctypes[next] & ctype_digit) == 0;
# Line 2290  if (next >= 0) switch(op_code) Line 2865  if (next >= 0) switch(op_code)
2865      case 0x202f:      case 0x202f:
2866      case 0x205f:      case 0x205f:
2867      case 0x3000:      case 0x3000:
2868      return op_code != OP_HSPACE;      return op_code == OP_NOT_HSPACE;
2869      default:      default:
2870      return op_code == OP_HSPACE;      return op_code != OP_NOT_HSPACE;
2871      }      }
2872    
2873      case OP_ANYNL:
2874    case OP_VSPACE:    case OP_VSPACE:
2875    case OP_NOT_VSPACE:    case OP_NOT_VSPACE:
2876    switch(next)    switch(next)
# Line 2306  if (next >= 0) switch(op_code) Line 2882  if (next >= 0) switch(op_code)
2882      case 0x85:      case 0x85:
2883      case 0x2028:      case 0x2028:
2884      case 0x2029:      case 0x2029:
2885      return op_code != OP_VSPACE;      return op_code == OP_NOT_VSPACE;
2886      default:      default:
2887      return op_code == OP_VSPACE;      return op_code != OP_NOT_VSPACE;
2888      }      }
2889    
2890    #ifdef SUPPORT_UCP
2891      case OP_PROP:
2892      return check_char_prop(next, previous[0], previous[1], FALSE);
2893    
2894      case OP_NOTPROP:
2895      return check_char_prop(next, previous[0], previous[1], TRUE);
2896    #endif
2897    
2898    default:    default:
2899    return FALSE;    return FALSE;
2900    }    }
2901    
2902    
2903  /* Handle the case when the next item is \d, \s, etc. */  /* Handle the case when the next item is \d, \s, etc. Note that when PCRE_UCP
2904    is set, \d turns into ESC_du rather than ESC_d, etc., so ESC_d etc. are
2905    generated only when PCRE_UCP is *not* set, that is, when only ASCII
2906    characteristics are recognized. Similarly, the opcodes OP_DIGIT etc. are
2907    replaced by OP_PROP codes when PCRE_UCP is set. */
2908    
2909  switch(op_code)  switch(op_code)
2910    {    {
2911    case OP_CHAR:    case OP_CHAR:
2912    case OP_CHARNC:    case OP_CHARI:
2913  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2914    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2915    #else
2916      c = *previous;
2917  #endif  #endif
2918    switch(-next)    switch(-next)
2919      {      {
2920      case ESC_d:      case ESC_d:
2921      return item > 127 || (cd->ctypes[item] & ctype_digit) == 0;      return c > 127 || (cd->ctypes[c] & ctype_digit) == 0;
2922    
2923      case ESC_D:      case ESC_D:
2924      return item <= 127 && (cd->ctypes[item] & ctype_digit) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_digit) != 0;
2925    
2926      case ESC_s:      case ESC_s:
2927      return item > 127 || (cd->ctypes[item] & ctype_space) == 0;      return c > 127 || (cd->ctypes[c] & ctype_space) == 0;
2928    
2929      case ESC_S:      case ESC_S:
2930      return item <= 127 && (cd->ctypes[item] & ctype_space) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_space) != 0;
2931    
2932      case ESC_w:      case ESC_w:
2933      return item > 127 || (cd->ctypes[item] & ctype_word) == 0;      return c > 127 || (cd->ctypes[c] & ctype_word) == 0;
2934    
2935      case ESC_W:      case ESC_W:
2936      return item <= 127 && (cd->ctypes[item] & ctype_word) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_word) != 0;
2937    
2938      case ESC_h:      case ESC_h:
2939      case ESC_H:      case ESC_H:
2940      switch(item)      switch(c)
2941        {        {
2942        case 0x09:        case 0x09:
2943        case 0x20:        case 0x20:
# Line 2375  switch(op_code) Line 2965  switch(op_code)
2965    
2966      case ESC_v:      case ESC_v:
2967      case ESC_V:      case ESC_V:
2968      switch(item)      switch(c)
2969        {        {
2970        case 0x0a:        case 0x0a:
2971        case 0x0b:        case 0x0b:
# Line 2389  switch(op_code) Line 2979  switch(op_code)
2979        return -next == ESC_v;        return -next == ESC_v;
2980        }        }
2981    
2982        /* When PCRE_UCP is set, these values get generated for \d etc. Find
2983        their substitutions and process them. The result will always be either
2984        -ESC_p or -ESC_P. Then fall through to process those values. */
2985    
2986    #ifdef SUPPORT_UCP
2987        case ESC_du:
2988        case ESC_DU:
2989        case ESC_wu:
2990        case ESC_WU:
2991        case ESC_su:
2992        case ESC_SU:
2993          {
2994          int temperrorcode = 0;
2995          ptr = substitutes[-next - ESC_DU];
2996          next = check_escape(&ptr, &temperrorcode, 0, options, FALSE);
2997          if (temperrorcode != 0) return FALSE;
2998          ptr++;    /* For compatibility */
2999          }
3000        /* Fall through */
3001    
3002        case ESC_p:
3003        case ESC_P:
3004          {
3005          int ptype, pdata, errorcodeptr;
3006          BOOL negated;
3007    
3008          ptr--;      /* Make ptr point at the p or P */
3009          ptype = get_ucp(&ptr, &negated, &pdata, &errorcodeptr);
3010          if (ptype < 0) return FALSE;
3011          ptr++;      /* Point past the final curly ket */
3012    
3013          /* If the property item is optional, we have to give up. (When generated
3014          from \d etc by PCRE_UCP, this test will have been applied much earlier,
3015          to the original \d etc. At this point, ptr will point to a zero byte. */
3016    
3017          if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK ||
3018            strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
3019              return FALSE;
3020    
3021          /* Do the property check. */
3022    
3023          return check_char_prop(c, ptype, pdata, (next == -ESC_P) != negated);
3024          }
3025    #endif
3026    
3027      default:      default:
3028      return FALSE;      return FALSE;
3029      }      }
3030    
3031      /* In principle, support for Unicode properties should be integrated here as
3032      well. It means re-organizing the above code so as to get hold of the property
3033      values before switching on the op-code. However, I wonder how many patterns
3034      combine ASCII \d etc with Unicode properties? (Note that if PCRE_UCP is set,
3035      these op-codes are never generated.) */
3036    
3037    case OP_DIGIT:    case OP_DIGIT:
3038    return next == -ESC_D || next == -ESC_s || next == -ESC_W ||    return next == -ESC_D || next == -ESC_s || next == -ESC_W ||
3039           next == -ESC_h || next == -ESC_v;           next == -ESC_h || next == -ESC_v || next == -ESC_R;
3040    
3041    case OP_NOT_DIGIT:    case OP_NOT_DIGIT:
3042    return next == -ESC_d;    return next == -ESC_d;
3043    
3044    case OP_WHITESPACE:    case OP_WHITESPACE:
3045    return next == -ESC_S || next == -ESC_d || next == -ESC_w;    return next == -ESC_S || next == -ESC_d || next == -ESC_w || next == -ESC_R;
3046    
3047    case OP_NOT_WHITESPACE:    case OP_NOT_WHITESPACE:
3048    return next == -ESC_s || next == -ESC_h || next == -ESC_v;    return next == -ESC_s || next == -ESC_h || next == -ESC_v;
3049    
3050    case OP_HSPACE:    case OP_HSPACE:
3051    return next == -ESC_S || next == -ESC_H || next == -ESC_d || next == -ESC_w;    return next == -ESC_S || next == -ESC_H || next == -ESC_d ||
3052             next == -ESC_w || next == -ESC_v || next == -ESC_R;
3053    
3054    case OP_NOT_HSPACE:    case OP_NOT_HSPACE:
3055    return next == -ESC_h;    return next == -ESC_h;
3056    
3057    /* Can't have \S in here because VT matches \S (Perl anomaly) */    /* Can't have \S in here because VT matches \S (Perl anomaly) */
3058      case OP_ANYNL:
3059    case OP_VSPACE:    case OP_VSPACE:
3060    return next == -ESC_V || next == -ESC_d || next == -ESC_w;    return next == -ESC_V || next == -ESC_d || next == -ESC_w;
3061    
3062    case OP_NOT_VSPACE:    case OP_NOT_VSPACE:
3063    return next == -ESC_v;    return next == -ESC_v || next == -ESC_R;
3064    
3065    case OP_WORDCHAR:    case OP_WORDCHAR:
3066    return next == -ESC_W || next == -ESC_s || next == -ESC_h || next == -ESC_v;    return next == -ESC_W || next == -ESC_s || next == -ESC_h ||
3067             next == -ESC_v || next == -ESC_R;
3068    
3069    case OP_NOT_WORDCHAR:    case OP_NOT_WORDCHAR:
3070    return next == -ESC_w || next == -ESC_d;    return next == -ESC_w || next == -ESC_d;
# Line 2452  Arguments: Line 3096  Arguments:
3096    firstbyteptr   set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE)    firstbyteptr   set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE)
3097    reqbyteptr     set to the last literal character required, else < 0    reqbyteptr     set to the last literal character required, else < 0
3098    bcptr          points to current branch chain    bcptr          points to current branch chain
3099      cond_depth     conditional nesting depth
3100    cd             contains pointers to tables etc.    cd             contains pointers to tables etc.
3101    lengthptr      NULL during the real compile phase    lengthptr      NULL during the real compile phase
3102                   points to length accumulator during pre-compile phase                   points to length accumulator during pre-compile phase
# Line 2463  Returns:         TRUE on success Line 3108  Returns:         TRUE on success
3108  static BOOL  static BOOL
3109  compile_branch(int *optionsptr, uschar **codeptr, const uschar **ptrptr,  compile_branch(int *optionsptr, uschar **codeptr, const uschar **ptrptr,
3110    int *errorcodeptr, int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr,    int *errorcodeptr, int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr,
3111    compile_data *cd, int *lengthptr)    int cond_depth, compile_data *cd, int *lengthptr)
3112  {  {
3113  int repeat_type, op_type;  int repeat_type, op_type;
3114  int repeat_min = 0, repeat_max = 0;      /* To please picky compilers */  int repeat_min = 0, repeat_max = 0;      /* To please picky compilers */
# Line 2472  int greedy_default, greedy_non_default; Line 3117  int greedy_default, greedy_non_default;
3117  int firstbyte, reqbyte;  int firstbyte, reqbyte;
3118  int zeroreqbyte, zerofirstbyte;  int zeroreqbyte, zerofirstbyte;
3119  int req_caseopt, reqvary, tempreqvary;  int req_caseopt, reqvary, tempreqvary;
3120  int options = *optionsptr;  int options = *optionsptr;               /* May change dynamically */
3121  int after_manual_callout = 0;  int after_manual_callout = 0;
3122  int length_prevgroup = 0;  int length_prevgroup = 0;
3123  register int c;  register int c;
# Line 2484  BOOL inescq = FALSE; Line 3129  BOOL inescq = FALSE;
3129  BOOL groupsetfirstbyte = FALSE;  BOOL groupsetfirstbyte = FALSE;
3130  const uschar *ptr = *ptrptr;  const uschar *ptr = *ptrptr;
3131  const uschar *tempptr;  const uschar *tempptr;
3132    const uschar *nestptr = NULL;
3133  uschar *previous = NULL;  uschar *previous = NULL;
3134  uschar *previous_callout = NULL;  uschar *previous_callout = NULL;
3135  uschar *save_hwm = NULL;  uschar *save_hwm = NULL;
3136  uschar classbits[32];  uschar classbits[32];
3137    
3138    /* We can fish out the UTF-8 setting once and for all into a BOOL, but we
3139    must not do this for other options (e.g. PCRE_EXTENDED) because they may change
3140    dynamically as we process the pattern. */
3141    
3142  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
3143  BOOL class_utf8;  BOOL class_utf8;
3144  BOOL utf8 = (options & PCRE_UTF8) != 0;  BOOL utf8 = (options & PCRE_UTF8) != 0;
# Line 2497  uschar *class_utf8data_base; Line 3147  uschar *class_utf8data_base;
3147  uschar utf8_char[6];  uschar utf8_char[6];
3148  #else  #else
3149  BOOL utf8 = FALSE;  BOOL utf8 = FALSE;
 uschar *utf8_char = NULL;  
3150  #endif  #endif
3151    
3152  #ifdef DEBUG  #ifdef PCRE_DEBUG
3153  if (lengthptr != NULL) DPRINTF((">> start branch\n"));  if (lengthptr != NULL) DPRINTF((">> start branch\n"));
3154  #endif  #endif
3155    
# Line 2548  for (;; ptr++) Line 3197  for (;; ptr++)
3197    int subfirstbyte;    int subfirstbyte;
3198    int terminator;    int terminator;
3199    int mclength;    int mclength;
3200      int tempbracount;
3201    uschar mcbuffer[8];    uschar mcbuffer[8];
3202    
3203    /* Get next byte in the pattern */    /* Get next byte in the pattern */
3204    
3205    c = *ptr;    c = *ptr;
3206    
3207      /* If we are at the end of a nested substitution, revert to the outer level
3208      string. Nesting only happens one level deep. */
3209    
3210      if (c == 0 && nestptr != NULL)
3211        {
3212        ptr = nestptr;
3213        nestptr = NULL;
3214        c = *ptr;
3215        }
3216    
3217    /* If we are in the pre-compile phase, accumulate the length used for the    /* If we are in the pre-compile phase, accumulate the length used for the
3218    previous cycle of this loop. */    previous cycle of this loop. */
3219    
3220    if (lengthptr != NULL)    if (lengthptr != NULL)
3221      {      {
3222  #ifdef DEBUG  #ifdef PCRE_DEBUG
3223      if (code > cd->hwm) cd->hwm = code;                 /* High water info */      if (code > cd->hwm) cd->hwm = code;                 /* High water info */
3224  #endif  #endif
3225      if (code > cd->start_workspace + COMPILE_WORK_SIZE) /* Check for overrun */      if (code > cd->start_workspace + WORK_SIZE_CHECK)   /* Check for overrun */
3226        {        {
3227        *errorcodeptr = ERR52;        *errorcodeptr = ERR52;
3228        goto FAILED;        goto FAILED;
# Line 2584  for (;; ptr++) Line 3244  for (;; ptr++)
3244        goto FAILED;        goto FAILED;
3245        }        }
3246    
3247      *lengthptr += code - last_code;      *lengthptr += (int)(code - last_code);
3248      DPRINTF(("length=%d added %d c=%c\n", *lengthptr, code - last_code, c));      DPRINTF(("length=%d added %d c=%c\n", *lengthptr, code - last_code, c));
3249    
3250      /* If "previous" is set and it is not at the start of the work space, move      /* If "previous" is set and it is not at the start of the work space, move
# Line 2611  for (;; ptr++) Line 3271  for (;; ptr++)
3271    /* In the real compile phase, just check the workspace used by the forward    /* In the real compile phase, just check the workspace used by the forward
3272    reference list. */    reference list. */
3273    
3274    else if (cd->hwm > cd->start_workspace + COMPILE_WORK_SIZE)    else if (cd->hwm > cd->start_workspace + WORK_SIZE_CHECK)
3275      {      {
3276      *errorcodeptr = ERR52;      *errorcodeptr = ERR52;
3277      goto FAILED;      goto FAILED;
# Line 2659  for (;; ptr++) Line 3319  for (;; ptr++)
3319      previous_callout = NULL;      previous_callout = NULL;
3320      }      }
3321    
3322    /* In extended mode, skip white space and comments */    /* In extended mode, skip white space and comments. */
3323    
3324    if ((options & PCRE_EXTENDED) != 0)    if ((options & PCRE_EXTENDED) != 0)
3325      {      {
3326      if ((cd->ctypes[c] & ctype_space) != 0) continue;      if ((cd->ctypes[c] & ctype_space) != 0) continue;
3327      if (c == CHAR_NUMBER_SIGN)      if (c == CHAR_NUMBER_SIGN)
3328        {        {
3329        while (*(++ptr) != 0)        ptr++;
3330          while (*ptr != 0)
3331          {          {
3332          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
3333            ptr++;
3334    #ifdef SUPPORT_UTF8
3335            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
3336    #endif
3337          }          }
3338        if (*ptr != 0) continue;        if (*ptr != 0) continue;
3339    
# Line 2702  for (;; ptr++) Line 3367  for (;; ptr++)
3367          *errorcodeptr = ERR20;          *errorcodeptr = ERR20;
3368          goto FAILED;          goto FAILED;
3369          }          }
3370        *lengthptr += code - last_code;   /* To include callout length */        *lengthptr += (int)(code - last_code);   /* To include callout length */
3371        DPRINTF((">> end branch\n"));        DPRINTF((">> end branch\n"));
3372        }        }
3373      return TRUE;      return TRUE;
# Line 2713  for (;; ptr++) Line 3378  for (;; ptr++)
3378      the setting of any following char as a first character. */      the setting of any following char as a first character. */
3379    
3380      case CHAR_CIRCUMFLEX_ACCENT:      case CHAR_CIRCUMFLEX_ACCENT:
3381        previous = NULL;
3382      if ((options & PCRE_MULTILINE) != 0)      if ((options & PCRE_MULTILINE) != 0)
3383        {        {
3384        if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;        if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
3385          *code++ = OP_CIRCM;
3386        }        }
3387      previous = NULL;      else *code++ = OP_CIRC;
     *code++ = OP_CIRC;  
3388      break;      break;
3389    
3390      case CHAR_DOLLAR_SIGN:      case CHAR_DOLLAR_SIGN:
3391      previous = NULL;      previous = NULL;
3392      *code++ = OP_DOLL;      *code++ = ((options & PCRE_MULTILINE) != 0)? OP_DOLLM : OP_DOLL;
3393      break;      break;
3394    
3395      /* There can never be a first char if '.' is first, whatever happens about      /* There can never be a first char if '.' is first, whatever happens about
# Line 2907  for (;; ptr++) Line 3573  for (;; ptr++)
3573            ptr++;            ptr++;
3574            }            }
3575    
3576          posix_class = check_posix_name(ptr, tempptr - ptr);          posix_class = check_posix_name(ptr, (int)(tempptr - ptr));
3577          if (posix_class < 0)          if (posix_class < 0)
3578            {            {
3579            *errorcodeptr = ERR30;            *errorcodeptr = ERR30;
# Line 2921  for (;; ptr++) Line 3587  for (;; ptr++)
3587          if ((options & PCRE_CASELESS) != 0 && posix_class <= 2)          if ((options & PCRE_CASELESS) != 0 && posix_class <= 2)
3588            posix_class = 0;            posix_class = 0;
3589    
3590          /* We build the bit map for the POSIX class in a chunk of local store          /* When PCRE_UCP is set, some of the POSIX classes are converted to
3591          because we may be adding and subtracting from it, and we don't want to          different escape sequences that use Unicode properties. */
3592          subtract bits that may be in the main map already. At the end we or the  
3593          result into the bit map that is being built. */  #ifdef SUPPORT_UCP
3594            if ((options & PCRE_UCP) != 0)
3595              {
3596              int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0);
3597              if (posix_substitutes[pc] != NULL)
3598                {
3599                nestptr = tempptr + 1;
3600                ptr = posix_substitutes[pc] - 1;
3601                continue;
3602                }
3603              }
3604    #endif
3605            /* In the non-UCP case, we build the bit map for the POSIX class in a
3606            chunk of local store because we may be adding and subtracting from it,
3607            and we don't want to subtract bits that may be in the main map already.
3608            At the end we or the result into the bit map that is being built. */
3609    
3610          posix_class *= 3;          posix_class *= 3;
3611    
# Line 2968  for (;; ptr++) Line 3649  for (;; ptr++)
3649    
3650        /* Backslash may introduce a single character, or it may introduce one        /* Backslash may introduce a single character, or it may introduce one
3651        of the specials, which just set a flag. The sequence \b is a special        of the specials, which just set a flag. The sequence \b is a special
3652        case. Inside a class (and only there) it is treated as backspace.        case. Inside a class (and only there) it is treated as backspace. We
3653        Elsewhere it marks a word boundary. Other escapes have preset maps ready        assume that other escapes have more than one character in them, so set
3654        to 'or' into the one we are building. We assume they have more than one        class_charcount bigger than one. Unrecognized escapes fall through and
3655        character in them, so set class_charcount bigger than one. */        are either treated as literal characters (by default), or are faulted if
3656          PCRE_EXTRA is set. */
3657    
3658        if (c == CHAR_BACKSLASH)        if (c == CHAR_BACKSLASH)
3659          {          {
3660          c = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE);          c = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE);
3661          if (*errorcodeptr != 0) goto FAILED;          if (*errorcodeptr != 0) goto FAILED;
3662    
3663          if (-c == ESC_b) c = CHAR_BS;       /* \b is backspace in a class */          if (-c == ESC_b) c = CHAR_BS;    /* \b is backspace in a class */
         else if (-c == ESC_X) c = CHAR_X;   /* \X is literal X in a class */  
         else if (-c == ESC_R) c = CHAR_R;   /* \R is literal R in a class */  
3664          else if (-c == ESC_Q)            /* Handle start of quoted string */          else if (-c == ESC_Q)            /* Handle start of quoted string */
3665            {            {
3666            if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E)            if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E)
# Line 2997  for (;; ptr++) Line 3677  for (;; ptr++)
3677            register const uschar *cbits = cd->cbits;            register const uschar *cbits = cd->cbits;
3678            class_charcount += 2;     /* Greater than 1 is what matters */            class_charcount += 2;     /* Greater than 1 is what matters */
3679    
3680            /* Save time by not doing this in the pre-compile phase. */            switch (-c)
   
           if (lengthptr == NULL) switch (-c)  
3681              {              {
3682    #ifdef SUPPORT_UCP
3683                case ESC_du:     /* These are the values given for \d etc */
3684                case ESC_DU:     /* when PCRE_UCP is set. We replace the */
3685                case ESC_wu:     /* escape sequence with an appropriate \p */
3686                case ESC_WU:     /* or \P to test Unicode properties instead */
3687                case ESC_su:     /* of the default ASCII testing. */
3688                case ESC_SU:
3689                nestptr = ptr;
3690                ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */
3691                class_charcount -= 2;                /* Undo! */
3692                continue;
3693    #endif
3694              case ESC_d:              case ESC_d:
3695              for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_digit];              for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_digit];
3696              continue;              continue;
# Line 3019  for (;; ptr++) Line 3709  for (;; ptr++)
3709              for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word];              for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word];
3710              continue;              continue;
3711    
3712                /* Perl 5.004 onwards omits VT from \s, but we must preserve it
3713                if it was previously set by something earlier in the character
3714                class. */
3715    
3716              case ESC_s:              case ESC_s:
3717              for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space];              classbits[0] |= cbits[cbit_space];
3718              classbits[1] &= ~0x08;   /* Perl 5.004 onwards omits VT from \s */              classbits[1] |= cbits[cbit_space+1] & ~0x08;
3719                for (c = 2; c < 32; c++) classbits[c] |= cbits[c+cbit_space];
3720              continue;              continue;
3721    
3722              case ESC_S:              case ESC_S:
# Line 3030  for (;; ptr++) Line 3725  for (;; ptr++)
3725              classbits[1] |= 0x08;    /* Perl 5.004 onwards omits VT from \s */              classbits[1] |= 0x08;    /* Perl 5.004 onwards omits VT from \s */
3726              continue;              continue;
3727    
3728              default:    /* Not recognized; fall through */              case ESC_h:
             break;      /* Need "default" setting to stop compiler warning. */  
             }  
   
           /* In the pre-compile phase, just do the recognition. */  
   
           else if (c == -ESC_d || c == -ESC_D || c == -ESC_w ||  
                    c == -ESC_W || c == -ESC_s || c == -ESC_S) continue;  
   
           /* We need to deal with \H, \h, \V, and \v in both phases because  
           they use extra memory. */  
   
           if (-c == ESC_h)  
             {  
3729              SETBIT(classbits, 0x09); /* VT */              SETBIT(classbits, 0x09); /* VT */
3730              SETBIT(classbits, 0x20); /* SPACE */              SETBIT(classbits, 0x20); /* SPACE */
3731              SETBIT(classbits, 0xa0); /* NSBP */              SETBIT(classbits, 0xa0); /* NSBP */
# Line 3067  for (;; ptr++) Line 3749  for (;; ptr++)
3749                }                }
3750  #endif  #endif
3751              continue;              continue;
             }  
3752    
3753            if (-c == ESC_H)              case ESC_H:
             {  
3754              for (c = 0; c < 32; c++)              for (c = 0; c < 32; c++)
3755                {                {
3756                int x = 0xff;                int x = 0xff;
# Line 3112  for (;; ptr++) Line 3792  for (;; ptr++)
3792                }                }
3793  #endif  #endif
3794              continue;              continue;
             }  
3795    
3796            if (-c == ESC_v)              case ESC_v:
             {  
3797              SETBIT(classbits, 0x0a); /* LF */              SETBIT(classbits, 0x0a); /* LF */
3798              SETBIT(classbits, 0x0b); /* VT */              SETBIT(classbits, 0x0b); /* VT */
3799              SETBIT(classbits, 0x0c); /* FF */              SETBIT(classbits, 0x0c); /* FF */
# Line 3131  for (;; ptr++) Line 3809  for (;; ptr++)
3809                }                }
3810  #endif  #endif
3811              continue;              continue;
             }  
3812    
3813            if (-c == ESC_V)              case ESC_V:
             {  
3814              for (c = 0; c < 32; c++)              for (c = 0; c < 32; c++)
3815                {                {
3816                int x = 0xff;                int x = 0xff;
# Line 3164  for (;; ptr++) Line 3840  for (;; ptr++)
3840                }                }
3841  #endif  #endif
3842              continue;              continue;
             }  
   
           /* We need to deal with \P and \p in both phases. */  
3843    
3844  #ifdef SUPPORT_UCP  #ifdef SUPPORT_UCP
3845            if (-c == ESC_p || -c == ESC_P)              case ESC_p:
3846              {              case ESC_P:
3847              BOOL negated;                {
3848              int pdata;                BOOL negated;
3849              int ptype = get_ucp(&ptr, &negated, &pdata, errorcodeptr);                int pdata;
3850              if (ptype < 0) goto FAILED;                int ptype = get_ucp(&ptr, &negated, &pdata, errorcodeptr);
3851              class_utf8 = TRUE;                if (ptype < 0) goto FAILED;
3852              *class_utf8data++ = ((-c == ESC_p) != negated)?                class_utf8 = TRUE;
3853                XCL_PROP : XCL_NOTPROP;                *class_utf8data++ = ((-c == ESC_p) != negated)?
3854              *class_utf8data++ = ptype;                  XCL_PROP : XCL_NOTPROP;
3855              *class_utf8data++ = pdata;                *class_utf8data++ = ptype;
3856              class_charcount -= 2;   /* Not a < 256 character */                *class_utf8data++ = pdata;
3857              continue;                class_charcount -= 2;   /* Not a < 256 character */
3858              }                continue;
3859                  }
3860  #endif  #endif
3861            /* Unrecognized escapes are faulted if PCRE is running in its              /* Unrecognized escapes are faulted if PCRE is running in its
3862            strict mode. By default, for compatibility with Perl, they are              strict mode. By default, for compatibility with Perl, they are
3863            treated as literals. */              treated as literals. */
3864    
3865            if ((options & PCRE_EXTRA) != 0)              default:
3866              {              if ((options & PCRE_EXTRA) != 0)
3867              *errorcodeptr = ERR7;                {
3868              goto FAILED;                *errorcodeptr = ERR7;
3869                  goto FAILED;
3870                  }
3871                class_charcount -= 2;  /* Undo the default count from above */
3872                c = *ptr;              /* Get the final character and fall through */
3873                break;
3874              }              }
   
           class_charcount -= 2;  /* Undo the default count from above */  
           c = *ptr;              /* Get the final character and fall through */  
3875            }            }
3876    
3877          /* Fall through if we have a single character (c >= 0). This may be          /* Fall through if we have a single character (c >= 0). This may be
# Line 3265  for (;; ptr++) Line 3941  for (;; ptr++)
3941            d = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE);            d = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE);
3942            if (*errorcodeptr != 0) goto FAILED;            if (*errorcodeptr != 0) goto FAILED;
3943    
3944            /* \b is backspace; \X is literal X; \R is literal R; any other            /* \b is backspace; any other special means the '-' was literal */
           special means the '-' was literal */  
3945    
3946            if (d < 0)            if (d < 0)
3947              {              {
3948              if (d == -ESC_b) d = CHAR_BS;              if (d == -ESC_b) d = CHAR_BS; else
             else if (d == -ESC_X) d = CHAR_X;  
             else if (d == -ESC_R) d = CHAR_R; else  
3949                {                {
3950                ptr = oldptr;                ptr = oldptr;
3951                goto LONE_SINGLE_CHARACTER;  /* A few lines below */                goto LONE_SINGLE_CHARACTER;  /* A few lines below */
# Line 3438  for (;; ptr++) Line 4111  for (;; ptr++)
4111          }          }
4112        }        }
4113    
4114      /* Loop until ']' reached. This "while" is the end of the "do" above. */      /* Loop until ']' reached. This "while" is the end of the "do" far above.
4115        If we are at the end of an internal nested string, revert to the outer
4116        string. */
4117    
4118        while (((c = *(++ptr)) != 0 ||
4119               (nestptr != NULL &&
4120                 (ptr = nestptr, nestptr = NULL, c = *(++ptr)) != 0)) &&
4121               (c != CHAR_RIGHT_SQUARE_BRACKET || inescq));
4122    
4123      while ((c = *(++ptr)) != 0 && (c != CHAR_RIGHT_SQUARE_BRACKET || inescq));      /* Check for missing terminating ']' */
4124    
4125      if (c == 0)                          /* Missing terminating ']' */      if (c == 0)
4126        {        {
4127        *errorcodeptr = ERR6;        *errorcodeptr = ERR6;
4128        goto FAILED;        goto FAILED;
4129        }        }
4130    
   
 /* This code has been disabled because it would mean that \s counts as  
 an explicit \r or \n reference, and that's not really what is wanted. Now  
 we set the flag only if there is a literal "\r" or "\n" in the class. */  
   
 #if 0  
     /* Remember whether \r or \n are in this class */  
   
     if (negate_class)  
       {  
       if ((classbits[1] & 0x24) != 0x24) cd->external_flags |= PCRE_HASCRORLF;  
       }  
     else  
       {  
       if ((classbits[1] & 0x24) != 0) cd->external_flags |= PCRE_HASCRORLF;  
       }  
 #endif  
   
   
4131      /* If class_charcount is 1, we saw precisely one character whose value is      /* If class_charcount is 1, we saw precisely one character whose value is
4132      less than 256. As long as there were no characters >= 128 and there was no      less than 256. As long as there were no characters >= 128 and there was no
4133      use of \p or \P, in other words, no use of any XCLASS features, we can      use of \p or \P, in other words, no use of any XCLASS features, we can
# Line 3474  we set the flag only if there is a liter Line 4135  we set the flag only if there is a liter
4135    
4136      In UTF-8 mode, we can optimize the negative case only if there were no      In UTF-8 mode, we can optimize the negative case only if there were no
4137      characters >= 128 because OP_NOT and the related opcodes like OP_NOTSTAR      characters >= 128 because OP_NOT and the related opcodes like OP_NOTSTAR
4138      operate on single-bytes only. This is an historical hangover. Maybe one day      operate on single-bytes characters only. This is an historical hangover.
4139      we can tidy these opcodes to handle multi-byte characters.      Maybe one day we can tidy these opcodes to handle multi-byte characters.
4140    
4141      The optimization throws away the bit map. We turn the item into a      The optimization throws away the bit map. We turn the item into a
4142      1-character OP_CHAR[NC] if it's positive, or OP_NOT if it's negative. Note      1-character OP_CHAR[I] if it's positive, or OP_NOT[I] if it's negative.
4143      that OP_NOT does not support multibyte characters. In the positive case, it      Note that OP_NOT[I] does not support multibyte characters. In the positive
4144      can cause firstbyte to be set. Otherwise, there can be no first char if      case, it can cause firstbyte to be set. Otherwise, there can be no first
4145      this item is first, whatever repeat count may follow. In the case of      char if this item is first, whatever repeat count may follow. In the case
4146      reqbyte, save the previous value for reinstating. */      of reqbyte, save the previous value for reinstating. */
4147    
4148  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
4149      if (class_charcount == 1 && !class_utf8 &&      if (class_charcount == 1 && !class_utf8 &&
# Line 3493  we set the flag only if there is a liter Line 4154  we set the flag only if there is a liter
4154        {        {
4155        zeroreqbyte = reqbyte;        zeroreqbyte = reqbyte;
4156    
4157        /* The OP_NOT opcode works on one-byte characters only. */        /* The OP_NOT[I] opcodes work on one-byte characters only. */
4158    
4159        if (negate_class)        if (negate_class)
4160          {          {
4161          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
4162          zerofirstbyte = firstbyte;          zerofirstbyte = firstbyte;
4163          *code++ = OP_NOT;          *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT;
4164          *code++ = class_lastchar;          *code++ = class_lastchar;
4165          break;          break;
4166          }          }
# Line 3530  we set the flag only if there is a liter Line 4191  we set the flag only if there is a liter
4191    
4192      /* If there are characters with values > 255, we have to compile an      /* If there are characters with values > 255, we have to compile an
4193      extended class, with its own opcode, unless there was a negated special      extended class, with its own opcode, unless there was a negated special
4194      such as \S in the class, because in that case all characters > 255 are in      such as \S in the class, and PCRE_UCP is not set, because in that case all
4195      the class, so any that were explicitly given as well can be ignored. If      characters > 255 are in the class, so any that were explicitly given as
4196      (when there are explicit characters > 255 that must be listed) there are no      well can be ignored. If (when there are explicit characters > 255 that must
4197      characters < 256, we can omit the bitmap in the actual compiled code. */      be listed) there are no characters < 256, we can omit the bitmap in the
4198        actual compiled code. */
4199    
4200  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
4201      if (class_utf8 && !should_flip_negation)      if (class_utf8 && (!should_flip_negation || (options & PCRE_UCP) != 0))
4202        {        {
4203        *class_utf8data++ = XCL_END;    /* Marks the end of extra data */        *class_utf8data++ = XCL_END;    /* Marks the end of extra data */
4204        *code++ = OP_XCLASS;        *code++ = OP_XCLASS;
# Line 3562  we set the flag only if there is a liter Line 4224  we set the flag only if there is a liter
4224        }        }
4225  #endif  #endif
4226    
4227      /* If there are no characters > 255, set the opcode to OP_CLASS or      /* If there are no characters > 255, or they are all to be included or
4228      OP_NCLASS, depending on whether the whole class was negated and whether      excluded, set the opcode to OP_CLASS or OP_NCLASS, depending on whether the
4229      there were negative specials such as \S in the class. Then copy the 32-byte      whole class was negated and whether there were negative specials such as \S
4230      map into the code vector, negating it if necessary. */      (non-UCP) in the class. Then copy the 32-byte map into the code vector,
4231        negating it if necessary. */
4232    
4233      *code++ = (negate_class == should_flip_negation) ? OP_CLASS : OP_NCLASS;      *code++ = (negate_class == should_flip_negation) ? OP_CLASS : OP_NCLASS;
4234      if (negate_class)      if (negate_class)
# Line 3625  we set the flag only if there is a liter Line 4288  we set the flag only if there is a liter
4288      op_type = 0;                    /* Default single-char op codes */      op_type = 0;                    /* Default single-char op codes */
4289      possessive_quantifier = FALSE;  /* Default not possessive quantifier */      possessive_quantifier = FALSE;  /* Default not possessive quantifier */
4290    
4291      /* Save start of previous item, in case we have to move it up to make space      /* Save start of previous item, in case we have to move it up in order to
4292      for an inserted OP_ONCE for the additional '+' extension. */      insert something before it. */
4293    
4294      tempcode = previous;      tempcode = previous;
4295    
# Line 3649  we set the flag only if there is a liter Line 4312  we set the flag only if there is a liter
4312        }        }
4313      else repeat_type = greedy_default;      else repeat_type = greedy_default;
4314    
4315        /* If previous was a recursion call, wrap it in atomic brackets so that
4316        previous becomes the atomic group. All recursions were so wrapped in the
4317        past, but it no longer happens for non-repeated recursions. In fact, the
4318        repeated ones could be re-implemented independently so as not to need this,
4319        but for the moment we rely on the code for repeating groups. */
4320    
4321        if (*previous == OP_RECURSE)
4322          {
4323          memmove(previous + 1 + LINK_SIZE, previous, 1 + LINK_SIZE);
4324          *previous = OP_ONCE;
4325          PUT(previous, 1, 2 + 2*LINK_SIZE);
4326          previous[2 + 2*LINK_SIZE] = OP_KET;
4327          PUT(previous, 3 + 2*LINK_SIZE, 2 + 2*LINK_SIZE);
4328          code += 2 + 2 * LINK_SIZE;
4329          length_prevgroup = 3 + 3*LINK_SIZE;
4330    
4331          /* When actually compiling, we need to check whether this was a forward
4332          reference, and if so, adjust the offset. */
4333    
4334          if (lengthptr == NULL && cd->hwm >= cd->start_workspace + LINK_SIZE)
4335            {
4336            int offset = GET(cd->hwm, -LINK_SIZE);
4337            if (offset == previous + 1 - cd->start_code)
4338              PUT(cd->hwm, -LINK_SIZE, offset + 1 + LINK_SIZE);
4339            }
4340          }
4341    
4342        /* Now handle repetition for the different types of item. */
4343    
4344      /* If previous was a character match, abolish the item and generate a      /* If previous was a character match, abolish the item and generate a
4345      repeat item instead. If a char item has a minumum of more than one, ensure      repeat item instead. If a char item has a minumum of more than one, ensure
4346      that it is set in reqbyte - it might not be if a sequence such as x{3} is      that it is set in reqbyte - it might not be if a sequence such as x{3} is
4347      the first thing in a branch because the x will have gone into firstbyte      the first thing in a branch because the x will have gone into firstbyte
4348      instead.  */      instead.  */
4349    
4350      if (*previous == OP_CHAR || *previous == OP_CHARNC)      if (*previous == OP_CHAR || *previous == OP_CHARI)
4351        {        {
4352          op_type = (*previous == OP_CHAR)? 0 : OP_STARI - OP_STAR;
4353    
4354        /* Deal with UTF-8 characters that take up more than one byte. It's        /* Deal with UTF-8 characters that take up more than one byte. It's
4355        easier to write this out separately than try to macrify it. Use c to        easier to write this out separately than try to macrify it. Use c to
4356        hold the length of the character in bytes, plus 0x80 to flag that it's a        hold the length of the character in bytes, plus 0x80 to flag that it's a
# Line 3689  we set the flag only if there is a liter Line 4383  we set the flag only if there is a liter
4383    
4384        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4385            repeat_max < 0 &&            repeat_max < 0 &&
4386            check_auto_possessive(*previous, c, utf8, utf8_char, ptr + 1,            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
             options, cd))  
4387          {          {
4388          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4389          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 3702  we set the flag only if there is a liter Line 4395  we set the flag only if there is a liter
4395      /* If previous was a single negated character ([^a] or similar), we use      /* If previous was a single negated character ([^a] or similar), we use
4396      one of the special opcodes, replacing it. The code is shared with single-      one of the special opcodes, replacing it. The code is shared with single-
4397      character repeats by setting opt_type to add a suitable offset into      character repeats by setting opt_type to add a suitable offset into
4398      repeat_type. We can also test for auto-possessification. OP_NOT is      repeat_type. We can also test for auto-possessification. OP_NOT and OP_NOTI
4399      currently used only for single-byte chars. */      are currently used only for single-byte chars. */
4400    
4401      else if (*previous == OP_NOT)      else if (*previous == OP_NOT || *previous == OP_NOTI)
4402        {        {
4403        op_type = OP_NOTSTAR - OP_STAR;  /* Use "not" opcodes */        op_type = ((*previous == OP_NOT)? OP_NOTSTAR : OP_NOTSTARI) - OP_STAR;
4404        c = previous[1];        c = previous[1];
4405        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4406            repeat_max < 0 &&            repeat_max < 0 &&
4407            check_auto_possessive(OP_NOT, c, utf8, NULL, ptr + 1, options, cd))            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
4408          {          {
4409          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4410          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 3735  we set the flag only if there is a liter Line 4428  we set the flag only if there is a liter
4428    
4429        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4430            repeat_max < 0 &&            repeat_max < 0 &&
4431            check_auto_possessive(c, 0, utf8, NULL, ptr + 1, options, cd))            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
4432          {          {
4433          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4434          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 3757  we set the flag only if there is a liter Line 4450  we set the flag only if there is a liter
4450    
4451        if (repeat_max == 0) goto END_REPEAT;        if (repeat_max == 0) goto END_REPEAT;
4452    
4453          /*--------------------------------------------------------------------*/
4454          /* This code is obsolete from release 8.00; the restriction was finally
4455          removed: */
4456    
4457        /* All real repeats make it impossible to handle partial matching (maybe        /* All real repeats make it impossible to handle partial matching (maybe
4458        one day we will be able to remove this restriction). */        one day we will be able to remove this restriction). */
4459    
4460        if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL;        /* if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL; */
4461          /*--------------------------------------------------------------------*/
4462    
4463        /* Combine the op_type with the repeat_type */        /* Combine the op_type with the repeat_type */
4464    
# Line 3899  we set the flag only if there is a liter Line 4597  we set the flag only if there is a liter
4597  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
4598               *previous == OP_XCLASS ||               *previous == OP_XCLASS ||
4599  #endif  #endif
4600               *previous == OP_REF)               *previous == OP_REF ||
4601                 *previous == OP_REFI)
4602        {        {
4603        if (repeat_max == 0)        if (repeat_max == 0)
4604          {          {
# Line 3907  we set the flag only if there is a liter Line 4606  we set the flag only if there is a liter
4606          goto END_REPEAT;          goto END_REPEAT;
4607          }          }
4608    
4609          /*--------------------------------------------------------------------*/
4610          /* This code is obsolete from release 8.00; the restriction was finally
4611          removed: */
4612    
4613        /* All real repeats make it impossible to handle partial matching (maybe        /* All real repeats make it impossible to handle partial matching (maybe
4614        one day we will be able to remove this restriction). */        one day we will be able to remove this restriction). */
4615    
4616        if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL;        /* if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL; */
4617          /*--------------------------------------------------------------------*/
4618    
4619        if (repeat_min == 0 && repeat_max == -1)        if (repeat_min == 0 && repeat_max == -1)
4620          *code++ = OP_CRSTAR + repeat_type;          *code++ = OP_CRSTAR + repeat_type;
# Line 3928  we set the flag only if there is a liter Line 4632  we set the flag only if there is a liter
4632        }        }
4633    
4634      /* If previous was a bracket group, we may have to replicate it in certain      /* If previous was a bracket group, we may have to replicate it in certain
4635      cases. */      cases. Note that at this point we can encounter only the "basic" bracket
4636        opcodes such as BRA and CBRA, as this is the place where they get converted
4637        into the more special varieties such as BRAPOS and SBRA. A test for >=
4638        OP_ASSERT and <= OP_COND includes ASSERT, ASSERT_NOT, ASSERTBACK,
4639        ASSERTBACK_NOT, ONCE, BRA, CBRA, and COND. Originally, PCRE did not allow
4640        repetition of assertions, but now it does, for Perl compatibility. */
4641    
4642      else if (*previous == OP_BRA  || *previous == OP_CBRA ||      else if (*previous >= OP_ASSERT && *previous <= OP_COND)
              *previous == OP_ONCE || *previous == OP_COND)  
4643        {        {
4644        register int i;        register int i;
4645        int ketoffset = 0;        int len = (int)(code - previous);
       int len = code - previous;  
4646        uschar *bralink = NULL;        uschar *bralink = NULL;
4647          uschar *brazeroptr = NULL;
4648    
4649        /* Repeating a DEFINE group is pointless */        /* Repeating a DEFINE group is pointless, but Perl allows the syntax, so
4650          we just ignore the repeat. */
4651    
4652        if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF)        if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF)
4653          {          goto END_REPEAT;
         *errorcodeptr = ERR55;  
         goto FAILED;  
         }  
4654    
4655        /* If the maximum repeat count is unlimited, find the end of the bracket        /* There is no sense in actually repeating assertions. The only potential
4656        by scanning through from the start, and compute the offset back to it        use of repetition is in cases when the assertion is optional. Therefore,
4657        from the current code pointer. There may be an OP_OPT setting following        if the minimum is greater than zero, just ignore the repeat. If the
4658        the final KET, so we can't find the end just by going back from the code        maximum is not not zero or one, set it to 1. */
4659        pointer. */  
4660          if (*previous < OP_ONCE)    /* Assertion */
4661        if (repeat_max == -1)          {
4662          {          if (repeat_min > 0) goto END_REPEAT;
4663          register uschar *ket = previous;          if (repeat_max < 0 || repeat_max > 1) repeat_max = 1;
         do ket += GET(ket, 1); while (*ket != OP_KET);  
         ketoffset = code - ket;  
4664          }          }
4665    
4666        /* The case of a zero minimum is special because of the need to stick        /* The case of a zero minimum is special because of the need to stick
# Line 3977  we set the flag only if there is a liter Line 4681  we set the flag only if there is a liter
4681          **   goto END_REPEAT;          **   goto END_REPEAT;
4682          **   }          **   }
4683    
4684          However, that fails when a group is referenced as a subroutine from          However, that fails when a group or a subgroup within it is referenced
4685          elsewhere in the pattern, so now we stick in OP_SKIPZERO in front of it          as a subroutine from elsewhere in the pattern, so now we stick in
4686          so that it is skipped on execution. As we don't have a list of which          OP_SKIPZERO in front of it so that it is skipped on execution. As we
4687          groups are referenced, we cannot do this selectively.          don't have a list of which groups are referenced, we cannot do this
4688            selectively.
4689    
4690          If the maximum is 1 or unlimited, we just have to stick in the BRAZERO          If the maximum is 1 or unlimited, we just have to stick in the BRAZERO
4691          and do no more at this point. However, we do need to adjust any          and do no more at this point. However, we do need to adjust any
# Line 4000  we set the flag only if there is a liter Line 4705  we set the flag only if there is a liter
4705              *previous++ = OP_SKIPZERO;              *previous++ = OP_SKIPZERO;
4706              goto END_REPEAT;              goto END_REPEAT;
4707              }              }
4708              brazeroptr = previous;    /* Save for possessive optimizing */
4709            *previous++ = OP_BRAZERO + repeat_type;            *previous++ = OP_BRAZERO + repeat_type;
4710            }            }
4711    
# Line 4024  we set the flag only if there is a liter Line 4730  we set the flag only if there is a liter
4730            /* We chain together the bracket offset fields that have to be            /* We chain together the bracket offset fields that have to be
4731            filled in later when the ends of the brackets are reached. */            filled in later when the ends of the brackets are reached. */
4732    
4733            offset = (bralink == NULL)? 0 : previous - bralink;            offset = (bralink == NULL)? 0 : (int)(previous - bralink);
4734            bralink = previous;            bralink = previous;
4735            PUTINC(previous, 0, offset);            PUTINC(previous, 0, offset);
4736            }            }
# Line 4045  we set the flag only if there is a liter Line 4751  we set the flag only if there is a liter
4751            {            {
4752            /* In the pre-compile phase, we don't actually do the replication. We            /* In the pre-compile phase, we don't actually do the replication. We
4753            just adjust the length as if we had. Do some paranoid checks for            just adjust the length as if we had. Do some paranoid checks for
4754            potential integer overflow. */            potential integer overflow. The INT64_OR_DOUBLE type is a 64-bit
4755              integer type when available, otherwise double. */
4756    
4757            if (lengthptr != NULL)            if (lengthptr != NULL)
4758              {              {
4759              int delta = (repeat_min - 1)*length_prevgroup;              int delta = (repeat_min - 1)*length_prevgroup;
4760              if ((double)(repeat_min - 1)*(double)length_prevgroup >              if ((INT64_OR_DOUBLE)(repeat_min - 1)*
4761                                                              (double)INT_MAX ||                    (INT64_OR_DOUBLE)length_prevgroup >
4762                        (INT64_OR_DOUBLE)INT_MAX ||
4763                  OFLOW_MAX - *lengthptr < delta)                  OFLOW_MAX - *lengthptr < delta)
4764                {                {
4765                *errorcodeptr = ERR20;                *errorcodeptr = ERR20;
# Line 4097  we set the flag only if there is a liter Line 4805  we set the flag only if there is a liter
4805          just adjust the length as if we had. For each repetition we must add 1          just adjust the length as if we had. For each repetition we must add 1
4806          to the length for BRAZERO and for all but the last repetition we must          to the length for BRAZERO and for all but the last repetition we must
4807          add 2 + 2*LINKSIZE to allow for the nesting that occurs. Do some          add 2 + 2*LINKSIZE to allow for the nesting that occurs. Do some
4808          paranoid checks to avoid integer overflow. */          paranoid checks to avoid integer overflow. The INT64_OR_DOUBLE type is
4809            a 64-bit integer type when available, otherwise double. */
4810    
4811          if (lengthptr != NULL && repeat_max > 0)          if (lengthptr != NULL && repeat_max > 0)
4812            {            {
4813            int delta = repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) -            int delta = repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) -
4814                        2 - 2*LINK_SIZE;   /* Last one doesn't nest */                        2 - 2*LINK_SIZE;   /* Last one doesn't nest */
4815            if ((double)repeat_max *            if ((INT64_OR_DOUBLE)repeat_max *
4816                  (double)(length_prevgroup + 1 + 2 + 2*LINK_SIZE)                  (INT64_OR_DOUBLE)(length_prevgroup + 1 + 2 + 2*LINK_SIZE)
4817                    > (double)INT_MAX ||                    > (INT64_OR_DOUBLE)INT_MAX ||
4818                OFLOW_MAX - *lengthptr < delta)                OFLOW_MAX - *lengthptr < delta)
4819              {              {
4820              *errorcodeptr = ERR20;              *errorcodeptr = ERR20;
# Line 4130  we set the flag only if there is a liter Line 4839  we set the flag only if there is a liter
4839              {              {
4840              int offset;              int offset;
4841              *code++ = OP_BRA;              *code++ = OP_BRA;
4842              offset = (bralink == NULL)? 0 : code - bralink;              offset = (bralink == NULL)? 0 : (int)(code - bralink);
4843              bralink = code;              bralink = code;
4844              PUTINC(code, 0, offset);              PUTINC(code, 0, offset);
4845              }              }
# Line 4151  we set the flag only if there is a liter Line 4860  we set the flag only if there is a liter
4860          while (bralink != NULL)          while (bralink != NULL)
4861            {            {
4862            int oldlinkoffset;            int oldlinkoffset;
4863            int offset = code - bralink + 1;            int offset = (int)(code - bralink + 1);
4864            uschar *bra = code - offset;            uschar *bra = code - offset;
4865            oldlinkoffset = GET(bra, 1);            oldlinkoffset = GET(bra, 1);
4866            bralink = (oldlinkoffset == 0)? NULL : bralink - oldlinkoffset;            bralink = (oldlinkoffset == 0)? NULL : bralink - oldlinkoffset;
# Line 4161  we set the flag only if there is a liter Line 4870  we set the flag only if there is a liter
4870            }            }
4871          }          }
4872    
4873        /* If the maximum is unlimited, set a repeater in the final copy. We        /* If the maximum is unlimited, set a repeater in the final copy. For
4874        can't just offset backwards from the current code point, because we        ONCE brackets, that's all we need to do. However, possessively repeated
4875        don't know if there's been an options resetting after the ket. The        ONCE brackets can be converted into non-capturing brackets, as the
4876        correct offset was computed above.        behaviour of (?:xx)++ is the same as (?>xx)++ and this saves having to
4877          deal with possessive ONCEs specially.
4878    
4879          Otherwise, if the quantifier was possessive, we convert the BRA code to
4880          the POS form, and the KET code to KETRPOS. (It turns out to be convenient
4881          at runtime to detect this kind of subpattern at both the start and at the
4882          end.) The use of special opcodes makes it possible to reduce greatly the
4883          stack usage in pcre_exec(). If the group is preceded by OP_BRAZERO,
4884          convert this to OP_BRAPOSZERO. Then cancel the possessive flag so that
4885          the default action below, of wrapping everything inside atomic brackets,
4886          does not happen.
4887    
4888        Then, when we are doing the actual compile phase, check to see whether        Then, when we are doing the actual compile phase, check to see whether
4889        this group is a non-atomic one that could match an empty string. If so,        this group is one that could match an empty string. If so, convert the
4890        convert the initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so        initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so that runtime
4891        that runtime checking can be done. [This check is also applied to        checking can be done. [This check is also applied to ONCE groups at
4892        atomic groups at runtime, but in a different way.] */        runtime, but in a different way.] */
4893    
4894        else        else
4895          {          {
4896          uschar *ketcode = code - ketoffset;          uschar *ketcode = code - 1 - LINK_SIZE;
4897          uschar *bracode = ketcode - GET(ketcode, 1);          uschar *bracode = ketcode - GET(ketcode, 1);
4898          *ketcode = OP_KETRMAX + repeat_type;  
4899          if (lengthptr == NULL && *bracode != OP_ONCE)          if ((*bracode == OP_ONCE || *bracode == OP_ONCE_NC) &&
4900                possessive_quantifier) *bracode = OP_BRA;
4901    
4902            if (*bracode == OP_ONCE || *bracode == OP_ONCE_NC)
4903              *ketcode = OP_KETRMAX + repeat_type;
4904            else
4905            {            {
4906            uschar *scode = bracode;            if (possessive_quantifier)
4907            do              {
4908                *bracode += 1;                   /* Switch to xxxPOS opcodes */
4909                *ketcode = OP_KETRPOS;
4910                if (brazeroptr != NULL) *brazeroptr = OP_BRAPOSZERO;
4911                possessive_quantifier = FALSE;
4912                }
4913              else *ketcode = OP_KETRMAX + repeat_type;
4914    
4915              if (lengthptr == NULL)
4916              {              {
4917              if (could_be_empty_branch(scode, ketcode, utf8))              uschar *scode = bracode;
4918                do
4919                {                {
4920                *bracode += OP_SBRA - OP_BRA;                if (could_be_empty_branch(scode, ketcode, utf8, cd))
4921                break;                  {
4922                    *bracode += OP_SBRA - OP_BRA;
4923                    break;
4924                    }
4925                  scode += GET(scode, 1);
4926                }                }
4927              scode += GET(scode, 1);              while (*scode == OP_ALT);
4928              }              }
           while (*scode == OP_ALT);  
4929            }            }
4930          }          }
4931        }        }
# Line 4210  we set the flag only if there is a liter Line 4946  we set the flag only if there is a liter
4946        }        }
4947    
4948      /* If the character following a repeat is '+', or if certain optimization      /* If the character following a repeat is '+', or if certain optimization
4949      tests above succeeded, possessive_quantifier is TRUE. For some of the      tests above succeeded, possessive_quantifier is TRUE. For some opcodes,
4950      simpler opcodes, there is an special alternative opcode for this. For      there are special alternative opcodes for this case. For anything else, we
4951      anything else, we wrap the entire repeated item inside OP_ONCE brackets.      wrap the entire repeated item inside OP_ONCE brackets. Logically, the '+'
4952      The '+' notation is just syntactic sugar, taken from Sun's Java package,      notation is just syntactic sugar, taken from Sun's Java package, but the
4953      but the special opcodes can optimize it a bit. The repeated item starts at      special opcodes can optimize it.
4954      tempcode, not at previous, which might be the first part of a string whose  
4955      (former) last char we repeated.      Possessively repeated subpatterns have already been handled in the code
4956        just above, so possessive_quantifier is always FALSE for them at this
4957        stage.
4958    
4959        Note that the repeated item starts at tempcode, not at previous, which
4960        might be the first part of a string whose (former) last char we repeated.
4961    
4962      Possessifying an 'exact' quantifier has no effect, so we can ignore it. But      Possessifying an 'exact' quantifier has no effect, so we can ignore it. But
4963      an 'upto' may follow. We skip over an 'exact' item, and then test the      an 'upto' may follow. We skip over an 'exact' item, and then test the
# Line 4225  we set the flag only if there is a liter Line 4966  we set the flag only if there is a liter
4966      if (possessive_quantifier)      if (possessive_quantifier)
4967        {        {
4968        int len;        int len;
4969        if (*tempcode == OP_EXACT || *tempcode == OP_TYPEEXACT ||  
4970            *tempcode == OP_NOTEXACT)        if (*tempcode == OP_TYPEEXACT)
4971          tempcode += _pcre_OP_lengths[*tempcode] +          tempcode += _pcre_OP_lengths[*tempcode] +
4972            ((*tempcode == OP_TYPEEXACT &&            ((tempcode[3] == OP_PROP || tempcode[3] == OP_NOTPROP)? 2 : 0);
4973               (tempcode[3] == OP_PROP || tempcode[3] == OP_NOTPROP))? 2:0);  
4974        len = code - tempcode;        else if (*tempcode == OP_EXACT || *tempcode == OP_NOTEXACT)
4975            {
4976            tempcode += _pcre_OP_lengths[*tempcode];
4977    #ifdef SUPPORT_UTF8
4978            if (utf8 && tempcode[-1] >= 0xc0)
4979              tempcode += _pcre_utf8_table4[tempcode[-1] & 0x3f];
4980    #endif
4981            }
4982    
4983          len = (int)(code - tempcode);
4984        if (len > 0) switch (*tempcode)        if (len > 0) switch (*tempcode)
4985          {          {
4986          case OP_STAR:  *tempcode = OP_POSSTAR; break;          case OP_STAR:  *tempcode = OP_POSSTAR; break;
# Line 4238  we set the flag only if there is a liter Line 4988  we set the flag only if there is a liter
4988          case OP_QUERY: *tempcode = OP_POSQUERY; break;          case OP_QUERY: *tempcode = OP_POSQUERY; break;
4989          case OP_UPTO:  *tempcode = OP_POSUPTO; break;          case OP_UPTO:  *tempcode = OP_POSUPTO; break;
4990    
4991          case OP_TYPESTAR:  *tempcode = OP_TYPEPOSSTAR; break;          case OP_STARI:  *tempcode = OP_POSSTARI; break;
4992          case OP_TYPEPLUS:  *tempcode = OP_TYPEPOSPLUS; break;          case OP_PLUSI:  *tempcode = OP_POSPLUSI; break;
4993          case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break;          case OP_QUERYI: *tempcode = OP_POSQUERYI; break;
4994          case OP_TYPEUPTO:  *tempcode = OP_TYPEPOSUPTO; break;          case OP_UPTOI:  *tempcode = OP_POSUPTOI; break;
4995    
4996          case OP_NOTSTAR:  *tempcode = OP_NOTPOSSTAR; break;          case OP_NOTSTAR:  *tempcode = OP_NOTPOSSTAR; break;
4997          case OP_NOTPLUS:  *tempcode = OP_NOTPOSPLUS; break;          case OP_NOTPLUS:  *tempcode = OP_NOTPOSPLUS; break;
4998          case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break;          case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break;
4999          case OP_NOTUPTO:  *tempcode = OP_NOTPOSUPTO; break;          case OP_NOTUPTO:  *tempcode = OP_NOTPOSUPTO; break;
5000    
5001            case OP_NOTSTARI:  *tempcode = OP_NOTPOSSTARI; break;
5002            case OP_NOTPLUSI:  *tempcode = OP_NOTPOSPLUSI; break;
5003            case OP_NOTQUERYI: *tempcode = OP_NOTPOSQUERYI; break;
5004            case OP_NOTUPTOI:  *tempcode = OP_NOTPOSUPTOI; break;
5005    
5006            case OP_TYPESTAR:  *tempcode = OP_TYPEPOSSTAR; break;
5007            case OP_TYPEPLUS:  *tempcode = OP_TYPEPOSPLUS; break;
5008            case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break;
5009            case OP_TYPEUPTO:  *tempcode = OP_TYPEPOSUPTO; break;
5010    
5011            /* Because we are moving code along, we must ensure that any
5012            pending recursive references are updated. */
5013    
5014          default:          default:
5015            *code = OP_END;
5016            adjust_recurse(tempcode, 1 + LINK_SIZE, utf8, cd, save_hwm);
5017          memmove(tempcode + 1+LINK_SIZE, tempcode, len);          memmove(tempcode + 1+LINK_SIZE, tempcode, len);
5018          code += 1 + LINK_SIZE;          code += 1 + LINK_SIZE;
5019          len += 1 + LINK_SIZE;          len += 1 + LINK_SIZE;
# Line 4284  we set the flag only if there is a liter Line 5049  we set the flag only if there is a liter
5049    
5050      /* First deal with various "verbs" that can be introduced by '*'. */      /* First deal with various "verbs" that can be introduced by '*'. */
5051    
5052      if (*(++ptr) == CHAR_ASTERISK && (cd->ctypes[ptr[1]] & ctype_letter) != 0)      if (*(++ptr) == CHAR_ASTERISK &&
5053             ((cd->ctypes[ptr[1]] & ctype_letter) != 0 || ptr[1] == ':'))
5054        {        {
5055        int i, namelen;        int i, namelen;
5056          int arglen = 0;
5057        const char *vn = verbnames;        const char *vn = verbnames;
5058        const uschar *name = ++ptr;        const uschar *name = ptr + 1;
5059          const uschar *arg = NULL;
5060        previous = NULL;        previous = NULL;
5061        while ((cd->ctypes[*++ptr] & ctype_letter) != 0) {};        while ((cd->ctypes[*++ptr] & ctype_letter) != 0) {};
5062          namelen = (int)(ptr - name);
5063    
5064          /* It appears that Perl allows any characters whatsoever, other than
5065          a closing parenthesis, to appear in arguments, so we no longer insist on
5066          letters, digits, and underscores. */
5067    
5068        if (*ptr == CHAR_COLON)        if (*ptr == CHAR_COLON)
5069          {          {
5070          *errorcodeptr = ERR59;   /* Not supported */          arg = ++ptr;
5071          goto FAILED;          while (*ptr != 0 && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
5072            arglen = (int)(ptr - arg);
5073          }          }
5074    
5075        if (*ptr != CHAR_RIGHT_PARENTHESIS)        if (*ptr != CHAR_RIGHT_PARENTHESIS)
5076          {          {
5077          *errorcodeptr = ERR60;          *errorcodeptr = ERR60;
5078          goto FAILED;          goto FAILED;
5079          }          }
5080        namelen = ptr - name;  
5081          /* Scan the table of verb names */
5082    
5083        for (i = 0; i < verbcount; i++)        for (i = 0; i < verbcount; i++)
5084          {          {
5085          if (namelen == verbs[i].len &&          if (namelen == verbs[i].len &&
5086              strncmp((char *)name, vn, namelen) == 0)              strncmp((char *)name, vn, namelen) == 0)
5087            {            {
5088            *code = verbs[i].op;            /* Check for open captures before ACCEPT and convert it to
5089            if (*code++ == OP_ACCEPT) cd->had_accept = TRUE;            ASSERT_ACCEPT if in an assertion. */
5090            break;  
5091              if (verbs[i].op == OP_ACCEPT)
5092                {
5093                open_capitem *oc;
5094                if (arglen != 0)
5095                  {
5096                  *errorcodeptr = ERR59;
5097                  goto FAILED;
5098                  }
5099                cd->had_accept = TRUE;
5100                for (oc = cd->open_caps; oc != NULL; oc = oc->next)
5101                  {
5102                  *code++ = OP_CLOSE;
5103                  PUT2INC(code, 0, oc->number);
5104                  }
5105                *code++ = (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT;
5106    
5107                /* Do not set firstbyte after *ACCEPT */
5108                if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
5109                }
5110    
5111              /* Handle other cases with/without an argument */
5112    
5113              else if (arglen == 0)
5114                {
5115                if (verbs[i].op < 0)   /* Argument is mandatory */
5116                  {
5117                  *errorcodeptr = ERR66;
5118                  goto FAILED;
5119                  }
5120                *code = verbs[i].op;
5121                if (*code++ == OP_THEN) cd->external_flags |= PCRE_HASTHEN;
5122                }
5123    
5124              else
5125                {
5126                if (verbs[i].op_arg < 0)   /* Argument is forbidden */
5127                  {
5128                  *errorcodeptr = ERR59;
5129                  goto FAILED;
5130                  }
5131                *code = verbs[i].op_arg;
5132                if (*code++ == OP_THEN_ARG) cd->external_flags |= PCRE_HASTHEN;
5133                *code++ = arglen;
5134                memcpy(code, arg, arglen);
5135                code += arglen;
5136                *code++ = 0;
5137                }
5138    
5139              break;  /* Found verb, exit loop */
5140            }            }
5141    
5142          vn += verbs[i].len + 1;          vn += verbs[i].len + 1;
5143          }          }
5144        if (i < verbcount) continue;  
5145        *errorcodeptr = ERR60;        if (i < verbcount) continue;    /* Successfully handled a verb */
5146          *errorcodeptr = ERR60;          /* Verb not recognized */
5147        goto FAILED;        goto FAILED;
5148        }        }
5149    
# Line 4433  we set the flag only if there is a liter Line 5262  we set the flag only if there is a liter
5262                recno * 10 + *ptr - CHAR_0 : -1;                recno * 10 + *ptr - CHAR_0 : -1;
5263            ptr++;            ptr++;
5264            }            }
5265          namelen = ptr - name;          namelen = (int)(ptr - name);
5266    
5267          if ((terminator > 0 && *ptr++ != terminator) ||          if ((terminator > 0 && *ptr++ != terminator) ||
5268              *ptr++ != CHAR_RIGHT_PARENTHESIS)              *ptr++ != CHAR_RIGHT_PARENTHESIS)
# Line 4470  we set the flag only if there is a liter Line 5299  we set the flag only if there is a liter
5299            }            }
5300    
5301          /* Otherwise (did not start with "+" or "-"), start by looking for the          /* Otherwise (did not start with "+" or "-"), start by looking for the
5302          name. */          name. If we find a name, add one to the opcode to change OP_CREF or
5303            OP_RREF into OP_NCREF or OP_NRREF. These behave exactly the same,
5304            except they record that the reference was originally to a name. The
5305            information is used to check duplicate names. */
5306    
5307          slot = cd->name_table;          slot = cd->name_table;
5308          for (i = 0; i < cd->names_found; i++)          for (i = 0; i < cd->names_found; i++)
# Line 4485  we set the flag only if there is a liter Line 5317  we set the flag only if there is a liter
5317            {            {
5318            recno = GET2(slot, 0);            recno = GET2(slot, 0);
5319            PUT2(code, 2+LINK_SIZE, recno);            PUT2(code, 2+LINK_SIZE, recno);
5320              code[1+LINK_SIZE]++;
5321            }            }
5322    
5323          /* Search the pattern for a forward reference */          /* Search the pattern for a forward reference */
5324    
5325          else if ((i = find_parens(ptr, cd, name, namelen,          else if ((i = find_parens(cd, name, namelen,
5326                          (options & PCRE_EXTENDED) != 0)) > 0)                          (options & PCRE_EXTENDED) != 0, utf8)) > 0)
5327            {            {
5328            PUT2(code, 2+LINK_SIZE, i);            PUT2(code, 2+LINK_SIZE, i);
5329              code[1+LINK_SIZE]++;
5330            }            }
5331    
5332          /* If terminator == 0 it means that the name followed directly after          /* If terminator == 0 it means that the name followed directly after
# Line 4557  we set the flag only if there is a liter Line 5391  we set the flag only if there is a liter
5391          /* ------------------------------------------------------------ */          /* ------------------------------------------------------------ */
5392          case CHAR_EQUALS_SIGN:                 /* Positive lookahead */          case CHAR_EQUALS_SIGN:                 /* Positive lookahead */
5393          bravalue = OP_ASSERT;          bravalue = OP_ASSERT;
5394            cd->assert_depth += 1;
5395          ptr++;          ptr++;
5396          break;          break;
5397    
# Line 4571  we set the flag only if there is a liter Line 5406  we set the flag only if there is a liter
5406            continue;            continue;
5407            }            }
5408          bravalue = OP_ASSERT_NOT;          bravalue = OP_ASSERT_NOT;
5409            cd->assert_depth += 1;
5410          break;          break;
5411    
5412    
# Line 4580  we set the flag only if there is a liter Line 5416  we set the flag only if there is a liter
5416            {            {
5417            case CHAR_EQUALS_SIGN:               /* Positive lookbehind */            case CHAR_EQUALS_SIGN:               /* Positive lookbehind */
5418            bravalue = OP_ASSERTBACK;            bravalue = OP_ASSERTBACK;
5419              cd->assert_depth += 1;
5420            ptr += 2;            ptr += 2;
5421            break;            break;
5422    
5423            case CHAR_EXCLAMATION_MARK:          /* Negative lookbehind */            case CHAR_EXCLAMATION_MARK:          /* Negative lookbehind */
5424            bravalue = OP_ASSERTBACK_NOT;            bravalue = OP_ASSERTBACK_NOT;
5425              cd->assert_depth += 1;
5426            ptr += 2;            ptr += 2;
5427            break;            break;
5428    
# Line 4624  we set the flag only if there is a liter Line 5462  we set the flag only if there is a liter
5462              goto FAILED;              goto FAILED;
5463              }              }
5464            *code++ = n;            *code++ = n;
5465            PUT(code, 0, ptr - cd->start_pattern + 1);  /* Pattern offset */            PUT(code, 0, (int)(ptr - cd->start_pattern + 1)); /* Pattern offset */
5466            PUT(code, LINK_SIZE, 0);                    /* Default length */            PUT(code, LINK_SIZE, 0);                          /* Default length */
5467            code += 2 * LINK_SIZE;            code += 2 * LINK_SIZE;
5468            }            }
5469          previous = NULL;          previous = NULL;
# Line 4658  we set the flag only if there is a liter Line 5496  we set the flag only if there is a liter
5496            name = ++ptr;            name = ++ptr;
5497    
5498            while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;            while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
5499            namelen = ptr - name;            namelen = (int)(ptr - name);
5500    
5501            /* In the pre-compile phase, just do a syntax check. */            /* In the pre-compile phase, just do a syntax check. */
5502    
# Line 4685  we set the flag only if there is a liter Line 5523  we set the flag only if there is a liter
5523                }                }
5524              }              }
5525    
5526            /* In the real compile, create the entry in the table */            /* In the real compile, create the entry in the table, maintaining
5527              alphabetical order. Duplicate names for different numbers are
5528              permitted only if PCRE_DUPNAMES is set. Duplicate names for the same
5529              number are always OK. (An existing number can be re-used if (?|
5530              appears in the pattern.) In either event, a duplicate name results in
5531              a duplicate entry in the table, even if the number is the same. This
5532              is because the number of names, and hence the table size, is computed
5533              in the pre-compile, and it affects various numbers and pointers which
5534              would all have to be modified, and the compiled code moved down, if
5535              duplicates with the same number were omitted from the table. This
5536              doesn't seem worth the hassle. However, *different* names for the
5537              same number are not permitted. */
5538    
5539            else            else
5540              {              {
5541                BOOL dupname = FALSE;
5542              slot = cd->name_table;              slot = cd->name_table;
5543    
5544              for (i = 0; i < cd->names_found; i++)              for (i = 0; i < cd->names_found; i++)
5545                {                {
5546                int crc = memcmp(name, slot+2, namelen);                int crc = memcmp(name, slot+2, namelen);
# Line 4697  we set the flag only if there is a liter Line 5548  we set the flag only if there is a liter
5548                  {                  {
5549                  if (slot[2+namelen] == 0)                  if (slot[2+namelen] == 0)
5550                    {                    {
5551                    if ((options & PCRE_DUPNAMES) == 0)                    if (GET2(slot, 0) != cd->bracount + 1 &&
5552                          (options & PCRE_DUPNAMES) == 0)
5553                      {                      {
5554                      *errorcodeptr = ERR43;                      *errorcodeptr = ERR43;
5555                      goto FAILED;                      goto FAILED;
5556                      }                      }
5557                      else dupname = TRUE;
5558                    }                    }
5559                  else crc = -1;      /* Current name is substring */                  else crc = -1;      /* Current name is a substring */
5560                  }                  }
5561    
5562                  /* Make space in the table and break the loop for an earlier
5563                  name. For a duplicate or later name, carry on. We do this for
5564                  duplicates so that in the simple case (when ?(| is not used) they
5565                  are in order of their numbers. */
5566    
5567                if (crc < 0)                if (crc < 0)
5568                  {                  {
5569                  memmove(slot + cd->name_entry_size, slot,                  memmove(slot + cd->name_entry_size, slot,
5570                    (cd->names_found - i) * cd->name_entry_size);                    (cd->names_found - i) * cd->name_entry_size);
5571                  break;                  break;
5572                  }                  }
5573    
5574                  /* Continue the loop for a later or duplicate name */
5575    
5576                slot += cd->name_entry_size;                slot += cd->name_entry_size;
5577                }                }
5578    
5579                /* For non-duplicate names, check for a duplicate number before
5580                adding the new name. */
5581    
5582                if (!dupname)
5583                  {
5584                  uschar *cslot = cd->name_table;
5585                  for (i = 0; i < cd->names_found; i++)
5586                    {
5587                    if (cslot != slot)
5588                      {
5589                      if (GET2(cslot, 0) == cd->bracount + 1)
5590                        {
5591                        *errorcodeptr = ERR65;
5592                        goto FAILED;
5593                        }
5594                      }
5595                    else i--;
5596                    cslot += cd->name_entry_size;
5597                    }
5598                  }
5599    
5600              PUT2(slot, 0, cd->bracount + 1);              PUT2(slot, 0, cd->bracount + 1);
5601              memcpy(slot + 2, name, namelen);              memcpy(slot + 2, name, namelen);
5602              slot[2+namelen] = 0;              slot[2+namelen] = 0;
5603              }              }
5604            }            }
5605    
5606          /* In both cases, count the number of names we've encountered. */          /* In both pre-compile and compile, count the number of names we've
5607            encountered. */
5608    
         ptr++;                    /* Move past > or ' */  
5609          cd->names_found++;          cd->names_found++;
5610            ptr++;                    /* Move past > or ' */
5611          goto NUMBERED_GROUP;          goto NUMBERED_GROUP;
5612    
5613    
# Line 4742  we set the flag only if there is a liter Line 5626  we set the flag only if there is a liter
5626          NAMED_REF_OR_RECURSE:          NAMED_REF_OR_RECURSE:
5627          name = ++ptr;          name = ++ptr;
5628          while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;          while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
5629          namelen = ptr - name;          namelen = (int)(ptr - name);
5630    
5631          /* In the pre-compile phase, do a syntax check and set a dummy          /* In the pre-compile phase, do a syntax check. We used to just set
5632          reference number. */          a dummy reference number, because it was not used in the first pass.
5633            However, with the change of recursive back references to be atomic,
5634            we have to look for the number so that this state can be identified, as
5635            otherwise the incorrect length is computed. If it's not a backwards
5636            reference, the dummy number will do. */
5637    
5638          if (lengthptr != NULL)          if (lengthptr != NULL)
5639            {            {
5640              const uschar *temp;
5641    
5642            if (namelen == 0)            if (namelen == 0)
5643              {              {
5644              *errorcodeptr = ERR62;              *errorcodeptr = ERR62;
# Line 4764  we set the flag only if there is a liter Line 5654  we set the flag only if there is a liter
5654              *errorcodeptr = ERR48;              *errorcodeptr = ERR48;
5655              goto FAILED;              goto FAILED;
5656              }              }
5657            recno = 0;  
5658              /* The name table does not exist in the first pass, so we cannot
5659              do a simple search as in the code below. Instead, we have to scan the
5660              pattern to find the number. It is important that we scan it only as
5661              far as we have got because the syntax of named subpatterns has not
5662              been checked for the rest of the pattern, and find_parens() assumes
5663              correct syntax. In any case, it's a waste of resources to scan
5664              further. We stop the scan at the current point by temporarily
5665              adjusting the value of cd->endpattern. */
5666    
5667              temp = cd->end_pattern;
5668              cd->end_pattern = ptr;
5669              recno = find_parens(cd, name, namelen,
5670                (options & PCRE_EXTENDED) != 0, utf8);
5671              cd->end_pattern = temp;
5672              if (recno < 0) recno = 0;    /* Forward ref; set dummy number */
5673            }            }
5674    
5675          /* In the real compile, seek the name in the table. We check the name          /* In the real compile, seek the name in the table. We check the name
# Line 4788  we set the flag only if there is a liter Line 5693  we set the flag only if there is a liter
5693              recno = GET2(slot, 0);              recno = GET2(slot, 0);
5694              }              }
5695            else if ((recno =                /* Forward back reference */            else if ((recno =                /* Forward back reference */
5696                      find_parens(ptr, cd, name, namelen,                      find_parens(cd, name, namelen,
5697                        (options & PCRE_EXTENDED) != 0)) <= 0)                        (options & PCRE_EXTENDED) != 0, utf8)) <= 0)
5698              {              {
5699              *errorcodeptr = ERR15;              *errorcodeptr = ERR15;
5700              goto FAILED;              goto FAILED;
# Line 4892  we set the flag only if there is a liter Line 5797  we set the flag only if there is a liter
5797            if (lengthptr == NULL)            if (lengthptr == NULL)
5798              {              {
5799              *code = OP_END;              *code = OP_END;
5800              if (recno != 0) called = find_bracket(cd->start_code, utf8, recno);              if (recno != 0)
5801                  called = _pcre_find_bracket(cd->start_code, utf8, recno);
5802    
5803              /* Forward reference */              /* Forward reference */
5804    
5805              if (called == NULL)              if (called == NULL)
5806                {                {
5807                if (find_parens(ptr, cd, NULL, recno,                if (find_parens(cd, NULL, recno,
5808                      (options & PCRE_EXTENDED) != 0) < 0)                      (options & PCRE_EXTENDED) != 0, utf8) < 0)
5809                  {                  {
5810                  *errorcodeptr = ERR15;                  *errorcodeptr = ERR15;
5811                  goto FAILED;                  goto FAILED;
5812                  }                  }
5813    
5814                  /* Fudge the value of "called" so that when it is inserted as an
5815                  offset below, what it actually inserted is the reference number
5816                  of the group. Then remember the forward reference. */
5817    
5818                called = cd->start_code + recno;                called = cd->start_code + recno;
5819                PUTINC(cd->hwm, 0, code + 2 + LINK_SIZE - cd->start_code);                PUTINC(cd->hwm, 0, (int)(code + 1 - cd->start_code));
5820                }                }
5821    
5822              /* If not a forward reference, and the subpattern is still open,              /* If not a forward reference, and the subpattern is still open,
5823              this is a recursive call. We check to see if this is a left              this is a recursive call. We check to see if this is a left
5824              recursion that could loop for ever, and diagnose that case. */              recursion that could loop for ever, and diagnose that case. We
5825                must not, however, do this check if we are in a conditional
5826                subpattern because the condition might be testing for recursion in
5827                a pattern such as /(?(R)a+|(?R)b)/, which is perfectly valid.
5828                Forever loops are also detected at runtime, so those that occur in
5829                conditional subpatterns will be picked up then. */
5830    
5831              else if (GET(called, 1) == 0 &&              else if (GET(called, 1) == 0 && cond_depth <= 0 &&
5832                       could_be_empty(called, code, bcptr, utf8))                       could_be_empty(called, code, bcptr, utf8, cd))
5833                {                {
5834                *errorcodeptr = ERR40;                *errorcodeptr = ERR40;
5835                goto FAILED;                goto FAILED;
5836                }                }
5837              }              }
5838    
5839            /* Insert the recursion/subroutine item, automatically wrapped inside            /* Insert the recursion/subroutine item. */
           "once" brackets. Set up a "previous group" length so that a  
           subsequent quantifier will work. */  
   
           *code = OP_ONCE;  
           PUT(code, 1, 2 + 2*LINK_SIZE);  
           code += 1 + LINK_SIZE;  
5840    
5841            *code = OP_RECURSE;            *code = OP_RECURSE;
5842            PUT(code, 1, called - cd->start_code);            PUT(code, 1, (int)(called - cd->start_code));
           code += 1 + LINK_SIZE;  
   
           *code = OP_KET;  
           PUT(code, 1, 2 + 2*LINK_SIZE);  
5843            code += 1 + LINK_SIZE;            code += 1 + LINK_SIZE;
   
           length_prevgroup = 3 + 3*LINK_SIZE;  
5844            }            }
5845    
5846          /* Can't determine a first byte now */          /* Can't determine a first byte now */
# Line 4997  we set the flag only if there is a liter Line 5901  we set the flag only if there is a liter
5901          is necessary to ensure we correctly detect the start of the pattern in          is necessary to ensure we correctly detect the start of the pattern in
5902          both phases.          both phases.
5903    
5904          If we are not at the pattern start, compile code to change the ims          If we are not at the pattern start, reset the greedy defaults and the
5905          options if this setting actually changes any of them, and reset the          case value for firstbyte and reqbyte. */
         greedy defaults and the case value for firstbyte and reqbyte. */  
5906    
5907          if (*ptr == CHAR_RIGHT_PARENTHESIS)          if (*ptr == CHAR_RIGHT_PARENTHESIS)
5908            {            {
# Line 5008  we set the flag only if there is a liter Line 5911  we set the flag only if there is a liter
5911              {              {
5912              cd->external_options = newoptions;              cd->external_options = newoptions;
5913              }              }
5914           else            else
5915              {              {
             if ((options & PCRE_IMS) != (newoptions & PCRE_IMS))  
               {  
               *code++ = OP_OPT;  
               *code++ = newoptions & PCRE_IMS;  
               }  
5916              greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);              greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);
5917              greedy_non_default = greedy_default ^ 1;              greedy_non_default = greedy_default ^ 1;
5918              req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;              req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;
5919              }              }
5920    
5921            /* Change options at this level, and pass them back for use            /* Change options at this level, and pass them back for use
5922            in subsequent branches. When not at the start of the pattern, this            in subsequent branches. */
           information is also necessary so that a resetting item can be  
           compiled at the end of a group (if we are in a group). */  
5923    
5924            *optionsptr = options = newoptions;            *optionsptr = options = newoptions;
5925            previous = NULL;       /* This item can't be repeated */            previous = NULL;       /* This item can't be repeated */
# Line 5040  we set the flag only if there is a liter Line 5936  we set the flag only if there is a liter
5936          }     /* End of switch for character following (? */          }     /* End of switch for character following (? */
5937        }       /* End of (? handling */        }       /* End of (? handling */
5938    
5939      /* Opening parenthesis not followed by '?'. If PCRE_NO_AUTO_CAPTURE is set,      /* Opening parenthesis not followed by '*' or '?'. If PCRE_NO_AUTO_CAPTURE
5940      all unadorned brackets become non-capturing and behave like (?:...)      is set, all unadorned brackets become non-capturing and behave like (?:...)
5941      brackets. */      brackets. */
5942    
5943      else if ((options & PCRE_NO_AUTO_CAPTURE) != 0)      else if ((options & PCRE_NO_AUTO_CAPTURE) != 0)
# Line 5059  we set the flag only if there is a liter Line 5955  we set the flag only if there is a liter
5955        skipbytes = 2;        skipbytes = 2;
5956        }        }
5957    
5958      /* Process nested bracketed regex. Assertions may not be repeated, but      /* Process nested bracketed regex. Assertions used not to be repeatable,
5959      other kinds can be. All their opcodes are >= OP_ONCE. We copy code into a      but this was changed for Perl compatibility, so all kinds can now be
5960      non-register variable in order to be able to pass its address because some      repeated. We copy code into a non-register variable (tempcode) in order to
5961      compilers complain otherwise. Pass in a new setting for the ims options if      be able to pass its address because some compilers complain otherwise. */
     they have changed. */  
5962    
5963      previous = (bravalue >= OP_ONCE)? code : NULL;      previous = code;                      /* For handling repetition */
5964      *code = bravalue;      *code = bravalue;
5965      tempcode = code;      tempcode = code;
5966      tempreqvary = cd->req_varyopt;     /* Save value before bracket */      tempreqvary = cd->req_varyopt;        /* Save value before bracket */
5967      length_prevgroup = 0;              /* Initialize for pre-compile phase */      tempbracount = cd->bracount;          /* Save value before bracket */
5968        length_prevgroup = 0;                 /* Initialize for pre-compile phase */
5969    
5970      if (!compile_regex(      if (!compile_regex(
5971           newoptions,                   /* The complete new option state */           newoptions,                      /* The complete new option state */
5972           options & PCRE_IMS,           /* The previous ims option state */           &tempcode,                       /* Where to put code (updated) */
5973           &tempcode,                    /* Where to put code (updated) */           &ptr,                            /* Input pointer (updated) */
5974           &ptr,                         /* Input pointer (updated) */           errorcodeptr,                    /* Where to put an error message */
          errorcodeptr,                 /* Where to put an error message */  
5975           (bravalue == OP_ASSERTBACK ||           (bravalue == OP_ASSERTBACK ||
5976            bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */            bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */
5977           reset_bracount,               /* True if (?| group */           reset_bracount,                  /* True if (?| group */
5978           skipbytes,                    /* Skip over bracket number */           skipbytes,                       /* Skip over bracket number */
5979           &subfirstbyte,                /* For possible first char */           cond_depth +
5980           &subreqbyte,                  /* For possible last char */             ((bravalue == OP_COND)?1:0),   /* Depth of condition subpatterns */
5981           bcptr,                        /* Current branch chain */           &subfirstbyte,                   /* For possible first char */
5982           cd,                           /* Tables block */           &subreqbyte,                     /* For possible last char */
5983           (lengthptr == NULL)? NULL :   /* Actual compile phase */           bcptr,                           /* Current branch chain */
5984             &length_prevgroup           /* Pre-compile phase */           cd,                              /* Tables block */
5985             (lengthptr == NULL)? NULL :      /* Actual compile phase */
5986               &length_prevgroup              /* Pre-compile phase */
5987           ))           ))
5988        goto FAILED;        goto FAILED;
5989    
5990        /* If this was an atomic group and there are no capturing groups within it,
5991        generate OP_ONCE_NC instead of OP_ONCE. */
5992    
5993        if (bravalue == OP_ONCE && cd->bracount <= tempbracount)
5994          *code = OP_ONCE_NC;
5995    
5996        if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT)
5997          cd->assert_depth -= 1;
5998    
5999      /* At the end of compiling, code is still pointing to the start of the      /* At the end of compiling, code is still pointing to the start of the
6000      group, while tempcode has been updated to point past the end of the group      group, while tempcode has been updated to point past the end of the group.
6001      and any option resetting that may follow it. The pattern pointer (ptr)      The pattern pointer (ptr) is on the bracket.
     is on the bracket. */  
6002    
6003      /* If this is a conditional bracket, check that there are no more than      If this is a conditional bracket, check that there are no more than
6004      two branches in the group, or just one if it's a DEFINE group. We do this      two branches in the group, or just one if it's a DEFINE group. We do this
6005      in the real compile phase, not in the pre-pass, where the whole group may      in the real compile phase, not in the pre-pass, where the whole group may
6006      not be available. */      not be available. */
# Line 5160  we set the flag only if there is a liter Line 6065  we set the flag only if there is a liter
6065          goto FAILED;          goto FAILED;
6066          }          }
6067        *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE;        *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE;
6068        *code++ = OP_BRA;        code++;   /* This already contains bravalue */
6069        PUTINC(code, 0, 1 + LINK_SIZE);        PUTINC(code, 0, 1 + LINK_SIZE);
6070        *code++ = OP_KET;        *code++ = OP_KET;
6071        PUTINC(code, 0, 1 + LINK_SIZE);        PUTINC(code, 0, 1 + LINK_SIZE);
# Line 5233  we set the flag only if there is a liter Line 6138  we set the flag only if there is a liter
6138    
6139      /* ===================================================================*/      /* ===================================================================*/
6140      /* Handle metasequences introduced by \. For ones like \d, the ESC_ values      /* Handle metasequences introduced by \. For ones like \d, the ESC_ values
6141      are arranged to be the negation of the corresponding OP_values. For the      are arranged to be the negation of the corresponding OP_values in the
6142      back references, the values are ESC_REF plus the reference number. Only      default case when PCRE_UCP is not set. For the back references, the values
6143      back references and those types that consume a character may be repeated.      are ESC_REF plus the reference number. Only back references and those types
6144      We can test for values between ESC_b and ESC_Z for the latter; this may      that consume a character may be repeated. We can test for values between
6145      have to change if any new ones are ever created. */      ESC_b and ESC_Z for the latter; this may have to change if any new ones are
6146        ever created. */
6147    
6148      case CHAR_BACKSLASH:      case CHAR_BACKSLASH:
6149      tempptr = ptr;      tempptr = ptr;
# Line 5327  we set the flag only if there is a liter Line 6233  we set the flag only if there is a liter
6233          }          }
6234    
6235        /* \k<name> or \k'name' is a back reference by name (Perl syntax).        /* \k<name> or \k'name' is a back reference by name (Perl syntax).
6236        We also support \k{name} (.NET syntax) */        We also support \k{name} (.NET syntax).  */
6237    
6238        if (-c == ESC_k && (ptr[1] == CHAR_LESS_THAN_SIGN ||        if (-c == ESC_k)
           ptr[1] == CHAR_APOSTROPHE || ptr[1] == CHAR_LEFT_CURLY_BRACKET))  
6239          {          {
6240            if ((ptr[1] != CHAR_LESS_THAN_SIGN &&
6241              ptr[1] != CHAR_APOSTROPHE && ptr[1] != CHAR_LEFT_CURLY_BRACKET))
6242              {
6243              *errorcodeptr = ERR69;
6244              break;
6245              }
6246          is_recurse = FALSE;          is_recurse = FALSE;
6247          terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)?          terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)?
6248            CHAR_GREATER_THAN_SIGN : (*ptr == CHAR_APOSTROPHE)?            CHAR_GREATER_THAN_SIGN : (*ptr == CHAR_APOSTROPHE)?
# Line 5345  we set the flag only if there is a liter Line 6256  we set the flag only if there is a liter
6256    
6257        if (-c >= ESC_REF)        if (-c >= ESC_REF)
6258          {          {
6259            open_capitem *oc;
6260          recno = -c - ESC_REF;          recno = -c - ESC_REF;
6261    
6262          HANDLE_REFERENCE:    /* Come here from named backref handling */          HANDLE_REFERENCE:    /* Come here from named backref handling */
6263          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
6264          previous = code;          previous = code;
6265          *code++ = OP_REF;          *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF;
6266          PUT2INC(code, 0, recno);          PUT2INC(code, 0, recno);
6267          cd->backref_map |= (recno < 32)? (1 << recno) : 1;          cd->backref_map |= (recno < 32)? (1 << recno) : 1;
6268          if (recno > cd->top_backref) cd->top_backref = recno;          if (recno > cd->top_backref) cd->top_backref = recno;
6269    
6270            /* Check to see if this back reference is recursive, that it, it
6271            is inside the group that it references. A flag is set so that the
6272            group can be made atomic. */
6273    
6274            for (oc = cd->open_caps; oc != NULL; oc = oc->next)
6275              {
6276              if (oc->number == recno)
6277                {
6278                oc->flag = TRUE;
6279                break;
6280                }
6281              }
6282          }          }
6283    
6284        /* So are Unicode property matches, if supported. */        /* So are Unicode property matches, if supported. */
# Line 5383  we set the flag only if there is a liter Line 6308  we set the flag only if there is a liter
6308  #endif  #endif
6309    
6310        /* For the rest (including \X when Unicode properties are supported), we        /* For the rest (including \X when Unicode properties are supported), we
6311        can obtain the OP value by negating the escape value. */        can obtain the OP value by negating the escape value in the default
6312          situation when PCRE_UCP is not set. When it *is* set, we substitute
6313          Unicode property tests. */
6314    
6315        else        else
6316          {          {
6317          previous = (-c > ESC_b && -c < ESC_Z)? code : NULL;  #ifdef SUPPORT_UCP
6318          *code++ = -c;          if (-c >= ESC_DU && -c <= ESC_wu)
6319              {
6320              nestptr = ptr + 1;                   /* Where to resume */
6321              ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */
6322              }
6323            else
6324    #endif
6325              {
6326              previous = (-c > ESC_b && -c < ESC_Z)? code : NULL;
6327              *code++ = -c;
6328              }
6329          }          }
6330        continue;        continue;
6331        }        }
# Line 5433  we set the flag only if there is a liter Line 6370  we set the flag only if there is a liter
6370    
6371      ONE_CHAR:      ONE_CHAR:
6372      previous = code;      previous = code;
6373      *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARNC : OP_CHAR;      *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARI : OP_CHAR;
6374      for (c = 0; c < mclength; c++) *code++ = mcbuffer[c];      for (c = 0; c < mclength; c++) *code++ = mcbuffer[c];
6375    
6376      /* Remember if \r or \n were seen */      /* Remember if \r or \n were seen */
# Line 5462  we set the flag only if there is a liter Line 6399  we set the flag only if there is a liter
6399        else firstbyte = reqbyte = REQ_NONE;        else firstbyte = reqbyte = REQ_NONE;
6400        }        }
6401    
6402      /* firstbyte was previously set; we can set reqbyte only the length is      /* firstbyte was previously set; we can set reqbyte only if the length is
6403      1 or the matching is caseful. */      1 or the matching is caseful. */
6404    
6405      else      else
# Line 5497  return FALSE; Line 6434  return FALSE;
6434  /* On entry, ptr is pointing past the bracket character, but on return it  /* On entry, ptr is pointing past the bracket character, but on return it
6435  points to the closing bracket, or vertical bar, or end of string. The code  points to the closing bracket, or vertical bar, or end of string. The code
6436  variable is pointing at the byte into which the BRA operator has been stored.  variable is pointing at the byte into which the BRA operator has been stored.
 If the ims options are changed at the start (for a (?ims: group) or during any  
 branch, we need to insert an OP_OPT item at the start of every following branch  
 to ensure they get set correctly at run time, and also pass the new options  
 into every subsequent branch compile.  
   
6437  This function is used during the pre-compile phase when we are trying to find  This function is used during the pre-compile phase when we are trying to find
6438  out the amount of memory needed, as well as during the real compile phase. The  out the amount of memory needed, as well as during the real compile phase. The
6439  value of lengthptr distinguishes the two phases.  value of lengthptr distinguishes the two phases.
6440    
6441  Arguments:  Arguments:
6442    options        option bits, including any changes for this subpattern    options        option bits, including any changes for this subpattern
   oldims         previous settings of ims option bits  
6443    codeptr        -> the address of the current code pointer    codeptr        -> the address of the current code pointer
6444    ptrptr         -> the address of the current pattern pointer    ptrptr         -> the address of the current pattern pointer
6445    errorcodeptr   -> pointer to error code variable    errorcodeptr   -> pointer to error code variable
6446    lookbehind     TRUE if this is a lookbehind assertion    lookbehind     TRUE if this is a lookbehind assertion
6447    reset_bracount TRUE to reset the count for each branch    reset_bracount TRUE to reset the count for each branch
6448    skipbytes      skip this many bytes at start (for brackets and OP_COND)    skipbytes      skip this many bytes at start (for brackets and OP_COND)
6449      cond_depth     depth of nesting for conditional subpatterns
6450    firstbyteptr   place to put the first required character, or a negative number    firstbyteptr   place to put the first required character, or a negative number
6451    reqbyteptr     place to put the last required character, or a negative number    reqbyteptr     place to put the last required character, or a negative number
6452    bcptr          pointer to the chain of currently open branches    bcptr          pointer to the chain of currently open branches
# Line 5526  Returns:         TRUE on success Line 6458  Returns:         TRUE on success
6458  */  */
6459    
6460  static BOOL  static BOOL
6461  compile_regex(int options, int oldims, uschar **codeptr, const uschar **ptrptr,  compile_regex(int options, uschar **codeptr, const uschar **ptrptr,
6462    int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes,    int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes,
6463    int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd,    int cond_depth, int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr,
6464    int *lengthptr)    compile_data *cd, int *lengthptr)
6465  {  {
6466  const uschar *ptr = *ptrptr;  const uschar *ptr = *ptrptr;
6467  uschar *code = *codeptr;  uschar *code = *codeptr;
6468  uschar *last_branch = code;  uschar *last_branch = code;
6469  uschar *start_bracket = code;  uschar *start_bracket = code;
6470  uschar *reverse_count = NULL;  uschar *reverse_count = NULL;
6471    open_capitem capitem;
6472    int capnumber = 0;
6473  int firstbyte, reqbyte;  int firstbyte, reqbyte;
6474  int branchfirstbyte, branchreqbyte;  int branchfirstbyte, branchreqbyte;
6475  int length;  int length;
# Line 5544  int max_bracount; Line 6478  int max_bracount;
6478  branch_chain bc;  branch_chain bc;
6479    
6480  bc.outer = bcptr;  bc.outer = bcptr;
6481  bc.current = code;  bc.current_branch = code;
6482    
6483  firstbyte = reqbyte = REQ_UNSET;  firstbyte = reqbyte = REQ_UNSET;
6484    
# Line 5562  the code that abstracts option settings Line 6496  the code that abstracts option settings
6496  them global. It tests the value of length for (2 + 2*LINK_SIZE) in the  them global. It tests the value of length for (2 + 2*LINK_SIZE) in the
6497  pre-compile phase to find out whether anything has yet been compiled or not. */  pre-compile phase to find out whether anything has yet been compiled or not. */
6498    
6499    /* If this is a capturing subpattern, add to the chain of open capturing items
6500    so that we can detect them if (*ACCEPT) is encountered. This is also used to
6501    detect groups that contain recursive back references to themselves. Note that
6502    only OP_CBRA need be tested here; changing this opcode to one of its variants,
6503    e.g. OP_SCBRAPOS, happens later, after the group has been compiled. */
6504    
6505    if (*code == OP_CBRA)
6506      {
6507      capnumber = GET2(code, 1 + LINK_SIZE);
6508      capitem.number = capnumber;
6509      capitem.next = cd->open_caps;
6510      capitem.flag = FALSE;
6511      cd->open_caps = &capitem;
6512      }
6513    
6514  /* Offset is set zero to mark that this bracket is still open */  /* Offset is set zero to mark that this bracket is still open */
6515    
6516  PUT(code, 1, 0);  PUT(code, 1, 0);
# Line 5577  for (;;) Line 6526  for (;;)
6526    
6527    if (reset_bracount) cd->bracount = orig_bracount;    if (reset_bracount) cd->bracount = orig_bracount;
6528    
   /* Handle a change of ims options at the start of the branch */  
   
   if ((options & PCRE_IMS) != oldims)  
     {  
     *code++ = OP_OPT;  
     *code++ = options & PCRE_IMS;  
     length += 2;  
     }  
   
6529    /* Set up dummy OP_REVERSE if lookbehind assertion */    /* Set up dummy OP_REVERSE if lookbehind assertion */
6530    
6531    if (lookbehind)    if (lookbehind)
# Line 5600  for (;;) Line 6540  for (;;)
6540    into the length. */    into the length. */
6541    
6542    if (!compile_branch(&options, &code, &ptr, errorcodeptr, &branchfirstbyte,    if (!compile_branch(&options, &code, &ptr, errorcodeptr, &branchfirstbyte,
6543          &branchreqbyte, &bc, cd, (lengthptr == NULL)? NULL : &length))          &branchreqbyte, &bc, cond_depth, cd,
6544            (lengthptr == NULL)? NULL : &length))
6545      {      {
6546      *ptrptr = ptr;      *ptrptr = ptr;
6547      return FALSE;      return FALSE;
# Line 5656  for (;;) Line 6597  for (;;)
6597    
6598      /* If lookbehind, check that this branch matches a fixed-length string, and      /* If lookbehind, check that this branch matches a fixed-length string, and
6599      put the length into the OP_REVERSE item. Temporarily mark the end of the      put the length into the OP_REVERSE item. Temporarily mark the end of the
6600      branch with OP_END. */      branch with OP_END. If the branch contains OP_RECURSE, the result is -3
6601        because there may be forward references that we can't check here. Set a
6602        flag to cause another lookbehind check at the end. Why not do it all at the
6603        end? Because common, erroneous checks are picked up here and the offset of
6604        the problem can be shown. */
6605    
6606      if (lookbehind)      if (lookbehind)
6607        {        {
6608        int fixed_length;        int fixed_length;
6609        *code = OP_END;        *code = OP_END;
6610        fixed_length = find_fixedlength(last_branch, options);        fixed_length = find_fixedlength(last_branch,  (options & PCRE_UTF8) != 0,
6611            FALSE, cd);
6612        DPRINTF(("fixed length = %d\n", fixed_length));        DPRINTF(("fixed length = %d\n", fixed_length));
6613        if (fixed_length < 0)        if (fixed_length == -3)
6614            {
6615            cd->check_lookbehind = TRUE;
6616            }
6617          else if (fixed_length < 0)
6618          {          {
6619          *errorcodeptr = (fixed_length == -2)? ERR36 : ERR25;          *errorcodeptr = (fixed_length == -2)? ERR36 : ERR25;
6620          *ptrptr = ptr;          *ptrptr = ptr;
6621          return FALSE;          return FALSE;
6622          }          }
6623        PUT(reverse_count, 0, fixed_length);        else { PUT(reverse_count, 0, fixed_length); }
6624        }        }
6625      }      }
6626    
# Line 5679  for (;;) Line 6629  for (;;)
6629    of offsets, with the field in the BRA item now becoming an offset to the    of offsets, w