/[pcre]/code/trunk/pcre_study.c
ViewVC logotype

Diff of /code/trunk/pcre_study.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 538 by ph10, Wed Jun 9 19:30:57 2010 UTC revision 1032 by ph10, Sat Sep 8 16:05:38 2012 UTC
# Line 6  Line 6 
6  and semantics are as close as possible to those of the Perl 5 language.  and semantics are as close as possible to those of the Perl 5 language.
7    
8                         Written by Philip Hazel                         Written by Philip Hazel
9             Copyright (c) 1997-2010 University of Cambridge             Copyright (c) 1997-2012 University of Cambridge
10    
11  -----------------------------------------------------------------------------  -----------------------------------------------------------------------------
12  Redistribution and use in source and binary forms, with or without  Redistribution and use in source and binary forms, with or without
# Line 52  supporting functions. */ Line 52  supporting functions. */
52    
53  /* Returns from set_start_bits() */  /* Returns from set_start_bits() */
54    
55  enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE };  enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE, SSB_UNKNOWN };
56    
57    
58    
# Line 66  string of that length that matches. In U Line 66  string of that length that matches. In U
66  rather than bytes.  rather than bytes.
67    
68  Arguments:  Arguments:
69    code       pointer to start of group (the bracket)    code            pointer to start of group (the bracket)
70    startcode  pointer to start of the whole pattern    startcode       pointer to start of the whole pattern
71    options    the compiling options    options         the compiling options
72      int             RECURSE depth
73    
74  Returns:   the minimum length  Returns:   the minimum length
75             -1 if \C was encountered             -1 if \C in UTF-8 mode or (*ACCEPT) was encountered
76             -2 internal error (missing capturing bracket)             -2 internal error (missing capturing bracket)
77               -3 internal error (opcode not listed)
78  */  */
79    
80  static int  static int
81  find_minlength(const uschar *code, const uschar *startcode, int options)  find_minlength(const pcre_uchar *code, const pcre_uchar *startcode, int options,
82      int recurse_depth)
83  {  {
84  int length = -1;  int length = -1;
85  BOOL utf8 = (options & PCRE_UTF8) != 0;  /* PCRE_UTF16 has the same value as PCRE_UTF8. */
86    BOOL utf = (options & PCRE_UTF8) != 0;
87  BOOL had_recurse = FALSE;  BOOL had_recurse = FALSE;
88  register int branchlength = 0;  register int branchlength = 0;
89  register uschar *cc = (uschar *)code + 1 + LINK_SIZE;  register pcre_uchar *cc = (pcre_uchar *)code + 1 + LINK_SIZE;
90    
91  if (*code == OP_CBRA || *code == OP_SCBRA) cc += 2;  if (*code == OP_CBRA || *code == OP_SCBRA ||
92        *code == OP_CBRAPOS || *code == OP_SCBRAPOS) cc += IMM2_SIZE;
93    
94  /* Scan along the opcodes for this branch. If we get to the end of the  /* Scan along the opcodes for this branch. If we get to the end of the
95  branch, check the length against that of the other branches. */  branch, check the length against that of the other branches. */
# Line 92  branch, check the length against that of Line 97  branch, check the length against that of
97  for (;;)  for (;;)
98    {    {
99    int d, min;    int d, min;
100    uschar *cs, *ce;    pcre_uchar *cs, *ce;
101    register int op = *cc;    register int op = *cc;
102    
103    switch (op)    switch (op)
# Line 118  for (;;) Line 123  for (;;)
123      case OP_SCBRA:      case OP_SCBRA:
124      case OP_BRA:      case OP_BRA:
125      case OP_SBRA:      case OP_SBRA:
126        case OP_CBRAPOS:
127        case OP_SCBRAPOS:
128        case OP_BRAPOS:
129        case OP_SBRAPOS:
130      case OP_ONCE:      case OP_ONCE:
131      d = find_minlength(cc, startcode, options);      case OP_ONCE_NC:
132        d = find_minlength(cc, startcode, options, recurse_depth);
133      if (d < 0) return d;      if (d < 0) return d;
134      branchlength += d;      branchlength += d;
135      do cc += GET(cc, 1); while (*cc == OP_ALT);      do cc += GET(cc, 1); while (*cc == OP_ALT);
136      cc += 1 + LINK_SIZE;      cc += 1 + LINK_SIZE;
137      break;      break;
138    
139        /* ACCEPT makes things far too complicated; we have to give up. */
140    
141        case OP_ACCEPT:
142        case OP_ASSERT_ACCEPT:
143        return -1;
144    
145      /* Reached end of a branch; if it's a ket it is the end of a nested      /* Reached end of a branch; if it's a ket it is the end of a nested
146      call. If it's ALT it is an alternation in a nested call. If it is      call. If it's ALT it is an alternation in a nested call. If it is END it's
147      END it's the end of the outer call. All can be handled by the same code. */      the end of the outer call. All can be handled by the same code. If an
148        ACCEPT was previously encountered, use the length that was in force at that
149        time, and pass back the shortest ACCEPT length. */
150    
151      case OP_ALT:      case OP_ALT:
152      case OP_KET:      case OP_KET:
153      case OP_KETRMAX:      case OP_KETRMAX:
154      case OP_KETRMIN:      case OP_KETRMIN:
155        case OP_KETRPOS:
156      case OP_END:      case OP_END:
157      if (length < 0 || (!had_recurse && branchlength < length))      if (length < 0 || (!had_recurse && branchlength < length))
158        length = branchlength;        length = branchlength;
159      if (*cc != OP_ALT) return length;      if (op != OP_ALT) return length;
160      cc += 1 + LINK_SIZE;      cc += 1 + LINK_SIZE;
161      branchlength = 0;      branchlength = 0;
162      had_recurse = FALSE;      had_recurse = FALSE;
# Line 160  for (;;) Line 179  for (;;)
179      case OP_RREF:      case OP_RREF:
180      case OP_NRREF:      case OP_NRREF:
181      case OP_DEF:      case OP_DEF:
     case OP_OPT:  
182      case OP_CALLOUT:      case OP_CALLOUT:
183      case OP_SOD:      case OP_SOD:
184      case OP_SOM:      case OP_SOM:
185      case OP_EOD:      case OP_EOD:
186      case OP_EODN:      case OP_EODN:
187      case OP_CIRC:      case OP_CIRC:
188        case OP_CIRCM:
189      case OP_DOLL:      case OP_DOLL:
190        case OP_DOLLM:
191      case OP_NOT_WORD_BOUNDARY:      case OP_NOT_WORD_BOUNDARY:
192      case OP_WORD_BOUNDARY:      case OP_WORD_BOUNDARY:
193      cc += _pcre_OP_lengths[*cc];      cc += PRIV(OP_lengths)[*cc];
194      break;      break;
195    
196      /* Skip over a subpattern that has a {0} or {0,x} quantifier */      /* Skip over a subpattern that has a {0} or {0,x} quantifier */
197    
198      case OP_BRAZERO:      case OP_BRAZERO:
199      case OP_BRAMINZERO:      case OP_BRAMINZERO:
200        case OP_BRAPOSZERO:
201      case OP_SKIPZERO:      case OP_SKIPZERO:
202      cc += _pcre_OP_lengths[*cc];      cc += PRIV(OP_lengths)[*cc];
203      do cc += GET(cc, 1); while (*cc == OP_ALT);      do cc += GET(cc, 1); while (*cc == OP_ALT);
204      cc += 1 + LINK_SIZE;      cc += 1 + LINK_SIZE;
205      break;      break;
# Line 186  for (;;) Line 207  for (;;)
207      /* Handle literal characters and + repetitions */      /* Handle literal characters and + repetitions */
208    
209      case OP_CHAR:      case OP_CHAR:
210      case OP_CHARNC:      case OP_CHARI:
211      case OP_NOT:      case OP_NOT:
212        case OP_NOTI:
213      case OP_PLUS:      case OP_PLUS:
214        case OP_PLUSI:
215      case OP_MINPLUS:      case OP_MINPLUS:
216        case OP_MINPLUSI:
217      case OP_POSPLUS:      case OP_POSPLUS:
218        case OP_POSPLUSI:
219      case OP_NOTPLUS:      case OP_NOTPLUS:
220        case OP_NOTPLUSI:
221      case OP_NOTMINPLUS:      case OP_NOTMINPLUS:
222        case OP_NOTMINPLUSI:
223      case OP_NOTPOSPLUS:      case OP_NOTPOSPLUS:
224        case OP_NOTPOSPLUSI:
225      branchlength++;      branchlength++;
226      cc += 2;      cc += 2;
227  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF
228      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];      if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
229  #endif  #endif
230      break;      break;
231    
# Line 212  for (;;) Line 240  for (;;)
240      need to skip over a multibyte character in UTF8 mode.  */      need to skip over a multibyte character in UTF8 mode.  */
241    
242      case OP_EXACT:      case OP_EXACT:
243        case OP_EXACTI:
244      case OP_NOTEXACT:      case OP_NOTEXACT:
245        case OP_NOTEXACTI:
246      branchlength += GET2(cc,1);      branchlength += GET2(cc,1);
247      cc += 4;      cc += 2 + IMM2_SIZE;
248  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF
249      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];      if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
250  #endif  #endif
251      break;      break;
252    
253      case OP_TYPEEXACT:      case OP_TYPEEXACT:
254      branchlength += GET2(cc,1);      branchlength += GET2(cc,1);
255      cc += (cc[3] == OP_PROP || cc[3] == OP_NOTPROP)? 6 : 4;      cc += 2 + IMM2_SIZE + ((cc[1 + IMM2_SIZE] == OP_PROP
256          || cc[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0);
257      break;      break;
258    
259      /* Handle single-char non-literal matchers */      /* Handle single-char non-literal matchers */
# Line 249  for (;;) Line 280  for (;;)
280      cc++;      cc++;
281      break;      break;
282    
283      /* "Any newline" might match two characters */      /* "Any newline" might match two characters, but it also might match just
284        one. */
285    
286      case OP_ANYNL:      case OP_ANYNL:
287      branchlength += 2;      branchlength += 1;
288      cc++;      cc++;
289      break;      break;
290    
291      /* The single-byte matcher means we can't proceed in UTF-8 mode */      /* The single-byte matcher means we can't proceed in UTF-8 mode. (In
292        non-UTF-8 mode \C will actually be turned into OP_ALLANY, so won't ever
293        appear, but leave the code, just in case.) */
294    
295      case OP_ANYBYTE:      case OP_ANYBYTE:
296  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF
297      if (utf8) return -1;      if (utf) return -1;
298  #endif  #endif
299      branchlength++;      branchlength++;
300      cc++;      cc++;
# Line 276  for (;;) Line 310  for (;;)
310      case OP_TYPEPOSSTAR:      case OP_TYPEPOSSTAR:
311      case OP_TYPEPOSQUERY:      case OP_TYPEPOSQUERY:
312      if (cc[1] == OP_PROP || cc[1] == OP_NOTPROP) cc += 2;      if (cc[1] == OP_PROP || cc[1] == OP_NOTPROP) cc += 2;
313      cc += _pcre_OP_lengths[op];      cc += PRIV(OP_lengths)[op];
314      break;      break;
315    
316      case OP_TYPEUPTO:      case OP_TYPEUPTO:
317      case OP_TYPEMINUPTO:      case OP_TYPEMINUPTO:
318      case OP_TYPEPOSUPTO:      case OP_TYPEPOSUPTO:
319      if (cc[3] == OP_PROP || cc[3] == OP_NOTPROP) cc += 2;      if (cc[1 + IMM2_SIZE] == OP_PROP
320      cc += _pcre_OP_lengths[op];        || cc[1 + IMM2_SIZE] == OP_NOTPROP) cc += 2;
321        cc += PRIV(OP_lengths)[op];
322      break;      break;
323    
324      /* Check a class for variable quantification */      /* Check a class for variable quantification */
325    
326  #ifdef SUPPORT_UTF8  #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
327      case OP_XCLASS:      case OP_XCLASS:
328      cc += GET(cc, 1) - 33;      cc += GET(cc, 1) - PRIV(OP_lengths)[OP_CLASS];
329      /* Fall through */      /* Fall through */
330  #endif  #endif
331    
332      case OP_CLASS:      case OP_CLASS:
333      case OP_NCLASS:      case OP_NCLASS:
334      cc += 33;      cc += PRIV(OP_lengths)[OP_CLASS];
335    
336      switch (*cc)      switch (*cc)
337        {        {
# Line 315  for (;;) Line 350  for (;;)
350        case OP_CRRANGE:        case OP_CRRANGE:
351        case OP_CRMINRANGE:        case OP_CRMINRANGE:
352        branchlength += GET2(cc,1);        branchlength += GET2(cc,1);
353        cc += 5;        cc += 1 + 2 * IMM2_SIZE;
354        break;        break;
355    
356        default:        default:
# Line 337  for (;;) Line 372  for (;;)
372      that case we must set the minimum length to zero. */      that case we must set the minimum length to zero. */
373    
374      case OP_REF:      case OP_REF:
375        case OP_REFI:
376      if ((options & PCRE_JAVASCRIPT_COMPAT) == 0)      if ((options & PCRE_JAVASCRIPT_COMPAT) == 0)
377        {        {
378        ce = cs = (uschar *)_pcre_find_bracket(startcode, utf8, GET2(cc, 1));        ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(cc, 1));
379        if (cs == NULL) return -2;        if (cs == NULL) return -2;
380        do ce += GET(ce, 1); while (*ce == OP_ALT);        do ce += GET(ce, 1); while (*ce == OP_ALT);
381        if (cc > cs && cc < ce)        if (cc > cs && cc < ce)
# Line 347  for (;;) Line 383  for (;;)
383          d = 0;          d = 0;
384          had_recurse = TRUE;          had_recurse = TRUE;
385          }          }
386        else d = find_minlength(cs, startcode, options);        else
387            {
388            d = find_minlength(cs, startcode, options, recurse_depth);
389            }
390        }        }
391      else d = 0;      else d = 0;
392      cc += 3;      cc += 1 + IMM2_SIZE;
393    
394      /* Handle repeated back references */      /* Handle repeated back references */
395    
# Line 364  for (;;) Line 403  for (;;)
403        cc++;        cc++;
404        break;        break;
405    
406          case OP_CRPLUS:
407          case OP_CRMINPLUS:
408          min = 1;
409          cc++;
410          break;
411    
412        case OP_CRRANGE:        case OP_CRRANGE:
413        case OP_CRMINRANGE:        case OP_CRMINRANGE:
414        min = GET2(cc, 1);        min = GET2(cc, 1);
415        cc += 5;        cc += 1 + 2 * IMM2_SIZE;
416        break;        break;
417    
418        default:        default:
# Line 378  for (;;) Line 423  for (;;)
423      branchlength += min * d;      branchlength += min * d;
424      break;      break;
425    
426        /* We can easily detect direct recursion, but not mutual recursion. This is
427        caught by a recursion depth count. */
428    
429      case OP_RECURSE:      case OP_RECURSE:
430      cs = ce = (uschar *)startcode + GET(cc, 1);      cs = ce = (pcre_uchar *)startcode + GET(cc, 1);
     if (cs == NULL) return -2;  
431      do ce += GET(ce, 1); while (*ce == OP_ALT);      do ce += GET(ce, 1); while (*ce == OP_ALT);
432      if (cc > cs && cc < ce)      if ((cc > cs && cc < ce) || recurse_depth > 10)
433        had_recurse = TRUE;        had_recurse = TRUE;
434      else      else
435        branchlength += find_minlength(cs, startcode, options);        {
436          branchlength += find_minlength(cs, startcode, options, recurse_depth + 1);
437          }
438      cc += 1 + LINK_SIZE;      cc += 1 + LINK_SIZE;
439      break;      break;
440    
441      /* Anything else does not or need not match a character. We can get the      /* Anything else does not or need not match a character. We can get the
442      item's length from the table, but for those that can match zero occurrences      item's length from the table, but for those that can match zero occurrences
443      of a character, we must take special action for UTF-8 characters. */      of a character, we must take special action for UTF-8 characters. As it
444        happens, the "NOT" versions of these opcodes are used at present only for
445        ASCII characters, so they could be omitted from this list. However, in
446        future that may change, so we include them here so as not to leave a
447        gotcha for a future maintainer. */
448    
449      case OP_UPTO:      case OP_UPTO:
450        case OP_UPTOI:
451      case OP_NOTUPTO:      case OP_NOTUPTO:
452        case OP_NOTUPTOI:
453      case OP_MINUPTO:      case OP_MINUPTO:
454        case OP_MINUPTOI:
455      case OP_NOTMINUPTO:      case OP_NOTMINUPTO:
456        case OP_NOTMINUPTOI:
457      case OP_POSUPTO:      case OP_POSUPTO:
458        case OP_POSUPTOI:
459        case OP_NOTPOSUPTO:
460        case OP_NOTPOSUPTOI:
461    
462      case OP_STAR:      case OP_STAR:
463        case OP_STARI:
464        case OP_NOTSTAR:
465        case OP_NOTSTARI:
466      case OP_MINSTAR:      case OP_MINSTAR:
467        case OP_MINSTARI:
468      case OP_NOTMINSTAR:      case OP_NOTMINSTAR:
469        case OP_NOTMINSTARI:
470      case OP_POSSTAR:      case OP_POSSTAR:
471        case OP_POSSTARI:
472      case OP_NOTPOSSTAR:      case OP_NOTPOSSTAR:
473        case OP_NOTPOSSTARI:
474    
475      case OP_QUERY:      case OP_QUERY:
476        case OP_QUERYI:
477        case OP_NOTQUERY:
478        case OP_NOTQUERYI:
479      case OP_MINQUERY:      case OP_MINQUERY:
480        case OP_MINQUERYI:
481      case OP_NOTMINQUERY:      case OP_NOTMINQUERY:
482        case OP_NOTMINQUERYI:
483      case OP_POSQUERY:      case OP_POSQUERY:
484        case OP_POSQUERYI:
485      case OP_NOTPOSQUERY:      case OP_NOTPOSQUERY:
486      cc += _pcre_OP_lengths[op];      case OP_NOTPOSQUERYI:
487  #ifdef SUPPORT_UTF8  
488      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];      cc += PRIV(OP_lengths)[op];
489    #ifdef SUPPORT_UTF
490        if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
491  #endif  #endif
492      break;      break;
493    
# Line 420  for (;;) Line 497  for (;;)
497      case OP_PRUNE_ARG:      case OP_PRUNE_ARG:
498      case OP_SKIP_ARG:      case OP_SKIP_ARG:
499      case OP_THEN_ARG:      case OP_THEN_ARG:
500      cc += _pcre_OP_lengths[op] + cc[1];      cc += PRIV(OP_lengths)[op] + cc[1];
501      break;      break;
502    
503      /* For the record, these are the opcodes that are matched by "default":      /* The remaining opcodes are just skipped over. */
     OP_ACCEPT, OP_CLOSE, OP_COMMIT, OP_FAIL, OP_PRUNE, OP_SET_SOM, OP_SKIP,  
     OP_THEN. */  
504    
505      default:      case OP_CLOSE:
506      cc += _pcre_OP_lengths[op];      case OP_COMMIT:
507        case OP_FAIL:
508        case OP_PRUNE:
509        case OP_SET_SOM:
510        case OP_SKIP:
511        case OP_THEN:
512        cc += PRIV(OP_lengths)[op];
513      break;      break;
514    
515        /* This should not occur: we list all opcodes explicitly so that when
516        new ones get added they are properly considered. */
517    
518        default:
519        return -3;
520      }      }
521    }    }
522  /* Control never gets here */  /* Control never gets here */
# Line 451  Arguments: Line 538  Arguments:
538    p             points to the character    p             points to the character
539    caseless      the caseless flag    caseless      the caseless flag
540    cd            the block with char table pointers    cd            the block with char table pointers
541    utf8          TRUE for UTF-8 mode    utf           TRUE for UTF-8 / UTF-16 mode
542    
543  Returns:        pointer after the character  Returns:        pointer after the character
544  */  */
545    
546  static const uschar *  static const pcre_uchar *
547  set_table_bit(uschar *start_bits, const uschar *p, BOOL caseless,  set_table_bit(pcre_uint8 *start_bits, const pcre_uchar *p, BOOL caseless,
548    compile_data *cd, BOOL utf8)    compile_data *cd, BOOL utf)
549  {  {
550  unsigned int c = *p;  unsigned int c = *p;
551    
552    #ifdef COMPILE_PCRE8
553  SET_BIT(c);  SET_BIT(c);
554    
555  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF
556  if (utf8 && c > 127)  if (utf && c > 127)
557    {    {
558    GETCHARINC(c, p);    GETCHARINC(c, p);
559  #ifdef SUPPORT_UCP  #ifdef SUPPORT_UCP
560    if (caseless)    if (caseless)
561      {      {
562      uschar buff[8];      pcre_uchar buff[6];
563      c = UCD_OTHERCASE(c);      c = UCD_OTHERCASE(c);
564      (void)_pcre_ord2utf8(c, buff);      (void)PRIV(ord2utf)(c, buff);
565      SET_BIT(buff[0]);      SET_BIT(buff[0]);
566      }      }
567  #endif  #endif  /* Not SUPPORT_UCP */
568    return p;    return p;
569    }    }
570    #else  /* Not SUPPORT_UTF */
571    (void)(utf);   /* Stops warning for unused parameter */
572  #endif  #endif
573    
574  /* Not UTF-8 mode, or character is less than 127. */  /* Not UTF-8 mode, or character is less than 127. */
575    
576  if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]);  if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]);
577  return p + 1;  return p + 1;
578    #endif
579    
580    #ifdef COMPILE_PCRE16
581    if (c > 0xff)
582      {
583      c = 0xff;
584      caseless = FALSE;
585      }
586    SET_BIT(c);
587    
588    #ifdef SUPPORT_UTF
589    if (utf && c > 127)
590      {
591      GETCHARINC(c, p);
592    #ifdef SUPPORT_UCP
593      if (caseless)
594        {
595        c = UCD_OTHERCASE(c);
596        if (c > 0xff)
597          c = 0xff;
598        SET_BIT(c);
599        }
600    #endif
601      return p;
602      }
603    #endif
604    
605    if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]);
606    return p + 1;
607    #endif
608    }
609    
610    
611    
612    /*************************************************
613    *     Set bits for a positive character type     *
614    *************************************************/
615    
616    /* This function sets starting bits for a character type. In UTF-8 mode, we can
617    only do a direct setting for bytes less than 128, as otherwise there can be
618    confusion with bytes in the middle of UTF-8 characters. In a "traditional"
619    environment, the tables will only recognize ASCII characters anyway, but in at
620    least one Windows environment, some higher bytes bits were set in the tables.
621    So we deal with that case by considering the UTF-8 encoding.
622    
623    Arguments:
624      start_bits     the starting bitmap
625      cbit type      the type of character wanted
626      table_limit    32 for non-UTF-8; 16 for UTF-8
627      cd             the block with char table pointers
628    
629    Returns:         nothing
630    */
631    
632    static void
633    set_type_bits(pcre_uint8 *start_bits, int cbit_type, int table_limit,
634      compile_data *cd)
635    {
636    register int c;
637    for (c = 0; c < table_limit; c++) start_bits[c] |= cd->cbits[c+cbit_type];
638    #if defined SUPPORT_UTF && defined COMPILE_PCRE8
639    if (table_limit == 32) return;
640    for (c = 128; c < 256; c++)
641      {
642      if ((cd->cbits[c/8] & (1 << (c&7))) != 0)
643        {
644        pcre_uchar buff[6];
645        (void)PRIV(ord2utf)(c, buff);
646        SET_BIT(buff[0]);
647        }
648      }
649    #endif
650    }
651    
652    
653    /*************************************************
654    *     Set bits for a negative character type     *
655    *************************************************/
656    
657    /* This function sets starting bits for a negative character type such as \D.
658    In UTF-8 mode, we can only do a direct setting for bytes less than 128, as
659    otherwise there can be confusion with bytes in the middle of UTF-8 characters.
660    Unlike in the positive case, where we can set appropriate starting bits for
661    specific high-valued UTF-8 characters, in this case we have to set the bits for
662    all high-valued characters. The lowest is 0xc2, but we overkill by starting at
663    0xc0 (192) for simplicity.
664    
665    Arguments:
666      start_bits     the starting bitmap
667      cbit type      the type of character wanted
668      table_limit    32 for non-UTF-8; 16 for UTF-8
669      cd             the block with char table pointers
670    
671    Returns:         nothing
672    */
673    
674    static void
675    set_nottype_bits(pcre_uint8 *start_bits, int cbit_type, int table_limit,
676      compile_data *cd)
677    {
678    register int c;
679    for (c = 0; c < table_limit; c++) start_bits[c] |= ~cd->cbits[c+cbit_type];
680    #if defined SUPPORT_UTF && defined COMPILE_PCRE8
681    if (table_limit != 32) for (c = 24; c < 32; c++) start_bits[c] = 0xff;
682    #endif
683  }  }
684    
685    
# Line 504  function fails unless the result is SSB_ Line 699  function fails unless the result is SSB_
699  Arguments:  Arguments:
700    code         points to an expression    code         points to an expression
701    start_bits   points to a 32-byte table, initialized to 0    start_bits   points to a 32-byte table, initialized to 0
702    caseless     the current state of the caseless flag    utf          TRUE if in UTF-8 / UTF-16 mode
   utf8         TRUE if in UTF-8 mode  
703    cd           the block with char table pointers    cd           the block with char table pointers
704    
705  Returns:       SSB_FAIL     => Failed to find any starting bytes  Returns:       SSB_FAIL     => Failed to find any starting bytes
706                 SSB_DONE     => Found mandatory starting bytes                 SSB_DONE     => Found mandatory starting bytes
707                 SSB_CONTINUE => Found optional starting bytes                 SSB_CONTINUE => Found optional starting bytes
708                   SSB_UNKNOWN  => Hit an unrecognized opcode
709  */  */
710    
711  static int  static int
712  set_start_bits(const uschar *code, uschar *start_bits, BOOL caseless,  set_start_bits(const pcre_uchar *code, pcre_uint8 *start_bits, BOOL utf,
713    BOOL utf8, compile_data *cd)    compile_data *cd)
714  {  {
715  register int c;  register int c;
716  int yield = SSB_DONE;  int yield = SSB_DONE;
717  int table_limit = utf8? 16:32;  #if defined SUPPORT_UTF && defined COMPILE_PCRE8
718    int table_limit = utf? 16:32;
719    #else
720    int table_limit = 32;
721    #endif
722    
723  #if 0  #if 0
724  /* ========================================================================= */  /* ========================================================================= */
# Line 540  volatile int dummy; Line 739  volatile int dummy;
739    
740  do  do
741    {    {
   const uschar *tcode = code + (((int)*code == OP_CBRA)? 3:1) + LINK_SIZE;  
742    BOOL try_next = TRUE;    BOOL try_next = TRUE;
743      const pcre_uchar *tcode = code + 1 + LINK_SIZE;
744    
745      if (*code == OP_CBRA || *code == OP_SCBRA ||
746          *code == OP_CBRAPOS || *code == OP_SCBRAPOS) tcode += IMM2_SIZE;
747    
748    while (try_next)    /* Loop for items in this branch */    while (try_next)    /* Loop for items in this branch */
749      {      {
750      int rc;      int rc;
751    
752      switch(*tcode)      switch(*tcode)
753        {        {
754        /* Fail if we reach something we don't understand */        /* If we reach something we don't understand, it means a new opcode has
755          been created that hasn't been added to this code. Hopefully this problem
756          will be discovered during testing. */
757    
758        default:        default:
759          return SSB_UNKNOWN;
760    
761          /* Fail for a valid opcode that implies no starting bits. */
762    
763          case OP_ACCEPT:
764          case OP_ASSERT_ACCEPT:
765          case OP_ALLANY:
766          case OP_ANY:
767          case OP_ANYBYTE:
768          case OP_CIRC:
769          case OP_CIRCM:
770          case OP_CLOSE:
771          case OP_COMMIT:
772          case OP_COND:
773          case OP_CREF:
774          case OP_DEF:
775          case OP_DOLL:
776          case OP_DOLLM:
777          case OP_END:
778          case OP_EOD:
779          case OP_EODN:
780          case OP_EXTUNI:
781          case OP_FAIL:
782          case OP_MARK:
783          case OP_NCREF:
784          case OP_NOT:
785          case OP_NOTEXACT:
786          case OP_NOTEXACTI:
787          case OP_NOTI:
788          case OP_NOTMINPLUS:
789          case OP_NOTMINPLUSI:
790          case OP_NOTMINQUERY:
791          case OP_NOTMINQUERYI:
792          case OP_NOTMINSTAR:
793          case OP_NOTMINSTARI:
794          case OP_NOTMINUPTO:
795          case OP_NOTMINUPTOI:
796          case OP_NOTPLUS:
797          case OP_NOTPLUSI:
798          case OP_NOTPOSPLUS:
799          case OP_NOTPOSPLUSI:
800          case OP_NOTPOSQUERY:
801          case OP_NOTPOSQUERYI:
802          case OP_NOTPOSSTAR:
803          case OP_NOTPOSSTARI:
804          case OP_NOTPOSUPTO:
805          case OP_NOTPOSUPTOI:
806          case OP_NOTPROP:
807          case OP_NOTQUERY:
808          case OP_NOTQUERYI:
809          case OP_NOTSTAR:
810          case OP_NOTSTARI:
811          case OP_NOTUPTO:
812          case OP_NOTUPTOI:
813          case OP_NOT_HSPACE:
814          case OP_NOT_VSPACE:
815          case OP_NRREF:
816          case OP_PROP:
817          case OP_PRUNE:
818          case OP_PRUNE_ARG:
819          case OP_RECURSE:
820          case OP_REF:
821          case OP_REFI:
822          case OP_REVERSE:
823          case OP_RREF:
824          case OP_SCOND:
825          case OP_SET_SOM:
826          case OP_SKIP:
827          case OP_SKIP_ARG:
828          case OP_SOD:
829          case OP_SOM:
830          case OP_THEN:
831          case OP_THEN_ARG:
832    #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
833          case OP_XCLASS:
834    #endif
835        return SSB_FAIL;        return SSB_FAIL;
836    
837          /* We can ignore word boundary tests. */
838    
839          case OP_WORD_BOUNDARY:
840          case OP_NOT_WORD_BOUNDARY:
841          tcode++;
842          break;
843    
844        /* If we hit a bracket or a positive lookahead assertion, recurse to set        /* If we hit a bracket or a positive lookahead assertion, recurse to set
845        bits from within the subpattern. If it can't find anything, we have to        bits from within the subpattern. If it can't find anything, we have to
846        give up. If it finds some mandatory character(s), we are done for this        give up. If it finds some mandatory character(s), we are done for this
# Line 562  do Line 850  do
850        case OP_SBRA:        case OP_SBRA:
851        case OP_CBRA:        case OP_CBRA:
852        case OP_SCBRA:        case OP_SCBRA:
853          case OP_BRAPOS:
854          case OP_SBRAPOS:
855          case OP_CBRAPOS:
856          case OP_SCBRAPOS:
857        case OP_ONCE:        case OP_ONCE:
858          case OP_ONCE_NC:
859        case OP_ASSERT:        case OP_ASSERT:
860        rc = set_start_bits(tcode, start_bits, caseless, utf8, cd);        rc = set_start_bits(tcode, start_bits, utf, cd);
861        if (rc == SSB_FAIL) return SSB_FAIL;        if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc;
862        if (rc == SSB_DONE) try_next = FALSE; else        if (rc == SSB_DONE) try_next = FALSE; else
863          {          {
864          do tcode += GET(tcode, 1); while (*tcode == OP_ALT);          do tcode += GET(tcode, 1); while (*tcode == OP_ALT);
# Line 588  do Line 881  do
881        case OP_KET:        case OP_KET:
882        case OP_KETRMAX:        case OP_KETRMAX:
883        case OP_KETRMIN:        case OP_KETRMIN:
884          case OP_KETRPOS:
885        return SSB_CONTINUE;        return SSB_CONTINUE;
886    
887        /* Skip over callout */        /* Skip over callout */
# Line 605  do Line 899  do
899        tcode += 1 + LINK_SIZE;        tcode += 1 + LINK_SIZE;
900        break;        break;
901    
       /* Skip over an option setting, changing the caseless flag */  
   
       case OP_OPT:  
       caseless = (tcode[1] & PCRE_CASELESS) != 0;  
       tcode += 2;  
       break;  
   
902        /* BRAZERO does the bracket, but carries on. */        /* BRAZERO does the bracket, but carries on. */
903    
904        case OP_BRAZERO:        case OP_BRAZERO:
905        case OP_BRAMINZERO:        case OP_BRAMINZERO:
906        if (set_start_bits(++tcode, start_bits, caseless, utf8, cd) == SSB_FAIL)        case OP_BRAPOSZERO:
907          return SSB_FAIL;        rc = set_start_bits(++tcode, start_bits, utf, cd);
908          if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc;
909  /* =========================================================================  /* =========================================================================
910        See the comment at the head of this function concerning the next line,        See the comment at the head of this function concerning the next line,
911        which was an old fudge for the benefit of OS/2.        which was an old fudge for the benefit of OS/2.
# Line 643  do Line 931  do
931        case OP_QUERY:        case OP_QUERY:
932        case OP_MINQUERY:        case OP_MINQUERY:
933        case OP_POSQUERY:        case OP_POSQUERY:
934        tcode = set_table_bit(start_bits, tcode + 1, caseless, cd, utf8);        tcode = set_table_bit(start_bits, tcode + 1, FALSE, cd, utf);
935          break;
936    
937          case OP_STARI:
938          case OP_MINSTARI:
939          case OP_POSSTARI:
940          case OP_QUERYI:
941          case OP_MINQUERYI:
942          case OP_POSQUERYI:
943          tcode = set_table_bit(start_bits, tcode + 1, TRUE, cd, utf);
944        break;        break;
945    
946        /* Single-char upto sets the bit and tries the next */        /* Single-char upto sets the bit and tries the next */
# Line 651  do Line 948  do
948        case OP_UPTO:        case OP_UPTO:
949        case OP_MINUPTO:        case OP_MINUPTO:
950        case OP_POSUPTO:        case OP_POSUPTO:
951        tcode = set_table_bit(start_bits, tcode + 3, caseless, cd, utf8);        tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, FALSE, cd, utf);
952        break;        break;
953    
954        /* At least one single char sets the bit and stops */        case OP_UPTOI:
955          case OP_MINUPTOI:
956          case OP_POSUPTOI:
957          tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, TRUE, cd, utf);
958          break;
959    
960        case OP_EXACT:       /* Fall through */        /* At least one single char sets the bit and stops */
       tcode += 2;  
961    
962          case OP_EXACT:
963          tcode += IMM2_SIZE;
964          /* Fall through */
965        case OP_CHAR:        case OP_CHAR:
       case OP_CHARNC:  
966        case OP_PLUS:        case OP_PLUS:
967        case OP_MINPLUS:        case OP_MINPLUS:
968        case OP_POSPLUS:        case OP_POSPLUS:
969        (void)set_table_bit(start_bits, tcode + 1, caseless, cd, utf8);        (void)set_table_bit(start_bits, tcode + 1, FALSE, cd, utf);
970          try_next = FALSE;
971          break;
972    
973          case OP_EXACTI:
974          tcode += IMM2_SIZE;
975          /* Fall through */
976          case OP_CHARI:
977          case OP_PLUSI:
978          case OP_MINPLUSI:
979          case OP_POSPLUSI:
980          (void)set_table_bit(start_bits, tcode + 1, TRUE, cd, utf);
981        try_next = FALSE;        try_next = FALSE;
982        break;        break;
983    
# Line 677  do Line 990  do
990        case OP_HSPACE:        case OP_HSPACE:
991        SET_BIT(0x09);        SET_BIT(0x09);
992        SET_BIT(0x20);        SET_BIT(0x20);
993        if (utf8)  #ifdef SUPPORT_UTF
994          if (utf)
995          {          {
996          SET_BIT(0xC2);  /* For U+00A0 */  #ifdef COMPILE_PCRE8
997            SET_BIT(0xC2);  /* For U+00A0 */
998          SET_BIT(0xE1);  /* For U+1680, U+180E */          SET_BIT(0xE1);  /* For U+1680, U+180E */
999          SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */          SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */
1000          SET_BIT(0xE3);  /* For U+3000 */          SET_BIT(0xE3);  /* For U+3000 */
1001    #endif
1002    #ifdef COMPILE_PCRE16
1003            SET_BIT(0xA0);
1004            SET_BIT(0xFF);  /* For characters > 255 */
1005    #endif
1006            }
1007          else
1008    #endif /* SUPPORT_UTF */
1009            {
1010            SET_BIT(0xA0);
1011    #ifdef COMPILE_PCRE16
1012            SET_BIT(0xFF);  /* For characters > 255 */
1013    #endif
1014          }          }
       else SET_BIT(0xA0);  
1015        try_next = FALSE;        try_next = FALSE;
1016        break;        break;
1017    
# Line 694  do Line 1021  do
1021        SET_BIT(0x0B);        SET_BIT(0x0B);
1022        SET_BIT(0x0C);        SET_BIT(0x0C);
1023        SET_BIT(0x0D);        SET_BIT(0x0D);
1024        if (utf8)  #ifdef SUPPORT_UTF
1025          {        if (utf)
1026          SET_BIT(0xC2);  /* For U+0085 */          {
1027    #ifdef COMPILE_PCRE8
1028            SET_BIT(0xC2);  /* For U+0085 */
1029          SET_BIT(0xE2);  /* For U+2028, U+2029 */          SET_BIT(0xE2);  /* For U+2028, U+2029 */
1030          }  #endif
1031        else SET_BIT(0x85);  #ifdef COMPILE_PCRE16
1032            SET_BIT(0x85);
1033            SET_BIT(0xFF);  /* For characters > 255 */
1034    #endif
1035            }
1036          else
1037    #endif /* SUPPORT_UTF */
1038            {
1039            SET_BIT(0x85);
1040    #ifdef COMPILE_PCRE16
1041            SET_BIT(0xFF);  /* For characters > 255 */
1042    #endif
1043            }
1044        try_next = FALSE;        try_next = FALSE;
1045        break;        break;
1046    
1047        /* Single character types set the bits and stop. Note that if PCRE_UCP        /* Single character types set the bits and stop. Note that if PCRE_UCP
1048        is set, we do not see these op codes because \d etc are converted to        is set, we do not see these op codes because \d etc are converted to
1049        properties. Therefore, these apply in the case when only ASCII characters        properties. Therefore, these apply in the case when only characters less
1050        are recognized to match the types. In UTF-8 mode, we must restrict        than 256 are recognized to match the types. */
       ourselves to bytes less than 128, as otherwise there can be confusion  
       with bytes in the middle of UTF-8 characters. (In a "traditional"  
       environment, the tables will only recognize ASCII characters anyway, but  
       in at least one Windows environment, some higher bytes bits were set in  
       the tables.) */  
1051    
1052        case OP_NOT_DIGIT:        case OP_NOT_DIGIT:
1053        for (c = 0; c < table_limit; c++)        set_nottype_bits(start_bits, cbit_digit, table_limit, cd);
         start_bits[c] |= ~cd->cbits[c+cbit_digit];  
1054        try_next = FALSE;        try_next = FALSE;
1055        break;        break;
1056    
1057        case OP_DIGIT:        case OP_DIGIT:
1058        for (c = 0; c < table_limit; c++)        set_type_bits(start_bits, cbit_digit, table_limit, cd);
         start_bits[c] |= cd->cbits[c+cbit_digit];  
1059        try_next = FALSE;        try_next = FALSE;
1060        break;        break;
1061    
1062        /* The cbit_space table has vertical tab as whitespace; we have to        /* The cbit_space table has vertical tab as whitespace; we have to
1063        discard it. */        ensure it is set as not whitespace. */
1064    
1065        case OP_NOT_WHITESPACE:        case OP_NOT_WHITESPACE:
1066        for (c = 0; c < table_limit; c++)        set_nottype_bits(start_bits, cbit_space, table_limit, cd);
1067          {        start_bits[1] |= 0x08;
         int d = cd->cbits[c+cbit_space];  
         if (c == 1) d &= ~0x08;  
         start_bits[c] |= ~d;  
         }  
1068        try_next = FALSE;        try_next = FALSE;
1069        break;        break;
1070    
1071        /* The cbit_space table has vertical tab as whitespace; we have to        /* The cbit_space table has vertical tab as whitespace; we have to
1072        discard it. */        not set it from the table. */
1073    
1074        case OP_WHITESPACE:        case OP_WHITESPACE:
1075        for (c = 0; c < table_limit; c++)        c = start_bits[1];    /* Save in case it was already set */
1076          {        set_type_bits(start_bits, cbit_space, table_limit, cd);
1077          int d = cd->cbits[c+cbit_space];        start_bits[1] = (start_bits[1] & ~0x08) | c;
         if (c == 1) d &= ~0x08;  
         start_bits[c] |= d;  
         }  
1078        try_next = FALSE;        try_next = FALSE;
1079        break;        break;
1080    
1081        case OP_NOT_WORDCHAR:        case OP_NOT_WORDCHAR:
1082        for (c = 0; c < table_limit; c++)        set_nottype_bits(start_bits, cbit_word, table_limit, cd);
         start_bits[c] |= ~cd->cbits[c+cbit_word];  
1083        try_next = FALSE;        try_next = FALSE;
1084        break;        break;
1085    
1086        case OP_WORDCHAR:        case OP_WORDCHAR:
1087        for (c = 0; c < table_limit; c++)        set_type_bits(start_bits, cbit_word, table_limit, cd);
         start_bits[c] |= cd->cbits[c+cbit_word];  
1088        try_next = FALSE;        try_next = FALSE;
1089        break;        break;
1090    
# Line 773  do Line 1098  do
1098        break;        break;
1099    
1100        case OP_TYPEEXACT:        case OP_TYPEEXACT:
1101        tcode += 3;        tcode += 1 + IMM2_SIZE;
1102        break;        break;
1103    
1104        /* Zero or more repeats of character types set the bits and then        /* Zero or more repeats of character types set the bits and then
# Line 782  do Line 1107  do
1107        case OP_TYPEUPTO:        case OP_TYPEUPTO:
1108        case OP_TYPEMINUPTO:        case OP_TYPEMINUPTO:
1109        case OP_TYPEPOSUPTO:        case OP_TYPEPOSUPTO:
1110        tcode += 2;               /* Fall through */        tcode += IMM2_SIZE;  /* Fall through */
1111    
1112        case OP_TYPESTAR:        case OP_TYPESTAR:
1113        case OP_TYPEMINSTAR:        case OP_TYPEMINSTAR:
# Line 800  do Line 1125  do
1125          case OP_HSPACE:          case OP_HSPACE:
1126          SET_BIT(0x09);          SET_BIT(0x09);
1127          SET_BIT(0x20);          SET_BIT(0x20);
1128          if (utf8)  #ifdef SUPPORT_UTF
1129            if (utf)
1130            {            {
1131            SET_BIT(0xC2);  /* For U+00A0 */  #ifdef COMPILE_PCRE8
1132              SET_BIT(0xC2);  /* For U+00A0 */
1133            SET_BIT(0xE1);  /* For U+1680, U+180E */            SET_BIT(0xE1);  /* For U+1680, U+180E */
1134            SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */            SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */
1135            SET_BIT(0xE3);  /* For U+3000 */            SET_BIT(0xE3);  /* For U+3000 */
1136    #endif
1137    #ifdef COMPILE_PCRE16
1138              SET_BIT(0xA0);
1139              SET_BIT(0xFF);  /* For characters > 255 */
1140    #endif
1141            }            }
1142          else SET_BIT(0xA0);          else
1143    #endif /* SUPPORT_UTF */
1144              SET_BIT(0xA0);
1145          break;          break;
1146    
1147          case OP_ANYNL:          case OP_ANYNL:
# Line 816  do Line 1150  do
1150          SET_BIT(0x0B);          SET_BIT(0x0B);
1151          SET_BIT(0x0C);          SET_BIT(0x0C);
1152          SET_BIT(0x0D);          SET_BIT(0x0D);
1153          if (utf8)  #ifdef SUPPORT_UTF
1154            if (utf)
1155            {            {
1156            SET_BIT(0xC2);  /* For U+0085 */  #ifdef COMPILE_PCRE8
1157              SET_BIT(0xC2);  /* For U+0085 */
1158            SET_BIT(0xE2);  /* For U+2028, U+2029 */            SET_BIT(0xE2);  /* For U+2028, U+2029 */
1159            }  #endif
1160          else SET_BIT(0x85);  #ifdef COMPILE_PCRE16
1161              SET_BIT(0x85);
1162              SET_BIT(0xFF);  /* For characters > 255 */
1163    #endif
1164              }
1165            else
1166    #endif /* SUPPORT_UTF */
1167              SET_BIT(0x85);
1168          break;          break;
1169    
1170          case OP_NOT_DIGIT:          case OP_NOT_DIGIT:
1171          for (c = 0; c < table_limit; c++)          set_nottype_bits(start_bits, cbit_digit, table_limit, cd);
           start_bits[c] |= ~cd->cbits[c+cbit_digit];  
1172          break;          break;
1173    
1174          case OP_DIGIT:          case OP_DIGIT:
1175          for (c = 0; c < table_limit; c++)          set_type_bits(start_bits, cbit_digit, table_limit, cd);
           start_bits[c] |= cd->cbits[c+cbit_digit];  
1176          break;          break;
1177    
1178          /* The cbit_space table has vertical tab as whitespace; we have to          /* The cbit_space table has vertical tab as whitespace; we have to
1179          discard it. */          ensure it gets set as not whitespace. */
1180    
1181          case OP_NOT_WHITESPACE:          case OP_NOT_WHITESPACE:
1182          for (c = 0; c < table_limit; c++)          set_nottype_bits(start_bits, cbit_space, table_limit, cd);
1183            {          start_bits[1] |= 0x08;
           int d = cd->cbits[c+cbit_space];  
           if (c == 1) d &= ~0x08;  
           start_bits[c] |= ~d;  
           }  
1184          break;          break;
1185    
1186          /* The cbit_space table has vertical tab as whitespace; we have to          /* The cbit_space table has vertical tab as whitespace; we have to
1187          discard it. */          avoid setting it. */
1188    
1189          case OP_WHITESPACE:          case OP_WHITESPACE:
1190          for (c = 0; c < table_limit; c++)          c = start_bits[1];    /* Save in case it was already set */
1191            {          set_type_bits(start_bits, cbit_space, table_limit, cd);
1192            int d = cd->cbits[c+cbit_space];          start_bits[1] = (start_bits[1] & ~0x08) | c;
           if (c == 1) d &= ~0x08;  
           start_bits[c] |= d;  
           }  
1193          break;          break;
1194    
1195          case OP_NOT_WORDCHAR:          case OP_NOT_WORDCHAR:
1196          for (c = 0; c < table_limit; c++)          set_nottype_bits(start_bits, cbit_word, table_limit, cd);
           start_bits[c] |= ~cd->cbits[c+cbit_word];  
1197          break;          break;
1198    
1199          case OP_WORDCHAR:          case OP_WORDCHAR:
1200          for (c = 0; c < table_limit; c++)          set_type_bits(start_bits, cbit_word, table_limit, cd);
           start_bits[c] |= cd->cbits[c+cbit_word];  
1201          break;          break;
1202          }          }
1203    
# Line 879  do Line 1211  do
1211        character with a value > 255. */        character with a value > 255. */
1212    
1213        case OP_NCLASS:        case OP_NCLASS:
1214  #ifdef SUPPORT_UTF8  #if defined SUPPORT_UTF && defined COMPILE_PCRE8
1215        if (utf8)        if (utf)
1216          {          {
1217          start_bits[24] |= 0xf0;              /* Bits for 0xc4 - 0xc8 */          start_bits[24] |= 0xf0;              /* Bits for 0xc4 - 0xc8 */
1218          memset(start_bits+25, 0xff, 7);      /* Bits for 0xc9 - 0xff */          memset(start_bits+25, 0xff, 7);      /* Bits for 0xc9 - 0xff */
1219          }          }
1220  #endif  #endif
1221    #ifdef COMPILE_PCRE16
1222          SET_BIT(0xFF);                         /* For characters > 255 */
1223    #endif
1224        /* Fall through */        /* Fall through */
1225    
1226        case OP_CLASS:        case OP_CLASS:
1227          {          {
1228            pcre_uint8 *map;
1229          tcode++;          tcode++;
1230            map = (pcre_uint8 *)tcode;
1231    
1232          /* In UTF-8 mode, the bits in a bit map correspond to character          /* In UTF-8 mode, the bits in a bit map correspond to character
1233          values, not to byte values. However, the bit map we are constructing is          values, not to byte values. However, the bit map we are constructing is
# Line 898  do Line 1235  do
1235          value is > 127. In fact, there are only two possible starting bytes for          value is > 127. In fact, there are only two possible starting bytes for
1236          characters in the range 128 - 255. */          characters in the range 128 - 255. */
1237    
1238  #ifdef SUPPORT_UTF8  #if defined SUPPORT_UTF && defined COMPILE_PCRE8
1239          if (utf8)          if (utf)
1240            {            {
1241            for (c = 0; c < 16; c++) start_bits[c] |= tcode[c];            for (c = 0; c < 16; c++) start_bits[c] |= map[c];
1242            for (c = 128; c < 256; c++)            for (c = 128; c < 256; c++)
1243              {              {
1244              if ((tcode[c/8] && (1 << (c&7))) != 0)              if ((map[c/8] && (1 << (c&7))) != 0)
1245                {                {
1246                int d = (c >> 6) | 0xc0;            /* Set bit for this starter */                int d = (c >> 6) | 0xc0;            /* Set bit for this starter */
1247                start_bits[d/8] |= (1 << (d&7));    /* and then skip on to the */                start_bits[d/8] |= (1 << (d&7));    /* and then skip on to the */
# Line 912  do Line 1249  do
1249                }                }
1250              }              }
1251            }            }
   
         /* In non-UTF-8 mode, the two bit maps are completely compatible. */  
   
1252          else          else
1253  #endif  #endif
1254            {            {
1255            for (c = 0; c < 32; c++) start_bits[c] |= tcode[c];            /* In non-UTF-8 mode, the two bit maps are completely compatible. */
1256              for (c = 0; c < 32; c++) start_bits[c] |= map[c];
1257            }            }
1258    
1259          /* Advance past the bit map, and act on what follows */          /* Advance past the bit map, and act on what follows. For a zero
1260            minimum repeat, continue; otherwise stop processing. */
1261    
1262          tcode += 32;          tcode += 32 / sizeof(pcre_uchar);
1263          switch (*tcode)          switch (*tcode)
1264            {            {
1265            case OP_CRSTAR:            case OP_CRSTAR:
# Line 935  do Line 1271  do
1271    
1272            case OP_CRRANGE:            case OP_CRRANGE:
1273            case OP_CRMINRANGE:            case OP_CRMINRANGE:
1274            if (((tcode[1] << 8) + tcode[2]) == 0) tcode += 5;            if (GET2(tcode, 1) == 0) tcode += 1 + 2 * IMM2_SIZE;
1275              else try_next = FALSE;              else try_next = FALSE;
1276            break;            break;
1277    
# Line 957  return yield; Line 1293  return yield;
1293    
1294    
1295    
1296    
1297    
1298  /*************************************************  /*************************************************
1299  *          Study a compiled expression           *  *          Study a compiled expression           *
1300  *************************************************/  *************************************************/
1301    
1302  /* This function is handed a compiled expression that it must study to produce  /* This function is handed a compiled expression that it must study to produce
1303  information that will speed up the matching. It returns a pcre_extra block  information that will speed up the matching. It returns a pcre[16]_extra block
1304  which then gets handed back to pcre_exec().  which then gets handed back to pcre_exec().
1305    
1306  Arguments:  Arguments:
# Line 971  Arguments: Line 1309  Arguments:
1309    errorptr  points to where to place error messages;    errorptr  points to where to place error messages;
1310              set NULL unless error              set NULL unless error
1311    
1312  Returns:    pointer to a pcre_extra block, with study_data filled in and the  Returns:    pointer to a pcre[16]_extra block, with study_data filled in and
1313                appropriate flags set;                the appropriate flags set;
1314              NULL on error or if no optimization possible              NULL on error or if no optimization possible
1315  */  */
1316    
1317    #ifdef COMPILE_PCRE8
1318  PCRE_EXP_DEFN pcre_extra * PCRE_CALL_CONVENTION  PCRE_EXP_DEFN pcre_extra * PCRE_CALL_CONVENTION
1319  pcre_study(const pcre *external_re, int options, const char **errorptr)  pcre_study(const pcre *external_re, int options, const char **errorptr)
1320    #else
1321    PCRE_EXP_DEFN pcre16_extra * PCRE_CALL_CONVENTION
1322    pcre16_study(const pcre16 *external_re, int options, const char **errorptr)
1323    #endif
1324  {  {
1325  int min;  int min;
1326  BOOL bits_set = FALSE;  BOOL bits_set = FALSE;
1327  uschar start_bits[32];  pcre_uint8 start_bits[32];
1328  pcre_extra *extra;  PUBL(extra) *extra = NULL;
1329  pcre_study_data *study;  pcre_study_data *study;
1330  const uschar *tables;  const pcre_uint8 *tables;
1331  uschar *code;  pcre_uchar *code;
1332  compile_data compile_block;  compile_data compile_block;
1333  const real_pcre *re = (const real_pcre *)external_re;  const REAL_PCRE *re = (const REAL_PCRE *)external_re;
1334    
1335  *errorptr = NULL;  *errorptr = NULL;
1336    
# Line 997  if (re == NULL || re->magic_number != MA Line 1340  if (re == NULL || re->magic_number != MA
1340    return NULL;    return NULL;
1341    }    }
1342    
1343    if ((re->flags & PCRE_MODE) == 0)
1344      {
1345    #ifdef COMPILE_PCRE8
1346      *errorptr = "argument is compiled in 16 bit mode";
1347    #else
1348      *errorptr = "argument is compiled in 8 bit mode";
1349    #endif
1350      return NULL;
1351      }
1352    
1353  if ((options & ~PUBLIC_STUDY_OPTIONS) != 0)  if ((options & ~PUBLIC_STUDY_OPTIONS) != 0)
1354    {    {
1355    *errorptr = "unknown or incorrect option bit(s) set";    *errorptr = "unknown or incorrect option bit(s) set";
1356    return NULL;    return NULL;
1357    }    }
1358    
1359  code = (uschar *)re + re->name_table_offset +  code = (pcre_uchar *)re + re->name_table_offset +
1360    (re->name_count * re->name_entry_size);    (re->name_count * re->name_entry_size);
1361    
1362  /* For an anchored pattern, or an unanchored pattern that has a first char, or  /* For an anchored pattern, or an unanchored pattern that has a first char, or
# Line 1013  seeking a list of starting bytes. */ Line 1366  seeking a list of starting bytes. */
1366  if ((re->options & PCRE_ANCHORED) == 0 &&  if ((re->options & PCRE_ANCHORED) == 0 &&
1367      (re->flags & (PCRE_FIRSTSET|PCRE_STARTLINE)) == 0)      (re->flags & (PCRE_FIRSTSET|PCRE_STARTLINE)) == 0)
1368    {    {
1369      int rc;
1370    
1371    /* Set the character tables in the block that is passed around */    /* Set the character tables in the block that is passed around */
1372    
1373    tables = re->tables;    tables = re->tables;
1374    
1375    #ifdef COMPILE_PCRE8
1376    if (tables == NULL)    if (tables == NULL)
1377      (void)pcre_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES,      (void)pcre_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES,
1378      (void *)(&tables));      (void *)(&tables));
1379    #else
1380      if (tables == NULL)
1381        (void)pcre16_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES,
1382        (void *)(&tables));
1383    #endif
1384    
1385    compile_block.lcc = tables + lcc_offset;    compile_block.lcc = tables + lcc_offset;
1386    compile_block.fcc = tables + fcc_offset;    compile_block.fcc = tables + fcc_offset;
# Line 1027  if ((re->options & PCRE_ANCHORED) == 0 & Line 1389  if ((re->options & PCRE_ANCHORED) == 0 &
1389    
1390    /* See if we can find a fixed set of initial characters for the pattern. */    /* See if we can find a fixed set of initial characters for the pattern. */
1391    
1392    memset(start_bits, 0, 32 * sizeof(uschar));    memset(start_bits, 0, 32 * sizeof(pcre_uint8));
1393    bits_set = set_start_bits(code, start_bits,    rc = set_start_bits(code, start_bits, (re->options & PCRE_UTF8) != 0,
1394      (re->options & PCRE_CASELESS) != 0, (re->options & PCRE_UTF8) != 0,      &compile_block);
1395      &compile_block) == SSB_DONE;    bits_set = rc == SSB_DONE;
1396      if (rc == SSB_UNKNOWN)
1397        {
1398        *errorptr = "internal error: opcode not recognized";
1399        return NULL;
1400        }
1401    }    }
1402    
1403  /* Find the minimum length of subject string. */  /* Find the minimum length of subject string. */
1404    
1405  min = find_minlength(code, code, re->options);  switch(min = find_minlength(code, code, re->options, 0))
1406      {
1407      case -2: *errorptr = "internal error: missing capturing bracket"; return NULL;
1408      case -3: *errorptr = "internal error: opcode not recognized"; return NULL;
1409      default: break;
1410      }
1411    
1412  /* Return NULL if no optimization is possible. */  /* If a set of starting bytes has been identified, or if the minimum length is
1413    greater than zero, or if JIT optimization has been requested, or if
1414    PCRE_STUDY_EXTRA_NEEDED is set, get a pcre[16]_extra block and a
1415    pcre_study_data block. The study data is put in the latter, which is pointed to
1416    by the former, which may also get additional data set later by the calling
1417    program. At the moment, the size of pcre_study_data is fixed. We nevertheless
1418    save it in a field for returning via the pcre_fullinfo() function so that if it
1419    becomes variable in the future, we don't have to change that code. */
1420    
1421    if (bits_set || min > 0 || (options & (
1422    #ifdef SUPPORT_JIT
1423        PCRE_STUDY_JIT_COMPILE | PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE |
1424        PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE |
1425    #endif
1426        PCRE_STUDY_EXTRA_NEEDED)) != 0)
1427      {
1428      extra = (PUBL(extra) *)(PUBL(malloc))
1429        (sizeof(PUBL(extra)) + sizeof(pcre_study_data));
1430      if (extra == NULL)
1431        {
1432        *errorptr = "failed to get memory";
1433        return NULL;
1434        }
1435    
1436  if (!bits_set && min < 0) return NULL;    study = (pcre_study_data *)((char *)extra + sizeof(PUBL(extra)));
1437      extra->flags = PCRE_EXTRA_STUDY_DATA;
1438      extra->study_data = study;
1439    
1440      study->size = sizeof(pcre_study_data);
1441      study->flags = 0;
1442    
1443      /* Set the start bits always, to avoid unset memory errors if the
1444      study data is written to a file, but set the flag only if any of the bits
1445      are set, to save time looking when none are. */
1446    
1447  /* Get a pcre_extra block and a pcre_study_data block. The study data is put in    if (bits_set)
1448  the latter, which is pointed to by the former, which may also get additional      {
1449  data set later by the calling program. At the moment, the size of      study->flags |= PCRE_STUDY_MAPPED;
1450  pcre_study_data is fixed. We nevertheless save it in a field for returning via      memcpy(study->start_bits, start_bits, sizeof(start_bits));
1451  the pcre_fullinfo() function so that if it becomes variable in the future, we      }
1452  don't have to change that code. */    else memset(study->start_bits, 0, 32 * sizeof(pcre_uint8));
1453    
1454  extra = (pcre_extra *)(pcre_malloc)  #ifdef PCRE_DEBUG
1455    (sizeof(pcre_extra) + sizeof(pcre_study_data));    if (bits_set)
1456        {
1457        pcre_uint8 *ptr = start_bits;
1458        int i;
1459    
1460  if (extra == NULL)      printf("Start bits:\n");
1461    {      for (i = 0; i < 32; i++)
1462    *errorptr = "failed to get memory";        printf("%3d: %02x%s", i * 8, *ptr++, ((i + 1) & 0x7) != 0? " " : "\n");
1463    return NULL;      }
1464    }  #endif
1465    
1466  study = (pcre_study_data *)((char *)extra + sizeof(pcre_extra));    /* Always set the minlength value in the block, because the JIT compiler
1467  extra->flags = PCRE_EXTRA_STUDY_DATA;    makes use of it. However, don't set the bit unless the length is greater than
1468  extra->study_data = study;    zero - the interpretive pcre_exec() and pcre_dfa_exec() needn't waste time
1469      checking the zero case. */
1470    
1471  study->size = sizeof(pcre_study_data);    if (min > 0)
1472  study->flags = 0;      {
1473        study->flags |= PCRE_STUDY_MINLEN;
1474        study->minlength = min;
1475        }
1476      else study->minlength = 0;
1477    
1478  if (bits_set)    /* If JIT support was compiled and requested, attempt the JIT compilation.
1479    {    If no starting bytes were found, and the minimum length is zero, and JIT
1480    study->flags |= PCRE_STUDY_MAPPED;    compilation fails, abandon the extra block and return NULL, unless
1481    memcpy(study->start_bits, start_bits, sizeof(start_bits));    PCRE_STUDY_EXTRA_NEEDED is set. */
1482    }  
1483    #ifdef SUPPORT_JIT
1484      extra->executable_jit = NULL;
1485      if ((options & PCRE_STUDY_JIT_COMPILE) != 0)
1486        PRIV(jit_compile)(re, extra, JIT_COMPILE);
1487      if ((options & PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE) != 0)
1488        PRIV(jit_compile)(re, extra, JIT_PARTIAL_SOFT_COMPILE);
1489      if ((options & PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE) != 0)
1490        PRIV(jit_compile)(re, extra, JIT_PARTIAL_HARD_COMPILE);
1491    
1492  if (min >= 0)    if (study->flags == 0 && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) == 0 &&
1493    {        (options & PCRE_STUDY_EXTRA_NEEDED) == 0)
1494    study->flags |= PCRE_STUDY_MINLEN;      {
1495    study->minlength = min;  #ifdef COMPILE_PCRE8
1496        pcre_free_study(extra);
1497    #endif
1498    #ifdef COMPILE_PCRE16
1499        pcre16_free_study(extra);
1500    #endif
1501        extra = NULL;
1502        }
1503    #endif
1504    }    }
1505    
1506  return extra;  return extra;
1507  }  }
1508    
1509    
1510    /*************************************************
1511    *          Free the study data                   *
1512    *************************************************/
1513    
1514    /* This function frees the memory that was obtained by pcre_study().
1515    
1516    Argument:   a pointer to the pcre[16]_extra block
1517    Returns:    nothing
1518    */
1519    
1520    #ifdef COMPILE_PCRE8
1521    PCRE_EXP_DEFN void
1522    pcre_free_study(pcre_extra *extra)
1523    #else
1524    PCRE_EXP_DEFN void
1525    pcre16_free_study(pcre16_extra *extra)
1526    #endif
1527    {
1528    if (extra == NULL)
1529      return;
1530    #ifdef SUPPORT_JIT
1531    if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 &&
1532         extra->executable_jit != NULL)
1533      PRIV(jit_free)(extra->executable_jit);
1534    #endif
1535    PUBL(free)(extra);
1536    }
1537    
1538  /* End of pcre_study.c */  /* End of pcre_study.c */
Legend:
Removed from v.538  
changed lines
  Added in v.1032
  ViewVC Help
Powered by ViewVC 1.1.5