/[pcre]/code/trunk/pcretest.c

Diff of /code/trunk/pcretest.c

Parent Directory | Revision Log | View Patch Patch

-revision 75 by nigel,
Sat Feb 24 21:40:37 2007 UTC
+revision 645 by ph10,
Sun Jul 31 17:02:18 2011 UTC
 Line 4
  /* This program was hacked up as a tester for PCRE. I really should have
  written it more tidily in the first place. Will I ever learn? It has grown and
- been extended and consequently is now rather untidy in places.
+ been extended and consequently is now rather, er, *very* untidy in places.
  -----------------------------------------------------------------------------
  Redistribution and use in source and binary forms, with or without
 Line 36 
 POSSIBILITY OF SUCH DAMAGE.
  */
+ #ifdef HAVE_CONFIG_H
+ #include "config.h"
+ #endif
  #include <ctype.h>
  #include <stdio.h>
  #include <string.h>
-Line 44 
 POSSIBILITY OF SUCH DAMAGE.
+Line 48 
 POSSIBILITY OF SUCH DAMAGE.
  #include <locale.h>
  #include <errno.h>
- /* We need the internal info for displaying the results of pcre_study(). Also
+ #ifdef SUPPORT_LIBREADLINE
- for getting the opcodes for showing compiled code. */
+ #ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+ #endif
+ #include <readline/readline.h>
+ #include <readline/history.h>
+ #endif
+ /* A number of things vary for Windows builds. Originally, pcretest opened its
+ input and output without "b"; then I was told that "b" was needed in some
+ environments, so it was added for release 5.0 to both the input and output. (It
+ makes no difference on Unix-like systems.) Later I was told that it is wrong
+ for the input on Windows. I've now abstracted the modes into two macros that
+ are set here, to make it easier to fiddle with them, and removed "b" from the
+ input mode under Windows. */
- #define PCRE_SPY        /* For Win32 build, import data, not export */
+ #if defined(_WIN32) || defined(WIN32)
- #include "internal.h"
+ #include <io.h>                /* For _setmode() */
+ #include <fcntl.h>             /* For _O_BINARY */
+ #define INPUT_MODE   "r"
+ #define OUTPUT_MODE  "wb"
+ #ifndef isatty
+ #define isatty _isatty         /* This is what Windows calls them, I'm told, */
+ #endif                         /* though in some environments they seem to   */
+                                /* be already defined, hence the #ifndefs.    */
+ #ifndef fileno
+ #define fileno _fileno
+ #endif
+ /* A user sent this fix for Borland Builder 5 under Windows. */
+ #ifdef __BORLANDC__
+ #define _setmode(handle, mode) setmode(handle, mode)
+ #endif
+ /* Not Windows */
+ #else
+ #include <sys/time.h>          /* These two includes are needed */
+ #include <sys/resource.h>      /* for setrlimit(). */
+ #define INPUT_MODE   "rb"
+ #define OUTPUT_MODE  "wb"
+ #endif
+ /* We have to include pcre_internal.h because we need the internal info for
+ displaying the results of pcre_study() and we also need to know about the
+ internal macros, structures, and other internal data values; pcretest has
+ "inside information" compared to a program that strictly follows the PCRE API.
+ Although pcre_internal.h does itself include pcre.h, we explicitly include it
+ here before pcre_internal.h so that the PCRE_EXP_xxx macros get set
+ appropriately for an application, not for building PCRE. */
+ #include "pcre.h"
+ #include "pcre_internal.h"
+ /* We need access to some of the data tables that PCRE uses. So as not to have
+ to keep two copies, we include the source file here, changing the names of the
+ external symbols to prevent clashes. */
+ #define _pcre_ucp_gentype      ucp_gentype
+ #define _pcre_utf8_table1      utf8_table1
+ #define _pcre_utf8_table1_size utf8_table1_size
+ #define _pcre_utf8_table2      utf8_table2
+ #define _pcre_utf8_table3      utf8_table3
+ #define _pcre_utf8_table4      utf8_table4
+ #define _pcre_utt              utt
+ #define _pcre_utt_size         utt_size
+ #define _pcre_utt_names        utt_names
+ #define _pcre_OP_lengths       OP_lengths
+ #include "pcre_tables.c"
+ /* We also need the pcre_printint() function for printing out compiled
+ patterns. This function is in a separate file so that it can be included in
+ pcre_compile.c when that module is compiled with debugging enabled. It needs to
+ know which case is being compiled. */
+ #define COMPILING_PCRETEST
+ #include "pcre_printint.src"
+ /* The definition of the macro PRINTABLE, which determines whether to print an
+ output character as-is or as a hex value when showing compiled patterns, is
+ contained in the printint.src file. We uses it here also, in cases when the
+ locale has not been explicitly changed, so as to get consistent output from
+ systems that differ in their output from isprint() even in the "C" locale. */
+ #define PRINTHEX(c) (locale_set? isprint(c) : PRINTABLE(c))
  /* It is possible to compile this test program without including support for
  testing the POSIX interface, though this is not available via the standard
-Line 58 
 Makefile. */
+Line 148 
 Makefile. */
  #include "pcreposix.h"
  #endif
+ /* It is also possible, for the benefit of the version currently imported into
+ Exim, to build pcretest without support for UTF8 (define NOUTF8), without the
+ interface to the DFA matcher (NODFA), and without the doublecheck of the old
+ "info" function (define NOINFOCHECK). In fact, we automatically cut out the
+ UTF8 support if PCRE is built without it. */
+ #ifndef SUPPORT_UTF8
+ #ifndef NOUTF8
+ #define NOUTF8
+ #endif
+ #endif
+ /* Other parameters */
  #ifndef CLOCKS_PER_SEC
  #ifdef CLK_TCK
  #define CLOCKS_PER_SEC CLK_TCK
-Line 66 
 Makefile. */
+Line 171 
 Makefile. */
  #endif
  #endif
- #define LOOPREPEAT 500000
+ /* This is the default loop count for timing. */
- #define BUFFER_SIZE 30000
+ #define LOOPREPEAT 500000
- #define PBUFFER_SIZE BUFFER_SIZE
- #define DBUFFER_SIZE BUFFER_SIZE
+ /* Static variables */
  static FILE *outfile;
  static int log_store = 0;
-Line 79 
 static int callout_count;
+Line 183 
 static int callout_count;
  static int callout_extra;
  static int callout_fail_count;
  static int callout_fail_id;
+ static int debug_lengths;
  static int first_callout;
+ static int locale_set = 0;
  static int show_malloc;
  static int use_utf8;
  static size_t gotten_store;
+ static const unsigned char *last_callout_mark = NULL;
+ /* The buffers grow automatically if very long input lines are encountered. */
+ static int buffer_size = 50000;
+ static uschar *buffer = NULL;
+ static uschar *dbuffer = NULL;
  static uschar *pbuffer = NULL;
+ /* Textual explanations for runtime error codes */
- static const int utf8_table1[] = {
+ static const char *errtexts[] = {
-x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff};
+   NULL,  /* 0 is no error */
+   NULL,  /* NOMATCH is handled specially */
+   "NULL argument passed",
+   "bad option value",
+   "magic number missing",
+   "unknown opcode - pattern overwritten?",
+   "no more memory",
+   NULL,  /* never returned by pcre_exec() or pcre_dfa_exec() */
+   "match limit exceeded",
+   "callout error code",
+   NULL,  /* BADUTF8 is handled specially */
+   "bad UTF-8 offset",
+   NULL,  /* PARTIAL is handled specially */
+   "not used - internal error",
+   "internal error - pattern overwritten?",
+   "bad count value",
+   "item unsupported for DFA matching",
+   "backreference condition or recursion test not supported for DFA matching",
+   "match limit not supported for DFA matching",
+   "workspace size exceeded in DFA matching",
+   "too much recursion for DFA matching",
+   "recursion limit exceeded",
+   "not used - internal error",
+   "invalid combination of newline options",
+   "bad offset value",
+   NULL,  /* SHORTUTF8 is handled specially */
+   "nested recursion at the same subject position"
+ };
- static const int utf8_table2[] = {
+ /*************************************************
-, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc};
+ *         Alternate character tables             *
+ *************************************************/
- static const int utf8_table3[] = {
+ /* By default, the "tables" pointer when calling PCRE is set to NULL, thereby
-xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
+ using the default tables of the library. However, the T option can be used to
+ select alternate sets of tables, for different kinds of testing. Note also that
+ the L (locale) option also adjusts the tables. */
+ /* This is the set of tables distributed as default with PCRE. It recognizes
+ only ASCII characters. */
+ static const unsigned char tables0[] = {
+ /* This table is a lower casing table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table is a case flipping table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 65, 66, 67, 68, 69, 70, 71,
+, 73, 74, 75, 76, 77, 78, 79,
+, 81, 82, 83, 84, 85, 86, 87,
+, 89, 90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table contains bit maps for various character classes. Each map is 32
+ bytes long and the bits run from the least significant end of each byte. The
+ classes that have their own maps are: space, xdigit, digit, upper, lower, word,
+ graph, print, punct, and cntrl. Other classes are built from combinations. */
+x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc,
+x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+ /* This table identifies various classes of character by individual bits:
+x01   white space character
+x02   letter
+x04   decimal digit
+x08   hexadecimal digit
+x10   alphanumeric or '_'
+x80   regular expression metacharacter or binary zero
+ */
+x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*   0-  7 */
+x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /*   8- 15 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  16- 23 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  24- 31 */
+x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /*    - '  */
+x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /*  ( - /  */
+x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /*  0 - 7  */
+x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /*  8 - ?  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  @ - G  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  H - O  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  P - W  */
+x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /*  X - _  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  ` - g  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  h - o  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  p - w  */
+x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /*  x -127 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */
+ /* This is a set of tables that came orginally from a Windows user. It seems to
+ be at least an approximation of ISO 8859. In particular, there are characters
+ greater than 128 that are marked as spaces, letters, etc. */
+ static const unsigned char tables1[] = {
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,65,66,67,68,69,70,71,
+,73,74,75,76,77,78,79,
+,81,82,83,84,85,86,87,
+,89,90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,247,
+,217,218,219,220,221,222,255,
+,62,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,126,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,12,2,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,255,7,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,127,127,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,254,255,255,7,
+,0,0,0,0,4,32,4,
+,0,0,128,255,255,127,255,
+,0,0,0,0,0,255,3,
+,255,255,135,254,255,255,7,
+,0,0,0,0,4,44,6,
+,255,127,255,255,255,127,255,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,255,
+,2,0,0,255,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,255,255,255,255,
+,255,255,255,255,255,255,255,
+,0,0,0,254,255,0,252,
+,0,0,248,1,0,0,120,
+,0,0,0,254,255,255,255,
+,0,128,0,0,0,128,0,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,128,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,1,1,0,1,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,128,0,0,0,
+,128,128,128,0,0,128,0,
+,28,28,28,28,28,28,28,
+,28,0,0,0,0,0,128,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,128,16,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,0,0,
+,0,0,0,0,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,18,0,0,0,0,0,
+,0,20,20,0,18,0,0,
+,20,18,0,0,0,0,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18
+ };
+ #ifndef HAVE_STRERROR
  /*************************************************
- *         Print compiled regex                   *
+ *     Provide strerror() for non-ANSI libraries  *
  *************************************************/
- /* The code for doing this is held in a separate file that is also included in
+ /* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror()
- pcre.c when it is compiled with the debug switch. It defines a function called
+ in their libraries, but can provide the same facility by this simple
- print_internals(), which uses a table of opcode lengths defined by the macro
+ alternative function. */
- OP_LENGTHS, whose name must be OP_lengths. It also uses a table that translates
- Unicode property names to numbers; this is kept in a separate file. */
+ extern int   sys_nerr;
+ extern char *sys_errlist[];
- static uschar OP_lengths[] = { OP_LENGTHS };
+ char *
- #include "ucp.h"
+ strerror(int n)
- #include "ucptypetable.c"
+ {
- #include "printint.c"
+ if (n < 0 || n >= sys_nerr) return "unknown error number";
+ return sys_errlist[n];
+ }
+ #endif /* HAVE_STRERROR */
+ /*************************************************
+ *        Read or extend an input line            *
+ *************************************************/
+ /* Input lines are read into buffer, but both patterns and data lines can be
+ continued over multiple input lines. In addition, if the buffer fills up, we
+ want to automatically expand it so as to be able to handle extremely large
+ lines that are needed for certain stress tests. When the input buffer is
+ expanded, the other two buffers must also be expanded likewise, and the
+ contents of pbuffer, which are a copy of the input for callouts, must be
+ preserved (for when expansion happens for a data line). This is not the most
+ optimal way of handling this, but hey, this is just a test program!
+ Arguments:
+   f            the file to read
+   start        where in buffer to start (this *must* be within buffer)
+   prompt       for stdin or readline()
+ Returns:       pointer to the start of new data
+                could be a copy of start, or could be moved
+                NULL if no data read and EOF reached
+ */
+ static uschar *
+ extend_inputline(FILE *f, uschar *start, const char *prompt)
+ {
+ uschar *here = start;
+ for (;;)
+   {
+   int rlen = (int)(buffer_size - (here - buffer));
+   if (rlen > 1000)
+     {
+     int dlen;
+     /* If libreadline support is required, use readline() to read a line if the
+     input is a terminal. Note that readline() removes the trailing newline, so
+     we must put it back again, to be compatible with fgets(). */
+ #ifdef SUPPORT_LIBREADLINE
+     if (isatty(fileno(f)))
+       {
+       size_t len;
+       char *s = readline(prompt);
+       if (s == NULL) return (here == start)? NULL : start;
+       len = strlen(s);
+       if (len > 0) add_history(s);
+       if (len > rlen - 1) len = rlen - 1;
+       memcpy(here, s, len);
+       here[len] = '\n';
+       here[len+1] = 0;
+       free(s);
+       }
+     else
+ #endif
+     /* Read the next line by normal means, prompting if the file is stdin. */
+       {
+       if (f == stdin) printf("%s", prompt);
+       if (fgets((char *)here, rlen,  f) == NULL)
+         return (here == start)? NULL : start;
+       }
+     dlen = (int)strlen((char *)here);
+     if (dlen > 0 && here[dlen - 1] == '\n') return start;
+     here += dlen;
+     }
+   else
+     {
+     int new_buffer_size = 2*buffer_size;
+     uschar *new_buffer = (unsigned char *)malloc(new_buffer_size);
+     uschar *new_dbuffer = (unsigned char *)malloc(new_buffer_size);
+     uschar *new_pbuffer = (unsigned char *)malloc(new_buffer_size);
+     if (new_buffer == NULL || new_dbuffer == NULL || new_pbuffer == NULL)
+       {
+       fprintf(stderr, "pcretest: malloc(%d) failed\n", new_buffer_size);
+       exit(1);
+       }
+     memcpy(new_buffer, buffer, buffer_size);
+     memcpy(new_pbuffer, pbuffer, buffer_size);
+     buffer_size = new_buffer_size;
+     start = new_buffer + (start - buffer);
+     here = new_buffer + (here - buffer);
+     free(buffer);
+     free(dbuffer);
+     free(pbuffer);
+     buffer = new_buffer;
+     dbuffer = new_dbuffer;
+     pbuffer = new_pbuffer;
+     }
+   }
+ return NULL;  /* Control never gets here */
+ }
-Line 122 
 static uschar OP_lengths[] = { OP_LENGTH
+Line 697 
 static uschar OP_lengths[] = { OP_LENGTH
  /* We don't use strtoul() because SunOS4 doesn't have it. Rather than mess
  around with conditional compilation, just do the job by hand. It is only used
- for unpicking the -o argument, so just keep it simple.
+ for unpicking arguments, so just keep it simple.
  Arguments:
    str           string to be converted
-Line 143 
 return(result);
+Line 718 
 return(result);
- /*************************************************
- *       Convert character value to UTF-8         *
- *************************************************/
- /* This function takes an integer value in the range 0 - 0x7fffffff
- and encodes it as a UTF-8 character in 0 to 6 bytes.
- Arguments:
-   cvalue     the character value
-   buffer     pointer to buffer for result - at least 6 bytes long
- Returns:     number of characters placed in the buffer
-              -1 if input character is negative
-if input character is positive but too big (only when
-              int is longer than 32 bits)
- */
- static int
- ord2utf8(int cvalue, unsigned char *buffer)
- {
- register int i, j;
- for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
-   if (cvalue <= utf8_table1[i]) break;
- if (i >= sizeof(utf8_table1)/sizeof(int)) return 0;
- if (cvalue < 0) return -1;
- buffer += i;
- for (j = i; j > 0; j--)
-  {
-  *buffer-- = 0x80 | (cvalue & 0x3f);
-  cvalue >>= 6;
-  }
- *buffer = utf8_table2[i] | cvalue;
- return i + 1;
- }
  /*************************************************
  *            Convert UTF-8 string to value       *
-Line 188 
 return i + 1;
+Line 727 
 return i + 1;
  and returns the value of the character.
  Argument:
-   buffer   a pointer to the byte vector
+   utf8bytes   a pointer to the byte vector
-   vptr     a pointer to an int to receive the value
+   vptr        a pointer to an int to receive the value
- Returns:   >  0 => the number of bytes consumed
+ Returns:      >  0 => the number of bytes consumed
-            -6 to 0 => malformed UTF-8 character at offset = (-return)
+               -6 to 0 => malformed UTF-8 character at offset = (-return)
  */
+ #if !defined NOUTF8
  static int
- utf82ord(unsigned char *buffer, int *vptr)
+ utf82ord(unsigned char *utf8bytes, int *vptr)
  {
- int c = *buffer++;
+ int c = *utf8bytes++;
  int d = c;
  int i, j, s;
-Line 218 
 d = (c & utf8_table3[i]) << s;
+Line 759 
 d = (c & utf8_table3[i]) << s;
  for (j = 0; j < i; j++)
    {
-   c = *buffer++;
+   c = *utf8bytes++;
    if ((c & 0xc0) != 0x80) return -(j+1);
    s -= 6;
    d |= (c & 0x3f) << s;
-Line 226 
 for (j = 0; j < i; j++)
+Line 767 
 for (j = 0; j < i; j++)
  /* Check that encoding was the correct unique one */
- for (j = 0; j < sizeof(utf8_table1)/sizeof(int); j++)
+ for (j = 0; j < utf8_table1_size; j++)
    if (d <= utf8_table1[j]) break;
  if (j != i) return -(i+1);
-Line 236 
 if (j != i) return -(i+1);
+Line 777 
 if (j != i) return -(i+1);
  return i+1;
  }
+ #endif
+ /*************************************************
+ *       Convert character value to UTF-8         *
+ *************************************************/
+ /* This function takes an integer value in the range 0 - 0x7fffffff
+ and encodes it as a UTF-8 character in 0 to 6 bytes.
+ Arguments:
+   cvalue     the character value
+   utf8bytes  pointer to buffer for result - at least 6 bytes long
+ Returns:     number of characters placed in the buffer
+ */
+ #if !defined NOUTF8
+ static int
+ ord2utf8(int cvalue, uschar *utf8bytes)
+ {
+ register int i, j;
+ for (i = 0; i < utf8_table1_size; i++)
+   if (cvalue <= utf8_table1[i]) break;
+ utf8bytes += i;
+ for (j = i; j > 0; j--)
+  {
+  *utf8bytes-- = 0x80 | (cvalue & 0x3f);
+  cvalue >>= 6;
+  }
+ *utf8bytes = utf8_table2[i] | cvalue;
+ return i + 1;
+ }
+ #endif
  /*************************************************
-Line 248 
 chars without printing. */
+Line 827 
 chars without printing. */
  static int pchars(unsigned char *p, int length, FILE *f)
  {
- int c;
+ int c = 0;
  int yield = 0;
  while (length-- > 0)
    {
+ #if !defined NOUTF8
    if (use_utf8)
      {
      int rc = utf82ord(p, &c);
-Line 261 
 while (length-- > 0)
+Line 841 
 while (length-- > 0)
        {
        length -= rc - 1;
        p += rc;
-       if (c < 256 && isprint(c))
+       if (PRINTHEX(c))
          {
          if (f != NULL) fprintf(f, "%c", c);
          yield++;
          }
        else
          {
-         int n;
+         int n = 4;
-         if (f != NULL) fprintf(f, "\\x{%02x}%n", c, &n);
+         if (f != NULL) fprintf(f, "\\x{%02x}", c);
-         yield += n;
+         yield += (n <= 0x000000ff)? 2 :
+                  (n <= 0x00000fff)? 3 :
+                  (n <= 0x0000ffff)? 4 :
+                  (n <= 0x000fffff)? 5 : 6;
          }
        continue;
        }
      }
+ #endif
     /* Not UTF-8, or malformed UTF-8  */
-   if (isprint(c = *(p++)))
+   c = *p++;
+   if (PRINTHEX(c))
      {
      if (f != NULL) fprintf(f, "%c", c);
      yield++;
-Line 376 
 fprintf(outfile, "%.*s", (cb->next_item_
+Line 961 
 fprintf(outfile, "%.*s", (cb->next_item_
  fprintf(outfile, "\n");
  first_callout = 0;
+ if (cb->mark != last_callout_mark)
+   {
+   fprintf(outfile, "Latest Mark: %s\n",
+     (cb->mark == NULL)? "<unset>" : (char *)(cb->mark));
+   last_callout_mark = cb->mark;
+   }
  if (cb->callout_data != NULL)
    {
    int callout_data = *((int *)(cb->callout_data));
-Line 403 
 static void *new_malloc(size_t size)
+Line 995 
 static void *new_malloc(size_t size)
  void *block = malloc(size);
  gotten_store = size;
  if (show_malloc)
-   fprintf(outfile, "malloc       %3d %p\n", size, block);
+   fprintf(outfile, "malloc       %3d %p\n", (int)size, block);
  return block;
  }
-Line 421 
 static void *stack_malloc(size_t size)
+Line 1013 
 static void *stack_malloc(size_t size)
  {
  void *block = malloc(size);
  if (show_malloc)
-   fprintf(outfile, "stack_malloc %3d %p\n", size, block);
+   fprintf(outfile, "stack_malloc %3d %p\n", (int)size, block);
  return block;
  }
-Line 452 
 if ((rc = pcre_fullinfo(re, study, optio
+Line 1044 
 if ((rc = pcre_fullinfo(re, study, optio
  *         Byte flipping function                 *
  *************************************************/
- static long int
+ static unsigned long int
- byteflip(long int value, int n)
+ byteflip(unsigned long int value, int n)
  {
  if (n == 2) return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8);
  return ((value & 0x000000ff) << 24) |
-Line 466 
 return ((value & 0x000000ff) << 24) |
+Line 1058 
 return ((value & 0x000000ff) << 24) |
  /*************************************************
+ *        Check match or recursion limit          *
+ *************************************************/
+ static int
+ check_match_limit(pcre *re, pcre_extra *extra, uschar *bptr, int len,
+   int start_offset, int options, int *use_offsets, int use_size_offsets,
+   int flag, unsigned long int *limit, int errnumber, const char *msg)
+ {
+ int count;
+ int min = 0;
+ int mid = 64;
+ int max = -1;
+ extra->flags |= flag;
+ for (;;)
+   {
+   *limit = mid;
+   count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options,
+     use_offsets, use_size_offsets);
+   if (count == errnumber)
+     {
+     /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */
+     min = mid;
+     mid = (mid == max - 1)? max : (max > 0)? (min + max)/2 : mid*2;
+     }
+   else if (count >= 0 || count == PCRE_ERROR_NOMATCH ||
+                          count == PCRE_ERROR_PARTIAL)
+     {
+     if (mid == min + 1)
+       {
+       fprintf(outfile, "Minimum %s limit = %d\n", msg, mid);
+       break;
+       }
+     /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */
+     max = mid;
+     mid = (min + mid)/2;
+     }
+   else break;    /* Some other error */
+   }
+ extra->flags &= ~flag;
+ return count;
+ }
+ /*************************************************
+ *         Case-independent strncmp() function    *
+ *************************************************/
+ /*
+ Arguments:
+   s         first string
+   t         second string
+   n         number of characters to compare
+ Returns:    < 0, = 0, or > 0, according to the comparison
+ */
+ static int
+ strncmpic(uschar *s, uschar *t, int n)
+ {
+ while (n--)
+   {
+   int c = tolower(*s++) - tolower(*t++);
+   if (c) return c;
+   }
+ return 0;
+ }
+ /*************************************************
+ *         Check newline indicator                *
+ *************************************************/
+ /* This is used both at compile and run-time to check for <xxx> escapes. Print
+ a message and return 0 if there is no match.
+ Arguments:
+   p           points after the leading '<'
+   f           file for error message
+ Returns:      appropriate PCRE_NEWLINE_xxx flags, or 0
+ */
+ static int
+ check_newline(uschar *p, FILE *f)
+ {
+ if (strncmpic(p, (uschar *)"cr>", 3) == 0) return PCRE_NEWLINE_CR;
+ if (strncmpic(p, (uschar *)"lf>", 3) == 0) return PCRE_NEWLINE_LF;
+ if (strncmpic(p, (uschar *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF;
+ if (strncmpic(p, (uschar *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF;
+ if (strncmpic(p, (uschar *)"any>", 4) == 0) return PCRE_NEWLINE_ANY;
+ if (strncmpic(p, (uschar *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF;
+ if (strncmpic(p, (uschar *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE;
+ fprintf(f, "Unknown newline type at: <%s\n", p);
+ return 0;
+ }
+ /*************************************************
+ *             Usage function                     *
+ *************************************************/
+ static void
+ usage(void)
+ {
+ printf("Usage:     pcretest [options] [<input file> [<output file>]]\n\n");
+ printf("Input and output default to stdin and stdout.\n");
+ #ifdef SUPPORT_LIBREADLINE
+ printf("If input is a terminal, readline() is used to read from it.\n");
+ #else
+ printf("This version of pcretest is not linked with readline().\n");
+ #endif
+ printf("\nOptions:\n");
+ printf("  -b       show compiled code (bytecode)\n");
+ printf("  -C       show PCRE compile-time options and exit\n");
+ printf("  -d       debug: show compiled code and information (-b and -i)\n");
+ #if !defined NODFA
+ printf("  -dfa     force DFA matching for all subjects\n");
+ #endif
+ printf("  -help    show usage information\n");
+ printf("  -i       show information about compiled patterns\n"
+        "  -M       find MATCH_LIMIT minimum for each subject\n"
+        "  -m       output memory used information\n"
+        "  -o <n>   set size of offsets vector to <n>\n");
+ #if !defined NOPOSIX
+ printf("  -p       use POSIX interface\n");
+ #endif
+ printf("  -q       quiet: do not output PCRE version number at start\n");
+ printf("  -S <n>   set stack size to <n> megabytes\n");
+ printf("  -s       force each pattern to be studied\n"
+        "  -t       time compilation and execution\n");
+ printf("  -t <n>   time compilation and execution, repeating <n> times\n");
+ printf("  -tm      time execution (matching) only\n");
+ printf("  -tm <n>  time execution (matching) only, repeating <n> times\n");
+ }
+ /*************************************************
  *                Main Program                    *
  *************************************************/
-Line 478 
 int main(int argc, char **argv)
+Line 1217 
 int main(int argc, char **argv)
  FILE *infile = stdin;
  int options = 0;
  int study_options = 0;
+ int default_find_match_limit = FALSE;
  int op = 1;
  int timeit = 0;
+ int timeitm = 0;
  int showinfo = 0;
  int showstore = 0;
+ int force_study = 0;
+ int quiet = 0;
  int size_offsets = 45;
  int size_offsets_max;
- int *offsets;
+ int *offsets = NULL;
  #if !defined NOPOSIX
  int posix = 0;
  #endif
  int debug = 0;
  int done = 0;
+ int all_use_dfa = 0;
+ int yield = 0;
+ int stack_size;
- unsigned char *buffer;
+ /* These vectors store, end-to-end, a list of captured substring names. Assume
- unsigned char *dbuffer;
+ that 1024 is plenty long enough for the few names we'll be testing. */
+ uschar copynames[1024];
+ uschar getnames[1024];
+ uschar *copynamesptr;
+ uschar *getnamesptr;
  /* Get buffers from malloc() so that Electric Fence will check their misuse
- when I am debugging. */
+ when I am debugging. They grow automatically when very long lines are read. */
- buffer = (unsigned char *)malloc(BUFFER_SIZE);
+ buffer = (unsigned char *)malloc(buffer_size);
- dbuffer = (unsigned char *)malloc(DBUFFER_SIZE);
+ dbuffer = (unsigned char *)malloc(buffer_size);
- pbuffer = (unsigned char *)malloc(PBUFFER_SIZE);
+ pbuffer = (unsigned char *)malloc(buffer_size);
- /* The outfile variable is static so that new_malloc can use it. The _setmode()
- stuff is some magic that I don't understand, but which apparently does good
- things in Windows. It's related to line terminations.  */
- #if defined(_WIN32) || defined(WIN32)
+ /* The outfile variable is static so that new_malloc can use it. */
- _setmode( _fileno( stdout ), 0x8000 );
- #endif  /* defined(_WIN32) || defined(WIN32) */
  outfile = stdout;
+ /* The following  _setmode() stuff is some Windows magic that tells its runtime
+ library to translate CRLF into a single LF character. At least, that's what
+ I've been told: never having used Windows I take this all on trust. Originally
+ it set 0x8000, but then I was advised that _O_BINARY was better. */
+ #if defined(_WIN32) || defined(WIN32)
+ _setmode( _fileno( stdout ), _O_BINARY );
+ #endif
  /* Scan options */
  while (argc > 1 && argv[op][0] == '-')
    {
    unsigned char *endptr;
-   if (strcmp(argv[op], "-s") == 0 || strcmp(argv[op], "-m") == 0)
+   if (strcmp(argv[op], "-m") == 0) showstore = 1;
-     showstore = 1;
+   else if (strcmp(argv[op], "-s") == 0) force_study = 1;
-   else if (strcmp(argv[op], "-t") == 0) timeit = 1;
+   else if (strcmp(argv[op], "-q") == 0) quiet = 1;
+   else if (strcmp(argv[op], "-b") == 0) debug = 1;
    else if (strcmp(argv[op], "-i") == 0) showinfo = 1;
    else if (strcmp(argv[op], "-d") == 0) showinfo = debug = 1;
+   else if (strcmp(argv[op], "-M") == 0) default_find_match_limit = TRUE;
+ #if !defined NODFA
+   else if (strcmp(argv[op], "-dfa") == 0) all_use_dfa = 1;
+ #endif
    else if (strcmp(argv[op], "-o") == 0 && argc > 2 &&
        ((size_offsets = get_value((unsigned char *)argv[op+1], &endptr)),
          *endptr == 0))
-Line 529 
 while (argc > 1 && argv[op][0] == '-')
+Line 1289 
 while (argc > 1 && argv[op][0] == '-')
      op++;
      argc--;
      }
+   else if (strcmp(argv[op], "-t") == 0 || strcmp(argv[op], "-tm") == 0)
+     {
+     int both = argv[op][2] == 0;
+     int temp;
+     if (argc > 2 && (temp = get_value((unsigned char *)argv[op+1], &endptr),
+                      *endptr == 0))
+       {
+       timeitm = temp;
+       op++;
+       argc--;
+       }
+     else timeitm = LOOPREPEAT;
+     if (both) timeit = timeitm;
+     }
+   else if (strcmp(argv[op], "-S") == 0 && argc > 2 &&
+       ((stack_size = get_value((unsigned char *)argv[op+1], &endptr)),
+         *endptr == 0))
+     {
+ #if defined(_WIN32) || defined(WIN32) || defined(__minix)
+     printf("PCRE: -S not supported on this OS\n");
+     exit(1);
+ #else
+     int rc;
+     struct rlimit rlim;
+     getrlimit(RLIMIT_STACK, &rlim);
+     rlim.rlim_cur = stack_size * 1024 * 1024;
+     rc = setrlimit(RLIMIT_STACK, &rlim);
+     if (rc != 0)
+       {
+     printf("PCRE: setrlimit() failed with error %d\n", rc);
+     exit(1);
+       }
+     op++;
+     argc--;
+ #endif
+     }
  #if !defined NOPOSIX
    else if (strcmp(argv[op], "-p") == 0) posix = 1;
  #endif
    else if (strcmp(argv[op], "-C") == 0)
      {
      int rc;
+     unsigned long int lrc;
      printf("PCRE version %s\n", pcre_version());
      printf("Compiled with\n");
      (void)pcre_config(PCRE_CONFIG_UTF8, &rc);
-Line 542 
 while (argc > 1 && argv[op][0] == '-')
+Line 1339 
 while (argc > 1 && argv[op][0] == '-')
      (void)pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &rc);
      printf("  %sUnicode properties support\n", rc? "" : "No ");
      (void)pcre_config(PCRE_CONFIG_NEWLINE, &rc);
-     printf("  Newline character is %s\n", (rc == '\r')? "CR" : "LF");
+     /* Note that these values are always the ASCII values, even
+     in EBCDIC environments. CR is 13 and NL is 10. */
+     printf("  Newline sequence is %s\n", (rc == 13)? "CR" :
+       (rc == 10)? "LF" : (rc == (13<<8 | 10))? "CRLF" :
+       (rc == -2)? "ANYCRLF" :
+       (rc == -1)? "ANY" : "???");
+     (void)pcre_config(PCRE_CONFIG_BSR, &rc);
+     printf("  \\R matches %s\n", rc? "CR, LF, or CRLF only" :
+                                      "all Unicode newlines");
      (void)pcre_config(PCRE_CONFIG_LINK_SIZE, &rc);
      printf("  Internal link size = %d\n", rc);
      (void)pcre_config(PCRE_CONFIG_POSIX_MALLOC_THRESHOLD, &rc);
      printf("  POSIX malloc threshold = %d\n", rc);
-     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT, &rc);
+     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT, &lrc);
-     printf("  Default match limit = %d\n", rc);
+     printf("  Default match limit = %ld\n", lrc);
+     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT_RECURSION, &lrc);
+     printf("  Default recursion depth limit = %ld\n", lrc);
      (void)pcre_config(PCRE_CONFIG_STACKRECURSE, &rc);
      printf("  Match recursion uses %s\n", rc? "stack" : "heap");
-     exit(0);
+     goto EXIT;
+     }
+   else if (strcmp(argv[op], "-help") == 0 ||
+            strcmp(argv[op], "--help") == 0)
+     {
+     usage();
+     goto EXIT;
      }
    else
      {
      printf("** Unknown or malformed option %s\n", argv[op]);
-     printf("Usage:   pcretest [-d] [-i] [-o <n>] [-p] [-s] [-t] [<input> [<output>]]\n");
+     usage();
-     printf("  -C     show PCRE compile-time options and exit\n");
+     yield = 1;
-     printf("  -d     debug: show compiled code; implies -i\n"
+     goto EXIT;
-            "  -i     show information about compiled pattern\n"
-            "  -m     output memory used information\n"
-            "  -o <n> set size of offsets vector to <n>\n");
- #if !defined NOPOSIX
-     printf("  -p     use POSIX interface\n");
- #endif
-     printf("  -s     output store (memory) used information\n"
-            "  -t     time compilation and execution\n");
-     return 1;
      }
    op++;
    argc--;
-Line 580 
 offsets = (int *)malloc(size_offsets_max
+Line 1384 
 offsets = (int *)malloc(size_offsets_max
  if (offsets == NULL)
    {
    printf("** Failed to get %d bytes of memory for offsets vector\n",
-     size_offsets_max * sizeof(int));
+     (int)(size_offsets_max * sizeof(int)));
-   return 1;
+   yield = 1;
+   goto EXIT;
    }
  /* Sort out the input and output files */
  if (argc > 1)
    {
-   infile = fopen(argv[op], "rb");
+   infile = fopen(argv[op], INPUT_MODE);
    if (infile == NULL)
      {
      printf("** Failed to open %s\n", argv[op]);
-     return 1;
+     yield = 1;
+     goto EXIT;
      }
    }
  if (argc > 2)
    {
-   outfile = fopen(argv[op+1], "wb");
+   outfile = fopen(argv[op+1], OUTPUT_MODE);
    if (outfile == NULL)
      {
      printf("** Failed to open %s\n", argv[op+1]);
-     return 1;
+     yield = 1;
+     goto EXIT;
      }
    }
-Line 613 
 pcre_free = new_free;
+Line 1420 
 pcre_free = new_free;
  pcre_stack_malloc = stack_malloc;
  pcre_stack_free = stack_free;
- /* Heading line, then prompt for first regex if stdin */
+ /* Heading line unless quiet, then prompt for first regex if stdin */
- fprintf(outfile, "PCRE version %s\n\n", pcre_version());
+ if (!quiet) fprintf(outfile, "PCRE version %s\n\n", pcre_version());
  /* Main loop */
-Line 630 
 while (!done)
+Line 1437 
 while (!done)
  #endif
    const char *error;
+   unsigned char *markptr;
    unsigned char *p, *pp, *ppp;
    unsigned char *to_file = NULL;
    const unsigned char *tables = NULL;
    unsigned long int true_size, true_study_size = 0;
    size_t size, regex_gotten_store;
+   int do_allcaps = 0;
+   int do_mark = 0;
    int do_study = 0;
+   int no_force_study = 0;
    int do_debug = debug;
    int do_G = 0;
    int do_g = 0;
    int do_showinfo = showinfo;
    int do_showrest = 0;
+   int do_showcaprest = 0;
    int do_flip = 0;
-   int erroroffset, len, delimiter;
+   int erroroffset, len, delimiter, poffset;
    use_utf8 = 0;
+   debug_lengths = 1;
-   if (infile == stdin) printf("  re> ");
+   if (extend_inputline(infile, buffer, "  re> ") == NULL) break;
-   if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL) break;
    if (infile != stdin) fprintf(outfile, "%s", (char *)buffer);
    fflush(outfile);
-Line 659 
 while (!done)
+Line 1471 
 while (!done)
    if (*p == '<' && strchr((char *)(p+1), '<') == NULL)
      {
-     unsigned long int magic;
+     unsigned long int magic, get_options;
      uschar sbuf[8];
      FILE *f;
-Line 702 
 while (!done)
+Line 1514 
 while (!done)
          }
        }
-     fprintf(outfile, "Compiled regex%s loaded from %s\n",
+     fprintf(outfile, "Compiled pattern%s loaded from %s\n",
        do_flip? " (byte-inverted)" : "", p);
      /* Need to know if UTF-8 for printing data strings */
-     new_info(re, NULL, PCRE_INFO_OPTIONS, &options);
+     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
-     use_utf8 = (options & PCRE_UTF8) != 0;
+     use_utf8 = (get_options & PCRE_UTF8) != 0;
-     /* Now see if there is any following study data */
+     /* Now see if there is any following study data. */
      if (true_study_size != 0)
        {
-Line 747 
 while (!done)
+Line 1559 
 while (!done)
    if (isalnum(delimiter) || delimiter == '\\')
      {
-     fprintf(outfile, "** Delimiter must not be alphameric or \\\n");
+     fprintf(outfile, "** Delimiter must not be alphanumeric or \\\n");
      goto SKIP_DATA;
      }
    pp = p;
+   poffset = (int)(p - buffer);
    for(;;)
      {
-Line 762 
 while (!done)
+Line 1575 
 while (!done)
        pp++;
        }
      if (*pp != 0) break;
+     if ((pp = extend_inputline(infile, pp, "    > ")) == NULL)
-     len = BUFFER_SIZE - (pp - buffer);
-     if (len < 256)
-       {
-       fprintf(outfile, "** Expression too long - missing delimiter?\n");
-       goto SKIP_DATA;
-       }
-     if (infile == stdin) printf("    > ");
-     if (fgets((char *)pp, len, infile) == NULL)
        {
        fprintf(outfile, "** Unexpected EOF\n");
        done = 1;
-Line 780 
 while (!done)
+Line 1584 
 while (!done)
      if (infile != stdin) fprintf(outfile, "%s", (char *)pp);
      }
+   /* The buffer may have moved while being extended; reset the start of data
+   pointer to the correct relative point in the buffer. */
+   p = buffer + poffset;
    /* If the first character after the delimiter is backslash, make
    the pattern end with backslash. This is purely to provide a way
    of testing for the error message when a pattern ends with backslash. */
-Line 802 
 while (!done)
+Line 1611 
 while (!done)
      {
      switch (*pp++)
        {
+       case 'f': options |= PCRE_FIRSTLINE; break;
        case 'g': do_g = 1; break;
        case 'i': options |= PCRE_CASELESS; break;
        case 'm': options |= PCRE_MULTILINE; break;
        case 's': options |= PCRE_DOTALL; break;
        case 'x': options |= PCRE_EXTENDED; break;
-       case '+': do_showrest = 1; break;
+       case '+':
+       if (do_showrest) do_showcaprest = 1; else do_showrest = 1;
+       break;
+       case '=': do_allcaps = 1; break;
        case 'A': options |= PCRE_ANCHORED; break;
+       case 'B': do_debug = 1; break;
        case 'C': options |= PCRE_AUTO_CALLOUT; break;
        case 'D': do_debug = do_showinfo = 1; break;
        case 'E': options |= PCRE_DOLLAR_ENDONLY; break;
        case 'F': do_flip = 1; break;
        case 'G': do_G = 1; break;
        case 'I': do_showinfo = 1; break;
+       case 'J': options |= PCRE_DUPNAMES; break;
+       case 'K': do_mark = 1; break;
        case 'M': log_store = 1; break;
        case 'N': options |= PCRE_NO_AUTO_CAPTURE; break;
-Line 823 
 while (!done)
+Line 1640 
 while (!done)
        case 'P': do_posix = 1; break;
  #endif
-       case 'S': do_study = 1; break;
+       case 'S':
+       if (do_study == 0) do_study = 1; else
+         {
+         do_study = 0;
+         no_force_study = 1;
+         }
+       break;
        case 'U': options |= PCRE_UNGREEDY; break;
+       case 'W': options |= PCRE_UCP; break;
        case 'X': options |= PCRE_EXTRA; break;
+       case 'Y': options |= PCRE_NO_START_OPTIMISE; break;
+       case 'Z': debug_lengths = 0; break;
        case '8': options |= PCRE_UTF8; use_utf8 = 1; break;
        case '?': options |= PCRE_NO_UTF8_CHECK; break;
+       case 'T':
+       switch (*pp++)
+         {
+         case '0': tables = tables0; break;
+         case '1': tables = tables1; break;
+         case '\r':
+         case '\n':
+         case ' ':
+         case 0:
+         fprintf(outfile, "** Missing table number after /T\n");
+         goto SKIP_DATA;
+         default:
+         fprintf(outfile, "** Bad table number \"%c\" after /T\n", pp[-1]);
+         goto SKIP_DATA;
+         }
+       break;
        case 'L':
        ppp = pp;
-       while (*ppp != '\n' && *ppp != ' ') ppp++;
+       /* The '\r' test here is so that it works on Windows. */
+       /* The '0' test is just in case this is an unterminated line. */
+       while (*ppp != 0 && *ppp != '\n' && *ppp != '\r' && *ppp != ' ') ppp++;
        *ppp = 0;
        if (setlocale(LC_CTYPE, (const char *)pp) == NULL)
          {
          fprintf(outfile, "** Failed to set locale \"%s\"\n", pp);
          goto SKIP_DATA;
          }
+       locale_set = 1;
        tables = pcre_maketables();
        pp = ppp;
        break;
-Line 849 
 while (!done)
+Line 1698 
 while (!done)
        *pp = 0;
        break;
-       case '\n': case ' ': break;
+       case '<':
+         {
+         if (strncmpic(pp, (uschar *)"JS>", 3) == 0)
+           {
+           options |= PCRE_JAVASCRIPT_COMPAT;
+           pp += 3;
+           }
+         else
+           {
+           int x = check_newline(pp, outfile);
+           if (x == 0) goto SKIP_DATA;
+           options |= x;
+           while (*pp++ != '>');
+           }
+         }
+       break;
+       case '\r':                      /* So that it works in Windows */
+       case '\n':
+       case ' ':
+       break;
        default:
        fprintf(outfile, "** Unknown option '%c'\n", pp[-1]);
-Line 869 
 while (!done)
+Line 1738 
 while (!done)
      if ((options & PCRE_CASELESS) != 0) cflags |= REG_ICASE;
      if ((options & PCRE_MULTILINE) != 0) cflags |= REG_NEWLINE;
+     if ((options & PCRE_DOTALL) != 0) cflags |= REG_DOTALL;
+     if ((options & PCRE_NO_AUTO_CAPTURE) != 0) cflags |= REG_NOSUB;
+     if ((options & PCRE_UTF8) != 0) cflags |= REG_UTF8;
+     if ((options & PCRE_UCP) != 0) cflags |= REG_UCP;
+     if ((options & PCRE_UNGREEDY) != 0) cflags |= REG_UNGREEDY;
      rc = regcomp(&preg, (char *)p, cflags);
      /* Compilation failed; go back for another re, skipping to blank line
-Line 876 
 while (!done)
+Line 1751 
 while (!done)
      if (rc != 0)
        {
-       (void)regerror(rc, &preg, (char *)buffer, BUFFER_SIZE);
+       (void)regerror(rc, &preg, (char *)buffer, buffer_size);
        fprintf(outfile, "Failed: POSIX code %d: %s\n", rc, buffer);
        goto SKIP_DATA;
        }
-Line 888 
 while (!done)
+Line 1763 
 while (!done)
  #endif  /* !defined NOPOSIX */
      {
-     if (timeit)
+     unsigned long int get_options;
+     if (timeit > 0)
        {
        register int i;
        clock_t time_taken;
        clock_t start_time = clock();
-       for (i = 0; i < LOOPREPEAT; i++)
+       for (i = 0; i < timeit; i++)
          {
          re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
          if (re != NULL) free(re);
          }
        time_taken = clock() - start_time;
-       fprintf(outfile, "Compile time %.3f milliseconds\n",
+       fprintf(outfile, "Compile time %.4f milliseconds\n",
-         (((double)time_taken * 1000.0) / (double)LOOPREPEAT) /
+         (((double)time_taken * 1000.0) / (double)timeit) /
            (double)CLOCKS_PER_SEC);
        }
-Line 917 
 while (!done)
+Line 1794 
 while (!done)
          {
          for (;;)
            {
-           if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL)
+           if (extend_inputline(infile, buffer, NULL) == NULL)
              {
              done = 1;
              goto CONTINUE;
-Line 931 
 while (!done)
+Line 1808 
 while (!done)
        goto CONTINUE;
        }
-     /* Compilation succeeded; print data if required. There are now two
+     /* Compilation succeeded. It is now possible to set the UTF-8 option from
-     info-returning functions. The old one has a limited interface and
+     within the regex; check for this so that we know how to process the data
-     returns only limited data. Check that it agrees with the newer one. */
+     lines. */
+     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
+     if ((get_options & PCRE_UTF8) != 0) use_utf8 = 1;
+     /* Print information if required. There are now two info-returning
+     functions. The old one has a limited interface and returns only limited
+     data. Check that it agrees with the newer one. */
      if (log_store)
        fprintf(outfile, "Memory allocation (code space): %d\n",
-Line 947 
 while (!done)
+Line 1831 
 while (!done)
      true_size = ((real_pcre *)re)->size;
      regex_gotten_store = gotten_store;
-     /* If /S was present, study the regexp to generate additional info to
+     /* If -s or /S was present, study the regex to generate additional info to
-     help with the matching. */
+     help with the matching, unless the pattern has the SS option, which
+     suppresses the effect of /S (used for a few test patterns where studying is
+     never sensible). */
-     if (do_study)
+     if (do_study || (force_study && !no_force_study))
        {
-       if (timeit)
+       if (timeit > 0)
          {
          register int i;
          clock_t time_taken;
          clock_t start_time = clock();
-         for (i = 0; i < LOOPREPEAT; i++)
+         for (i = 0; i < timeit; i++)
            extra = pcre_study(re, study_options, &error);
          time_taken = clock() - start_time;
          if (extra != NULL) free(extra);
-         fprintf(outfile, "  Study time %.3f milliseconds\n",
+         fprintf(outfile, "  Study time %.4f milliseconds\n",
-           (((double)time_taken * 1000.0) / (double)LOOPREPEAT) /
+           (((double)time_taken * 1000.0) / (double)timeit) /
              (double)CLOCKS_PER_SEC);
          }
        extra = pcre_study(re, study_options, &error);
-Line 972 
 while (!done)
+Line 1858 
 while (!done)
          true_study_size = ((pcre_study_data *)(extra->study_data))->size;
        }
+     /* If /K was present, we set up for handling MARK data. */
+     if (do_mark)
+       {
+       if (extra == NULL)
+         {
+         extra = (pcre_extra *)malloc(sizeof(pcre_extra));
+         extra->flags = 0;
+         }
+       extra->mark = &markptr;
+       extra->flags |= PCRE_EXTRA_MARK;
+       }
      /* If the 'F' option was present, we flip the bytes of all the integer
      fields in the regex data block and the study block. This is to make it
      possible to test PCRE's handling of byte-flipped patterns, e.g. those
-Line 980 
 while (!done)
+Line 1879 
 while (!done)
      if (do_flip)
        {
        real_pcre *rre = (real_pcre *)re;
-       rre->magic_number = byteflip(rre->magic_number, sizeof(rre->magic_number));
+       rre->magic_number =
+         byteflip(rre->magic_number, sizeof(rre->magic_number));
        rre->size = byteflip(rre->size, sizeof(rre->size));
        rre->options = byteflip(rre->options, sizeof(rre->options));
-       rre->top_bracket = byteflip(rre->top_bracket, sizeof(rre->top_bracket));
+       rre->flags = (pcre_uint16)byteflip(rre->flags, sizeof(rre->flags));
-       rre->top_backref = byteflip(rre->top_backref, sizeof(rre->top_backref));
+       rre->top_bracket =
-       rre->first_byte = byteflip(rre->first_byte, sizeof(rre->first_byte));
+         (pcre_uint16)byteflip(rre->top_bracket, sizeof(rre->top_bracket));
-       rre->req_byte = byteflip(rre->req_byte, sizeof(rre->req_byte));
+       rre->top_backref =
-       rre->name_table_offset = byteflip(rre->name_table_offset,
+         (pcre_uint16)byteflip(rre->top_backref, sizeof(rre->top_backref));
+       rre->first_byte =
+         (pcre_uint16)byteflip(rre->first_byte, sizeof(rre->first_byte));
+       rre->req_byte =
+         (pcre_uint16)byteflip(rre->req_byte, sizeof(rre->req_byte));
+       rre->name_table_offset = (pcre_uint16)byteflip(rre->name_table_offset,
          sizeof(rre->name_table_offset));
-       rre->name_entry_size = byteflip(rre->name_entry_size,
+       rre->name_entry_size = (pcre_uint16)byteflip(rre->name_entry_size,
          sizeof(rre->name_entry_size));
-       rre->name_count = byteflip(rre->name_count, sizeof(rre->name_count));
+       rre->name_count = (pcre_uint16)byteflip(rre->name_count,
+         sizeof(rre->name_count));
        if (extra != NULL)
          {
          pcre_study_data *rsd = (pcre_study_data *)(extra->study_data);
          rsd->size = byteflip(rsd->size, sizeof(rsd->size));
-         rsd->options = byteflip(rsd->options, sizeof(rsd->options));
+         rsd->flags = byteflip(rsd->flags, sizeof(rsd->flags));
+         rsd->minlength = byteflip(rsd->minlength, sizeof(rsd->minlength));
          }
        }
-Line 1005 
 while (!done)
+Line 1912 
 while (!done)
      SHOW_INFO:
+     if (do_debug)
+       {
+       fprintf(outfile, "------------------------------------------------------------------\n");
+       pcre_printint(re, outfile, debug_lengths);
+       }
+     /* We already have the options in get_options (see above) */
      if (do_showinfo)
        {
-       unsigned long int get_options, all_options;
+       unsigned long int all_options;
+ #if !defined NOINFOCHECK
        int old_first_char, old_options, old_count;
-       int count, backrefmax, first_char, need_char;
+ #endif
+       int count, backrefmax, first_char, need_char, okpartial, jchanged,
+         hascrorlf;
        int nameentrysize, namecount;
        const uschar *nametable;
-       if (do_debug)
-         {
-         fprintf(outfile, "------------------------------------------------------------------\n");
-         print_internals(re, outfile);
-         }
-       new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
        new_info(re, NULL, PCRE_INFO_SIZE, &size);
        new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
        new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax);
-Line 1028 
 while (!done)
+Line 1939 
 while (!done)
        new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &nameentrysize);
        new_info(re, NULL, PCRE_INFO_NAMECOUNT, &namecount);
        new_info(re, NULL, PCRE_INFO_NAMETABLE, (void *)&nametable);
+       new_info(re, NULL, PCRE_INFO_OKPARTIAL, &okpartial);
+       new_info(re, NULL, PCRE_INFO_JCHANGED, &jchanged);
+       new_info(re, NULL, PCRE_INFO_HASCRORLF, &hascrorlf);
+ #if !defined NOINFOCHECK
        old_count = pcre_info(re, &old_options, &old_first_char);
        if (count < 0) fprintf(outfile,
          "Error %d from pcre_info()\n", count);
-Line 1046 
 while (!done)
+Line 1961 
 while (!done)
            "Options disagreement: pcre_fullinfo=%ld pcre_info=%d\n",
              get_options, old_options);
          }
+ #endif
        if (size != regex_gotten_store) fprintf(outfile,
          "Size disagreement: pcre_fullinfo=%d call to malloc for %d\n",
-         size, regex_gotten_store);
+         (int)size, (int)regex_gotten_store);
        fprintf(outfile, "Capturing subpattern count = %d\n", count);
        if (backrefmax > 0)
-Line 1067 
 while (!done)
+Line 1983 
 while (!done)
            }
          }
-       /* The NOPARTIAL bit is a private bit in the options, so we have
+       if (!okpartial) fprintf(outfile, "Partial matching not supported\n");
-       to fish it out via out back door */
+       if (hascrorlf) fprintf(outfile, "Contains explicit CR or LF match\n");
        all_options = ((real_pcre *)re)->options;
-       if (do_flip)
+       if (do_flip) all_options = byteflip(all_options, sizeof(all_options));
-         {
-         all_options = byteflip(all_options, sizeof(all_options));
-         }
-       if ((all_options & PCRE_NOPARTIAL) != 0)
-         fprintf(outfile, "Partial matching not supported\n");
        if (get_options == 0) fprintf(outfile, "No options\n");
-         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s\n",
+         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
            ((get_options & PCRE_ANCHORED) != 0)? " anchored" : "",
            ((get_options & PCRE_CASELESS) != 0)? " caseless" : "",
            ((get_options & PCRE_EXTENDED) != 0)? " extended" : "",
            ((get_options & PCRE_MULTILINE) != 0)? " multiline" : "",
+           ((get_options & PCRE_FIRSTLINE) != 0)? " firstline" : "",
            ((get_options & PCRE_DOTALL) != 0)? " dotall" : "",
+           ((get_options & PCRE_BSR_ANYCRLF) != 0)? " bsr_anycrlf" : "",
+           ((get_options & PCRE_BSR_UNICODE) != 0)? " bsr_unicode" : "",
            ((get_options & PCRE_DOLLAR_ENDONLY) != 0)? " dollar_endonly" : "",
            ((get_options & PCRE_EXTRA) != 0)? " extra" : "",
            ((get_options & PCRE_UNGREEDY) != 0)? " ungreedy" : "",
+           ((get_options & PCRE_NO_AUTO_CAPTURE) != 0)? " no_auto_capture" : "",
            ((get_options & PCRE_UTF8) != 0)? " utf8" : "",
-           ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf8_check" : "");
+           ((get_options & PCRE_UCP) != 0)? " ucp" : "",
+           ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf8_check" : "",
+           ((get_options & PCRE_NO_START_OPTIMIZE) != 0)? " no_start_optimize" : "",
+           ((get_options & PCRE_DUPNAMES) != 0)? " dupnames" : "");
+       if (jchanged) fprintf(outfile, "Duplicate name status changes\n");
+       switch (get_options & PCRE_NEWLINE_BITS)
+         {
+         case PCRE_NEWLINE_CR:
+         fprintf(outfile, "Forced newline sequence: CR\n");
+         break;
+         case PCRE_NEWLINE_LF:
+         fprintf(outfile, "Forced newline sequence: LF\n");
+         break;
+         case PCRE_NEWLINE_CRLF:
+         fprintf(outfile, "Forced newline sequence: CRLF\n");
+         break;
+         case PCRE_NEWLINE_ANYCRLF:
+         fprintf(outfile, "Forced newline sequence: ANYCRLF\n");
+         break;
-       if (((((real_pcre *)re)->options) & PCRE_ICHANGED) != 0)
+         case PCRE_NEWLINE_ANY:
-         fprintf(outfile, "Case state changes\n");
+         fprintf(outfile, "Forced newline sequence: ANY\n");
+         break;
+         default:
+         break;
+         }
        if (first_char == -1)
          {
-         fprintf(outfile, "First char at start or follows \\n\n");
+         fprintf(outfile, "First char at start or follows newline\n");
          }
        else if (first_char < 0)
          {
-Line 1108 
 while (!done)
+Line 2050 
 while (!done)
          int ch = first_char & 255;
          const char *caseless = ((first_char & REQ_CASELESS) == 0)?
            "" : " (caseless)";
-         if (isprint(ch))
+         if (PRINTHEX(ch))
            fprintf(outfile, "First char = \'%c\'%s\n", ch, caseless);
          else
            fprintf(outfile, "First char = %d%s\n", ch, caseless);
-Line 1123 
 while (!done)
+Line 2065 
 while (!done)
          int ch = need_char & 255;
          const char *caseless = ((need_char & REQ_CASELESS) == 0)?
            "" : " (caseless)";
-         if (isprint(ch))
+         if (PRINTHEX(ch))
            fprintf(outfile, "Need char = \'%c\'%s\n", ch, caseless);
          else
            fprintf(outfile, "Need char = %d%s\n", ch, caseless);
-Line 1132 
 while (!done)
+Line 2074 
 while (!done)
        /* Don't output study size; at present it is in any case a fixed
        value, but it varies, depending on the computer architecture, and
        so messes up the test suite. (And with the /F option, it might be
-       flipped.) */
+       flipped.) If study was forced by an external -s, don't show this
+       information unless -i or -d was also present. This means that, except
+       when auto-callouts are involved, the output from runs with and without
+       -s should be identical. */
-       if (do_study)
+       if (do_study || (force_study && showinfo && !no_force_study))
          {
          if (extra == NULL)
            fprintf(outfile, "Study returned NULL\n");
          else
            {
            uschar *start_bits = NULL;
-           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
+           int minlength;
+           new_info(re, extra, PCRE_INFO_MINLENGTH, &minlength);
+           fprintf(outfile, "Subject length lower bound = %d\n", minlength);
+           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
            if (start_bits == NULL)
-             fprintf(outfile, "No starting byte set\n");
+             fprintf(outfile, "No set of starting bytes\n");
            else
              {
              int i;
-Line 1159 
 while (!done)
+Line 2108 
 while (!done)
                    fprintf(outfile, "\n  ");
                    c = 2;
                    }
-                 if (isprint(i) && i != ' ')
+                 if (PRINTHEX(i) && i != ' ')
                    {
                    fprintf(outfile, "%c ", i);
                    c += 2;
-Line 1191 
 while (!done)
+Line 2140 
 while (!done)
        else
          {
          uschar sbuf[8];
-         sbuf[0] = (true_size >> 24)  & 255;
+         sbuf[0] = (uschar)((true_size >> 24) & 255);
-         sbuf[1] = (true_size >> 16)  & 255;
+         sbuf[1] = (uschar)((true_size >> 16) & 255);
-         sbuf[2] = (true_size >>  8)  & 255;
+         sbuf[2] = (uschar)((true_size >>  8) & 255);
-         sbuf[3] = (true_size)  & 255;
+         sbuf[3] = (uschar)((true_size) & 255);
-         sbuf[4] = (true_study_size >> 24)  & 255;
+         sbuf[4] = (uschar)((true_study_size >> 24) & 255);
-         sbuf[5] = (true_study_size >> 16)  & 255;
+         sbuf[5] = (uschar)((true_study_size >> 16) & 255);
-         sbuf[6] = (true_study_size >>  8)  & 255;
+         sbuf[6] = (uschar)((true_study_size >>  8) & 255);
-         sbuf[7] = (true_study_size)  & 255;
+         sbuf[7] = (uschar)((true_study_size) & 255);
          if (fwrite(sbuf, 1, 8, f) < 8 ||
              fwrite(re, 1, true_size, f) < true_size)
-Line 1208 
 while (!done)
+Line 2157 
 while (!done)
            }
          else
            {
-           fprintf(outfile, "Compiled regex written to %s\n", to_file);
+           fprintf(outfile, "Compiled pattern written to %s\n", to_file);
+           /* If there is study data, write it, but verify the writing only
+           if the studying was requested by /S, not just by -s. */
            if (extra != NULL)
              {
              if (fwrite(extra->study_data, 1, true_study_size, f) <
-Line 1222 
 while (!done)
+Line 2175 
 while (!done)
            }
          fclose(f);
          }
+       new_free(re);
+       if (extra != NULL) new_free(extra);
+       if (locale_set)
+         {
+         new_free((void *)tables);
+         setlocale(LC_CTYPE, "C");
+         locale_set = 0;
+         }
        continue;  /* With next regex */
        }
      }        /* End of non-POSIX compile */
-Line 1230 
 while (!done)
+Line 2192 
 while (!done)
    for (;;)
      {
-     unsigned char *q;
+     uschar *q;
-     unsigned char *bptr = dbuffer;
+     uschar *bptr;
      int *use_offsets = offsets;
      int use_size_offsets = size_offsets;
      int callout_data = 0;
      int callout_data_set = 0;
      int count, c;
      int copystrings = 0;
-     int find_match_limit = 0;
+     int find_match_limit = default_find_match_limit;
      int getstrings = 0;
      int getlist = 0;
      int gmatched = 0;
      int start_offset = 0;
+     int start_offset_sign = 1;
      int g_notempty = 0;
+     int use_dfa = 0;
      options = 0;
+     *copynames = 0;
+     *getnames = 0;
+     copynamesptr = copynames;
+     getnamesptr = getnames;
      pcre_callout = callout;
      first_callout = 1;
+     last_callout_mark = NULL;
      callout_extra = 0;
      callout_count = 0;
      callout_fail_count = 999999;
      callout_fail_id = -1;
      show_malloc = 0;
-     if (infile == stdin) printf("data> ");
+     if (extra != NULL) extra->flags &=
-     if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL)
+       ~(PCRE_EXTRA_MATCH_LIMIT|PCRE_EXTRA_MATCH_LIMIT_RECURSION);
+     len = 0;
+     for (;;)
        {
-       done = 1;
+       if (extend_inputline(infile, buffer + len, "data> ") == NULL)
-       goto CONTINUE;
+         {
+         if (len > 0)    /* Reached EOF without hitting a newline */
+           {
+           fprintf(outfile, "\n");
+           break;
+           }
+         done = 1;
+         goto CONTINUE;
+         }
+       if (infile != stdin) fprintf(outfile, "%s", (char *)buffer);
+       len = (int)strlen((char *)buffer);
+       if (buffer[len-1] == '\n') break;
        }
-     if (infile != stdin) fprintf(outfile, "%s", (char *)buffer);
-     len = (int)strlen((char *)buffer);
      while (len > 0 && isspace(buffer[len-1])) len--;
      buffer[len] = 0;
      if (len == 0) break;
-Line 1271 
 while (!done)
+Line 2254 
 while (!done)
      p = buffer;
      while (isspace(*p)) p++;
-     q = dbuffer;
+     bptr = q = dbuffer;
      while ((c = *p++) != 0)
        {
        int i = 0;
-Line 1293 
 while (!done)
+Line 2276 
 while (!done)
          c -= '0';
          while (i++ < 2 && isdigit(*p) && *p != '8' && *p != '9')
            c = c * 8 + *p++ - '0';
+ #if !defined NOUTF8
+         if (use_utf8 && c > 255)
+           {
+           unsigned char buff8[8];
+           int ii, utn;
+           utn = ord2utf8(c, buff8);
+           for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
+           c = buff8[ii];   /* Last byte */
+           }
+ #endif
          break;
          case 'x':
          /* Handle \x{..} specially - new Perl thing for utf8 */
+ #if !defined NOUTF8
          if (*p == '{')
            {
            unsigned char *pt = p;
-Line 1309 
 while (!done)
+Line 2304 
 while (!done)
              {
              unsigned char buff8[8];
              int ii, utn;
-             utn = ord2utf8(c, buff8);
+             if (use_utf8)
-             for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
+               {
-             c = buff8[ii];   /* Last byte */
+               utn = ord2utf8(c, buff8);
+               for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
+               c = buff8[ii];   /* Last byte */
+               }
+             else
+              {
+              if (c > 255)
+                fprintf(outfile, "** Character \\x{%x} is greater than 255 and "
+                  "UTF-8 mode is not enabled.\n"
+                  "** Truncation will probably give the wrong result.\n", c);
+              }
              p = pt + 1;
              break;
              }
            /* Not correct form; fall through */
            }
+ #endif
          /* Ordinary \x */
-Line 1333 
 while (!done)
+Line 2339 
 while (!done)
          continue;
          case '>':
+         if (*p == '-')
+           {
+           start_offset_sign = -1;
+           p++;
+           }
          while(isdigit(*p)) start_offset = start_offset * 10 + *p++ - '0';
+         start_offset *= start_offset_sign;
          continue;
          case 'A':  /* Option setting */
-Line 1352 
 while (!done)
+Line 2364 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar name[256];
+           uschar *npp = copynamesptr;
-           uschar *npp = name;
            while (isalnum(*p)) *npp++ = *p++;
+           *npp++ = 0;
            *npp = 0;
-           n = pcre_get_stringnumber(re, (char *)name);
+           n = pcre_get_stringnumber(re, (char *)copynamesptr);
            if (n < 0)
-             fprintf(outfile, "no parentheses with name \"%s\"\n", name);
+             fprintf(outfile, "no parentheses with name \"%s\"\n", copynamesptr);
-           else copystrings |= 1 << n;
+           copynamesptr = npp;
            }
          else if (*p == '+')
            {
-Line 1397 
 while (!done)
+Line 2409 
 while (!done)
            }
          continue;
+ #if !defined NODFA
+         case 'D':
+ #if !defined NOPOSIX
+         if (posix || do_posix)
+           printf("** Can't use dfa matching in POSIX mode: \\D ignored\n");
+         else
+ #endif
+           use_dfa = 1;
+         continue;
+ #endif
+ #if !defined NODFA
+         case 'F':
+         options |= PCRE_DFA_SHORTEST;
+         continue;
+ #endif
          case 'G':
          if (isdigit(*p))
            {
-Line 1405 
 while (!done)
+Line 2434 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar name[256];
+           uschar *npp = getnamesptr;
-           uschar *npp = name;
            while (isalnum(*p)) *npp++ = *p++;
+           *npp++ = 0;
            *npp = 0;
-           n = pcre_get_stringnumber(re, (char *)name);
+           n = pcre_get_stringnumber(re, (char *)getnamesptr);
            if (n < 0)
-             fprintf(outfile, "no parentheses with name \"%s\"\n", name);
+             fprintf(outfile, "no parentheses with name \"%s\"\n", getnamesptr);
-           else getstrings |= 1 << n;
+           getnamesptr = npp;
            }
          continue;
-Line 1425 
 while (!done)
+Line 2454 
 while (!done)
          continue;
          case 'N':
-         options |= PCRE_NOTEMPTY;
+         if ((options & PCRE_NOTEMPTY) != 0)
+           options = (options & ~PCRE_NOTEMPTY) | PCRE_NOTEMPTY_ATSTART;
+         else
+           options |= PCRE_NOTEMPTY;
          continue;
          case 'O':
-Line 1438 
 while (!done)
+Line 2470 
 while (!done)
            if (offsets == NULL)
              {
              printf("** Failed to get %d bytes of memory for offsets vector\n",
-               size_offsets_max * sizeof(int));
+               (int)(size_offsets_max * sizeof(int)));
-             return 1;
+             yield = 1;
+             goto EXIT;
              }
            }
          use_size_offsets = n;
-Line 1447 
 while (!done)
+Line 2480 
 while (!done)
          continue;
          case 'P':
-         options |= PCRE_PARTIAL;
+         options |= ((options & PCRE_PARTIAL_SOFT) == 0)?
+           PCRE_PARTIAL_SOFT : PCRE_PARTIAL_HARD;
          continue;
+         case 'Q':
+         while(isdigit(*p)) n = n * 10 + *p++ - '0';
+         if (extra == NULL)
+           {
+           extra = (pcre_extra *)malloc(sizeof(pcre_extra));
+           extra->flags = 0;
+           }
+         extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+         extra->match_limit_recursion = n;
+         continue;
+         case 'q':
+         while(isdigit(*p)) n = n * 10 + *p++ - '0';
+         if (extra == NULL)
+           {
+           extra = (pcre_extra *)malloc(sizeof(pcre_extra));
+           extra->flags = 0;
+           }
+         extra->flags |= PCRE_EXTRA_MATCH_LIMIT;
+         extra->match_limit = n;
+         continue;
+ #if !defined NODFA
+         case 'R':
+         options |= PCRE_DFA_RESTART;
+         continue;
+ #endif
          case 'S':
          show_malloc = 1;
          continue;
+         case 'Y':
+         options |= PCRE_NO_START_OPTIMIZE;
+         continue;
          case 'Z':
          options |= PCRE_NOTEOL;
          continue;
-Line 1461 
 while (!done)
+Line 2527 
 while (!done)
          case '?':
          options |= PCRE_NO_UTF8_CHECK;
          continue;
+         case '<':
+           {
+           int x = check_newline(p, outfile);
+           if (x == 0) goto NEXT_DATA;
+           options |= x;
+           while (*p++ != '>');
+           }
+         continue;
          }
        *q++ = c;
        }
      *q = 0;
-     len = q - dbuffer;
+     len = (int)(q - dbuffer);
+     /* Move the data to the end of the buffer so that a read over the end of
+     the buffer will be seen by valgrind, even if it doesn't cause a crash. If
+     we are using the POSIX interface, we must include the terminating zero. */
+ #if !defined NOPOSIX
+     if (posix || do_posix)
+       {
+       memmove(bptr + buffer_size - len - 1, bptr, len + 1);
+       bptr += buffer_size - len - 1;
+       }
+     else
+ #endif
+       {
+       memmove(bptr + buffer_size - len, bptr, len);
+       bptr += buffer_size - len;
+       }
+     if ((all_use_dfa || use_dfa) && find_match_limit)
+       {
+       printf("**Match limit not relevant for DFA matching: ignored\n");
+       find_match_limit = 0;
+       }
      /* Handle matching via the POSIX interface, which does not
      support timing or playing with the match limit or callout data. */
-Line 1480 
 while (!done)
+Line 2578 
 while (!done)
          pmatch = (regmatch_t *)malloc(sizeof(regmatch_t) * use_size_offsets);
        if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL;
        if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL;
+       if ((options & PCRE_NOTEMPTY) != 0) eflags |= REG_NOTEMPTY;
        rc = regexec(&preg, (const char *)bptr, use_size_offsets, pmatch, eflags);
        if (rc != 0)
          {
-         (void)regerror(rc, &preg, (char *)buffer, BUFFER_SIZE);
+         (void)regerror(rc, &preg, (char *)buffer, buffer_size);
          fprintf(outfile, "No match: POSIX code %d: %s\n", rc, buffer);
          }
+       else if ((((const pcre *)preg.re_pcre)->options & PCRE_NO_AUTO_CAPTURE)
+               != 0)
+         {
+         fprintf(outfile, "Matched with REG_NOSUB\n");
+         }
        else
          {
          size_t i;
-Line 1499 
 while (!done)
+Line 2603 
 while (!done)
              (void)pchars(dbuffer + pmatch[i].rm_so,
                pmatch[i].rm_eo - pmatch[i].rm_so, outfile);
              fprintf(outfile, "\n");
-             if (i == 0 && do_showrest)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               fprintf(outfile, " 0+ ");
+               fprintf(outfile, "%2d+ ", (int)i);
                (void)pchars(dbuffer + pmatch[i].rm_eo, len - pmatch[i].rm_eo,
                  outfile);
                fprintf(outfile, "\n");
-Line 1519 
 while (!done)
+Line 2623 
 while (!done)
      for (;; gmatched++)    /* Loop for /g or /G */
        {
-       if (timeit)
+       markptr = NULL;
+       if (timeitm > 0)
          {
          register int i;
          clock_t time_taken;
          clock_t start_time = clock();
-         for (i = 0; i < LOOPREPEAT; i++)
+ #if !defined NODFA
+         if (all_use_dfa || use_dfa)
+           {
+           int workspace[1000];
+           for (i = 0; i < timeitm; i++)
+             count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset,
+               options | g_notempty, use_offsets, use_size_offsets, workspace,
+               sizeof(workspace)/sizeof(int));
+           }
+         else
+ #endif
+         for (i = 0; i < timeitm; i++)
            count = pcre_exec(re, extra, (char *)bptr, len,
              start_offset, options | g_notempty, use_offsets, use_size_offsets);
          time_taken = clock() - start_time;
-         fprintf(outfile, "Execute time %.3f milliseconds\n",
+         fprintf(outfile, "Execute time %.4f milliseconds\n",
-           (((double)time_taken * 1000.0) / (double)LOOPREPEAT) /
+           (((double)time_taken * 1000.0) / (double)timeitm) /
              (double)CLOCKS_PER_SEC);
          }
        /* If find_match_limit is set, we want to do repeated matches with
-       varying limits in order to find the minimum value. */
+       varying limits in order to find the minimum value for the match limit and
+       for the recursion limit. */
        if (find_match_limit)
          {
-         int min = 0;
-         int mid = 64;
-         int max = -1;
          if (extra == NULL)
            {
            extra = (pcre_extra *)malloc(sizeof(pcre_extra));
            extra->flags = 0;
            }
-         extra->flags |= PCRE_EXTRA_MATCH_LIMIT;
-         for (;;)
-           {
-           extra->match_limit = mid;
-           count = pcre_exec(re, extra, (char *)bptr, len, start_offset,
-             options | g_notempty, use_offsets, use_size_offsets);
-           if (count == PCRE_ERROR_MATCHLIMIT)
-             {
-             /* fprintf(outfile, "Testing match limit = %d\n", mid); */
-             min = mid;
-             mid = (mid == max - 1)? max : (max > 0)? (min + max)/2 : mid*2;
-             }
-           else if (count >= 0 || count == PCRE_ERROR_NOMATCH ||
-                                  count == PCRE_ERROR_PARTIAL)
-             {
-             if (mid == min + 1)
-               {
-               fprintf(outfile, "Minimum match limit = %d\n", mid);
-               break;
-               }
-             /* fprintf(outfile, "Testing match limit = %d\n", mid); */
-             max = mid;
-             mid = (min + mid)/2;
-             }
-           else break;    /* Some other error */
-           }
-         extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT;
+         (void)check_match_limit(re, extra, bptr, len, start_offset,
+           options|g_notempty, use_offsets, use_size_offsets,
+           PCRE_EXTRA_MATCH_LIMIT, &(extra->match_limit),
+           PCRE_ERROR_MATCHLIMIT, "match()");
+         count = check_match_limit(re, extra, bptr, len, start_offset,
+           options|g_notempty, use_offsets, use_size_offsets,
+           PCRE_EXTRA_MATCH_LIMIT_RECURSION, &(extra->match_limit_recursion),
+           PCRE_ERROR_RECURSIONLIMIT, "match() recursion");
          }
        /* If callout_data is set, use the interface with additional data */
-Line 1597 
 while (!done)
+Line 2695 
 while (!done)
        /* The normal case is just to do the match once, with the default
        value of match_limit. */
-       else
+ #if !defined NODFA
+       else if (all_use_dfa || use_dfa)
          {
-         count = pcre_exec(re, extra, (char *)bptr, len,
+         int workspace[1000];
-           start_offset, options | g_notempty, use_offsets, use_size_offsets);
+         count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset,
+           options | g_notempty, use_offsets, use_size_offsets, workspace,
+           sizeof(workspace)/sizeof(int));
+         if (count == 0)
+           {
+           fprintf(outfile, "Matched, but too many subsidiary matches\n");
+           count = use_size_offsets/2;
+           }
          }
+ #endif
-       if (count == 0)
+       else
          {
-         fprintf(outfile, "Matched, but too many substrings\n");
+         count = pcre_exec(re, extra, (char *)bptr, len,
-         count = use_size_offsets/3;
+           start_offset, options | g_notempty, use_offsets, use_size_offsets);
+         if (count == 0)
+           {
+           fprintf(outfile, "Matched, but too many substrings\n");
+           count = use_size_offsets/3;
+           }
          }
        /* Matched */
        if (count >= 0)
          {
-         int i;
+         int i, maxcount;
+ #if !defined NODFA
+         if (all_use_dfa || use_dfa) maxcount = use_size_offsets/2; else
+ #endif
+           maxcount = use_size_offsets/3;
+         /* This is a check against a lunatic return value. */
+         if (count > maxcount)
+           {
+           fprintf(outfile,
+             "** PCRE error: returned count %d is too big for offset size %d\n",
+             count, use_size_offsets);
+           count = use_size_offsets/3;
+           if (do_g || do_G)
+             {
+             fprintf(outfile, "** /%c loop abandoned\n", do_g? 'g' : 'G');
+             do_g = do_G = FALSE;        /* Break g/G loop */
+             }
+           }
+         /* do_allcaps requests showing of all captures in the pattern, to check
+         unset ones at the end. */
+         if (do_allcaps)
+           {
+           new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
+           count++;   /* Allow for full match */
+           if (count * 2 > use_size_offsets) count = use_size_offsets/2;
+           }
+         /* Output the captured substrings */
          for (i = 0; i < count * 2; i += 2)
            {
            if (use_offsets[i] < 0)
+             {
+             if (use_offsets[i] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i], i);
+             if (use_offsets[i+1] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i+1], i+1);
              fprintf(outfile, "%2d: <unset>\n", i/2);
+             }
            else
              {
              fprintf(outfile, "%2d: ", i/2);
              (void)pchars(bptr + use_offsets[i],
                use_offsets[i+1] - use_offsets[i], outfile);
              fprintf(outfile, "\n");
-             if (i == 0)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               if (do_showrest)
+               fprintf(outfile, "%2d+ ", i/2);
-                 {
+               (void)pchars(bptr + use_offsets[i+1], len - use_offsets[i+1],
-                 fprintf(outfile, " 0+ ");
+                 outfile);
-                 (void)pchars(bptr + use_offsets[i+1], len - use_offsets[i+1],
+               fprintf(outfile, "\n");
-                   outfile);
-                 fprintf(outfile, "\n");
-                 }
                }
              }
            }
+         if (markptr != NULL) fprintf(outfile, "MK: %s\n", markptr);
          for (i = 0; i < 32; i++)
            {
            if ((copystrings & (1 << i)) != 0)
              {
-             char copybuffer[16];
+             char copybuffer[256];
              int rc = pcre_copy_substring((char *)bptr, use_offsets, count,
                i, copybuffer, sizeof(copybuffer));
              if (rc < 0)
-Line 1651 
 while (!done)
+Line 2803 
 while (!done)
              }
            }
+         for (copynamesptr = copynames;
+              *copynamesptr != 0;
+              copynamesptr += (int)strlen((char*)copynamesptr) + 1)
+           {
+           char copybuffer[256];
+           int rc = pcre_copy_named_substring(re, (char *)bptr, use_offsets,
+             count, (char *)copynamesptr, copybuffer, sizeof(copybuffer));
+           if (rc < 0)
+             fprintf(outfile, "copy substring %s failed %d\n", copynamesptr, rc);
+           else
+             fprintf(outfile, "  C %s (%d) %s\n", copybuffer, rc, copynamesptr);
+           }
          for (i = 0; i < 32; i++)
            {
            if ((getstrings & (1 << i)) != 0)
-Line 1663 
 while (!done)
+Line 2828 
 while (!done)
              else
                {
                fprintf(outfile, "%2dG %s (%d)\n", i, substring, rc);
-               /* free((void *)substring); */
                pcre_free_substring(substring);
                }
              }
            }
+         for (getnamesptr = getnames;
+              *getnamesptr != 0;
+              getnamesptr += (int)strlen((char*)getnamesptr) + 1)
+           {
+           const char *substring;
+           int rc = pcre_get_named_substring(re, (char *)bptr, use_offsets,
+             count, (char *)getnamesptr, &substring);
+           if (rc < 0)
+             fprintf(outfile, "copy substring %s failed %d\n", getnamesptr, rc);
+           else
+             {
+             fprintf(outfile, "  G %s (%d) %s\n", substring, rc, getnamesptr);
+             pcre_free_substring(substring);
+             }
+           }
          if (getlist)
            {
            const char **stringlist;
-Line 1692 
 while (!done)
+Line 2872 
 while (!done)
        else if (count == PCRE_ERROR_PARTIAL)
          {
-         fprintf(outfile, "Partial match\n");
+         if (markptr == NULL) fprintf(outfile, "Partial match");
+           else fprintf(outfile, "Partial match, mark=%s", markptr);
+         if (use_size_offsets > 1)
+           {
+           fprintf(outfile, ": ");
+           pchars(bptr + use_offsets[0], use_offsets[1] - use_offsets[0],
+             outfile);
+           }
+         fprintf(outfile, "\n");
          break;  /* Out of the /g loop */
          }
        /* Failed to match. If this is a /g or /G loop and we previously set
-       g_notempty after a null match, this is not necessarily the end.
+       g_notempty after a null match, this is not necessarily the end. We want
-       We want to advance the start offset, and continue. In the case of UTF-8
+       to advance the start offset, and continue. We won't be at the end of the
-       matching, the advance must be one character, not one byte. Fudge the
+       string - that was checked before setting g_notempty.
-       offset values to achieve this. We won't be at the end of the string -
-       that was checked before setting g_notempty. */
+       Complication arises in the case when the newline convention is "any",
+       "crlf", or "anycrlf". If the previous match was at the end of a line
+       terminated by CRLF, an advance of one character just passes the \r,
+       whereas we should prefer the longer newline sequence, as does the code in
+       pcre_exec(). Fudge the offset value to achieve this. We check for a
+       newline setting in the pattern; if none was set, use pcre_config() to
+       find the default.
+       Otherwise, in the case of UTF-8 matching, the advance must be one
+       character, not one byte. */
        else
          {
          if (g_notempty != 0)
            {
            int onechar = 1;
+           unsigned int obits = ((real_pcre *)re)->options;
            use_offsets[0] = start_offset;
-           if (use_utf8)
+           if ((obits & PCRE_NEWLINE_BITS) == 0)
+             {
+             int d;
+             (void)pcre_config(PCRE_CONFIG_NEWLINE, &d);
+             /* Note that these values are always the ASCII ones, even in
+             EBCDIC environments. CR = 13, NL = 10. */
+             obits = (d == 13)? PCRE_NEWLINE_CR :
+                     (d == 10)? PCRE_NEWLINE_LF :
+                     (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF :
+                     (d == -2)? PCRE_NEWLINE_ANYCRLF :
+                     (d == -1)? PCRE_NEWLINE_ANY : 0;
+             }
+           if (((obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANY ||
+                (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_CRLF ||
+                (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANYCRLF)
+               &&
+               start_offset < len - 1 &&
+               bptr[start_offset] == '\r' &&
+               bptr[start_offset+1] == '\n')
+             onechar++;
+           else if (use_utf8)
              {
              while (start_offset + onechar < len)
                {
-               int tb = bptr[start_offset+onechar];
+               if ((bptr[start_offset+onechar] & 0xc0) != 0x80) break;
-               if (tb <= 127) break;
+               onechar++;
-               tb &= 0xc0;
-               if (tb != 0 && tb != 0xc0) onechar++;
                }
              }
            use_offsets[1] = start_offset + onechar;
            }
          else
            {
-           if (count == PCRE_ERROR_NOMATCH)
+           switch(count)
              {
-             if (gmatched == 0) fprintf(outfile, "No match\n");
+             case PCRE_ERROR_NOMATCH:
+             if (gmatched == 0)
+               {
+               if (markptr == NULL) fprintf(outfile, "No match\n");
+                 else fprintf(outfile, "No match, mark = %s\n", markptr);
+               }
+             break;
+             case PCRE_ERROR_BADUTF8:
+             case PCRE_ERROR_SHORTUTF8:
+             fprintf(outfile, "Error %d (%s UTF-8 string)", count,
+               (count == PCRE_ERROR_BADUTF8)? "bad" : "short");
+             if (use_size_offsets >= 2)
+               fprintf(outfile, " offset=%d reason=%d", use_offsets[0],
+                 use_offsets[1]);
+             fprintf(outfile, "\n");
+             break;
+             default:
+             if (count < 0 && (-count) < sizeof(errtexts)/sizeof(const char *))
+               fprintf(outfile, "Error %d (%s)\n", count, errtexts[-count]);
+             else
+               fprintf(outfile, "Error %d (Unexpected value)\n", count);
+             break;
              }
-           else fprintf(outfile, "Error %d\n", count);
            break;  /* Out of the /g loop */
            }
          }
-Line 1737 
 while (!done)
+Line 2976 
 while (!done)
        if (!do_g && !do_G) break;
        /* If we have matched an empty string, first check to see if we are at
-       the end of the subject. If so, the /g loop is over. Otherwise, mimic
+       the end of the subject. If so, the /g loop is over. Otherwise, mimic what
-       what Perl's /g options does. This turns out to be rather cunning. First
+       Perl's /g options does. This turns out to be rather cunning. First we set
-       we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match again at the
+       PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED and try the match again at the
        same point. If this fails (picked up above) we advance to the next
        character. */
        g_notempty = 0;
        if (use_offsets[0] == use_offsets[1])
          {
          if (use_offsets[0] == len) break;
-         g_notempty = PCRE_NOTEMPTY | PCRE_ANCHORED;
+         g_notempty = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED;
          }
        /* For /g, update the start offset, leaving the rest alone */
-Line 1762 
 while (!done)
+Line 3002 
 while (!done)
          len -= use_offsets[1];
          }
        }  /* End of loop for /g and /G */
+     NEXT_DATA: continue;
      }    /* End of loop for data lines */
    CONTINUE:
-Line 1770 
 while (!done)
+Line 3012 
 while (!done)
    if (posix || do_posix) regfree(&preg);
  #endif
-   if (re != NULL) free(re);
+   if (re != NULL) new_free(re);
-   if (extra != NULL) free(extra);
+   if (extra != NULL) new_free(extra);
-   if (tables != NULL)
+   if (locale_set)
      {
-     free((void *)tables);
+     new_free((void *)tables);
      setlocale(LC_CTYPE, "C");
+     locale_set = 0;
      }
    }
  if (infile == stdin) fprintf(outfile, "\n");
- return 0;
+ EXIT:
+ if (infile != NULL && infile != stdin) fclose(infile);
+ if (outfile != NULL && outfile != stdout) fclose(outfile);
+ free(buffer);
+ free(dbuffer);
+ free(pbuffer);
+ free(offsets);
+ return yield;
  }
- /* End */
+ /* End of pcretest.c */

 Legend:



Removed from v.75
 


changed lines


 
Added in v.645
 Legend:



Removed from v.75
 


changed lines


 
Added in v.645
-Removed from v.75
+Added in v.645

	ViewVC Help
Powered by ViewVC 1.1.5