/[pcre]/code/trunk/pcre_jit_compile.c
ViewVC logotype

Contents of /code/trunk/pcre_jit_compile.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 788 - (show annotations)
Tue Dec 6 15:38:01 2011 UTC (3 years, 5 months ago) by ph10
File MIME type: text/plain
File size: 202150 byte(s)
Error occurred while calculating annotation data.
Source tidies for 8.21-RC1
1 /*************************************************
2 * Perl-Compatible Regular Expressions *
3 *************************************************/
4
5 /* PCRE is a library of functions to support regular expressions whose syntax
6 and semantics are as close as possible to those of the Perl 5 language.
7
8 Written by Philip Hazel
9 Copyright (c) 1997-2008 University of Cambridge
10
11 The machine code generator part (this module) was written by Zoltan Herczeg
12 Copyright (c) 2010-2011
13
14 -----------------------------------------------------------------------------
15 Redistribution and use in source and binary forms, with or without
16 modification, are permitted provided that the following conditions are met:
17
18 * Redistributions of source code must retain the above copyright notice,
19 this list of conditions and the following disclaimer.
20
21 * Redistributions in binary form must reproduce the above copyright
22 notice, this list of conditions and the following disclaimer in the
23 documentation and/or other materials provided with the distribution.
24
25 * Neither the name of the University of Cambridge nor the names of its
26 contributors may be used to endorse or promote products derived from
27 this software without specific prior written permission.
28
29 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
36 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
37 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 POSSIBILITY OF SUCH DAMAGE.
40 -----------------------------------------------------------------------------
41 */
42
43 #ifdef HAVE_CONFIG_H
44 #include "config.h"
45 #endif
46
47 #include "pcre_internal.h"
48
49 #ifdef SUPPORT_JIT
50
51 /* All-in-one: Since we use the JIT compiler only from here,
52 we just include it. This way we don't need to touch the build
53 system files. */
54
55 #define SLJIT_MALLOC(size) (pcre_malloc)(size)
56 #define SLJIT_FREE(ptr) (pcre_free)(ptr)
57 #define SLJIT_CONFIG_AUTO 1
58 #define SLJIT_CONFIG_STATIC 1
59 #define SLJIT_VERBOSE 0
60 #define SLJIT_DEBUG 0
61
62 #include "sljit/sljitLir.c"
63
64 #if defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED
65 #error "Unsupported architecture"
66 #endif
67
68 /* Allocate memory on the stack. Fast, but limited size. */
69 #define LOCAL_SPACE_SIZE 32768
70
71 #define STACK_GROWTH_RATE 8192
72
73 /* Enable to check that the allocation could destroy temporaries. */
74 #if defined SLJIT_DEBUG && SLJIT_DEBUG
75 #define DESTROY_REGISTERS 1
76 #endif
77
78 /*
79 Short summary about the backtracking mechanism empolyed by the jit code generator:
80
81 The code generator follows the recursive nature of the PERL compatible regular
82 expressions. The basic blocks of regular expressions are condition checkers
83 whose execute different commands depending on the result of the condition check.
84 The relationship between the operators can be horizontal (concatenation) and
85 vertical (sub-expression) (See struct fallback_common for more details).
86
87 'ab' - 'a' and 'b' regexps are concatenated
88 'a+' - 'a' is the sub-expression of the '+' operator
89
90 The condition checkers are boolean (true/false) checkers. Machine code is generated
91 for the checker itself and for the actions depending on the result of the checker.
92 The 'true' case is called as the hot path (expected path), and the other is called as
93 the 'fallback' path. Branch instructions are expesive for all CPUs, so we avoid taken
94 branches on the hot path.
95
96 Greedy star operator (*) :
97 Hot path: match happens.
98 Fallback path: match failed.
99 Non-greedy star operator (*?) :
100 Hot path: no need to perform a match.
101 Fallback path: match is required.
102
103 The following example shows how the code generated for a capturing bracket
104 with two alternatives. Let A, B, C, D are arbirary regular expressions, and
105 we have the following regular expression:
106
107 A(B|C)D
108
109 The generated code will be the following:
110
111 A hot path
112 '(' hot path (pushing arguments to the stack)
113 B hot path
114 ')' hot path (pushing arguments to the stack)
115 D hot path
116 return with successful match
117
118 D fallback path
119 ')' fallback path (If we arrived from "C" jump to the fallback of "C")
120 B fallback path
121 C expected path
122 jump to D hot path
123 C fallback path
124 A fallback path
125
126 Notice, that the order of fallback code paths are the opposite of the fast
127 code paths. In this way the topmost value on the stack is always belong
128 to the current fallback code path. The fallback code path must check
129 whether there is a next alternative. If so, it needs to jump back to
130 the hot path eventually. Otherwise it needs to clear out its own stack
131 frame and continue the execution on the fallback code paths.
132 */
133
134 /*
135 Saved stack frames:
136
137 Atomic blocks and asserts require reloading the values of local variables
138 when the fallback mechanism performed. Because of OP_RECURSE, the locals
139 are not necessarly known in compile time, thus we need a dynamic restore
140 mechanism.
141
142 The stack frames are stored in a chain list, and have the following format:
143 ([ capturing bracket offset ][ start value ][ end value ])+ ... [ 0 ] [ previous head ]
144
145 Thus we can restore the locals to a particular point in the stack.
146 */
147
148 typedef struct jit_arguments {
149 /* Pointers first. */
150 struct sljit_stack *stack;
151 PCRE_SPTR str;
152 PCRE_SPTR begin;
153 PCRE_SPTR end;
154 int *offsets;
155 uschar *ptr;
156 /* Everything else after. */
157 int offsetcount;
158 int calllimit;
159 uschar notbol;
160 uschar noteol;
161 uschar notempty;
162 uschar notempty_atstart;
163 } jit_arguments;
164
165 typedef struct executable_function {
166 void *executable_func;
167 pcre_jit_callback callback;
168 void *userdata;
169 sljit_uw executable_size;
170 } executable_function;
171
172 typedef struct jump_list {
173 struct sljit_jump *jump;
174 struct jump_list *next;
175 } jump_list;
176
177 enum stub_types { stack_alloc };
178
179 typedef struct stub_list {
180 enum stub_types type;
181 int data;
182 struct sljit_jump *start;
183 struct sljit_label *leave;
184 struct stub_list *next;
185 } stub_list;
186
187 typedef int (SLJIT_CALL *jit_function)(jit_arguments *args);
188
189 /* The following structure is the key data type for the recursive
190 code generator. It is allocated by compile_hotpath, and contains
191 the aguments for compile_fallbackpath. Must be the first member
192 of its descendants. */
193 typedef struct fallback_common {
194 /* Concatenation stack. */
195 struct fallback_common *prev;
196 jump_list *nextfallbacks;
197 /* Internal stack (for component operators). */
198 struct fallback_common *top;
199 jump_list *topfallbacks;
200 /* Opcode pointer. */
201 uschar *cc;
202 } fallback_common;
203
204 typedef struct assert_fallback {
205 fallback_common common;
206 jump_list *condfailed;
207 /* Less than 0 (-1) if a frame is not needed. */
208 int framesize;
209 /* Points to our private memory word on the stack. */
210 int localptr;
211 /* For iterators. */
212 struct sljit_label *hotpath;
213 } assert_fallback;
214
215 typedef struct bracket_fallback {
216 fallback_common common;
217 /* Where to coninue if an alternative is successfully matched. */
218 struct sljit_label *althotpath;
219 /* For rmin and rmax iterators. */
220 struct sljit_label *recursivehotpath;
221 /* For greedy ? operator. */
222 struct sljit_label *zerohotpath;
223 /* Contains the branches of a failed condition. */
224 union {
225 /* Both for OP_COND, OP_SCOND. */
226 jump_list *condfailed;
227 assert_fallback *assert;
228 /* For OP_ONCE. -1 if not needed. */
229 int framesize;
230 } u;
231 /* Points to our private memory word on the stack. */
232 int localptr;
233 } bracket_fallback;
234
235 typedef struct bracketpos_fallback {
236 fallback_common common;
237 /* Points to our private memory word on the stack. */
238 int localptr;
239 /* Reverting stack is needed. */
240 int framesize;
241 /* Allocated stack size. */
242 int stacksize;
243 } bracketpos_fallback;
244
245 typedef struct braminzero_fallback {
246 fallback_common common;
247 struct sljit_label *hotpath;
248 } braminzero_fallback;
249
250 typedef struct iterator_fallback {
251 fallback_common common;
252 /* Next iteration. */
253 struct sljit_label *hotpath;
254 } iterator_fallback;
255
256 typedef struct recurse_entry {
257 struct recurse_entry *next;
258 /* Contains the function entry. */
259 struct sljit_label *entry;
260 /* Collects the calls until the function is not created. */
261 jump_list *calls;
262 /* Points to the starting opcode. */
263 int start;
264 } recurse_entry;
265
266 typedef struct recurse_fallback {
267 fallback_common common;
268 } recurse_fallback;
269
270 typedef struct compiler_common {
271 struct sljit_compiler *compiler;
272 uschar *start;
273 int localsize;
274 int *localptrs;
275 const uschar *fcc;
276 sljit_w lcc;
277 int cbraptr;
278 int nltype;
279 int newline;
280 int bsr_nltype;
281 int endonly;
282 sljit_w ctypes;
283 sljit_uw name_table;
284 sljit_w name_count;
285 sljit_w name_entry_size;
286 struct sljit_label *acceptlabel;
287 stub_list *stubs;
288 recurse_entry *entries;
289 recurse_entry *currententry;
290 jump_list *accept;
291 jump_list *calllimit;
292 jump_list *stackalloc;
293 jump_list *revertframes;
294 jump_list *wordboundary;
295 jump_list *anynewline;
296 jump_list *hspace;
297 jump_list *vspace;
298 jump_list *casefulcmp;
299 jump_list *caselesscmp;
300 BOOL jscript_compat;
301 #ifdef SUPPORT_UTF8
302 BOOL utf8;
303 #ifdef SUPPORT_UCP
304 BOOL useucp;
305 #endif
306 jump_list *utf8readchar;
307 jump_list *utf8readtype8;
308 #endif
309 #ifdef SUPPORT_UCP
310 jump_list *getucd;
311 #endif
312 } compiler_common;
313
314 /* For byte_sequence_compare. */
315
316 typedef struct compare_context {
317 int length;
318 int sourcereg;
319 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
320 int byteptr;
321 union {
322 int asint;
323 short asshort;
324 sljit_ub asbyte;
325 sljit_ub asbytes[4];
326 } c;
327 union {
328 int asint;
329 short asshort;
330 sljit_ub asbyte;
331 sljit_ub asbytes[4];
332 } oc;
333 #endif
334 } compare_context;
335
336 enum {
337 frame_end = 0,
338 frame_setstrbegin = -1
339 };
340
341 /* Used for accessing the elements of the stack. */
342 #define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_w))
343
344 #define TMP1 SLJIT_TEMPORARY_REG1
345 #define TMP2 SLJIT_TEMPORARY_REG3
346 #define TMP3 SLJIT_TEMPORARY_EREG2
347 #define STR_PTR SLJIT_GENERAL_REG1
348 #define STR_END SLJIT_GENERAL_REG2
349 #define STACK_TOP SLJIT_TEMPORARY_REG2
350 #define STACK_LIMIT SLJIT_GENERAL_REG3
351 #define ARGUMENTS SLJIT_GENERAL_EREG1
352 #define CALL_COUNT SLJIT_GENERAL_EREG2
353 #define RETURN_ADDR SLJIT_TEMPORARY_EREG1
354
355 /* Locals layout. */
356 /* These two locals can be used by the current opcode. */
357 #define LOCALS0 (0 * sizeof(sljit_w))
358 #define LOCALS1 (1 * sizeof(sljit_w))
359 /* Two local variables for possessive quantifiers (char1 cannot use them). */
360 #define POSSESSIVE0 (2 * sizeof(sljit_w))
361 #define POSSESSIVE1 (3 * sizeof(sljit_w))
362 /* Head of the last recursion. */
363 #define RECURSIVE_HEAD (4 * sizeof(sljit_w))
364 /* Max limit of recursions. */
365 #define CALL_LIMIT (5 * sizeof(sljit_w))
366 /* Last known position of the requested byte. */
367 #define REQ_BYTE_PTR (6 * sizeof(sljit_w))
368 /* End pointer of the first line. */
369 #define FIRSTLINE_END (7 * sizeof(sljit_w))
370 /* The output vector is stored on the stack, and contains pointers
371 to characters. The vector data is divided into two groups: the first
372 group contains the start / end character pointers, and the second is
373 the start pointers when the end of the capturing group has not yet reached. */
374 #define OVECTOR_START (8 * sizeof(sljit_w))
375 #define OVECTOR(i) (OVECTOR_START + (i) * sizeof(sljit_w))
376 #define OVECTOR_PRIV(i) (common->cbraptr + (i) * sizeof(sljit_w))
377 #define PRIV(cc) (common->localptrs[(cc) - common->start])
378
379 /* Shortcuts. */
380 #define DEFINE_COMPILER \
381 struct sljit_compiler *compiler = common->compiler
382 #define OP1(op, dst, dstw, src, srcw) \
383 sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw))
384 #define OP2(op, dst, dstw, src1, src1w, src2, src2w) \
385 sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w))
386 #define LABEL() \
387 sljit_emit_label(compiler)
388 #define JUMP(type) \
389 sljit_emit_jump(compiler, (type))
390 #define JUMPTO(type, label) \
391 sljit_set_label(sljit_emit_jump(compiler, (type)), (label))
392 #define JUMPHERE(jump) \
393 sljit_set_label((jump), sljit_emit_label(compiler))
394 #define CMP(type, src1, src1w, src2, src2w) \
395 sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w))
396 #define CMPTO(type, src1, src1w, src2, src2w, label) \
397 sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label))
398 #define COND_VALUE(op, dst, dstw, type) \
399 sljit_emit_cond_value(compiler, (op), (dst), (dstw), (type))
400
401 static uschar* bracketend(uschar* cc)
402 {
403 SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
404 do cc += GET(cc, 1); while (*cc == OP_ALT);
405 SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS);
406 cc += 1 + LINK_SIZE;
407 return cc;
408 }
409
410 /* Functions whose might need modification for all new supported opcodes:
411 next_opcode
412 get_localspace
413 set_localptrs
414 get_framesize
415 init_frame
416 get_localsize
417 copy_locals
418 compile_hotpath
419 compile_fallbackpath
420 */
421
422 static uschar *next_opcode(compiler_common *common, uschar *cc)
423 {
424 SLJIT_UNUSED_ARG(common);
425 switch(*cc)
426 {
427 case OP_SOD:
428 case OP_SOM:
429 case OP_SET_SOM:
430 case OP_NOT_WORD_BOUNDARY:
431 case OP_WORD_BOUNDARY:
432 case OP_NOT_DIGIT:
433 case OP_DIGIT:
434 case OP_NOT_WHITESPACE:
435 case OP_WHITESPACE:
436 case OP_NOT_WORDCHAR:
437 case OP_WORDCHAR:
438 case OP_ANY:
439 case OP_ALLANY:
440 case OP_ANYNL:
441 case OP_NOT_HSPACE:
442 case OP_HSPACE:
443 case OP_NOT_VSPACE:
444 case OP_VSPACE:
445 case OP_EXTUNI:
446 case OP_EODN:
447 case OP_EOD:
448 case OP_CIRC:
449 case OP_CIRCM:
450 case OP_DOLL:
451 case OP_DOLLM:
452 case OP_TYPESTAR:
453 case OP_TYPEMINSTAR:
454 case OP_TYPEPLUS:
455 case OP_TYPEMINPLUS:
456 case OP_TYPEQUERY:
457 case OP_TYPEMINQUERY:
458 case OP_TYPEPOSSTAR:
459 case OP_TYPEPOSPLUS:
460 case OP_TYPEPOSQUERY:
461 case OP_CRSTAR:
462 case OP_CRMINSTAR:
463 case OP_CRPLUS:
464 case OP_CRMINPLUS:
465 case OP_CRQUERY:
466 case OP_CRMINQUERY:
467 case OP_DEF:
468 case OP_BRAZERO:
469 case OP_BRAMINZERO:
470 case OP_BRAPOSZERO:
471 case OP_FAIL:
472 case OP_ACCEPT:
473 case OP_ASSERT_ACCEPT:
474 case OP_SKIPZERO:
475 return cc + 1;
476
477 case OP_ANYBYTE:
478 #ifdef SUPPORT_UTF8
479 if (common->utf8) return NULL;
480 #endif
481 return cc + 1;
482
483 case OP_CHAR:
484 case OP_CHARI:
485 case OP_NOT:
486 case OP_NOTI:
487
488 case OP_STAR:
489 case OP_MINSTAR:
490 case OP_PLUS:
491 case OP_MINPLUS:
492 case OP_QUERY:
493 case OP_MINQUERY:
494 case OP_POSSTAR:
495 case OP_POSPLUS:
496 case OP_POSQUERY:
497 case OP_STARI:
498 case OP_MINSTARI:
499 case OP_PLUSI:
500 case OP_MINPLUSI:
501 case OP_QUERYI:
502 case OP_MINQUERYI:
503 case OP_POSSTARI:
504 case OP_POSPLUSI:
505 case OP_POSQUERYI:
506 case OP_NOTSTAR:
507 case OP_NOTMINSTAR:
508 case OP_NOTPLUS:
509 case OP_NOTMINPLUS:
510 case OP_NOTQUERY:
511 case OP_NOTMINQUERY:
512 case OP_NOTPOSSTAR:
513 case OP_NOTPOSPLUS:
514 case OP_NOTPOSQUERY:
515 case OP_NOTSTARI:
516 case OP_NOTMINSTARI:
517 case OP_NOTPLUSI:
518 case OP_NOTMINPLUSI:
519 case OP_NOTQUERYI:
520 case OP_NOTMINQUERYI:
521 case OP_NOTPOSSTARI:
522 case OP_NOTPOSPLUSI:
523 case OP_NOTPOSQUERYI:
524 cc += 2;
525 #ifdef SUPPORT_UTF8
526 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
527 #endif
528 return cc;
529
530 case OP_UPTO:
531 case OP_MINUPTO:
532 case OP_EXACT:
533 case OP_POSUPTO:
534 case OP_UPTOI:
535 case OP_MINUPTOI:
536 case OP_EXACTI:
537 case OP_POSUPTOI:
538 case OP_NOTUPTO:
539 case OP_NOTMINUPTO:
540 case OP_NOTEXACT:
541 case OP_NOTPOSUPTO:
542 case OP_NOTUPTOI:
543 case OP_NOTMINUPTOI:
544 case OP_NOTEXACTI:
545 case OP_NOTPOSUPTOI:
546 cc += 4;
547 #ifdef SUPPORT_UTF8
548 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
549 #endif
550 return cc;
551
552 case OP_NOTPROP:
553 case OP_PROP:
554 case OP_TYPEUPTO:
555 case OP_TYPEMINUPTO:
556 case OP_TYPEEXACT:
557 case OP_TYPEPOSUPTO:
558 case OP_REF:
559 case OP_REFI:
560 case OP_CREF:
561 case OP_NCREF:
562 case OP_RREF:
563 case OP_NRREF:
564 case OP_CLOSE:
565 cc += 3;
566 return cc;
567
568 case OP_CRRANGE:
569 case OP_CRMINRANGE:
570 return cc + 5;
571
572 case OP_CLASS:
573 case OP_NCLASS:
574 return cc + 33;
575
576 #ifdef SUPPORT_UTF8
577 case OP_XCLASS:
578 return cc + GET(cc, 1);
579 #endif
580
581 case OP_RECURSE:
582 case OP_ASSERT:
583 case OP_ASSERT_NOT:
584 case OP_ASSERTBACK:
585 case OP_ASSERTBACK_NOT:
586 case OP_REVERSE:
587 case OP_ONCE:
588 case OP_ONCE_NC:
589 case OP_BRA:
590 case OP_BRAPOS:
591 case OP_COND:
592 case OP_SBRA:
593 case OP_SBRAPOS:
594 case OP_SCOND:
595 case OP_ALT:
596 case OP_KET:
597 case OP_KETRMAX:
598 case OP_KETRMIN:
599 case OP_KETRPOS:
600 return cc + 1 + LINK_SIZE;
601
602 case OP_CBRA:
603 case OP_CBRAPOS:
604 case OP_SCBRA:
605 case OP_SCBRAPOS:
606 return cc + 1 + LINK_SIZE + 2;
607
608 default:
609 return NULL;
610 }
611 }
612
613 static int get_localspace(compiler_common *common, uschar *cc, uschar *ccend)
614 {
615 int localspace = 0;
616 uschar *alternative;
617 /* Calculate important variables (like stack size) and checks whether all opcodes are supported. */
618 while (cc < ccend)
619 {
620 switch(*cc)
621 {
622 case OP_ASSERT:
623 case OP_ASSERT_NOT:
624 case OP_ASSERTBACK:
625 case OP_ASSERTBACK_NOT:
626 case OP_ONCE:
627 case OP_ONCE_NC:
628 case OP_BRAPOS:
629 case OP_SBRA:
630 case OP_SBRAPOS:
631 case OP_SCOND:
632 localspace += sizeof(sljit_w);
633 cc += 1 + LINK_SIZE;
634 break;
635
636 case OP_CBRAPOS:
637 case OP_SCBRAPOS:
638 localspace += sizeof(sljit_w);
639 cc += 1 + LINK_SIZE + 2;
640 break;
641
642 case OP_COND:
643 /* Might be a hidden SCOND. */
644 alternative = cc + GET(cc, 1);
645 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
646 localspace += sizeof(sljit_w);
647 cc += 1 + LINK_SIZE;
648 break;
649
650 default:
651 cc = next_opcode(common, cc);
652 if (cc == NULL)
653 return -1;
654 break;
655 }
656 }
657 return localspace;
658 }
659
660 static void set_localptrs(compiler_common *common, int localptr, uschar *ccend)
661 {
662 uschar *cc = common->start;
663 uschar *alternative;
664 while (cc < ccend)
665 {
666 switch(*cc)
667 {
668 case OP_ASSERT:
669 case OP_ASSERT_NOT:
670 case OP_ASSERTBACK:
671 case OP_ASSERTBACK_NOT:
672 case OP_ONCE:
673 case OP_ONCE_NC:
674 case OP_BRAPOS:
675 case OP_SBRA:
676 case OP_SBRAPOS:
677 case OP_SCOND:
678 common->localptrs[cc - common->start] = localptr;
679 localptr += sizeof(sljit_w);
680 cc += 1 + LINK_SIZE;
681 break;
682
683 case OP_CBRAPOS:
684 case OP_SCBRAPOS:
685 common->localptrs[cc - common->start] = localptr;
686 localptr += sizeof(sljit_w);
687 cc += 1 + LINK_SIZE + 2;
688 break;
689
690 case OP_COND:
691 /* Might be a hidden SCOND. */
692 alternative = cc + GET(cc, 1);
693 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
694 {
695 common->localptrs[cc - common->start] = localptr;
696 localptr += sizeof(sljit_w);
697 }
698 cc += 1 + LINK_SIZE;
699 break;
700
701 default:
702 cc = next_opcode(common, cc);
703 SLJIT_ASSERT(cc != NULL);
704 break;
705 }
706 }
707 }
708
709 /* Returns with -1 if no need for frame. */
710 static int get_framesize(compiler_common *common, uschar *cc, BOOL recursive)
711 {
712 uschar *ccend = bracketend(cc);
713 int length = 0;
714 BOOL possessive = FALSE;
715 BOOL setsom_found = FALSE;
716
717 if (!recursive && (*cc == OP_CBRAPOS || *cc == OP_SCBRAPOS))
718 {
719 length = 3;
720 possessive = TRUE;
721 }
722
723 cc = next_opcode(common, cc);
724 SLJIT_ASSERT(cc != NULL);
725 while (cc < ccend)
726 switch(*cc)
727 {
728 case OP_SET_SOM:
729 case OP_RECURSE:
730 if (!setsom_found)
731 {
732 length += 2;
733 setsom_found = TRUE;
734 }
735 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
736 break;
737
738 case OP_CBRA:
739 case OP_CBRAPOS:
740 case OP_SCBRA:
741 case OP_SCBRAPOS:
742 length += 3;
743 cc += 1 + LINK_SIZE + 2;
744 break;
745
746 default:
747 cc = next_opcode(common, cc);
748 SLJIT_ASSERT(cc != NULL);
749 break;
750 }
751
752 /* Possessive quantifiers can use a special case. */
753 if (SLJIT_UNLIKELY(possessive) && length == 3)
754 return -1;
755
756 if (length > 0)
757 return length + 1;
758 return -1;
759 }
760
761 static void init_frame(compiler_common *common, uschar *cc, int stackpos, int stacktop, BOOL recursive)
762 {
763 DEFINE_COMPILER;
764 uschar *ccend = bracketend(cc);
765 BOOL setsom_found = FALSE;
766 int offset;
767
768 /* >= 1 + shortest item size (2) */
769 SLJIT_ASSERT(stackpos >= stacktop + 2);
770
771 stackpos = STACK(stackpos);
772 if (recursive || (*cc != OP_CBRAPOS && *cc != OP_SCBRAPOS))
773 cc = next_opcode(common, cc);
774 SLJIT_ASSERT(cc != NULL);
775 while (cc < ccend)
776 switch(*cc)
777 {
778 case OP_SET_SOM:
779 case OP_RECURSE:
780 if (!setsom_found)
781 {
782 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
783 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_setstrbegin);
784 stackpos += (int)sizeof(sljit_w);
785 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
786 stackpos += (int)sizeof(sljit_w);
787 setsom_found = TRUE;
788 }
789 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
790 break;
791
792 case OP_CBRA:
793 case OP_CBRAPOS:
794 case OP_SCBRA:
795 case OP_SCBRAPOS:
796 offset = (GET2(cc, 1 + LINK_SIZE)) << 1;
797 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset));
798 stackpos += (int)sizeof(sljit_w);
799 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
800 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
801 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
802 stackpos += (int)sizeof(sljit_w);
803 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0);
804 stackpos += (int)sizeof(sljit_w);
805
806 cc += 1 + LINK_SIZE + 2;
807 break;
808
809 default:
810 cc = next_opcode(common, cc);
811 SLJIT_ASSERT(cc != NULL);
812 break;
813 }
814
815 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_end);
816 SLJIT_ASSERT(stackpos == STACK(stacktop));
817 }
818
819 static SLJIT_INLINE int get_localsize(compiler_common *common, uschar *cc, uschar *ccend)
820 {
821 int localsize = 2;
822 uschar *alternative;
823 /* Calculate the sum of the local variables. */
824 while (cc < ccend)
825 {
826 switch(*cc)
827 {
828 case OP_ASSERT:
829 case OP_ASSERT_NOT:
830 case OP_ASSERTBACK:
831 case OP_ASSERTBACK_NOT:
832 case OP_ONCE:
833 case OP_ONCE_NC:
834 case OP_BRAPOS:
835 case OP_SBRA:
836 case OP_SBRAPOS:
837 case OP_SCOND:
838 localsize++;
839 cc += 1 + LINK_SIZE;
840 break;
841
842 case OP_CBRA:
843 case OP_SCBRA:
844 localsize++;
845 cc += 1 + LINK_SIZE + 2;
846 break;
847
848 case OP_CBRAPOS:
849 case OP_SCBRAPOS:
850 localsize += 2;
851 cc += 1 + LINK_SIZE + 2;
852 break;
853
854 case OP_COND:
855 /* Might be a hidden SCOND. */
856 alternative = cc + GET(cc, 1);
857 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
858 localsize++;
859 cc += 1 + LINK_SIZE;
860 break;
861
862 default:
863 cc = next_opcode(common, cc);
864 SLJIT_ASSERT(cc != NULL);
865 break;
866 }
867 }
868 SLJIT_ASSERT(cc == ccend);
869 return localsize;
870 }
871
872 static void copy_locals(compiler_common *common, uschar *cc, uschar *ccend,
873 BOOL save, int stackptr, int stacktop)
874 {
875 DEFINE_COMPILER;
876 int srcw[2];
877 int count;
878 BOOL tmp1next = TRUE;
879 BOOL tmp1empty = TRUE;
880 BOOL tmp2empty = TRUE;
881 uschar *alternative;
882 enum {
883 start,
884 loop,
885 end
886 } status;
887
888 status = save ? start : loop;
889 stackptr = STACK(stackptr - 2);
890 stacktop = STACK(stacktop - 1);
891
892 if (!save)
893 {
894 stackptr += sizeof(sljit_w);
895 if (stackptr < stacktop)
896 {
897 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
898 stackptr += sizeof(sljit_w);
899 tmp1empty = FALSE;
900 }
901 if (stackptr < stacktop)
902 {
903 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
904 stackptr += sizeof(sljit_w);
905 tmp2empty = FALSE;
906 }
907 /* The tmp1next must be TRUE in either way. */
908 }
909
910 while (status != end)
911 {
912 count = 0;
913 switch(status)
914 {
915 case start:
916 SLJIT_ASSERT(save);
917 count = 1;
918 srcw[0] = RECURSIVE_HEAD;
919 status = loop;
920 break;
921
922 case loop:
923 if (cc >= ccend)
924 {
925 status = end;
926 break;
927 }
928
929 switch(*cc)
930 {
931 case OP_ASSERT:
932 case OP_ASSERT_NOT:
933 case OP_ASSERTBACK:
934 case OP_ASSERTBACK_NOT:
935 case OP_ONCE:
936 case OP_ONCE_NC:
937 case OP_BRAPOS:
938 case OP_SBRA:
939 case OP_SBRAPOS:
940 case OP_SCOND:
941 count = 1;
942 srcw[0] = PRIV(cc);
943 SLJIT_ASSERT(srcw[0] != 0);
944 cc += 1 + LINK_SIZE;
945 break;
946
947 case OP_CBRA:
948 case OP_SCBRA:
949 count = 1;
950 srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
951 cc += 1 + LINK_SIZE + 2;
952 break;
953
954 case OP_CBRAPOS:
955 case OP_SCBRAPOS:
956 count = 2;
957 srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
958 srcw[0] = PRIV(cc);
959 SLJIT_ASSERT(srcw[0] != 0);
960 cc += 1 + LINK_SIZE + 2;
961 break;
962
963 case OP_COND:
964 /* Might be a hidden SCOND. */
965 alternative = cc + GET(cc, 1);
966 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
967 {
968 count = 1;
969 srcw[0] = PRIV(cc);
970 SLJIT_ASSERT(srcw[0] != 0);
971 }
972 cc += 1 + LINK_SIZE;
973 break;
974
975 default:
976 cc = next_opcode(common, cc);
977 SLJIT_ASSERT(cc != NULL);
978 break;
979 }
980 break;
981
982 case end:
983 SLJIT_ASSERT_STOP();
984 break;
985 }
986
987 while (count > 0)
988 {
989 count--;
990 if (save)
991 {
992 if (tmp1next)
993 {
994 if (!tmp1empty)
995 {
996 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
997 stackptr += sizeof(sljit_w);
998 }
999 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
1000 tmp1empty = FALSE;
1001 tmp1next = FALSE;
1002 }
1003 else
1004 {
1005 if (!tmp2empty)
1006 {
1007 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1008 stackptr += sizeof(sljit_w);
1009 }
1010 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
1011 tmp2empty = FALSE;
1012 tmp1next = TRUE;
1013 }
1014 }
1015 else
1016 {
1017 if (tmp1next)
1018 {
1019 SLJIT_ASSERT(!tmp1empty);
1020 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP1, 0);
1021 tmp1empty = stackptr >= stacktop;
1022 if (!tmp1empty)
1023 {
1024 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1025 stackptr += sizeof(sljit_w);
1026 }
1027 tmp1next = FALSE;
1028 }
1029 else
1030 {
1031 SLJIT_ASSERT(!tmp2empty);
1032 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP2, 0);
1033 tmp2empty = stackptr >= stacktop;
1034 if (!tmp2empty)
1035 {
1036 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1037 stackptr += sizeof(sljit_w);
1038 }
1039 tmp1next = TRUE;
1040 }
1041 }
1042 }
1043 }
1044
1045 if (save)
1046 {
1047 if (tmp1next)
1048 {
1049 if (!tmp1empty)
1050 {
1051 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1052 stackptr += sizeof(sljit_w);
1053 }
1054 if (!tmp2empty)
1055 {
1056 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1057 stackptr += sizeof(sljit_w);
1058 }
1059 }
1060 else
1061 {
1062 if (!tmp2empty)
1063 {
1064 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1065 stackptr += sizeof(sljit_w);
1066 }
1067 if (!tmp1empty)
1068 {
1069 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1070 stackptr += sizeof(sljit_w);
1071 }
1072 }
1073 }
1074 SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty)));
1075 }
1076
1077 static SLJIT_INLINE BOOL ispowerof2(unsigned int value)
1078 {
1079 return (value & (value - 1)) == 0;
1080 }
1081
1082 static SLJIT_INLINE void set_jumps(jump_list *list, struct sljit_label *label)
1083 {
1084 while (list)
1085 {
1086 /* sljit_set_label is clever enough to do nothing
1087 if either the jump or the label is NULL */
1088 sljit_set_label(list->jump, label);
1089 list = list->next;
1090 }
1091 }
1092
1093 static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump* jump)
1094 {
1095 jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list));
1096 if (list_item)
1097 {
1098 list_item->next = *list;
1099 list_item->jump = jump;
1100 *list = list_item;
1101 }
1102 }
1103
1104 static void add_stub(compiler_common *common, enum stub_types type, int data, struct sljit_jump *start)
1105 {
1106 DEFINE_COMPILER;
1107 stub_list* list_item = sljit_alloc_memory(compiler, sizeof(stub_list));
1108
1109 if (list_item)
1110 {
1111 list_item->type = type;
1112 list_item->data = data;
1113 list_item->start = start;
1114 list_item->leave = LABEL();
1115 list_item->next = common->stubs;
1116 common->stubs = list_item;
1117 }
1118 }
1119
1120 static void flush_stubs(compiler_common *common)
1121 {
1122 DEFINE_COMPILER;
1123 stub_list* list_item = common->stubs;
1124
1125 while (list_item)
1126 {
1127 JUMPHERE(list_item->start);
1128 switch(list_item->type)
1129 {
1130 case stack_alloc:
1131 add_jump(compiler, &common->stackalloc, JUMP(SLJIT_FAST_CALL));
1132 break;
1133 }
1134 JUMPTO(SLJIT_JUMP, list_item->leave);
1135 list_item = list_item->next;
1136 }
1137 common->stubs = NULL;
1138 }
1139
1140 static SLJIT_INLINE void decrease_call_count(compiler_common *common)
1141 {
1142 DEFINE_COMPILER;
1143
1144 OP2(SLJIT_SUB | SLJIT_SET_E, CALL_COUNT, 0, CALL_COUNT, 0, SLJIT_IMM, 1);
1145 add_jump(compiler, &common->calllimit, JUMP(SLJIT_C_ZERO));
1146 }
1147
1148 static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
1149 {
1150 /* May destroy all locals and registers except TMP2. */
1151 DEFINE_COMPILER;
1152
1153 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1154 #ifdef DESTROY_REGISTERS
1155 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345);
1156 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
1157 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
1158 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, TMP1, 0);
1159 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
1160 #endif
1161 add_stub(common, stack_alloc, 0, CMP(SLJIT_C_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
1162 }
1163
1164 static SLJIT_INLINE void free_stack(compiler_common *common, int size)
1165 {
1166 DEFINE_COMPILER;
1167 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1168 }
1169
1170 static SLJIT_INLINE void reset_ovector(compiler_common *common, int length)
1171 {
1172 DEFINE_COMPILER;
1173 struct sljit_label *loop;
1174 int i;
1175 /* At this point we can freely use all temporary registers. */
1176 /* TMP1 returns with begin - 1. */
1177 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, 1);
1178 if (length < 8)
1179 {
1180 for (i = 0; i < length; i++)
1181 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(i), SLJIT_TEMPORARY_REG1, 0);
1182 }
1183 else
1184 {
1185 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START - sizeof(sljit_w));
1186 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, length);
1187 loop = LABEL();
1188 OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), sizeof(sljit_w), SLJIT_TEMPORARY_REG1, 0);
1189 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG3, 0, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, 1);
1190 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1191 }
1192 }
1193
1194 static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket)
1195 {
1196 DEFINE_COMPILER;
1197 struct sljit_label *loop;
1198 struct sljit_jump *earlyexit;
1199
1200 /* At this point we can freely use all registers. */
1201 OP1(SLJIT_MOV, SLJIT_GENERAL_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
1202 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1), STR_PTR, 0);
1203
1204 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, ARGUMENTS, 0);
1205 OP1(SLJIT_MOV_SI, SLJIT_TEMPORARY_REG2, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsetcount));
1206 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
1207 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, begin));
1208 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START);
1209 /* Unlikely, but possible */
1210 earlyexit = CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 0);
1211 loop = LABEL();
1212 OP2(SLJIT_SUB, SLJIT_GENERAL_REG2, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), 0, SLJIT_TEMPORARY_REG1, 0);
1213 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_GENERAL_REG1, 0, SLJIT_IMM, sizeof(sljit_w));
1214 /* Copy the integer value to the output buffer */
1215 OP1(SLJIT_MOVU_SI, SLJIT_MEM1(SLJIT_TEMPORARY_REG3), sizeof(int), SLJIT_GENERAL_REG2, 0);
1216 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1217 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1218 JUMPHERE(earlyexit);
1219
1220 /* Calculate the return value, which is the maximum ovector value. */
1221 if (topbracket > 1)
1222 {
1223 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START + topbracket * 2 * sizeof(sljit_w));
1224 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, topbracket + 1);
1225
1226 /* OVECTOR(0) is never equal to SLJIT_GENERAL_REG3. */
1227 loop = LABEL();
1228 OP1(SLJIT_MOVU, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), -(2 * (sljit_w)sizeof(sljit_w)));
1229 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1230 CMPTO(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG3, 0, SLJIT_GENERAL_REG3, 0, loop);
1231 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_TEMPORARY_REG2, 0);
1232 }
1233 else
1234 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
1235 }
1236
1237 static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, uschar* cc)
1238 {
1239 /* Detects if the character has an othercase. */
1240 unsigned int c;
1241
1242 #ifdef SUPPORT_UTF8
1243 if (common->utf8)
1244 {
1245 GETCHAR(c, cc);
1246 if (c > 127)
1247 {
1248 #ifdef SUPPORT_UCP
1249 return c != UCD_OTHERCASE(c);
1250 #else
1251 return FALSE;
1252 #endif
1253 }
1254 }
1255 else
1256 #endif
1257 c = *cc;
1258 return common->fcc[c] != c;
1259 }
1260
1261 static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigned int c)
1262 {
1263 /* Returns with the othercase. */
1264 #ifdef SUPPORT_UTF8
1265 if (common->utf8 && c > 127)
1266 {
1267 #ifdef SUPPORT_UCP
1268 return UCD_OTHERCASE(c);
1269 #else
1270 return c;
1271 #endif
1272 }
1273 #endif
1274 return common->fcc[c];
1275 }
1276
1277 static unsigned int char_get_othercase_bit(compiler_common *common, uschar* cc)
1278 {
1279 /* Detects if the character and its othercase has only 1 bit difference. */
1280 unsigned int c, oc, bit;
1281 #ifdef SUPPORT_UTF8
1282 int n;
1283 #endif
1284
1285 #ifdef SUPPORT_UTF8
1286 if (common->utf8)
1287 {
1288 GETCHAR(c, cc);
1289 if (c <= 127)
1290 oc = common->fcc[c];
1291 else
1292 {
1293 #ifdef SUPPORT_UCP
1294 oc = UCD_OTHERCASE(c);
1295 #else
1296 oc = c;
1297 #endif
1298 }
1299 }
1300 else
1301 {
1302 c = *cc;
1303 oc = common->fcc[c];
1304 }
1305 #else
1306 c = *cc;
1307 oc = common->fcc[c];
1308 #endif
1309
1310 SLJIT_ASSERT(c != oc);
1311
1312 bit = c ^ oc;
1313 /* Optimized for English alphabet. */
1314 if (c <= 127 && bit == 0x20)
1315 return (0 << 8) | 0x20;
1316
1317 /* Since c != oc, they must have at least 1 bit difference. */
1318 if (!ispowerof2(bit))
1319 return 0;
1320
1321 #ifdef SUPPORT_UTF8
1322 if (common->utf8 && c > 127)
1323 {
1324 n = _pcre_utf8_table4[*cc & 0x3f];
1325 while ((bit & 0x3f) == 0)
1326 {
1327 n--;
1328 bit >>= 6;
1329 }
1330 return (n << 8) | bit;
1331 }
1332 #endif
1333 return (0 << 8) | bit;
1334 }
1335
1336 static SLJIT_INLINE void check_input_end(compiler_common *common, jump_list **fallbacks)
1337 {
1338 DEFINE_COMPILER;
1339 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1340 }
1341
1342 static void read_char(compiler_common *common)
1343 {
1344 /* Reads the character into TMP1, updates STR_PTR.
1345 Does not check STR_END. TMP2 Destroyed. */
1346 DEFINE_COMPILER;
1347 #ifdef SUPPORT_UTF8
1348 struct sljit_jump *jump;
1349 #endif
1350
1351 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1352 #ifdef SUPPORT_UTF8
1353 if (common->utf8)
1354 {
1355 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1356 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1357 JUMPHERE(jump);
1358 }
1359 #endif
1360 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1361 }
1362
1363 static void peek_char(compiler_common *common)
1364 {
1365 /* Reads the character into TMP1, keeps STR_PTR.
1366 Does not check STR_END. TMP2 Destroyed. */
1367 DEFINE_COMPILER;
1368 #ifdef SUPPORT_UTF8
1369 struct sljit_jump *jump;
1370 #endif
1371
1372 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1373 #ifdef SUPPORT_UTF8
1374 if (common->utf8)
1375 {
1376 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1377 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1378 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1379 JUMPHERE(jump);
1380 }
1381 #endif
1382 }
1383
1384 static void read_char8_type(compiler_common *common)
1385 {
1386 /* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
1387 DEFINE_COMPILER;
1388 #ifdef SUPPORT_UTF8
1389 struct sljit_jump *jump;
1390 #endif
1391
1392 #ifdef SUPPORT_UTF8
1393 if (common->utf8)
1394 {
1395 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1396 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1397 /* This can be an extra read in some situations, but hopefully
1398 it is a clever early read in most cases. */
1399 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1400 jump = CMP(SLJIT_C_LESS, TMP2, 0, SLJIT_IMM, 0xc0);
1401 add_jump(compiler, &common->utf8readtype8, JUMP(SLJIT_FAST_CALL));
1402 JUMPHERE(jump);
1403 return;
1404 }
1405 #endif
1406 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1407 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1408 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
1409 }
1410
1411 static void skip_char_back(compiler_common *common)
1412 {
1413 /* Goes one character back. Only affects STR_PTR. Does not check begin. */
1414 DEFINE_COMPILER;
1415 #ifdef SUPPORT_UTF8
1416 struct sljit_label *label;
1417
1418 if (common->utf8)
1419 {
1420 label = LABEL();
1421 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1422 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1423 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
1424 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label);
1425 return;
1426 }
1427 #endif
1428 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1429 }
1430
1431 static void check_newlinechar(compiler_common *common, int nltype, jump_list **fallbacks, BOOL jumpiftrue)
1432 {
1433 /* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */
1434 DEFINE_COMPILER;
1435
1436 if (nltype == NLTYPE_ANY)
1437 {
1438 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
1439 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1440 }
1441 else if (nltype == NLTYPE_ANYCRLF)
1442 {
1443 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_CR);
1444 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1445 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1446 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1447 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1448 }
1449 else
1450 {
1451 SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline <= 255);
1452 add_jump(compiler, fallbacks, CMP(jumpiftrue ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
1453 }
1454 }
1455
1456 #ifdef SUPPORT_UTF8
1457 static void do_utf8readchar(compiler_common *common)
1458 {
1459 /* Fast decoding an utf8 character. TMP1 contains the first byte
1460 of the character (>= 0xc0). Return char value in TMP1, length - 1 in TMP2. */
1461 DEFINE_COMPILER;
1462 struct sljit_jump *jump;
1463
1464 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1465 /* Searching for the first zero. */
1466 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
1467 jump = JUMP(SLJIT_C_NOT_ZERO);
1468 /* 2 byte sequence */
1469 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1470 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1471 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1f);
1472 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
1473 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1474 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1475 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
1476 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1477 JUMPHERE(jump);
1478
1479 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10);
1480 jump = JUMP(SLJIT_C_NOT_ZERO);
1481 /* 3 byte sequence */
1482 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1483 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0f);
1484 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 12);
1485 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1486 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1487 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1488 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1489 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 2);
1490 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1491 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1492 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 2);
1493 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1494 JUMPHERE(jump);
1495
1496 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x08);
1497 jump = JUMP(SLJIT_C_NOT_ZERO);
1498 /* 4 byte sequence */
1499 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1500 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x07);
1501 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 18);
1502 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1503 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1504 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1505 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1506 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1507 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1508 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1509 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1510 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 3);
1511 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1512 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1513 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 3);
1514 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1515 JUMPHERE(jump);
1516
1517 /* 5 byte sequence */
1518 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1519 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x03);
1520 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 24);
1521 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1522 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 18);
1523 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1524 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1525 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1526 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1527 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1528 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1529 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1530 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1531 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1532 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 4);
1533 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 4);
1534 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1535 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1536 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 4);
1537 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1538 }
1539
1540 static void do_utf8readtype8(compiler_common *common)
1541 {
1542 /* Fast decoding an utf8 character type. TMP2 contains the first byte
1543 of the character (>= 0xc0) and TMP1 is destroyed. Return value in TMP1. */
1544 DEFINE_COMPILER;
1545 struct sljit_jump *jump;
1546 struct sljit_jump *compare;
1547
1548 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1549
1550 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
1551 jump = JUMP(SLJIT_C_NOT_ZERO);
1552 /* 2 byte sequence */
1553 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1554 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1555 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f);
1556 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1557 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
1558 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0);
1559 compare = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
1560 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1561 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1562
1563 JUMPHERE(compare);
1564 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1565 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1566 JUMPHERE(jump);
1567
1568 /* We only have types for characters less than 256. */
1569 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1570 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1571 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1572 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1573 }
1574
1575 #endif
1576
1577 #ifdef SUPPORT_UCP
1578
1579 /* UCD_BLOCK_SIZE must be 128 (see the assert below). */
1580 #define UCD_BLOCK_MASK 127
1581 #define UCD_BLOCK_SHIFT 7
1582
1583 static void do_getucd(compiler_common *common)
1584 {
1585 /* Search the UCD record for the character comes in TMP1.
1586 Returns chartype in TMP1 and UCD offset in TMP2. */
1587 DEFINE_COMPILER;
1588
1589 SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8);
1590
1591 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1592 OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1593 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_ucd_stage1);
1594 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
1595 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1596 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
1597 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_stage2);
1598 OP1(SLJIT_MOV_UH, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
1599 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, chartype));
1600 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
1601 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1602 }
1603 #endif
1604
1605 static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf, BOOL firstline)
1606 {
1607 DEFINE_COMPILER;
1608 struct sljit_label *mainloop;
1609 struct sljit_label *newlinelabel = NULL;
1610 struct sljit_jump *start;
1611 struct sljit_jump *end = NULL;
1612 struct sljit_jump *nl = NULL;
1613 #ifdef SUPPORT_UTF8
1614 struct sljit_jump *singlebyte;
1615 #endif
1616 jump_list *newline = NULL;
1617 BOOL newlinecheck = FALSE;
1618 BOOL readbyte = FALSE;
1619
1620 if (!(hascrorlf || firstline) && (common->nltype == NLTYPE_ANY ||
1621 common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
1622 newlinecheck = TRUE;
1623
1624 if (firstline)
1625 {
1626 /* Search for the end of the first line. */
1627 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STR_PTR, 0);
1628 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_END, 0);
1629
1630 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1631 {
1632 mainloop = LABEL();
1633 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1634 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1635 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
1636 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1637 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
1638 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
1639 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0, SLJIT_IMM, 1);
1640 }
1641 else
1642 {
1643 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1644 mainloop = LABEL();
1645 /* Continual stores does not cause data dependency. */
1646 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1647 read_char(common);
1648 check_newlinechar(common, common->nltype, &newline, TRUE);
1649 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
1650 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1651 set_jumps(newline, LABEL());
1652 }
1653
1654 JUMPHERE(end);
1655 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
1656 }
1657
1658 start = JUMP(SLJIT_JUMP);
1659
1660 if (newlinecheck)
1661 {
1662 newlinelabel = LABEL();
1663 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1664 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1665 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1666 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
1667 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1668 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1669 nl = JUMP(SLJIT_JUMP);
1670 }
1671
1672 mainloop = LABEL();
1673
1674 /* Increasing the STR_PTR here requires one less jump in the most common case. */
1675 #ifdef SUPPORT_UTF8
1676 if (common->utf8) readbyte = TRUE;
1677 #endif
1678 if (newlinecheck) readbyte = TRUE;
1679
1680 if (readbyte)
1681 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1682
1683 if (newlinecheck)
1684 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);
1685
1686 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1687 #ifdef SUPPORT_UTF8
1688 if (common->utf8)
1689 {
1690 singlebyte = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1691 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1692 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1693 JUMPHERE(singlebyte);
1694 }
1695 #endif
1696 JUMPHERE(start);
1697
1698 if (newlinecheck)
1699 {
1700 JUMPHERE(end);
1701 JUMPHERE(nl);
1702 }
1703
1704 return mainloop;
1705 }
1706
1707 static SLJIT_INLINE void fast_forward_first_byte(compiler_common *common, pcre_uint16 firstbyte, BOOL firstline)
1708 {
1709 DEFINE_COMPILER;
1710 struct sljit_label *start;
1711 struct sljit_jump *leave;
1712 struct sljit_jump *found;
1713 pcre_uint16 oc, bit;
1714
1715 if (firstline)
1716 {
1717 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1718 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1719 }
1720
1721 start = LABEL();
1722 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1723 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1724
1725 if ((firstbyte & REQ_CASELESS) == 0)
1726 found = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, firstbyte & 0xff);
1727 else
1728 {
1729 firstbyte &= 0xff;
1730 oc = common->fcc[firstbyte];
1731 bit = firstbyte ^ oc;
1732 if (ispowerof2(bit))
1733 {
1734 OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, bit);
1735 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, firstbyte | bit);
1736 }
1737 else
1738 {
1739 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, firstbyte);
1740 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1741 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc);
1742 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1743 found = JUMP(SLJIT_C_NOT_ZERO);
1744 }
1745 }
1746
1747 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1748 #ifdef SUPPORT_UTF8
1749 if (common->utf8)
1750 {
1751 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
1752 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1753 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1754 }
1755 #endif
1756 JUMPTO(SLJIT_JUMP, start);
1757 JUMPHERE(found);
1758 JUMPHERE(leave);
1759
1760 if (firstline)
1761 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1762 }
1763
1764 static SLJIT_INLINE void fast_forward_newline(compiler_common *common, BOOL firstline)
1765 {
1766 DEFINE_COMPILER;
1767 struct sljit_label *loop;
1768 struct sljit_jump *lastchar;
1769 struct sljit_jump *firstchar;
1770 struct sljit_jump *leave;
1771 struct sljit_jump *foundcr = NULL;
1772 struct sljit_jump *notfoundnl;
1773 jump_list *newline = NULL;
1774
1775 if (firstline)
1776 {
1777 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1778 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1779 }
1780
1781 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1782 {
1783 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1784 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1785 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1786 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
1787 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1788
1789 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2);
1790 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
1791 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER_EQUAL);
1792 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1793
1794 loop = LABEL();
1795 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1796 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1797 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
1798 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
1799 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
1800 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);
1801
1802 JUMPHERE(leave);
1803 JUMPHERE(firstchar);
1804 JUMPHERE(lastchar);
1805
1806 if (firstline)
1807 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1808 return;
1809 }
1810
1811 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1812 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1813 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1814 skip_char_back(common);
1815
1816 loop = LABEL();
1817 read_char(common);
1818 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1819 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1820 foundcr = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
1821 check_newlinechar(common, common->nltype, &newline, FALSE);
1822 set_jumps(newline, loop);
1823
1824 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1825 {
1826 leave = JUMP(SLJIT_JUMP);
1827 JUMPHERE(foundcr);
1828 notfoundnl = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1829 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1830 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1831 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1832 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1833 JUMPHERE(notfoundnl);
1834 JUMPHERE(leave);
1835 }
1836 JUMPHERE(lastchar);
1837 JUMPHERE(firstchar);
1838
1839 if (firstline)
1840 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1841 }
1842
1843 static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, sljit_uw start_bits, BOOL firstline)
1844 {
1845 DEFINE_COMPILER;
1846 struct sljit_label *start;
1847 struct sljit_jump *leave;
1848 struct sljit_jump *found;
1849
1850 if (firstline)
1851 {
1852 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1853 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1854 }
1855
1856 start = LABEL();
1857 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1858 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1859 #ifdef SUPPORT_UTF8
1860 if (common->utf8)
1861 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
1862 #endif
1863 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
1864 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
1865 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), start_bits);
1866 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
1867 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
1868 found = JUMP(SLJIT_C_NOT_ZERO);
1869
1870 #ifdef SUPPORT_UTF8
1871 if (common->utf8)
1872 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
1873 #endif
1874 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1875 #ifdef SUPPORT_UTF8
1876 if (common->utf8)
1877 {
1878 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
1879 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1880 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1881 }
1882 #endif
1883 JUMPTO(SLJIT_JUMP, start);
1884 JUMPHERE(found);
1885 JUMPHERE(leave);
1886
1887 if (firstline)
1888 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1889 }
1890
1891 static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, pcre_uint16 reqbyte, BOOL has_firstbyte)
1892 {
1893 DEFINE_COMPILER;
1894 struct sljit_label *loop;
1895 struct sljit_jump *toolong;
1896 struct sljit_jump *alreadyfound;
1897 struct sljit_jump *found;
1898 struct sljit_jump *foundoc = NULL;
1899 struct sljit_jump *notfound;
1900 pcre_uint16 oc, bit;
1901
1902 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR);
1903 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX);
1904 toolong = CMP(SLJIT_C_LESS, TMP1, 0, STR_END, 0);
1905 alreadyfound = CMP(SLJIT_C_LESS, STR_PTR, 0, TMP2, 0);
1906
1907 if (has_firstbyte)
1908 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, 1);
1909 else
1910 OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0);
1911
1912 loop = LABEL();
1913 notfound = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, STR_END, 0);
1914
1915 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1916 if ((reqbyte & REQ_CASELESS) == 0)
1917 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte & 0xff);
1918 else
1919 {
1920 reqbyte &= 0xff;
1921 oc = common->fcc[reqbyte];
1922 bit = reqbyte ^ oc;
1923 if (ispowerof2(bit))
1924 {
1925 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit);
1926 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte | bit);
1927 }
1928 else
1929 {
1930 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte);
1931 foundoc = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, oc);
1932 }
1933 }
1934 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
1935 JUMPTO(SLJIT_JUMP, loop);
1936
1937 JUMPHERE(found);
1938 if (foundoc)
1939 JUMPHERE(foundoc);
1940 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, TMP1, 0);
1941 JUMPHERE(alreadyfound);
1942 JUMPHERE(toolong);
1943 return notfound;
1944 }
1945
1946 static void do_revertframes(compiler_common *common)
1947 {
1948 DEFINE_COMPILER;
1949 struct sljit_jump *jump;
1950 struct sljit_label *mainloop;
1951
1952 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1953 OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0);
1954
1955 /* Drop frames until we reach STACK_TOP. */
1956 mainloop = LABEL();
1957 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1958 jump = CMP(SLJIT_C_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1959 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_LOCALS_REG, 0);
1960 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1961 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_w), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_w));
1962 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_w));
1963 JUMPTO(SLJIT_JUMP, mainloop);
1964
1965 JUMPHERE(jump);
1966 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1967 /* End of dropping frames. */
1968 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1969
1970 JUMPHERE(jump);
1971 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_setstrbegin);
1972 /* Set string begin. */
1973 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1974 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
1975 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
1976 JUMPTO(SLJIT_JUMP, mainloop);
1977
1978 JUMPHERE(jump);
1979 /* Unknown command. */
1980 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
1981 JUMPTO(SLJIT_JUMP, mainloop);
1982 }
1983
1984 static void check_wordboundary(compiler_common *common)
1985 {
1986 DEFINE_COMPILER;
1987 struct sljit_jump *beginend;
1988 #ifdef SUPPORT_UTF8
1989 struct sljit_jump *jump;
1990 #endif
1991
1992 SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16);
1993
1994 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
1995 /* Get type of the previous char, and put it to LOCALS1. */
1996 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1997 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
1998 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, 0);
1999 beginend = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
2000 skip_char_back(common);
2001 read_char(common);
2002
2003 /* Testing char type. */
2004 #ifdef SUPPORT_UCP
2005 if (common->useucp)
2006 {
2007 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2008 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2009 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2010 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2011 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2012 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2013 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2014 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2015 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2016 JUMPHERE(jump);
2017 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
2018 }
2019 else
2020 #endif
2021 {
2022 #ifdef SUPPORT_UTF8
2023 /* Here LOCALS1 has already been zeroed. */
2024 jump = NULL;
2025 if (common->utf8)
2026 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2027 #endif
2028 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
2029 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */);
2030 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2031 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
2032 #ifdef SUPPORT_UTF8
2033 if (jump != NULL)
2034 JUMPHERE(jump);
2035 #endif
2036 }
2037 JUMPHERE(beginend);
2038
2039 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2040 beginend = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2041 peek_char(common);
2042
2043 /* Testing char type. This is a code duplication. */
2044 #ifdef SUPPORT_UCP
2045 if (common->useucp)
2046 {
2047 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2048 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2049 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2050 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2051 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2052 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2053 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2054 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2055 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2056 JUMPHERE(jump);
2057 }
2058 else
2059 #endif
2060 {
2061 #ifdef SUPPORT_UTF8
2062 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2063 jump = NULL;
2064 if (common->utf8)
2065 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2066 #endif
2067 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes);
2068 OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */);
2069 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2070 #ifdef SUPPORT_UTF8
2071 if (jump != NULL)
2072 JUMPHERE(jump);
2073 #endif
2074 }
2075 JUMPHERE(beginend);
2076
2077 OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2078 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2079 }
2080
2081 static void check_anynewline(compiler_common *common)
2082 {
2083 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2084 DEFINE_COMPILER;
2085
2086 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2087
2088 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2089 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2090 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2091 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2092 #ifdef SUPPORT_UTF8
2093 if (common->utf8)
2094 {
2095 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2096 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2097 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2098 }
2099 #endif
2100 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2101 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2102 }
2103
2104 static void check_hspace(compiler_common *common)
2105 {
2106 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2107 DEFINE_COMPILER;
2108
2109 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2110
2111 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
2112 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2113 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
2114 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2115 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
2116 #ifdef SUPPORT_UTF8
2117 if (common->utf8)
2118 {
2119 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2120 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
2121 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2122 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
2123 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2124 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000);
2125 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
2126 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2127 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
2128 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2129 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
2130 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2131 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
2132 }
2133 #endif
2134 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2135
2136 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2137 }
2138
2139 static void check_vspace(compiler_common *common)
2140 {
2141 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2142 DEFINE_COMPILER;
2143
2144 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2145
2146 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2147 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2148 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2149 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2150 #ifdef SUPPORT_UTF8
2151 if (common->utf8)
2152 {
2153 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2154 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2155 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2156 }
2157 #endif
2158 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2159
2160 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2161 }
2162
2163 #define CHAR1 STR_END
2164 #define CHAR2 STACK_TOP
2165
2166 static void do_casefulcmp(compiler_common *common)
2167 {
2168 DEFINE_COMPILER;
2169 struct sljit_jump *jump;
2170 struct sljit_label *label;
2171
2172 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2173 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2174 OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0);
2175 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR2, 0);
2176 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2177 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2178
2179 label = LABEL();
2180 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2181 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2182 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2183 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2184 JUMPTO(SLJIT_C_NOT_ZERO, label);
2185
2186 JUMPHERE(jump);
2187 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2188 OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0);
2189 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2190 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2191 }
2192
2193 #define LCC_TABLE STACK_LIMIT
2194
2195 static void do_caselesscmp(compiler_common *common)
2196 {
2197 DEFINE_COMPILER;
2198 struct sljit_jump *jump;
2199 struct sljit_label *label;
2200
2201 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2202 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2203
2204 OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0);
2205 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR1, 0);
2206 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, CHAR2, 0);
2207 OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc);
2208 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2209 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2210
2211 label = LABEL();
2212 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2213 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2214 OP1(SLJIT_MOV_UB, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
2215 OP1(SLJIT_MOV_UB, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
2216 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2217 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2218 JUMPTO(SLJIT_C_NOT_ZERO, label);
2219
2220 JUMPHERE(jump);
2221 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2222 OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0);
2223 OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2224 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2225 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2226 }
2227
2228 #undef LCC_TABLE
2229 #undef CHAR1
2230 #undef CHAR2
2231
2232 #ifdef SUPPORT_UTF8
2233 #ifdef SUPPORT_UCP
2234
2235 static uschar * SLJIT_CALL do_utf8caselesscmp(uschar *src1, jit_arguments *args, uschar *end1)
2236 {
2237 /* This function would be ineffective to do in JIT level. */
2238 int c1, c2;
2239 uschar *src2 = args->ptr;
2240 uschar *end2 = (uschar*)args->end;
2241
2242 while (src1 < end1)
2243 {
2244 if (src2 >= end2)
2245 return 0;
2246 GETCHARINC(c1, src1);
2247 GETCHARINC(c2, src2);
2248 if (c1 != c2 && c1 != UCD_OTHERCASE(c2)) return 0;
2249 }
2250 return src2;
2251 }
2252
2253 #endif
2254 #endif
2255
2256 static uschar *byte_sequence_compare(compiler_common *common, BOOL caseless, uschar *cc,
2257 compare_context* context, jump_list **fallbacks)
2258 {
2259 DEFINE_COMPILER;
2260 unsigned int othercasebit = 0;
2261 uschar *othercasebyte = NULL;
2262 #ifdef SUPPORT_UTF8
2263 int utf8length;
2264 #endif
2265
2266 if (caseless && char_has_othercase(common, cc))
2267 {
2268 othercasebit = char_get_othercase_bit(common, cc);
2269 SLJIT_ASSERT(othercasebit);
2270 /* Extracting bit difference info. */
2271 othercasebyte = cc + (othercasebit >> 8);
2272 othercasebit &= 0xff;
2273 }
2274
2275 if (context->sourcereg == -1)
2276 {
2277 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2278 if (context->length >= 4)
2279 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2280 else if (context->length >= 2)
2281 OP1(SLJIT_MOV_SH, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2282 else
2283 #endif
2284 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2285 context->sourcereg = TMP2;
2286 }
2287
2288 #ifdef SUPPORT_UTF8
2289 utf8length = 1;
2290 if (common->utf8 && *cc >= 0xc0)
2291 utf8length += _pcre_utf8_table4[*cc & 0x3f];
2292
2293 do
2294 {
2295 #endif
2296
2297 context->length--;
2298 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2299
2300 /* Unaligned read is supported. */
2301 if (othercasebit != 0 && othercasebyte == cc)
2302 {
2303 context->c.asbytes[context->byteptr] = *cc | othercasebit;
2304 context->oc.asbytes[context->byteptr] = othercasebit;
2305 }
2306 else
2307 {
2308 context->c.asbytes[context->byteptr] = *cc;
2309 context->oc.asbytes[context->byteptr] = 0;
2310 }
2311 context->byteptr++;
2312
2313 if (context->byteptr >= 4 || context->length == 0 || (context->byteptr == 2 && context->length == 1))
2314 {
2315 if (context->length >= 4)
2316 OP1(SLJIT_MOV_SI, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2317 else if (context->length >= 2)
2318 OP1(SLJIT_MOV_SH, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2319 else if (context->length >= 1)
2320 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2321 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2322
2323 switch(context->byteptr)
2324 {
2325 case 4:
2326 if (context->oc.asint != 0)
2327 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint);
2328 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint));
2329 break;
2330
2331 case 2:
2332 if (context->oc.asshort != 0)
2333 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asshort);
2334 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asshort | context->oc.asshort));
2335 break;
2336
2337 case 1:
2338 if (context->oc.asbyte != 0)
2339 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte);
2340 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte));
2341 break;
2342
2343 default:
2344 SLJIT_ASSERT_STOP();
2345 break;
2346 }
2347 context->byteptr = 0;
2348 }
2349
2350 #else
2351
2352 /* Unaligned read is unsupported. */
2353 if (context->length > 0)
2354 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2355 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2356
2357 if (othercasebit != 0 && othercasebyte == cc)
2358 {
2359 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit);
2360 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit));
2361 }
2362 else
2363 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc));
2364
2365 #endif
2366
2367 cc++;
2368 #ifdef SUPPORT_UTF8
2369 utf8length--;
2370 }
2371 while (utf8length > 0);
2372 #endif
2373
2374 return cc;
2375 }
2376
2377 #ifdef SUPPORT_UTF8
2378
2379 #define SET_TYPE_OFFSET(value) \
2380 if ((value) != typeoffset) \
2381 { \
2382 if ((value) > typeoffset) \
2383 OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \
2384 else \
2385 OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \
2386 } \
2387 typeoffset = (value);
2388
2389 #define SET_CHAR_OFFSET(value) \
2390 if ((value) != charoffset) \
2391 { \
2392 if ((value) > charoffset) \
2393 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (value) - charoffset); \
2394 else \
2395 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, charoffset - (value)); \
2396 } \
2397 charoffset = (value);
2398
2399 static void compile_xclass_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks)
2400 {
2401 DEFINE_COMPILER;
2402 jump_list *found = NULL;
2403 jump_list **list = (*cc & XCL_NOT) == 0 ? &found : fallbacks;
2404 unsigned int c;
2405 int compares;
2406 struct sljit_jump *jump = NULL;
2407 uschar *ccbegin;
2408 #ifdef SUPPORT_UCP
2409 BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE;
2410 BOOL charsaved = FALSE;
2411 int typereg = TMP1, scriptreg = TMP1;
2412 unsigned int typeoffset;
2413 #endif
2414 int invertcmp, numberofcmps;
2415 unsigned int charoffset;
2416
2417 /* Although SUPPORT_UTF8 must be defined, we are not necessary in utf8 mode. */
2418 check_input_end(common, fallbacks);
2419 read_char(common);
2420
2421 if ((*cc++ & XCL_MAP) != 0)
2422 {
2423 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2424 if (common->utf8)
2425 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2426
2427 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
2428 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
2429 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
2430 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
2431 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
2432 add_jump(compiler, list, JUMP(SLJIT_C_NOT_ZERO));
2433
2434 if (common->utf8)
2435 JUMPHERE(jump);
2436 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2437 #ifdef SUPPORT_UCP
2438 charsaved = TRUE;
2439 #endif
2440 cc += 32;
2441 }
2442
2443 /* Scanning the necessary info. */
2444 ccbegin = cc;
2445 compares = 0;
2446 while (*cc != XCL_END)
2447 {
2448 compares++;
2449 if (*cc == XCL_SINGLE)
2450 {
2451 cc += 2;
2452 #ifdef SUPPORT_UTF8
2453 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2454 #endif
2455 #ifdef SUPPORT_UCP
2456 needschar = TRUE;
2457 #endif
2458 }
2459 else if (*cc == XCL_RANGE)
2460 {
2461 cc += 2;
2462 #ifdef SUPPORT_UTF8
2463 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2464 #endif
2465 cc++;
2466 #ifdef SUPPORT_UTF8
2467 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2468 #endif
2469 #ifdef SUPPORT_UCP
2470 needschar = TRUE;
2471 #endif
2472 }
2473 #ifdef SUPPORT_UCP
2474 else
2475 {
2476 SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
2477 cc++;
2478 switch(*cc)
2479 {
2480 case PT_ANY:
2481 break;
2482
2483 case PT_LAMP:
2484 case PT_GC:
2485 case PT_PC:
2486 case PT_ALNUM:
2487 needstype = TRUE;
2488 break;
2489
2490 case PT_SC:
2491 needsscript = TRUE;
2492 break;
2493
2494 case PT_SPACE:
2495 case PT_PXSPACE:
2496 case PT_WORD:
2497 needstype = TRUE;
2498 needschar = TRUE;
2499 break;
2500
2501 default:
2502 SLJIT_ASSERT_STOP();
2503 break;
2504 }
2505 cc += 2;
2506 }
2507 #endif
2508 }
2509
2510 #ifdef SUPPORT_UCP
2511 /* Simple register allocation. TMP1 is preferred if possible. */
2512 if (needstype || needsscript)
2513 {
2514 if (needschar && !charsaved)
2515 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2516 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2517 if (needschar)
2518 {
2519 if (needstype)
2520 {
2521 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
2522 typereg = RETURN_ADDR;
2523 }
2524
2525 if (needsscript)
2526 scriptreg = TMP3;
2527 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2528 }
2529 else if (needstype && needsscript)
2530 scriptreg = TMP3;
2531 /* In all other cases only one of them was specified, and that can goes to TMP1. */
2532
2533 if (needsscript)
2534 {
2535 if (scriptreg == TMP1)
2536 {
2537 OP1(SLJIT_MOV, scriptreg, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2538 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM2(scriptreg, TMP2), 3);
2539 }
2540 else
2541 {
2542 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3);
2543 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2544 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM1(TMP2), 0);
2545 }
2546 }
2547 }
2548 #endif
2549
2550 /* Generating code. */
2551 cc = ccbegin;
2552 charoffset = 0;
2553 numberofcmps = 0;
2554 #ifdef SUPPORT_UCP
2555 typeoffset = 0;
2556 #endif
2557
2558 while (*cc != XCL_END)
2559 {
2560 compares--;
2561 invertcmp = (compares == 0 && list != fallbacks);
2562 jump = NULL;
2563
2564 if (*cc == XCL_SINGLE)
2565 {
2566 cc ++;
2567 #ifdef SUPPORT_UTF8
2568 if (common->utf8)
2569 {
2570 GETCHARINC(c, cc);
2571 }
2572 else
2573 #endif
2574 c = *cc++;
2575
2576 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2577 {
2578 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2579 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2580 numberofcmps++;
2581 }
2582 else if (numberofcmps > 0)
2583 {
2584 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2585 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2586 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2587 numberofcmps = 0;
2588 }
2589 else
2590 {
2591 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2592 numberofcmps = 0;
2593 }
2594 }
2595 else if (*cc == XCL_RANGE)
2596 {
2597 cc ++;
2598 #ifdef SUPPORT_UTF8
2599 if (common->utf8)
2600 {
2601 GETCHARINC(c, cc);
2602 }
2603 else
2604 #endif
2605 c = *cc++;
2606 SET_CHAR_OFFSET(c);
2607 #ifdef SUPPORT_UTF8
2608 if (common->utf8)
2609 {
2610 GETCHARINC(c, cc);
2611 }
2612 else
2613 #endif
2614 c = *cc++;
2615 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2616 {
2617 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2618 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2619 numberofcmps++;
2620 }
2621 else if (numberofcmps > 0)
2622 {
2623 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2624 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2625 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2626 numberofcmps = 0;
2627 }
2628 else
2629 {
2630 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2631 numberofcmps = 0;
2632 }
2633 }
2634 #ifdef SUPPORT_UCP
2635 else
2636 {
2637 if (*cc == XCL_NOTPROP)
2638 invertcmp ^= 0x1;
2639 cc++;
2640 switch(*cc)
2641 {
2642 case PT_ANY:
2643 if (list != fallbacks)
2644 {
2645 if ((cc[-1] == XCL_NOTPROP && compares > 0) || (cc[-1] == XCL_PROP && compares == 0))
2646 continue;
2647 }
2648 else if (cc[-1] == XCL_NOTPROP)
2649 continue;
2650 jump = JUMP(SLJIT_JUMP);
2651 break;
2652
2653 case PT_LAMP:
2654 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
2655 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2656 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
2657 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2658 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
2659 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2660 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2661 break;
2662
2663 case PT_GC:
2664 c = _pcre_ucp_typerange[(int)cc[1] * 2];
2665 SET_TYPE_OFFSET(c);
2666 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, _pcre_ucp_typerange[(int)cc[1] * 2 + 1] - c);
2667 break;
2668
2669 case PT_PC:
2670 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
2671 break;
2672
2673 case PT_SC:
2674 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, scriptreg, 0, SLJIT_IMM, (int)cc[1]);
2675 break;
2676
2677 case PT_SPACE:
2678 case PT_PXSPACE:
2679 if (*cc == PT_SPACE)
2680 {
2681 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2682 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 11 - charoffset);
2683 }
2684 SET_CHAR_OFFSET(9);
2685 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 13 - 9);
2686 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2687 if (*cc == PT_SPACE)
2688 JUMPHERE(jump);
2689
2690 SET_TYPE_OFFSET(ucp_Zl);
2691 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
2692 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2693 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2694 break;
2695
2696 case PT_WORD:
2697 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE - charoffset);
2698 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2699 /* ... fall through */
2700
2701 case PT_ALNUM:
2702 SET_TYPE_OFFSET(ucp_Ll);
2703 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2704 COND_VALUE((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2705 SET_TYPE_OFFSET(ucp_Nd);
2706 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2707 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2708 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2709 break;
2710 }
2711 cc += 2;
2712 }
2713 #endif
2714
2715 if (jump != NULL)
2716 add_jump(compiler, compares > 0 ? list : fallbacks, jump);
2717 }
2718
2719 if (found != NULL)
2720 set_jumps(found, LABEL());
2721 }
2722
2723 #undef SET_TYPE_OFFSET
2724 #undef SET_CHAR_OFFSET
2725
2726 #endif
2727
2728 static uschar *compile_char1_hotpath(compiler_common *common, uschar type, uschar *cc, jump_list **fallbacks)
2729 {
2730 DEFINE_COMPILER;
2731 int length;
2732 unsigned int c, oc, bit;
2733 compare_context context;
2734 struct sljit_jump *jump[4];
2735 #ifdef SUPPORT_UTF8
2736 struct sljit_label *label;
2737 #ifdef SUPPORT_UCP
2738 uschar propdata[5];
2739 #endif
2740 #endif
2741
2742 switch(type)
2743 {
2744 case OP_SOD:
2745 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2746 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
2747 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2748 return cc;
2749
2750 case OP_SOM:
2751 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2752 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
2753 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2754 return cc;
2755
2756 case OP_NOT_WORD_BOUNDARY:
2757 case OP_WORD_BOUNDARY:
2758 add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
2759 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2760 return cc;
2761
2762 case OP_NOT_DIGIT:
2763 case OP_DIGIT:
2764 check_input_end(common, fallbacks);
2765 read_char8_type(common);
2766 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
2767 add_jump(compiler, fallbacks, JUMP(type == OP_DIGIT ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2768 return cc;
2769
2770 case OP_NOT_WHITESPACE:
2771 case OP_WHITESPACE:
2772 check_input_end(common, fallbacks);
2773 read_char8_type(common);
2774 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
2775 add_jump(compiler, fallbacks, JUMP(type == OP_WHITESPACE ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2776 return cc;
2777
2778 case OP_NOT_WORDCHAR:
2779 case OP_WORDCHAR:
2780 check_input_end(common, fallbacks);
2781 read_char8_type(common);
2782 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
2783 add_jump(compiler, fallbacks, JUMP(type == OP_WORDCHAR ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2784 return cc;
2785
2786 case OP_ANY:
2787 check_input_end(common, fallbacks);
2788 read_char(common);
2789 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2790 {
2791 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
2792 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2793 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2794 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
2795 JUMPHERE(jump[1]);
2796 JUMPHERE(jump[0]);
2797 }
2798 else
2799 check_newlinechar(common, common->nltype, fallbacks, TRUE);
2800 return cc;
2801
2802 case OP_ALLANY:
2803 check_input_end(common, fallbacks);
2804 #ifdef SUPPORT_UTF8
2805 if (common->utf8)
2806 {
2807 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2808 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2809 jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
2810 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
2811 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
2812 JUMPHERE(jump[0]);
2813 return cc;
2814 }
2815 #endif
2816 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2817 return cc;
2818
2819 case OP_ANYBYTE:
2820 check_input_end(common, fallbacks);
2821 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2822 return cc;
2823
2824 #ifdef SUPPORT_UTF8
2825 #ifdef SUPPORT_UCP
2826 case OP_NOTPROP:
2827 case OP_PROP:
2828 propdata[0] = 0;
2829 propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP;
2830 propdata[2] = cc[0];
2831 propdata[3] = cc[1];
2832 propdata[4] = XCL_END;
2833 compile_xclass_hotpath(common, propdata, fallbacks);
2834 return cc + 2;
2835 #endif
2836 #endif
2837
2838 case OP_ANYNL:
2839 check_input_end(common, fallbacks);
2840 read_char(common);
2841 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2842 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2843 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2844 jump[2] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2845 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2846 jump[3] = JUMP(SLJIT_JUMP);
2847 JUMPHERE(jump[0]);
2848 check_newlinechar(common, common->bsr_nltype, fallbacks, FALSE);
2849 JUMPHERE(jump[1]);
2850 JUMPHERE(jump[2]);
2851 JUMPHERE(jump[3]);
2852 return cc;
2853
2854 case OP_NOT_HSPACE:
2855 case OP_HSPACE:
2856 check_input_end(common, fallbacks);
2857 read_char(common);
2858 add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
2859 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_HSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2860 return cc;
2861
2862 case OP_NOT_VSPACE:
2863 case OP_VSPACE:
2864 check_input_end(common, fallbacks);
2865 read_char(common);
2866 add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
2867 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_VSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2868 return cc;
2869
2870 #ifdef SUPPORT_UCP
2871 case OP_EXTUNI:
2872 check_input_end(common, fallbacks);
2873 read_char(common);
2874 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2875 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2876 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc));
2877
2878 label = LABEL();
2879 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2880 OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
2881 read_char(common);
2882 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2883 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2884 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc, label);
2885
2886 OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
2887 JUMPHERE(jump[0]);
2888 return cc;
2889 #endif
2890
2891 case OP_EODN:
2892 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2893 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2894 {
2895 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2896 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2897 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2898 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
2899 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2900 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2901 }
2902 else if (common->nltype == NLTYPE_FIXED)
2903 {
2904 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2905 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2906 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2907 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
2908 }
2909 else
2910 {
2911 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2912 jump[1] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2913 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2914 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
2915 jump[2] = JUMP(SLJIT_C_GREATER);
2916 add_jump(compiler, fallbacks, JUMP(SLJIT_C_LESS));
2917 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 1);
2918 jump[3] = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2919 add_jump(compiler, fallbacks, JUMP(SLJIT_JUMP));
2920
2921 JUMPHERE(jump[1]);
2922 if (common->nltype == NLTYPE_ANYCRLF)
2923 {
2924 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2925 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, STR_END, 0));
2926 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
2927 }
2928 else
2929 {
2930 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, STR_PTR, 0);
2931 read_char(common);
2932 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2933 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
2934 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
2935 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2936 }
2937 JUMPHERE(jump[2]);
2938 JUMPHERE(jump[3]);
2939 }
2940 JUMPHERE(jump[0]);
2941 return cc;
2942
2943 case OP_EOD:
2944 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2945 return cc;
2946
2947 case OP_CIRC:
2948 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2949 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2950 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0));
2951 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2952 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2953 return cc;
2954
2955 case OP_CIRCM:
2956 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2957 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2958 jump[1] = CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0);
2959 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2960 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2961 jump[0] = JUMP(SLJIT_JUMP);
2962 JUMPHERE(jump[1]);
2963
2964 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, end));
2965 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP2, 0, STR_PTR, 0));
2966
2967 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2968 {
2969 OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2970 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, TMP1, 0));
2971 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
2972 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
2973 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2974 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2975 }
2976 else
2977 {
2978 skip_char_back(common);
2979 read_char(common);
2980 check_newlinechar(common, common->nltype, fallbacks, FALSE);
2981 }
2982 JUMPHERE(jump[0]);
2983 return cc;
2984
2985 case OP_DOLL:
2986 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2987 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
2988 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2989
2990 if (!common->endonly)
2991 compile_char1_hotpath(common, OP_EODN, cc, fallbacks);
2992 else
2993 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0));
2994 return cc;
2995
2996 case OP_DOLLM:
2997 jump[1] = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
2998 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2999 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
3000 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3001 jump[0] = JUMP(SLJIT_JUMP);
3002 JUMPHERE(jump[1]);
3003
3004 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
3005 {
3006 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
3007 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, TMP2, 0, STR_END, 0));
3008 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3009 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
3010 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
3011 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
3012 }
3013 else
3014 {
3015 peek_char(common);
3016 check_newlinechar(common, common->nltype, fallbacks, FALSE);
3017 }
3018 JUMPHERE(jump[0]);
3019 return cc;
3020
3021 case OP_CHAR:
3022 case OP_CHARI:
3023 length = 1;
3024 #ifdef SUPPORT_UTF8
3025 if (common->utf8 && *cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3026 #endif
3027 if (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)
3028 {
3029 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3030 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3031
3032 context.length = length;
3033 context.sourcereg = -1;
3034 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3035 context.byteptr = 0;
3036 #endif
3037 return byte_sequence_compare(common, type == OP_CHARI, cc, &context, fallbacks);
3038 }
3039 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
3040 read_char(common);
3041 #ifdef SUPPORT_UTF8
3042 if (common->utf8)
3043 {
3044 GETCHAR(c, cc);
3045 }
3046 else
3047 #endif
3048 c = *cc;
3049 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c);
3050 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3051 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char_othercase(common, c));
3052 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3053 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3054 return cc + length;
3055
3056 case OP_NOT:
3057 case OP_NOTI:
3058 length = 1;
3059 #ifdef SUPPORT_UTF8
3060 if (common->utf8)
3061 {
3062 if (*cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3063
3064 check_input_end(common, fallbacks);
3065 GETCHAR(c, cc);
3066
3067 if (c <= 127)
3068 {
3069 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3070 if (type == OP_NOT || !char_has_othercase(common, cc))
3071 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3072 else
3073 {
3074 /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */
3075 OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20);
3076 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20));
3077 }
3078 /* Skip the variable-length character. */
3079 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
3080 jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
3081 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
3082 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
3083 JUMPHERE(jump[0]);
3084 return cc + length;
3085 }
3086 else
3087 read_char(common);
3088 }
3089 else
3090 #endif
3091 {
3092 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
3093 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3094 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
3095 c = *cc;
3096 }
3097
3098 if (type == OP_NOT || !char_has_othercase(common, cc))
3099 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3100 else
3101 {
3102 oc = char_othercase(common, c);
3103 bit = c ^ oc;
3104 if (ispowerof2(bit))
3105 {
3106 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
3107 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
3108 }
3109 else
3110 {
3111 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3112 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, oc));
3113 }
3114 }
3115 return cc + length;
3116
3117 case OP_CLASS:
3118 case OP_NCLASS:
3119 check_input_end(common, fallbacks);
3120 read_char(common);
3121 #ifdef SUPPORT_UTF8
3122 jump[0] = NULL;
3123 if (common->utf8)
3124 {
3125 jump[0] = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
3126 if (type == OP_CLASS)
3127 {
3128 add_jump(compiler, fallbacks, jump[0]);
3129 jump[0] = NULL;
3130 }
3131 }
3132 #endif
3133 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
3134 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
3135 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
3136 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
3137 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
3138 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3139 #ifdef SUPPORT_UTF8
3140 if (jump[0] != NULL)
3141 JUMPHERE(jump[0]);
3142 #endif
3143 return cc + 32;
3144
3145 #ifdef SUPPORT_UTF8
3146 case OP_XCLASS:
3147 compile_xclass_hotpath(common, cc + LINK_SIZE, fallbacks);
3148 return cc + GET(cc, 0) - 1;
3149 #endif
3150
3151 case OP_REVERSE:
3152 length = GET(cc, 0);
3153 SLJIT_ASSERT(length > 0);
3154 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
3155 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
3156 #ifdef SUPPORT_UTF8
3157 if (common->utf8)
3158 {
3159 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length);
3160 label = LABEL();
3161 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0));
3162 skip_char_back(common);
3163 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
3164 JUMPTO(SLJIT_C_NOT_ZERO, label);
3165 return cc + LINK_SIZE;
3166 }
3167 #endif
3168 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3169 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, TMP1, 0));
3170 return cc + LINK_SIZE;
3171 }
3172 SLJIT_ASSERT_STOP();
3173 return cc;
3174 }
3175
3176 static SLJIT_INLINE uschar *compile_charn_hotpath(compiler_common *common, uschar *cc, uschar *ccend, jump_list **fallbacks)
3177 {
3178 /* This function consumes at least one input character. */
3179 /* To decrease the number of length checks, we try to concatenate the fixed length character sequences. */
3180 DEFINE_COMPILER;
3181 uschar *ccbegin = cc;
3182 compare_context context;
3183 int size;
3184
3185 context.length = 0;
3186 do
3187 {
3188 if (cc >= ccend)
3189 break;
3190
3191 if (*cc == OP_CHAR)
3192 {
3193 size = 1;
3194 #ifdef SUPPORT_UTF8
3195 if (common->utf8 && cc[1] >= 0xc0)
3196 size += _pcre_utf8_table4[cc[1] & 0x3f];
3197 #endif
3198 }
3199 else if (*cc == OP_CHARI)
3200 {
3201 size = 1;
3202 #ifdef SUPPORT_UTF8
3203 if (common->utf8)
3204 {
3205 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3206 size = 0;
3207 else if (cc[1] >= 0xc0)
3208 size += _pcre_utf8_table4[cc[1] & 0x3f];
3209 }
3210 else
3211 #endif
3212 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3213 size = 0;
3214 }
3215 else
3216 size = 0;
3217
3218 cc += 1 + size;
3219 context.length += size;
3220 }
3221 while (size > 0 && context.length <= 128);
3222
3223 cc = ccbegin;
3224 if (context.length > 0)
3225 {
3226 /* We have a fixed-length byte sequence. */
3227 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length);
3228 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3229
3230 context.sourcereg = -1;
3231 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3232 context.byteptr = 0;
3233 #endif
3234 do cc = byte_sequence_compare(common, *cc == OP_CHARI, cc + 1, &context, fallbacks); while (context.length > 0);
3235 return cc;
3236 }
3237
3238 /* A non-fixed length character will be checked if length == 0. */
3239 return compile_char1_hotpath(common, *cc, cc + 1, fallbacks);
3240 }
3241
3242 static struct sljit_jump *compile_ref_checks(compiler_common *common, uschar *cc, jump_list **fallbacks)
3243 {
3244 DEFINE_COMPILER;
3245 int offset = GET2(cc, 1) << 1;
3246
3247 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3248 if (!common->jscript_compat)
3249 {
3250 if (fallbacks == NULL)
3251 {
3252 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
3253 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3254 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3255 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3256 return JUMP(SLJIT_C_NOT_ZERO);
3257 }
3258 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3259 }
3260 return CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3261 }
3262
3263 /* Forward definitions. */
3264 static void compile_hotpath(compiler_common *, uschar *, uschar *, fallback_common *);
3265 static void compile_fallbackpath(compiler_common *, struct fallback_common *);
3266
3267 #define PUSH_FALLBACK(size, ccstart, error) \
3268 do \
3269 { \
3270 fallback = sljit_alloc_memory(compiler, (size)); \
3271 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3272 return error; \
3273 memset(fallback, 0, size); \
3274 fallback->prev = parent->top; \
3275 fallback->cc = (ccstart); \
3276 parent->top = fallback; \
3277 } \
3278 while (0)
3279
3280 #define PUSH_FALLBACK_NOVALUE(size, ccstart) \
3281 do \
3282 { \
3283 fallback = sljit_alloc_memory(compiler, (size)); \
3284 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3285 return; \
3286 memset(fallback, 0, size); \
3287 fallback->prev = parent->top; \
3288 fallback->cc = (ccstart); \
3289 parent->top = fallback; \
3290 } \
3291 while (0)
3292
3293 #define FALLBACK_AS(type) ((type*)fallback)
3294
3295 static uschar *compile_ref_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks, BOOL withchecks, BOOL emptyfail)
3296 {
3297 DEFINE_COMPILER;
3298 int offset = GET2(cc, 1) << 1;
3299 struct sljit_jump *jump = NULL;
3300
3301 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3302 if (withchecks && !common->jscript_compat)
3303 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3304
3305 #ifdef SUPPORT_UTF8
3306 #ifdef SUPPORT_UCP
3307 if (common->utf8 && *cc == OP_REFI)
3308 {
3309 SLJIT_ASSERT(TMP1 == SLJIT_TEMPORARY_REG1 && STACK_TOP == SLJIT_TEMPORARY_REG2 && TMP2 == SLJIT_TEMPORARY_REG3);
3310 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3311 if (withchecks)
3312 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, TMP2, 0);
3313
3314 /* Needed to save important temporary registers. */
3315 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
3316 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, ARGUMENTS, 0);
3317 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), SLJIT_OFFSETOF(jit_arguments, ptr), STR_PTR, 0);
3318 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf8caselesscmp));
3319 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
3320 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
3321 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
3322 }
3323 else
3324 #endif
3325 #endif
3326 {
3327 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP1, 0);
3328 if (withchecks)
3329 jump = JUMP(SLJIT_C_ZERO);
3330 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
3331
3332 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3333 add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL));
3334 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3335 }
3336
3337 if (jump != NULL)
3338 {
3339 if (emptyfail)
3340 add_jump(compiler, fallbacks, jump);
3341 else
3342 JUMPHERE(jump);
3343 }
3344 return cc + 3;
3345 }
3346
3347 static SLJIT_INLINE uschar *compile_ref_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3348 {
3349 DEFINE_COMPILER;
3350 fallback_common *fallback;
3351 uschar type;
3352 struct sljit_label *label;
3353 struct sljit_jump *zerolength;
3354 struct sljit_jump *jump = NULL;
3355 uschar *ccbegin = cc;
3356 int min = 0, max = 0;
3357 BOOL minimize;
3358
3359 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
3360
3361 type = cc[3];
3362 minimize = (type & 0x1) != 0;
3363 switch(type)
3364 {
3365 case OP_CRSTAR:
3366 case OP_CRMINSTAR:
3367 min = 0;
3368 max = 0;
3369 cc += 4;
3370 break;
3371 case OP_CRPLUS:
3372 case OP_CRMINPLUS:
3373 min = 1;
3374 max = 0;
3375 cc += 4;
3376 break;
3377 case OP_CRQUERY:
3378 case OP_CRMINQUERY:
3379 min = 0;
3380 max = 1;
3381 cc += 4;
3382 break;
3383 case OP_CRRANGE:
3384 case OP_CRMINRANGE:
3385 min = GET2(cc, 3 + 1);
3386 max = GET2(cc, 3 + 3);
3387 cc += 8;
3388 break;
3389 default:
3390 SLJIT_ASSERT_STOP();
3391 break;
3392 }
3393
3394 if (!minimize)
3395 {
3396 if (min == 0)
3397 {
3398 allocate_stack(common, 2);
3399 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3400 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3401 /* Temporary release of STR_PTR. */
3402 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3403 zerolength = compile_ref_checks(common, ccbegin, NULL);
3404 /* Restore if not zero length. */
3405 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3406 }
3407 else
3408 {
3409 allocate_stack(common, 1);
3410 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3411 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3412 }
3413
3414 if (min > 1 || max > 1)
3415 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
3416
3417 label = LABEL();
3418 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, FALSE, FALSE);
3419
3420 if (min > 1 || max > 1)
3421 {
3422 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
3423 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3424 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
3425 if (min > 1)
3426 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, label);
3427 if (max > 1)
3428 {
3429 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
3430 allocate_stack(common, 1);
3431 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3432 JUMPTO(SLJIT_JUMP, label);
3433 JUMPHERE(jump);
3434 }
3435 }
3436
3437 if (max == 0)
3438 {
3439 /* Includes min > 1 case as well. */
3440 allocate_stack(common, 1);
3441 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3442 JUMPTO(SLJIT_JUMP, label);
3443 }
3444
3445 JUMPHERE(zerolength);
3446 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3447
3448 decrease_call_count(common);
3449 return cc;
3450 }
3451
3452 allocate_stack(common, 2);
3453 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3454 if (type != OP_CRMINSTAR)
3455 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3456
3457 if (min == 0)
3458 {
3459 zerolength = compile_ref_checks(common, ccbegin, NULL);
3460 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3461 jump = JUMP(SLJIT_JUMP);
3462 }
3463 else
3464 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3465
3466 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3467 if (max > 0)
3468 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max));
3469
3470 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, TRUE, TRUE);
3471 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3472
3473 if (min > 1)
3474 {
3475 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3476 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3477 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3478 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, FALLBACK_AS(iterator_fallback)->hotpath);
3479 }
3480 else if (max > 0)
3481 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
3482
3483 if (jump != NULL)
3484 JUMPHERE(jump);
3485 JUMPHERE(zerolength);
3486
3487 decrease_call_count(common);
3488 return cc;
3489 }
3490
3491 static SLJIT_INLINE uschar *compile_recurse_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3492 {
3493 DEFINE_COMPILER;
3494 fallback_common *fallback;
3495 recurse_entry *entry = common->entries;
3496 recurse_entry *prev = NULL;
3497 int start = GET(cc, 1);
3498
3499 PUSH_FALLBACK(sizeof(recurse_fallback), cc, NULL);
3500 while (entry != NULL)
3501 {
3502 if (entry->start == start)
3503 break;
3504 prev = entry;
3505 entry = entry->next;
3506 }
3507
3508 if (entry == NULL)
3509 {
3510 entry = sljit_alloc_memory(compiler, sizeof(recurse_entry));
3511 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3512 return NULL;
3513 entry->next = NULL;
3514 entry->entry = NULL;
3515 entry->calls = NULL;
3516 entry->start = start;
3517
3518 if (prev != NULL)
3519 prev->next = entry;
3520 else
3521 common->entries = entry;
3522 }
3523
3524 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
3525 allocate_stack(common, 1);
3526 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
3527
3528 if (entry->entry == NULL)
3529 add_jump(compiler, &entry->calls, JUMP(SLJIT_FAST_CALL));
3530 else
3531 JUMPTO(SLJIT_FAST_CALL, entry->entry);
3532 /* Leave if the match is failed. */
3533 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0));
3534 return cc + 1 + LINK_SIZE;
3535 }
3536
3537 static uschar *compile_assert_hotpath(compiler_common *common, uschar *cc, assert_fallback *fallback, BOOL conditional)
3538 {
3539 DEFINE_COMPILER;
3540 int framesize;
3541 int localptr;
3542 fallback_common altfallback;
3543 uschar *ccbegin;
3544 uschar opcode;
3545 uschar bra = OP_BRA;
3546 jump_list *tmp = NULL;
3547 jump_list **target = (conditional) ? &fallback->condfailed : &fallback->common.topfallbacks;
3548 jump_list **found;
3549 /* Saving previous accept variables. */
3550 struct sljit_label *save_acceptlabel = common->acceptlabel;
3551 struct sljit_jump *jump;
3552 struct sljit_jump *brajump = NULL;
3553 jump_list *save_accept = common->accept;
3554
3555 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3556 {
3557 SLJIT_ASSERT(!conditional);
3558 bra = *cc;
3559 cc++;
3560 }
3561 localptr = PRIV(cc);
3562 SLJIT_ASSERT(localptr != 0);
3563 framesize = get_framesize(common, cc, FALSE);
3564 fallback->framesize = framesize;
3565 fallback->localptr = localptr;
3566 opcode = *cc;
3567 SLJIT_ASSERT(opcode >= OP_ASSERT && opcode <= OP_ASSERTBACK_NOT);
3568 found = (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) ? &tmp : target;
3569 ccbegin = cc;
3570 cc += GET(cc, 1);
3571
3572 if (bra == OP_BRAMINZERO)
3573 {
3574 /* This is a braminzero fallback path. */
3575 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3576 free_stack(common, 1);
3577 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
3578 }
3579
3580 if (framesize < 0)
3581 {
3582 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
3583 allocate_stack(common, 1);
3584 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3585 }
3586 else
3587 {
3588 allocate_stack(common, framesize + 2);
3589 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3590 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(framesize + 1));
3591 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
3592 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3593 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3594 init_frame(common, ccbegin, framesize + 1, 2, FALSE);
3595 }
3596
3597 memset(&altfallback, 0, sizeof(fallback_common));
3598 while (1)
3599 {
3600 common->acceptlabel = NULL;
3601 common->accept = NULL;
3602 altfallback.top = NULL;
3603 altfallback.topfallbacks = NULL;
3604
3605 if (*ccbegin == OP_ALT)
3606 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3607
3608 altfallback.cc = ccbegin;
3609 compile_hotpath(common, ccbegin + 1 + LINK_SIZE, cc, &altfallback);
3610 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3611 {
3612 common->acceptlabel = save_acceptlabel;
3613 common->accept = save_accept;
3614 return NULL;
3615 }
3616 common->acceptlabel = LABEL();
3617 if (common->accept != NULL)
3618 set_jumps(common->accept, common->acceptlabel);
3619
3620 /* Reset stack. */
3621 if (framesize < 0)
3622 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3623 else {
3624 if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional)
3625 {
3626 /* We don't need to keep the STR_PTR, only the previous localptr. */
3627 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_w));
3628 }
3629 else
3630 {
3631 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3632 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3633 }
3634 }
3635
3636 if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT)
3637 {
3638 /* We know that STR_PTR was stored on the top of the stack. */
3639 if (conditional)
3640 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3641 else if (bra == OP_BRAZERO)
3642 {
3643 if (framesize < 0)
3644 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3645 else
3646 {
3647 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3648 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_w));
3649 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3650 }
3651 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3652 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3653 }
3654 else if (framesize >= 0)
3655 {
3656 /* For OP_BRA and OP_BRAMINZERO. */
3657 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3658 }
3659 }
3660 add_jump(compiler, found, JUMP(SLJIT_JUMP));
3661
3662 compile_fallbackpath(common, altfallback.top);
3663 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3664 {
3665 common->acceptlabel = save_acceptlabel;
3666 common->accept = save_accept;
3667 return NULL;
3668 }
3669 set_jumps(altfallback.topfallbacks, LABEL());
3670
3671 if (*cc != OP_ALT)
3672 break;
3673
3674 ccbegin = cc;
3675 cc += GET(cc, 1);
3676 }
3677 /* None of them matched. */
3678
3679 if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
3680 {
3681 /* Assert is failed. */
3682 if (conditional || bra == OP_BRAZERO)
3683 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3684
3685 if (framesize < 0)
3686 {
3687 /* The topmost item should be 0. */
3688 if (bra == OP_BRAZERO)
3689 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3690 else
3691 free_stack(common, 1);
3692 }
3693 else
3694 {
3695 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3696 /* The topmost item should be 0. */
3697 if (bra == OP_BRAZERO)
3698 {
3699 free_stack(common, framesize + 1);
3700 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3701 }
3702 else
3703 free_stack(common, framesize + 2);
3704 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3705 }
3706 jump = JUMP(SLJIT_JUMP);
3707 if (bra != OP_BRAZERO)
3708 add_jump(compiler, target, jump);
3709
3710 /* Assert is successful. */
3711 set_jumps(tmp, LABEL());
3712 if (framesize < 0)
3713 {
3714 /* We know that STR_PTR was stored on the top of the stack. */
3715 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3716 /* Keep the STR_PTR on the top of the stack. */
3717 if (bra == OP_BRAZERO)
3718 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3719 else if (bra == OP_BRAMINZERO)
3720 {
3721 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3722 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3723 }
3724 }
3725 else
3726 {
3727 if (bra == OP_BRA)
3728 {
3729 /* We don't need to keep the STR_PTR, only the previous localptr. */
3730 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_w));
3731 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3732 }
3733 else
3734 {
3735 /* We don't need to keep the STR_PTR, only the previous localptr. */
3736 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_w));
3737 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3738 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), bra == OP_BRAZERO ? STR_PTR : SLJIT_IMM, 0);
3739 }
3740 }
3741
3742 if (bra == OP_BRAZERO)
3743 {
3744 fallback->hotpath = LABEL();
3745 sljit_set_label(jump, fallback->hotpath);
3746 }
3747 else if (bra == OP_BRAMINZERO)
3748 {
3749 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3750 JUMPHERE(brajump);
3751 if (framesize >= 0)
3752 {
3753 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3754 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3755 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3756 }
3757 set_jumps(fallback->common.topfallbacks, LABEL());
3758 }
3759 }
3760 else
3761 {
3762 /* AssertNot is successful. */
3763 if (framesize < 0)
3764 {
3765 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3766 if (bra != OP_BRA)
3767 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3768 else
3769 free_stack(common, 1);
3770 }
3771 else
3772 {
3773 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3774 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3775 /* The topmost item should be 0. */
3776 if (bra != OP_BRA)
3777 {
3778 free_stack(common, framesize + 1);
3779 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3780 }
3781 else
3782 free_stack(common, framesize + 2);
3783 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3784 }
3785
3786 if (bra == OP_BRAZERO)
3787 fallback->hotpath = LABEL();
3788 else if (bra == OP_BRAMINZERO)
3789 {
3790 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3791 JUMPHERE(brajump);
3792 }
3793
3794 if (bra != OP_BRA)
3795 {
3796 SLJIT_ASSERT(found == &fallback->common.topfallbacks);
3797 set_jumps(fallback->common.topfallbacks, LABEL());
3798 fallback->common.topfallbacks = NULL;
3799 }
3800 }
3801
3802 common->acceptlabel = save_acceptlabel;
3803 common->accept = save_accept;
3804 return cc + 1 + LINK_SIZE;
3805 }
3806
3807 static sljit_w SLJIT_CALL do_searchovector(sljit_w refno, sljit_w* locals, uschar *name_table)
3808 {
3809 int condition = FALSE;
3810 uschar *slotA = name_table;
3811 uschar *slotB;
3812 sljit_w name_count = locals[LOCALS0 / sizeof(sljit_w)];
3813 sljit_w name_entry_size = locals[LOCALS1 / sizeof(sljit_w)];
3814 sljit_w no_capture;
3815 int i;
3816
3817 locals += OVECTOR_START / sizeof(sljit_w);
3818 no_capture = locals[1];
3819
3820 for (i = 0; i < name_count; i++)
3821 {
3822 if (GET2(slotA, 0) == refno) break;
3823 slotA += name_entry_size;
3824 }
3825
3826 if (i < name_count)
3827 {
3828 /* Found a name for the number - there can be only one; duplicate names
3829 for different numbers are allowed, but not vice versa. First scan down
3830 for duplicates. */
3831
3832 slotB = slotA;
3833 while (slotB > name_table)
3834 {
3835 slotB -= name_entry_size;
3836 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3837 {
3838 condition = locals[GET2(slotB, 0) << 1] != no_capture;
3839 if (condition) break;
3840 }
3841 else break;
3842 }
3843
3844 /* Scan up for duplicates */
3845 if (!condition)
3846 {
3847 slotB = slotA;
3848 for (i++; i < name_count; i++)
3849 {
3850 slotB += name_entry_size;
3851 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3852 {
3853 condition = locals[GET2(slotB, 0) << 1] != no_capture;
3854 if (condition) break;
3855 }
3856 else break;
3857 }
3858 }
3859 }
3860 return condition;
3861 }
3862
3863 static sljit_w SLJIT_CALL do_searchgroups(sljit_w recno, sljit_w* locals, uschar *name_table)
3864 {
3865 int condition = FALSE;
3866 uschar *slotA = name_table;
3867 uschar *slotB;
3868 sljit_w name_count = locals[LOCALS0 / sizeof(sljit_w)];
3869 sljit_w name_entry_size = locals[LOCALS1 / sizeof(sljit_w)];
3870 sljit_w group_num = locals[POSSESSIVE0 / sizeof(sljit_w)];
3871 int i;
3872
3873 for (i = 0; i < name_count; i++)
3874 {
3875 if (GET2(slotA, 0) == recno) break;
3876 slotA += name_entry_size;
3877 }
3878
3879 if (i < name_count)
3880 {
3881 /* Found a name for the number - there can be only one; duplicate
3882 names for different numbers are allowed, but not vice versa. First
3883 scan down for duplicates. */
3884
3885 slotB = slotA;
3886 while (slotB > name_table)
3887 {
3888 slotB -= name_entry_size;
3889 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3890 {
3891 condition = GET2(slotB, 0) == group_num;
3892 if (condition) break;
3893 }
3894 else break;
3895 }
3896
3897 /* Scan up for duplicates */
3898 if (!condition)
3899 {
3900 slotB = slotA;
3901 for (i++; i < name_count; i++)
3902 {
3903 slotB += name_entry_size;
3904 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3905 {
3906 condition = GET2(slotB, 0) == group_num;
3907 if (condition) break;
3908 }
3909 else break;
3910 }
3911 }
3912 }
3913 return condition;
3914 }
3915
3916 /*
3917 Handling bracketed expressions is probably the most complex part.
3918
3919 Stack layout naming characters:
3920 S - Push the current STR_PTR
3921 0 - Push a 0 (NULL)
3922 A - Push the current STR_PTR. Needed for restoring the STR_PTR
3923 before the next alternative. Not pushed if there are no alternatives.
3924 M - Any values pushed by the current alternative. Can be empty, or anything.
3925 C - Push the previous OVECTOR(i), OVECTOR(i+1) and OVECTOR_PRIV(i) to the stack.
3926 L - Push the previous local (pointed by localptr) to the stack
3927 () - opional values stored on the stack
3928 ()* - optonal, can be stored multiple times
3929
3930 The following list shows the regular expression templates, their PCRE byte codes
3931 and stack layout supported by pcre-sljit.
3932
3933 (?:) OP_BRA | OP_KET A M
3934 () OP_CBRA | OP_KET C M
3935 (?:)+ OP_BRA | OP_KETRMAX 0 A M S ( A M S )*
3936 OP_SBRA | OP_KETRMAX 0 L M S ( L M S )*
3937 (?:)+? OP_BRA | OP_KETRMIN 0 A M S ( A M S )*
3938 OP_SBRA | OP_KETRMIN 0 L M S ( L M S )*
3939 ()+ OP_CBRA | OP_KETRMAX 0 C M S ( C M S )*
3940 OP_SCBRA | OP_KETRMAX 0 C M S ( C M S )*
3941 ()+? OP_CBRA | OP_KETRMIN 0 C M S ( C M S )*
3942 OP_SCBRA | OP_KETRMIN 0 C M S ( C M S )*
3943 (?:)? OP_BRAZERO | OP_BRA | OP_KET S ( A M 0 )
3944 (?:)?? OP_BRAMINZERO | OP_BRA | OP_KET S ( A M 0 )
3945 ()? OP_BRAZERO | OP_CBRA | OP_KET S ( C M 0 )
3946 ()?? OP_BRAMINZERO | OP_CBRA | OP_KET S ( C M 0 )
3947 (?:)* OP_BRAZERO | OP_BRA | OP_KETRMAX S 0 ( A M S )*
3948 OP_BRAZERO | OP_SBRA | OP_KETRMAX S 0 ( L M S )*
3949 (?:)*? OP_BRAMINZERO | OP_BRA | OP_KETRMIN S 0 ( A M S )*
3950 OP_BRAMINZERO | OP_SBRA | OP_KETRMIN S 0 ( L M S )*
3951 ()* OP_BRAZERO | OP_CBRA | OP_KETRMAX S 0 ( C M S )*
3952 OP_BRAZERO | OP_SCBRA | OP_KETRMAX S 0 ( C M S )*
3953 ()*? OP_BRAMINZERO | OP_CBRA | OP_KETRMIN S 0 ( C M S )*
3954 OP_BRAMINZERO | OP_SCBRA | OP_KETRMIN S 0 ( C M S )*
3955
3956
3957 Stack layout naming characters:
3958 A - Push the alternative index (starting from 0) on the stack.
3959 Not pushed if there is no alternatives.
3960 M - Any values pushed by the current alternative. Can be empty, or anything.
3961
3962 The next list shows the possible content of a bracket:
3963 (|) OP_*BRA | OP_ALT ... M A
3964 (?()|) OP_*COND | OP_ALT M A
3965 (?>|) OP_ONCE | OP_ALT ... [stack trace] M A
3966 (?>|) OP_ONCE_NC | OP_ALT ... [stack trace] M A
3967 Or nothing, if trace is unnecessary
3968 */
3969
3970 static uschar *compile_bracket_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3971 {
3972 DEFINE_COMPILER;
3973 fallback_common *fallback;
3974 uschar opcode;
3975 int localptr = 0;
3976 int offset = 0;
3977 int stacksize;
3978 uschar *ccbegin;
3979 uschar *hotpath;
3980 uschar bra = OP_BRA;
3981 uschar ket;
3982 assert_fallback *assert;
3983 BOOL has_alternatives;
3984 struct sljit_jump *jump;
3985 struct sljit_jump *skip;
3986 struct sljit_label *rmaxlabel = NULL;
3987 struct sljit_jump *braminzerojump = NULL;
3988
3989 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
3990
3991 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3992 {
3993 bra = *cc;
3994 cc++;
3995 opcode = *cc;
3996 }
3997
3998 opcode = *cc;
3999 ccbegin = cc;
4000 hotpath = ccbegin + 1 + LINK_SIZE;
4001
4002 if ((opcode == OP_COND || opcode == OP_SCOND) && cc[1 + LINK_SIZE] == OP_DEF)
4003 {
4004 /* Drop this bracket_fallback. */
4005 parent->top = fallback->prev;
4006 return bracketend(cc);
4007 }
4008
4009 ket = *(bracketend(cc) - 1 - LINK_SIZE);
4010 SLJIT_ASSERT(ket == OP_KET || ket == OP_KETRMAX || ket == OP_KETRMIN);
4011 SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO && ket == OP_KETRMAX)));
4012 cc += GET(cc, 1);
4013
4014 has_alternatives = *cc == OP_ALT;
4015 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
4016 {
4017 has_alternatives = (*hotpath == OP_RREF) ? FALSE : TRUE;
4018 if (*hotpath == OP_NRREF)
4019 {
4020 stacksize = GET2(hotpath, 1);
4021 if (common->currententry == NULL || stacksize == RREF_ANY)
4022 has_alternatives = FALSE;
4023 else if (common->currententry->start == 0)
4024 has_alternatives = stacksize != 0;
4025 else
4026 has_alternatives = stacksize != GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
4027 }
4028 }
4029
4030 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
4031 opcode = OP_SCOND;
4032 if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC))
4033 opcode = OP_ONCE;
4034
4035 if (opcode == OP_CBRA || opcode == OP_SCBRA)
4036 {
4037 /* Capturing brackets has a pre-allocated space. */
4038 offset = GET2(ccbegin, 1 + LINK_SIZE);
4039 localptr = OVECTOR_PRIV(offset);
4040 offset <<= 1;
4041 FALLBACK_AS(bracket_fallback)->localptr = localptr;
4042 hotpath += 2;
4043 }
4044 else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND)
4045 {
4046 /* Other brackets simply allocate the next entry. */
4047 localptr = PRIV(ccbegin);
4048 SLJIT_ASSERT(localptr != 0);
4049 FALLBACK_AS(bracket_fallback)->localptr = localptr;
4050 if (opcode == OP_ONCE)
4051 FALLBACK_AS(bracket_fallback)->u.framesize = get_framesize(common, ccbegin, FALSE);
4052 }
4053
4054 /* Instructions before the first alternative. */
4055 stacksize = 0;
4056 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
4057 stacksize++;
4058 if (bra == OP_BRAZERO)
4059 stacksize++;
4060
4061 if (stacksize > 0)
4062 allocate_stack(common, stacksize);
4063
4064 stacksize = 0;
4065 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
4066 {
4067 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4068 stacksize++;
4069 }
4070
4071 if (bra == OP_BRAZERO)
4072 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
4073
4074 if (bra == OP_BRAMINZERO)
4075 {
4076 /* This is a fallback path! (Since the hot-path of OP_BRAMINZERO matches to the empty string) */
4077 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4078 if (ket != OP_KETRMIN)
4079 {
4080 free_stack(common, 1);
4081 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4082 }
4083 else
4084 {
4085 if (opcode == OP_ONCE || opcode >= OP_SBRA)
4086 {
4087 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4088 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4089 /* Nothing stored during the first run. */
4090 skip = JUMP(SLJIT_JUMP);
4091 JUMPHERE(jump);
4092 /* Checking zero-length iteration. */
4093 if (opcode != OP_ONCE || FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4094 {
4095 /* When we come from outside, localptr contains the previous STR_PTR. */
4096 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4097 }
4098 else
4099 {
4100 /* Except when the whole stack frame must be saved. */
4101 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4102 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (FALLBACK_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w));
4103 }
4104 JUMPHERE(skip);
4105 }
4106 else
4107 {
4108 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4109 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4110 JUMPHERE(jump);
4111 }
4112 }
4113 }
4114
4115 if (ket == OP_KETRMIN)
4116 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4117
4118 if (ket == OP_KETRMAX)
4119 {
4120 rmaxlabel = LABEL();
4121 if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA)
4122 FALLBACK_AS(bracket_fallback)->althotpath = rmaxlabel;
4123 }
4124
4125 /* Handling capturing brackets and alternatives. */
4126 if (opcode == OP_ONCE)
4127 {
4128 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4129 {
4130 /* Neither capturing brackets nor recursions are not found in the block. */
4131 if (ket == OP_KETRMIN)
4132 {
4133 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4134 allocate_stack(common, 2);
4135 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4136 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4137 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
4138 }
4139 else if (ket == OP_KETRMAX || has_alternatives)
4140 {
4141 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4142 allocate_stack(common, 1);
4143 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4144 }
4145 else
4146 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4147 }
4148 else
4149 {
4150 if (ket == OP_KETRMIN || ket == OP_KETRMAX || has_alternatives)
4151 {
4152 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 2);
4153 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4154 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize + 1));
4155 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4156 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4157 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4158 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize + 1, 2, FALSE);
4159 }
4160 else
4161 {
4162 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 1);
4163 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4164 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize));
4165 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4166 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4167 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize, 1, FALSE);
4168 }
4169 }
4170 }
4171 else if (opcode == OP_CBRA || opcode == OP_SCBRA)
4172 {
4173 /* Saving the previous values. */
4174 allocate_stack(common, 3);
4175 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4176 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4177 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4178 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4179 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4180 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4181 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0);
4182 }
4183 else if (opcode == OP_SBRA || opcode == OP_SCOND)
4184 {
4185 /* Saving the previous value. */
4186 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4187 allocate_stack(common, 1);
4188 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4189 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
4190 }
4191 else if (has_alternatives)
4192 {
4193 /* Pushing the starting string pointer. */
4194 allocate_stack(common, 1);
4195 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4196 }
4197
4198 /* Generating code for the first alternative. */
4199 if (opcode == OP_COND || opcode == OP_SCOND)
4200 {
4201 if (*hotpath == OP_CREF)
4202 {
4203 SLJIT_ASSERT(has_alternatives);
4204 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed),
4205 CMP(SLJIT_C_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(GET2(hotpath, 1) << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
4206 hotpath += 3;
4207 }
4208 else if (*hotpath == OP_NCREF)
4209 {
4210 SLJIT_ASSERT(has_alternatives);
4211 stacksize = GET2(hotpath, 1);
4212 jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(stacksize << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
4213
4214 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
4215 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
4216 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
4217 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, stacksize);
4218 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0);
4219 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, common->name_table);
4220 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchovector));
4221 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4222 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, 0));
4223
4224 JUMPHERE(jump);
4225 hotpath += 3;
4226 }
4227 else if (*hotpath == OP_RREF || *hotpath == OP_NRREF)
4228 {
4229 /* Never has other case. */
4230 FALLBACK_AS(bracket_fallback)->u.condfailed = NULL;
4231
4232 stacksize = GET2(hotpath, 1);
4233 if (common->currententry == NULL)
4234 stacksize = 0;
4235 else if (stacksize == RREF_ANY)
4236 stacksize = 1;
4237 else if (common->currententry->start == 0)
4238 stacksize = stacksize == 0;
4239 else
4240 stacksize = stacksize == GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
4241
4242 if (*hotpath == OP_RREF || stacksize || common->currententry == NULL)
4243 {
4244 SLJIT_ASSERT(!has_alternatives);
4245 if (stacksize != 0)
4246 hotpath += 3;
4247 else
4248 {
4249 if (*cc == OP_ALT)
4250 {
4251 hotpath = cc + 1 + LINK_SIZE;
4252 cc += GET(cc, 1);
4253 }
4254 else
4255 hotpath = cc;
4256 }
4257 }
4258 else
4259 {
4260 SLJIT_ASSERT(has_alternatives);
4261
4262 stacksize = GET2(hotpath, 1);
4263 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
4264 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
4265 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
4266 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, GET2(common->start, common->currententry->start + 1 + LINK_SIZE));
4267 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, stacksize);
4268 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0);
4269 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, common->name_table);
4270 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchgroups));
4271 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4272 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, 0));
4273 hotpath += 3;
4274 }
4275 }
4276 else
4277 {
4278 SLJIT_ASSERT(has_alternatives && *hotpath >= OP_ASSERT && *hotpath <= OP_ASSERTBACK_NOT);
4279 /* Similar code as PUSH_FALLBACK macro. */
4280 assert = sljit_alloc_memory(compiler, sizeof(assert_fallback));
4281 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4282 return NULL;
4283 memset(assert, 0, sizeof(assert_fallback));
4284 assert->common.cc = hotpath;
4285 FALLBACK_AS(bracket_fallback)->u.assert = assert;
4286 hotpath = compile_assert_hotpath(common, hotpath, assert, TRUE);
4287 }
4288 }
4289
4290 compile_hotpath(common, hotpath, cc, fallback);
4291 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4292 return NULL;
4293
4294 if (opcode == OP_ONCE)
4295 {
4296 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4297 {
4298 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4299 /* TMP2 which is set here used by OP_KETRMAX below. */
4300 if (ket == OP_KETRMAX)
4301 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
4302 else if (ket == OP_KETRMIN)
4303 {
4304 /* Move the STR_PTR to the localptr. */
4305 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
4306 }
4307 }
4308 else
4309 {
4310 stacksize = (ket == OP_KETRMIN || ket == OP_KETRMAX || has_alternatives) ? 2 : 1;
4311 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (FALLBACK_AS(bracket_fallback)->u.framesize + stacksize) * sizeof(sljit_w));
4312 if (ket == OP_KETRMAX)
4313 {
4314 /* TMP2 which is set here used by OP_KETRMAX below. */
4315 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4316 }
4317 }
4318 }
4319
4320 stacksize = 0;
4321 if (ket != OP_KET || bra != OP_BRA)
4322 stacksize++;
4323 if (has_alternatives && opcode != OP_ONCE)
4324 stacksize++;
4325
4326 if (stacksize > 0)
4327 allocate_stack(common, stacksize);
4328
4329 stacksize = 0;
4330 if (ket != OP_KET)
4331 {
4332 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
4333 stacksize++;
4334 }
4335 else if (bra != OP_BRA)
4336 {
4337 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4338 stacksize++;
4339 }
4340
4341 if (has_alternatives)
4342 {
4343 if (opcode != OP_ONCE)
4344 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4345 if (ket != OP_KETRMAX)
4346 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4347 }
4348
4349 /* Must be after the hotpath label. */
4350 if (offset != 0)
4351 {
4352 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4353 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4354 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
4355 }
4356
4357 if (ket == OP_KETRMAX)
4358 {
4359 if (opcode == OP_ONCE || opcode >= OP_SBRA)
4360 {
4361 if (has_alternatives)
4362 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4363 /* Checking zero-length iteration. */
4364 if (opcode != OP_ONCE)
4365 CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0, rmaxlabel);
4366 else
4367 /* TMP2 must contain the starting STR_PTR. */
4368 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmaxlabel);
4369 }
4370 else
4371 JUMPTO(SLJIT_JUMP, rmaxlabel);
4372 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4373 }
4374
4375 if (bra == OP_BRAZERO)
4376 FALLBACK_AS(bracket_fallback)->zerohotpath = LABEL();
4377
4378 if (bra == OP_BRAMINZERO)
4379 {
4380 /* This is a fallback path! (From the viewpoint of OP_BRAMINZERO) */
4381 JUMPTO(SLJIT_JUMP, ((braminzero_fallback*)parent)->hotpath);
4382 if (braminzerojump != NULL)
4383 {
4384 JUMPHERE(braminzerojump);
4385 /* We need to release the end pointer to perform the
4386 fallback for the zero-length iteration. When
4387 framesize is < 0, OP_ONCE will do the release itself. */
4388 if (opcode == OP_ONCE && FALLBACK_AS(bracket_fallback)->u.framesize >= 0)
4389 {
4390 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4391 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
4392 }
4393 else if (ket == OP_KETRMIN && opcode != OP_ONCE)
4394 free_stack(common, 1);
4395 }
4396 /* Continue to the normal fallback. */
4397 }
4398
4399 if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO)
4400 decrease_call_count(common);
4401
4402 /* Skip the other alternatives. */
4403 while (*cc == OP_ALT)
4404 cc += GET(cc, 1);
4405 cc += 1 + LINK_SIZE;
4406 return cc;
4407 }
4408
4409 static uschar *compile_bracketpos_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4410 {
4411 DEFINE_COMPILER;
4412 fallback_common *fallback;
4413 uschar opcode;
4414 int localptr;
4415 int cbraprivptr = 0;
4416 int framesize;
4417 int stacksize;
4418 int offset = 0;
4419 BOOL zero = FALSE;
4420 uschar *ccbegin = NULL;
4421 int stack;
4422 struct sljit_label *loop = NULL;
4423 struct jump_list *emptymatch = NULL;
4424
4425 PUSH_FALLBACK(sizeof(bracketpos_fallback), cc, NULL);
4426 if (*cc == OP_BRAPOSZERO)
4427 {
4428 zero = TRUE;
4429 cc++;
4430 }
4431
4432 opcode = *cc;
4433 localptr = PRIV(cc);
4434 SLJIT_ASSERT(localptr != 0);
4435 FALLBACK_AS(bracketpos_fallback)->localptr = localptr;
4436 switch(opcode)
4437 {
4438 case OP_BRAPOS:
4439 case OP_SBRAPOS:
4440 ccbegin = cc + 1 + LINK_SIZE;
4441 break;
4442
4443 case OP_CBRAPOS:
4444 case OP_SCBRAPOS:
4445 offset = GET2(cc, 1 + LINK_SIZE);
4446 cbraprivptr = OVECTOR_PRIV(offset);
4447 offset <<= 1;
4448 ccbegin = cc + 1 + LINK_SIZE + 2;
4449 break;
4450
4451 default:
4452 SLJIT_ASSERT_STOP();
4453 break;
4454 }
4455
4456 framesize = get_framesize(common, cc, FALSE);
4457 FALLBACK_AS(bracketpos_fallback)->framesize = framesize;
4458 if (framesize < 0)
4459 {
4460 stacksize = (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS) ? 2 : 1;
4461 if (!zero)
4462 stacksize++;
4463 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4464 allocate_stack(common, stacksize);
4465 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4466
4467 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4468 {
4469 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4470 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4471 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4472 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4473 }
4474 else
4475 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4476
4477 if (!zero)
4478 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 1);
4479 }
4480 else
4481 {
4482 stacksize = framesize + 1;
4483 if (!zero)
4484 stacksize++;
4485 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4486 stacksize++;
4487 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4488 allocate_stack(common, stacksize);
4489
4490 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4491 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
4492 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4493 stack = 0;
4494 if (!zero)
4495 {
4496 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 1);
4497 stack++;
4498 }
4499 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4500 {
4501 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), STR_PTR, 0);
4502 stack++;
4503 }
4504 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP1, 0);
4505 init_frame(common, cc, stacksize - 1, stacksize - framesize, FALSE);
4506 }
4507
4508 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4509 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4510
4511 loop = LABEL();
4512 while (*cc != OP_KETRPOS)
4513 {
4514 fallback->top = NULL;
4515 fallback->topfallbacks = NULL;
4516 cc += GET(cc, 1);
4517
4518 compile_hotpath(common, ccbegin, cc, fallback);
4519 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4520 return NULL;
4521
4522 if (framesize < 0)
4523 {
4524 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4525
4526 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4527 {
4528 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4529 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4530 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4531 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4532 }
4533 else
4534 {
4535 if (opcode == OP_SBRAPOS)
4536 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4537 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4538 }
4539
4540 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4541 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4542
4543 if (!zero)
4544 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4545 }
4546 else
4547 {
4548 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4549 {
4550 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, stacksize * sizeof(sljit_w));
4551 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4552 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4553 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4554 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4555 }
4556 else
4557 {
4558 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4559 OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_w));
4560 if (opcode == OP_SBRAPOS)
4561 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4562 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w), STR_PTR, 0);
4563 }
4564
4565 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4566 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4567
4568 if (!zero)
4569 {
4570 if (framesize < 0)
4571 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4572 else
4573 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4574 }
4575 }
4576 JUMPTO(SLJIT_JUMP, loop);
4577 flush_stubs(common);
4578
4579 compile_fallbackpath(common, fallback->top);
4580 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4581 return NULL;
4582 set_jumps(fallback->topfallbacks, LABEL());
4583
4584 if (framesize < 0)
4585 {
4586 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4587 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4588 else
4589 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4590 }
4591 else
4592 {
4593 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4594 {
4595 /* Last alternative. */
4596 if (*cc == OP_KETRPOS)
4597 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4598 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4599 }
4600 else
4601 {
4602 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4603 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4604 }
4605 }
4606
4607 if (*cc == OP_KETRPOS)
4608 break;
4609 ccbegin = cc + 1 + LINK_SIZE;
4610 }
4611
4612 fallback->topfallbacks = NULL;
4613 if (!zero)
4614 {
4615 if (framesize < 0)
4616 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
4617 else /* TMP2 is set to [localptr] above. */
4618 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_w), SLJIT_IMM, 0));
4619 }
4620
4621 /* None of them matched. */
4622 set_jumps(emptymatch, LABEL());
4623 decrease_call_count(common);
4624 return cc + 1 + LINK_SIZE;
4625 }
4626
4627 static SLJIT_INLINE uschar *get_iterator_parameters(compiler_common *common, uschar *cc, uschar *opcode, uschar *type, int *arg1, int *arg2, uschar **end)
4628 {
4629 int class_len;
4630
4631 *opcode = *cc;
4632 if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO)
4633 {
4634 cc++;
4635 *type = OP_CHAR;
4636 }
4637 else if (*opcode >= OP_STARI && *opcode <= OP_POSUPTOI)
4638 {
4639 cc++;
4640 *type = OP_CHARI;
4641 *opcode -= OP_STARI - OP_STAR;
4642 }
4643 else if (*opcode >= OP_NOTSTAR && *opcode <= OP_NOTPOSUPTO)
4644 {
4645 cc++;
4646 *type = OP_NOT;
4647 *opcode -= OP_NOTSTAR - OP_STAR;
4648 }
4649 else if (*opcode >= OP_NOTSTARI && *opcode <= OP_NOTPOSUPTOI)
4650 {
4651 cc++;
4652 *type = OP_NOTI;
4653 *opcode -= OP_NOTSTARI - OP_STAR;
4654 }
4655 else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO)
4656 {
4657 cc++;
4658 *opcode -= OP_TYPESTAR - OP_STAR;
4659 *type = 0;
4660 }
4661 else
4662 {
4663 SLJIT_ASSERT(*opcode >= OP_CLASS || *opcode <= OP_XCLASS);
4664 *type = *opcode;
4665 cc++;
4666 class_len = (*type < OP_XCLASS) ? 33 : GET(cc, 0);
4667 *opcode = cc[class_len - 1];
4668 if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY)
4669 {
4670 *opcode -= OP_CRSTAR - OP_STAR;
4671 if (end != NULL)
4672 *end = cc + class_len;
4673 }
4674 else
4675 {
4676 SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE);
4677 *arg1 = GET2(cc, (class_len + 2));
4678 *arg2 = GET2(cc, class_len);
4679
4680 if (*arg2 == 0)
4681 {
4682 SLJIT_ASSERT(*arg1 != 0);
4683 *opcode = (*opcode == OP_CRRANGE) ? OP_UPTO : OP_MINUPTO;
4684 }
4685 if (*arg1 == *arg2)
4686 *opcode = OP_EXACT;
4687
4688 if (end != NULL)
4689 *end = cc + class_len + 4;
4690 }
4691 return cc;
4692 }
4693
4694 if (*opcode == OP_UPTO || *opcode == OP_MINUPTO || *opcode == OP_EXACT || *opcode == OP_POSUPTO)
4695 {
4696 *arg1 = GET2(cc, 0);
4697 cc += 2;
4698 }
4699
4700 if (*type == 0)
4701 {
4702 *type = *cc;
4703 if (end != NULL)
4704 *end = next_opcode(common, cc);
4705 cc++;
4706 return cc;
4707 }
4708
4709 if (end != NULL)
4710 {
4711 *end = cc + 1;
4712 #ifdef SUPPORT_UTF8
4713 if (common->utf8 && *cc >= 0xc0) *end += _pcre_utf8_table4[*cc & 0x3f];
4714 #endif
4715 }
4716 return cc;
4717 }
4718
4719 static uschar *compile_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4720 {
4721 DEFINE_COMPILER;
4722 fallback_common *fallback;
4723 uschar opcode;
4724 uschar type;
4725 int arg1 = -1, arg2 = -1;
4726 uschar* end;
4727 jump_list *nomatch = NULL;
4728 struct sljit_jump *jump = NULL;
4729 struct sljit_label *label;
4730
4731 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
4732
4733 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, &end);
4734
4735 switch(opcode)
4736 {
4737 case OP_STAR:
4738 case OP_PLUS:
4739 case OP_UPTO:
4740 case OP_CRRANGE:
4741 if (type == OP_ANYNL || type == OP_EXTUNI)
4742 {
4743 if (opcode == OP_STAR || opcode == OP_UPTO)
4744 {
4745 allocate_stack(common, 2);
4746 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4747 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
4748 }
4749 else
4750 {
4751 allocate_stack(common, 1);
4752 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4753 }
4754 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4755 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
4756
4757 label = LABEL();
4758 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4759 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4760 {
4761 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4762 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4763 if (opcode == OP_CRRANGE && arg2 > 0)
4764 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2, label);
4765 if (opcode == OP_UPTO || (opcode == OP_CRRANGE && arg1 > 0))
4766 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, arg1);
4767 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4768 }
4769
4770 allocate_stack(common, 1);
4771 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4772 JUMPTO(SLJIT_JUMP, label);
4773 if (jump != NULL)
4774 JUMPHERE(jump);
4775 }
4776 else
4777 {
4778 allocate_stack(common, 2);
4779 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4780 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4781 label = LABEL();
4782 compile_char1_hotpath(common, type, cc, &nomatch);
4783 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4784 if (opcode <= OP_PLUS || (opcode == OP_CRRANGE && arg1 == 0))
4785 {
4786 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4787 JUMPTO(SLJIT_JUMP, label);
4788 }
4789 else
4790 {
4791 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4792 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4793 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4794 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4795 }
4796 set_jumps(nomatch, LABEL());
4797 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
4798 add_jump(compiler, &fallback->topfallbacks,
4799 CMP(SLJIT_C_LESS, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, opcode == OP_PLUS ? 2 : arg2 + 1));
4800 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4801 }
4802 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4803 break;
4804
4805 case OP_MINSTAR:
4806 case OP_MINPLUS:
4807 allocate_stack(common, 1);
4808 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4809 if (opcode == OP_MINPLUS)
4810 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4811 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4812 break;
4813
4814 case OP_MINUPTO:
4815 case OP_CRMINRANGE:
4816 allocate_stack(common, 2);
4817 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4818 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4819 if (opcode == OP_CRMINRANGE)
4820 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4821 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4822 break;
4823
4824 case OP_QUERY:
4825 case OP_MINQUERY:
4826 allocate_stack(common, 1);
4827 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4828 if (opcode == OP_QUERY)
4829 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4830 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4831 break;
4832
4833 case OP_EXACT:
4834 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4835 label = LABEL();
4836 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4837 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4838 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4839 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4840 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4841 break;
4842
4843 case OP_POSSTAR:
4844 case OP_POSPLUS:
4845 case OP_POSUPTO:
4846 if (opcode != OP_POSSTAR)
4847 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4848 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4849 label = LABEL();
4850 compile_char1_hotpath(common, type, cc, &nomatch);
4851 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4852 if (opcode != OP_POSUPTO)
4853 {
4854 if (opcode == OP_POSPLUS)
4855 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2);
4856 JUMPTO(SLJIT_JUMP, label);
4857 }
4858 else
4859 {
4860 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4861 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4862 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4863 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4864 }
4865 set_jumps(nomatch, LABEL());
4866 if (opcode == OP_POSPLUS)
4867 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_LESS, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2));
4868 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4869 break;
4870
4871 case OP_POSQUERY:
4872 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4873 compile_char1_hotpath(common, type, cc, &nomatch);
4874 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4875 set_jumps(nomatch, LABEL());
4876 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4877 break;
4878
4879 default:
4880 SLJIT_ASSERT_STOP();
4881 break;
4882 }
4883
4884 decrease_call_count(common);
4885 return end;
4886 }
4887
4888 static SLJIT_INLINE uschar *compile_fail_accept_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4889 {
4890 DEFINE_COMPILER;
4891 fallback_common *fallback;
4892
4893 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
4894
4895 if (*cc == OP_FAIL)
4896 {
4897 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4898 return cc + 1;
4899 }
4900
4901 if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL)
4902 {
4903 /* No need to check notempty conditions. */
4904 if (common->acceptlabel == NULL)
4905 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
4906 else
4907 JUMPTO(SLJIT_JUMP, common->acceptlabel);
4908 return cc + 1;
4909 }
4910
4911 if (common->acceptlabel == NULL)
4912 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0)));
4913 else
4914 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), common->acceptlabel);
4915 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
4916 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
4917 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4918 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
4919 if (common->acceptlabel == NULL)
4920 add_jump(compiler, &common->accept, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4921 else
4922 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->acceptlabel);
4923 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
4924 if (common->acceptlabel == NULL)
4925 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0));
4926 else
4927 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->acceptlabel);
4928 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4929 return cc + 1;
4930 }
4931
4932 static SLJIT_INLINE uschar *compile_close_hotpath(compiler_common *common, uschar *cc)
4933 {
4934 DEFINE_COMPILER;
4935 int offset = GET2(cc, 1);
4936
4937 /* Data will be discarded anyway... */
4938 if (common->currententry != NULL)
4939 return cc + 3;
4940
4941 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR_PRIV(offset));
4942 offset <<= 1;
4943 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4944 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4945 return cc + 3;
4946 }
4947
4948 static void compile_hotpath(compiler_common *common, uschar *cc, uschar *ccend, fallback_common *parent)
4949 {
4950 DEFINE_COMPILER;
4951 fallback_common *fallback;
4952
4953 while (cc < ccend)
4954 {
4955 switch(*cc)
4956 {
4957 case OP_SOD:
4958 case OP_SOM:
4959 case OP_NOT_WORD_BOUNDARY:
4960 case OP_WORD_BOUNDARY:
4961 case OP_NOT_DIGIT:
4962 case OP_DIGIT:
4963 case OP_NOT_WHITESPACE:
4964 case OP_WHITESPACE:
4965 case OP_NOT_WORDCHAR:
4966 case OP_WORDCHAR:
4967 case OP_ANY:
4968 case OP_ALLANY:
4969 case OP_ANYBYTE:
4970 case OP_NOTPROP:
4971 case OP_PROP:
4972 case OP_ANYNL:
4973 case OP_NOT_HSPACE:
4974 case OP_HSPACE:
4975 case OP_NOT_VSPACE:
4976 case OP_VSPACE:
4977 case OP_EXTUNI:
4978 case OP_EODN:
4979 case OP_EOD:
4980 case OP_CIRC:
4981 case OP_CIRCM:
4982 case OP_DOLL:
4983 case OP_DOLLM:
4984 case OP_NOT:
4985 case OP_NOTI:
4986 case OP_REVERSE:
4987 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4988 break;
4989
4990 case OP_SET_SOM:
4991 PUSH_FALLBACK_NOVALUE(sizeof(fallback_common), cc);
4992 allocate_stack(common, 1);
4993 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
4994 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
4995 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4996 cc++;
4997 break;
4998
4999 case OP_CHAR:
5000 case OP_CHARI:
5001 cc = compile_charn_hotpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
5002 break;
5003
5004 case OP_STAR:
5005 case OP_MINSTAR:
5006 case OP_PLUS:
5007 case OP_MINPLUS:
5008 case OP_QUERY:
5009 case OP_MINQUERY:
5010 case OP_UPTO:
5011 case OP_MINUPTO:
5012 case OP_EXACT:
5013 case OP_POSSTAR:
5014 case OP_POSPLUS:
5015 case OP_POSQUERY:
5016 case OP_POSUPTO:
5017 case OP_STARI:
5018 case OP_MINSTARI:
5019 case OP_PLUSI:
5020 case OP_MINPLUSI:
5021 case OP_QUERYI:
5022 case OP_MINQUERYI:
5023 case OP_UPTOI:
5024 case OP_MINUPTOI:
5025 case OP_EXACTI:
5026 case OP_POSSTARI:
5027 case OP_POSPLUSI:
5028 case OP_POSQUERYI:
5029 case OP_POSUPTOI:
5030 case OP_NOTSTAR:
5031 case OP_NOTMINSTAR:
5032 case OP_NOTPLUS:
5033 case OP_NOTMINPLUS:
5034 case OP_NOTQUERY:
5035 case OP_NOTMINQUERY:
5036 case OP_NOTUPTO:
5037 case OP_NOTMINUPTO:
5038 case OP_NOTEXACT:
5039 case OP_NOTPOSSTAR:
5040 case OP_NOTPOSPLUS:
5041 case OP_NOTPOSQUERY:
5042 case OP_NOTPOSUPTO:
5043 case OP_NOTSTARI:
5044 case OP_NOTMINSTARI:
5045 case OP_NOTPLUSI:
5046 case OP_NOTMINPLUSI:
5047 case OP_NOTQUERYI:
5048 case OP_NOTMINQUERYI:
5049 case OP_NOTUPTOI:
5050 case OP_NOTMINUPTOI:
5051 case OP_NOTEXACTI:
5052 case OP_NOTPOSSTARI:
5053 case OP_NOTPOSPLUSI:
5054 case OP_NOTPOSQUERYI:
5055 case OP_NOTPOSUPTOI:
5056 case OP_TYPESTAR:
5057 case OP_TYPEMINSTAR:
5058 case OP_TYPEPLUS:
5059 case OP_TYPEMINPLUS:
5060 case OP_TYPEQUERY:
5061 case OP_TYPEMINQUERY:
5062 case OP_TYPEUPTO:
5063 case OP_TYPEMINUPTO:
5064 case OP_TYPEEXACT:
5065 case OP_TYPEPOSSTAR:
5066 case OP_TYPEPOSPLUS:
5067 case OP_TYPEPOSQUERY:
5068 case OP_TYPEPOSUPTO:
5069 cc = compile_iterator_hotpath(common, cc, parent);
5070 break;
5071
5072 case OP_CLASS:
5073 case OP_NCLASS:
5074 if (cc[33] >= OP_CRSTAR && cc[33] <= OP_CRMINRANGE)
5075 cc = compile_iterator_hotpath(common, cc, parent);
5076 else
5077 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
5078 break;
5079
5080 #ifdef SUPPORT_UTF8
5081 case OP_XCLASS:
5082 if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRMINRANGE)
5083 cc = compile_iterator_hotpath(common, cc, parent);
5084 else
5085 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
5086 break;
5087 #endif
5088
5089 case OP_REF:
5090 case OP_REFI:
5091 if (cc[3] >= OP_CRSTAR && cc[3] <= OP_CRMINRANGE)
5092 cc = compile_ref_iterator_hotpath(common, cc, parent);
5093 else
5094 cc = compile_ref_hotpath(common, cc, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks, TRUE, FALSE);
5095 break;
5096
5097 case OP_RECURSE:
5098 cc = compile_recurse_hotpath(common, cc, parent);
5099 break;
5100
5101 case OP_ASSERT:
5102 case OP_ASSERT_NOT:
5103 case OP_ASSERTBACK:
5104 case OP_ASSERTBACK_NOT:
5105 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
5106 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
5107 break;
5108
5109 case OP_BRAMINZERO:
5110 PUSH_FALLBACK_NOVALUE(sizeof(braminzero_fallback), cc);
5111 cc = bracketend(cc + 1);
5112 if (*(cc - 1 - LINK_SIZE) != OP_KETRMIN)
5113 {
5114 allocate_stack(common, 1);
5115 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5116 }
5117 else
5118 {
5119 allocate_stack(common, 2);
5120 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5121 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0);
5122 }
5123 FALLBACK_AS(braminzero_fallback)->hotpath = LABEL();
5124 if (cc[1] > OP_ASSERTBACK_NOT)
5125 decrease_call_count(common);
5126 break;
5127
5128 case OP_ONCE:
5129 case OP_ONCE_NC:
5130 case OP_BRA:
5131 case OP_CBRA:
5132 case OP_COND:
5133 case OP_SBRA:
5134 case OP_SCBRA:
5135 case OP_SCOND:
5136 cc = compile_bracket_hotpath(common, cc, parent);
5137 break;
5138
5139 case OP_BRAZERO:
5140 if (cc[1] > OP_ASSERTBACK_NOT)
5141 cc = compile_bracket_hotpath(common, cc, parent);
5142 else
5143 {
5144 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
5145 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
5146 }
5147 break;
5148
5149 case OP_BRAPOS:
5150 case OP_CBRAPOS:
5151 case OP_SBRAPOS:
5152 case OP_SCBRAPOS:
5153 case OP_BRAPOSZERO:
5154 cc = compile_bracketpos_hotpath(common, cc, parent);
5155 break;
5156
5157 case OP_FAIL:
5158 case OP_ACCEPT:
5159 case OP_ASSERT_ACCEPT:
5160 cc = compile_fail_accept_hotpath(common, cc, parent);
5161 break;
5162
5163 case OP_CLOSE:
5164 cc = compile_close_hotpath(common, cc);
5165 break;
5166
5167 case OP_SKIPZERO:
5168 cc = bracketend(cc + 1);
5169 break;
5170
5171 default:
5172 SLJIT_ASSERT_STOP();
5173 return;
5174 }
5175 if (cc == NULL)
5176 return;
5177 }
5178 SLJIT_ASSERT(cc == ccend);
5179 }
5180
5181 #undef PUSH_FALLBACK
5182 #undef PUSH_FALLBACK_NOVALUE
5183 #undef FALLBACK_AS
5184
5185 #define COMPILE_FALLBACKPATH(current) \
5186 do \
5187 { \
5188 compile_fallbackpath(common, (current)); \
5189 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
5190 return; \
5191 } \
5192 while (0)
5193
5194 #define CURRENT_AS(type) ((type*)current)
5195
5196 static void compile_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5197 {
5198 DEFINE_COMPILER;
5199 uschar *cc = current->cc;
5200 uschar opcode;
5201 uschar type;
5202 int arg1 = -1, arg2 = -1;
5203 struct sljit_label *label = NULL;
5204 struct sljit_jump *jump = NULL;
5205
5206 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, NULL);
5207
5208 switch(opcode)
5209 {
5210 case OP_STAR:
5211 case OP_PLUS:
5212 case OP_UPTO:
5213 case OP_CRRANGE:
5214 if (type == OP_ANYNL || type == OP_EXTUNI)
5215 {
5216 set_jumps(current->topfallbacks, LABEL());
5217 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5218 free_stack(common, 1);
5219 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5220 }
5221 else
5222 {
5223 if (opcode == OP_STAR || opcode == OP_UPTO)
5224 arg2 = 0;
5225 else if (opcode == OP_PLUS)
5226 arg2 = 1;
5227 jump = CMP(SLJIT_C_LESS_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, arg2 + 1);
5228 OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
5229 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5230 skip_char_back(common);
5231 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5232 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5233 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
5234 set_jumps(current->topfallbacks, LABEL());
5235 JUMPHERE(jump);
5236 free_stack(common, 2);
5237 }
5238 break;
5239
5240 case OP_MINSTAR:
5241 case OP_MINPLUS:
5242 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5243 if (opcode == OP_MINPLUS)
5244 {
5245 set_jumps(current->topfallbacks, LABEL());
5246 current->topfallbacks = NULL;
5247 }
5248 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5249 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5250 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5251 set_jumps(current->topfallbacks, LABEL());
5252 free_stack(common, 1);
5253 break;
5254
5255 case OP_MINUPTO:
5256 case OP_CRMINRANGE:
5257 if (opcode == OP_CRMINRANGE)
5258 {
5259 set_jumps(current->topfallbacks, LABEL());
5260 current->topfallbacks = NULL;
5261 label = LABEL();
5262 }
5263 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5264 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5265
5266 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5267 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5268 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
5269 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
5270
5271 if (opcode == OP_CRMINRANGE)
5272 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2 + 1, label);
5273
5274 if (opcode == OP_CRMINRANGE && arg1 == 0)
5275 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5276 else
5277 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 2, CURRENT_AS(iterator_fallback)->hotpath);
5278
5279 set_jumps(current->topfallbacks, LABEL());
5280 free_stack(common, 2);
5281 break;
5282
5283 case OP_QUERY:
5284 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5285 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5286 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5287 jump = JUMP(SLJIT_JUMP);
5288 set_jumps(current->topfallbacks, LABEL());
5289 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5290 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5291 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5292 JUMPHERE(jump);
5293 free_stack(common, 1);
5294 break;
5295
5296 case OP_MINQUERY:
5297 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5298 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5299 jump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5300 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5301 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5302 set_jumps(current->topfallbacks, LABEL());
5303 JUMPHERE(jump);
5304 free_stack(common, 1);
5305 break;
5306
5307 case OP_EXACT:
5308 case OP_POSPLUS:
5309 set_jumps(current->topfallbacks, LABEL());
5310 break;
5311
5312 case OP_POSSTAR:
5313 case OP_POSQUERY:
5314 case OP_POSUPTO:
5315 break;
5316
5317 default:
5318 SLJIT_ASSERT_STOP();
5319 break;
5320 }
5321 }
5322
5323 static void compile_ref_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5324 {
5325 DEFINE_COMPILER;
5326 uschar *cc = current->cc;
5327 uschar type;
5328
5329 type = cc[3];
5330 if ((type & 0x1) == 0)
5331 {
5332 set_jumps(current->topfallbacks, LABEL());
5333 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5334 free_stack(common, 1);
5335 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5336 return;
5337 }
5338
5339 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5340 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5341 set_jumps(current->topfallbacks, LABEL());
5342 free_stack(common, 2);
5343 }
5344
5345 static void compile_recurse_fallbackpath(compiler_common *common, struct fallback_common *current)
5346 {
5347 DEFINE_COMPILER;
5348
5349 set_jumps(current->topfallbacks, LABEL());
5350 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5351 free_stack(common, 1);
5352 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
5353 }
5354
5355 static void compile_assert_fallbackpath(compiler_common *common, struct fallback_common *current)
5356 {
5357 DEFINE_COMPILER;
5358 uschar *cc = current->cc;
5359 uschar bra = OP_BRA;
5360 struct sljit_jump *brajump = NULL;
5361
5362 SLJIT_ASSERT(*cc != OP_BRAMINZERO);
5363 if (*cc == OP_BRAZERO)
5364 {
5365 bra = *cc;
5366 cc++;
5367 }
5368
5369 if (bra == OP_BRAZERO)
5370 {
5371 SLJIT_ASSERT(current->topfallbacks == NULL);
5372 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5373 }
5374
5375 if (CURRENT_AS(assert_fallback)->framesize < 0)
5376 {
5377 set_jumps(current->topfallbacks, LABEL());
5378
5379 if (bra == OP_BRAZERO)
5380 {
5381 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5382 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5383 free_stack(common, 1);
5384 }
5385 return;
5386 }
5387
5388 if (bra == OP_BRAZERO)
5389 {
5390 if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT)
5391 {
5392 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5393 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5394 free_stack(common, 1);
5395 return;
5396 }
5397 free_stack(common, 1);
5398 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5399 }
5400
5401 if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK)
5402 {
5403 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr);
5404 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5405 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_fallback)->framesize * sizeof(sljit_w));
5406
5407 set_jumps(current->topfallbacks, LABEL());
5408 }
5409 else
5410 set_jumps(current->topfallbacks, LABEL());
5411
5412 if (bra == OP_BRAZERO)
5413 {
5414 /* We know there is enough place on the stack. */
5415 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5416 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5417 JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_fallback)->hotpath);
5418 JUMPHERE(brajump);
5419 }
5420 }
5421
5422 static void compile_bracket_fallbackpath(compiler_common *common, struct fallback_common *current)
5423 {
5424 DEFINE_COMPILER;
5425 int opcode;
5426 int offset = 0;
5427 int localptr = CURRENT_AS(bracket_fallback)->localptr;
5428 int stacksize;
5429 int count;
5430 uschar *cc = current->cc;
5431 uschar *ccbegin;
5432 uschar *ccprev;
5433 jump_list *jumplist = NULL;
5434 jump_list *jumplistitem = NULL;
5435 uschar bra = OP_BRA;
5436 uschar ket;
5437 assert_fallback *assert;
5438 BOOL has_alternatives;
5439 struct sljit_jump *brazero = NULL;
5440 struct sljit_jump *once = NULL;
5441 struct sljit_jump *cond = NULL;
5442 struct sljit_label *rminlabel = NULL;
5443
5444 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
5445 {
5446 bra = *cc;
5447 cc++;
5448 }
5449
5450 opcode = *cc;
5451 ccbegin = cc;
5452 ket = *(bracketend(ccbegin) - 1 - LINK_SIZE);
5453 cc += GET(cc, 1);
5454 has_alternatives = *cc == OP_ALT;
5455 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5456 has_alternatives = (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) || CURRENT_AS(bracket_fallback)->u.condfailed != NULL;
5457 if (opcode == OP_CBRA || opcode == OP_SCBRA)
5458 offset = (GET2(ccbegin, 1 + LINK_SIZE)) << 1;
5459 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
5460 opcode = OP_SCOND;
5461 if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC))
5462 opcode = OP_ONCE;
5463
5464 if (ket == OP_KETRMAX)
5465 {
5466 if (bra != OP_BRAZERO)
5467 free_stack(common, 1);
5468 else
5469 {
5470 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5471 free_stack(common, 1);
5472 brazero = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5473 }
5474 }
5475 else if (ket == OP_KETRMIN)
5476 {
5477 if (bra != OP_BRAMINZERO)
5478 {
5479 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5480 if (opcode >= OP_SBRA || opcode == OP_ONCE)
5481 {
5482 /* Checking zero-length iteration. */
5483 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize < 0)
5484 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, CURRENT_AS(bracket_fallback)->recursivehotpath);
5485 else
5486 {
5487 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5488 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w), CURRENT_AS(bracket_fallback)->recursivehotpath);
5489 }
5490 if (opcode != OP_ONCE)
5491 free_stack(common, 1);
5492 }
5493 else
5494 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->recursivehotpath);
5495 }
5496 rminlabel = LABEL();
5497 }
5498 else if (bra == OP_BRAZERO)
5499 {
5500 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5501 free_stack(common, 1);
5502 brazero = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5503 }
5504
5505 if (SLJIT_UNLIKELY(opcode == OP_ONCE))
5506 {
5507 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5508 {
5509 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5510 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5511 }
5512 once = JUMP(SLJIT_JUMP);
5513 }
5514 else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5515 {
5516 if (has_alternatives)
5517 {
5518 /* Always exactly one alternative. */
5519 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5520 free_stack(common, 1);
5521
5522 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5523 if (SLJIT_UNLIKELY(!jumplistitem))
5524 return;
5525 jumplist = jumplistitem;
5526 jumplistitem->next = NULL;
5527 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 1);
5528 }
5529 }
5530 else if (*cc == OP_ALT)
5531 {
5532 /* Build a jump list. Get the last successfully matched branch index. */
5533 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5534 free_stack(common, 1);
5535 count = 1;
5536 do
5537 {
5538 /* Append as the last item. */
5539 if (jumplist != NULL)
5540 {
5541 jumplistitem->next = sljit_alloc_memory(compiler, sizeof(jump_list));
5542 jumplistitem = jumplistitem->next;
5543 }
5544 else
5545 {
5546 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5547 jumplist = jumplistitem;
5548 }
5549
5550 if (SLJIT_UNLIKELY(!jumplistitem))
5551 return;
5552
5553 jumplistitem->next = NULL;
5554 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, count++);
5555 cc += GET(cc, 1);
5556 }
5557 while (*cc == OP_ALT);
5558
5559 cc = ccbegin + GET(ccbegin, 1);
5560 }
5561
5562 COMPILE_FALLBACKPATH(current->top);
5563 if (current->topfallbacks)
5564 set_jumps(current->topfallbacks, LABEL());
5565
5566 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5567 {
5568 /* Conditional block always has at most one alternative. */
5569 if (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT)
5570 {
5571 SLJIT_ASSERT(has_alternatives);
5572 assert = CURRENT_AS(bracket_fallback)->u.assert;
5573 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK))
5574 {
5575 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5576 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5577 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5578 }
5579 cond = JUMP(SLJIT_JUMP);
5580 set_jumps(CURRENT_AS(bracket_fallback)->u.assert->condfailed, LABEL());
5581 }
5582 else if (CURRENT_AS(bracket_fallback)->u.condfailed != NULL)
5583 {
5584 SLJIT_ASSERT(has_alternatives);
5585 cond = JUMP(SLJIT_JUMP);
5586 set_jumps(CURRENT_AS(bracket_fallback)->u.condfailed, LABEL());
5587 }
5588 else
5589 SLJIT_ASSERT(!has_alternatives);
5590 }
5591
5592 if (has_alternatives)
5593 {
5594 count = 1;
5595 do
5596 {
5597 current->top = NULL;
5598 current->topfallbacks = NULL;
5599 current->nextfallbacks = NULL;
5600 if (*cc == OP_ALT)
5601 {
5602 ccprev = cc + 1 + LINK_SIZE;
5603 cc += GET(cc, 1);
5604 if (opcode != OP_COND && opcode != OP_SCOND)
5605 {
5606 if (localptr != 0 && opcode != OP_ONCE)
5607 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5608 else
5609 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5610 }
5611 compile_hotpath(common, ccprev, cc, current);
5612 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
5613 return;
5614 }
5615
5616 /* Instructions after the current alternative is succesfully matched. */
5617 /* There is a similar code in compile_bracket_hotpath. */
5618 if (opcode == OP_ONCE)
5619 {
5620 if (CURRENT_AS(bracket_fallback)->u.framesize < 0)
5621 {
5622 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5623 /* TMP2 which is set here used by OP_KETRMAX below. */
5624 if (ket == OP_KETRMAX)
5625 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
5626 else if (ket == OP_KETRMIN)
5627 {
5628 /* Move the STR_PTR to the localptr. */
5629 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
5630 }
5631 }
5632 else
5633 {
5634 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (CURRENT_AS(bracket_fallback)->u.framesize + 2) * sizeof(sljit_w));
5635 if (ket == OP_KETRMAX)
5636 {
5637 /* TMP2 which is set here used by OP_KETRMAX below. */
5638 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5639 }
5640 }
5641 }
5642
5643 stacksize = 0;
5644 if (opcode != OP_ONCE)
5645 stacksize++;
5646 if (ket != OP_KET || bra != OP_BRA)
5647 stacksize++;
5648
5649 if (stacksize > 0) {
5650 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5651 allocate_stack(common, stacksize);
5652 else
5653 {
5654 /* We know we have place at least for one item on the top of the stack. */
5655 SLJIT_ASSERT(stacksize == 1);
5656 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5657 }
5658 }
5659
5660 stacksize = 0;
5661 if (ket != OP_KET || bra != OP_BRA)
5662 {
5663 if (ket != OP_KET)
5664 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
5665 else
5666 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
5667 stacksize++;
5668 }
5669
5670 if (opcode != OP_ONCE)
5671 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, count++);
5672
5673 if (offset != 0)
5674 {
5675 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5676 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
5677 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
5678 }
5679
5680 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->althotpath);
5681
5682 if (opcode != OP_ONCE)
5683 {
5684 SLJIT_ASSERT(jumplist);
5685 JUMPHERE(jumplist->jump);
5686 jumplist = jumplist->next;
5687 }
5688
5689 COMPILE_FALLBACKPATH(current->top);
5690 if (current->topfallbacks)
5691 set_jumps(current->topfallbacks, LABEL());
5692 SLJIT_ASSERT(!current->nextfallbacks);
5693 }
5694 while (*cc == OP_ALT);
5695 SLJIT_ASSERT(!jumplist);
5696
5697 if (cond != NULL)
5698 {
5699 SLJIT_ASSERT(opcode == OP_COND || opcode == OP_SCOND);
5700 assert = CURRENT_AS(bracket_fallback)->u.assert;
5701 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT))
5702 {
5703 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5704 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5705 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5706 }
5707 JUMPHERE(cond);
5708 }
5709
5710 /* Free the STR_PTR. */
5711 if (localptr == 0)
5712 free_stack(common, 1);
5713 }
5714
5715 if (offset != 0)
5716 {
5717 /* Using both tmp register is better for instruction scheduling. */
5718 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5719 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5720 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5721 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5722 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(2));
5723 free_stack(common, 3);
5724 }
5725 else if (opcode == OP_SBRA || opcode == OP_SCOND)
5726 {
5727 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(0));
5728 free_stack(common, 1);
5729 }
5730 else if (opcode == OP_ONCE)
5731 {
5732 cc = ccbegin + GET(ccbegin, 1);
5733 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5734 {
5735 /* Reset head and drop saved frame. */
5736 stacksize = (ket == OP_KETRMAX || ket == OP_KETRMIN || *cc == OP_ALT) ? 2 : 1;
5737 free_stack(common, CURRENT_AS(bracket_fallback)->u.framesize + stacksize);
5738 }
5739 else if (ket == OP_KETRMAX || (*cc == OP_ALT && ket != OP_KETRMIN))
5740 {
5741 /* The STR_PTR must be released. */
5742 free_stack(common, 1);
5743 }
5744
5745 JUMPHERE(once);
5746 /* Restore previous localptr */
5747 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5748 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_fallback)->u.framesize * sizeof(sljit_w));
5749 else if (ket == OP_KETRMIN)
5750 {
5751 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5752 /* See the comment below. */
5753 free_stack(common, 2);
5754 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
5755 }
5756 }
5757
5758 if (ket == OP_KETRMAX)
5759 {
5760 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5761 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_fallback)->recursivehotpath);
5762 if (bra == OP_BRAZERO)
5763 {
5764 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5765 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5766 JUMPHERE(brazero);
5767 }
5768 free_stack(common, 1);
5769 }
5770 else if (ket == OP_KETRMIN)
5771 {
5772 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5773
5774 /* OP_ONCE removes everything in case of a fallback, so we don't
5775 need to explicitly release the STR_PTR. The extra release would
5776 affect badly the free_stack(2) above. */
5777 if (opcode != OP_ONCE)
5778 free_stack(common, 1);
5779 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rminlabel);
5780 if (opcode == OP_ONCE)
5781 free_stack(common, bra == OP_BRAMINZERO ? 2 : 1);
5782 else if (bra == OP_BRAMINZERO)
5783 free_stack(common, 1);
5784 }
5785 else if (bra == OP_BRAZERO)
5786 {
5787 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5788 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5789 JUMPHERE(brazero);
5790 }
5791 }
5792
5793 static void compile_bracketpos_fallbackpath(compiler_common *common, struct fallback_common *current)
5794 {
5795 DEFINE_COMPILER;
5796 int offset;
5797 struct sljit_jump *jump;
5798
5799 if (CURRENT_AS(bracketpos_fallback)->framesize < 0)
5800 {
5801 if (*current->cc == OP_CBRAPOS || *current->cc == OP_SCBRAPOS)
5802 {
5803 offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1;
5804 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5805 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5806 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5807 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5808 }
5809 set_jumps(current->topfallbacks, LABEL());
5810 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5811 return;
5812 }
5813
5814 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr);
5815 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5816
5817 if (current->topfallbacks)
5818 {
5819 jump = JUMP(SLJIT_JUMP);
5820 set_jumps(current->topfallbacks, LABEL());
5821 /* Drop the stack frame. */
5822 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5823 JUMPHERE(jump);
5824 }
5825 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_fallback)->framesize * sizeof(sljit_w));
5826 }
5827
5828 static void compile_braminzero_fallbackpath(compiler_common *common, struct fallback_common *current)
5829 {
5830 assert_fallback fallback;
5831
5832 current->top = NULL;
5833 current->topfallbacks = NULL;
5834 current->nextfallbacks = NULL;
5835 if (current->cc[1] > OP_ASSERTBACK_NOT)
5836 {
5837 /* Manual call of compile_bracket_hotpath and compile_bracket_fallbackpath. */
5838 compile_bracket_hotpath(common, current->cc, current);
5839 compile_bracket_fallbackpath(common, current->top);
5840 }
5841 else
5842 {
5843 memset(&fallback, 0, sizeof(fallback));
5844 fallback.common.cc = current->cc;
5845 fallback.hotpath = CURRENT_AS(braminzero_fallback)->hotpath;
5846 /* Manual call of compile_assert_hotpath. */
5847 compile_assert_hotpath(common, current->cc, &fallback, FALSE);
5848 }
5849 SLJIT_ASSERT(!current->nextfallbacks && !current->topfallbacks);
5850 }
5851
5852 static void compile_fallbackpath(compiler_common *common, struct fallback_common *current)
5853 {
5854 DEFINE_COMPILER;
5855
5856 while (current)
5857 {
5858 if (current->nextfallbacks != NULL)
5859 set_jumps(current->nextfallbacks, LABEL());
5860 switch(*current->cc)
5861 {
5862 case OP_SET_SOM:
5863 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5864 free_stack(common, 1);
5865 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP1, 0);
5866 break;
5867
5868 case OP_STAR:
5869 case OP_MINSTAR:
5870 case OP_PLUS:
5871 case OP_MINPLUS:
5872 case OP_QUERY:
5873 case OP_MINQUERY:
5874 case OP_UPTO:
5875 case OP_MINUPTO:
5876 case OP_EXACT:
5877 case OP_POSSTAR:
5878 case OP_POSPLUS:
5879 case OP_POSQUERY:
5880 case OP_POSUPTO:
5881 case OP_STARI:
5882 case OP_MINSTARI:
5883 case OP_PLUSI:
5884 case OP_MINPLUSI:
5885 case OP_QUERYI:
5886 case OP_MINQUERYI:
5887 case OP_UPTOI:
5888 case OP_MINUPTOI:
5889 case OP_EXACTI:
5890 case OP_POSSTARI:
5891 case OP_POSPLUSI:
5892 case OP_POSQUERYI:
5893 case OP_POSUPTOI:
5894 case OP_NOTSTAR:
5895 case OP_NOTMINSTAR:
5896 case OP_NOTPLUS:
5897 case OP_NOTMINPLUS:
5898 case OP_NOTQUERY:
5899 case OP_NOTMINQUERY:
5900 case OP_NOTUPTO:
5901 case OP_NOTMINUPTO:
5902 case OP_NOTEXACT:
5903 case OP_NOTPOSSTAR:
5904 case OP_NOTPOSPLUS:
5905 case OP_NOTPOSQUERY:
5906 case OP_NOTPOSUPTO:
5907 case OP_NOTSTARI:
5908 case OP_NOTMINSTARI:
5909 case OP_NOTPLUSI:
5910 case OP_NOTMINPLUSI:
5911 case OP_NOTQUERYI:
5912 case OP_NOTMINQUERYI:
5913 case OP_NOTUPTOI:
5914 case OP_NOTMINUPTOI:
5915 case OP_NOTEXACTI:
5916 case OP_NOTPOSSTARI:
5917 case OP_NOTPOSPLUSI:
5918 case OP_NOTPOSQUERYI:
5919 case OP_NOTPOSUPTOI:
5920 case OP_TYPESTAR:
5921 case OP_TYPEMINSTAR:
5922 case OP_TYPEPLUS:
5923 case OP_TYPEMINPLUS:
5924 case OP_TYPEQUERY:
5925 case OP_TYPEMINQUERY:
5926 case OP_TYPEUPTO:
5927 case OP_TYPEMINUPTO:
5928 case OP_TYPEEXACT:
5929 case OP_TYPEPOSSTAR:
5930 case OP_TYPEPOSPLUS:
5931 case OP_TYPEPOSQUERY:
5932 case OP_TYPEPOSUPTO:
5933 case OP_CLASS:
5934 case OP_NCLASS:
5935 case OP_XCLASS:
5936 compile_iterator_fallbackpath(common, current);
5937 break;
5938
5939 case OP_REF:
5940 case OP_REFI:
5941 compile_ref_iterator_fallbackpath(common, current);
5942 break;
5943
5944 case OP_RECURSE:
5945 compile_recurse_fallbackpath(common, current);
5946 break;
5947
5948 case OP_ASSERT:
5949 case OP_ASSERT_NOT:
5950 case OP_ASSERTBACK:
5951 case OP_ASSERTBACK_NOT:
5952 compile_assert_fallbackpath(common, current);
5953 break;
5954
5955 case OP_ONCE:
5956 case OP_ONCE_NC:
5957 case OP_BRA:
5958 case OP_CBRA:
5959 case OP_COND:
5960 case OP_SBRA:
5961 case OP_SCBRA:
5962 case OP_SCOND:
5963 compile_bracket_fallbackpath(common, current);
5964 break;
5965
5966 case OP_BRAZERO:
5967 if (current->cc[1] > OP_ASSERTBACK_NOT)
5968 compile_bracket_fallbackpath(common, current);
5969 else
5970 compile_assert_fallbackpath(common, current);
5971 break;
5972
5973 case OP_BRAPOS:
5974 case OP_CBRAPOS:
5975 case OP_SBRAPOS:
5976 case OP_SCBRAPOS:
5977 case OP_BRAPOSZERO:
5978 compile_bracketpos_fallbackpath(common, current);
5979 break;
5980
5981 case OP_BRAMINZERO:
5982 compile_braminzero_fallbackpath(common, current);
5983 break;
5984
5985 case OP_FAIL:
5986 case OP_ACCEPT:
5987 case OP_ASSERT_ACCEPT:
5988 set_jumps(current->topfallbacks, LABEL());
5989 break;
5990
5991 default:
5992 SLJIT_ASSERT_STOP();
5993 break;
5994 }
5995 current = current->prev;
5996 }
5997 }
5998
5999 static SLJIT_INLINE void compile_recurse(compiler_common *common)
6000 {
6001 DEFINE_COMPILER;
6002 uschar *cc = common->start + common->currententry->start;
6003 uschar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : 2);
6004 uschar *ccend = bracketend(cc);
6005 int localsize = get_localsize(common, ccbegin, ccend);
6006 int framesize = get_framesize(common, cc, TRUE);
6007 int alternativesize;
6008 BOOL needsframe;
6009 fallback_common altfallback;
6010 struct sljit_jump *jump;
6011
6012 SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS);
6013 needsframe = framesize >= 0;
6014 if (!needsframe)
6015 framesize = 0;
6016 alternativesize = *(cc + GET(cc, 1)) == OP_ALT ? 1 : 0;
6017
6018 SLJIT_ASSERT(common->currententry->entry == NULL);
6019 common->currententry->entry = LABEL();
6020 set_jumps(common->currententry->calls, common->currententry->entry);
6021
6022 sljit_emit_fast_enter(compiler, TMP2, 0, 1, 5, 5, common->localsize);
6023 allocate_stack(common, localsize + framesize + alternativesize);
6024 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(localsize + framesize + alternativesize - 1), TMP2, 0);
6025 copy_locals(common, ccbegin, ccend, TRUE, localsize + framesize + alternativesize, framesize + alternativesize);
6026 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, STACK_TOP, 0);
6027 if (needsframe)
6028 init_frame(common, cc, framesize + alternativesize - 1, alternativesize, FALSE);
6029
6030 if (alternativesize > 0)
6031 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
6032
6033 memset(&altfallback, 0, sizeof(fallback_common));
6034 common->acceptlabel = NULL;
6035 common->accept = NULL;
6036 altfallback.cc = ccbegin;
6037 cc += GET(cc, 1);
6038 while (1)
6039 {
6040 altfallback.top = NULL;
6041 altfallback.topfallbacks = NULL;
6042
6043 if (altfallback.cc != ccbegin)
6044 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
6045
6046 compile_hotpath(common, altfallback.cc, cc, &altfallback);
6047 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6048 return;
6049
6050 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
6051
6052 compile_fallbackpath(common, altfallback.top);
6053 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6054 return;
6055 set_jumps(altfallback.topfallbacks, LABEL());
6056
6057 if (*cc != OP_ALT)
6058 break;
6059
6060 altfallback.cc = cc + 1 + LINK_SIZE;
6061 cc += GET(cc, 1);
6062 }
6063 /* None of them matched. */
6064 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0);
6065 jump = JUMP(SLJIT_JUMP);
6066
6067 set_jumps(common->accept, LABEL());
6068 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD);
6069 if (needsframe)
6070 {
6071 OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6072 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_w));
6073 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
6074 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_w));
6075 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP3, 0);
6076 }
6077 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1);
6078
6079 JUMPHERE(jump);
6080 copy_locals(common, ccbegin, ccend, FALSE, localsize + framesize + alternativesize, framesize + alternativesize);
6081 free_stack(common, localsize + framesize + alternativesize);
6082 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_w));
6083 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
6084 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, TMP2, 0);
6085 sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0);
6086 }
6087
6088 #undef COMPILE_FALLBACKPATH
6089 #undef CURRENT_AS
6090
6091 void
6092 _pcre_jit_compile(const real_pcre *re, pcre_extra *extra)
6093 {
6094 struct sljit_compiler *compiler;
6095 fallback_common rootfallback;
6096 compiler_common common_data;
6097 compiler_common *common = &common_data;
6098 const uschar *tables = re->tables;
6099 pcre_study_data *study;
6100 uschar *ccend;
6101 executable_function *function;
6102 void *executable_func;
6103 sljit_uw executable_size;
6104 struct sljit_label *leave;
6105 struct sljit_label *mainloop = NULL;
6106 struct sljit_label *empty_match_found;
6107 struct sljit_label *empty_match_fallback;
6108 struct sljit_jump *alloc_error;
6109 struct sljit_jump *reqbyte_notfound = NULL;
6110 struct sljit_jump *empty_match;
6111
6112 SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0);
6113 study = extra->study_data;
6114
6115 if (!tables)
6116 tables = _pcre_default_tables;
6117
6118 memset(&rootfallback, 0, sizeof(fallback_common));
6119 rootfallback.cc = (uschar *)re + re->name_table_offset + re->name_count * re->name_entry_size;
6120
6121 common->compiler = NULL;
6122 common->start = rootfallback.cc;
6123 common->cbraptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_w);
6124 common->fcc = tables + fcc_offset;
6125 common->lcc = (sljit_w)(tables + lcc_offset);
6126 common->nltype = NLTYPE_FIXED;
6127 switch(re->options & PCRE_NEWLINE_BITS)
6128 {
6129 case 0:
6130 /* Compile-time default */
6131 switch (NEWLINE)
6132 {
6133 case -1: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
6134 case -2: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
6135 default: common->newline = NEWLINE; break;
6136 }
6137 break;
6138 case PCRE_NEWLINE_CR: common->newline = CHAR_CR; break;
6139 case PCRE_NEWLINE_LF: common->newline = CHAR_NL; break;
6140 case PCRE_NEWLINE_CR+
6141 PCRE_NEWLINE_LF: common->newline = (CHAR_CR << 8) | CHAR_NL; break;
6142 case PCRE_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
6143 case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
6144 default: return;
6145 }
6146 if ((re->options & PCRE_BSR_ANYCRLF) != 0)
6147 common->bsr_nltype = NLTYPE_ANYCRLF;
6148 else if ((re->options & PCRE_BSR_UNICODE) != 0)
6149 common->bsr_nltype = NLTYPE_ANY;
6150 else
6151 {
6152 #ifdef BSR_ANYCRLF
6153 common->bsr_nltype = NLTYPE_ANYCRLF;
6154 #else
6155 common->bsr_nltype = NLTYPE_ANY;
6156 #endif
6157 }
6158 common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0;
6159 common->ctypes = (sljit_w)(tables + ctypes_offset);
6160 common->name_table = (sljit_w)re + re->name_table_offset;
6161 common->name_count = re->name_count;
6162 common->name_entry_size = re->name_entry_size;
6163 common->acceptlabel = NULL;
6164 common->stubs = NULL;
6165 common->entries = NULL;
6166 common->currententry = NULL;
6167 common->accept = NULL;
6168 common->calllimit = NULL;
6169 common->stackalloc = NULL;
6170 common->revertframes = NULL;
6171 common->wordboundary = NULL;
6172 common->anynewline = NULL;
6173 common->hspace = NULL;
6174 common->vspace = NULL;
6175 common->casefulcmp = NULL;
6176 common->caselesscmp = NULL;
6177 common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0;
6178 #ifdef SUPPORT_UTF8
6179 common->utf8 = (re->options & PCRE_UTF8) != 0;
6180 #ifdef SUPPORT_UCP
6181 common->useucp = (re->options & PCRE_UCP) != 0;
6182 #endif
6183 common->utf8readchar = NULL;
6184 common->utf8readtype8 = NULL;
6185 #endif
6186 #ifdef SUPPORT_UCP
6187 common->getucd = NULL;
6188 #endif
6189 ccend = bracketend(rootfallback.cc);
6190 SLJIT_ASSERT(*rootfallback.cc == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET);
6191 common->localsize = get_localspace(common, rootfallback.cc, ccend);
6192 if (common->localsize < 0)
6193 return;
6194 common->localsize += common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w);
6195 if (common->localsize > SLJIT_MAX_LOCAL_SIZE)
6196 return;
6197 common->localptrs = (int*)SLJIT_MALLOC((ccend - rootfallback.cc) * sizeof(int));
6198 if (!common->localptrs)
6199 return;
6200 memset(common->localptrs, 0, (ccend - rootfallback.cc) * sizeof(int));
6201 set_localptrs(common, common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w), ccend);
6202
6203 compiler = sljit_create_compiler();
6204 if (!compiler)
6205 {
6206 SLJIT_FREE(common->localptrs);
6207 return;
6208 }
6209 common->compiler = compiler;
6210
6211 /* Main pcre_jit_exec entry. */
6212 sljit_emit_enter(compiler, 1, 5, 5, common->localsize);
6213
6214 /* Register init. */
6215 reset_ovector(common, (re->top_bracket + 1) * 2);
6216 if ((re->flags & PCRE_REQCHSET) != 0)
6217 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, SLJIT_TEMPORARY_REG1, 0);
6218
6219 OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_GENERAL_REG1, 0);
6220 OP1(SLJIT_MOV, TMP1, 0, SLJIT_GENERAL_REG1, 0);
6221 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6222 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end));
6223 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6224 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, calllimit));
6225 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base));
6226 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit));
6227 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT, TMP1, 0);
6228
6229 /* Main part of the matching */
6230 if ((re->options & PCRE_ANCHORED) == 0)
6231 {
6232 mainloop = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0, (re->options & PCRE_FIRSTLINE) != 0);
6233 /* Forward search if possible. */
6234 if ((re->flags & PCRE_FIRSTSET) != 0)
6235 fast_forward_first_byte(common, re->first_byte, (re->options & PCRE_FIRSTLINE) != 0);
6236 else if ((re->flags & PCRE_STARTLINE) != 0)
6237 fast_forward_newline(common, (re->options & PCRE_FIRSTLINE) != 0);
6238 else if ((re->flags & PCRE_STARTLINE) == 0 && study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)
6239 fast_forward_start_bits(common, (sljit_uw)study->start_bits, (re->options & PCRE_FIRSTLINE) != 0);
6240 }
6241 if ((re->flags & PCRE_REQCHSET) != 0)
6242 reqbyte_notfound = search_requested_char(common, re->req_byte, (re->flags & PCRE_FIRSTSET) != 0);
6243
6244 /* Store the current STR_PTR in OVECTOR(0). */
6245 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
6246 /* Copy the limit of allowed recursions. */
6247 OP1(SLJIT_MOV, CALL_COUNT, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT);
6248
6249 compile_hotpath(common, rootfallback.cc, ccend, &rootfallback);
6250 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6251 {
6252 sljit_free_compiler(compiler);
6253 SLJIT_FREE(common->localptrs);
6254 return;
6255 }
6256
6257 empty_match = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6258 empty_match_found = LABEL();
6259
6260 common->acceptlabel = LABEL();
6261 if (common->accept != NULL)
6262 set_jumps(common->accept, common->acceptlabel);
6263
6264 /* This means we have a match. Update the ovector. */
6265 copy_ovector(common, re->top_bracket + 1);
6266 leave = LABEL();
6267 sljit_emit_return(compiler, SLJIT_UNUSED, 0);
6268
6269 empty_match_fallback = LABEL();
6270 compile_fallbackpath(common, rootfallback.top);
6271 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6272 {
6273 sljit_free_compiler(compiler);
6274 SLJIT_FREE(common->localptrs);
6275 return;
6276 }
6277
6278 SLJIT_ASSERT(rootfallback.prev == NULL);
6279
6280 /* Check we have remaining characters. */
6281 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6282
6283 if ((re->options & PCRE_ANCHORED) == 0)
6284 {
6285 if ((re->options & PCRE_FIRSTLINE) == 0)
6286 {
6287 if (study != NULL && study->minlength > 1)
6288 {
6289 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6290 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, STR_END, 0, mainloop);
6291 }
6292 else
6293 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
6294 }
6295 else
6296 {
6297 if (study != NULL && study->minlength > 1)
6298 {
6299 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6300 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, STR_END, 0);
6301 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER);
6302 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
6303 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_GREATER_EQUAL);
6304 JUMPTO(SLJIT_C_ZERO, mainloop);
6305 }
6306 else
6307 CMPTO(SLJIT_C_LESS, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, mainloop);
6308 }
6309 }
6310
6311 if (reqbyte_notfound != NULL)
6312 JUMPHERE(reqbyte_notfound);
6313 /* Copy OVECTOR(1) to OVECTOR(0) */
6314 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
6315 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
6316 JUMPTO(SLJIT_JUMP, leave);
6317
6318 flush_stubs(common);
6319
6320 JUMPHERE(empty_match);
6321 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6322 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
6323 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_fallback);
6324 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
6325 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found);
6326 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6327 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found);
6328 JUMPTO(SLJIT_JUMP, empty_match_fallback);
6329
6330 common->currententry = common->entries;
6331 while (common->currententry != NULL)
6332 {
6333 /* Might add new entries. */
6334 compile_recurse(common);
6335 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6336 {
6337 sljit_free_compiler(compiler);
6338 SLJIT_FREE(common->localptrs);
6339 return;
6340 }
6341 flush_stubs(common);
6342 common->currententry = common->currententry->next;
6343 }
6344
6345 /* Allocating stack, returns with PCRE_ERROR_JIT_STACKLIMIT if fails. */
6346 /* This is a (really) rare case. */
6347 set_jumps(common->stackalloc, LABEL());
6348 /* RETURN_ADDR is not a saved register. */
6349 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
6350 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
6351 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6352 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6353 OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
6354 OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
6355
6356 sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
6357 alloc_error = CMP(SLJIT_C_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
6358 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6359 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6360 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top));
6361 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit));
6362 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
6363 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
6364
6365 /* Allocation failed. */
6366 JUMPHERE(alloc_error);
6367 /* We break the return address cache here, but this is a really rare case. */
6368 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_JIT_STACKLIMIT);
6369 JUMPTO(SLJIT_JUMP, leave);
6370
6371 /* Call limit reached. */
6372 set_jumps(common->calllimit, LABEL());
6373 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_MATCHLIMIT);
6374 JUMPTO(SLJIT_JUMP, leave);
6375
6376 if (common->revertframes != NULL)
6377 {
6378 set_jumps(common->revertframes, LABEL());
6379 do_revertframes(common);
6380 }
6381 if (common->wordboundary != NULL)
6382 {
6383 set_jumps(common->wordboundary, LABEL());
6384 check_wordboundary(common);
6385 }
6386 if (common->anynewline != NULL)
6387 {
6388 set_jumps(common->anynewline, LABEL());
6389 check_anynewline(common);
6390 }
6391 if (common->hspace != NULL)
6392 {
6393 set_jumps(common->hspace, LABEL());
6394 check_hspace(common);
6395 }
6396 if (common->vspace != NULL)
6397 {
6398 set_jumps(common->vspace, LABEL());
6399 check_vspace(common);
6400 }
6401 if (common->casefulcmp != NULL)
6402 {
6403 set_jumps(common->casefulcmp, LABEL());
6404 do_casefulcmp(common);
6405 }
6406 if (common->caselesscmp != NULL)
6407 {
6408 set_jumps(common->caselesscmp, LABEL());
6409 do_caselesscmp(common);
6410 }
6411 #ifdef SUPPORT_UTF8
6412 if (common->utf8readchar != NULL)
6413 {
6414 set_jumps(common->utf8readchar, LABEL());
6415 do_utf8readchar(common);
6416 }
6417 if (common->utf8readtype8 != NULL)
6418 {
6419 set_jumps(common->utf8readtype8, LABEL());
6420 do_utf8readtype8(common);
6421 }
6422 #endif
6423 #ifdef SUPPORT_UCP
6424 if (common->getucd != NULL)
6425 {
6426 set_jumps(common->getucd, LABEL());
6427 do_getucd(common);
6428 }
6429 #endif
6430
6431 SLJIT_FREE(common->localptrs);
6432 executable_func = sljit_generate_code(compiler);
6433 executable_size = sljit_get_generated_code_size(compiler);
6434 sljit_free_compiler(compiler);
6435 if (executable_func == NULL)
6436 return;
6437
6438 function = SLJIT_MALLOC(sizeof(executable_function));
6439 if (function == NULL)
6440 {
6441 /* This case is highly unlikely since we just recently
6442 freed a lot of memory. Although not impossible. */
6443 sljit_free_code(executable_func);
6444 return;
6445 }
6446
6447 function->executable_func = executable_func;
6448 function->executable_size = executable_size;
6449 function->callback = NULL;
6450 function->userdata = NULL;
6451 extra->executable_jit = function;
6452 extra->flags |= PCRE_EXTRA_EXECUTABLE_JIT;
6453 }
6454
6455 static int jit_machine_stack_exec(jit_arguments *arguments, executable_function *function)
6456 {
6457 union {
6458 void* executable_func;
6459 jit_function call_executable_func;
6460 } convert_executable_func;
6461 uschar local_area[LOCAL_SPACE_SIZE];
6462 struct sljit_stack local_stack;
6463
6464 local_stack.top = (sljit_w)&local_area;
6465 local_stack.base = local_stack.top;
6466 local_stack.limit = local_stack.base + LOCAL_SPACE_SIZE;
6467 local_stack.max_limit = local_stack.limit;
6468 arguments->stack = &local_stack;
6469 convert_executable_func.executable_func = function->executable_func;
6470 return convert_executable_func.call_executable_func(arguments);
6471 }
6472
6473 int
6474 _pcre_jit_exec(const real_pcre *re, void *executable_func,
6475 PCRE_SPTR subject, int length, int start_offset, int options,
6476 int match_limit, int *offsets, int offsetcount)
6477 {
6478 executable_function *function = (executable_function*)executable_func;
6479 union {
6480 void* executable_func;
6481 jit_function call_executable_func;
6482 } convert_executable_func;
6483 jit_arguments arguments;
6484 int maxoffsetcount;
6485 int retval;
6486
6487 /* Sanity checks should be handled by pcre_exec. */
6488 arguments.stack = NULL;
6489 arguments.str = subject + start_offset;
6490 arguments.begin = subject;
6491 arguments.end = subject + length;
6492 arguments.calllimit = match_limit; /* JIT decreases this value less times. */
6493 arguments.notbol = (options & PCRE_NOTBOL) != 0;
6494 arguments.noteol = (options & PCRE_NOTEOL) != 0;
6495 arguments.notempty = (options & PCRE_NOTEMPTY) != 0;
6496 arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0;
6497 arguments.offsets = offsets;
6498
6499 /* pcre_exec() rounds offsetcount to a multiple of 3, and then uses only 2/3 of
6500 the output vector for storing captured strings, with the remainder used as
6501 workspace. We don't need the workspace here. For compatibility, we limit the
6502 number of captured strings in the same way as pcre_exec(), so that the user
6503 gets the same result with and without JIT. */
6504
6505 offsetcount = ((offsetcount - (offsetcount % 3)) * 2)/3;
6506 maxoffsetcount = (re->top_bracket + 1) * 2;
6507 if (offsetcount > maxoffsetcount)
6508 offsetcount = maxoffsetcount;
6509 arguments.offsetcount = offsetcount;
6510
6511 if (function->callback)
6512 arguments.stack = (struct sljit_stack*)function->callback(function->userdata);
6513 else
6514 arguments.stack = (struct sljit_stack*)function->userdata;
6515
6516 if (arguments.stack == NULL)
6517 retval = jit_machine_stack_exec(&arguments, function);
6518 else
6519 {
6520 convert_executable_func.executable_func = function->executable_func;
6521 retval = convert_executable_func.call_executable_func(&arguments);
6522 }
6523
6524 if (retval * 2 > offsetcount)
6525 retval = 0;
6526 return retval;
6527 }
6528
6529 void
6530 _pcre_jit_free(void *executable_func)
6531 {
6532 executable_function *function = (executable_function*)executable_func;
6533 sljit_free_code(function->executable_func);
6534 SLJIT_FREE(function);
6535 }
6536
6537 int
6538 _pcre_jit_get_size(void *executable_func)
6539 {
6540 return ((executable_function*)executable_func)->executable_size;
6541 }
6542
6543 PCRE_EXP_DECL pcre_jit_stack *
6544 pcre_jit_stack_alloc(int startsize, int maxsize)
6545 {
6546 if (startsize < 1 || maxsize < 1)
6547 return NULL;
6548 if (startsize > maxsize)
6549 startsize = maxsize;
6550 startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6551 maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6552 return (pcre_jit_stack*)sljit_allocate_stack(startsize, maxsize);
6553 }
6554
6555 PCRE_EXP_DECL void
6556 pcre_jit_stack_free(pcre_jit_stack *stack)
6557 {
6558 sljit_free_stack((struct sljit_stack*)stack);
6559 }
6560
6561 PCRE_EXP_DECL void
6562 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6563 {
6564 executable_function *function;
6565 if (extra != NULL &&
6566 (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 &&
6567 extra->executable_jit != NULL)
6568 {
6569 function = (executable_function*)extra->executable_jit;
6570 function->callback = callback;
6571 function->userdata = userdata;
6572 }
6573 }
6574
6575 #else /* SUPPORT_JIT */
6576
6577 /* These are dummy functions to avoid linking errors when JIT support is not
6578 being compiled. */
6579
6580 PCRE_EXP_DECL pcre_jit_stack *
6581 pcre_jit_stack_alloc(int startsize, int maxsize)
6582 {
6583 (void)startsize;
6584 (void)maxsize;
6585 return NULL;
6586 }
6587
6588 PCRE_EXP_DECL void
6589 pcre_jit_stack_free(pcre_jit_stack *stack)
6590 {
6591 (void)stack;
6592 }
6593
6594 PCRE_EXP_DECL void
6595 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6596 {
6597 (void)extra;
6598 (void)callback;
6599 (void)userdata;
6600 }
6601
6602 #endif
6603
6604 /* End of pcre_jit_compile.c */

  ViewVC Help
Powered by ViewVC 1.1.5