/[pcre]/code/trunk/pcre_jit_compile.c
ViewVC logotype

Contents of /code/trunk/pcre_jit_compile.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 715 - (show annotations)
Sat Oct 1 06:42:38 2011 UTC (3 years, 9 months ago) by zherczeg
File MIME type: text/plain
File size: 196267 byte(s)
Error occurred while calculating annotation data.
A fix for signed/unsigned comparison warnings reported by Visual C++ in the JIT compiler
1 /*************************************************
2 * Perl-Compatible Regular Expressions *
3 *************************************************/
4
5 /* PCRE is a library of functions to support regular expressions whose syntax
6 and semantics are as close as possible to those of the Perl 5 language.
7
8 Written by Philip Hazel
9 Copyright (c) 1997-2008 University of Cambridge
10
11 The machine code generator part (this module) was written by Zoltan Herczeg
12 Copyright (c) 2010-2011
13
14 -----------------------------------------------------------------------------
15 Redistribution and use in source and binary forms, with or without
16 modification, are permitted provided that the following conditions are met:
17
18 * Redistributions of source code must retain the above copyright notice,
19 this list of conditions and the following disclaimer.
20
21 * Redistributions in binary form must reproduce the above copyright
22 notice, this list of conditions and the following disclaimer in the
23 documentation and/or other materials provided with the distribution.
24
25 * Neither the name of the University of Cambridge nor the names of its
26 contributors may be used to endorse or promote products derived from
27 this software without specific prior written permission.
28
29 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
36 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
37 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 POSSIBILITY OF SUCH DAMAGE.
40 -----------------------------------------------------------------------------
41 */
42
43 #ifdef HAVE_CONFIG_H
44 #include "config.h"
45 #endif
46
47 #include "pcre_internal.h"
48
49 #ifdef SUPPORT_JIT
50
51 /* All-in-one: Since we use the JIT compiler only from here,
52 we just include it. This way we don't need to touch the build
53 system files. */
54
55 #define SLJIT_CONFIG_AUTO 1
56 #define SLJIT_VERBOSE 0
57 #define SLJIT_DEBUG 0
58
59 #include "sljit/sljitLir.c"
60
61 #if defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED
62 #error "Unsupported architecture"
63 #endif
64
65 /* Allocate memory on the stack. Fast, but limited size. */
66 #define LOCAL_SPACE_SIZE 32768
67
68 #define STACK_GROWTH_RATE 8192
69
70 /* Enable to check that the allocation could destroy temporaries. */
71 #if defined SLJIT_DEBUG && SLJIT_DEBUG
72 #define DESTROY_REGISTERS 1
73 #endif
74
75 /*
76 Short summary about the backtracking mechanism empolyed by the jit code generator:
77
78 The code generator follows the recursive nature of the PERL compatible regular
79 expressions. The basic blocks of regular expressions are condition checkers
80 whose execute different commands depending on the result of the condition check.
81 The relationship between the operators can be horizontal (concatenation) and
82 vertical (sub-expression) (See struct fallback_common for more details).
83
84 'ab' - 'a' and 'b' regexps are concatenated
85 'a+' - 'a' is the sub-expression of the '+' operator
86
87 The condition checkers are boolean (true/false) checkers. Machine code is generated
88 for the checker itself and for the actions depending on the result of the checker.
89 The 'true' case is called as the hot path (expected path), and the other is called as
90 the 'fallback' path. Branch instructions are expesive for all CPUs, so we avoid taken
91 branches on the hot path.
92
93 Greedy star operator (*) :
94 Hot path: match happens.
95 Fallback path: match failed.
96 Non-greedy star operator (*?) :
97 Hot path: no need to perform a match.
98 Fallback path: match is required.
99
100 The following example shows how the code generated for a capturing bracket
101 with two alternatives. Let A, B, C, D are arbirary regular expressions, and
102 we have the following regular expression:
103
104 A(B|C)D
105
106 The generated code will be the following:
107
108 A hot path
109 '(' hot path (pushing arguments to the stack)
110 B hot path
111 ')' hot path (pushing arguments to the stack)
112 D hot path
113 return with successful match
114
115 D fallback path
116 ')' fallback path (If we arrived from "C" jump to the fallback of "C")
117 B fallback path
118 C expected path
119 jump to D hot path
120 C fallback path
121 A fallback path
122
123 Notice, that the order of fallback code paths are the opposite of the fast
124 code paths. In this way the topmost value on the stack is always belong
125 to the current fallback code path. The fallback code path must check
126 whether there is a next alternative. If so, it needs to jump back to
127 the hot path eventually. Otherwise it needs to clear out its own stack
128 frame and continue the execution on the fallback code paths.
129 */
130
131 /*
132 Saved stack frames:
133
134 Atomic blocks and asserts require reloading the values of local variables
135 when the fallback mechanism performed. Because of OP_RECURSE, the locals
136 are not necessarly known in compile time, thus we need a dynamic restore
137 mechanism.
138
139 The stack frames are stored in a chain list, and have the following format:
140 ([ capturing bracket offset ][ start value ][ end value ])+ ... [ 0 ] [ previous head ]
141
142 Thus we can restore the locals to a particular point in the stack.
143 */
144
145 typedef struct jit_arguments {
146 /* Pointers first. */
147 struct sljit_stack *stack;
148 PCRE_SPTR str;
149 PCRE_SPTR begin;
150 PCRE_SPTR end;
151 int *offsets;
152 uschar *ptr;
153 /* Everything else after. */
154 int offsetcount;
155 int calllimit;
156 uschar notbol;
157 uschar noteol;
158 uschar notempty;
159 uschar notempty_atstart;
160 } jit_arguments;
161
162 typedef struct executable_function {
163 void *executable_func;
164 pcre_jit_callback callback;
165 void *userdata;
166 } executable_function;
167
168 typedef struct jump_list {
169 struct sljit_jump *jump;
170 struct jump_list *next;
171 } jump_list;
172
173 enum stub_types { stack_alloc };
174
175 typedef struct stub_list {
176 enum stub_types type;
177 int data;
178 struct sljit_jump *start;
179 struct sljit_label *leave;
180 struct stub_list *next;
181 } stub_list;
182
183 typedef int (SLJIT_CALL *jit_function)(jit_arguments *args);
184
185 /* The following structure is the key data type for the recursive
186 code generator. It is allocated by compile_hotpath, and contains
187 the aguments for compile_fallbackpath. Must be the first member
188 of its descendants. */
189 typedef struct fallback_common {
190 /* Concatenation stack. */
191 struct fallback_common *prev;
192 jump_list *nextfallbacks;
193 /* Internal stack (for component operators). */
194 struct fallback_common *top;
195 jump_list *topfallbacks;
196 /* Opcode pointer. */
197 uschar *cc;
198 } fallback_common;
199
200 typedef struct assert_fallback {
201 fallback_common common;
202 jump_list *condfailed;
203 /* Less than 0 (-1) if a frame is not needed. */
204 int framesize;
205 /* Points to our private memory word on the stack. */
206 int localptr;
207 /* For iterators. */
208 struct sljit_label *hotpath;
209 } assert_fallback;
210
211 typedef struct bracket_fallback {
212 fallback_common common;
213 /* Where to coninue if an alternative is successfully matched. */
214 struct sljit_label *althotpath;
215 /* For rmin and rmax iterators. */
216 struct sljit_label *recursivehotpath;
217 /* For greedy ? operator. */
218 struct sljit_label *zerohotpath;
219 /* Contains the branches of a failed condition. */
220 union {
221 /* Both for OP_COND, OP_SCOND. */
222 jump_list *condfailed;
223 assert_fallback *assert;
224 /* For OP_ONCE. -1 if not needed. */
225 int framesize;
226 } u;
227 /* Points to our private memory word on the stack. */
228 int localptr;
229 } bracket_fallback;
230
231 typedef struct bracketpos_fallback {
232 fallback_common common;
233 /* Points to our private memory word on the stack. */
234 int localptr;
235 /* Reverting stack is needed. */
236 int framesize;
237 /* Allocated stack size. */
238 int stacksize;
239 } bracketpos_fallback;
240
241 typedef struct braminzero_fallback {
242 fallback_common common;
243 struct sljit_label *hotpath;
244 } braminzero_fallback;
245
246 typedef struct iterator_fallback {
247 fallback_common common;
248 /* Next iteration. */
249 struct sljit_label *hotpath;
250 } iterator_fallback;
251
252 typedef struct recurse_entry {
253 struct recurse_entry *next;
254 /* Contains the function entry. */
255 struct sljit_label *entry;
256 /* Collects the calls until the function is not created. */
257 jump_list *calls;
258 /* Points to the starting opcode. */
259 int start;
260 } recurse_entry;
261
262 typedef struct recurse_fallback {
263 fallback_common common;
264 } recurse_fallback;
265
266 typedef struct compiler_common {
267 struct sljit_compiler *compiler;
268 uschar *start;
269 int localsize;
270 int *localptrs;
271 const uschar *fcc;
272 sljit_w lcc;
273 int cbraptr;
274 int nltype;
275 int newline;
276 int bsr_nltype;
277 int endonly;
278 sljit_w ctypes;
279 struct sljit_label *acceptlabel;
280 stub_list *stubs;
281 recurse_entry *entries;
282 recurse_entry *currententry;
283 jump_list *accept;
284 jump_list *calllimit;
285 jump_list *stackalloc;
286 jump_list *revertframes;
287 jump_list *wordboundary;
288 jump_list *anynewline;
289 jump_list *hspace;
290 jump_list *vspace;
291 jump_list *casefulcmp;
292 jump_list *caselesscmp;
293 BOOL jscript_compat;
294 #ifdef SUPPORT_UTF8
295 BOOL utf8;
296 #ifdef SUPPORT_UCP
297 BOOL useucp;
298 #endif
299 jump_list *utf8readchar;
300 jump_list *utf8readtype8;
301 #endif
302 #ifdef SUPPORT_UCP
303 jump_list *getucd;
304 #endif
305 } compiler_common;
306
307 /* For byte_sequence_compare. */
308
309 typedef struct compare_context {
310 int length;
311 int sourcereg;
312 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
313 int byteptr;
314 union {
315 int asint;
316 short asshort;
317 sljit_ub asbyte;
318 sljit_ub asbytes[4];
319 } c;
320 union {
321 int asint;
322 short asshort;
323 sljit_ub asbyte;
324 sljit_ub asbytes[4];
325 } oc;
326 #endif
327 } compare_context;
328
329 enum {
330 frame_end = 0,
331 frame_setstrbegin = -1
332 };
333
334 /* Used for accessing the elements of the stack. */
335 #define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_w))
336
337 #define TMP1 SLJIT_TEMPORARY_REG1
338 #define TMP2 SLJIT_TEMPORARY_REG3
339 #define TMP3 SLJIT_TEMPORARY_EREG2
340 #define STR_PTR SLJIT_GENERAL_REG1
341 #define STR_END SLJIT_GENERAL_REG2
342 #define STACK_TOP SLJIT_TEMPORARY_REG2
343 #define STACK_LIMIT SLJIT_GENERAL_REG3
344 #define ARGUMENTS SLJIT_GENERAL_EREG1
345 #define CALL_COUNT SLJIT_GENERAL_EREG2
346 #define RETURN_ADDR SLJIT_TEMPORARY_EREG1
347
348 /* Locals layout. */
349 /* These two locals can be used by the current opcode. */
350 #define LOCALS0 (0 * sizeof(sljit_w))
351 #define LOCALS1 (1 * sizeof(sljit_w))
352 /* Two local variables for possessive quantifiers (char1 cannot use them). */
353 #define POSSESSIVE0 (2 * sizeof(sljit_w))
354 #define POSSESSIVE1 (3 * sizeof(sljit_w))
355 /* Head of the saved local variables */
356 #define LOCALS_HEAD (4 * sizeof(sljit_w))
357 /* Head of the last recursion. */
358 #define RECURSIVE_HEAD (5 * sizeof(sljit_w))
359 /* Max limit of recursions. */
360 #define CALL_LIMIT (7 * sizeof(sljit_w))
361 /* Last known position of the requested byte. */
362 #define REQ_BYTE_PTR (8 * sizeof(sljit_w))
363 /* End pointer of the first line. */
364 #define FIRSTLINE_END (9 * sizeof(sljit_w))
365 /* The output vector is stored on the stack, and contains pointers
366 to characters. The vector data is divided into two groups: the first
367 group contains the start / end character pointers, and the second is
368 the start pointers when the end of the capturing group has not yet reached. */
369 #define OVECTOR_START (10 * sizeof(sljit_w))
370 #define OVECTOR(i) (OVECTOR_START + (i) * sizeof(sljit_w))
371 #define OVECTOR_PRIV(i) (common->cbraptr + (i) * sizeof(sljit_w))
372 #define PRIV(cc) (common->localptrs[(cc) - common->start])
373
374 /* Shortcuts. */
375 #define DEFINE_COMPILER \
376 struct sljit_compiler *compiler = common->compiler
377 #define OP1(op, dst, dstw, src, srcw) \
378 sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw))
379 #define OP2(op, dst, dstw, src1, src1w, src2, src2w) \
380 sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w))
381 #define LABEL() \
382 sljit_emit_label(compiler)
383 #define JUMP(type) \
384 sljit_emit_jump(compiler, (type))
385 #define JUMPTO(type, label) \
386 sljit_set_label(sljit_emit_jump(compiler, (type)), (label))
387 #define JUMPHERE(jump) \
388 sljit_set_label((jump), sljit_emit_label(compiler))
389 #define CMP(type, src1, src1w, src2, src2w) \
390 sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w))
391 #define CMPTO(type, src1, src1w, src2, src2w, label) \
392 sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label))
393 #define COND_VALUE(op, dst, dstw, type) \
394 sljit_emit_cond_value(compiler, (op), (dst), (dstw), (type))
395
396 static uschar* bracketend(uschar* cc)
397 {
398 SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
399 do cc += GET(cc, 1); while (*cc == OP_ALT);
400 SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS);
401 cc += 1 + LINK_SIZE;
402 return cc;
403 }
404
405 /* Functions whose might need modification for all new supported opcodes:
406 next_opcode
407 get_localspace
408 set_localptrs
409 get_framesize
410 init_frame
411 get_localsize
412 copy_locals
413 compile_hotpath
414 compile_fallbackpath
415 */
416
417 static uschar *next_opcode(compiler_common *common, uschar *cc)
418 {
419 SLJIT_UNUSED_ARG(common);
420 switch(*cc)
421 {
422 case OP_SOD:
423 case OP_SOM:
424 case OP_SET_SOM:
425 case OP_NOT_WORD_BOUNDARY:
426 case OP_WORD_BOUNDARY:
427 case OP_NOT_DIGIT:
428 case OP_DIGIT:
429 case OP_NOT_WHITESPACE:
430 case OP_WHITESPACE:
431 case OP_NOT_WORDCHAR:
432 case OP_WORDCHAR:
433 case OP_ANY:
434 case OP_ALLANY:
435 case OP_ANYNL:
436 case OP_NOT_HSPACE:
437 case OP_HSPACE:
438 case OP_NOT_VSPACE:
439 case OP_VSPACE:
440 case OP_EXTUNI:
441 case OP_EODN:
442 case OP_EOD:
443 case OP_CIRC:
444 case OP_CIRCM:
445 case OP_DOLL:
446 case OP_DOLLM:
447 case OP_TYPESTAR:
448 case OP_TYPEMINSTAR:
449 case OP_TYPEPLUS:
450 case OP_TYPEMINPLUS:
451 case OP_TYPEQUERY:
452 case OP_TYPEMINQUERY:
453 case OP_TYPEPOSSTAR:
454 case OP_TYPEPOSPLUS:
455 case OP_TYPEPOSQUERY:
456 case OP_CRSTAR:
457 case OP_CRMINSTAR:
458 case OP_CRPLUS:
459 case OP_CRMINPLUS:
460 case OP_CRQUERY:
461 case OP_CRMINQUERY:
462 case OP_DEF:
463 case OP_BRAZERO:
464 case OP_BRAMINZERO:
465 case OP_BRAPOSZERO:
466 case OP_FAIL:
467 case OP_ACCEPT:
468 case OP_ASSERT_ACCEPT:
469 case OP_SKIPZERO:
470 return cc + 1;
471
472 case OP_CHAR:
473 case OP_CHARI:
474 case OP_NOT:
475 case OP_NOTI:
476
477 case OP_STAR:
478 case OP_MINSTAR:
479 case OP_PLUS:
480 case OP_MINPLUS:
481 case OP_QUERY:
482 case OP_MINQUERY:
483 case OP_POSSTAR:
484 case OP_POSPLUS:
485 case OP_POSQUERY:
486 case OP_STARI:
487 case OP_MINSTARI:
488 case OP_PLUSI:
489 case OP_MINPLUSI:
490 case OP_QUERYI:
491 case OP_MINQUERYI:
492 case OP_POSSTARI:
493 case OP_POSPLUSI:
494 case OP_POSQUERYI:
495 case OP_NOTSTAR:
496 case OP_NOTMINSTAR:
497 case OP_NOTPLUS:
498 case OP_NOTMINPLUS:
499 case OP_NOTQUERY:
500 case OP_NOTMINQUERY:
501 case OP_NOTPOSSTAR:
502 case OP_NOTPOSPLUS:
503 case OP_NOTPOSQUERY:
504 case OP_NOTSTARI:
505 case OP_NOTMINSTARI:
506 case OP_NOTPLUSI:
507 case OP_NOTMINPLUSI:
508 case OP_NOTQUERYI:
509 case OP_NOTMINQUERYI:
510 case OP_NOTPOSSTARI:
511 case OP_NOTPOSPLUSI:
512 case OP_NOTPOSQUERYI:
513 cc += 2;
514 #ifdef SUPPORT_UTF8
515 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
516 #endif
517 return cc;
518
519 case OP_UPTO:
520 case OP_MINUPTO:
521 case OP_EXACT:
522 case OP_POSUPTO:
523 case OP_UPTOI:
524 case OP_MINUPTOI:
525 case OP_EXACTI:
526 case OP_POSUPTOI:
527 case OP_NOTUPTO:
528 case OP_NOTMINUPTO:
529 case OP_NOTEXACT:
530 case OP_NOTPOSUPTO:
531 case OP_NOTUPTOI:
532 case OP_NOTMINUPTOI:
533 case OP_NOTEXACTI:
534 case OP_NOTPOSUPTOI:
535 cc += 4;
536 #ifdef SUPPORT_UTF8
537 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
538 #endif
539 return cc;
540
541 case OP_NOTPROP:
542 case OP_PROP:
543 case OP_TYPEUPTO:
544 case OP_TYPEMINUPTO:
545 case OP_TYPEEXACT:
546 case OP_TYPEPOSUPTO:
547 case OP_REF:
548 case OP_REFI:
549 case OP_CREF:
550 case OP_CLOSE:
551 cc += 3;
552 return cc;
553
554 case OP_CRRANGE:
555 case OP_CRMINRANGE:
556 return cc + 5;
557
558 case OP_CLASS:
559 case OP_NCLASS:
560 return cc + 33;
561
562 #ifdef SUPPORT_UTF8
563 case OP_XCLASS:
564 return cc + GET(cc, 1);
565 #endif
566
567 case OP_RECURSE:
568 case OP_ASSERT:
569 case OP_ASSERT_NOT:
570 case OP_ASSERTBACK:
571 case OP_ASSERTBACK_NOT:
572 case OP_REVERSE:
573 case OP_ONCE:
574 case OP_BRA:
575 case OP_BRAPOS:
576 case OP_COND:
577 case OP_SBRA:
578 case OP_SBRAPOS:
579 case OP_SCOND:
580 case OP_ALT:
581 case OP_KET:
582 case OP_KETRMAX:
583 case OP_KETRMIN:
584 case OP_KETRPOS:
585 return cc + 1 + LINK_SIZE;
586
587 case OP_CBRA:
588 case OP_CBRAPOS:
589 case OP_SCBRA:
590 case OP_SCBRAPOS:
591 return cc + 1 + LINK_SIZE + 2;
592
593 default:
594 return NULL;
595 }
596 }
597
598 static int get_localspace(compiler_common *common, uschar *cc, uschar *ccend)
599 {
600 int localspace = 0;
601 uschar *alternative;
602 /* Calculate important variables (like stack size) and checks whether all opcodes are supported. */
603 while (cc < ccend)
604 {
605 switch(*cc)
606 {
607 case OP_ASSERT:
608 case OP_ASSERT_NOT:
609 case OP_ASSERTBACK:
610 case OP_ASSERTBACK_NOT:
611 case OP_ONCE:
612 case OP_BRAPOS:
613 case OP_SBRA:
614 case OP_SBRAPOS:
615 case OP_SCOND:
616 localspace += sizeof(sljit_w);
617 cc += 1 + LINK_SIZE;
618 break;
619
620 case OP_CBRAPOS:
621 case OP_SCBRAPOS:
622 localspace += sizeof(sljit_w);
623 cc += 1 + LINK_SIZE + 2;
624 break;
625
626 case OP_COND:
627 /* Might be a hidden SCOND. */
628 alternative = cc + GET(cc, 1);
629 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
630 localspace += sizeof(sljit_w);
631 cc += 1 + LINK_SIZE;
632 break;
633
634 default:
635 cc = next_opcode(common, cc);
636 if (cc == NULL)
637 return -1;
638 break;
639 }
640 }
641 return localspace;
642 }
643
644 static void set_localptrs(compiler_common *common, int localptr, uschar *ccend)
645 {
646 uschar *cc = common->start;
647 uschar *alternative;
648 while (cc < ccend)
649 {
650 switch(*cc)
651 {
652 case OP_ASSERT:
653 case OP_ASSERT_NOT:
654 case OP_ASSERTBACK:
655 case OP_ASSERTBACK_NOT:
656 case OP_ONCE:
657 case OP_BRAPOS:
658 case OP_SBRA:
659 case OP_SBRAPOS:
660 case OP_SCOND:
661 common->localptrs[cc - common->start] = localptr;
662 localptr += sizeof(sljit_w);
663 cc += 1 + LINK_SIZE;
664 break;
665
666 case OP_CBRAPOS:
667 case OP_SCBRAPOS:
668 common->localptrs[cc - common->start] = localptr;
669 localptr += sizeof(sljit_w);
670 cc += 1 + LINK_SIZE + 2;
671 break;
672
673 case OP_COND:
674 /* Might be a hidden SCOND. */
675 alternative = cc + GET(cc, 1);
676 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
677 {
678 common->localptrs[cc - common->start] = localptr;
679 localptr += sizeof(sljit_w);
680 }
681 cc += 1 + LINK_SIZE;
682 break;
683
684 default:
685 cc = next_opcode(common, cc);
686 SLJIT_ASSERT(cc != NULL);
687 break;
688 }
689 }
690 }
691
692 /* Returns with -1 if no need for frame. */
693 static int get_framesize(compiler_common *common, uschar *cc, BOOL recursive)
694 {
695 uschar *ccend = bracketend(cc);
696 uschar *end;
697 int length = 0;
698 BOOL possessive = FALSE;
699 BOOL needs_frame = FALSE;
700 BOOL setsom_found = FALSE;
701
702 if (!recursive && (*cc == OP_CBRAPOS || *cc == OP_SCBRAPOS))
703 {
704 length = 3;
705 possessive = TRUE;
706 }
707
708 cc = next_opcode(common, cc);
709 SLJIT_ASSERT(cc != NULL);
710 while (cc < ccend)
711 switch(*cc)
712 {
713 case OP_SET_SOM:
714 case OP_RECURSE:
715 if (!setsom_found)
716 {
717 length += 2;
718 setsom_found = TRUE;
719 }
720 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
721 break;
722
723 case OP_ASSERT:
724 case OP_ASSERT_NOT:
725 case OP_ASSERTBACK:
726 case OP_ASSERTBACK_NOT:
727 case OP_ONCE:
728 if (needs_frame || length > 0)
729 {
730 cc = bracketend(cc);
731 break;
732 }
733 /* Check whether a frame must be created. */
734 end = bracketend(cc);
735 while (cc < end)
736 {
737 if (*cc == OP_SET_SOM || *cc == OP_CBRA || *cc == OP_CBRAPOS
738 || *cc == OP_SCBRA || *cc == OP_SCBRAPOS || *cc == OP_RECURSE)
739 needs_frame = TRUE;
740 cc = next_opcode(common, cc);
741 SLJIT_ASSERT(cc != NULL);
742 }
743 break;
744
745 case OP_CBRA:
746 case OP_CBRAPOS:
747 case OP_SCBRA:
748 case OP_SCBRAPOS:
749 length += 3;
750 cc += 1 + LINK_SIZE + 2;
751 break;
752
753 default:
754 cc = next_opcode(common, cc);
755 SLJIT_ASSERT(cc != NULL);
756 break;
757 }
758
759 /* Possessive quantifiers can use a special case. */
760 if (SLJIT_UNLIKELY(possessive) && !needs_frame && length == 3 + 2)
761 return -1;
762
763 if (length > 0)
764 return length + 2;
765 return needs_frame ? 0 : -1;
766 }
767
768 static void init_frame(compiler_common *common, uschar *cc, int stackpos, int stacktop, BOOL recursive)
769 {
770 /* TMP2 must contain STACK_TOP - (-STACK(stackpos)) */
771 DEFINE_COMPILER;
772 uschar *ccend = bracketend(cc);
773 BOOL setsom_found = FALSE;
774 int offset;
775
776 if (stackpos < stacktop)
777 {
778 SLJIT_ASSERT(stackpos + 1 == stacktop);
779 return;
780 }
781
782 stackpos = STACK(stackpos);
783 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD);
784 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP2, 0);
785 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacktop), TMP1, 0);
786
787 if (recursive || (*cc != OP_CBRAPOS && *cc != OP_SCBRAPOS))
788 cc = next_opcode(common, cc);
789 SLJIT_ASSERT(cc != NULL);
790 while (cc < ccend)
791 switch(*cc)
792 {
793 case OP_SET_SOM:
794 case OP_RECURSE:
795 if (!setsom_found)
796 {
797 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
798 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_setstrbegin);
799 stackpos += (int)sizeof(sljit_w);
800 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
801 stackpos += (int)sizeof(sljit_w);
802 setsom_found = TRUE;
803 }
804 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
805 break;
806
807 case OP_ASSERT:
808 case OP_ASSERT_NOT:
809 case OP_ASSERTBACK:
810 case OP_ASSERTBACK_NOT:
811 case OP_ONCE:
812 cc = bracketend(cc);
813 break;
814
815 case OP_CBRA:
816 case OP_CBRAPOS:
817 case OP_SCBRA:
818 case OP_SCBRAPOS:
819 offset = (GET2(cc, 1 + LINK_SIZE)) << 1;
820 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset));
821 stackpos += (int)sizeof(sljit_w);
822 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
823 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
824 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
825 stackpos += (int)sizeof(sljit_w);
826 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0);
827 stackpos += (int)sizeof(sljit_w);
828
829 cc += 1 + LINK_SIZE + 2;
830 break;
831
832 default:
833 cc = next_opcode(common, cc);
834 SLJIT_ASSERT(cc != NULL);
835 break;
836 }
837
838 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_end);
839 SLJIT_ASSERT(stackpos == STACK(stacktop + 1));
840 }
841
842 static SLJIT_INLINE int get_localsize(compiler_common *common, uschar *cc, uschar *ccend)
843 {
844 int localsize = 2;
845 uschar *alternative;
846 /* Calculate the sum of the local variables. */
847 while (cc < ccend)
848 {
849 switch(*cc)
850 {
851 case OP_ASSERT:
852 case OP_ASSERT_NOT:
853 case OP_ASSERTBACK:
854 case OP_ASSERTBACK_NOT:
855 case OP_ONCE:
856 case OP_BRAPOS:
857 case OP_SBRA:
858 case OP_SBRAPOS:
859 case OP_SCOND:
860 localsize++;
861 cc += 1 + LINK_SIZE;
862 break;
863
864 case OP_CBRA:
865 case OP_SCBRA:
866 localsize++;
867 cc += 1 + LINK_SIZE + 2;
868 break;
869
870 case OP_CBRAPOS:
871 case OP_SCBRAPOS:
872 localsize += 2;
873 cc += 1 + LINK_SIZE + 2;
874 break;
875
876 case OP_COND:
877 /* Might be a hidden SCOND. */
878 alternative = cc + GET(cc, 1);
879 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
880 localsize++;
881 cc += 1 + LINK_SIZE;
882 break;
883
884 default:
885 cc = next_opcode(common, cc);
886 SLJIT_ASSERT(cc != NULL);
887 break;
888 }
889 }
890 SLJIT_ASSERT(cc == ccend);
891 return localsize;
892 }
893
894 static void copy_locals(compiler_common *common, uschar *cc, uschar *ccend,
895 BOOL save, int stackptr, int stacktop)
896 {
897 DEFINE_COMPILER;
898 int srcw[2];
899 int count;
900 BOOL tmp1next = TRUE;
901 BOOL tmp1empty = TRUE;
902 BOOL tmp2empty = TRUE;
903 uschar *alternative;
904 enum {
905 start,
906 loop,
907 end
908 } status;
909
910 status = save ? start : loop;
911 stackptr = STACK(stackptr - 2);
912 stacktop = STACK(stacktop - 1);
913
914 if (!save)
915 {
916 stackptr += sizeof(sljit_w);
917 if (stackptr < stacktop)
918 {
919 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
920 stackptr += sizeof(sljit_w);
921 tmp1empty = FALSE;
922 }
923 if (stackptr < stacktop)
924 {
925 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
926 stackptr += sizeof(sljit_w);
927 tmp2empty = FALSE;
928 }
929 /* The tmp1next must be TRUE in either way. */
930 }
931
932 while (status != end)
933 {
934 count = 0;
935 switch(status)
936 {
937 case start:
938 SLJIT_ASSERT(save);
939 count = 1;
940 srcw[0] = RECURSIVE_HEAD;
941 status = loop;
942 break;
943
944 case loop:
945 if (cc >= ccend)
946 {
947 status = end;
948 break;
949 }
950
951 switch(*cc)
952 {
953 case OP_ASSERT:
954 case OP_ASSERT_NOT:
955 case OP_ASSERTBACK:
956 case OP_ASSERTBACK_NOT:
957 case OP_ONCE:
958 case OP_BRAPOS:
959 case OP_SBRA:
960 case OP_SBRAPOS:
961 case OP_SCOND:
962 count = 1;
963 srcw[0] = PRIV(cc);
964 SLJIT_ASSERT(srcw[0] != 0);
965 cc += 1 + LINK_SIZE;
966 break;
967
968 case OP_CBRA:
969 case OP_SCBRA:
970 count = 1;
971 srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
972 cc += 1 + LINK_SIZE + 2;
973 break;
974
975 case OP_CBRAPOS:
976 case OP_SCBRAPOS:
977 count = 2;
978 srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
979 srcw[0] = PRIV(cc);
980 SLJIT_ASSERT(srcw[0] != 0);
981 cc += 1 + LINK_SIZE + 2;
982 break;
983
984 case OP_COND:
985 /* Might be a hidden SCOND. */
986 alternative = cc + GET(cc, 1);
987 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
988 {
989 count = 1;
990 srcw[0] = PRIV(cc);
991 SLJIT_ASSERT(srcw[0] != 0);
992 }
993 cc += 1 + LINK_SIZE;
994 break;
995
996 default:
997 cc = next_opcode(common, cc);
998 SLJIT_ASSERT(cc != NULL);
999 break;
1000 }
1001 break;
1002
1003 case end:
1004 SLJIT_ASSERT_STOP();
1005 break;
1006 }
1007
1008 while (count > 0)
1009 {
1010 count--;
1011 if (save)
1012 {
1013 if (tmp1next)
1014 {
1015 if (!tmp1empty)
1016 {
1017 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1018 stackptr += sizeof(sljit_w);
1019 }
1020 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
1021 tmp1empty = FALSE;
1022 tmp1next = FALSE;
1023 }
1024 else
1025 {
1026 if (!tmp2empty)
1027 {
1028 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1029 stackptr += sizeof(sljit_w);
1030 }
1031 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
1032 tmp2empty = FALSE;
1033 tmp1next = TRUE;
1034 }
1035 }
1036 else
1037 {
1038 if (tmp1next)
1039 {
1040 SLJIT_ASSERT(!tmp1empty);
1041 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP1, 0);
1042 tmp1empty = stackptr >= stacktop;
1043 if (!tmp1empty)
1044 {
1045 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1046 stackptr += sizeof(sljit_w);
1047 }
1048 tmp1next = FALSE;
1049 }
1050 else
1051 {
1052 SLJIT_ASSERT(!tmp2empty);
1053 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP2, 0);
1054 tmp2empty = stackptr >= stacktop;
1055 if (!tmp2empty)
1056 {
1057 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1058 stackptr += sizeof(sljit_w);
1059 }
1060 tmp1next = TRUE;
1061 }
1062 }
1063 }
1064 }
1065
1066 if (save)
1067 {
1068 if (tmp1next)
1069 {
1070 if (!tmp1empty)
1071 {
1072 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1073 stackptr += sizeof(sljit_w);
1074 }
1075 if (!tmp2empty)
1076 {
1077 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1078 stackptr += sizeof(sljit_w);
1079 }
1080 }
1081 else
1082 {
1083 if (!tmp2empty)
1084 {
1085 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1086 stackptr += sizeof(sljit_w);
1087 }
1088 if (!tmp1empty)
1089 {
1090 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1091 stackptr += sizeof(sljit_w);
1092 }
1093 }
1094 }
1095 SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty)));
1096 }
1097
1098 static SLJIT_INLINE BOOL ispowerof2(unsigned int value)
1099 {
1100 return (value & (value - 1)) == 0;
1101 }
1102
1103 static SLJIT_INLINE void set_jumps(jump_list *list, struct sljit_label *label)
1104 {
1105 while (list)
1106 {
1107 /* sljit_set_label is clever enough to do nothing
1108 if either the jump or the label is NULL */
1109 sljit_set_label(list->jump, label);
1110 list = list->next;
1111 }
1112 }
1113
1114 static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump* jump)
1115 {
1116 jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list));
1117 if (list_item)
1118 {
1119 list_item->next = *list;
1120 list_item->jump = jump;
1121 *list = list_item;
1122 }
1123 }
1124
1125 static void add_stub(compiler_common *common, enum stub_types type, int data, struct sljit_jump *start)
1126 {
1127 DEFINE_COMPILER;
1128 stub_list* list_item = sljit_alloc_memory(compiler, sizeof(stub_list));
1129
1130 if (list_item)
1131 {
1132 list_item->type = type;
1133 list_item->data = data;
1134 list_item->start = start;
1135 list_item->leave = LABEL();
1136 list_item->next = common->stubs;
1137 common->stubs = list_item;
1138 }
1139 }
1140
1141 static void flush_stubs(compiler_common *common)
1142 {
1143 DEFINE_COMPILER;
1144 stub_list* list_item = common->stubs;
1145
1146 while (list_item)
1147 {
1148 JUMPHERE(list_item->start);
1149 switch(list_item->type)
1150 {
1151 case stack_alloc:
1152 add_jump(compiler, &common->stackalloc, JUMP(SLJIT_FAST_CALL));
1153 break;
1154 }
1155 JUMPTO(SLJIT_JUMP, list_item->leave);
1156 list_item = list_item->next;
1157 }
1158 common->stubs = NULL;
1159 }
1160
1161 static SLJIT_INLINE void decrease_call_count(compiler_common *common)
1162 {
1163 DEFINE_COMPILER;
1164
1165 OP2(SLJIT_SUB | SLJIT_SET_E, CALL_COUNT, 0, CALL_COUNT, 0, SLJIT_IMM, 1);
1166 add_jump(compiler, &common->calllimit, JUMP(SLJIT_C_ZERO));
1167 }
1168
1169 static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
1170 {
1171 /* May destroy all locals and registers except TMP2. */
1172 DEFINE_COMPILER;
1173
1174 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1175 #ifdef DESTROY_REGISTERS
1176 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345);
1177 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
1178 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
1179 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, TMP1, 0);
1180 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
1181 #endif
1182 add_stub(common, stack_alloc, 0, CMP(SLJIT_C_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
1183 }
1184
1185 static SLJIT_INLINE void free_stack(compiler_common *common, int size)
1186 {
1187 DEFINE_COMPILER;
1188 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1189 }
1190
1191 static SLJIT_INLINE void reset_ovector(compiler_common *common, int length)
1192 {
1193 DEFINE_COMPILER;
1194 struct sljit_label *loop;
1195 int i;
1196 /* At this point we can freely use all temporary registers. */
1197 /* TMP1 returns with begin - 1. */
1198 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, 1);
1199 if (length < 8)
1200 {
1201 for (i = 0; i < length; i++)
1202 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(i), SLJIT_TEMPORARY_REG1, 0);
1203 }
1204 else
1205 {
1206 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START - sizeof(sljit_w));
1207 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, length);
1208 loop = LABEL();
1209 OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), sizeof(sljit_w), SLJIT_TEMPORARY_REG1, 0);
1210 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG3, 0, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, 1);
1211 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1212 }
1213 }
1214
1215 static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket)
1216 {
1217 DEFINE_COMPILER;
1218 struct sljit_label *loop;
1219 struct sljit_jump *earlyexit;
1220
1221 /* At this point we can freely use all registers. */
1222 OP1(SLJIT_MOV, SLJIT_GENERAL_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
1223 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1), STR_PTR, 0);
1224
1225 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, ARGUMENTS, 0);
1226 OP1(SLJIT_MOV_SI, SLJIT_TEMPORARY_REG2, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsetcount));
1227 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
1228 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, begin));
1229 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START);
1230 /* Unlikely, but possible */
1231 earlyexit = CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 0);
1232 loop = LABEL();
1233 OP2(SLJIT_SUB, SLJIT_GENERAL_REG2, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), 0, SLJIT_TEMPORARY_REG1, 0);
1234 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_GENERAL_REG1, 0, SLJIT_IMM, sizeof(sljit_w));
1235 /* Copy the integer value to the output buffer */
1236 OP1(SLJIT_MOVU_SI, SLJIT_MEM1(SLJIT_TEMPORARY_REG3), sizeof(int), SLJIT_GENERAL_REG2, 0);
1237 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1238 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1239 JUMPHERE(earlyexit);
1240
1241 /* Calculate the return value, which is the maximum ovector value. */
1242 if (topbracket > 1)
1243 {
1244 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START + topbracket * 2 * sizeof(sljit_w));
1245 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, topbracket + 1);
1246
1247 /* OVECTOR(0) is never equal to SLJIT_GENERAL_REG3. */
1248 loop = LABEL();
1249 OP1(SLJIT_MOVU, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), -(2 * (sljit_w)sizeof(sljit_w)));
1250 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1251 CMPTO(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG3, 0, SLJIT_GENERAL_REG3, 0, loop);
1252 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_TEMPORARY_REG2, 0);
1253 }
1254 else
1255 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
1256 }
1257
1258 static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, uschar* cc)
1259 {
1260 /* Detects if the character has an othercase. */
1261 unsigned int c;
1262
1263 #ifdef SUPPORT_UTF8
1264 if (common->utf8)
1265 {
1266 GETCHAR(c, cc);
1267 if (c > 127)
1268 {
1269 #ifdef SUPPORT_UCP
1270 return c != UCD_OTHERCASE(c);
1271 #else
1272 return FALSE;
1273 #endif
1274 }
1275 }
1276 else
1277 #endif
1278 c = *cc;
1279 return common->fcc[c] != c;
1280 }
1281
1282 static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigned int c)
1283 {
1284 /* Returns with the othercase. */
1285 #ifdef SUPPORT_UTF8
1286 if (common->utf8 && c > 127)
1287 {
1288 #ifdef SUPPORT_UCP
1289 return UCD_OTHERCASE(c);
1290 #else
1291 return c;
1292 #endif
1293 }
1294 #endif
1295 return common->fcc[c];
1296 }
1297
1298 static unsigned int char_get_othercase_bit(compiler_common *common, uschar* cc)
1299 {
1300 /* Detects if the character and its othercase has only 1 bit difference. */
1301 unsigned int c, oc, bit;
1302 #ifdef SUPPORT_UTF8
1303 int n;
1304 #endif
1305
1306 #ifdef SUPPORT_UTF8
1307 if (common->utf8)
1308 {
1309 GETCHAR(c, cc);
1310 if (c <= 127)
1311 oc = common->fcc[c];
1312 else
1313 {
1314 #ifdef SUPPORT_UCP
1315 oc = UCD_OTHERCASE(c);
1316 #else
1317 oc = c;
1318 #endif
1319 }
1320 }
1321 else
1322 {
1323 c = *cc;
1324 oc = common->fcc[c];
1325 }
1326 #else
1327 c = *cc;
1328 oc = common->fcc[c];
1329 #endif
1330
1331 SLJIT_ASSERT(c != oc);
1332
1333 bit = c ^ oc;
1334 /* Optimized for English alphabet. */
1335 if (c <= 127 && bit == 0x20)
1336 return (0 << 8) | 0x20;
1337
1338 /* Since c != oc, they must have at least 1 bit difference. */
1339 if (!ispowerof2(bit))
1340 return 0;
1341
1342 #ifdef SUPPORT_UTF8
1343 if (common->utf8 && c > 127)
1344 {
1345 n = _pcre_utf8_table4[*cc & 0x3f];
1346 while ((bit & 0x3f) == 0)
1347 {
1348 n--;
1349 bit >>= 6;
1350 }
1351 return (n << 8) | bit;
1352 }
1353 #endif
1354 return (0 << 8) | bit;
1355 }
1356
1357 static SLJIT_INLINE void check_input_end(compiler_common *common, jump_list **fallbacks)
1358 {
1359 DEFINE_COMPILER;
1360 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1361 }
1362
1363 static void read_char(compiler_common *common)
1364 {
1365 /* Reads the character into TMP1, updates STR_PTR.
1366 Does not check STR_END. TMP2 Destroyed. */
1367 DEFINE_COMPILER;
1368 #ifdef SUPPORT_UTF8
1369 struct sljit_jump *jump;
1370 #endif
1371
1372 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1373 #ifdef SUPPORT_UTF8
1374 if (common->utf8)
1375 {
1376 /* Should not found a value between 128 and 192 here. */
1377 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 192);
1378 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1379 JUMPHERE(jump);
1380 }
1381 #endif
1382 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1383 }
1384
1385 static void peek_char(compiler_common *common)
1386 {
1387 /* Reads the character into TMP1, keeps STR_PTR.
1388 Does not check STR_END. TMP2 Destroyed. */
1389 DEFINE_COMPILER;
1390 #ifdef SUPPORT_UTF8
1391 struct sljit_jump *jump;
1392 #endif
1393
1394 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1395 #ifdef SUPPORT_UTF8
1396 if (common->utf8)
1397 {
1398 /* Should not found a value between 128 and 192 here. */
1399 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 192);
1400 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1401 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1402 JUMPHERE(jump);
1403 }
1404 #endif
1405 }
1406
1407 static void read_char8_type(compiler_common *common)
1408 {
1409 /* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
1410 DEFINE_COMPILER;
1411 #ifdef SUPPORT_UTF8
1412 struct sljit_jump *jump;
1413 #endif
1414
1415 #ifdef SUPPORT_UTF8
1416 if (common->utf8)
1417 {
1418 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1419 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1420 /* This can be an extra read in some situations, but hopefully
1421 it is a clever early read in most cases. */
1422 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1423 /* Should not found a value between 128 and 192 here. */
1424 jump = CMP(SLJIT_C_LESS, TMP2, 0, SLJIT_IMM, 192);
1425 add_jump(compiler, &common->utf8readtype8, JUMP(SLJIT_FAST_CALL));
1426 JUMPHERE(jump);
1427 return;
1428 }
1429 #endif
1430 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1431 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1432 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
1433 }
1434
1435 static void skip_char_back(compiler_common *common)
1436 {
1437 /* Goes one character back. Only affects STR_PTR. Does not check begin. */
1438 DEFINE_COMPILER;
1439 #ifdef SUPPORT_UTF8
1440 struct sljit_label *label;
1441
1442 if (common->utf8)
1443 {
1444 label = LABEL();
1445 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1446 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1447 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
1448 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label);
1449 return;
1450 }
1451 #endif
1452 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1453 }
1454
1455 static void check_newlinechar(compiler_common *common, int nltype, jump_list **fallbacks, BOOL jumpiftrue)
1456 {
1457 /* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */
1458 DEFINE_COMPILER;
1459
1460 if (nltype == NLTYPE_ANY)
1461 {
1462 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
1463 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1464 }
1465 else if (nltype == NLTYPE_ANYCRLF)
1466 {
1467 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_CR);
1468 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1469 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1470 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1471 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1472 }
1473 else
1474 {
1475 SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline <= 255);
1476 add_jump(compiler, fallbacks, CMP(jumpiftrue ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
1477 }
1478 }
1479
1480 #ifdef SUPPORT_UTF8
1481 static void do_utf8readchar(compiler_common *common)
1482 {
1483 /* Fast decoding an utf8 character. TMP1 contains the first byte
1484 of the character (>= 192). Return char value in TMP1, length - 1 in TMP2. */
1485 DEFINE_COMPILER;
1486 struct sljit_jump *jump;
1487
1488 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1489 /* Searching for the first zero. */
1490 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
1491 jump = JUMP(SLJIT_C_NOT_ZERO);
1492 /* 2 byte sequence */
1493 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1494 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1495 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1f);
1496 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
1497 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1498 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1499 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
1500 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1501 JUMPHERE(jump);
1502
1503 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10);
1504 jump = JUMP(SLJIT_C_NOT_ZERO);
1505 /* 3 byte sequence */
1506 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1507 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0f);
1508 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 12);
1509 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1510 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1511 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1512 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1513 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 2);
1514 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1515 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1516 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 2);
1517 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1518 JUMPHERE(jump);
1519
1520 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x08);
1521 jump = JUMP(SLJIT_C_NOT_ZERO);
1522 /* 4 byte sequence */
1523 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1524 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x07);
1525 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 18);
1526 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1527 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1528 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1529 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1530 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1531 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1532 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1533 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1534 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 3);
1535 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1536 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1537 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 3);
1538 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1539 JUMPHERE(jump);
1540
1541 /* 5 byte sequence */
1542 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1543 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x03);
1544 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 24);
1545 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1546 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 18);
1547 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1548 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1549 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1550 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1551 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1552 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1553 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1554 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1555 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1556 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 4);
1557 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 4);
1558 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1559 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1560 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 4);
1561 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1562 }
1563
1564 static void do_utf8readtype8(compiler_common *common)
1565 {
1566 /* Fast decoding an utf8 character type. TMP2 contains the first byte
1567 of the character (>= 192) and TMP1 is destroyed. Return value in TMP1. */
1568 DEFINE_COMPILER;
1569 struct sljit_jump *jump;
1570 struct sljit_jump *compare;
1571
1572 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1573
1574 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
1575 jump = JUMP(SLJIT_C_NOT_ZERO);
1576 /* 2 byte sequence */
1577 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1578 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1579 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f);
1580 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1581 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
1582 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0);
1583 compare = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
1584 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1585 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1586
1587 JUMPHERE(compare);
1588 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1589 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1590 JUMPHERE(jump);
1591
1592 /* We only have types for characters less than 256. */
1593 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_utf8_char_sizes);
1594 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
1595 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1596 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1597 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1598 }
1599
1600 #endif
1601
1602 #ifdef SUPPORT_UCP
1603
1604 /* UCD_BLOCK_SIZE must be 128 (see the assert below). */
1605 #define UCD_BLOCK_MASK 127
1606 #define UCD_BLOCK_SHIFT 7
1607
1608 static void do_getucd(compiler_common *common)
1609 {
1610 /* Search the UCD record for the character comes in TMP1.
1611 Returns chartype in TMP1 and UCD offset in TMP2. */
1612 DEFINE_COMPILER;
1613
1614 SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8);
1615
1616 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1617 OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1618 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_ucd_stage1);
1619 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
1620 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1621 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
1622 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_stage2);
1623 OP1(SLJIT_MOV_UH, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
1624 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, chartype));
1625 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
1626 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1627 }
1628 #endif
1629
1630 static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf, BOOL firstline)
1631 {
1632 DEFINE_COMPILER;
1633 struct sljit_label *mainloop;
1634 struct sljit_label *newlinelabel = NULL;
1635 struct sljit_jump *start;
1636 struct sljit_jump *end = NULL;
1637 struct sljit_jump *nl = NULL;
1638 jump_list *newline = NULL;
1639 BOOL newlinecheck = FALSE;
1640 BOOL readbyte = FALSE;
1641
1642 if (!(hascrorlf || firstline) && (common->nltype == NLTYPE_ANY ||
1643 common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
1644 newlinecheck = TRUE;
1645
1646 if (firstline)
1647 {
1648 /* Search for the end of the first line. */
1649 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STR_PTR, 0);
1650 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_END, 0);
1651
1652 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1653 {
1654 mainloop = LABEL();
1655 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1656 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1657 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
1658 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1659 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
1660 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
1661 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0, SLJIT_IMM, 1);
1662 }
1663 else
1664 {
1665 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1666 mainloop = LABEL();
1667 /* Continual stores does not cause data dependency. */
1668 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1669 read_char(common);
1670 check_newlinechar(common, common->nltype, &newline, TRUE);
1671 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
1672 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1673 set_jumps(newline, LABEL());
1674 }
1675
1676 JUMPHERE(end);
1677 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
1678 }
1679
1680 start = JUMP(SLJIT_JUMP);
1681
1682 if (newlinecheck)
1683 {
1684 newlinelabel = LABEL();
1685 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1686 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1687 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1688 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
1689 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1690 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1691 nl = JUMP(SLJIT_JUMP);
1692 }
1693
1694 mainloop = LABEL();
1695
1696 /* Increasing the STR_PTR here requires one less jump in the most common case. */
1697 #ifdef SUPPORT_UTF8
1698 if (common->utf8) readbyte = TRUE;
1699 #endif
1700 if (newlinecheck) readbyte = TRUE;
1701
1702 if (readbyte)
1703 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1704
1705 if (newlinecheck)
1706 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);
1707
1708 #ifdef SUPPORT_UTF8
1709 if (common->utf8)
1710 {
1711 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes);
1712 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1713 }
1714 else
1715 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1716 #else
1717 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1718 #endif
1719 JUMPHERE(start);
1720
1721 if (newlinecheck)
1722 {
1723 JUMPHERE(end);
1724 JUMPHERE(nl);
1725 }
1726
1727 return mainloop;
1728 }
1729
1730 static SLJIT_INLINE void fast_forward_first_byte(compiler_common *common, pcre_uint16 firstbyte, BOOL firstline)
1731 {
1732 DEFINE_COMPILER;
1733 struct sljit_label *start;
1734 struct sljit_jump *leave;
1735 struct sljit_jump *found;
1736 pcre_uint16 oc, bit;
1737
1738 if (firstline)
1739 {
1740 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1741 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1742 }
1743
1744 start = LABEL();
1745 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1746 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1747
1748 if ((firstbyte & REQ_CASELESS) == 0)
1749 found = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, firstbyte & 0xff);
1750 else
1751 {
1752 firstbyte &= 0xff;
1753 oc = common->fcc[firstbyte];
1754 bit = firstbyte ^ oc;
1755 if (ispowerof2(bit))
1756 {
1757 OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, bit);
1758 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, firstbyte | bit);
1759 }
1760 else
1761 {
1762 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, firstbyte);
1763 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1764 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc);
1765 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1766 found = JUMP(SLJIT_C_NOT_ZERO);
1767 }
1768 }
1769
1770 #ifdef SUPPORT_UTF8
1771 if (common->utf8)
1772 {
1773 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes);
1774 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1775 }
1776 else
1777 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1778 #else
1779 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1780 #endif
1781 JUMPTO(SLJIT_JUMP, start);
1782 JUMPHERE(found);
1783 JUMPHERE(leave);
1784
1785 if (firstline)
1786 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1787 }
1788
1789 static SLJIT_INLINE void fast_forward_newline(compiler_common *common, BOOL firstline)
1790 {
1791 DEFINE_COMPILER;
1792 struct sljit_label *loop;
1793 struct sljit_jump *lastchar;
1794 struct sljit_jump *firstchar;
1795 struct sljit_jump *leave;
1796 struct sljit_jump *foundcr = NULL;
1797 struct sljit_jump *notfoundnl;
1798 jump_list *newline = NULL;
1799
1800 if (firstline)
1801 {
1802 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1803 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1804 }
1805
1806 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1807 {
1808 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1809 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1810 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1811 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
1812 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1813
1814 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2);
1815 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
1816 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER_EQUAL);
1817 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1818
1819 loop = LABEL();
1820 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1821 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1822 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
1823 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
1824 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
1825 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);
1826
1827 JUMPHERE(leave);
1828 JUMPHERE(firstchar);
1829 JUMPHERE(lastchar);
1830
1831 if (firstline)
1832 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1833 return;
1834 }
1835
1836 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1837 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1838 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1839 skip_char_back(common);
1840
1841 loop = LABEL();
1842 read_char(common);
1843 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1844 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1845 foundcr = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
1846 check_newlinechar(common, common->nltype, &newline, FALSE);
1847 set_jumps(newline, loop);
1848
1849 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1850 {
1851 leave = JUMP(SLJIT_JUMP);
1852 JUMPHERE(foundcr);
1853 notfoundnl = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1854 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1855 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1856 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1857 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1858 JUMPHERE(notfoundnl);
1859 JUMPHERE(leave);
1860 }
1861 JUMPHERE(lastchar);
1862 JUMPHERE(firstchar);
1863
1864 if (firstline)
1865 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1866 }
1867
1868 static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, sljit_uw start_bits, BOOL firstline)
1869 {
1870 DEFINE_COMPILER;
1871 struct sljit_label *start;
1872 struct sljit_jump *leave;
1873 struct sljit_jump *found;
1874
1875 if (firstline)
1876 {
1877 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1878 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1879 }
1880
1881 start = LABEL();
1882 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1883 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1884 #ifdef SUPPORT_UTF8
1885 if (common->utf8)
1886 OP1(SLJIT_MOV_UB, TMP3, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes);
1887 #endif
1888 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
1889 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
1890 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), start_bits);
1891 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
1892 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
1893 found = JUMP(SLJIT_C_NOT_ZERO);
1894
1895 #ifdef SUPPORT_UTF8
1896 if (common->utf8)
1897 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP3, 0);
1898 else
1899 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1900 #else
1901 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1902 #endif
1903 JUMPTO(SLJIT_JUMP, start);
1904 JUMPHERE(found);
1905 JUMPHERE(leave);
1906
1907 if (firstline)
1908 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1909 }
1910
1911 static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, pcre_uint16 reqbyte, BOOL has_firstbyte)
1912 {
1913 DEFINE_COMPILER;
1914 struct sljit_label *loop;
1915 struct sljit_jump *toolong;
1916 struct sljit_jump *alreadyfound;
1917 struct sljit_jump *found;
1918 struct sljit_jump *foundoc = NULL;
1919 struct sljit_jump *notfound;
1920 pcre_uint16 oc, bit;
1921
1922 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR);
1923 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX);
1924 toolong = CMP(SLJIT_C_LESS, TMP1, 0, STR_END, 0);
1925 alreadyfound = CMP(SLJIT_C_LESS, STR_PTR, 0, TMP2, 0);
1926
1927 if (has_firstbyte)
1928 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, 1);
1929 else
1930 OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0);
1931
1932 loop = LABEL();
1933 notfound = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, STR_END, 0);
1934
1935 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1936 if ((reqbyte & REQ_CASELESS) == 0)
1937 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte & 0xff);
1938 else
1939 {
1940 reqbyte &= 0xff;
1941 oc = common->fcc[reqbyte];
1942 bit = reqbyte ^ oc;
1943 if (ispowerof2(bit))
1944 {
1945 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit);
1946 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte | bit);
1947 }
1948 else
1949 {
1950 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte);
1951 foundoc = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, oc);
1952 }
1953 }
1954 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
1955 JUMPTO(SLJIT_JUMP, loop);
1956
1957 JUMPHERE(found);
1958 if (foundoc)
1959 JUMPHERE(foundoc);
1960 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, TMP1, 0);
1961 JUMPHERE(alreadyfound);
1962 JUMPHERE(toolong);
1963 return notfound;
1964 }
1965
1966 static void do_revertframes(compiler_common *common)
1967 {
1968 DEFINE_COMPILER;
1969 struct sljit_jump *earlyexit;
1970 struct sljit_jump *jump;
1971 struct sljit_label *mainloop;
1972
1973 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1974 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD);
1975
1976 /* Drop frames until we reach STACK_TOP. */
1977 earlyexit = CMP(SLJIT_C_LESS, TMP1, 0, STACK_TOP, 0);
1978 mainloop = LABEL();
1979 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1980 jump = CMP(SLJIT_C_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1981 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_LOCALS_REG, 0);
1982 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1983 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_w), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_w));
1984 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_w));
1985 JUMPTO(SLJIT_JUMP, mainloop);
1986
1987 JUMPHERE(jump);
1988 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1989 /* End of dropping frames. */
1990 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1991 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP1, 0);
1992 CMPTO(SLJIT_C_GREATER_EQUAL, TMP1, 0, STACK_TOP, 0, mainloop);
1993 JUMPHERE(earlyexit);
1994 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1995
1996 JUMPHERE(jump);
1997 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_setstrbegin);
1998 /* Set string begin. */
1999 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
2000 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
2001 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
2002 JUMPTO(SLJIT_JUMP, mainloop);
2003
2004 JUMPHERE(jump);
2005 /* Unknown command. */
2006 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
2007 JUMPTO(SLJIT_JUMP, mainloop);
2008 }
2009
2010 static void check_wordboundary(compiler_common *common)
2011 {
2012 DEFINE_COMPILER;
2013 struct sljit_jump *beginend;
2014 #ifdef SUPPORT_UTF8
2015 struct sljit_jump *jump;
2016 #endif
2017
2018 SLJIT_ASSERT(ctype_word == 0x10);
2019
2020 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
2021 /* Get type of the previous char, and put it to LOCALS1. */
2022 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2023 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
2024 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, 0);
2025 beginend = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
2026 skip_char_back(common);
2027 read_char(common);
2028
2029 /* Testing char type. */
2030 #ifdef SUPPORT_UCP
2031 if (common->useucp)
2032 {
2033 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2034 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2035 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2036 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2037 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2038 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2039 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2040 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2041 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2042 JUMPHERE(jump);
2043 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
2044 }
2045 else
2046 #endif
2047 {
2048 #ifdef SUPPORT_UTF8
2049 /* Here LOCALS1 has already been zeroed. */
2050 jump = NULL;
2051 if (common->utf8)
2052 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2053 #endif
2054 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
2055 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */);
2056 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2057 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
2058 #ifdef SUPPORT_UTF8
2059 if (jump != NULL)
2060 JUMPHERE(jump);
2061 #endif
2062 }
2063 JUMPHERE(beginend);
2064
2065 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2066 beginend = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2067 peek_char(common);
2068
2069 /* Testing char type. This is a code duplication. */
2070 #ifdef SUPPORT_UCP
2071 if (common->useucp)
2072 {
2073 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2074 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2075 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2076 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2077 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2078 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2079 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2080 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2081 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2082 JUMPHERE(jump);
2083 }
2084 else
2085 #endif
2086 {
2087 #ifdef SUPPORT_UTF8
2088 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2089 jump = NULL;
2090 if (common->utf8)
2091 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2092 #endif
2093 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes);
2094 OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */);
2095 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2096 #ifdef SUPPORT_UTF8
2097 if (jump != NULL)
2098 JUMPHERE(jump);
2099 #endif
2100 }
2101 JUMPHERE(beginend);
2102
2103 OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2104 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2105 }
2106
2107 static void check_anynewline(compiler_common *common)
2108 {
2109 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2110 DEFINE_COMPILER;
2111
2112 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2113
2114 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2115 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2116 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2117 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2118 #ifdef SUPPORT_UTF8
2119 if (common->utf8)
2120 {
2121 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2122 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2123 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2124 }
2125 #endif
2126 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2127 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2128 }
2129
2130 static void check_hspace(compiler_common *common)
2131 {
2132 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2133 DEFINE_COMPILER;
2134
2135 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2136
2137 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
2138 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2139 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
2140 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2141 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
2142 #ifdef SUPPORT_UTF8
2143 if (common->utf8)
2144 {
2145 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2146 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
2147 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2148 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
2149 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2150 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000);
2151 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
2152 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2153 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
2154 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2155 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
2156 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2157 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
2158 }
2159 #endif
2160 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2161
2162 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2163 }
2164
2165 static void check_vspace(compiler_common *common)
2166 {
2167 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2168 DEFINE_COMPILER;
2169
2170 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2171
2172 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2173 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2174 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2175 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2176 #ifdef SUPPORT_UTF8
2177 if (common->utf8)
2178 {
2179 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2180 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2181 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2182 }
2183 #endif
2184 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2185
2186 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2187 }
2188
2189 #define CHAR1 STR_END
2190 #define CHAR2 STACK_TOP
2191
2192 static void do_casefulcmp(compiler_common *common)
2193 {
2194 DEFINE_COMPILER;
2195 struct sljit_jump *jump;
2196 struct sljit_label *label;
2197
2198 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2199 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2200 OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0);
2201 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR2, 0);
2202 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2203 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2204
2205 label = LABEL();
2206 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2207 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2208 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2209 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2210 JUMPTO(SLJIT_C_NOT_ZERO, label);
2211
2212 JUMPHERE(jump);
2213 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2214 OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0);
2215 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2216 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2217 }
2218
2219 #define LCC_TABLE STACK_LIMIT
2220
2221 static void do_caselesscmp(compiler_common *common)
2222 {
2223 DEFINE_COMPILER;
2224 struct sljit_jump *jump;
2225 struct sljit_label *label;
2226
2227 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2228 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2229
2230 OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0);
2231 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR1, 0);
2232 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, CHAR2, 0);
2233 OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc);
2234 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2235 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2236
2237 label = LABEL();
2238 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2239 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2240 OP1(SLJIT_MOV_UB, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
2241 OP1(SLJIT_MOV_UB, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
2242 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2243 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2244 JUMPTO(SLJIT_C_NOT_ZERO, label);
2245
2246 JUMPHERE(jump);
2247 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2248 OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0);
2249 OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2250 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2251 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2252 }
2253
2254 #undef LCC_TABLE
2255 #undef CHAR1
2256 #undef CHAR2
2257
2258 #ifdef SUPPORT_UTF8
2259 #ifdef SUPPORT_UCP
2260
2261 static uschar * SLJIT_CALL do_utf8caselesscmp(uschar *src1, jit_arguments *args, uschar *end1)
2262 {
2263 /* This function would be ineffective to do in JIT level. */
2264 int c1, c2;
2265 uschar *src2 = args->ptr;
2266 uschar *end2 = (uschar*)args->end;
2267
2268 while (src1 < end1)
2269 {
2270 if (src2 >= end2)
2271 return 0;
2272 GETCHARINC(c1, src1);
2273 GETCHARINC(c2, src2);
2274 if (c1 != c2 && c1 != UCD_OTHERCASE(c2)) return 0;
2275 }
2276 return src2;
2277 }
2278
2279 #endif
2280 #endif
2281
2282 static uschar *byte_sequence_compare(compiler_common *common, BOOL caseless, uschar *cc,
2283 compare_context* context, jump_list **fallbacks)
2284 {
2285 DEFINE_COMPILER;
2286 unsigned int othercasebit = 0;
2287 uschar *othercasebyte = NULL;
2288 #ifdef SUPPORT_UTF8
2289 int utf8length;
2290 #endif
2291
2292 if (caseless && char_has_othercase(common, cc))
2293 {
2294 othercasebit = char_get_othercase_bit(common, cc);
2295 SLJIT_ASSERT(othercasebit);
2296 /* Extracting bit difference info. */
2297 othercasebyte = cc + (othercasebit >> 8);
2298 othercasebit &= 0xff;
2299 }
2300
2301 if (context->sourcereg == -1)
2302 {
2303 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2304 if (context->length >= 4)
2305 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2306 else if (context->length >= 2)
2307 OP1(SLJIT_MOV_SH, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2308 else
2309 #endif
2310 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2311 context->sourcereg = TMP2;
2312 }
2313
2314 #ifdef SUPPORT_UTF8
2315 utf8length = 1;
2316 if (common->utf8 && *cc >= 0xc0)
2317 utf8length += _pcre_utf8_table4[*cc & 0x3f];
2318
2319 do
2320 {
2321 #endif
2322
2323 context->length--;
2324 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2325
2326 /* Unaligned read is supported. */
2327 if (othercasebit != 0 && othercasebyte == cc)
2328 {
2329 context->c.asbytes[context->byteptr] = *cc | othercasebit;
2330 context->oc.asbytes[context->byteptr] = othercasebit;
2331 }
2332 else
2333 {
2334 context->c.asbytes[context->byteptr] = *cc;
2335 context->oc.asbytes[context->byteptr] = 0;
2336 }
2337 context->byteptr++;
2338
2339 if (context->byteptr >= 4 || context->length == 0 || (context->byteptr == 2 && context->length == 1))
2340 {
2341 if (context->length >= 4)
2342 OP1(SLJIT_MOV_SI, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2343 else if (context->length >= 2)
2344 OP1(SLJIT_MOV_SH, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2345 else if (context->length >= 1)
2346 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2347 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2348
2349 switch(context->byteptr)
2350 {
2351 case 4:
2352 if (context->oc.asint != 0)
2353 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint);
2354 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint));
2355 break;
2356
2357 case 2:
2358 if (context->oc.asshort != 0)
2359 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asshort);
2360 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asshort | context->oc.asshort));
2361 break;
2362
2363 case 1:
2364 if (context->oc.asbyte != 0)
2365 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte);
2366 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte));
2367 break;
2368
2369 default:
2370 SLJIT_ASSERT_STOP();
2371 break;
2372 }
2373 context->byteptr = 0;
2374 }
2375
2376 #else
2377
2378 /* Unaligned read is unsupported. */
2379 if (context->length > 0)
2380 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2381 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2382
2383 if (othercasebit != 0 && othercasebyte == cc)
2384 {
2385 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit);
2386 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit));
2387 }
2388 else
2389 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc));
2390
2391 #endif
2392
2393 cc++;
2394 #ifdef SUPPORT_UTF8
2395 utf8length--;
2396 }
2397 while (utf8length > 0);
2398 #endif
2399
2400 return cc;
2401 }
2402
2403 #ifdef SUPPORT_UTF8
2404
2405 #define SET_TYPE_OFFSET(value) \
2406 if ((value) != typeoffset) \
2407 { \
2408 if ((value) > typeoffset) \
2409 OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \
2410 else \
2411 OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \
2412 } \
2413 typeoffset = (value);
2414
2415 #define SET_CHAR_OFFSET(value) \
2416 if ((value) != charoffset) \
2417 { \
2418 if ((value) > charoffset) \
2419 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (value) - charoffset); \
2420 else \
2421 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, charoffset - (value)); \
2422 } \
2423 charoffset = (value);
2424
2425 static void compile_xclass_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks)
2426 {
2427 DEFINE_COMPILER;
2428 jump_list *found = NULL;
2429 jump_list **list = (*cc & XCL_NOT) == 0 ? &found : fallbacks;
2430 unsigned int c;
2431 int compares;
2432 struct sljit_jump *jump = NULL;
2433 uschar *ccbegin;
2434 #ifdef SUPPORT_UCP
2435 BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE;
2436 BOOL charsaved = FALSE;
2437 int typereg = TMP1, scriptreg = TMP1;
2438 unsigned int typeoffset;
2439 #endif
2440 int invertcmp, numberofcmps;
2441 unsigned int charoffset;
2442
2443 /* Although SUPPORT_UTF8 must be defined, we are not necessary in utf8 mode. */
2444 check_input_end(common, fallbacks);
2445 read_char(common);
2446
2447 if ((*cc++ & XCL_MAP) != 0)
2448 {
2449 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2450 if (common->utf8)
2451 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2452
2453 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
2454 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
2455 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
2456 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
2457 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
2458 add_jump(compiler, list, JUMP(SLJIT_C_NOT_ZERO));
2459
2460 if (common->utf8)
2461 JUMPHERE(jump);
2462 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2463 #ifdef SUPPORT_UCP
2464 charsaved = TRUE;
2465 #endif
2466 cc += 32;
2467 }
2468
2469 /* Scanning the necessary info. */
2470 ccbegin = cc;
2471 compares = 0;
2472 while (*cc != XCL_END)
2473 {
2474 compares++;
2475 if (*cc == XCL_SINGLE)
2476 {
2477 cc += 2;
2478 #ifdef SUPPORT_UTF8
2479 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2480 #endif
2481 #ifdef SUPPORT_UCP
2482 needschar = TRUE;
2483 #endif
2484 }
2485 else if (*cc == XCL_RANGE)
2486 {
2487 cc += 2;
2488 #ifdef SUPPORT_UTF8
2489 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2490 #endif
2491 cc++;
2492 #ifdef SUPPORT_UTF8
2493 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2494 #endif
2495 #ifdef SUPPORT_UCP
2496 needschar = TRUE;
2497 #endif
2498 }
2499 #ifdef SUPPORT_UCP
2500 else
2501 {
2502 SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
2503 cc++;
2504 switch(*cc)
2505 {
2506 case PT_ANY:
2507 break;
2508
2509 case PT_LAMP:
2510 case PT_GC:
2511 case PT_PC:
2512 case PT_ALNUM:
2513 needstype = TRUE;
2514 break;
2515
2516 case PT_SC:
2517 needsscript = TRUE;
2518 break;
2519
2520 case PT_SPACE:
2521 case PT_PXSPACE:
2522 case PT_WORD:
2523 needstype = TRUE;
2524 needschar = TRUE;
2525 break;
2526
2527 default:
2528 SLJIT_ASSERT_STOP();
2529 break;
2530 }
2531 cc += 2;
2532 }
2533 #endif
2534 }
2535
2536 #ifdef SUPPORT_UCP
2537 /* Simple register allocation. TMP1 is preferred if possible. */
2538 if (needstype || needsscript)
2539 {
2540 if (needschar && !charsaved)
2541 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2542 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2543 if (needschar)
2544 {
2545 if (needstype)
2546 {
2547 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
2548 typereg = RETURN_ADDR;
2549 }
2550
2551 if (needsscript)
2552 scriptreg = TMP3;
2553 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2554 }
2555 else if (needstype && needsscript)
2556 scriptreg = TMP3;
2557 /* In all other cases only one of them was specified, and that can goes to TMP1. */
2558
2559 if (needsscript)
2560 {
2561 if (scriptreg == TMP1)
2562 {
2563 OP1(SLJIT_MOV, scriptreg, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2564 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM2(scriptreg, TMP2), 3);
2565 }
2566 else
2567 {
2568 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3);
2569 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2570 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM1(TMP2), 0);
2571 }
2572 }
2573 }
2574 #endif
2575
2576 /* Generating code. */
2577 cc = ccbegin;
2578 charoffset = 0;
2579 numberofcmps = 0;
2580 #ifdef SUPPORT_UCP
2581 typeoffset = 0;
2582 #endif
2583
2584 while (*cc != XCL_END)
2585 {
2586 compares--;
2587 invertcmp = (compares == 0 && list != fallbacks);
2588 jump = NULL;
2589
2590 if (*cc == XCL_SINGLE)
2591 {
2592 cc ++;
2593 #ifdef SUPPORT_UTF8
2594 if (common->utf8)
2595 {
2596 GETCHARINC(c, cc);
2597 }
2598 else
2599 #endif
2600 c = *cc++;
2601
2602 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2603 {
2604 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2605 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2606 numberofcmps++;
2607 }
2608 else if (numberofcmps > 0)
2609 {
2610 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2611 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2612 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2613 numberofcmps = 0;
2614 }
2615 else
2616 {
2617 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2618 numberofcmps = 0;
2619 }
2620 }
2621 else if (*cc == XCL_RANGE)
2622 {
2623 cc ++;
2624 #ifdef SUPPORT_UTF8
2625 if (common->utf8)
2626 {
2627 GETCHARINC(c, cc);
2628 }
2629 else
2630 #endif
2631 c = *cc++;
2632 SET_CHAR_OFFSET(c);
2633 #ifdef SUPPORT_UTF8
2634 if (common->utf8)
2635 {
2636 GETCHARINC(c, cc);
2637 }
2638 else
2639 #endif
2640 c = *cc++;
2641 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2642 {
2643 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2644 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2645 numberofcmps++;
2646 }
2647 else if (numberofcmps > 0)
2648 {
2649 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2650 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2651 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2652 numberofcmps = 0;
2653 }
2654 else
2655 {
2656 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2657 numberofcmps = 0;
2658 }
2659 }
2660 #ifdef SUPPORT_UCP
2661 else
2662 {
2663 if (*cc == XCL_NOTPROP)
2664 invertcmp ^= 0x1;
2665 cc++;
2666 switch(*cc)
2667 {
2668 case PT_ANY:
2669 if (list != fallbacks)
2670 {
2671 if ((cc[-1] == XCL_NOTPROP && compares > 0) || (cc[-1] == XCL_PROP && compares == 0))
2672 continue;
2673 }
2674 else if (cc[-1] == XCL_NOTPROP)
2675 continue;
2676 jump = JUMP(SLJIT_JUMP);
2677 break;
2678
2679 case PT_LAMP:
2680 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
2681 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2682 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
2683 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2684 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
2685 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2686 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2687 break;
2688
2689 case PT_GC:
2690 c = _pcre_ucp_typerange[(int)cc[1] * 2];
2691 SET_TYPE_OFFSET(c);
2692 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, _pcre_ucp_typerange[(int)cc[1] * 2 + 1] - c);
2693 break;
2694
2695 case PT_PC:
2696 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
2697 break;
2698
2699 case PT_SC:
2700 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, scriptreg, 0, SLJIT_IMM, (int)cc[1]);
2701 break;
2702
2703 case PT_SPACE:
2704 case PT_PXSPACE:
2705 if (*cc == PT_SPACE)
2706 {
2707 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2708 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 11 - charoffset);
2709 }
2710 SET_CHAR_OFFSET(9);
2711 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 13 - 9);
2712 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2713 if (*cc == PT_SPACE)
2714 JUMPHERE(jump);
2715
2716 SET_TYPE_OFFSET(ucp_Zl);
2717 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
2718 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2719 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2720 break;
2721
2722 case PT_WORD:
2723 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE - charoffset);
2724 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2725 /* ... fall through */
2726
2727 case PT_ALNUM:
2728 SET_TYPE_OFFSET(ucp_Ll);
2729 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2730 COND_VALUE((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2731 SET_TYPE_OFFSET(ucp_Nd);
2732 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2733 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2734 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2735 break;
2736 }
2737 cc += 2;
2738 }
2739 #endif
2740
2741 if (jump != NULL)
2742 add_jump(compiler, compares > 0 ? list : fallbacks, jump);
2743 }
2744
2745 if (found != NULL)
2746 set_jumps(found, LABEL());
2747 }
2748
2749 #undef SET_TYPE_OFFSET
2750 #undef SET_CHAR_OFFSET
2751
2752 #endif
2753
2754 static uschar *compile_char1_hotpath(compiler_common *common, uschar type, uschar *cc, jump_list **fallbacks)
2755 {
2756 DEFINE_COMPILER;
2757 int length;
2758 unsigned int c, oc, bit;
2759 compare_context context;
2760 struct sljit_jump *jump[4];
2761 #ifdef SUPPORT_UTF8
2762 struct sljit_label *label;
2763 #ifdef SUPPORT_UCP
2764 uschar propdata[5];
2765 #endif
2766 #endif
2767
2768 switch(type)
2769 {
2770 case OP_SOD:
2771 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2772 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
2773 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2774 return cc;
2775
2776 case OP_SOM:
2777 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2778 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
2779 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2780 return cc;
2781
2782 case OP_NOT_WORD_BOUNDARY:
2783 case OP_WORD_BOUNDARY:
2784 add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
2785 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2786 return cc;
2787
2788 case OP_NOT_DIGIT:
2789 case OP_DIGIT:
2790 check_input_end(common, fallbacks);
2791 read_char8_type(common);
2792 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
2793 add_jump(compiler, fallbacks, JUMP(type == OP_DIGIT ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2794 return cc;
2795
2796 case OP_NOT_WHITESPACE:
2797 case OP_WHITESPACE:
2798 check_input_end(common, fallbacks);
2799 read_char8_type(common);
2800 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
2801 add_jump(compiler, fallbacks, JUMP(type == OP_WHITESPACE ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2802 return cc;
2803
2804 case OP_NOT_WORDCHAR:
2805 case OP_WORDCHAR:
2806 check_input_end(common, fallbacks);
2807 read_char8_type(common);
2808 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
2809 add_jump(compiler, fallbacks, JUMP(type == OP_WORDCHAR ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2810 return cc;
2811
2812 case OP_ANY:
2813 check_input_end(common, fallbacks);
2814 read_char(common);
2815 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2816 {
2817 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
2818 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2819 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2820 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
2821 JUMPHERE(jump[1]);
2822 JUMPHERE(jump[0]);
2823 }
2824 else
2825 check_newlinechar(common, common->nltype, fallbacks, TRUE);
2826 return cc;
2827
2828 case OP_ALLANY:
2829 check_input_end(common, fallbacks);
2830 #ifdef SUPPORT_UTF8
2831 if (common->utf8)
2832 {
2833 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2834 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes);
2835 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
2836 return cc;
2837 }
2838 #endif
2839 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2840 return cc;
2841
2842 #ifdef SUPPORT_UTF8
2843 #ifdef SUPPORT_UCP
2844 case OP_NOTPROP:
2845 case OP_PROP:
2846 propdata[0] = 0;
2847 propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP;
2848 propdata[2] = cc[0];
2849 propdata[3] = cc[1];
2850 propdata[4] = XCL_END;
2851 compile_xclass_hotpath(common, propdata, fallbacks);
2852 return cc + 2;
2853 #endif
2854 #endif
2855
2856 case OP_ANYNL:
2857 check_input_end(common, fallbacks);
2858 read_char(common);
2859 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2860 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2861 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2862 jump[2] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2863 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2864 jump[3] = JUMP(SLJIT_JUMP);
2865 JUMPHERE(jump[0]);
2866 check_newlinechar(common, common->bsr_nltype, fallbacks, FALSE);
2867 JUMPHERE(jump[1]);
2868 JUMPHERE(jump[2]);
2869 JUMPHERE(jump[3]);
2870 return cc;
2871
2872 case OP_NOT_HSPACE:
2873 case OP_HSPACE:
2874 check_input_end(common, fallbacks);
2875 read_char(common);
2876 add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
2877 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_HSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2878 return cc;
2879
2880 case OP_NOT_VSPACE:
2881 case OP_VSPACE:
2882 check_input_end(common, fallbacks);
2883 read_char(common);
2884 add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
2885 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_VSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2886 return cc;
2887
2888 #ifdef SUPPORT_UCP
2889 case OP_EXTUNI:
2890 check_input_end(common, fallbacks);
2891 read_char(common);
2892 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2893 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2894 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc));
2895
2896 label = LABEL();
2897 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2898 OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
2899 read_char(common);
2900 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2901 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2902 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc, label);
2903
2904 OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
2905 JUMPHERE(jump[0]);
2906 return cc;
2907 #endif
2908
2909 case OP_EODN:
2910 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2911 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2912 {
2913 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2914 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2915 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2916 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
2917 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2918 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2919 }
2920 else if (common->nltype == NLTYPE_FIXED)
2921 {
2922 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2923 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2924 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2925 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
2926 }
2927 else
2928 {
2929 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2930 jump[1] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2931 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2932 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
2933 jump[2] = JUMP(SLJIT_C_GREATER);
2934 add_jump(compiler, fallbacks, JUMP(SLJIT_C_LESS));
2935 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 1);
2936 jump[3] = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2937 add_jump(compiler, fallbacks, JUMP(SLJIT_JUMP));
2938
2939 JUMPHERE(jump[1]);
2940 if (common->nltype == NLTYPE_ANYCRLF)
2941 {
2942 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2943 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, STR_END, 0));
2944 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
2945 }
2946 else
2947 {
2948 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, STR_PTR, 0);
2949 read_char(common);
2950 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2951 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
2952 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
2953 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2954 }
2955 JUMPHERE(jump[2]);
2956 JUMPHERE(jump[3]);
2957 }
2958 JUMPHERE(jump[0]);
2959 return cc;
2960
2961 case OP_EOD:
2962 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2963 return cc;
2964
2965 case OP_CIRC:
2966 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2967 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2968 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0));
2969 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2970 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2971 return cc;
2972
2973 case OP_CIRCM:
2974 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2975 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2976 jump[1] = CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0);
2977 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2978 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2979 jump[0] = JUMP(SLJIT_JUMP);
2980 JUMPHERE(jump[1]);
2981
2982 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, end));
2983 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP2, 0, STR_PTR, 0));
2984
2985 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2986 {
2987 OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2988 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, TMP1, 0));
2989 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
2990 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
2991 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2992 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2993 }
2994 else
2995 {
2996 skip_char_back(common);
2997 read_char(common);
2998 check_newlinechar(common, common->nltype, fallbacks, FALSE);
2999 }
3000 JUMPHERE(jump[0]);
3001 return cc;
3002
3003 case OP_DOLL:
3004 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
3005 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
3006 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3007
3008 if (!common->endonly)
3009 compile_char1_hotpath(common, OP_EODN, cc, fallbacks);
3010 else
3011 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0));
3012 return cc;
3013
3014 case OP_DOLLM:
3015 jump[1] = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
3016 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
3017 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
3018 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3019 jump[0] = JUMP(SLJIT_JUMP);
3020 JUMPHERE(jump[1]);
3021
3022 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
3023 {
3024 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
3025 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, TMP2, 0, STR_END, 0));
3026 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3027 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
3028 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
3029 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
3030 }
3031 else
3032 {
3033 peek_char(common);
3034 check_newlinechar(common, common->nltype, fallbacks, FALSE);
3035 }
3036 JUMPHERE(jump[0]);
3037 return cc;
3038
3039 case OP_CHAR:
3040 case OP_CHARI:
3041 length = 1;
3042 #ifdef SUPPORT_UTF8
3043 if (common->utf8 && *cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3044 #endif
3045 if (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)
3046 {
3047 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3048 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3049
3050 context.length = length;
3051 context.sourcereg = -1;
3052 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3053 context.byteptr = 0;
3054 #endif
3055 return byte_sequence_compare(common, type == OP_CHARI, cc, &context, fallbacks);
3056 }
3057 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
3058 read_char(common);
3059 #ifdef SUPPORT_UTF8
3060 if (common->utf8)
3061 {
3062 GETCHAR(c, cc);
3063 }
3064 else
3065 #endif
3066 c = *cc;
3067 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c);
3068 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3069 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char_othercase(common, c));
3070 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3071 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3072 return cc + length;
3073
3074 case OP_NOT:
3075 case OP_NOTI:
3076 length = 1;
3077 #ifdef SUPPORT_UTF8
3078 if (common->utf8)
3079 {
3080 if (*cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3081
3082 check_input_end(common, fallbacks);
3083 GETCHAR(c, cc);
3084
3085 if (c <= 127)
3086 {
3087 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3088 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes);
3089 if (type == OP_NOT || !char_has_othercase(common, cc))
3090 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3091 else
3092 {
3093 /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */
3094 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x20);
3095 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c | 0x20));
3096 }
3097 /* Skip the variable-length character. */
3098 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
3099 return cc + length;
3100 }
3101 else
3102 read_char(common);
3103 }
3104 else
3105 #endif
3106 {
3107 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
3108 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3109 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
3110 c = *cc;
3111 }
3112
3113 if (type == OP_NOT || !char_has_othercase(common, cc))
3114 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3115 else
3116 {
3117 oc = char_othercase(common, c);
3118 bit = c ^ oc;
3119 if (ispowerof2(bit))
3120 {
3121 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
3122 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
3123 }
3124 else
3125 {
3126 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3127 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, oc));
3128 }
3129 }
3130 return cc + length;
3131
3132 case OP_CLASS:
3133 case OP_NCLASS:
3134 check_input_end(common, fallbacks);
3135 read_char(common);
3136 #ifdef SUPPORT_UTF8
3137 jump[0] = NULL;
3138 if (common->utf8)
3139 {
3140 jump[0] = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
3141 if (type == OP_CLASS)
3142 {
3143 add_jump(compiler, fallbacks, jump[0]);
3144 jump[0] = NULL;
3145 }
3146 }
3147 #endif
3148 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
3149 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
3150 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
3151 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
3152 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
3153 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3154 #ifdef SUPPORT_UTF8
3155 if (jump[0] != NULL)
3156 JUMPHERE(jump[0]);
3157 #endif
3158 return cc + 32;
3159
3160 #ifdef SUPPORT_UTF8
3161 case OP_XCLASS:
3162 compile_xclass_hotpath(common, cc + LINK_SIZE, fallbacks);
3163 return cc + GET(cc, 0) - 1;
3164 #endif
3165
3166 case OP_REVERSE:
3167 length = GET(cc, 0);
3168 SLJIT_ASSERT(length > 0);
3169 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
3170 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
3171 #ifdef SUPPORT_UTF8
3172 if (common->utf8)
3173 {
3174 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length);
3175 label = LABEL();
3176 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0));
3177 skip_char_back(common);
3178 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
3179 JUMPTO(SLJIT_C_NOT_ZERO, label);
3180 return cc + LINK_SIZE;
3181 }
3182 #endif
3183 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3184 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, TMP1, 0));
3185 return cc + LINK_SIZE;
3186 }
3187 SLJIT_ASSERT_STOP();
3188 return cc;
3189 }
3190
3191 static SLJIT_INLINE uschar *compile_charn_hotpath(compiler_common *common, uschar *cc, uschar *ccend, jump_list **fallbacks)
3192 {
3193 /* This function consumes at least one input character. */
3194 /* To decrease the number of length checks, we try to concatenate the fixed length character sequences. */
3195 DEFINE_COMPILER;
3196 uschar *ccbegin = cc;
3197 compare_context context;
3198 int size;
3199
3200 context.length = 0;
3201 do
3202 {
3203 if (cc >= ccend)
3204 break;
3205
3206 if (*cc == OP_CHAR)
3207 {
3208 size = 1;
3209 #ifdef SUPPORT_UTF8
3210 if (common->utf8 && cc[1] >= 0xc0)
3211 size += _pcre_utf8_table4[cc[1] & 0x3f];
3212 #endif
3213 }
3214 else if (*cc == OP_CHARI)
3215 {
3216 size = 1;
3217 #ifdef SUPPORT_UTF8
3218 if (common->utf8)
3219 {
3220 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3221 size = 0;
3222 else if (cc[1] >= 0xc0)
3223 size += _pcre_utf8_table4[cc[1] & 0x3f];
3224 }
3225 else
3226 #endif
3227 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3228 size = 0;
3229 }
3230 else
3231 size = 0;
3232
3233 cc += 1 + size;
3234 context.length += size;
3235 }
3236 while (size > 0 && context.length <= 128);
3237
3238 cc = ccbegin;
3239 if (context.length > 0)
3240 {
3241 /* We have a fixed-length byte sequence. */
3242 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length);
3243 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3244
3245 context.sourcereg = -1;
3246 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3247 context.byteptr = 0;
3248 #endif
3249 do cc = byte_sequence_compare(common, *cc == OP_CHARI, cc + 1, &context, fallbacks); while (context.length > 0);
3250 return cc;
3251 }
3252
3253 /* A non-fixed length character will be checked if length == 0. */
3254 return compile_char1_hotpath(common, *cc, cc + 1, fallbacks);
3255 }
3256
3257 static struct sljit_jump *compile_ref_checks(compiler_common *common, uschar *cc, jump_list **fallbacks)
3258 {
3259 DEFINE_COMPILER;
3260 int offset = GET2(cc, 1) << 1;
3261
3262 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3263 if (!common->jscript_compat)
3264 {
3265 if (fallbacks == NULL)
3266 {
3267 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
3268 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3269 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3270 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3271 return JUMP(SLJIT_C_NOT_ZERO);
3272 }
3273 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3274 }
3275 return CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3276 }
3277
3278 /* Forward definitions. */
3279 static void compile_hotpath(compiler_common *, uschar *, uschar *, fallback_common *);
3280 static void compile_fallbackpath(compiler_common *, struct fallback_common *);
3281
3282 #define PUSH_FALLBACK(size, ccstart, error) \
3283 do \
3284 { \
3285 fallback = sljit_alloc_memory(compiler, (size)); \
3286 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3287 return error; \
3288 memset(fallback, 0, size); \
3289 fallback->prev = parent->top; \
3290 fallback->cc = (ccstart); \
3291 parent->top = fallback; \
3292 } \
3293 while (0)
3294
3295 #define PUSH_FALLBACK_NOVALUE(size, ccstart) \
3296 do \
3297 { \
3298 fallback = sljit_alloc_memory(compiler, (size)); \
3299 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3300 return; \
3301 memset(fallback, 0, size); \
3302 fallback->prev = parent->top; \
3303 fallback->cc = (ccstart); \
3304 parent->top = fallback; \
3305 } \
3306 while (0)
3307
3308 #define FALLBACK_AS(type) ((type*)fallback)
3309
3310 static uschar *compile_ref_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks, BOOL withchecks, BOOL emptyfail)
3311 {
3312 DEFINE_COMPILER;
3313 int offset = GET2(cc, 1) << 1;
3314 struct sljit_jump *jump = NULL;
3315
3316 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3317 if (withchecks && !common->jscript_compat)
3318 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3319
3320 #ifdef SUPPORT_UTF8
3321 #ifdef SUPPORT_UCP
3322 if (common->utf8 && *cc == OP_REFI)
3323 {
3324 SLJIT_ASSERT(TMP1 == SLJIT_TEMPORARY_REG1 && STACK_TOP == SLJIT_TEMPORARY_REG2 && TMP2 == SLJIT_TEMPORARY_REG3);
3325 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3326 if (withchecks)
3327 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, TMP2, 0);
3328
3329 /* Needed to save important temporary registers. */
3330 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
3331 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, ARGUMENTS, 0);
3332 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), SLJIT_OFFSETOF(jit_arguments, ptr), STR_PTR, 0);
3333 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf8caselesscmp));
3334 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
3335 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
3336 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
3337 }
3338 else
3339 #endif
3340 #endif
3341 {
3342 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP1, 0);
3343 if (withchecks)
3344 jump = JUMP(SLJIT_C_ZERO);
3345 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
3346
3347 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3348 add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL));
3349 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3350 }
3351
3352 if (jump != NULL)
3353 {
3354 if (emptyfail)
3355 add_jump(compiler, fallbacks, jump);
3356 else
3357 JUMPHERE(jump);
3358 }
3359 return cc + 3;
3360 }
3361
3362 static SLJIT_INLINE uschar *compile_ref_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3363 {
3364 DEFINE_COMPILER;
3365 fallback_common *fallback;
3366 uschar type;
3367 struct sljit_label *label;
3368 struct sljit_jump *zerolength;
3369 struct sljit_jump *jump = NULL;
3370 uschar *ccbegin = cc;
3371 int min = 0, max = 0;
3372 BOOL minimize;
3373
3374 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
3375
3376 type = cc[3];
3377 minimize = (type & 0x1) != 0;
3378 switch(type)
3379 {
3380 case OP_CRSTAR:
3381 case OP_CRMINSTAR:
3382 min = 0;
3383 max = 0;
3384 cc += 4;
3385 break;
3386 case OP_CRPLUS:
3387 case OP_CRMINPLUS:
3388 min = 1;
3389 max = 0;
3390 cc += 4;
3391 break;
3392 case OP_CRQUERY:
3393 case OP_CRMINQUERY:
3394 min = 0;
3395 max = 1;
3396 cc += 4;
3397 break;
3398 case OP_CRRANGE:
3399 case OP_CRMINRANGE:
3400 min = GET2(cc, 3 + 1);
3401 max = GET2(cc, 3 + 3);
3402 cc += 8;
3403 break;
3404 default:
3405 SLJIT_ASSERT_STOP();
3406 break;
3407 }
3408
3409 if (!minimize)
3410 {
3411 if (min == 0)
3412 {
3413 allocate_stack(common, 2);
3414 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3415 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3416 /* Temporary release of STR_PTR. */
3417 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3418 zerolength = compile_ref_checks(common, ccbegin, NULL);
3419 /* Restore if not zero length. */
3420 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3421 }
3422 else
3423 {
3424 allocate_stack(common, 1);
3425 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3426 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3427 }
3428
3429 if (min > 1 || max > 1)
3430 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
3431
3432 label = LABEL();
3433 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, FALSE, FALSE);
3434
3435 if (min > 1 || max > 1)
3436 {
3437 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
3438 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3439 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
3440 if (min > 1)
3441 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, label);
3442 if (max > 1)
3443 {
3444 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
3445 allocate_stack(common, 1);
3446 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3447 JUMPTO(SLJIT_JUMP, label);
3448 JUMPHERE(jump);
3449 }
3450 }
3451
3452 if (max == 0)
3453 {
3454 /* Includes min > 1 case as well. */
3455 allocate_stack(common, 1);
3456 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3457 JUMPTO(SLJIT_JUMP, label);
3458 }
3459
3460 JUMPHERE(zerolength);
3461 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3462
3463 decrease_call_count(common);
3464 return cc;
3465 }
3466
3467 allocate_stack(common, 2);
3468 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3469 if (type != OP_CRMINSTAR)
3470 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3471
3472 if (min == 0)
3473 {
3474 zerolength = compile_ref_checks(common, ccbegin, NULL);
3475 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3476 jump = JUMP(SLJIT_JUMP);
3477 }
3478 else
3479 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3480
3481 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3482 if (max > 0)
3483 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max));
3484
3485 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, TRUE, TRUE);
3486 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3487
3488 if (min > 1)
3489 {
3490 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3491 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3492 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3493 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, FALLBACK_AS(iterator_fallback)->hotpath);
3494 }
3495 else if (max > 0)
3496 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
3497
3498 if (jump != NULL)
3499 JUMPHERE(jump);
3500 JUMPHERE(zerolength);
3501
3502 decrease_call_count(common);
3503 return cc;
3504 }
3505
3506 static SLJIT_INLINE uschar *compile_recurse_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3507 {
3508 DEFINE_COMPILER;
3509 fallback_common *fallback;
3510 recurse_entry *entry = common->entries;
3511 recurse_entry *prev = NULL;
3512 int start = GET(cc, 1);
3513
3514 PUSH_FALLBACK(sizeof(recurse_fallback), cc, NULL);
3515 while (entry != NULL)
3516 {
3517 if (entry->start == start)
3518 break;
3519 prev = entry;
3520 entry = entry->next;
3521 }
3522
3523 if (entry == NULL)
3524 {
3525 entry = sljit_alloc_memory(compiler, sizeof(recurse_entry));
3526 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3527 return NULL;
3528 entry->next = NULL;
3529 entry->entry = NULL;
3530 entry->calls = NULL;
3531 entry->start = start;
3532
3533 if (prev != NULL)
3534 prev->next = entry;
3535 else
3536 common->entries = entry;
3537 }
3538
3539 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
3540 allocate_stack(common, 1);
3541 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
3542
3543 if (entry->entry == NULL)
3544 add_jump(compiler, &entry->calls, JUMP(SLJIT_FAST_CALL));
3545 else
3546 JUMPTO(SLJIT_FAST_CALL, entry->entry);
3547 /* Leave if the match is failed. */
3548 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0));
3549 return cc + 1 + LINK_SIZE;
3550 }
3551
3552 static uschar *compile_assert_hotpath(compiler_common *common, uschar *cc, assert_fallback *fallback, BOOL conditional)
3553 {
3554 DEFINE_COMPILER;
3555 int framesize;
3556 int localptr;
3557 fallback_common altfallback;
3558 uschar *ccbegin;
3559 uschar opcode;
3560 uschar bra = OP_BRA;
3561 jump_list *tmp = NULL;
3562 jump_list **target = (conditional) ? &fallback->condfailed : &fallback->common.topfallbacks;
3563 jump_list **found;
3564 /* Saving previous accept variables. */
3565 struct sljit_label *save_acceptlabel = common->acceptlabel;
3566 struct sljit_jump *jump;
3567 struct sljit_jump *brajump = NULL;
3568 jump_list *save_accept = common->accept;
3569
3570 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3571 {
3572 SLJIT_ASSERT(!conditional);
3573 bra = *cc;
3574 cc++;
3575 }
3576 localptr = PRIV(cc);
3577 SLJIT_ASSERT(localptr != 0);
3578 framesize = get_framesize(common, cc, FALSE);
3579 fallback->framesize = framesize;
3580 fallback->localptr = localptr;
3581 opcode = *cc;
3582 SLJIT_ASSERT(opcode >= OP_ASSERT && opcode <= OP_ASSERTBACK_NOT);
3583 found = (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) ? &tmp : target;
3584 ccbegin = cc;
3585 cc += GET(cc, 1);
3586
3587 if (bra == OP_BRAMINZERO)
3588 {
3589 /* This is a braminzero fallback path. */
3590 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3591 free_stack(common, 1);
3592 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
3593 }
3594
3595 if (framesize < 0)
3596 {
3597 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
3598 allocate_stack(common, 1);
3599 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3600 }
3601 else
3602 {
3603 allocate_stack(common, framesize + 2);
3604 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3605 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(framesize + 1));
3606 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
3607 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3608 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3609 init_frame(common, ccbegin, framesize + 1, 2, FALSE);
3610 }
3611
3612 memset(&altfallback, 0, sizeof(fallback_common));
3613 while (1)
3614 {
3615 common->acceptlabel = NULL;
3616 common->accept = NULL;
3617 altfallback.top = NULL;
3618 altfallback.topfallbacks = NULL;
3619
3620 if (*ccbegin == OP_ALT)
3621 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3622
3623 altfallback.cc = ccbegin;
3624 compile_hotpath(common, ccbegin + 1 + LINK_SIZE, cc, &altfallback);
3625 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3626 {
3627 common->acceptlabel = save_acceptlabel;
3628 common->accept = save_accept;
3629 return NULL;
3630 }
3631 common->acceptlabel = LABEL();
3632 if (common->accept != NULL)
3633 set_jumps(common->accept, common->acceptlabel);
3634
3635 if (framesize < 0)
3636 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3637
3638 if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT)
3639 {
3640 /* We know that STR_PTR was stored on the top of the stack. */
3641 if (conditional)
3642 {
3643 if (framesize < 0)
3644 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3645 else
3646 {
3647 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3648 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), (framesize + 1) * sizeof(sljit_w));
3649 }
3650 }
3651 else if (bra == OP_BRAZERO)
3652 {
3653 if (framesize < 0)
3654 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3655 else
3656 {
3657 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3658 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3659 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3660 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_w));
3661 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3662 }
3663 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3664 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3665 }
3666 else if (bra == OP_BRAMINZERO)
3667 {
3668 if (framesize >= 0)
3669 {
3670 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3671 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3672 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3673 }
3674 }
3675 }
3676 add_jump(compiler, found, JUMP(SLJIT_JUMP));
3677
3678 compile_fallbackpath(common, altfallback.top);
3679 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3680 {
3681 common->acceptlabel = save_acceptlabel;
3682 common->accept = save_accept;
3683 return NULL;
3684 }
3685 set_jumps(altfallback.topfallbacks, LABEL());
3686
3687 if (*cc != OP_ALT)
3688 break;
3689
3690 ccbegin = cc;
3691 cc += GET(cc, 1);
3692 }
3693 /* None of them matched. */
3694
3695 if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
3696 {
3697 /* Assert is failed. */
3698 if (conditional || bra == OP_BRAZERO)
3699 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3700 if (framesize < 0)
3701 {
3702 /* The topmost item should be 0. */
3703 if (bra == OP_BRAZERO)
3704 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3705 else
3706 free_stack(common, 1);
3707 }
3708 else
3709 {
3710 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3711 if (framesize > 0)
3712 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(2));
3713 /* The topmost item should be 0. */
3714 if (bra == OP_BRAZERO)
3715 {
3716 free_stack(common, framesize + 1);
3717 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3718 }
3719 else
3720 free_stack(common, framesize + 2);
3721 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3722 if (framesize > 0)
3723 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP2, 0);
3724 }
3725 jump = JUMP(SLJIT_JUMP);
3726 if (bra != OP_BRAZERO)
3727 add_jump(compiler, target, jump);
3728
3729 /* Assert is successful. */
3730 set_jumps(tmp, LABEL());
3731 if (framesize < 0)
3732 {
3733 /* We know that STR_PTR was stored on the top of the stack. */
3734 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3735 /* Keep the STR_PTR on the top of the stack. */
3736 if (bra == OP_BRAZERO)
3737 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3738 else if (bra == OP_BRAMINZERO)
3739 {
3740 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3741 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3742 }
3743 }
3744 else
3745 {
3746 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3747 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), (framesize + 1) * sizeof(sljit_w));
3748 if (bra == OP_BRAZERO)
3749 {
3750 allocate_stack(common, 1);
3751 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3752 }
3753 else if (bra == OP_BRAMINZERO)
3754 {
3755 allocate_stack(common, 1);
3756 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3757 }
3758 }
3759
3760 if (bra == OP_BRAZERO)
3761 {
3762 fallback->hotpath = LABEL();
3763 sljit_set_label(jump, fallback->hotpath);
3764 }
3765 else if (bra == OP_BRAMINZERO)
3766 {
3767 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3768 JUMPHERE(brajump);
3769 if (framesize >= 0)
3770 {
3771 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3772 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3773 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3774 }
3775 set_jumps(fallback->common.topfallbacks, LABEL());
3776 }
3777 }
3778 else
3779 {
3780 /* AssertNot is successful. */
3781 if (framesize < 0)
3782 {
3783 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3784 if (bra != OP_BRA)
3785 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3786 else
3787 free_stack(common, 1);
3788 }
3789 else
3790 {
3791 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3792 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3793 if (framesize > 0)
3794 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(2));
3795 /* The topmost item should be 0. */
3796 if (bra != OP_BRA)
3797 {
3798 free_stack(common, framesize + 1);
3799 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3800 }
3801 else
3802 free_stack(common, framesize + 2);
3803 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3804 if (framesize > 0)
3805 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP2, 0);
3806 }
3807
3808 if (bra == OP_BRAZERO)
3809 fallback->hotpath = LABEL();
3810 else if (bra == OP_BRAMINZERO)
3811 {
3812 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3813 JUMPHERE(brajump);
3814 }
3815
3816 if (bra != OP_BRA)
3817 {
3818 SLJIT_ASSERT(found == &fallback->common.topfallbacks);
3819 set_jumps(fallback->common.topfallbacks, LABEL());
3820 fallback->common.topfallbacks = NULL;
3821 }
3822 }
3823
3824 common->acceptlabel = save_acceptlabel;
3825 common->accept = save_accept;
3826 return cc + 1 + LINK_SIZE;
3827 }
3828
3829 /*
3830 Handling bracketed expressions is probably the most complex part.
3831
3832 Stack layout naming characters:
3833 S - Push the current STR_PTR
3834 0 - Push a 0 (NULL)
3835 A - Push the current STR_PTR. Needed for restoring the STR_PTR
3836 before the next alternative. Not pushed if there are no alternatives.
3837 M - Any values pushed by the current alternative. Can be empty, or anything.
3838 C - Push the previous OVECTOR(i), OVECTOR(i+1) and OVECTOR_PRIV(i) to the stack.
3839 L - Push the previous local (pointed by localptr) to the stack
3840 () - opional values stored on the stack
3841 ()* - optonal, can be stored multiple times
3842
3843 The following list shows the regular expression templates, their PCRE byte codes
3844 and stack layout supported by pcre-sljit.
3845
3846 (?:) OP_BRA | OP_KET A M
3847 () OP_CBRA | OP_KET C M
3848 (?:)+ OP_BRA | OP_KETRMAX 0 A M S ( A M S )*
3849 OP_SBRA | OP_KETRMAX 0 L M S ( L M S )*
3850 (?:)+? OP_BRA | OP_KETRMIN 0 A M S ( A M S )*
3851 OP_SBRA | OP_KETRMIN 0 L M S ( L M S )*
3852 ()+ OP_CBRA | OP_KETRMAX 0 C M S ( C M S )*
3853 OP_SCBRA | OP_KETRMAX 0 C M S ( C M S )*
3854 ()+? OP_CBRA | OP_KETRMIN 0 C M S ( C M S )*
3855 OP_SCBRA | OP_KETRMIN 0 C M S ( C M S )*
3856 (?:)? OP_BRAZERO | OP_BRA | OP_KET S ( A M 0 )
3857 (?:)?? OP_BRAMINZERO | OP_BRA | OP_KET S ( A M 0 )
3858 ()? OP_BRAZERO | OP_CBRA | OP_KET S ( C M 0 )
3859 ()?? OP_BRAMINZERO | OP_CBRA | OP_KET S ( C M 0 )
3860 (?:)* OP_BRAZERO | OP_BRA | OP_KETRMAX S 0 ( A M S )*
3861 OP_BRAZERO | OP_SBRA | OP_KETRMAX S 0 ( L M S )*
3862 (?:)*? OP_BRAMINZERO | OP_BRA | OP_KETRMIN S 0 ( A M S )*
3863 OP_BRAMINZERO | OP_SBRA | OP_KETRMIN S 0 ( L M S )*
3864 ()* OP_BRAZERO | OP_CBRA | OP_KETRMAX S 0 ( C M S )*
3865 OP_BRAZERO | OP_SCBRA | OP_KETRMAX S 0 ( C M S )*
3866 ()*? OP_BRAMINZERO | OP_CBRA | OP_KETRMIN S 0 ( C M S )*
3867 OP_BRAMINZERO | OP_SCBRA | OP_KETRMIN S 0 ( C M S )*
3868
3869
3870 Stack layout naming characters:
3871 A - Push the alternative index (starting from 0) on the stack.
3872 Not pushed if there is no alternatives.
3873 M - Any values pushed by the current alternative. Can be empty, or anything.
3874
3875 The next list shows the possible content of a bracket:
3876 (|) OP_*BRA | OP_ALT ... M A
3877 (?()|) OP_*COND | OP_ALT M A
3878 (?>|) OP_ONCE | OP_ALT ... [stack trace] M A
3879 Or nothing, if trace is unnecessary
3880 */
3881
3882 static uschar *compile_bracket_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3883 {
3884 DEFINE_COMPILER;
3885 fallback_common *fallback;
3886 uschar opcode;
3887 int localptr = 0;
3888 int offset = 0;
3889 int stacksize;
3890 uschar *ccbegin;
3891 uschar *hotpath;
3892 uschar bra = OP_BRA;
3893 uschar ket;
3894 assert_fallback *assert;
3895 BOOL has_alternatives;
3896 struct sljit_jump *jump;
3897 struct sljit_jump *skip;
3898 struct sljit_label *rmaxlabel = NULL;
3899 struct sljit_jump *braminzerojump = NULL;
3900
3901 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
3902
3903 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3904 {
3905 bra = *cc;
3906 cc++;
3907 opcode = *cc;
3908 }
3909
3910 opcode = *cc;
3911 ccbegin = cc;
3912 if ((opcode == OP_COND || opcode == OP_SCOND) && cc[1 + LINK_SIZE] == OP_DEF)
3913 {
3914 /* Drop this bracket_fallback. */
3915 parent->top = fallback->prev;
3916 return bracketend(cc);
3917 }
3918
3919 ket = *(bracketend(cc) - 1 - LINK_SIZE);
3920 SLJIT_ASSERT(ket == OP_KET || ket == OP_KETRMAX || ket == OP_KETRMIN);
3921 SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO && ket == OP_KETRMAX)));
3922 cc += GET(cc, 1);
3923 has_alternatives = *cc == OP_ALT || opcode == OP_COND || opcode == OP_SCOND;
3924 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
3925 opcode = OP_SCOND;
3926
3927 if (opcode == OP_CBRA || opcode == OP_SCBRA)
3928 {
3929 /* Capturing brackets has a pre-allocated space. */
3930 offset = GET2(ccbegin, 1 + LINK_SIZE);
3931 localptr = OVECTOR_PRIV(offset);
3932 offset <<= 1;
3933 FALLBACK_AS(bracket_fallback)->localptr = localptr;
3934 }
3935 else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND)
3936 {
3937 /* Other brackets simply allocate the next entry. */
3938 localptr = PRIV(ccbegin);
3939 SLJIT_ASSERT(localptr != 0);
3940 FALLBACK_AS(bracket_fallback)->localptr = localptr;
3941 if (opcode == OP_ONCE)
3942 FALLBACK_AS(bracket_fallback)->u.framesize = get_framesize(common, ccbegin, FALSE);
3943 }
3944
3945 /* Instructions before the first alternative. */
3946 stacksize = 0;
3947 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
3948 stacksize++;
3949 if (bra == OP_BRAZERO)
3950 stacksize++;
3951
3952 if (stacksize > 0)
3953 allocate_stack(common, stacksize);
3954
3955 stacksize = 0;
3956 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
3957 {
3958 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
3959 stacksize++;
3960 }
3961
3962 if (bra == OP_BRAZERO)
3963 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
3964
3965 if (bra == OP_BRAMINZERO)
3966 {
3967 /* This is a fallback path! (Since the hot-path of OP_BRAMINZERO matches to the empty string) */
3968 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3969 if (ket != OP_KETRMIN)
3970 {
3971 free_stack(common, 1);
3972 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
3973 }
3974 else
3975 {
3976 if (opcode == OP_ONCE || opcode >= OP_SBRA)
3977 {
3978 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
3979 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3980 /* Nothing stored during the first run. */
3981 skip = JUMP(SLJIT_JUMP);
3982 JUMPHERE(jump);
3983 /* Checking zero-length iteration. */
3984 if (opcode != OP_ONCE || FALLBACK_AS(bracket_fallback)->u.framesize < 0)
3985 {
3986 /* When we come from outside, localptr contains the previous STR_PTR. */
3987 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3988 }
3989 else
3990 {
3991 /* Except when the whole stack frame must be saved. */
3992 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3993 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (FALLBACK_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w));
3994 }
3995 JUMPHERE(skip);
3996 }
3997 else
3998 {
3999 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4000 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4001 JUMPHERE(jump);
4002 }
4003 }
4004 }
4005
4006 if (ket == OP_KETRMIN)
4007 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4008
4009 if (ket == OP_KETRMAX)
4010 {
4011 rmaxlabel = LABEL();
4012 if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA)
4013 FALLBACK_AS(bracket_fallback)->althotpath = rmaxlabel;
4014 }
4015
4016 /* Handling capturing brackets and alternatives. */
4017 if (opcode == OP_ONCE)
4018 {
4019 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4020 {
4021 /* Neither capturing brackets nor recursions are not found in the block. */
4022 if (ket == OP_KETRMIN)
4023 {
4024 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4025 allocate_stack(common, 2);
4026 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4027 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4028 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
4029 }
4030 else if (ket == OP_KETRMAX || has_alternatives)
4031 {
4032 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4033 allocate_stack(common, 1);
4034 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4035 }
4036 else
4037 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4038 }
4039 else
4040 {
4041 if (ket == OP_KETRMIN || ket == OP_KETRMAX || has_alternatives)
4042 {
4043 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 2);
4044 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4045 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize + 1));
4046 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4047 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4048 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4049 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize + 1, 2, FALSE);
4050 }
4051 else
4052 {
4053 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 1);
4054 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4055 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize));
4056 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4057 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4058 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize, 1, FALSE);
4059 }
4060 }
4061 }
4062 else if (opcode == OP_CBRA || opcode == OP_SCBRA)
4063 {
4064 /* Saving the previous values. */
4065 allocate_stack(common, 3);
4066 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4067 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4068 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4069 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4070 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4071 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4072 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0);
4073 }
4074 else if (opcode == OP_SBRA || opcode == OP_SCOND)
4075 {
4076 /* Saving the previous value. */
4077 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4078 allocate_stack(common, 1);
4079 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4080 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
4081 }
4082 else if (has_alternatives)
4083 {
4084 /* Pushing the starting string pointer. */
4085 allocate_stack(common, 1);
4086 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4087 }
4088
4089 /* Generating code for the first alternative. */
4090 hotpath = ccbegin + 1 + LINK_SIZE;
4091 if (offset != 0)
4092 hotpath += 2;
4093 if (opcode == OP_COND || opcode == OP_SCOND)
4094 {
4095 if (*hotpath == OP_CREF)
4096 {
4097 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed),
4098 CMP(SLJIT_C_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR((GET2(hotpath, 1) << 1)), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
4099 hotpath += 3;
4100 }
4101 else
4102 {
4103 SLJIT_ASSERT(*hotpath >= OP_ASSERT && *hotpath <= OP_ASSERTBACK_NOT);
4104 /* Similar code as PUSH_FALLBACK macro. */
4105 assert = sljit_alloc_memory(compiler, sizeof(assert_fallback));
4106 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4107 return NULL;
4108 memset(assert, 0, sizeof(assert_fallback));
4109 assert->common.cc = hotpath;
4110 FALLBACK_AS(bracket_fallback)->u.assert = assert;
4111 hotpath = compile_assert_hotpath(common, hotpath, assert, TRUE);
4112 }
4113 }
4114
4115 compile_hotpath(common, hotpath, cc, fallback);
4116 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4117 return NULL;
4118
4119 if (opcode == OP_ONCE)
4120 {
4121 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4122 {
4123 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4124 /* TMP2 which is set here used by OP_KETRMAX below. */
4125 if (ket == OP_KETRMAX)
4126 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
4127 else if (ket == OP_KETRMIN)
4128 {
4129 /* Move the STR_PTR to the localptr. */
4130 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
4131 }
4132 }
4133 else if (ket == OP_KETRMAX)
4134 {
4135 /* TMP2 which is set here used by OP_KETRMAX below. */
4136 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4137 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), (FALLBACK_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w));
4138 }
4139 }
4140
4141 stacksize = 0;
4142 if (ket != OP_KET || bra != OP_BRA)
4143 stacksize++;
4144 if (has_alternatives && opcode != OP_ONCE)
4145 stacksize++;
4146
4147 if (stacksize > 0)
4148 allocate_stack(common, stacksize);
4149
4150 stacksize = 0;
4151 if (ket != OP_KET)
4152 {
4153 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
4154 stacksize++;
4155 }
4156 else if (bra != OP_BRA)
4157 {
4158 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4159 stacksize++;
4160 }
4161
4162 if (has_alternatives)
4163 {
4164 if (opcode != OP_ONCE)
4165 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4166 if (ket != OP_KETRMAX)
4167 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4168 }
4169
4170 /* Must be after the hotpath label. */
4171 if (offset != 0)
4172 {
4173 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4174 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4175 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
4176 }
4177
4178 if (ket == OP_KETRMAX)
4179 {
4180 if (opcode == OP_ONCE || opcode >= OP_SBRA)
4181 {
4182 if (has_alternatives)
4183 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4184 /* Checking zero-length iteration. */
4185 if (opcode != OP_ONCE)
4186 CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0, rmaxlabel);
4187 else
4188 /* TMP2 must contain the starting STR_PTR. */
4189 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmaxlabel);
4190 }
4191 else
4192 JUMPTO(SLJIT_JUMP, rmaxlabel);
4193 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4194 }
4195
4196 if (bra == OP_BRAZERO)
4197 FALLBACK_AS(bracket_fallback)->zerohotpath = LABEL();
4198
4199 if (bra == OP_BRAMINZERO)
4200 {
4201 /* This is a fallback path! (From the viewpoint of OP_BRAMINZERO) */
4202 JUMPTO(SLJIT_JUMP, ((braminzero_fallback*)parent)->hotpath);
4203 if (braminzerojump != NULL)
4204 {
4205 JUMPHERE(braminzerojump);
4206 /* We need to release the end pointer to perform the
4207 fallback for the zero-length iteration. When
4208 framesize is < 0, OP_ONCE will do the release itself. */
4209 if (opcode == OP_ONCE && FALLBACK_AS(bracket_fallback)->u.framesize >= 0)
4210 {
4211 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4212 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
4213 }
4214 else if (ket == OP_KETRMIN && opcode != OP_ONCE)
4215 free_stack(common, 1);
4216 }
4217 /* Continue to the normal fallback. */
4218 }
4219
4220 if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO)
4221 decrease_call_count(common);
4222
4223 /* Skip the other alternatives. */
4224 while (*cc == OP_ALT)
4225 cc += GET(cc, 1);
4226 cc += 1 + LINK_SIZE;
4227 return cc;
4228 }
4229
4230 static uschar *compile_bracketpos_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4231 {
4232 DEFINE_COMPILER;
4233 fallback_common *fallback;
4234 uschar opcode;
4235 int localptr;
4236 int cbraprivptr = 0;
4237 int framesize;
4238 int stacksize;
4239 int offset = 0;
4240 BOOL zero = FALSE;
4241 uschar *ccbegin = NULL;
4242 int stack;
4243 struct sljit_label *loop = NULL;
4244 struct jump_list *emptymatch = NULL;
4245
4246 PUSH_FALLBACK(sizeof(bracketpos_fallback), cc, NULL);
4247 if (*cc == OP_BRAPOSZERO)
4248 {
4249 zero = TRUE;
4250 cc++;
4251 }
4252
4253 opcode = *cc;
4254 localptr = PRIV(cc);
4255 SLJIT_ASSERT(localptr != 0);
4256 FALLBACK_AS(bracketpos_fallback)->localptr = localptr;
4257 switch(opcode)
4258 {
4259 case OP_BRAPOS:
4260 case OP_SBRAPOS:
4261 ccbegin = cc + 1 + LINK_SIZE;
4262 break;
4263
4264 case OP_CBRAPOS:
4265 case OP_SCBRAPOS:
4266 offset = GET2(cc, 1 + LINK_SIZE);
4267 cbraprivptr = OVECTOR_PRIV(offset);
4268 offset <<= 1;
4269 ccbegin = cc + 1 + LINK_SIZE + 2;
4270 break;
4271
4272 default:
4273 SLJIT_ASSERT_STOP();
4274 break;
4275 }
4276
4277 framesize = get_framesize(common, cc, FALSE);
4278 FALLBACK_AS(bracketpos_fallback)->framesize = framesize;
4279 if (framesize < 0)
4280 {
4281 stacksize = (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS) ? 2 : 1;
4282 if (!zero)
4283 stacksize++;
4284 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4285 allocate_stack(common, stacksize);
4286 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4287
4288 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4289 {
4290 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4291 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4292 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4293 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4294 }
4295 else
4296 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4297
4298 if (!zero)
4299 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 1);
4300 }
4301 else
4302 {
4303 stacksize = framesize + 1;
4304 if (!zero)
4305 stacksize++;
4306 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4307 stacksize++;
4308 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4309 allocate_stack(common, stacksize);
4310
4311 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4312 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
4313 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4314 stack = 0;
4315 if (!zero)
4316 {
4317 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 1);
4318 stack++;
4319 }
4320 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4321 {
4322 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), STR_PTR, 0);
4323 stack++;
4324 }
4325 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP1, 0);
4326 init_frame(common, cc, stacksize - 1, stacksize - framesize, FALSE);
4327 }
4328
4329 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4330 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4331
4332 loop = LABEL();
4333 while (*cc != OP_KETRPOS)
4334 {
4335 fallback->top = NULL;
4336 fallback->topfallbacks = NULL;
4337 cc += GET(cc, 1);
4338
4339 compile_hotpath(common, ccbegin, cc, fallback);
4340 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4341 return NULL;
4342
4343 if (framesize < 0)
4344 {
4345 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4346
4347 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4348 {
4349 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4350 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4351 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4352 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4353 }
4354 else
4355 {
4356 if (opcode == OP_SBRAPOS)
4357 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4358 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4359 }
4360
4361 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4362 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4363
4364 if (!zero)
4365 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4366 }
4367 else
4368 {
4369 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4370 {
4371 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4372 if (!zero)
4373 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4374 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4375 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4376 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4377 }
4378 else
4379 {
4380 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4381 if (opcode == OP_SBRAPOS)
4382 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4383 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w), STR_PTR, 0);
4384 }
4385
4386 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4387 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4388
4389 /* TMP2 must be set above. */
4390 if (!zero)
4391 {
4392 if (framesize < 0)
4393 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4394 else
4395 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_w), SLJIT_IMM, 0);
4396 }
4397 }
4398 JUMPTO(SLJIT_JUMP, loop);
4399 flush_stubs(common);
4400
4401 compile_fallbackpath(common, fallback->top);
4402 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4403 return NULL;
4404 set_jumps(fallback->topfallbacks, LABEL());
4405
4406 if (framesize < 0)
4407 {
4408 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4409 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4410 else
4411 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4412 }
4413 else
4414 {
4415 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4416 {
4417 if (*cc == OP_KETRPOS)
4418 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4419 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4420 }
4421 else
4422 {
4423 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4424 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4425 }
4426 }
4427
4428 if (*cc == OP_KETRPOS)
4429 break;
4430 ccbegin = cc + 1 + LINK_SIZE;
4431 }
4432
4433 fallback->topfallbacks = NULL;
4434 if (!zero)
4435 {
4436 if (framesize < 0)
4437 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
4438 else /* TMP2 is set to [localptr] above. */
4439 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_w), SLJIT_IMM, 0));
4440 }
4441
4442 /* None of them matched. */
4443 set_jumps(emptymatch, LABEL());
4444 decrease_call_count(common);
4445 return cc + 1 + LINK_SIZE;
4446 }
4447
4448 static SLJIT_INLINE uschar *get_iterator_parameters(compiler_common *common, uschar *cc, uschar *opcode, uschar *type, int *arg1, int *arg2, uschar **end)
4449 {
4450 int class_len;
4451
4452 *opcode = *cc;
4453 if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO)
4454 {
4455 cc++;
4456 *type = OP_CHAR;
4457 }
4458 else if (*opcode >= OP_STARI && *opcode <= OP_POSUPTOI)
4459 {
4460 cc++;
4461 *type = OP_CHARI;
4462 *opcode -= OP_STARI - OP_STAR;
4463 }
4464 else if (*opcode >= OP_NOTSTAR && *opcode <= OP_NOTPOSUPTO)
4465 {
4466 cc++;
4467 *type = OP_NOT;
4468 *opcode -= OP_NOTSTAR - OP_STAR;
4469 }
4470 else if (*opcode >= OP_NOTSTARI && *opcode <= OP_NOTPOSUPTOI)
4471 {
4472 cc++;
4473 *type = OP_NOTI;
4474 *opcode -= OP_NOTSTARI - OP_STAR;
4475 }
4476 else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO)
4477 {
4478 cc++;
4479 *opcode -= OP_TYPESTAR - OP_STAR;
4480 *type = 0;
4481 }
4482 else
4483 {
4484 SLJIT_ASSERT(*opcode >= OP_CLASS || *opcode <= OP_XCLASS);
4485 *type = *opcode;
4486 cc++;
4487 class_len = (*type < OP_XCLASS) ? 33 : GET(cc, 0);
4488 *opcode = cc[class_len - 1];
4489 if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY)
4490 {
4491 *opcode -= OP_CRSTAR - OP_STAR;
4492 if (end != NULL)
4493 *end = cc + class_len;
4494 }
4495 else
4496 {
4497 SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE);
4498 *arg1 = GET2(cc, (class_len + 2));
4499 *arg2 = GET2(cc, class_len);
4500
4501 if (*arg2 == 0)
4502 {
4503 SLJIT_ASSERT(*arg1 != 0);
4504 *opcode = (*opcode == OP_CRRANGE) ? OP_UPTO : OP_MINUPTO;
4505 }
4506 if (*arg1 == *arg2)
4507 *opcode = OP_EXACT;
4508
4509 if (end != NULL)
4510 *end = cc + class_len + 4;
4511 }
4512 return cc;
4513 }
4514
4515 if (*opcode == OP_UPTO || *opcode == OP_MINUPTO || *opcode == OP_EXACT || *opcode == OP_POSUPTO)
4516 {
4517 *arg1 = GET2(cc, 0);
4518 cc += 2;
4519 }
4520
4521 if (*type == 0)
4522 {
4523 *type = *cc;
4524 if (end != NULL)
4525 *end = next_opcode(common, cc);
4526 cc++;
4527 return cc;
4528 }
4529
4530 if (end != NULL)
4531 {
4532 *end = cc + 1;
4533 #ifdef SUPPORT_UTF8
4534 if (common->utf8 && *cc >= 0xc0) *end += _pcre_utf8_table4[*cc & 0x3f];
4535 #endif
4536 }
4537 return cc;
4538 }
4539
4540 static uschar *compile_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4541 {
4542 DEFINE_COMPILER;
4543 fallback_common *fallback;
4544 uschar opcode;
4545 uschar type;
4546 int arg1 = -1, arg2 = -1;
4547 uschar* end;
4548 jump_list *nomatch = NULL;
4549 struct sljit_jump *jump = NULL;
4550 struct sljit_label *label;
4551
4552 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
4553
4554 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, &end);
4555
4556 switch(opcode)
4557 {
4558 case OP_STAR:
4559 case OP_PLUS:
4560 case OP_UPTO:
4561 case OP_CRRANGE:
4562 if (type == OP_ANYNL || type == OP_EXTUNI)
4563 {
4564 if (opcode == OP_STAR || opcode == OP_UPTO)
4565 {
4566 allocate_stack(common, 2);
4567 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4568 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
4569 }
4570 else
4571 {
4572 allocate_stack(common, 1);
4573 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4574 }
4575 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4576 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
4577
4578 label = LABEL();
4579 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4580 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4581 {
4582 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4583 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4584 if (opcode == OP_CRRANGE && arg2 > 0)
4585 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2, label);
4586 if (opcode == OP_UPTO || (opcode == OP_CRRANGE && arg1 > 0))
4587 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, arg1);
4588 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4589 }
4590
4591 allocate_stack(common, 1);
4592 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4593 JUMPTO(SLJIT_JUMP, label);
4594 if (jump != NULL)
4595 JUMPHERE(jump);
4596 }
4597 else
4598 {
4599 allocate_stack(common, 2);
4600 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4601 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4602 label = LABEL();
4603 compile_char1_hotpath(common, type, cc, &nomatch);
4604 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4605 if (opcode <= OP_PLUS || (opcode == OP_CRRANGE && arg1 == 0))
4606 {
4607 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4608 JUMPTO(SLJIT_JUMP, label);
4609 }
4610 else
4611 {
4612 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4613 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4614 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4615 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4616 }
4617 set_jumps(nomatch, LABEL());
4618 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
4619 add_jump(compiler, &fallback->topfallbacks,
4620 CMP(SLJIT_C_LESS, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, opcode == OP_PLUS ? 2 : arg2 + 1));
4621 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4622 }
4623 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4624 break;
4625
4626 case OP_MINSTAR:
4627 case OP_MINPLUS:
4628 allocate_stack(common, 1);
4629 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4630 if (opcode == OP_MINPLUS)
4631 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4632 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4633 break;
4634
4635 case OP_MINUPTO:
4636 case OP_CRMINRANGE:
4637 allocate_stack(common, 2);
4638 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4639 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4640 if (opcode == OP_CRMINRANGE)
4641 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4642 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4643 break;
4644
4645 case OP_QUERY:
4646 case OP_MINQUERY:
4647 allocate_stack(common, 1);
4648 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4649 if (opcode == OP_QUERY)
4650 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4651 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4652 break;
4653
4654 case OP_EXACT:
4655 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4656 label = LABEL();
4657 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4658 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4659 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4660 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4661 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4662 break;
4663
4664 case OP_POSSTAR:
4665 case OP_POSPLUS:
4666 case OP_POSUPTO:
4667 if (opcode != OP_POSSTAR)
4668 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4669 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4670 label = LABEL();
4671 compile_char1_hotpath(common, type, cc, &nomatch);
4672 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4673 if (opcode != OP_POSUPTO)
4674 {
4675 if (opcode == OP_POSPLUS)
4676 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2);
4677 JUMPTO(SLJIT_JUMP, label);
4678 }
4679 else
4680 {
4681 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4682 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4683 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4684 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4685 }
4686 set_jumps(nomatch, LABEL());
4687 if (opcode == OP_POSPLUS)
4688 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_LESS, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2));
4689 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4690 break;
4691
4692 case OP_POSQUERY:
4693 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4694 compile_char1_hotpath(common, type, cc, &nomatch);
4695 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4696 set_jumps(nomatch, LABEL());
4697 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4698 break;
4699
4700 default:
4701 SLJIT_ASSERT_STOP();
4702 break;
4703 }
4704
4705 decrease_call_count(common);
4706 return end;
4707 }
4708
4709 static SLJIT_INLINE uschar *compile_fail_accept_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4710 {
4711 DEFINE_COMPILER;
4712 fallback_common *fallback;
4713
4714 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
4715
4716 if (*cc == OP_FAIL)
4717 {
4718 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4719 return cc + 1;
4720 }
4721
4722 if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL)
4723 {
4724 /* No need to check notempty conditions. */
4725 if (common->acceptlabel == NULL)
4726 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
4727 else
4728 JUMPTO(SLJIT_JUMP, common->acceptlabel);
4729 return cc + 1;
4730 }
4731
4732 if (common->acceptlabel == NULL)
4733 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0)));
4734 else
4735 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), common->acceptlabel);
4736 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
4737 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
4738 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4739 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
4740 if (common->acceptlabel == NULL)
4741 add_jump(compiler, &common->accept, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4742 else
4743 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->acceptlabel);
4744 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
4745 if (common->acceptlabel == NULL)
4746 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0));
4747 else
4748 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->acceptlabel);
4749 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4750 return cc + 1;
4751 }
4752
4753 static SLJIT_INLINE uschar *compile_close_hotpath(compiler_common *common, uschar *cc)
4754 {
4755 DEFINE_COMPILER;
4756 int offset = GET2(cc, 1);
4757
4758 /* Data will be discarded anyway... */
4759 if (common->currententry != NULL)
4760 return cc + 3;
4761
4762 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR_PRIV(offset));
4763 offset <<= 1;
4764 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4765 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4766 return cc + 3;
4767 }
4768
4769 static void compile_hotpath(compiler_common *common, uschar *cc, uschar *ccend, fallback_common *parent)
4770 {
4771 DEFINE_COMPILER;
4772 fallback_common *fallback;
4773
4774 while (cc < ccend)
4775 {
4776 switch(*cc)
4777 {
4778 case OP_SOD:
4779 case OP_SOM:
4780 case OP_NOT_WORD_BOUNDARY:
4781 case OP_WORD_BOUNDARY:
4782 case OP_NOT_DIGIT:
4783 case OP_DIGIT:
4784 case OP_NOT_WHITESPACE:
4785 case OP_WHITESPACE:
4786 case OP_NOT_WORDCHAR:
4787 case OP_WORDCHAR:
4788 case OP_ANY:
4789 case OP_ALLANY:
4790 case OP_NOTPROP:
4791 case OP_PROP:
4792 case OP_ANYNL:
4793 case OP_NOT_HSPACE:
4794 case OP_HSPACE:
4795 case OP_NOT_VSPACE:
4796 case OP_VSPACE:
4797 case OP_EXTUNI:
4798 case OP_EODN:
4799 case OP_EOD:
4800 case OP_CIRC:
4801 case OP_CIRCM:
4802 case OP_DOLL:
4803 case OP_DOLLM:
4804 case OP_NOT:
4805 case OP_NOTI:
4806 case OP_REVERSE:
4807 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4808 break;
4809
4810 case OP_SET_SOM:
4811 PUSH_FALLBACK_NOVALUE(sizeof(fallback_common), cc);
4812 allocate_stack(common, 1);
4813 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
4814 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
4815 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4816 cc++;
4817 break;
4818
4819 case OP_CHAR:
4820 case OP_CHARI:
4821 cc = compile_charn_hotpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4822 break;
4823
4824 case OP_STAR:
4825 case OP_MINSTAR:
4826 case OP_PLUS:
4827 case OP_MINPLUS:
4828 case OP_QUERY:
4829 case OP_MINQUERY:
4830 case OP_UPTO:
4831 case OP_MINUPTO:
4832 case OP_EXACT:
4833 case OP_POSSTAR:
4834 case OP_POSPLUS:
4835 case OP_POSQUERY:
4836 case OP_POSUPTO:
4837 case OP_STARI:
4838 case OP_MINSTARI:
4839 case OP_PLUSI:
4840 case OP_MINPLUSI:
4841 case OP_QUERYI:
4842 case OP_MINQUERYI:
4843 case OP_UPTOI:
4844 case OP_MINUPTOI:
4845 case OP_EXACTI:
4846 case OP_POSSTARI:
4847 case OP_POSPLUSI:
4848 case OP_POSQUERYI:
4849 case OP_POSUPTOI:
4850 case OP_NOTSTAR:
4851 case OP_NOTMINSTAR:
4852 case OP_NOTPLUS:
4853 case OP_NOTMINPLUS:
4854 case OP_NOTQUERY:
4855 case OP_NOTMINQUERY:
4856 case OP_NOTUPTO:
4857 case OP_NOTMINUPTO:
4858 case OP_NOTEXACT:
4859 case OP_NOTPOSSTAR:
4860 case OP_NOTPOSPLUS:
4861 case OP_NOTPOSQUERY:
4862 case OP_NOTPOSUPTO:
4863 case OP_NOTSTARI:
4864 case OP_NOTMINSTARI:
4865 case OP_NOTPLUSI:
4866 case OP_NOTMINPLUSI:
4867 case OP_NOTQUERYI:
4868 case OP_NOTMINQUERYI:
4869 case OP_NOTUPTOI:
4870 case OP_NOTMINUPTOI:
4871 case OP_NOTEXACTI:
4872 case OP_NOTPOSSTARI:
4873 case OP_NOTPOSPLUSI:
4874 case OP_NOTPOSQUERYI:
4875 case OP_NOTPOSUPTOI:
4876 case OP_TYPESTAR:
4877 case OP_TYPEMINSTAR:
4878 case OP_TYPEPLUS:
4879 case OP_TYPEMINPLUS:
4880 case OP_TYPEQUERY:
4881 case OP_TYPEMINQUERY:
4882 case OP_TYPEUPTO:
4883 case OP_TYPEMINUPTO:
4884 case OP_TYPEEXACT:
4885 case OP_TYPEPOSSTAR:
4886 case OP_TYPEPOSPLUS:
4887 case OP_TYPEPOSQUERY:
4888 case OP_TYPEPOSUPTO:
4889 cc = compile_iterator_hotpath(common, cc, parent);
4890 break;
4891
4892 case OP_CLASS:
4893 case OP_NCLASS:
4894 if (cc[33] >= OP_CRSTAR && cc[33] <= OP_CRMINRANGE)
4895 cc = compile_iterator_hotpath(common, cc, parent);
4896 else
4897 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4898 break;
4899
4900 #ifdef SUPPORT_UTF8
4901 case OP_XCLASS:
4902 if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRMINRANGE)
4903 cc = compile_iterator_hotpath(common, cc, parent);
4904 else
4905 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4906 break;
4907 #endif
4908
4909 case OP_REF:
4910 case OP_REFI:
4911 if (cc[3] >= OP_CRSTAR && cc[3] <= OP_CRMINRANGE)
4912 cc = compile_ref_iterator_hotpath(common, cc, parent);
4913 else
4914 cc = compile_ref_hotpath(common, cc, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks, TRUE, FALSE);
4915 break;
4916
4917 case OP_RECURSE:
4918 cc = compile_recurse_hotpath(common, cc, parent);
4919 break;
4920
4921 case OP_ASSERT:
4922 case OP_ASSERT_NOT:
4923 case OP_ASSERTBACK:
4924 case OP_ASSERTBACK_NOT:
4925 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
4926 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
4927 break;
4928
4929 case OP_BRAMINZERO:
4930 PUSH_FALLBACK_NOVALUE(sizeof(braminzero_fallback), cc);
4931 cc = bracketend(cc + 1);
4932 if (*(cc - 1 - LINK_SIZE) != OP_KETRMIN)
4933 {
4934 allocate_stack(common, 1);
4935 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4936 }
4937 else
4938 {
4939 allocate_stack(common, 2);
4940 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4941 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0);
4942 }
4943 FALLBACK_AS(braminzero_fallback)->hotpath = LABEL();
4944 if (cc[1] > OP_ASSERTBACK_NOT)
4945 decrease_call_count(common);
4946 break;
4947
4948 case OP_ONCE:
4949 case OP_BRA:
4950 case OP_CBRA:
4951 case OP_COND:
4952 case OP_SBRA:
4953 case OP_SCBRA:
4954 case OP_SCOND:
4955 cc = compile_bracket_hotpath(common, cc, parent);
4956 break;
4957
4958 case OP_BRAZERO:
4959 if (cc[1] > OP_ASSERTBACK_NOT)
4960 cc = compile_bracket_hotpath(common, cc, parent);
4961 else
4962 {
4963 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
4964 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
4965 }
4966 break;
4967
4968 case OP_BRAPOS:
4969 case OP_CBRAPOS:
4970 case OP_SBRAPOS:
4971 case OP_SCBRAPOS:
4972 case OP_BRAPOSZERO:
4973 cc = compile_bracketpos_hotpath(common, cc, parent);
4974 break;
4975
4976 case OP_FAIL:
4977 case OP_ACCEPT:
4978 case OP_ASSERT_ACCEPT:
4979 cc = compile_fail_accept_hotpath(common, cc, parent);
4980 break;
4981
4982 case OP_CLOSE:
4983 cc = compile_close_hotpath(common, cc);
4984 break;
4985
4986 case OP_SKIPZERO:
4987 cc = bracketend(cc + 1);
4988 break;
4989
4990 default:
4991 SLJIT_ASSERT_STOP();
4992 return;
4993 }
4994 if (cc == NULL)
4995 return;
4996 }
4997 SLJIT_ASSERT(cc == ccend);
4998 }
4999
5000 #undef PUSH_FALLBACK
5001 #undef PUSH_FALLBACK_NOVALUE
5002 #undef FALLBACK_AS
5003
5004 #define COMPILE_FALLBACKPATH(current) \
5005 do \
5006 { \
5007 compile_fallbackpath(common, (current)); \
5008 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
5009 return; \
5010 } \
5011 while (0)
5012
5013 #define CURRENT_AS(type) ((type*)current)
5014
5015 static void compile_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5016 {
5017 DEFINE_COMPILER;
5018 uschar *cc = current->cc;
5019 uschar opcode;
5020 uschar type;
5021 int arg1 = -1, arg2 = -1;
5022 struct sljit_label *label = NULL;
5023 struct sljit_jump *jump = NULL;
5024
5025 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, NULL);
5026
5027 switch(opcode)
5028 {
5029 case OP_STAR:
5030 case OP_PLUS:
5031 case OP_UPTO:
5032 case OP_CRRANGE:
5033 if (type == OP_ANYNL || type == OP_EXTUNI)
5034 {
5035 set_jumps(current->topfallbacks, LABEL());
5036 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5037 free_stack(common, 1);
5038 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5039 }
5040 else
5041 {
5042 if (opcode == OP_STAR || opcode == OP_UPTO)
5043 arg2 = 0;
5044 else if (opcode == OP_PLUS)
5045 arg2 = 1;
5046 jump = CMP(SLJIT_C_LESS_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, arg2 + 1);
5047 OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
5048 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5049 skip_char_back(common);
5050 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5051 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5052 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
5053 set_jumps(current->topfallbacks, LABEL());
5054 JUMPHERE(jump);
5055 free_stack(common, 2);
5056 }
5057 break;
5058
5059 case OP_MINSTAR:
5060 case OP_MINPLUS:
5061 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5062 if (opcode == OP_MINPLUS)
5063 {
5064 set_jumps(current->topfallbacks, LABEL());
5065 current->topfallbacks = NULL;
5066 }
5067 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5068 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5069 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5070 set_jumps(current->topfallbacks, LABEL());
5071 free_stack(common, 1);
5072 break;
5073
5074 case OP_MINUPTO:
5075 case OP_CRMINRANGE:
5076 if (opcode == OP_CRMINRANGE)
5077 {
5078 set_jumps(current->topfallbacks, LABEL());
5079 current->topfallbacks = NULL;
5080 label = LABEL();
5081 }
5082 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5083 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5084
5085 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5086 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5087 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
5088 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
5089
5090 if (opcode == OP_CRMINRANGE)
5091 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2 + 1, label);
5092
5093 if (opcode == OP_CRMINRANGE && arg1 == 0)
5094 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5095 else
5096 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 2, CURRENT_AS(iterator_fallback)->hotpath);
5097
5098 set_jumps(current->topfallbacks, LABEL());
5099 free_stack(common, 2);
5100 break;
5101
5102 case OP_QUERY:
5103 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5104 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5105 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5106 jump = JUMP(SLJIT_JUMP);
5107 set_jumps(current->topfallbacks, LABEL());
5108 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5109 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5110 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5111 JUMPHERE(jump);
5112 free_stack(common, 1);
5113 break;
5114
5115 case OP_MINQUERY:
5116 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5117 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5118 jump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5119 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5120 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5121 set_jumps(current->topfallbacks, LABEL());
5122 JUMPHERE(jump);
5123 free_stack(common, 1);
5124 break;
5125
5126 case OP_EXACT:
5127 case OP_POSPLUS:
5128 set_jumps(current->topfallbacks, LABEL());
5129 break;
5130
5131 case OP_POSSTAR:
5132 case OP_POSQUERY:
5133 case OP_POSUPTO:
5134 break;
5135
5136 default:
5137 SLJIT_ASSERT_STOP();
5138 break;
5139 }
5140 }
5141
5142 static void compile_ref_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5143 {
5144 DEFINE_COMPILER;
5145 uschar *cc = current->cc;
5146 uschar type;
5147
5148 type = cc[3];
5149 if ((type & 0x1) == 0)
5150 {
5151 set_jumps(current->topfallbacks, LABEL());
5152 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5153 free_stack(common, 1);
5154 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5155 return;
5156 }
5157
5158 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5159 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5160 set_jumps(current->topfallbacks, LABEL());
5161 free_stack(common, 2);
5162 }
5163
5164 static void compile_recurse_fallbackpath(compiler_common *common, struct fallback_common *current)
5165 {
5166 DEFINE_COMPILER;
5167
5168 set_jumps(current->topfallbacks, LABEL());
5169 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5170 free_stack(common, 1);
5171 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
5172 }
5173
5174 static void compile_assert_fallbackpath(compiler_common *common, struct fallback_common *current)
5175 {
5176 DEFINE_COMPILER;
5177 uschar *cc = current->cc;
5178 uschar bra = OP_BRA;
5179 struct sljit_jump *jump;
5180 struct sljit_jump *brajump = NULL;
5181
5182 SLJIT_ASSERT(*cc != OP_BRAMINZERO);
5183 if (*cc == OP_BRAZERO)
5184 {
5185 bra = *cc;
5186 cc++;
5187 }
5188
5189 if (bra == OP_BRAZERO)
5190 {
5191 SLJIT_ASSERT(current->topfallbacks == NULL);
5192 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5193 }
5194
5195 if (CURRENT_AS(assert_fallback)->framesize < 0)
5196 {
5197 set_jumps(current->topfallbacks, LABEL());
5198
5199 if (bra == OP_BRAZERO)
5200 {
5201 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5202 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5203 free_stack(common, 1);
5204 }
5205 return;
5206 }
5207
5208 if (bra == OP_BRAZERO)
5209 {
5210 if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT)
5211 {
5212 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5213 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5214 free_stack(common, 1);
5215 return;
5216 }
5217 free_stack(common, 1);
5218 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5219 }
5220
5221 if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK)
5222 {
5223 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr);
5224 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5225
5226 set_jumps(current->topfallbacks, LABEL());
5227 }
5228 else
5229 {
5230 jump = JUMP(SLJIT_JUMP);
5231
5232 set_jumps(current->topfallbacks, LABEL());
5233 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr);
5234 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5235 JUMPHERE(jump);
5236 }
5237 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_fallback)->framesize * sizeof(sljit_w));
5238
5239 if (bra == OP_BRAZERO)
5240 {
5241 /* We know there is enough place on the stack. */
5242 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5243 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5244 JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_fallback)->hotpath);
5245 JUMPHERE(brajump);
5246 }
5247 }
5248
5249 static void compile_bracket_fallbackpath(compiler_common *common, struct fallback_common *current)
5250 {
5251 DEFINE_COMPILER;
5252 int opcode;
5253 int offset = 0;
5254 int localptr = CURRENT_AS(bracket_fallback)->localptr;
5255 int stacksize;
5256 int count;
5257 uschar *cc = current->cc;
5258 uschar *ccbegin;
5259 uschar *ccprev;
5260 jump_list *jumplist = NULL;
5261 jump_list *jumplistitem = NULL;
5262 uschar bra = OP_BRA;
5263 uschar ket;
5264 assert_fallback *assert;
5265 struct sljit_jump *brazero = NULL;
5266 struct sljit_jump *once = NULL;
5267 struct sljit_jump *cond = NULL;
5268 struct sljit_label *rminlabel = NULL;
5269
5270 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
5271 {
5272 bra = *cc;
5273 cc++;
5274 }
5275
5276 opcode = *cc;
5277 ccbegin = cc;
5278 ket = *(bracketend(ccbegin) - 1 - LINK_SIZE);
5279 cc += GET(cc, 1);
5280 if (opcode == OP_CBRA || opcode == OP_SCBRA)
5281 offset = (GET2(ccbegin, 1 + LINK_SIZE)) << 1;
5282 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
5283 opcode = OP_SCOND;
5284
5285 if (ket == OP_KETRMAX)
5286 {
5287 if (bra != OP_BRAZERO)
5288 free_stack(common, 1);
5289 else
5290 {
5291 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5292 free_stack(common, 1);
5293 brazero = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5294 }
5295 }
5296 else if (ket == OP_KETRMIN)
5297 {
5298 if (bra != OP_BRAMINZERO)
5299 {
5300 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5301 if (opcode >= OP_SBRA || opcode == OP_ONCE)
5302 {
5303 /* Checking zero-length iteration. */
5304 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize < 0)
5305 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, CURRENT_AS(bracket_fallback)->recursivehotpath);
5306 else
5307 {
5308 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5309 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w), CURRENT_AS(bracket_fallback)->recursivehotpath);
5310 }
5311 if (opcode != OP_ONCE)
5312 free_stack(common, 1);
5313 }
5314 else
5315 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->recursivehotpath);
5316 }
5317 rminlabel = LABEL();
5318 }
5319 else if (bra == OP_BRAZERO)
5320 {
5321 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5322 free_stack(common, 1);
5323 brazero = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5324 }
5325
5326 if (opcode == OP_ONCE)
5327 {
5328 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5329 {
5330 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5331 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5332 }
5333 once = JUMP(SLJIT_JUMP);
5334 }
5335 else if (*cc == OP_ALT)
5336 {
5337 /* Build a jump list. Get the last successfully matched branch index. */
5338 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5339 free_stack(common, 1);
5340 count = 1;
5341 do
5342 {
5343 /* Append as the last item. */
5344 if (jumplist != NULL)
5345 {
5346 jumplistitem->next = sljit_alloc_memory(compiler, sizeof(jump_list));
5347 jumplistitem = jumplistitem->next;
5348 }
5349 else
5350 {
5351 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5352 jumplist = jumplistitem;
5353 }
5354
5355 if (SLJIT_UNLIKELY(!jumplistitem))
5356 return;
5357
5358 jumplistitem->next = NULL;
5359 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, count++);
5360 cc += GET(cc, 1);
5361 }
5362 while (*cc == OP_ALT);
5363
5364 cc = ccbegin + GET(ccbegin, 1);
5365 }
5366 else if (opcode == OP_COND || opcode == OP_SCOND)
5367 {
5368 /* Always one. */
5369 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5370 free_stack(common, 1);
5371
5372 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5373 if (SLJIT_UNLIKELY(!jumplistitem))
5374 return;
5375 jumplist = jumplistitem;
5376 jumplistitem->next = NULL;
5377 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 1);
5378 }
5379
5380 COMPILE_FALLBACKPATH(current->top);
5381 if (current->topfallbacks)
5382 set_jumps(current->topfallbacks, LABEL());
5383
5384 if (opcode == OP_COND || opcode == OP_SCOND)
5385 {
5386 /* Conditional block always has at most one alternative. */
5387 if (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT)
5388 {
5389 assert = CURRENT_AS(bracket_fallback)->u.assert;
5390 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK))
5391 {
5392 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5393 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5394 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5395 }
5396 cond = JUMP(SLJIT_JUMP);
5397 set_jumps(CURRENT_AS(bracket_fallback)->u.assert->condfailed, LABEL());
5398 }
5399 else
5400 {
5401 cond = JUMP(SLJIT_JUMP);
5402 set_jumps(CURRENT_AS(bracket_fallback)->u.condfailed, LABEL());
5403 }
5404 }
5405
5406 if (*cc == OP_ALT || opcode == OP_COND || opcode == OP_SCOND)
5407 {
5408 count = 1;
5409 do
5410 {
5411 current->top = NULL;
5412 current->topfallbacks = NULL;
5413 current->nextfallbacks = NULL;
5414 if (*cc == OP_ALT)
5415 {
5416 ccprev = cc + 1 + LINK_SIZE;
5417 cc += GET(cc, 1);
5418 if (opcode != OP_COND && opcode != OP_SCOND)
5419 {
5420 if (localptr != 0 && opcode != OP_ONCE)
5421 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5422 else
5423 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5424 }
5425 compile_hotpath(common, ccprev, cc, current);
5426 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
5427 return;
5428 }
5429
5430 /* Instructions after the current alternative is succesfully matched. */
5431 /* There is a similar code in compile_bracket_hotpath. */
5432 if (opcode == OP_ONCE)
5433 {
5434 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5435 {
5436 if (ket == OP_KETRMAX)
5437 {
5438 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5439 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), (CURRENT_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w));
5440 }
5441 }
5442 else
5443 {
5444 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5445 /* The register which is set here used by OP_KETRMAX below. */
5446 if (ket == OP_KETRMAX)
5447 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
5448 else if (ket == OP_KETRMIN)
5449 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
5450 }
5451 }
5452
5453 stacksize = 0;
5454 if (opcode != OP_ONCE)
5455 stacksize++;
5456 if (ket != OP_KET || bra != OP_BRA)
5457 stacksize++;
5458
5459 if (stacksize > 0) {
5460 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5461 allocate_stack(common, stacksize);
5462 else
5463 {
5464 /* We know we have place at least for one item on the top of the stack. */
5465 SLJIT_ASSERT(stacksize == 1);
5466 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5467 }
5468 }
5469
5470 stacksize = 0;
5471 if (ket != OP_KET || bra != OP_BRA)
5472 {
5473 if (ket != OP_KET)
5474 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
5475 else
5476 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
5477 stacksize++;
5478 }
5479
5480 if (opcode != OP_ONCE)
5481 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, count++);
5482
5483 if (offset != 0)
5484 {
5485 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5486 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
5487 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
5488 }
5489
5490 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->althotpath);
5491
5492 if (opcode != OP_ONCE)
5493 {
5494 SLJIT_ASSERT(jumplist);
5495 JUMPHERE(jumplist->jump);
5496 jumplist = jumplist->next;
5497 }
5498
5499 COMPILE_FALLBACKPATH(current->top);
5500 if (current->topfallbacks)
5501 set_jumps(current->topfallbacks, LABEL());
5502 SLJIT_ASSERT(!current->nextfallbacks);
5503 }
5504 while (*cc == OP_ALT);
5505 SLJIT_ASSERT(!jumplist);
5506
5507 if (cond != NULL)
5508 {
5509 SLJIT_ASSERT(opcode == OP_COND || opcode == OP_SCOND);
5510 assert = CURRENT_AS(bracket_fallback)->u.assert;
5511 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT))
5512 {
5513 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5514 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5515 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5516 }
5517 JUMPHERE(cond);
5518 }
5519
5520 /* Free the STR_PTR. */
5521 if (localptr == 0)
5522 free_stack(common, 1);
5523 }
5524
5525 if (offset != 0)
5526 {
5527 /* Using both tmp register is better for instruction scheduling. */
5528 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5529 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5530 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5531 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5532 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(2));
5533 free_stack(common, 3);
5534 }
5535 else if (opcode == OP_SBRA || opcode == OP_SCOND)
5536 {
5537 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(0));
5538 free_stack(common, 1);
5539 }
5540 else if (opcode == OP_ONCE)
5541 {
5542 cc = ccbegin + GET(ccbegin, 1);
5543 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5544 {
5545 /* Reset head and drop saved frame. */
5546 stacksize = (ket == OP_KETRMAX || ket == OP_KETRMIN || *cc == OP_ALT) ? 2 : 1;
5547 if (CURRENT_AS(bracket_fallback)->u.framesize > 0)
5548 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(stacksize));
5549 free_stack(common, CURRENT_AS(bracket_fallback)->u.framesize + stacksize);
5550 if (CURRENT_AS(bracket_fallback)->u.framesize > 0)
5551 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP1, 0);
5552 }
5553 else if (ket == OP_KETRMAX || (*cc == OP_ALT && ket != OP_KETRMIN))
5554 {
5555 /* The STR_PTR must be released. */
5556 free_stack(common, 1);
5557 }
5558
5559 JUMPHERE(once);
5560 /* Restore previous localptr */
5561 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5562 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_fallback)->u.framesize * sizeof(sljit_w));
5563 else if (ket == OP_KETRMIN)
5564 {
5565 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5566 /* See the comment below. */
5567 free_stack(common, 2);
5568 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
5569 }
5570 }
5571
5572 if (ket == OP_KETRMAX)
5573 {
5574 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5575 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_fallback)->recursivehotpath);
5576 if (bra == OP_BRAZERO)
5577 {
5578 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5579 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5580 JUMPHERE(brazero);
5581 }
5582 free_stack(common, 1);
5583 }
5584 else if (ket == OP_KETRMIN)
5585 {
5586 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5587
5588 /* OP_ONCE removes everything in case of a fallback, so we don't
5589 need to explicitly release the STR_PTR. The extra release would
5590 affect badly the free_stack(2) above. */
5591 if (opcode != OP_ONCE)
5592 free_stack(common, 1);
5593 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rminlabel);
5594 if (opcode == OP_ONCE)
5595 free_stack(common, bra == OP_BRAMINZERO ? 2 : 1);
5596 else if (bra == OP_BRAMINZERO)
5597 free_stack(common, 1);
5598 }
5599 else if (bra == OP_BRAZERO)
5600 {
5601 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5602 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5603 JUMPHERE(brazero);
5604 }
5605 }
5606
5607 static void compile_bracketpos_fallbackpath(compiler_common *common, struct fallback_common *current)
5608 {
5609 DEFINE_COMPILER;
5610 int offset;
5611 struct sljit_jump *jump;
5612
5613 if (CURRENT_AS(bracketpos_fallback)->framesize < 0)
5614 {
5615 if (*current->cc == OP_CBRAPOS || *current->cc == OP_SCBRAPOS)
5616 {
5617 offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1;
5618 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5619 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5620 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5621 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5622 }
5623 set_jumps(current->topfallbacks, LABEL());
5624 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5625 return;
5626 }
5627
5628 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr);
5629 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5630
5631 if (current->topfallbacks)
5632 {
5633 jump = JUMP(SLJIT_JUMP);
5634 set_jumps(current->topfallbacks, LABEL());
5635 /* Drop the stack frame and restore LOCALS_HEAD. */
5636 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(CURRENT_AS(bracketpos_fallback)->stacksize - CURRENT_AS(bracketpos_fallback)->framesize));
5637 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5638 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, TMP1, 0);
5639 JUMPHERE(jump);
5640 }
5641 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_fallback)->framesize * sizeof(sljit_w));
5642 }
5643
5644 static void compile_braminzero_fallbackpath(compiler_common *common, struct fallback_common *current)
5645 {
5646 assert_fallback fallback;
5647
5648 current->top = NULL;
5649 current->topfallbacks = NULL;
5650 current->nextfallbacks = NULL;
5651 if (current->cc[1] > OP_ASSERTBACK_NOT)
5652 {
5653 /* Manual call of compile_bracket_hotpath and compile_bracket_fallbackpath. */
5654 compile_bracket_hotpath(common, current->cc, current);
5655 compile_bracket_fallbackpath(common, current->top);
5656 }
5657 else
5658 {
5659 memset(&fallback, 0, sizeof(fallback));
5660 fallback.common.cc = current->cc;
5661 fallback.hotpath = CURRENT_AS(braminzero_fallback)->hotpath;
5662 /* Manual call of compile_assert_hotpath. */
5663 compile_assert_hotpath(common, current->cc, &fallback, FALSE);
5664 }
5665 SLJIT_ASSERT(!current->nextfallbacks && !current->topfallbacks);
5666 }
5667
5668 static void compile_fallbackpath(compiler_common *common, struct fallback_common *current)
5669 {
5670 DEFINE_COMPILER;
5671
5672 while (current)
5673 {
5674 if (current->nextfallbacks != NULL)
5675 set_jumps(current->nextfallbacks, LABEL());
5676 switch(*current->cc)
5677 {
5678 case OP_SET_SOM:
5679 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5680 free_stack(common, 1);
5681 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP1, 0);
5682 break;
5683
5684 case OP_STAR:
5685 case OP_MINSTAR:
5686 case OP_PLUS:
5687 case OP_MINPLUS:
5688 case OP_QUERY:
5689 case OP_MINQUERY:
5690 case OP_UPTO:
5691 case OP_MINUPTO:
5692 case OP_EXACT:
5693 case OP_POSSTAR:
5694 case OP_POSPLUS:
5695 case OP_POSQUERY:
5696 case OP_POSUPTO:
5697 case OP_STARI:
5698 case OP_MINSTARI:
5699 case OP_PLUSI:
5700 case OP_MINPLUSI:
5701 case OP_QUERYI:
5702 case OP_MINQUERYI:
5703 case OP_UPTOI:
5704 case OP_MINUPTOI:
5705 case OP_EXACTI:
5706 case OP_POSSTARI:
5707 case OP_POSPLUSI:
5708 case OP_POSQUERYI:
5709 case OP_POSUPTOI:
5710 case OP_NOTSTAR:
5711 case OP_NOTMINSTAR:
5712 case OP_NOTPLUS:
5713 case OP_NOTMINPLUS:
5714 case OP_NOTQUERY:
5715 case OP_NOTMINQUERY:
5716 case OP_NOTUPTO:
5717 case OP_NOTMINUPTO:
5718 case OP_NOTEXACT:
5719 case OP_NOTPOSSTAR:
5720 case OP_NOTPOSPLUS:
5721 case OP_NOTPOSQUERY:
5722 case OP_NOTPOSUPTO:
5723 case OP_NOTSTARI:
5724 case OP_NOTMINSTARI:
5725 case OP_NOTPLUSI:
5726 case OP_NOTMINPLUSI:
5727 case OP_NOTQUERYI:
5728 case OP_NOTMINQUERYI:
5729 case OP_NOTUPTOI:
5730 case OP_NOTMINUPTOI:
5731 case OP_NOTEXACTI:
5732 case OP_NOTPOSSTARI:
5733 case OP_NOTPOSPLUSI:
5734 case OP_NOTPOSQUERYI:
5735 case OP_NOTPOSUPTOI:
5736 case OP_TYPESTAR:
5737 case OP_TYPEMINSTAR:
5738 case OP_TYPEPLUS:
5739 case OP_TYPEMINPLUS:
5740 case OP_TYPEQUERY:
5741 case OP_TYPEMINQUERY:
5742 case OP_TYPEUPTO:
5743 case OP_TYPEMINUPTO:
5744 case OP_TYPEEXACT:
5745 case OP_TYPEPOSSTAR:
5746 case OP_TYPEPOSPLUS:
5747 case OP_TYPEPOSQUERY:
5748 case OP_TYPEPOSUPTO:
5749 case OP_CLASS:
5750 case OP_NCLASS:
5751 case OP_XCLASS:
5752 compile_iterator_fallbackpath(common, current);
5753 break;
5754
5755 case OP_REF:
5756 case OP_REFI:
5757 compile_ref_iterator_fallbackpath(common, current);
5758 break;
5759
5760 case OP_RECURSE:
5761 compile_recurse_fallbackpath(common, current);
5762 break;
5763
5764 case OP_ASSERT:
5765 case OP_ASSERT_NOT:
5766 case OP_ASSERTBACK:
5767 case OP_ASSERTBACK_NOT:
5768 compile_assert_fallbackpath(common, current);
5769 break;
5770
5771 case OP_ONCE:
5772 case OP_BRA:
5773 case OP_CBRA:
5774 case OP_COND:
5775 case OP_SBRA:
5776 case OP_SCBRA:
5777 case OP_SCOND:
5778 compile_bracket_fallbackpath(common, current);
5779 break;
5780
5781 case OP_BRAZERO:
5782 if (current->cc[1] > OP_ASSERTBACK_NOT)
5783 compile_bracket_fallbackpath(common, current);
5784 else
5785 compile_assert_fallbackpath(common, current);
5786 break;
5787
5788 case OP_BRAPOS:
5789 case OP_CBRAPOS:
5790 case OP_SBRAPOS:
5791 case OP_SCBRAPOS:
5792 case OP_BRAPOSZERO:
5793 compile_bracketpos_fallbackpath(common, current);
5794 break;
5795
5796 case OP_BRAMINZERO:
5797 compile_braminzero_fallbackpath(common, current);
5798 break;
5799
5800 case OP_FAIL:
5801 case OP_ACCEPT:
5802 case OP_ASSERT_ACCEPT:
5803 set_jumps(current->topfallbacks, LABEL());
5804 break;
5805
5806 default:
5807 SLJIT_ASSERT_STOP();
5808 break;
5809 }
5810 current = current->prev;
5811 }
5812 }
5813
5814 static SLJIT_INLINE void compile_recurse(compiler_common *common)
5815 {
5816 DEFINE_COMPILER;
5817 uschar *cc = common->start + common->currententry->start;
5818 uschar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : 2);
5819 uschar *ccend = bracketend(cc);
5820 int localsize = get_localsize(common, ccbegin, ccend);
5821 int framesize = get_framesize(common, cc, TRUE);
5822 int alternativesize;
5823 BOOL needsframe;
5824 fallback_common altfallback;
5825 struct sljit_jump *jump;
5826
5827 SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS);
5828 needsframe = framesize >= 0;
5829 if (!needsframe)
5830 framesize = 0;
5831 alternativesize = *(cc + GET(cc, 1)) == OP_ALT ? 1 : 0;
5832
5833 SLJIT_ASSERT(common->currententry->entry == NULL);
5834 common->currententry->entry = LABEL();
5835 set_jumps(common->currententry->calls, common->currententry->entry);
5836
5837 sljit_emit_fast_enter(compiler, TMP2, 0, 1, 5, 5, common->localsize);
5838 allocate_stack(common, localsize + framesize + alternativesize);
5839 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(localsize + framesize + alternativesize - 1), TMP2, 0);
5840 copy_locals(common, ccbegin, ccend, TRUE, localsize + framesize + alternativesize, framesize + alternativesize);
5841 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, STACK_TOP, 0);
5842 if (needsframe)
5843 {
5844 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(framesize + alternativesize - 1));
5845 init_frame(common, cc, framesize + alternativesize - 1, alternativesize, FALSE);
5846 }
5847
5848 if (alternativesize > 0)
5849 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5850
5851 memset(&altfallback, 0, sizeof(fallback_common));
5852 common->acceptlabel = NULL;
5853 common->accept = NULL;
5854 altfallback.cc = ccbegin;
5855 cc += GET(cc, 1);
5856 while (1)
5857 {
5858 altfallback.top = NULL;
5859 altfallback.topfallbacks = NULL;
5860
5861 if (altfallback.cc != ccbegin)
5862 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5863
5864 compile_hotpath(common, altfallback.cc, cc, &altfallback);
5865 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
5866 return;
5867
5868 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
5869
5870 compile_fallbackpath(common, altfallback.top);
5871 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
5872 return;
5873 set_jumps(altfallback.topfallbacks, LABEL());
5874
5875 if (*cc != OP_ALT)
5876 break;
5877
5878 altfallback.cc = cc + 1 + LINK_SIZE;
5879 cc += GET(cc, 1);
5880 }
5881 /* None of them matched. */
5882 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0);
5883 if (needsframe)
5884 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, SLJIT_MEM1(STACK_TOP), STACK(alternativesize));
5885 jump = JUMP(SLJIT_JUMP);
5886
5887 set_jumps(common->accept, LABEL());
5888 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD);
5889 if (needsframe)
5890 {
5891 OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
5892 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (localsize + framesize + alternativesize) * sizeof(sljit_w));
5893 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5894 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (localsize + framesize + alternativesize) * sizeof(sljit_w));
5895 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP3, 0);
5896 }
5897 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1);
5898
5899 JUMPHERE(jump);
5900 copy_locals(common, ccbegin, ccend, FALSE, localsize + framesize + alternativesize, framesize + alternativesize);
5901 free_stack(common, localsize + framesize + alternativesize);
5902 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_w));
5903 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
5904 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, TMP2, 0);
5905 sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0);
5906 }
5907
5908 #undef COMPILE_FALLBACKPATH
5909 #undef CURRENT_AS
5910
5911 void
5912 _pcre_jit_compile(const real_pcre *re, pcre_extra *extra)
5913 {
5914 struct sljit_compiler *compiler;
5915 fallback_common rootfallback;
5916 compiler_common common_data;
5917 compiler_common *common = &common_data;
5918 const unsigned char *tables = re->tables;
5919 pcre_study_data *study;
5920 uschar *ccend;
5921 executable_function *function;
5922 void *executable_func;
5923 struct sljit_label *leave;
5924 struct sljit_label *mainloop = NULL;
5925 struct sljit_label *empty_match_found;
5926 struct sljit_label *empty_match_fallback;
5927 struct sljit_jump *alloc_error;
5928 struct sljit_jump *reqbyte_notfound = NULL;
5929 struct sljit_jump *empty_match;
5930
5931 SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0);
5932 study = extra->study_data;
5933
5934 if (!tables)
5935 tables = _pcre_default_tables;
5936
5937 memset(&rootfallback, 0, sizeof(fallback_common));
5938 rootfallback.cc = (uschar *)re + re->name_table_offset + re->name_count * re->name_entry_size;
5939
5940 common->compiler = NULL;
5941 common->start = rootfallback.cc;
5942 common->cbraptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_w);
5943 common->fcc = tables + fcc_offset;
5944 common->lcc = (sljit_w)(tables + lcc_offset);
5945 common->nltype = NLTYPE_FIXED;
5946 switch(re->options & PCRE_NEWLINE_BITS)
5947 {
5948 case 0:
5949 /* Compile-time default */
5950 switch (NEWLINE)
5951 {
5952 case -1: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
5953 case -2: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
5954 default: common->newline = NEWLINE; break;
5955 }
5956 break;
5957 case PCRE_NEWLINE_CR: common->newline = CHAR_CR; break;
5958 case PCRE_NEWLINE_LF: common->newline = CHAR_NL; break;
5959 case PCRE_NEWLINE_CR+
5960 PCRE_NEWLINE_LF: common->newline = (CHAR_CR << 8) | CHAR_NL; break;
5961 case PCRE_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
5962 case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
5963 default: return;
5964 }
5965 if ((re->options & PCRE_BSR_ANYCRLF) != 0)
5966 common->bsr_nltype = NLTYPE_ANYCRLF;
5967 else if ((re->options & PCRE_BSR_UNICODE) != 0)
5968 common->bsr_nltype = NLTYPE_ANY;
5969 else
5970 {
5971 #ifdef BSR_ANYCRLF
5972 common->bsr_nltype = NLTYPE_ANYCRLF;
5973 #else
5974 common->bsr_nltype = NLTYPE_ANY;
5975 #endif
5976 }
5977 common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0;
5978 common->ctypes = (sljit_w)(tables + ctypes_offset);
5979 common->acceptlabel = NULL;
5980 common->stubs = NULL;
5981 common->entries = NULL;
5982 common->currententry = NULL;
5983 common->accept = NULL;
5984 common->calllimit = NULL;
5985 common->stackalloc = NULL;
5986 common->revertframes = NULL;
5987 common->wordboundary = NULL;
5988 common->anynewline = NULL;
5989 common->hspace = NULL;
5990 common->vspace = NULL;
5991 common->casefulcmp = NULL;
5992 common->caselesscmp = NULL;
5993 common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0;
5994 #ifdef SUPPORT_UTF8
5995 common->utf8 = (re->options & PCRE_UTF8) != 0;
5996 #ifdef SUPPORT_UCP
5997 common->useucp = (re->options & PCRE_UCP) != 0;
5998 #endif
5999 common->utf8readchar = NULL;
6000 common->utf8readtype8 = NULL;
6001 #endif
6002 #ifdef SUPPORT_UCP
6003 common->getucd = NULL;
6004 #endif
6005 ccend = bracketend(rootfallback.cc);
6006 SLJIT_ASSERT(*rootfallback.cc == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET);
6007 common->localsize = get_localspace(common, rootfallback.cc, ccend);
6008 if (common->localsize < 0)
6009 return;
6010 common->localsize += common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w);
6011 if (common->localsize > SLJIT_MAX_LOCAL_SIZE)
6012 return;
6013 common->localptrs = (int*)SLJIT_MALLOC((ccend - rootfallback.cc) * sizeof(int));
6014 if (!common->localptrs)
6015 return;
6016 memset(common->localptrs, 0, (ccend - rootfallback.cc) * sizeof(int));
6017 set_localptrs(common, common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w), ccend);
6018
6019 compiler = sljit_create_compiler();
6020 if (!compiler)
6021 {
6022 SLJIT_FREE(common->localptrs);
6023 return;
6024 }
6025 common->compiler = compiler;
6026
6027 /* Main pcre_jit_exec entry. */
6028 sljit_emit_enter(compiler, 1, 5, 5, common->localsize);
6029
6030 /* Register init. */
6031 reset_ovector(common, (re->top_bracket + 1) * 2);
6032 if ((re->flags & PCRE_REQCHSET) != 0)
6033 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, SLJIT_TEMPORARY_REG1, 0);
6034 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS_HEAD, SLJIT_IMM, 0);
6035
6036 OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_GENERAL_REG1, 0);
6037 OP1(SLJIT_MOV, TMP1, 0, SLJIT_GENERAL_REG1, 0);
6038 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6039 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end));
6040 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6041 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, calllimit));
6042 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base));
6043 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit));
6044 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT, TMP1, 0);
6045
6046 /* Main part of the matching */
6047 if ((re->options & PCRE_ANCHORED) == 0)
6048 {
6049 mainloop = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0, (re->options & PCRE_FIRSTLINE) != 0);
6050 /* Forward search if possible. */
6051 if ((re->flags & PCRE_FIRSTSET) != 0)
6052 fast_forward_first_byte(common, re->first_byte, (re->options & PCRE_FIRSTLINE) != 0);
6053 else if ((re->flags & PCRE_STARTLINE) != 0)
6054 fast_forward_newline(common, (re->options & PCRE_FIRSTLINE) != 0);
6055 else if ((re->flags & PCRE_STARTLINE) == 0 && study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)
6056 fast_forward_start_bits(common, (sljit_uw)study->start_bits, (re->options & PCRE_FIRSTLINE) != 0);
6057 }
6058 if ((re->flags & PCRE_REQCHSET) != 0)
6059 reqbyte_notfound = search_requested_char(common, re->req_byte, (re->flags & PCRE_FIRSTSET) != 0);
6060
6061 /* Store the current STR_PTR in OVECTOR(0). */
6062 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
6063 /* Copy the limit of allowed recursions. */
6064 OP1(SLJIT_MOV, CALL_COUNT, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT);
6065
6066 compile_hotpath(common, rootfallback.cc, ccend, &rootfallback);
6067 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6068 {
6069 sljit_free_compiler(compiler);
6070 SLJIT_FREE(common->localptrs);
6071 return;
6072 }
6073
6074 empty_match = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6075 empty_match_found = LABEL();
6076
6077 common->acceptlabel = LABEL();
6078 if (common->accept != NULL)
6079 set_jumps(common->accept, common->acceptlabel);
6080
6081 /* This means we have a match. Update the ovector. */
6082 copy_ovector(common, re->top_bracket + 1);
6083 leave = LABEL();
6084 sljit_emit_return(compiler, SLJIT_UNUSED, 0);
6085
6086 empty_match_fallback = LABEL();
6087 compile_fallbackpath(common, rootfallback.top);
6088 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6089 {
6090 sljit_free_compiler(compiler);
6091 SLJIT_FREE(common->localptrs);
6092 return;
6093 }
6094
6095 SLJIT_ASSERT(rootfallback.prev == NULL);
6096
6097 /* Check we have remaining characters. */
6098 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6099
6100 if ((re->options & PCRE_ANCHORED) == 0)
6101 {
6102 if ((re->options & PCRE_FIRSTLINE) == 0)
6103 {
6104 if (study != NULL && study->minlength > 1)
6105 {
6106 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6107 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, STR_END, 0, mainloop);
6108 }
6109 else
6110 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
6111 }
6112 else
6113 {
6114 if (study != NULL && study->minlength > 1)
6115 {
6116 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6117 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, STR_END, 0);
6118 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER);
6119 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
6120 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_GREATER_EQUAL);
6121 JUMPTO(SLJIT_C_ZERO, mainloop);
6122 }
6123 else
6124 CMPTO(SLJIT_C_LESS, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, mainloop);
6125 }
6126 }
6127
6128 if (reqbyte_notfound != NULL)
6129 JUMPHERE(reqbyte_notfound);
6130 /* Copy OVECTOR(1) to OVECTOR(0) */
6131 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
6132 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
6133 JUMPTO(SLJIT_JUMP, leave);
6134
6135 flush_stubs(common);
6136
6137 JUMPHERE(empty_match);
6138 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6139 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
6140 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_fallback);
6141 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
6142 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found);
6143 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6144 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found);
6145 JUMPTO(SLJIT_JUMP, empty_match_fallback);
6146
6147 common->currententry = common->entries;
6148 while (common->currententry != NULL)
6149 {
6150 /* Might add new entries. */
6151 compile_recurse(common);
6152 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6153 {
6154 sljit_free_compiler(compiler);
6155 SLJIT_FREE(common->localptrs);
6156 return;
6157 }
6158 flush_stubs(common);
6159 common->currententry = common->currententry->next;
6160 }
6161
6162 /* Allocating stack, returns with PCRE_ERROR_JIT_STACKLIMIT if fails. */
6163 /* This is a (really) rare case. */
6164 set_jumps(common->stackalloc, LABEL());
6165 /* RETURN_ADDR is not a saved register. */
6166 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
6167 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
6168 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6169 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6170 OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
6171 OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
6172
6173 sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
6174 alloc_error = CMP(SLJIT_C_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
6175 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6176 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6177 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top));
6178 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit));
6179 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
6180 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
6181
6182 /* Allocation failed. */
6183 JUMPHERE(alloc_error);
6184 /* We break the return address cache here, but this is a really rare case. */
6185 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_JIT_STACKLIMIT);
6186 JUMPTO(SLJIT_JUMP, leave);
6187
6188 /* Call limit reached. */
6189 set_jumps(common->calllimit, LABEL());
6190 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_MATCHLIMIT);
6191 JUMPTO(SLJIT_JUMP, leave);
6192
6193 if (common->revertframes != NULL)
6194 {
6195 set_jumps(common->revertframes, LABEL());
6196 do_revertframes(common);
6197 }
6198 if (common->wordboundary != NULL)
6199 {
6200 set_jumps(common->wordboundary, LABEL());
6201 check_wordboundary(common);
6202 }
6203 if (common->anynewline != NULL)
6204 {
6205 set_jumps(common->anynewline, LABEL());
6206 check_anynewline(common);
6207 }
6208 if (common->hspace != NULL)
6209 {
6210 set_jumps(common->hspace, LABEL());
6211 check_hspace(common);
6212 }
6213 if (common->vspace != NULL)
6214 {
6215 set_jumps(common->vspace, LABEL());
6216 check_vspace(common);
6217 }
6218 if (common->casefulcmp != NULL)
6219 {
6220 set_jumps(common->casefulcmp, LABEL());
6221 do_casefulcmp(common);
6222 }
6223 if (common->caselesscmp != NULL)
6224 {
6225 set_jumps(common->caselesscmp, LABEL());
6226 do_caselesscmp(common);
6227 }
6228 #ifdef SUPPORT_UTF8
6229 if (common->utf8readchar != NULL)
6230 {
6231 set_jumps(common->utf8readchar, LABEL());
6232 do_utf8readchar(common);
6233 }
6234 if (common->utf8readtype8 != NULL)
6235 {
6236 set_jumps(common->utf8readtype8, LABEL());
6237 do_utf8readtype8(common);
6238 }
6239 #endif
6240 #ifdef SUPPORT_UCP
6241 if (common->getucd != NULL)
6242 {
6243 set_jumps(common->getucd, LABEL());
6244 do_getucd(common);
6245 }
6246 #endif
6247
6248 SLJIT_FREE(common->localptrs);
6249 executable_func = sljit_generate_code(compiler);
6250 sljit_free_compiler(compiler);
6251 if (executable_func == NULL)
6252 return;
6253
6254 function = SLJIT_MALLOC(sizeof(executable_function));
6255 if (function == NULL)
6256 {
6257 /* This case is highly unlikely since we just recently
6258 freed a lot of memory. Although not impossible. */
6259 sljit_free_code(executable_func);
6260 return;
6261 }
6262
6263 function->executable_func = executable_func;
6264 function->callback = NULL;
6265 function->userdata = NULL;
6266 extra->executable_jit = function;
6267 extra->flags |= PCRE_EXTRA_EXECUTABLE_JIT;
6268 }
6269
6270 static int jit_machine_stack_exec(jit_arguments *arguments, executable_function *function)
6271 {
6272 union {
6273 void* executable_func;
6274 jit_function call_executable_func;
6275 } convert_executable_func;
6276 uschar local_area[LOCAL_SPACE_SIZE];
6277 struct sljit_stack local_stack;
6278
6279 local_stack.top = (sljit_w)&local_area;
6280 local_stack.base = local_stack.top;
6281 local_stack.limit = local_stack.base + LOCAL_SPACE_SIZE;
6282 local_stack.max_limit = local_stack.limit;
6283 arguments->stack = &local_stack;
6284 convert_executable_func.executable_func = function->executable_func;
6285 return convert_executable_func.call_executable_func(arguments);
6286 }
6287
6288 int
6289 _pcre_jit_exec(const real_pcre *re, void *executable_func,
6290 PCRE_SPTR subject, int length, int start_offset, int options,
6291 int match_limit, int *offsets, int offsetcount)
6292 {
6293 executable_function *function = (executable_function*)executable_func;
6294 union {
6295 void* executable_func;
6296 jit_function call_executable_func;
6297 } convert_executable_func;
6298 jit_arguments arguments;
6299 int maxoffsetcount;
6300 int retval;
6301
6302 /* Sanity checks should be handled by pcre_exec. */
6303 arguments.stack = NULL;
6304 arguments.str = subject + start_offset;
6305 arguments.begin = subject;
6306 arguments.end = subject + length;
6307 arguments.calllimit = match_limit; /* JIT decreases this value less times. */
6308 arguments.notbol = (options & PCRE_NOTBOL) != 0;
6309 arguments.noteol = (options & PCRE_NOTEOL) != 0;
6310 arguments.notempty = (options & PCRE_NOTEMPTY) != 0;
6311 arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0;
6312 arguments.offsets = offsets;
6313
6314 /* pcre_exec() rounds offsetcount to a multiple of 3, and then uses only 2/3 of
6315 the output vector for storing captured strings, with the remainder used as
6316 workspace. We don't need the workspace here. For compatibility, we limit the
6317 number of captured strings in the same way as pcre_exec(), so that the user
6318 gets the same result with and without JIT. */
6319
6320 offsetcount = ((offsetcount - (offsetcount % 3)) * 2)/3;
6321 maxoffsetcount = (re->top_bracket + 1) * 2;
6322 if (offsetcount > maxoffsetcount)
6323 offsetcount = maxoffsetcount;
6324 arguments.offsetcount = offsetcount;
6325
6326 if (function->callback)
6327 arguments.stack = (struct sljit_stack*)function->callback(function->userdata);
6328 else
6329 arguments.stack = (struct sljit_stack*)function->userdata;
6330
6331 if (arguments.stack == NULL)
6332 retval = jit_machine_stack_exec(&arguments, function);
6333 else
6334 {
6335 convert_executable_func.executable_func = function->executable_func;
6336 retval = convert_executable_func.call_executable_func(&arguments);
6337 }
6338
6339 if (retval * 2 > offsetcount)
6340 retval = 0;
6341 return retval;
6342 }
6343
6344 void
6345 _pcre_jit_free(void *executable_func)
6346 {
6347 executable_function *function = (executable_function*)executable_func;
6348 sljit_free_code(function->executable_func);
6349 SLJIT_FREE(function);
6350 }
6351
6352 PCRE_EXP_DECL pcre_jit_stack *
6353 pcre_jit_stack_alloc(int startsize, int maxsize)
6354 {
6355 if (startsize < 1 || maxsize < 1)
6356 return NULL;
6357 if (startsize > maxsize)
6358 startsize = maxsize;
6359 startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6360 maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6361 return (pcre_jit_stack*)sljit_allocate_stack(startsize, maxsize);
6362 }
6363
6364 PCRE_EXP_DECL void
6365 pcre_jit_stack_free(pcre_jit_stack *stack)
6366 {
6367 sljit_free_stack((struct sljit_stack*)stack);
6368 }
6369
6370 PCRE_EXP_DECL void
6371 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6372 {
6373 executable_function *function;
6374 if (extra != NULL &&
6375 (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 &&
6376 extra->executable_jit != NULL)
6377 {
6378 function = (executable_function*)extra->executable_jit;
6379 function->callback = callback;
6380 function->userdata = userdata;
6381 }
6382 }
6383
6384 #else /* SUPPORT_JIT */
6385
6386 /* These are dummy functions to avoid linking errors when JIT support is not
6387 being compiled. */
6388
6389 PCRE_EXP_DECL pcre_jit_stack *
6390 pcre_jit_stack_alloc(int startsize, int maxsize)
6391 {
6392 (void)startsize;
6393 (void)maxsize;
6394 return NULL;
6395 }
6396
6397 PCRE_EXP_DECL void
6398 pcre_jit_stack_free(pcre_jit_stack *stack)
6399 {
6400 (void)stack;
6401 }
6402
6403 PCRE_EXP_DECL void
6404 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6405 {
6406 (void)extra;
6407 (void)callback;
6408 (void)userdata;
6409 }
6410
6411 #endif
6412
6413 /* End of pcre_jit_compile.c */

  ViewVC Help
Powered by ViewVC 1.1.5