/[pcre]/code/trunk/pcre_jit_compile.c
ViewVC logotype

Contents of /code/trunk/pcre_jit_compile.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 741 - (show annotations)
Mon Oct 31 09:31:46 2011 UTC (7 years, 11 months ago) by zherczeg
File MIME type: text/plain
File size: 201788 byte(s)
Error occurred while calculating annotation data.
Supporting OP_NCREF, OP_RREF and OP_NRREF by the JIT compiler
1 /*************************************************
2 * Perl-Compatible Regular Expressions *
3 *************************************************/
4
5 /* PCRE is a library of functions to support regular expressions whose syntax
6 and semantics are as close as possible to those of the Perl 5 language.
7
8 Written by Philip Hazel
9 Copyright (c) 1997-2008 University of Cambridge
10
11 The machine code generator part (this module) was written by Zoltan Herczeg
12 Copyright (c) 2010-2011
13
14 -----------------------------------------------------------------------------
15 Redistribution and use in source and binary forms, with or without
16 modification, are permitted provided that the following conditions are met:
17
18 * Redistributions of source code must retain the above copyright notice,
19 this list of conditions and the following disclaimer.
20
21 * Redistributions in binary form must reproduce the above copyright
22 notice, this list of conditions and the following disclaimer in the
23 documentation and/or other materials provided with the distribution.
24
25 * Neither the name of the University of Cambridge nor the names of its
26 contributors may be used to endorse or promote products derived from
27 this software without specific prior written permission.
28
29 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
36 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
37 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 POSSIBILITY OF SUCH DAMAGE.
40 -----------------------------------------------------------------------------
41 */
42
43 #ifdef HAVE_CONFIG_H
44 #include "config.h"
45 #endif
46
47 #include "pcre_internal.h"
48
49 #ifdef SUPPORT_JIT
50
51 /* All-in-one: Since we use the JIT compiler only from here,
52 we just include it. This way we don't need to touch the build
53 system files. */
54
55 #define SLJIT_CONFIG_AUTO 1
56 #define SLJIT_CONFIG_STATIC 1
57 #define SLJIT_VERBOSE 0
58 #define SLJIT_DEBUG 0
59
60 #include "sljit/sljitLir.c"
61
62 #if defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED
63 #error "Unsupported architecture"
64 #endif
65
66 /* Allocate memory on the stack. Fast, but limited size. */
67 #define LOCAL_SPACE_SIZE 32768
68
69 #define STACK_GROWTH_RATE 8192
70
71 /* Enable to check that the allocation could destroy temporaries. */
72 #if defined SLJIT_DEBUG && SLJIT_DEBUG
73 #define DESTROY_REGISTERS 1
74 #endif
75
76 /*
77 Short summary about the backtracking mechanism empolyed by the jit code generator:
78
79 The code generator follows the recursive nature of the PERL compatible regular
80 expressions. The basic blocks of regular expressions are condition checkers
81 whose execute different commands depending on the result of the condition check.
82 The relationship between the operators can be horizontal (concatenation) and
83 vertical (sub-expression) (See struct fallback_common for more details).
84
85 'ab' - 'a' and 'b' regexps are concatenated
86 'a+' - 'a' is the sub-expression of the '+' operator
87
88 The condition checkers are boolean (true/false) checkers. Machine code is generated
89 for the checker itself and for the actions depending on the result of the checker.
90 The 'true' case is called as the hot path (expected path), and the other is called as
91 the 'fallback' path. Branch instructions are expesive for all CPUs, so we avoid taken
92 branches on the hot path.
93
94 Greedy star operator (*) :
95 Hot path: match happens.
96 Fallback path: match failed.
97 Non-greedy star operator (*?) :
98 Hot path: no need to perform a match.
99 Fallback path: match is required.
100
101 The following example shows how the code generated for a capturing bracket
102 with two alternatives. Let A, B, C, D are arbirary regular expressions, and
103 we have the following regular expression:
104
105 A(B|C)D
106
107 The generated code will be the following:
108
109 A hot path
110 '(' hot path (pushing arguments to the stack)
111 B hot path
112 ')' hot path (pushing arguments to the stack)
113 D hot path
114 return with successful match
115
116 D fallback path
117 ')' fallback path (If we arrived from "C" jump to the fallback of "C")
118 B fallback path
119 C expected path
120 jump to D hot path
121 C fallback path
122 A fallback path
123
124 Notice, that the order of fallback code paths are the opposite of the fast
125 code paths. In this way the topmost value on the stack is always belong
126 to the current fallback code path. The fallback code path must check
127 whether there is a next alternative. If so, it needs to jump back to
128 the hot path eventually. Otherwise it needs to clear out its own stack
129 frame and continue the execution on the fallback code paths.
130 */
131
132 /*
133 Saved stack frames:
134
135 Atomic blocks and asserts require reloading the values of local variables
136 when the fallback mechanism performed. Because of OP_RECURSE, the locals
137 are not necessarly known in compile time, thus we need a dynamic restore
138 mechanism.
139
140 The stack frames are stored in a chain list, and have the following format:
141 ([ capturing bracket offset ][ start value ][ end value ])+ ... [ 0 ] [ previous head ]
142
143 Thus we can restore the locals to a particular point in the stack.
144 */
145
146 typedef struct jit_arguments {
147 /* Pointers first. */
148 struct sljit_stack *stack;
149 PCRE_SPTR str;
150 PCRE_SPTR begin;
151 PCRE_SPTR end;
152 int *offsets;
153 uschar *ptr;
154 /* Everything else after. */
155 int offsetcount;
156 int calllimit;
157 uschar notbol;
158 uschar noteol;
159 uschar notempty;
160 uschar notempty_atstart;
161 } jit_arguments;
162
163 typedef struct executable_function {
164 void *executable_func;
165 pcre_jit_callback callback;
166 void *userdata;
167 } executable_function;
168
169 typedef struct jump_list {
170 struct sljit_jump *jump;
171 struct jump_list *next;
172 } jump_list;
173
174 enum stub_types { stack_alloc };
175
176 typedef struct stub_list {
177 enum stub_types type;
178 int data;
179 struct sljit_jump *start;
180 struct sljit_label *leave;
181 struct stub_list *next;
182 } stub_list;
183
184 typedef int (SLJIT_CALL *jit_function)(jit_arguments *args);
185
186 /* The following structure is the key data type for the recursive
187 code generator. It is allocated by compile_hotpath, and contains
188 the aguments for compile_fallbackpath. Must be the first member
189 of its descendants. */
190 typedef struct fallback_common {
191 /* Concatenation stack. */
192 struct fallback_common *prev;
193 jump_list *nextfallbacks;
194 /* Internal stack (for component operators). */
195 struct fallback_common *top;
196 jump_list *topfallbacks;
197 /* Opcode pointer. */
198 uschar *cc;
199 } fallback_common;
200
201 typedef struct assert_fallback {
202 fallback_common common;
203 jump_list *condfailed;
204 /* Less than 0 (-1) if a frame is not needed. */
205 int framesize;
206 /* Points to our private memory word on the stack. */
207 int localptr;
208 /* For iterators. */
209 struct sljit_label *hotpath;
210 } assert_fallback;
211
212 typedef struct bracket_fallback {
213 fallback_common common;
214 /* Where to coninue if an alternative is successfully matched. */
215 struct sljit_label *althotpath;
216 /* For rmin and rmax iterators. */
217 struct sljit_label *recursivehotpath;
218 /* For greedy ? operator. */
219 struct sljit_label *zerohotpath;
220 /* Contains the branches of a failed condition. */
221 union {
222 /* Both for OP_COND, OP_SCOND. */
223 jump_list *condfailed;
224 assert_fallback *assert;
225 /* For OP_ONCE. -1 if not needed. */
226 int framesize;
227 } u;
228 /* Points to our private memory word on the stack. */
229 int localptr;
230 } bracket_fallback;
231
232 typedef struct bracketpos_fallback {
233 fallback_common common;
234 /* Points to our private memory word on the stack. */
235 int localptr;
236 /* Reverting stack is needed. */
237 int framesize;
238 /* Allocated stack size. */
239 int stacksize;
240 } bracketpos_fallback;
241
242 typedef struct braminzero_fallback {
243 fallback_common common;
244 struct sljit_label *hotpath;
245 } braminzero_fallback;
246
247 typedef struct iterator_fallback {
248 fallback_common common;
249 /* Next iteration. */
250 struct sljit_label *hotpath;
251 } iterator_fallback;
252
253 typedef struct recurse_entry {
254 struct recurse_entry *next;
255 /* Contains the function entry. */
256 struct sljit_label *entry;
257 /* Collects the calls until the function is not created. */
258 jump_list *calls;
259 /* Points to the starting opcode. */
260 int start;
261 } recurse_entry;
262
263 typedef struct recurse_fallback {
264 fallback_common common;
265 } recurse_fallback;
266
267 typedef struct compiler_common {
268 struct sljit_compiler *compiler;
269 uschar *start;
270 int localsize;
271 int *localptrs;
272 const uschar *fcc;
273 sljit_w lcc;
274 int cbraptr;
275 int nltype;
276 int newline;
277 int bsr_nltype;
278 int endonly;
279 sljit_w ctypes;
280 sljit_uw name_table;
281 sljit_w name_count;
282 sljit_w name_entry_size;
283 struct sljit_label *acceptlabel;
284 stub_list *stubs;
285 recurse_entry *entries;
286 recurse_entry *currententry;
287 jump_list *accept;
288 jump_list *calllimit;
289 jump_list *stackalloc;
290 jump_list *revertframes;
291 jump_list *wordboundary;
292 jump_list *anynewline;
293 jump_list *hspace;
294 jump_list *vspace;
295 jump_list *casefulcmp;
296 jump_list *caselesscmp;
297 BOOL jscript_compat;
298 #ifdef SUPPORT_UTF8
299 BOOL utf8;
300 #ifdef SUPPORT_UCP
301 BOOL useucp;
302 #endif
303 jump_list *utf8readchar;
304 jump_list *utf8readtype8;
305 #endif
306 #ifdef SUPPORT_UCP
307 jump_list *getucd;
308 #endif
309 } compiler_common;
310
311 /* For byte_sequence_compare. */
312
313 typedef struct compare_context {
314 int length;
315 int sourcereg;
316 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
317 int byteptr;
318 union {
319 int asint;
320 short asshort;
321 sljit_ub asbyte;
322 sljit_ub asbytes[4];
323 } c;
324 union {
325 int asint;
326 short asshort;
327 sljit_ub asbyte;
328 sljit_ub asbytes[4];
329 } oc;
330 #endif
331 } compare_context;
332
333 enum {
334 frame_end = 0,
335 frame_setstrbegin = -1
336 };
337
338 /* Used for accessing the elements of the stack. */
339 #define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_w))
340
341 #define TMP1 SLJIT_TEMPORARY_REG1
342 #define TMP2 SLJIT_TEMPORARY_REG3
343 #define TMP3 SLJIT_TEMPORARY_EREG2
344 #define STR_PTR SLJIT_GENERAL_REG1
345 #define STR_END SLJIT_GENERAL_REG2
346 #define STACK_TOP SLJIT_TEMPORARY_REG2
347 #define STACK_LIMIT SLJIT_GENERAL_REG3
348 #define ARGUMENTS SLJIT_GENERAL_EREG1
349 #define CALL_COUNT SLJIT_GENERAL_EREG2
350 #define RETURN_ADDR SLJIT_TEMPORARY_EREG1
351
352 /* Locals layout. */
353 /* These two locals can be used by the current opcode. */
354 #define LOCALS0 (0 * sizeof(sljit_w))
355 #define LOCALS1 (1 * sizeof(sljit_w))
356 /* Two local variables for possessive quantifiers (char1 cannot use them). */
357 #define POSSESSIVE0 (2 * sizeof(sljit_w))
358 #define POSSESSIVE1 (3 * sizeof(sljit_w))
359 /* Head of the last recursion. */
360 #define RECURSIVE_HEAD (4 * sizeof(sljit_w))
361 /* Max limit of recursions. */
362 #define CALL_LIMIT (5 * sizeof(sljit_w))
363 /* Last known position of the requested byte. */
364 #define REQ_BYTE_PTR (6 * sizeof(sljit_w))
365 /* End pointer of the first line. */
366 #define FIRSTLINE_END (7 * sizeof(sljit_w))
367 /* The output vector is stored on the stack, and contains pointers
368 to characters. The vector data is divided into two groups: the first
369 group contains the start / end character pointers, and the second is
370 the start pointers when the end of the capturing group has not yet reached. */
371 #define OVECTOR_START (8 * sizeof(sljit_w))
372 #define OVECTOR(i) (OVECTOR_START + (i) * sizeof(sljit_w))
373 #define OVECTOR_PRIV(i) (common->cbraptr + (i) * sizeof(sljit_w))
374 #define PRIV(cc) (common->localptrs[(cc) - common->start])
375
376 /* Shortcuts. */
377 #define DEFINE_COMPILER \
378 struct sljit_compiler *compiler = common->compiler
379 #define OP1(op, dst, dstw, src, srcw) \
380 sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw))
381 #define OP2(op, dst, dstw, src1, src1w, src2, src2w) \
382 sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w))
383 #define LABEL() \
384 sljit_emit_label(compiler)
385 #define JUMP(type) \
386 sljit_emit_jump(compiler, (type))
387 #define JUMPTO(type, label) \
388 sljit_set_label(sljit_emit_jump(compiler, (type)), (label))
389 #define JUMPHERE(jump) \
390 sljit_set_label((jump), sljit_emit_label(compiler))
391 #define CMP(type, src1, src1w, src2, src2w) \
392 sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w))
393 #define CMPTO(type, src1, src1w, src2, src2w, label) \
394 sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label))
395 #define COND_VALUE(op, dst, dstw, type) \
396 sljit_emit_cond_value(compiler, (op), (dst), (dstw), (type))
397
398 static uschar* bracketend(uschar* cc)
399 {
400 SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
401 do cc += GET(cc, 1); while (*cc == OP_ALT);
402 SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS);
403 cc += 1 + LINK_SIZE;
404 return cc;
405 }
406
407 /* Functions whose might need modification for all new supported opcodes:
408 next_opcode
409 get_localspace
410 set_localptrs
411 get_framesize
412 init_frame
413 get_localsize
414 copy_locals
415 compile_hotpath
416 compile_fallbackpath
417 */
418
419 static uschar *next_opcode(compiler_common *common, uschar *cc)
420 {
421 SLJIT_UNUSED_ARG(common);
422 switch(*cc)
423 {
424 case OP_SOD:
425 case OP_SOM:
426 case OP_SET_SOM:
427 case OP_NOT_WORD_BOUNDARY:
428 case OP_WORD_BOUNDARY:
429 case OP_NOT_DIGIT:
430 case OP_DIGIT:
431 case OP_NOT_WHITESPACE:
432 case OP_WHITESPACE:
433 case OP_NOT_WORDCHAR:
434 case OP_WORDCHAR:
435 case OP_ANY:
436 case OP_ALLANY:
437 case OP_ANYNL:
438 case OP_NOT_HSPACE:
439 case OP_HSPACE:
440 case OP_NOT_VSPACE:
441 case OP_VSPACE:
442 case OP_EXTUNI:
443 case OP_EODN:
444 case OP_EOD:
445 case OP_CIRC:
446 case OP_CIRCM:
447 case OP_DOLL:
448 case OP_DOLLM:
449 case OP_TYPESTAR:
450 case OP_TYPEMINSTAR:
451 case OP_TYPEPLUS:
452 case OP_TYPEMINPLUS:
453 case OP_TYPEQUERY:
454 case OP_TYPEMINQUERY:
455 case OP_TYPEPOSSTAR:
456 case OP_TYPEPOSPLUS:
457 case OP_TYPEPOSQUERY:
458 case OP_CRSTAR:
459 case OP_CRMINSTAR:
460 case OP_CRPLUS:
461 case OP_CRMINPLUS:
462 case OP_CRQUERY:
463 case OP_CRMINQUERY:
464 case OP_DEF:
465 case OP_BRAZERO:
466 case OP_BRAMINZERO:
467 case OP_BRAPOSZERO:
468 case OP_FAIL:
469 case OP_ACCEPT:
470 case OP_ASSERT_ACCEPT:
471 case OP_SKIPZERO:
472 return cc + 1;
473
474 case OP_ANYBYTE:
475 #ifdef SUPPORT_UTF8
476 if (common->utf8) return NULL;
477 #endif
478 return cc + 1;
479
480 case OP_CHAR:
481 case OP_CHARI:
482 case OP_NOT:
483 case OP_NOTI:
484
485 case OP_STAR:
486 case OP_MINSTAR:
487 case OP_PLUS:
488 case OP_MINPLUS:
489 case OP_QUERY:
490 case OP_MINQUERY:
491 case OP_POSSTAR:
492 case OP_POSPLUS:
493 case OP_POSQUERY:
494 case OP_STARI:
495 case OP_MINSTARI:
496 case OP_PLUSI:
497 case OP_MINPLUSI:
498 case OP_QUERYI:
499 case OP_MINQUERYI:
500 case OP_POSSTARI:
501 case OP_POSPLUSI:
502 case OP_POSQUERYI:
503 case OP_NOTSTAR:
504 case OP_NOTMINSTAR:
505 case OP_NOTPLUS:
506 case OP_NOTMINPLUS:
507 case OP_NOTQUERY:
508 case OP_NOTMINQUERY:
509 case OP_NOTPOSSTAR:
510 case OP_NOTPOSPLUS:
511 case OP_NOTPOSQUERY:
512 case OP_NOTSTARI:
513 case OP_NOTMINSTARI:
514 case OP_NOTPLUSI:
515 case OP_NOTMINPLUSI:
516 case OP_NOTQUERYI:
517 case OP_NOTMINQUERYI:
518 case OP_NOTPOSSTARI:
519 case OP_NOTPOSPLUSI:
520 case OP_NOTPOSQUERYI:
521 cc += 2;
522 #ifdef SUPPORT_UTF8
523 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
524 #endif
525 return cc;
526
527 case OP_UPTO:
528 case OP_MINUPTO:
529 case OP_EXACT:
530 case OP_POSUPTO:
531 case OP_UPTOI:
532 case OP_MINUPTOI:
533 case OP_EXACTI:
534 case OP_POSUPTOI:
535 case OP_NOTUPTO:
536 case OP_NOTMINUPTO:
537 case OP_NOTEXACT:
538 case OP_NOTPOSUPTO:
539 case OP_NOTUPTOI:
540 case OP_NOTMINUPTOI:
541 case OP_NOTEXACTI:
542 case OP_NOTPOSUPTOI:
543 cc += 4;
544 #ifdef SUPPORT_UTF8
545 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
546 #endif
547 return cc;
548
549 case OP_NOTPROP:
550 case OP_PROP:
551 case OP_TYPEUPTO:
552 case OP_TYPEMINUPTO:
553 case OP_TYPEEXACT:
554 case OP_TYPEPOSUPTO:
555 case OP_REF:
556 case OP_REFI:
557 case OP_CREF:
558 case OP_NCREF:
559 case OP_RREF:
560 case OP_NRREF:
561 case OP_CLOSE:
562 cc += 3;
563 return cc;
564
565 case OP_CRRANGE:
566 case OP_CRMINRANGE:
567 return cc + 5;
568
569 case OP_CLASS:
570 case OP_NCLASS:
571 return cc + 33;
572
573 #ifdef SUPPORT_UTF8
574 case OP_XCLASS:
575 return cc + GET(cc, 1);
576 #endif
577
578 case OP_RECURSE:
579 case OP_ASSERT:
580 case OP_ASSERT_NOT:
581 case OP_ASSERTBACK:
582 case OP_ASSERTBACK_NOT:
583 case OP_REVERSE:
584 case OP_ONCE:
585 case OP_ONCE_NC:
586 case OP_BRA:
587 case OP_BRAPOS:
588 case OP_COND:
589 case OP_SBRA:
590 case OP_SBRAPOS:
591 case OP_SCOND:
592 case OP_ALT:
593 case OP_KET:
594 case OP_KETRMAX:
595 case OP_KETRMIN:
596 case OP_KETRPOS:
597 return cc + 1 + LINK_SIZE;
598
599 case OP_CBRA:
600 case OP_CBRAPOS:
601 case OP_SCBRA:
602 case OP_SCBRAPOS:
603 return cc + 1 + LINK_SIZE + 2;
604
605 default:
606 return NULL;
607 }
608 }
609
610 static int get_localspace(compiler_common *common, uschar *cc, uschar *ccend)
611 {
612 int localspace = 0;
613 uschar *alternative;
614 /* Calculate important variables (like stack size) and checks whether all opcodes are supported. */
615 while (cc < ccend)
616 {
617 switch(*cc)
618 {
619 case OP_ASSERT:
620 case OP_ASSERT_NOT:
621 case OP_ASSERTBACK:
622 case OP_ASSERTBACK_NOT:
623 case OP_ONCE:
624 case OP_ONCE_NC:
625 case OP_BRAPOS:
626 case OP_SBRA:
627 case OP_SBRAPOS:
628 case OP_SCOND:
629 localspace += sizeof(sljit_w);
630 cc += 1 + LINK_SIZE;
631 break;
632
633 case OP_CBRAPOS:
634 case OP_SCBRAPOS:
635 localspace += sizeof(sljit_w);
636 cc += 1 + LINK_SIZE + 2;
637 break;
638
639 case OP_COND:
640 /* Might be a hidden SCOND. */
641 alternative = cc + GET(cc, 1);
642 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
643 localspace += sizeof(sljit_w);
644 cc += 1 + LINK_SIZE;
645 break;
646
647 default:
648 cc = next_opcode(common, cc);
649 if (cc == NULL)
650 return -1;
651 break;
652 }
653 }
654 return localspace;
655 }
656
657 static void set_localptrs(compiler_common *common, int localptr, uschar *ccend)
658 {
659 uschar *cc = common->start;
660 uschar *alternative;
661 while (cc < ccend)
662 {
663 switch(*cc)
664 {
665 case OP_ASSERT:
666 case OP_ASSERT_NOT:
667 case OP_ASSERTBACK:
668 case OP_ASSERTBACK_NOT:
669 case OP_ONCE:
670 case OP_ONCE_NC:
671 case OP_BRAPOS:
672 case OP_SBRA:
673 case OP_SBRAPOS:
674 case OP_SCOND:
675 common->localptrs[cc - common->start] = localptr;
676 localptr += sizeof(sljit_w);
677 cc += 1 + LINK_SIZE;
678 break;
679
680 case OP_CBRAPOS:
681 case OP_SCBRAPOS:
682 common->localptrs[cc - common->start] = localptr;
683 localptr += sizeof(sljit_w);
684 cc += 1 + LINK_SIZE + 2;
685 break;
686
687 case OP_COND:
688 /* Might be a hidden SCOND. */
689 alternative = cc + GET(cc, 1);
690 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
691 {
692 common->localptrs[cc - common->start] = localptr;
693 localptr += sizeof(sljit_w);
694 }
695 cc += 1 + LINK_SIZE;
696 break;
697
698 default:
699 cc = next_opcode(common, cc);
700 SLJIT_ASSERT(cc != NULL);
701 break;
702 }
703 }
704 }
705
706 /* Returns with -1 if no need for frame. */
707 static int get_framesize(compiler_common *common, uschar *cc, BOOL recursive)
708 {
709 uschar *ccend = bracketend(cc);
710 int length = 0;
711 BOOL possessive = FALSE;
712 BOOL setsom_found = FALSE;
713
714 if (!recursive && (*cc == OP_CBRAPOS || *cc == OP_SCBRAPOS))
715 {
716 length = 3;
717 possessive = TRUE;
718 }
719
720 cc = next_opcode(common, cc);
721 SLJIT_ASSERT(cc != NULL);
722 while (cc < ccend)
723 switch(*cc)
724 {
725 case OP_SET_SOM:
726 case OP_RECURSE:
727 if (!setsom_found)
728 {
729 length += 2;
730 setsom_found = TRUE;
731 }
732 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
733 break;
734
735 case OP_CBRA:
736 case OP_CBRAPOS:
737 case OP_SCBRA:
738 case OP_SCBRAPOS:
739 length += 3;
740 cc += 1 + LINK_SIZE + 2;
741 break;
742
743 default:
744 cc = next_opcode(common, cc);
745 SLJIT_ASSERT(cc != NULL);
746 break;
747 }
748
749 /* Possessive quantifiers can use a special case. */
750 if (SLJIT_UNLIKELY(possessive) && length == 3)
751 return -1;
752
753 if (length > 0)
754 return length + 1;
755 return -1;
756 }
757
758 static void init_frame(compiler_common *common, uschar *cc, int stackpos, int stacktop, BOOL recursive)
759 {
760 DEFINE_COMPILER;
761 uschar *ccend = bracketend(cc);
762 BOOL setsom_found = FALSE;
763 int offset;
764
765 /* >= 1 + shortest item size (2) */
766 SLJIT_ASSERT(stackpos >= stacktop + 2);
767
768 stackpos = STACK(stackpos);
769 if (recursive || (*cc != OP_CBRAPOS && *cc != OP_SCBRAPOS))
770 cc = next_opcode(common, cc);
771 SLJIT_ASSERT(cc != NULL);
772 while (cc < ccend)
773 switch(*cc)
774 {
775 case OP_SET_SOM:
776 case OP_RECURSE:
777 if (!setsom_found)
778 {
779 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
780 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_setstrbegin);
781 stackpos += (int)sizeof(sljit_w);
782 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
783 stackpos += (int)sizeof(sljit_w);
784 setsom_found = TRUE;
785 }
786 cc += (*cc == OP_SET_SOM) ? 1 : 1 + LINK_SIZE;
787 break;
788
789 case OP_CBRA:
790 case OP_CBRAPOS:
791 case OP_SCBRA:
792 case OP_SCBRAPOS:
793 offset = (GET2(cc, 1 + LINK_SIZE)) << 1;
794 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset));
795 stackpos += (int)sizeof(sljit_w);
796 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
797 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
798 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
799 stackpos += (int)sizeof(sljit_w);
800 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0);
801 stackpos += (int)sizeof(sljit_w);
802
803 cc += 1 + LINK_SIZE + 2;
804 break;
805
806 default:
807 cc = next_opcode(common, cc);
808 SLJIT_ASSERT(cc != NULL);
809 break;
810 }
811
812 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, frame_end);
813 SLJIT_ASSERT(stackpos == STACK(stacktop));
814 }
815
816 static SLJIT_INLINE int get_localsize(compiler_common *common, uschar *cc, uschar *ccend)
817 {
818 int localsize = 2;
819 uschar *alternative;
820 /* Calculate the sum of the local variables. */
821 while (cc < ccend)
822 {
823 switch(*cc)
824 {
825 case OP_ASSERT:
826 case OP_ASSERT_NOT:
827 case OP_ASSERTBACK:
828 case OP_ASSERTBACK_NOT:
829 case OP_ONCE:
830 case OP_ONCE_NC:
831 case OP_BRAPOS:
832 case OP_SBRA:
833 case OP_SBRAPOS:
834 case OP_SCOND:
835 localsize++;
836 cc += 1 + LINK_SIZE;
837 break;
838
839 case OP_CBRA:
840 case OP_SCBRA:
841 localsize++;
842 cc += 1 + LINK_SIZE + 2;
843 break;
844
845 case OP_CBRAPOS:
846 case OP_SCBRAPOS:
847 localsize += 2;
848 cc += 1 + LINK_SIZE + 2;
849 break;
850
851 case OP_COND:
852 /* Might be a hidden SCOND. */
853 alternative = cc + GET(cc, 1);
854 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
855 localsize++;
856 cc += 1 + LINK_SIZE;
857 break;
858
859 default:
860 cc = next_opcode(common, cc);
861 SLJIT_ASSERT(cc != NULL);
862 break;
863 }
864 }
865 SLJIT_ASSERT(cc == ccend);
866 return localsize;
867 }
868
869 static void copy_locals(compiler_common *common, uschar *cc, uschar *ccend,
870 BOOL save, int stackptr, int stacktop)
871 {
872 DEFINE_COMPILER;
873 int srcw[2];
874 int count;
875 BOOL tmp1next = TRUE;
876 BOOL tmp1empty = TRUE;
877 BOOL tmp2empty = TRUE;
878 uschar *alternative;
879 enum {
880 start,
881 loop,
882 end
883 } status;
884
885 status = save ? start : loop;
886 stackptr = STACK(stackptr - 2);
887 stacktop = STACK(stacktop - 1);
888
889 if (!save)
890 {
891 stackptr += sizeof(sljit_w);
892 if (stackptr < stacktop)
893 {
894 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
895 stackptr += sizeof(sljit_w);
896 tmp1empty = FALSE;
897 }
898 if (stackptr < stacktop)
899 {
900 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
901 stackptr += sizeof(sljit_w);
902 tmp2empty = FALSE;
903 }
904 /* The tmp1next must be TRUE in either way. */
905 }
906
907 while (status != end)
908 {
909 count = 0;
910 switch(status)
911 {
912 case start:
913 SLJIT_ASSERT(save);
914 count = 1;
915 srcw[0] = RECURSIVE_HEAD;
916 status = loop;
917 break;
918
919 case loop:
920 if (cc >= ccend)
921 {
922 status = end;
923 break;
924 }
925
926 switch(*cc)
927 {
928 case OP_ASSERT:
929 case OP_ASSERT_NOT:
930 case OP_ASSERTBACK:
931 case OP_ASSERTBACK_NOT:
932 case OP_ONCE:
933 case OP_ONCE_NC:
934 case OP_BRAPOS:
935 case OP_SBRA:
936 case OP_SBRAPOS:
937 case OP_SCOND:
938 count = 1;
939 srcw[0] = PRIV(cc);
940 SLJIT_ASSERT(srcw[0] != 0);
941 cc += 1 + LINK_SIZE;
942 break;
943
944 case OP_CBRA:
945 case OP_SCBRA:
946 count = 1;
947 srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
948 cc += 1 + LINK_SIZE + 2;
949 break;
950
951 case OP_CBRAPOS:
952 case OP_SCBRAPOS:
953 count = 2;
954 srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
955 srcw[0] = PRIV(cc);
956 SLJIT_ASSERT(srcw[0] != 0);
957 cc += 1 + LINK_SIZE + 2;
958 break;
959
960 case OP_COND:
961 /* Might be a hidden SCOND. */
962 alternative = cc + GET(cc, 1);
963 if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
964 {
965 count = 1;
966 srcw[0] = PRIV(cc);
967 SLJIT_ASSERT(srcw[0] != 0);
968 }
969 cc += 1 + LINK_SIZE;
970 break;
971
972 default:
973 cc = next_opcode(common, cc);
974 SLJIT_ASSERT(cc != NULL);
975 break;
976 }
977 break;
978
979 case end:
980 SLJIT_ASSERT_STOP();
981 break;
982 }
983
984 while (count > 0)
985 {
986 count--;
987 if (save)
988 {
989 if (tmp1next)
990 {
991 if (!tmp1empty)
992 {
993 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
994 stackptr += sizeof(sljit_w);
995 }
996 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
997 tmp1empty = FALSE;
998 tmp1next = FALSE;
999 }
1000 else
1001 {
1002 if (!tmp2empty)
1003 {
1004 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1005 stackptr += sizeof(sljit_w);
1006 }
1007 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
1008 tmp2empty = FALSE;
1009 tmp1next = TRUE;
1010 }
1011 }
1012 else
1013 {
1014 if (tmp1next)
1015 {
1016 SLJIT_ASSERT(!tmp1empty);
1017 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP1, 0);
1018 tmp1empty = stackptr >= stacktop;
1019 if (!tmp1empty)
1020 {
1021 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1022 stackptr += sizeof(sljit_w);
1023 }
1024 tmp1next = FALSE;
1025 }
1026 else
1027 {
1028 SLJIT_ASSERT(!tmp2empty);
1029 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP2, 0);
1030 tmp2empty = stackptr >= stacktop;
1031 if (!tmp2empty)
1032 {
1033 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr);
1034 stackptr += sizeof(sljit_w);
1035 }
1036 tmp1next = TRUE;
1037 }
1038 }
1039 }
1040 }
1041
1042 if (save)
1043 {
1044 if (tmp1next)
1045 {
1046 if (!tmp1empty)
1047 {
1048 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1049 stackptr += sizeof(sljit_w);
1050 }
1051 if (!tmp2empty)
1052 {
1053 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1054 stackptr += sizeof(sljit_w);
1055 }
1056 }
1057 else
1058 {
1059 if (!tmp2empty)
1060 {
1061 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
1062 stackptr += sizeof(sljit_w);
1063 }
1064 if (!tmp1empty)
1065 {
1066 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
1067 stackptr += sizeof(sljit_w);
1068 }
1069 }
1070 }
1071 SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty)));
1072 }
1073
1074 static SLJIT_INLINE BOOL ispowerof2(unsigned int value)
1075 {
1076 return (value & (value - 1)) == 0;
1077 }
1078
1079 static SLJIT_INLINE void set_jumps(jump_list *list, struct sljit_label *label)
1080 {
1081 while (list)
1082 {
1083 /* sljit_set_label is clever enough to do nothing
1084 if either the jump or the label is NULL */
1085 sljit_set_label(list->jump, label);
1086 list = list->next;
1087 }
1088 }
1089
1090 static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump* jump)
1091 {
1092 jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list));
1093 if (list_item)
1094 {
1095 list_item->next = *list;
1096 list_item->jump = jump;
1097 *list = list_item;
1098 }
1099 }
1100
1101 static void add_stub(compiler_common *common, enum stub_types type, int data, struct sljit_jump *start)
1102 {
1103 DEFINE_COMPILER;
1104 stub_list* list_item = sljit_alloc_memory(compiler, sizeof(stub_list));
1105
1106 if (list_item)
1107 {
1108 list_item->type = type;
1109 list_item->data = data;
1110 list_item->start = start;
1111 list_item->leave = LABEL();
1112 list_item->next = common->stubs;
1113 common->stubs = list_item;
1114 }
1115 }
1116
1117 static void flush_stubs(compiler_common *common)
1118 {
1119 DEFINE_COMPILER;
1120 stub_list* list_item = common->stubs;
1121
1122 while (list_item)
1123 {
1124 JUMPHERE(list_item->start);
1125 switch(list_item->type)
1126 {
1127 case stack_alloc:
1128 add_jump(compiler, &common->stackalloc, JUMP(SLJIT_FAST_CALL));
1129 break;
1130 }
1131 JUMPTO(SLJIT_JUMP, list_item->leave);
1132 list_item = list_item->next;
1133 }
1134 common->stubs = NULL;
1135 }
1136
1137 static SLJIT_INLINE void decrease_call_count(compiler_common *common)
1138 {
1139 DEFINE_COMPILER;
1140
1141 OP2(SLJIT_SUB | SLJIT_SET_E, CALL_COUNT, 0, CALL_COUNT, 0, SLJIT_IMM, 1);
1142 add_jump(compiler, &common->calllimit, JUMP(SLJIT_C_ZERO));
1143 }
1144
1145 static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
1146 {
1147 /* May destroy all locals and registers except TMP2. */
1148 DEFINE_COMPILER;
1149
1150 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1151 #ifdef DESTROY_REGISTERS
1152 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345);
1153 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
1154 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
1155 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, TMP1, 0);
1156 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
1157 #endif
1158 add_stub(common, stack_alloc, 0, CMP(SLJIT_C_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
1159 }
1160
1161 static SLJIT_INLINE void free_stack(compiler_common *common, int size)
1162 {
1163 DEFINE_COMPILER;
1164 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_w));
1165 }
1166
1167 static SLJIT_INLINE void reset_ovector(compiler_common *common, int length)
1168 {
1169 DEFINE_COMPILER;
1170 struct sljit_label *loop;
1171 int i;
1172 /* At this point we can freely use all temporary registers. */
1173 /* TMP1 returns with begin - 1. */
1174 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, 1);
1175 if (length < 8)
1176 {
1177 for (i = 0; i < length; i++)
1178 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(i), SLJIT_TEMPORARY_REG1, 0);
1179 }
1180 else
1181 {
1182 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START - sizeof(sljit_w));
1183 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, length);
1184 loop = LABEL();
1185 OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), sizeof(sljit_w), SLJIT_TEMPORARY_REG1, 0);
1186 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG3, 0, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, 1);
1187 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1188 }
1189 }
1190
1191 static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket)
1192 {
1193 DEFINE_COMPILER;
1194 struct sljit_label *loop;
1195 struct sljit_jump *earlyexit;
1196
1197 /* At this point we can freely use all registers. */
1198 OP1(SLJIT_MOV, SLJIT_GENERAL_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
1199 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1), STR_PTR, 0);
1200
1201 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, ARGUMENTS, 0);
1202 OP1(SLJIT_MOV_SI, SLJIT_TEMPORARY_REG2, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsetcount));
1203 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
1204 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), SLJIT_OFFSETOF(jit_arguments, begin));
1205 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START);
1206 /* Unlikely, but possible */
1207 earlyexit = CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 0);
1208 loop = LABEL();
1209 OP2(SLJIT_SUB, SLJIT_GENERAL_REG2, 0, SLJIT_MEM1(SLJIT_GENERAL_REG1), 0, SLJIT_TEMPORARY_REG1, 0);
1210 OP2(SLJIT_ADD, SLJIT_GENERAL_REG1, 0, SLJIT_GENERAL_REG1, 0, SLJIT_IMM, sizeof(sljit_w));
1211 /* Copy the integer value to the output buffer */
1212 OP1(SLJIT_MOVU_SI, SLJIT_MEM1(SLJIT_TEMPORARY_REG3), sizeof(int), SLJIT_GENERAL_REG2, 0);
1213 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1214 JUMPTO(SLJIT_C_NOT_ZERO, loop);
1215 JUMPHERE(earlyexit);
1216
1217 /* Calculate the return value, which is the maximum ovector value. */
1218 if (topbracket > 1)
1219 {
1220 OP2(SLJIT_ADD, SLJIT_TEMPORARY_REG1, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, OVECTOR_START + topbracket * 2 * sizeof(sljit_w));
1221 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, topbracket + 1);
1222
1223 /* OVECTOR(0) is never equal to SLJIT_GENERAL_REG3. */
1224 loop = LABEL();
1225 OP1(SLJIT_MOVU, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_TEMPORARY_REG1), -(2 * (sljit_w)sizeof(sljit_w)));
1226 OP2(SLJIT_SUB, SLJIT_TEMPORARY_REG2, 0, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, 1);
1227 CMPTO(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG3, 0, SLJIT_GENERAL_REG3, 0, loop);
1228 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_TEMPORARY_REG2, 0);
1229 }
1230 else
1231 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
1232 }
1233
1234 static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, uschar* cc)
1235 {
1236 /* Detects if the character has an othercase. */
1237 unsigned int c;
1238
1239 #ifdef SUPPORT_UTF8
1240 if (common->utf8)
1241 {
1242 GETCHAR(c, cc);
1243 if (c > 127)
1244 {
1245 #ifdef SUPPORT_UCP
1246 return c != UCD_OTHERCASE(c);
1247 #else
1248 return FALSE;
1249 #endif
1250 }
1251 }
1252 else
1253 #endif
1254 c = *cc;
1255 return common->fcc[c] != c;
1256 }
1257
1258 static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigned int c)
1259 {
1260 /* Returns with the othercase. */
1261 #ifdef SUPPORT_UTF8
1262 if (common->utf8 && c > 127)
1263 {
1264 #ifdef SUPPORT_UCP
1265 return UCD_OTHERCASE(c);
1266 #else
1267 return c;
1268 #endif
1269 }
1270 #endif
1271 return common->fcc[c];
1272 }
1273
1274 static unsigned int char_get_othercase_bit(compiler_common *common, uschar* cc)
1275 {
1276 /* Detects if the character and its othercase has only 1 bit difference. */
1277 unsigned int c, oc, bit;
1278 #ifdef SUPPORT_UTF8
1279 int n;
1280 #endif
1281
1282 #ifdef SUPPORT_UTF8
1283 if (common->utf8)
1284 {
1285 GETCHAR(c, cc);
1286 if (c <= 127)
1287 oc = common->fcc[c];
1288 else
1289 {
1290 #ifdef SUPPORT_UCP
1291 oc = UCD_OTHERCASE(c);
1292 #else
1293 oc = c;
1294 #endif
1295 }
1296 }
1297 else
1298 {
1299 c = *cc;
1300 oc = common->fcc[c];
1301 }
1302 #else
1303 c = *cc;
1304 oc = common->fcc[c];
1305 #endif
1306
1307 SLJIT_ASSERT(c != oc);
1308
1309 bit = c ^ oc;
1310 /* Optimized for English alphabet. */
1311 if (c <= 127 && bit == 0x20)
1312 return (0 << 8) | 0x20;
1313
1314 /* Since c != oc, they must have at least 1 bit difference. */
1315 if (!ispowerof2(bit))
1316 return 0;
1317
1318 #ifdef SUPPORT_UTF8
1319 if (common->utf8 && c > 127)
1320 {
1321 n = _pcre_utf8_table4[*cc & 0x3f];
1322 while ((bit & 0x3f) == 0)
1323 {
1324 n--;
1325 bit >>= 6;
1326 }
1327 return (n << 8) | bit;
1328 }
1329 #endif
1330 return (0 << 8) | bit;
1331 }
1332
1333 static SLJIT_INLINE void check_input_end(compiler_common *common, jump_list **fallbacks)
1334 {
1335 DEFINE_COMPILER;
1336 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1337 }
1338
1339 static void read_char(compiler_common *common)
1340 {
1341 /* Reads the character into TMP1, updates STR_PTR.
1342 Does not check STR_END. TMP2 Destroyed. */
1343 DEFINE_COMPILER;
1344 #ifdef SUPPORT_UTF8
1345 struct sljit_jump *jump;
1346 #endif
1347
1348 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1349 #ifdef SUPPORT_UTF8
1350 if (common->utf8)
1351 {
1352 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1353 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1354 JUMPHERE(jump);
1355 }
1356 #endif
1357 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1358 }
1359
1360 static void peek_char(compiler_common *common)
1361 {
1362 /* Reads the character into TMP1, keeps STR_PTR.
1363 Does not check STR_END. TMP2 Destroyed. */
1364 DEFINE_COMPILER;
1365 #ifdef SUPPORT_UTF8
1366 struct sljit_jump *jump;
1367 #endif
1368
1369 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1370 #ifdef SUPPORT_UTF8
1371 if (common->utf8)
1372 {
1373 jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1374 add_jump(compiler, &common->utf8readchar, JUMP(SLJIT_FAST_CALL));
1375 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1376 JUMPHERE(jump);
1377 }
1378 #endif
1379 }
1380
1381 static void read_char8_type(compiler_common *common)
1382 {
1383 /* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
1384 DEFINE_COMPILER;
1385 #ifdef SUPPORT_UTF8
1386 struct sljit_jump *jump;
1387 #endif
1388
1389 #ifdef SUPPORT_UTF8
1390 if (common->utf8)
1391 {
1392 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1393 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1394 /* This can be an extra read in some situations, but hopefully
1395 it is a clever early read in most cases. */
1396 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1397 jump = CMP(SLJIT_C_LESS, TMP2, 0, SLJIT_IMM, 0xc0);
1398 add_jump(compiler, &common->utf8readtype8, JUMP(SLJIT_FAST_CALL));
1399 JUMPHERE(jump);
1400 return;
1401 }
1402 #endif
1403 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1404 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1405 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
1406 }
1407
1408 static void skip_char_back(compiler_common *common)
1409 {
1410 /* Goes one character back. Only affects STR_PTR. Does not check begin. */
1411 DEFINE_COMPILER;
1412 #ifdef SUPPORT_UTF8
1413 struct sljit_label *label;
1414
1415 if (common->utf8)
1416 {
1417 label = LABEL();
1418 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1419 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1420 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
1421 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label);
1422 return;
1423 }
1424 #endif
1425 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1426 }
1427
1428 static void check_newlinechar(compiler_common *common, int nltype, jump_list **fallbacks, BOOL jumpiftrue)
1429 {
1430 /* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */
1431 DEFINE_COMPILER;
1432
1433 if (nltype == NLTYPE_ANY)
1434 {
1435 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
1436 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1437 }
1438 else if (nltype == NLTYPE_ANYCRLF)
1439 {
1440 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_CR);
1441 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1442 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1443 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1444 add_jump(compiler, fallbacks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
1445 }
1446 else
1447 {
1448 SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline <= 255);
1449 add_jump(compiler, fallbacks, CMP(jumpiftrue ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
1450 }
1451 }
1452
1453 #ifdef SUPPORT_UTF8
1454 static void do_utf8readchar(compiler_common *common)
1455 {
1456 /* Fast decoding an utf8 character. TMP1 contains the first byte
1457 of the character (>= 0xc0). Return char value in TMP1, length - 1 in TMP2. */
1458 DEFINE_COMPILER;
1459 struct sljit_jump *jump;
1460
1461 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1462 /* Searching for the first zero. */
1463 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
1464 jump = JUMP(SLJIT_C_NOT_ZERO);
1465 /* 2 byte sequence */
1466 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1467 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1468 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1f);
1469 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
1470 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1471 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1472 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
1473 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1474 JUMPHERE(jump);
1475
1476 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10);
1477 jump = JUMP(SLJIT_C_NOT_ZERO);
1478 /* 3 byte sequence */
1479 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1480 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0f);
1481 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 12);
1482 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1483 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1484 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1485 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1486 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 2);
1487 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1488 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1489 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 2);
1490 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1491 JUMPHERE(jump);
1492
1493 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x08);
1494 jump = JUMP(SLJIT_C_NOT_ZERO);
1495 /* 4 byte sequence */
1496 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1497 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x07);
1498 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 18);
1499 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1500 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1501 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1502 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1503 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1504 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1505 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1506 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1507 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 3);
1508 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1509 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1510 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 3);
1511 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1512 JUMPHERE(jump);
1513
1514 /* 5 byte sequence */
1515 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
1516 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x03);
1517 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 24);
1518 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1519 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 18);
1520 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1521 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 2);
1522 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1523 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
1524 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1525 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 3);
1526 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1527 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1528 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1529 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 4);
1530 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 4);
1531 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
1532 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
1533 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 4);
1534 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1535 }
1536
1537 static void do_utf8readtype8(compiler_common *common)
1538 {
1539 /* Fast decoding an utf8 character type. TMP2 contains the first byte
1540 of the character (>= 0xc0) and TMP1 is destroyed. Return value in TMP1. */
1541 DEFINE_COMPILER;
1542 struct sljit_jump *jump;
1543 struct sljit_jump *compare;
1544
1545 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1546
1547 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
1548 jump = JUMP(SLJIT_C_NOT_ZERO);
1549 /* 2 byte sequence */
1550 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1551 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1552 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f);
1553 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
1554 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
1555 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0);
1556 compare = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
1557 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
1558 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1559
1560 JUMPHERE(compare);
1561 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1562 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1563 JUMPHERE(jump);
1564
1565 /* We only have types for characters less than 256. */
1566 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1567 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1568 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
1569 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1570 }
1571
1572 #endif
1573
1574 #ifdef SUPPORT_UCP
1575
1576 /* UCD_BLOCK_SIZE must be 128 (see the assert below). */
1577 #define UCD_BLOCK_MASK 127
1578 #define UCD_BLOCK_SHIFT 7
1579
1580 static void do_getucd(compiler_common *common)
1581 {
1582 /* Search the UCD record for the character comes in TMP1.
1583 Returns chartype in TMP1 and UCD offset in TMP2. */
1584 DEFINE_COMPILER;
1585
1586 SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8);
1587
1588 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1589 OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1590 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_w)_pcre_ucd_stage1);
1591 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
1592 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
1593 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
1594 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_stage2);
1595 OP1(SLJIT_MOV_UH, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
1596 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, chartype));
1597 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
1598 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1599 }
1600 #endif
1601
1602 static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf, BOOL firstline)
1603 {
1604 DEFINE_COMPILER;
1605 struct sljit_label *mainloop;
1606 struct sljit_label *newlinelabel = NULL;
1607 struct sljit_jump *start;
1608 struct sljit_jump *end = NULL;
1609 struct sljit_jump *nl = NULL;
1610 #ifdef SUPPORT_UTF8
1611 struct sljit_jump *singlebyte;
1612 #endif
1613 jump_list *newline = NULL;
1614 BOOL newlinecheck = FALSE;
1615 BOOL readbyte = FALSE;
1616
1617 if (!(hascrorlf || firstline) && (common->nltype == NLTYPE_ANY ||
1618 common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
1619 newlinecheck = TRUE;
1620
1621 if (firstline)
1622 {
1623 /* Search for the end of the first line. */
1624 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STR_PTR, 0);
1625 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_END, 0);
1626
1627 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1628 {
1629 mainloop = LABEL();
1630 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1631 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1632 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
1633 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
1634 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
1635 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
1636 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0, SLJIT_IMM, 1);
1637 }
1638 else
1639 {
1640 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1641 mainloop = LABEL();
1642 /* Continual stores does not cause data dependency. */
1643 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1644 read_char(common);
1645 check_newlinechar(common, common->nltype, &newline, TRUE);
1646 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
1647 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, STR_PTR, 0);
1648 set_jumps(newline, LABEL());
1649 }
1650
1651 JUMPHERE(end);
1652 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
1653 }
1654
1655 start = JUMP(SLJIT_JUMP);
1656
1657 if (newlinecheck)
1658 {
1659 newlinelabel = LABEL();
1660 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1661 end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1662 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1663 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
1664 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1665 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1666 nl = JUMP(SLJIT_JUMP);
1667 }
1668
1669 mainloop = LABEL();
1670
1671 /* Increasing the STR_PTR here requires one less jump in the most common case. */
1672 #ifdef SUPPORT_UTF8
1673 if (common->utf8) readbyte = TRUE;
1674 #endif
1675 if (newlinecheck) readbyte = TRUE;
1676
1677 if (readbyte)
1678 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1679
1680 if (newlinecheck)
1681 CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);
1682
1683 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1684 #ifdef SUPPORT_UTF8
1685 if (common->utf8)
1686 {
1687 singlebyte = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
1688 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1689 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1690 JUMPHERE(singlebyte);
1691 }
1692 #endif
1693 JUMPHERE(start);
1694
1695 if (newlinecheck)
1696 {
1697 JUMPHERE(end);
1698 JUMPHERE(nl);
1699 }
1700
1701 return mainloop;
1702 }
1703
1704 static SLJIT_INLINE void fast_forward_first_byte(compiler_common *common, pcre_uint16 firstbyte, BOOL firstline)
1705 {
1706 DEFINE_COMPILER;
1707 struct sljit_label *start;
1708 struct sljit_jump *leave;
1709 struct sljit_jump *found;
1710 pcre_uint16 oc, bit;
1711
1712 if (firstline)
1713 {
1714 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1715 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1716 }
1717
1718 start = LABEL();
1719 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1720 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1721
1722 if ((firstbyte & REQ_CASELESS) == 0)
1723 found = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, firstbyte & 0xff);
1724 else
1725 {
1726 firstbyte &= 0xff;
1727 oc = common->fcc[firstbyte];
1728 bit = firstbyte ^ oc;
1729 if (ispowerof2(bit))
1730 {
1731 OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, bit);
1732 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, firstbyte | bit);
1733 }
1734 else
1735 {
1736 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, firstbyte);
1737 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
1738 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc);
1739 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
1740 found = JUMP(SLJIT_C_NOT_ZERO);
1741 }
1742 }
1743
1744 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1745 #ifdef SUPPORT_UTF8
1746 if (common->utf8)
1747 {
1748 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
1749 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1750 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1751 }
1752 #endif
1753 JUMPTO(SLJIT_JUMP, start);
1754 JUMPHERE(found);
1755 JUMPHERE(leave);
1756
1757 if (firstline)
1758 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1759 }
1760
1761 static SLJIT_INLINE void fast_forward_newline(compiler_common *common, BOOL firstline)
1762 {
1763 DEFINE_COMPILER;
1764 struct sljit_label *loop;
1765 struct sljit_jump *lastchar;
1766 struct sljit_jump *firstchar;
1767 struct sljit_jump *leave;
1768 struct sljit_jump *foundcr = NULL;
1769 struct sljit_jump *notfoundnl;
1770 jump_list *newline = NULL;
1771
1772 if (firstline)
1773 {
1774 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1775 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1776 }
1777
1778 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
1779 {
1780 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1781 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1782 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1783 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
1784 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1785
1786 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2);
1787 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
1788 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER_EQUAL);
1789 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1790
1791 loop = LABEL();
1792 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1793 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1794 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
1795 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
1796 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
1797 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);
1798
1799 JUMPHERE(leave);
1800 JUMPHERE(firstchar);
1801 JUMPHERE(lastchar);
1802
1803 if (firstline)
1804 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1805 return;
1806 }
1807
1808 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1809 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
1810 firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
1811 skip_char_back(common);
1812
1813 loop = LABEL();
1814 read_char(common);
1815 lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1816 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1817 foundcr = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
1818 check_newlinechar(common, common->nltype, &newline, FALSE);
1819 set_jumps(newline, loop);
1820
1821 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
1822 {
1823 leave = JUMP(SLJIT_JUMP);
1824 JUMPHERE(foundcr);
1825 notfoundnl = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1826 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1827 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
1828 COND_VALUE(SLJIT_MOV, TMP1, 0, SLJIT_C_EQUAL);
1829 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1830 JUMPHERE(notfoundnl);
1831 JUMPHERE(leave);
1832 }
1833 JUMPHERE(lastchar);
1834 JUMPHERE(firstchar);
1835
1836 if (firstline)
1837 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1838 }
1839
1840 static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, sljit_uw start_bits, BOOL firstline)
1841 {
1842 DEFINE_COMPILER;
1843 struct sljit_label *start;
1844 struct sljit_jump *leave;
1845 struct sljit_jump *found;
1846
1847 if (firstline)
1848 {
1849 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, STR_END, 0);
1850 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
1851 }
1852
1853 start = LABEL();
1854 leave = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1855 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
1856 #ifdef SUPPORT_UTF8
1857 if (common->utf8)
1858 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
1859 #endif
1860 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
1861 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
1862 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), start_bits);
1863 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
1864 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
1865 found = JUMP(SLJIT_C_NOT_ZERO);
1866
1867 #ifdef SUPPORT_UTF8
1868 if (common->utf8)
1869 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
1870 #endif
1871 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
1872 #ifdef SUPPORT_UTF8
1873 if (common->utf8)
1874 {
1875 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
1876 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
1877 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1878 }
1879 #endif
1880 JUMPTO(SLJIT_JUMP, start);
1881 JUMPHERE(found);
1882 JUMPHERE(leave);
1883
1884 if (firstline)
1885 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
1886 }
1887
1888 static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, pcre_uint16 reqbyte, BOOL has_firstbyte)
1889 {
1890 DEFINE_COMPILER;
1891 struct sljit_label *loop;
1892 struct sljit_jump *toolong;
1893 struct sljit_jump *alreadyfound;
1894 struct sljit_jump *found;
1895 struct sljit_jump *foundoc = NULL;
1896 struct sljit_jump *notfound;
1897 pcre_uint16 oc, bit;
1898
1899 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR);
1900 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX);
1901 toolong = CMP(SLJIT_C_LESS, TMP1, 0, STR_END, 0);
1902 alreadyfound = CMP(SLJIT_C_LESS, STR_PTR, 0, TMP2, 0);
1903
1904 if (has_firstbyte)
1905 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, 1);
1906 else
1907 OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0);
1908
1909 loop = LABEL();
1910 notfound = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, STR_END, 0);
1911
1912 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1913 if ((reqbyte & REQ_CASELESS) == 0)
1914 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte & 0xff);
1915 else
1916 {
1917 reqbyte &= 0xff;
1918 oc = common->fcc[reqbyte];
1919 bit = reqbyte ^ oc;
1920 if (ispowerof2(bit))
1921 {
1922 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit);
1923 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte | bit);
1924 }
1925 else
1926 {
1927 found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, reqbyte);
1928 foundoc = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, oc);
1929 }
1930 }
1931 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
1932 JUMPTO(SLJIT_JUMP, loop);
1933
1934 JUMPHERE(found);
1935 if (foundoc)
1936 JUMPHERE(foundoc);
1937 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, TMP1, 0);
1938 JUMPHERE(alreadyfound);
1939 JUMPHERE(toolong);
1940 return notfound;
1941 }
1942
1943 static void do_revertframes(compiler_common *common)
1944 {
1945 DEFINE_COMPILER;
1946 struct sljit_jump *jump;
1947 struct sljit_label *mainloop;
1948
1949 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
1950 OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0);
1951
1952 /* Drop frames until we reach STACK_TOP. */
1953 mainloop = LABEL();
1954 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0);
1955 jump = CMP(SLJIT_C_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1956 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_LOCALS_REG, 0);
1957 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1958 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_w), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_w));
1959 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_w));
1960 JUMPTO(SLJIT_JUMP, mainloop);
1961
1962 JUMPHERE(jump);
1963 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_end);
1964 /* End of dropping frames. */
1965 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
1966
1967 JUMPHERE(jump);
1968 jump = CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, frame_setstrbegin);
1969 /* Set string begin. */
1970 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), sizeof(sljit_w));
1971 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
1972 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
1973 JUMPTO(SLJIT_JUMP, mainloop);
1974
1975 JUMPHERE(jump);
1976 /* Unknown command. */
1977 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_w));
1978 JUMPTO(SLJIT_JUMP, mainloop);
1979 }
1980
1981 static void check_wordboundary(compiler_common *common)
1982 {
1983 DEFINE_COMPILER;
1984 struct sljit_jump *beginend;
1985 #ifdef SUPPORT_UTF8
1986 struct sljit_jump *jump;
1987 #endif
1988
1989 SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16);
1990
1991 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
1992 /* Get type of the previous char, and put it to LOCALS1. */
1993 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
1994 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
1995 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, 0);
1996 beginend = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
1997 skip_char_back(common);
1998 read_char(common);
1999
2000 /* Testing char type. */
2001 #ifdef SUPPORT_UCP
2002 if (common->useucp)
2003 {
2004 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2005 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2006 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2007 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2008 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2009 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2010 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2011 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2012 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2013 JUMPHERE(jump);
2014 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
2015 }
2016 else
2017 #endif
2018 {
2019 #ifdef SUPPORT_UTF8
2020 /* Here LOCALS1 has already been zeroed. */
2021 jump = NULL;
2022 if (common->utf8)
2023 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2024 #endif
2025 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
2026 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */);
2027 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2028 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
2029 #ifdef SUPPORT_UTF8
2030 if (jump != NULL)
2031 JUMPHERE(jump);
2032 #endif
2033 }
2034 JUMPHERE(beginend);
2035
2036 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2037 beginend = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2038 peek_char(common);
2039
2040 /* Testing char type. This is a code duplication. */
2041 #ifdef SUPPORT_UCP
2042 if (common->useucp)
2043 {
2044 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
2045 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
2046 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2047 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
2048 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2049 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2050 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
2051 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2052 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2053 JUMPHERE(jump);
2054 }
2055 else
2056 #endif
2057 {
2058 #ifdef SUPPORT_UTF8
2059 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2060 jump = NULL;
2061 if (common->utf8)
2062 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2063 #endif
2064 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes);
2065 OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */);
2066 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2067 #ifdef SUPPORT_UTF8
2068 if (jump != NULL)
2069 JUMPHERE(jump);
2070 #endif
2071 }
2072 JUMPHERE(beginend);
2073
2074 OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2075 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2076 }
2077
2078 static void check_anynewline(compiler_common *common)
2079 {
2080 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2081 DEFINE_COMPILER;
2082
2083 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2084
2085 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2086 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2087 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2088 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2089 #ifdef SUPPORT_UTF8
2090 if (common->utf8)
2091 {
2092 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2093 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2094 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2095 }
2096 #endif
2097 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2098 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2099 }
2100
2101 static void check_hspace(compiler_common *common)
2102 {
2103 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2104 DEFINE_COMPILER;
2105
2106 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2107
2108 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
2109 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2110 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
2111 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2112 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
2113 #ifdef SUPPORT_UTF8
2114 if (common->utf8)
2115 {
2116 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2117 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
2118 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2119 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
2120 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2121 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000);
2122 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
2123 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2124 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
2125 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2126 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
2127 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2128 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
2129 }
2130 #endif
2131 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2132
2133 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2134 }
2135
2136 static void check_vspace(compiler_common *common)
2137 {
2138 /* Check whether TMP1 contains a newline character. TMP2 destroyed. */
2139 DEFINE_COMPILER;
2140
2141 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2142
2143 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
2144 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
2145 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2146 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
2147 #ifdef SUPPORT_UTF8
2148 if (common->utf8)
2149 {
2150 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2151 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
2152 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
2153 }
2154 #endif
2155 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2156
2157 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2158 }
2159
2160 #define CHAR1 STR_END
2161 #define CHAR2 STACK_TOP
2162
2163 static void do_casefulcmp(compiler_common *common)
2164 {
2165 DEFINE_COMPILER;
2166 struct sljit_jump *jump;
2167 struct sljit_label *label;
2168
2169 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2170 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2171 OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0);
2172 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR2, 0);
2173 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2174 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2175
2176 label = LABEL();
2177 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2178 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2179 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2180 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2181 JUMPTO(SLJIT_C_NOT_ZERO, label);
2182
2183 JUMPHERE(jump);
2184 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2185 OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0);
2186 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2187 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2188 }
2189
2190 #define LCC_TABLE STACK_LIMIT
2191
2192 static void do_caselesscmp(compiler_common *common)
2193 {
2194 DEFINE_COMPILER;
2195 struct sljit_jump *jump;
2196 struct sljit_label *label;
2197
2198 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0, 1, 5, 5, common->localsize);
2199 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
2200
2201 OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0);
2202 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR1, 0);
2203 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, CHAR2, 0);
2204 OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc);
2205 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
2206 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2207
2208 label = LABEL();
2209 OP1(SLJIT_MOVU_UB, CHAR1, 0, SLJIT_MEM1(TMP1), 1);
2210 OP1(SLJIT_MOVU_UB, CHAR2, 0, SLJIT_MEM1(STR_PTR), 1);
2211 OP1(SLJIT_MOV_UB, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
2212 OP1(SLJIT_MOV_UB, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
2213 jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
2214 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
2215 JUMPTO(SLJIT_C_NOT_ZERO, label);
2216
2217 JUMPHERE(jump);
2218 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2219 OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0);
2220 OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
2221 OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2222 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
2223 }
2224
2225 #undef LCC_TABLE
2226 #undef CHAR1
2227 #undef CHAR2
2228
2229 #ifdef SUPPORT_UTF8
2230 #ifdef SUPPORT_UCP
2231
2232 static uschar * SLJIT_CALL do_utf8caselesscmp(uschar *src1, jit_arguments *args, uschar *end1)
2233 {
2234 /* This function would be ineffective to do in JIT level. */
2235 int c1, c2;
2236 uschar *src2 = args->ptr;
2237 uschar *end2 = (uschar*)args->end;
2238
2239 while (src1 < end1)
2240 {
2241 if (src2 >= end2)
2242 return 0;
2243 GETCHARINC(c1, src1);
2244 GETCHARINC(c2, src2);
2245 if (c1 != c2 && c1 != UCD_OTHERCASE(c2)) return 0;
2246 }
2247 return src2;
2248 }
2249
2250 #endif
2251 #endif
2252
2253 static uschar *byte_sequence_compare(compiler_common *common, BOOL caseless, uschar *cc,
2254 compare_context* context, jump_list **fallbacks)
2255 {
2256 DEFINE_COMPILER;
2257 unsigned int othercasebit = 0;
2258 uschar *othercasebyte = NULL;
2259 #ifdef SUPPORT_UTF8
2260 int utf8length;
2261 #endif
2262
2263 if (caseless && char_has_othercase(common, cc))
2264 {
2265 othercasebit = char_get_othercase_bit(common, cc);
2266 SLJIT_ASSERT(othercasebit);
2267 /* Extracting bit difference info. */
2268 othercasebyte = cc + (othercasebit >> 8);
2269 othercasebit &= 0xff;
2270 }
2271
2272 if (context->sourcereg == -1)
2273 {
2274 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2275 if (context->length >= 4)
2276 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2277 else if (context->length >= 2)
2278 OP1(SLJIT_MOV_SH, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2279 else
2280 #endif
2281 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
2282 context->sourcereg = TMP2;
2283 }
2284
2285 #ifdef SUPPORT_UTF8
2286 utf8length = 1;
2287 if (common->utf8 && *cc >= 0xc0)
2288 utf8length += _pcre_utf8_table4[*cc & 0x3f];
2289
2290 do
2291 {
2292 #endif
2293
2294 context->length--;
2295 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
2296
2297 /* Unaligned read is supported. */
2298 if (othercasebit != 0 && othercasebyte == cc)
2299 {
2300 context->c.asbytes[context->byteptr] = *cc | othercasebit;
2301 context->oc.asbytes[context->byteptr] = othercasebit;
2302 }
2303 else
2304 {
2305 context->c.asbytes[context->byteptr] = *cc;
2306 context->oc.asbytes[context->byteptr] = 0;
2307 }
2308 context->byteptr++;
2309
2310 if (context->byteptr >= 4 || context->length == 0 || (context->byteptr == 2 && context->length == 1))
2311 {
2312 if (context->length >= 4)
2313 OP1(SLJIT_MOV_SI, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2314 else if (context->length >= 2)
2315 OP1(SLJIT_MOV_SH, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2316 else if (context->length >= 1)
2317 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2318 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2319
2320 switch(context->byteptr)
2321 {
2322 case 4:
2323 if (context->oc.asint != 0)
2324 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint);
2325 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint));
2326 break;
2327
2328 case 2:
2329 if (context->oc.asshort != 0)
2330 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asshort);
2331 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asshort | context->oc.asshort));
2332 break;
2333
2334 case 1:
2335 if (context->oc.asbyte != 0)
2336 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte);
2337 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte));
2338 break;
2339
2340 default:
2341 SLJIT_ASSERT_STOP();
2342 break;
2343 }
2344 context->byteptr = 0;
2345 }
2346
2347 #else
2348
2349 /* Unaligned read is unsupported. */
2350 if (context->length > 0)
2351 OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
2352 context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
2353
2354 if (othercasebit != 0 && othercasebyte == cc)
2355 {
2356 OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit);
2357 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit));
2358 }
2359 else
2360 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc));
2361
2362 #endif
2363
2364 cc++;
2365 #ifdef SUPPORT_UTF8
2366 utf8length--;
2367 }
2368 while (utf8length > 0);
2369 #endif
2370
2371 return cc;
2372 }
2373
2374 #ifdef SUPPORT_UTF8
2375
2376 #define SET_TYPE_OFFSET(value) \
2377 if ((value) != typeoffset) \
2378 { \
2379 if ((value) > typeoffset) \
2380 OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \
2381 else \
2382 OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \
2383 } \
2384 typeoffset = (value);
2385
2386 #define SET_CHAR_OFFSET(value) \
2387 if ((value) != charoffset) \
2388 { \
2389 if ((value) > charoffset) \
2390 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (value) - charoffset); \
2391 else \
2392 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, charoffset - (value)); \
2393 } \
2394 charoffset = (value);
2395
2396 static void compile_xclass_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks)
2397 {
2398 DEFINE_COMPILER;
2399 jump_list *found = NULL;
2400 jump_list **list = (*cc & XCL_NOT) == 0 ? &found : fallbacks;
2401 unsigned int c;
2402 int compares;
2403 struct sljit_jump *jump = NULL;
2404 uschar *ccbegin;
2405 #ifdef SUPPORT_UCP
2406 BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE;
2407 BOOL charsaved = FALSE;
2408 int typereg = TMP1, scriptreg = TMP1;
2409 unsigned int typeoffset;
2410 #endif
2411 int invertcmp, numberofcmps;
2412 unsigned int charoffset;
2413
2414 /* Although SUPPORT_UTF8 must be defined, we are not necessary in utf8 mode. */
2415 check_input_end(common, fallbacks);
2416 read_char(common);
2417
2418 if ((*cc++ & XCL_MAP) != 0)
2419 {
2420 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2421 if (common->utf8)
2422 jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
2423
2424 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
2425 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
2426 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
2427 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
2428 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
2429 add_jump(compiler, list, JUMP(SLJIT_C_NOT_ZERO));
2430
2431 if (common->utf8)
2432 JUMPHERE(jump);
2433 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2434 #ifdef SUPPORT_UCP
2435 charsaved = TRUE;
2436 #endif
2437 cc += 32;
2438 }
2439
2440 /* Scanning the necessary info. */
2441 ccbegin = cc;
2442 compares = 0;
2443 while (*cc != XCL_END)
2444 {
2445 compares++;
2446 if (*cc == XCL_SINGLE)
2447 {
2448 cc += 2;
2449 #ifdef SUPPORT_UTF8
2450 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2451 #endif
2452 #ifdef SUPPORT_UCP
2453 needschar = TRUE;
2454 #endif
2455 }
2456 else if (*cc == XCL_RANGE)
2457 {
2458 cc += 2;
2459 #ifdef SUPPORT_UTF8
2460 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2461 #endif
2462 cc++;
2463 #ifdef SUPPORT_UTF8
2464 if (common->utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
2465 #endif
2466 #ifdef SUPPORT_UCP
2467 needschar = TRUE;
2468 #endif
2469 }
2470 #ifdef SUPPORT_UCP
2471 else
2472 {
2473 SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
2474 cc++;
2475 switch(*cc)
2476 {
2477 case PT_ANY:
2478 break;
2479
2480 case PT_LAMP:
2481 case PT_GC:
2482 case PT_PC:
2483 case PT_ALNUM:
2484 needstype = TRUE;
2485 break;
2486
2487 case PT_SC:
2488 needsscript = TRUE;
2489 break;
2490
2491 case PT_SPACE:
2492 case PT_PXSPACE:
2493 case PT_WORD:
2494 needstype = TRUE;
2495 needschar = TRUE;
2496 break;
2497
2498 default:
2499 SLJIT_ASSERT_STOP();
2500 break;
2501 }
2502 cc += 2;
2503 }
2504 #endif
2505 }
2506
2507 #ifdef SUPPORT_UCP
2508 /* Simple register allocation. TMP1 is preferred if possible. */
2509 if (needstype || needsscript)
2510 {
2511 if (needschar && !charsaved)
2512 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
2513 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2514 if (needschar)
2515 {
2516 if (needstype)
2517 {
2518 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
2519 typereg = RETURN_ADDR;
2520 }
2521
2522 if (needsscript)
2523 scriptreg = TMP3;
2524 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
2525 }
2526 else if (needstype && needsscript)
2527 scriptreg = TMP3;
2528 /* In all other cases only one of them was specified, and that can goes to TMP1. */
2529
2530 if (needsscript)
2531 {
2532 if (scriptreg == TMP1)
2533 {
2534 OP1(SLJIT_MOV, scriptreg, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2535 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM2(scriptreg, TMP2), 3);
2536 }
2537 else
2538 {
2539 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3);
2540 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_w)_pcre_ucd_records + SLJIT_OFFSETOF(ucd_record, script));
2541 OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM1(TMP2), 0);
2542 }
2543 }
2544 }
2545 #endif
2546
2547 /* Generating code. */
2548 cc = ccbegin;
2549 charoffset = 0;
2550 numberofcmps = 0;
2551 #ifdef SUPPORT_UCP
2552 typeoffset = 0;
2553 #endif
2554
2555 while (*cc != XCL_END)
2556 {
2557 compares--;
2558 invertcmp = (compares == 0 && list != fallbacks);
2559 jump = NULL;
2560
2561 if (*cc == XCL_SINGLE)
2562 {
2563 cc ++;
2564 #ifdef SUPPORT_UTF8
2565 if (common->utf8)
2566 {
2567 GETCHARINC(c, cc);
2568 }
2569 else
2570 #endif
2571 c = *cc++;
2572
2573 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2574 {
2575 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2576 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2577 numberofcmps++;
2578 }
2579 else if (numberofcmps > 0)
2580 {
2581 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2582 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2583 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2584 numberofcmps = 0;
2585 }
2586 else
2587 {
2588 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2589 numberofcmps = 0;
2590 }
2591 }
2592 else if (*cc == XCL_RANGE)
2593 {
2594 cc ++;
2595 #ifdef SUPPORT_UTF8
2596 if (common->utf8)
2597 {
2598 GETCHARINC(c, cc);
2599 }
2600 else
2601 #endif
2602 c = *cc++;
2603 SET_CHAR_OFFSET(c);
2604 #ifdef SUPPORT_UTF8
2605 if (common->utf8)
2606 {
2607 GETCHARINC(c, cc);
2608 }
2609 else
2610 #endif
2611 c = *cc++;
2612 if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
2613 {
2614 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2615 COND_VALUE(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2616 numberofcmps++;
2617 }
2618 else if (numberofcmps > 0)
2619 {
2620 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
2621 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2622 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2623 numberofcmps = 0;
2624 }
2625 else
2626 {
2627 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
2628 numberofcmps = 0;
2629 }
2630 }
2631 #ifdef SUPPORT_UCP
2632 else
2633 {
2634 if (*cc == XCL_NOTPROP)
2635 invertcmp ^= 0x1;
2636 cc++;
2637 switch(*cc)
2638 {
2639 case PT_ANY:
2640 if (list != fallbacks)
2641 {
2642 if ((cc[-1] == XCL_NOTPROP && compares > 0) || (cc[-1] == XCL_PROP && compares == 0))
2643 continue;
2644 }
2645 else if (cc[-1] == XCL_NOTPROP)
2646 continue;
2647 jump = JUMP(SLJIT_JUMP);
2648 break;
2649
2650 case PT_LAMP:
2651 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
2652 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2653 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
2654 COND_VALUE(SLJIT_OR, TMP2, 0, SLJIT_C_EQUAL);
2655 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
2656 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
2657 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2658 break;
2659
2660 case PT_GC:
2661 c = _pcre_ucp_typerange[(int)cc[1] * 2];
2662 SET_TYPE_OFFSET(c);
2663 jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, _pcre_ucp_typerange[(int)cc[1] * 2 + 1] - c);
2664 break;
2665
2666 case PT_PC:
2667 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
2668 break;
2669
2670 case PT_SC:
2671 jump = CMP(SLJIT_C_EQUAL ^ invertcmp, scriptreg, 0, SLJIT_IMM, (int)cc[1]);
2672 break;
2673
2674 case PT_SPACE:
2675 case PT_PXSPACE:
2676 if (*cc == PT_SPACE)
2677 {
2678 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
2679 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 11 - charoffset);
2680 }
2681 SET_CHAR_OFFSET(9);
2682 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 13 - 9);
2683 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_LESS_EQUAL);
2684 if (*cc == PT_SPACE)
2685 JUMPHERE(jump);
2686
2687 SET_TYPE_OFFSET(ucp_Zl);
2688 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
2689 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2690 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2691 break;
2692
2693 case PT_WORD:
2694 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE - charoffset);
2695 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
2696 /* ... fall through */
2697
2698 case PT_ALNUM:
2699 SET_TYPE_OFFSET(ucp_Ll);
2700 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
2701 COND_VALUE((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_C_LESS_EQUAL);
2702 SET_TYPE_OFFSET(ucp_Nd);
2703 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
2704 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_LESS_EQUAL);
2705 jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
2706 break;
2707 }
2708 cc += 2;
2709 }
2710 #endif
2711
2712 if (jump != NULL)
2713 add_jump(compiler, compares > 0 ? list : fallbacks, jump);
2714 }
2715
2716 if (found != NULL)
2717 set_jumps(found, LABEL());
2718 }
2719
2720 #undef SET_TYPE_OFFSET
2721 #undef SET_CHAR_OFFSET
2722
2723 #endif
2724
2725 static uschar *compile_char1_hotpath(compiler_common *common, uschar type, uschar *cc, jump_list **fallbacks)
2726 {
2727 DEFINE_COMPILER;
2728 int length;
2729 unsigned int c, oc, bit;
2730 compare_context context;
2731 struct sljit_jump *jump[4];
2732 #ifdef SUPPORT_UTF8
2733 struct sljit_label *label;
2734 #ifdef SUPPORT_UCP
2735 uschar propdata[5];
2736 #endif
2737 #endif
2738
2739 switch(type)
2740 {
2741 case OP_SOD:
2742 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2743 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
2744 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2745 return cc;
2746
2747 case OP_SOM:
2748 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
2749 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
2750 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
2751 return cc;
2752
2753 case OP_NOT_WORD_BOUNDARY:
2754 case OP_WORD_BOUNDARY:
2755 add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
2756 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2757 return cc;
2758
2759 case OP_NOT_DIGIT:
2760 case OP_DIGIT:
2761 check_input_end(common, fallbacks);
2762 read_char8_type(common);
2763 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
2764 add_jump(compiler, fallbacks, JUMP(type == OP_DIGIT ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2765 return cc;
2766
2767 case OP_NOT_WHITESPACE:
2768 case OP_WHITESPACE:
2769 check_input_end(common, fallbacks);
2770 read_char8_type(common);
2771 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
2772 add_jump(compiler, fallbacks, JUMP(type == OP_WHITESPACE ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2773 return cc;
2774
2775 case OP_NOT_WORDCHAR:
2776 case OP_WORDCHAR:
2777 check_input_end(common, fallbacks);
2778 read_char8_type(common);
2779 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
2780 add_jump(compiler, fallbacks, JUMP(type == OP_WORDCHAR ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
2781 return cc;
2782
2783 case OP_ANY:
2784 check_input_end(common, fallbacks);
2785 read_char(common);
2786 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2787 {
2788 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
2789 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2790 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2791 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
2792 JUMPHERE(jump[1]);
2793 JUMPHERE(jump[0]);
2794 }
2795 else
2796 check_newlinechar(common, common->nltype, fallbacks, TRUE);
2797 return cc;
2798
2799 case OP_ALLANY:
2800 check_input_end(common, fallbacks);
2801 #ifdef SUPPORT_UTF8
2802 if (common->utf8)
2803 {
2804 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2805 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2806 jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
2807 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
2808 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
2809 JUMPHERE(jump[0]);
2810 return cc;
2811 }
2812 #endif
2813 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2814 return cc;
2815
2816 case OP_ANYBYTE:
2817 check_input_end(common, fallbacks);
2818 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2819 return cc;
2820
2821 #ifdef SUPPORT_UTF8
2822 #ifdef SUPPORT_UCP
2823 case OP_NOTPROP:
2824 case OP_PROP:
2825 propdata[0] = 0;
2826 propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP;
2827 propdata[2] = cc[0];
2828 propdata[3] = cc[1];
2829 propdata[4] = XCL_END;
2830 compile_xclass_hotpath(common, propdata, fallbacks);
2831 return cc + 2;
2832 #endif
2833 #endif
2834
2835 case OP_ANYNL:
2836 check_input_end(common, fallbacks);
2837 read_char(common);
2838 jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2839 jump[1] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2840 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2841 jump[2] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2842 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
2843 jump[3] = JUMP(SLJIT_JUMP);
2844 JUMPHERE(jump[0]);
2845 check_newlinechar(common, common->bsr_nltype, fallbacks, FALSE);
2846 JUMPHERE(jump[1]);
2847 JUMPHERE(jump[2]);
2848 JUMPHERE(jump[3]);
2849 return cc;
2850
2851 case OP_NOT_HSPACE:
2852 case OP_HSPACE:
2853 check_input_end(common, fallbacks);
2854 read_char(common);
2855 add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
2856 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_HSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2857 return cc;
2858
2859 case OP_NOT_VSPACE:
2860 case OP_VSPACE:
2861 check_input_end(common, fallbacks);
2862 read_char(common);
2863 add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
2864 add_jump(compiler, fallbacks, JUMP(type == OP_NOT_VSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
2865 return cc;
2866
2867 #ifdef SUPPORT_UCP
2868 case OP_EXTUNI:
2869 check_input_end(common, fallbacks);
2870 read_char(common);
2871 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2872 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2873 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc));
2874
2875 label = LABEL();
2876 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2877 OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
2878 read_char(common);
2879 add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
2880 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Mc);
2881 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, ucp_Mn - ucp_Mc, label);
2882
2883 OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
2884 JUMPHERE(jump[0]);
2885 return cc;
2886 #endif
2887
2888 case OP_EODN:
2889 jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
2890 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2891 {
2892 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2893 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2894 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2895 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
2896 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2897 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2898 }
2899 else if (common->nltype == NLTYPE_FIXED)
2900 {
2901 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2902 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2903 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
2904 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
2905 }
2906 else
2907 {
2908 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
2909 jump[1] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
2910 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2911 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
2912 jump[2] = JUMP(SLJIT_C_GREATER);
2913 add_jump(compiler, fallbacks, JUMP(SLJIT_C_LESS));
2914 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 1);
2915 jump[3] = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
2916 add_jump(compiler, fallbacks, JUMP(SLJIT_JUMP));
2917
2918 JUMPHERE(jump[1]);
2919 if (common->nltype == NLTYPE_ANYCRLF)
2920 {
2921 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 1);
2922 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, STR_END, 0));
2923 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
2924 }
2925 else
2926 {
2927 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, STR_PTR, 0);
2928 read_char(common);
2929 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2930 add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
2931 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
2932 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
2933 }
2934 JUMPHERE(jump[2]);
2935 JUMPHERE(jump[3]);
2936 }
2937 JUMPHERE(jump[0]);
2938 return cc;
2939
2940 case OP_EOD:
2941 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
2942 return cc;
2943
2944 case OP_CIRC:
2945 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2946 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2947 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0));
2948 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2949 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2950 return cc;
2951
2952 case OP_CIRCM:
2953 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2954 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
2955 jump[1] = CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0);
2956 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
2957 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2958 jump[0] = JUMP(SLJIT_JUMP);
2959 JUMPHERE(jump[1]);
2960
2961 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, end));
2962 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP2, 0, STR_PTR, 0));
2963
2964 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
2965 {
2966 OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
2967 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, TMP2, 0, TMP1, 0));
2968 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -2);
2969 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), -1);
2970 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
2971 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
2972 }
2973 else
2974 {
2975 skip_char_back(common);
2976 read_char(common);
2977 check_newlinechar(common, common->nltype, fallbacks, FALSE);
2978 }
2979 JUMPHERE(jump[0]);
2980 return cc;
2981
2982 case OP_DOLL:
2983 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2984 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
2985 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2986
2987 if (!common->endonly)
2988 compile_char1_hotpath(common, OP_EODN, cc, fallbacks);
2989 else
2990 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0));
2991 return cc;
2992
2993 case OP_DOLLM:
2994 jump[1] = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
2995 OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
2996 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
2997 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
2998 jump[0] = JUMP(SLJIT_JUMP);
2999 JUMPHERE(jump[1]);
3000
3001 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
3002 {
3003 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 2);
3004 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, TMP2, 0, STR_END, 0));
3005 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3006 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(STR_PTR), 1);
3007 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
3008 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
3009 }
3010 else
3011 {
3012 peek_char(common);
3013 check_newlinechar(common, common->nltype, fallbacks, FALSE);
3014 }
3015 JUMPHERE(jump[0]);
3016 return cc;
3017
3018 case OP_CHAR:
3019 case OP_CHARI:
3020 length = 1;
3021 #ifdef SUPPORT_UTF8
3022 if (common->utf8 && *cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3023 #endif
3024 if (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)
3025 {
3026 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3027 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3028
3029 context.length = length;
3030 context.sourcereg = -1;
3031 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3032 context.byteptr = 0;
3033 #endif
3034 return byte_sequence_compare(common, type == OP_CHARI, cc, &context, fallbacks);
3035 }
3036 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
3037 read_char(common);
3038 #ifdef SUPPORT_UTF8
3039 if (common->utf8)
3040 {
3041 GETCHAR(c, cc);
3042 }
3043 else
3044 #endif
3045 c = *cc;
3046 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c);
3047 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3048 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char_othercase(common, c));
3049 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3050 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3051 return cc + length;
3052
3053 case OP_NOT:
3054 case OP_NOTI:
3055 length = 1;
3056 #ifdef SUPPORT_UTF8
3057 if (common->utf8)
3058 {
3059 if (*cc >= 0xc0) length += _pcre_utf8_table4[*cc & 0x3f];
3060
3061 check_input_end(common, fallbacks);
3062 GETCHAR(c, cc);
3063
3064 if (c <= 127)
3065 {
3066 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
3067 if (type == OP_NOT || !char_has_othercase(common, cc))
3068 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3069 else
3070 {
3071 /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */
3072 OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20);
3073 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20));
3074 }
3075 /* Skip the variable-length character. */
3076 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
3077 jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
3078 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)_pcre_utf8_char_sizes - 0xc0);
3079 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
3080 JUMPHERE(jump[0]);
3081 return cc + length;
3082 }
3083 else
3084 read_char(common);
3085 }
3086 else
3087 #endif
3088 {
3089 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 1);
3090 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3091 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -1);
3092 c = *cc;
3093 }
3094
3095 if (type == OP_NOT || !char_has_othercase(common, cc))
3096 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3097 else
3098 {
3099 oc = char_othercase(common, c);
3100 bit = c ^ oc;
3101 if (ispowerof2(bit))
3102 {
3103 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
3104 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
3105 }
3106 else
3107 {
3108 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
3109 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, oc));
3110 }
3111 }
3112 return cc + length;
3113
3114 case OP_CLASS:
3115 case OP_NCLASS:
3116 check_input_end(common, fallbacks);
3117 read_char(common);
3118 #ifdef SUPPORT_UTF8
3119 jump[0] = NULL;
3120 if (common->utf8)
3121 {
3122 jump[0] = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
3123 if (type == OP_CLASS)
3124 {
3125 add_jump(compiler, fallbacks, jump[0]);
3126 jump[0] = NULL;
3127 }
3128 }
3129 #endif
3130 OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
3131 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
3132 OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_w)cc);
3133 OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
3134 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
3135 add_jump(compiler, fallbacks, JUMP(SLJIT_C_ZERO));
3136 #ifdef SUPPORT_UTF8
3137 if (jump[0] != NULL)
3138 JUMPHERE(jump[0]);
3139 #endif
3140 return cc + 32;
3141
3142 #ifdef SUPPORT_UTF8
3143 case OP_XCLASS:
3144 compile_xclass_hotpath(common, cc + LINK_SIZE, fallbacks);
3145 return cc + GET(cc, 0) - 1;
3146 #endif
3147
3148 case OP_REVERSE:
3149 length = GET(cc, 0);
3150 SLJIT_ASSERT(length > 0);
3151 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
3152 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
3153 #ifdef SUPPORT_UTF8
3154 if (common->utf8)
3155 {
3156 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length);
3157 label = LABEL();
3158 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0));
3159 skip_char_back(common);
3160 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
3161 JUMPTO(SLJIT_C_NOT_ZERO, label);
3162 return cc + LINK_SIZE;
3163 }
3164 #endif
3165 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, length);
3166 add_jump(compiler, fallbacks, CMP(SLJIT_C_LESS, STR_PTR, 0, TMP1, 0));
3167 return cc + LINK_SIZE;
3168 }
3169 SLJIT_ASSERT_STOP();
3170 return cc;
3171 }
3172
3173 static SLJIT_INLINE uschar *compile_charn_hotpath(compiler_common *common, uschar *cc, uschar *ccend, jump_list **fallbacks)
3174 {
3175 /* This function consumes at least one input character. */
3176 /* To decrease the number of length checks, we try to concatenate the fixed length character sequences. */
3177 DEFINE_COMPILER;
3178 uschar *ccbegin = cc;
3179 compare_context context;
3180 int size;
3181
3182 context.length = 0;
3183 do
3184 {
3185 if (cc >= ccend)
3186 break;
3187
3188 if (*cc == OP_CHAR)
3189 {
3190 size = 1;
3191 #ifdef SUPPORT_UTF8
3192 if (common->utf8 && cc[1] >= 0xc0)
3193 size += _pcre_utf8_table4[cc[1] & 0x3f];
3194 #endif
3195 }
3196 else if (*cc == OP_CHARI)
3197 {
3198 size = 1;
3199 #ifdef SUPPORT_UTF8
3200 if (common->utf8)
3201 {
3202 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3203 size = 0;
3204 else if (cc[1] >= 0xc0)
3205 size += _pcre_utf8_table4[cc[1] & 0x3f];
3206 }
3207 else
3208 #endif
3209 if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0)
3210 size = 0;
3211 }
3212 else
3213 size = 0;
3214
3215 cc += 1 + size;
3216 context.length += size;
3217 }
3218 while (size > 0 && context.length <= 128);
3219
3220 cc = ccbegin;
3221 if (context.length > 0)
3222 {
3223 /* We have a fixed-length byte sequence. */
3224 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length);
3225 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3226
3227 context.sourcereg = -1;
3228 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
3229 context.byteptr = 0;
3230 #endif
3231 do cc = byte_sequence_compare(common, *cc == OP_CHARI, cc + 1, &context, fallbacks); while (context.length > 0);
3232 return cc;
3233 }
3234
3235 /* A non-fixed length character will be checked if length == 0. */
3236 return compile_char1_hotpath(common, *cc, cc + 1, fallbacks);
3237 }
3238
3239 static struct sljit_jump *compile_ref_checks(compiler_common *common, uschar *cc, jump_list **fallbacks)
3240 {
3241 DEFINE_COMPILER;
3242 int offset = GET2(cc, 1) << 1;
3243
3244 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3245 if (!common->jscript_compat)
3246 {
3247 if (fallbacks == NULL)
3248 {
3249 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
3250 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_EQUAL);
3251 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3252 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_EQUAL);
3253 return JUMP(SLJIT_C_NOT_ZERO);
3254 }
3255 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3256 }
3257 return CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3258 }
3259
3260 /* Forward definitions. */
3261 static void compile_hotpath(compiler_common *, uschar *, uschar *, fallback_common *);
3262 static void compile_fallbackpath(compiler_common *, struct fallback_common *);
3263
3264 #define PUSH_FALLBACK(size, ccstart, error) \
3265 do \
3266 { \
3267 fallback = sljit_alloc_memory(compiler, (size)); \
3268 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3269 return error; \
3270 memset(fallback, 0, size); \
3271 fallback->prev = parent->top; \
3272 fallback->cc = (ccstart); \
3273 parent->top = fallback; \
3274 } \
3275 while (0)
3276
3277 #define PUSH_FALLBACK_NOVALUE(size, ccstart) \
3278 do \
3279 { \
3280 fallback = sljit_alloc_memory(compiler, (size)); \
3281 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
3282 return; \
3283 memset(fallback, 0, size); \
3284 fallback->prev = parent->top; \
3285 fallback->cc = (ccstart); \
3286 parent->top = fallback; \
3287 } \
3288 while (0)
3289
3290 #define FALLBACK_AS(type) ((type*)fallback)
3291
3292 static uschar *compile_ref_hotpath(compiler_common *common, uschar *cc, jump_list **fallbacks, BOOL withchecks, BOOL emptyfail)
3293 {
3294 DEFINE_COMPILER;
3295 int offset = GET2(cc, 1) << 1;
3296 struct sljit_jump *jump = NULL;
3297
3298 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
3299 if (withchecks && !common->jscript_compat)
3300 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
3301
3302 #ifdef SUPPORT_UTF8
3303 #ifdef SUPPORT_UCP
3304 if (common->utf8 && *cc == OP_REFI)
3305 {
3306 SLJIT_ASSERT(TMP1 == SLJIT_TEMPORARY_REG1 && STACK_TOP == SLJIT_TEMPORARY_REG2 && TMP2 == SLJIT_TEMPORARY_REG3);
3307 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
3308 if (withchecks)
3309 jump = CMP(SLJIT_C_EQUAL, TMP1, 0, TMP2, 0);
3310
3311 /* Needed to save important temporary registers. */
3312 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
3313 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, ARGUMENTS, 0);
3314 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_TEMPORARY_REG2), SLJIT_OFFSETOF(jit_arguments, ptr), STR_PTR, 0);
3315 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf8caselesscmp));
3316 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
3317 add_jump(compiler, fallbacks, CMP(SLJIT_C_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
3318 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
3319 }
3320 else
3321 #endif
3322 #endif
3323 {
3324 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP1, 0);
3325 if (withchecks)
3326 jump = JUMP(SLJIT_C_ZERO);
3327 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
3328
3329 add_jump(compiler, fallbacks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
3330 add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL));
3331 add_jump(compiler, fallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
3332 }
3333
3334 if (jump != NULL)
3335 {
3336 if (emptyfail)
3337 add_jump(compiler, fallbacks, jump);
3338 else
3339 JUMPHERE(jump);
3340 }
3341 return cc + 3;
3342 }
3343
3344 static SLJIT_INLINE uschar *compile_ref_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3345 {
3346 DEFINE_COMPILER;
3347 fallback_common *fallback;
3348 uschar type;
3349 struct sljit_label *label;
3350 struct sljit_jump *zerolength;
3351 struct sljit_jump *jump = NULL;
3352 uschar *ccbegin = cc;
3353 int min = 0, max = 0;
3354 BOOL minimize;
3355
3356 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
3357
3358 type = cc[3];
3359 minimize = (type & 0x1) != 0;
3360 switch(type)
3361 {
3362 case OP_CRSTAR:
3363 case OP_CRMINSTAR:
3364 min = 0;
3365 max = 0;
3366 cc += 4;
3367 break;
3368 case OP_CRPLUS:
3369 case OP_CRMINPLUS:
3370 min = 1;
3371 max = 0;
3372 cc += 4;
3373 break;
3374 case OP_CRQUERY:
3375 case OP_CRMINQUERY:
3376 min = 0;
3377 max = 1;
3378 cc += 4;
3379 break;
3380 case OP_CRRANGE:
3381 case OP_CRMINRANGE:
3382 min = GET2(cc, 3 + 1);
3383 max = GET2(cc, 3 + 3);
3384 cc += 8;
3385 break;
3386 default:
3387 SLJIT_ASSERT_STOP();
3388 break;
3389 }
3390
3391 if (!minimize)
3392 {
3393 if (min == 0)
3394 {
3395 allocate_stack(common, 2);
3396 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3397 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3398 /* Temporary release of STR_PTR. */
3399 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3400 zerolength = compile_ref_checks(common, ccbegin, NULL);
3401 /* Restore if not zero length. */
3402 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3403 }
3404 else
3405 {
3406 allocate_stack(common, 1);
3407 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3408 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3409 }
3410
3411 if (min > 1 || max > 1)
3412 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
3413
3414 label = LABEL();
3415 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, FALSE, FALSE);
3416
3417 if (min > 1 || max > 1)
3418 {
3419 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
3420 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3421 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
3422 if (min > 1)
3423 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, label);
3424 if (max > 1)
3425 {
3426 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
3427 allocate_stack(common, 1);
3428 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3429 JUMPTO(SLJIT_JUMP, label);
3430 JUMPHERE(jump);
3431 }
3432 }
3433
3434 if (max == 0)
3435 {
3436 /* Includes min > 1 case as well. */
3437 allocate_stack(common, 1);
3438 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3439 JUMPTO(SLJIT_JUMP, label);
3440 }
3441
3442 JUMPHERE(zerolength);
3443 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3444
3445 decrease_call_count(common);
3446 return cc;
3447 }
3448
3449 allocate_stack(common, 2);
3450 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3451 if (type != OP_CRMINSTAR)
3452 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
3453
3454 if (min == 0)
3455 {
3456 zerolength = compile_ref_checks(common, ccbegin, NULL);
3457 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3458 jump = JUMP(SLJIT_JUMP);
3459 }
3460 else
3461 zerolength = compile_ref_checks(common, ccbegin, &fallback->topfallbacks);
3462
3463 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
3464 if (max > 0)
3465 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max));
3466
3467 compile_ref_hotpath(common, ccbegin, &fallback->topfallbacks, TRUE, TRUE);
3468 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3469
3470 if (min > 1)
3471 {
3472 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3473 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
3474 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3475 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, FALLBACK_AS(iterator_fallback)->hotpath);
3476 }
3477 else if (max > 0)
3478 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
3479
3480 if (jump != NULL)
3481 JUMPHERE(jump);
3482 JUMPHERE(zerolength);
3483
3484 decrease_call_count(common);
3485 return cc;
3486 }
3487
3488 static SLJIT_INLINE uschar *compile_recurse_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3489 {
3490 DEFINE_COMPILER;
3491 fallback_common *fallback;
3492 recurse_entry *entry = common->entries;
3493 recurse_entry *prev = NULL;
3494 int start = GET(cc, 1);
3495
3496 PUSH_FALLBACK(sizeof(recurse_fallback), cc, NULL);
3497 while (entry != NULL)
3498 {
3499 if (entry->start == start)
3500 break;
3501 prev = entry;
3502 entry = entry->next;
3503 }
3504
3505 if (entry == NULL)
3506 {
3507 entry = sljit_alloc_memory(compiler, sizeof(recurse_entry));
3508 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3509 return NULL;
3510 entry->next = NULL;
3511 entry->entry = NULL;
3512 entry->calls = NULL;
3513 entry->start = start;
3514
3515 if (prev != NULL)
3516 prev->next = entry;
3517 else
3518 common->entries = entry;
3519 }
3520
3521 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
3522 allocate_stack(common, 1);
3523 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
3524
3525 if (entry->entry == NULL)
3526 add_jump(compiler, &entry->calls, JUMP(SLJIT_FAST_CALL));
3527 else
3528 JUMPTO(SLJIT_FAST_CALL, entry->entry);
3529 /* Leave if the match is failed. */
3530 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0));
3531 return cc + 1 + LINK_SIZE;
3532 }
3533
3534 static uschar *compile_assert_hotpath(compiler_common *common, uschar *cc, assert_fallback *fallback, BOOL conditional)
3535 {
3536 DEFINE_COMPILER;
3537 int framesize;
3538 int localptr;
3539 fallback_common altfallback;
3540 uschar *ccbegin;
3541 uschar opcode;
3542 uschar bra = OP_BRA;
3543 jump_list *tmp = NULL;
3544 jump_list **target = (conditional) ? &fallback->condfailed : &fallback->common.topfallbacks;
3545 jump_list **found;
3546 /* Saving previous accept variables. */
3547 struct sljit_label *save_acceptlabel = common->acceptlabel;
3548 struct sljit_jump *jump;
3549 struct sljit_jump *brajump = NULL;
3550 jump_list *save_accept = common->accept;
3551
3552 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3553 {
3554 SLJIT_ASSERT(!conditional);
3555 bra = *cc;
3556 cc++;
3557 }
3558 localptr = PRIV(cc);
3559 SLJIT_ASSERT(localptr != 0);
3560 framesize = get_framesize(common, cc, FALSE);
3561 fallback->framesize = framesize;
3562 fallback->localptr = localptr;
3563 opcode = *cc;
3564 SLJIT_ASSERT(opcode >= OP_ASSERT && opcode <= OP_ASSERTBACK_NOT);
3565 found = (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) ? &tmp : target;
3566 ccbegin = cc;
3567 cc += GET(cc, 1);
3568
3569 if (bra == OP_BRAMINZERO)
3570 {
3571 /* This is a braminzero fallback path. */
3572 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3573 free_stack(common, 1);
3574 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
3575 }
3576
3577 if (framesize < 0)
3578 {
3579 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
3580 allocate_stack(common, 1);
3581 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3582 }
3583 else
3584 {
3585 allocate_stack(common, framesize + 2);
3586 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3587 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(framesize + 1));
3588 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
3589 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
3590 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
3591 init_frame(common, ccbegin, framesize + 1, 2, FALSE);
3592 }
3593
3594 memset(&altfallback, 0, sizeof(fallback_common));
3595 while (1)
3596 {
3597 common->acceptlabel = NULL;
3598 common->accept = NULL;
3599 altfallback.top = NULL;
3600 altfallback.topfallbacks = NULL;
3601
3602 if (*ccbegin == OP_ALT)
3603 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3604
3605 altfallback.cc = ccbegin;
3606 compile_hotpath(common, ccbegin + 1 + LINK_SIZE, cc, &altfallback);
3607 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3608 {
3609 common->acceptlabel = save_acceptlabel;
3610 common->accept = save_accept;
3611 return NULL;
3612 }
3613 common->acceptlabel = LABEL();
3614 if (common->accept != NULL)
3615 set_jumps(common->accept, common->acceptlabel);
3616
3617 /* Reset stack. */
3618 if (framesize < 0)
3619 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3620 else {
3621 if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional)
3622 {
3623 /* We don't need to keep the STR_PTR, only the previous localptr. */
3624 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_w));
3625 }
3626 else
3627 {
3628 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3629 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3630 }
3631 }
3632
3633 if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT)
3634 {
3635 /* We know that STR_PTR was stored on the top of the stack. */
3636 if (conditional)
3637 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3638 else if (bra == OP_BRAZERO)
3639 {
3640 if (framesize < 0)
3641 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3642 else
3643 {
3644 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3645 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_w));
3646 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3647 }
3648 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3649 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3650 }
3651 else if (framesize >= 0)
3652 {
3653 /* For OP_BRA and OP_BRAMINZERO. */
3654 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3655 }
3656 }
3657 add_jump(compiler, found, JUMP(SLJIT_JUMP));
3658
3659 compile_fallbackpath(common, altfallback.top);
3660 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
3661 {
3662 common->acceptlabel = save_acceptlabel;
3663 common->accept = save_accept;
3664 return NULL;
3665 }
3666 set_jumps(altfallback.topfallbacks, LABEL());
3667
3668 if (*cc != OP_ALT)
3669 break;
3670
3671 ccbegin = cc;
3672 cc += GET(cc, 1);
3673 }
3674 /* None of them matched. */
3675
3676 if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
3677 {
3678 /* Assert is failed. */
3679 if (conditional || bra == OP_BRAZERO)
3680 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3681
3682 if (framesize < 0)
3683 {
3684 /* The topmost item should be 0. */
3685 if (bra == OP_BRAZERO)
3686 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3687 else
3688 free_stack(common, 1);
3689 }
3690 else
3691 {
3692 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3693 /* The topmost item should be 0. */
3694 if (bra == OP_BRAZERO)
3695 {
3696 free_stack(common, framesize + 1);
3697 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3698 }
3699 else
3700 free_stack(common, framesize + 2);
3701 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3702 }
3703 jump = JUMP(SLJIT_JUMP);
3704 if (bra != OP_BRAZERO)
3705 add_jump(compiler, target, jump);
3706
3707 /* Assert is successful. */
3708 set_jumps(tmp, LABEL());
3709 if (framesize < 0)
3710 {
3711 /* We know that STR_PTR was stored on the top of the stack. */
3712 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3713 /* Keep the STR_PTR on the top of the stack. */
3714 if (bra == OP_BRAZERO)
3715 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3716 else if (bra == OP_BRAMINZERO)
3717 {
3718 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
3719 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3720 }
3721 }
3722 else
3723 {
3724 if (bra == OP_BRA)
3725 {
3726 /* We don't need to keep the STR_PTR, only the previous localptr. */
3727 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_w));
3728 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0);
3729 }
3730 else
3731 {
3732 /* We don't need to keep the STR_PTR, only the previous localptr. */
3733 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_w));
3734 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3735 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), bra == OP_BRAZERO ? STR_PTR : SLJIT_IMM, 0);
3736 }
3737 }
3738
3739 if (bra == OP_BRAZERO)
3740 {
3741 fallback->hotpath = LABEL();
3742 sljit_set_label(jump, fallback->hotpath);
3743 }
3744 else if (bra == OP_BRAMINZERO)
3745 {
3746 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3747 JUMPHERE(brajump);
3748 if (framesize >= 0)
3749 {
3750 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
3751 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
3752 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_w));
3753 }
3754 set_jumps(fallback->common.topfallbacks, LABEL());
3755 }
3756 }
3757 else
3758 {
3759 /* AssertNot is successful. */
3760 if (framesize < 0)
3761 {
3762 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3763 if (bra != OP_BRA)
3764 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3765 else
3766 free_stack(common, 1);
3767 }
3768 else
3769 {
3770 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
3771 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
3772 /* The topmost item should be 0. */
3773 if (bra != OP_BRA)
3774 {
3775 free_stack(common, framesize + 1);
3776 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
3777 }
3778 else
3779 free_stack(common, framesize + 2);
3780 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
3781 }
3782
3783 if (bra == OP_BRAZERO)
3784 fallback->hotpath = LABEL();
3785 else if (bra == OP_BRAMINZERO)
3786 {
3787 JUMPTO(SLJIT_JUMP, fallback->hotpath);
3788 JUMPHERE(brajump);
3789 }
3790
3791 if (bra != OP_BRA)
3792 {
3793 SLJIT_ASSERT(found == &fallback->common.topfallbacks);
3794 set_jumps(fallback->common.topfallbacks, LABEL());
3795 fallback->common.topfallbacks = NULL;
3796 }
3797 }
3798
3799 common->acceptlabel = save_acceptlabel;
3800 common->accept = save_accept;
3801 return cc + 1 + LINK_SIZE;
3802 }
3803
3804 static sljit_w SLJIT_CALL do_searchovector(sljit_w refno, sljit_w* locals, uschar *name_table)
3805 {
3806 int condition = FALSE;
3807 uschar *slotA = name_table;
3808 uschar *slotB;
3809 sljit_w name_count = locals[LOCALS0 / sizeof(sljit_w)];
3810 sljit_w name_entry_size = locals[LOCALS1 / sizeof(sljit_w)];
3811 sljit_w no_capture;
3812 int i;
3813
3814 locals += OVECTOR_START / sizeof(sljit_w);
3815 no_capture = locals[1];
3816
3817 for (i = 0; i < name_count; i++)
3818 {
3819 if (GET2(slotA, 0) == refno) break;
3820 slotA += name_entry_size;
3821 }
3822
3823 if (i < name_count)
3824 {
3825 /* Found a name for the number - there can be only one; duplicate names
3826 for different numbers are allowed, but not vice versa. First scan down
3827 for duplicates. */
3828
3829 slotB = slotA;
3830 while (slotB > name_table)
3831 {
3832 slotB -= name_entry_size;
3833 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3834 {
3835 condition = locals[GET2(slotB, 0) << 1] != no_capture;
3836 if (condition) break;
3837 }
3838 else break;
3839 }
3840
3841 /* Scan up for duplicates */
3842 if (!condition)
3843 {
3844 slotB = slotA;
3845 for (i++; i < name_count; i++)
3846 {
3847 slotB += name_entry_size;
3848 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3849 {
3850 condition = locals[GET2(slotB, 0) << 1] != no_capture;
3851 if (condition) break;
3852 }
3853 else break;
3854 }
3855 }
3856 }
3857 return condition;
3858 }
3859
3860 static sljit_w SLJIT_CALL do_searchgroups(sljit_w recno, sljit_w* locals, uschar *name_table)
3861 {
3862 int condition = FALSE;
3863 uschar *slotA = name_table;
3864 uschar *slotB;
3865 sljit_w name_count = locals[LOCALS0 / sizeof(sljit_w)];
3866 sljit_w name_entry_size = locals[LOCALS1 / sizeof(sljit_w)];
3867 sljit_w group_num = locals[POSSESSIVE0 / sizeof(sljit_w)];
3868 int i;
3869
3870 for (i = 0; i < name_count; i++)
3871 {
3872 if (GET2(slotA, 0) == recno) break;
3873 slotA += name_entry_size;
3874 }
3875
3876 if (i < name_count)
3877 {
3878 /* Found a name for the number - there can be only one; duplicate
3879 names for different numbers are allowed, but not vice versa. First
3880 scan down for duplicates. */
3881
3882 slotB = slotA;
3883 while (slotB > name_table)
3884 {
3885 slotB -= name_entry_size;
3886 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3887 {
3888 condition = GET2(slotB, 0) == group_num;
3889 if (condition) break;
3890 }
3891 else break;
3892 }
3893
3894 /* Scan up for duplicates */
3895 if (!condition)
3896 {
3897 slotB = slotA;
3898 for (i++; i < name_count; i++)
3899 {
3900 slotB += name_entry_size;
3901 if (strcmp((char *)slotA + 2, (char *)slotB + 2) == 0)
3902 {
3903 condition = GET2(slotB, 0) == group_num;
3904 if (condition) break;
3905 }
3906 else break;
3907 }
3908 }
3909 }
3910 return condition;
3911 }
3912
3913 /*
3914 Handling bracketed expressions is probably the most complex part.
3915
3916 Stack layout naming characters:
3917 S - Push the current STR_PTR
3918 0 - Push a 0 (NULL)
3919 A - Push the current STR_PTR. Needed for restoring the STR_PTR
3920 before the next alternative. Not pushed if there are no alternatives.
3921 M - Any values pushed by the current alternative. Can be empty, or anything.
3922 C - Push the previous OVECTOR(i), OVECTOR(i+1) and OVECTOR_PRIV(i) to the stack.
3923 L - Push the previous local (pointed by localptr) to the stack
3924 () - opional values stored on the stack
3925 ()* - optonal, can be stored multiple times
3926
3927 The following list shows the regular expression templates, their PCRE byte codes
3928 and stack layout supported by pcre-sljit.
3929
3930 (?:) OP_BRA | OP_KET A M
3931 () OP_CBRA | OP_KET C M
3932 (?:)+ OP_BRA | OP_KETRMAX 0 A M S ( A M S )*
3933 OP_SBRA | OP_KETRMAX 0 L M S ( L M S )*
3934 (?:)+? OP_BRA | OP_KETRMIN 0 A M S ( A M S )*
3935 OP_SBRA | OP_KETRMIN 0 L M S ( L M S )*
3936 ()+ OP_CBRA | OP_KETRMAX 0 C M S ( C M S )*
3937 OP_SCBRA | OP_KETRMAX 0 C M S ( C M S )*
3938 ()+? OP_CBRA | OP_KETRMIN 0 C M S ( C M S )*
3939 OP_SCBRA | OP_KETRMIN 0 C M S ( C M S )*
3940 (?:)? OP_BRAZERO | OP_BRA | OP_KET S ( A M 0 )
3941 (?:)?? OP_BRAMINZERO | OP_BRA | OP_KET S ( A M 0 )
3942 ()? OP_BRAZERO | OP_CBRA | OP_KET S ( C M 0 )
3943 ()?? OP_BRAMINZERO | OP_CBRA | OP_KET S ( C M 0 )
3944 (?:)* OP_BRAZERO | OP_BRA | OP_KETRMAX S 0 ( A M S )*
3945 OP_BRAZERO | OP_SBRA | OP_KETRMAX S 0 ( L M S )*
3946 (?:)*? OP_BRAMINZERO | OP_BRA | OP_KETRMIN S 0 ( A M S )*
3947 OP_BRAMINZERO | OP_SBRA | OP_KETRMIN S 0 ( L M S )*
3948 ()* OP_BRAZERO | OP_CBRA | OP_KETRMAX S 0 ( C M S )*
3949 OP_BRAZERO | OP_SCBRA | OP_KETRMAX S 0 ( C M S )*
3950 ()*? OP_BRAMINZERO | OP_CBRA | OP_KETRMIN S 0 ( C M S )*
3951 OP_BRAMINZERO | OP_SCBRA | OP_KETRMIN S 0 ( C M S )*
3952
3953
3954 Stack layout naming characters:
3955 A - Push the alternative index (starting from 0) on the stack.
3956 Not pushed if there is no alternatives.
3957 M - Any values pushed by the current alternative. Can be empty, or anything.
3958
3959 The next list shows the possible content of a bracket:
3960 (|) OP_*BRA | OP_ALT ... M A
3961 (?()|) OP_*COND | OP_ALT M A
3962 (?>|) OP_ONCE | OP_ALT ... [stack trace] M A
3963 (?>|) OP_ONCE_NC | OP_ALT ... [stack trace] M A
3964 Or nothing, if trace is unnecessary
3965 */
3966
3967 static uschar *compile_bracket_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
3968 {
3969 DEFINE_COMPILER;
3970 fallback_common *fallback;
3971 uschar opcode;
3972 int localptr = 0;
3973 int offset = 0;
3974 int stacksize;
3975 uschar *ccbegin;
3976 uschar *hotpath;
3977 uschar bra = OP_BRA;
3978 uschar ket;
3979 assert_fallback *assert;
3980 BOOL has_alternatives;
3981 struct sljit_jump *jump;
3982 struct sljit_jump *skip;
3983 struct sljit_label *rmaxlabel = NULL;
3984 struct sljit_jump *braminzerojump = NULL;
3985
3986 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
3987
3988 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
3989 {
3990 bra = *cc;
3991 cc++;
3992 opcode = *cc;
3993 }
3994
3995 opcode = *cc;
3996 ccbegin = cc;
3997 hotpath = ccbegin + 1 + LINK_SIZE;
3998
3999 if ((opcode == OP_COND || opcode == OP_SCOND) && cc[1 + LINK_SIZE] == OP_DEF)
4000 {
4001 /* Drop this bracket_fallback. */
4002 parent->top = fallback->prev;
4003 return bracketend(cc);
4004 }
4005
4006 ket = *(bracketend(cc) - 1 - LINK_SIZE);
4007 SLJIT_ASSERT(ket == OP_KET || ket == OP_KETRMAX || ket == OP_KETRMIN);
4008 SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO && ket == OP_KETRMAX)));
4009 cc += GET(cc, 1);
4010
4011 has_alternatives = *cc == OP_ALT;
4012 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
4013 {
4014 has_alternatives = (*hotpath == OP_RREF) ? FALSE : TRUE;
4015 if (*hotpath == OP_NRREF)
4016 {
4017 stacksize = GET2(hotpath, 1);
4018 if (common->currententry == NULL || stacksize == RREF_ANY)
4019 has_alternatives = FALSE;
4020 else if (common->currententry->start == 0)
4021 has_alternatives = stacksize != 0;
4022 else
4023 has_alternatives = stacksize != GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
4024 }
4025 }
4026
4027 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
4028 opcode = OP_SCOND;
4029 if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC))
4030 opcode = OP_ONCE;
4031
4032 if (opcode == OP_CBRA || opcode == OP_SCBRA)
4033 {
4034 /* Capturing brackets has a pre-allocated space. */
4035 offset = GET2(ccbegin, 1 + LINK_SIZE);
4036 localptr = OVECTOR_PRIV(offset);
4037 offset <<= 1;
4038 FALLBACK_AS(bracket_fallback)->localptr = localptr;
4039 hotpath += 2;
4040 }
4041 else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND)
4042 {
4043 /* Other brackets simply allocate the next entry. */
4044 localptr = PRIV(ccbegin);
4045 SLJIT_ASSERT(localptr != 0);
4046 FALLBACK_AS(bracket_fallback)->localptr = localptr;
4047 if (opcode == OP_ONCE)
4048 FALLBACK_AS(bracket_fallback)->u.framesize = get_framesize(common, ccbegin, FALSE);
4049 }
4050
4051 /* Instructions before the first alternative. */
4052 stacksize = 0;
4053 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
4054 stacksize++;
4055 if (bra == OP_BRAZERO)
4056 stacksize++;
4057
4058 if (stacksize > 0)
4059 allocate_stack(common, stacksize);
4060
4061 stacksize = 0;
4062 if ((ket == OP_KETRMAX) || (ket == OP_KETRMIN && bra != OP_BRAMINZERO))
4063 {
4064 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4065 stacksize++;
4066 }
4067
4068 if (bra == OP_BRAZERO)
4069 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
4070
4071 if (bra == OP_BRAMINZERO)
4072 {
4073 /* This is a fallback path! (Since the hot-path of OP_BRAMINZERO matches to the empty string) */
4074 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4075 if (ket != OP_KETRMIN)
4076 {
4077 free_stack(common, 1);
4078 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4079 }
4080 else
4081 {
4082 if (opcode == OP_ONCE || opcode >= OP_SBRA)
4083 {
4084 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4085 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4086 /* Nothing stored during the first run. */
4087 skip = JUMP(SLJIT_JUMP);
4088 JUMPHERE(jump);
4089 /* Checking zero-length iteration. */
4090 if (opcode != OP_ONCE || FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4091 {
4092 /* When we come from outside, localptr contains the previous STR_PTR. */
4093 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4094 }
4095 else
4096 {
4097 /* Except when the whole stack frame must be saved. */
4098 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4099 braminzerojump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (FALLBACK_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w));
4100 }
4101 JUMPHERE(skip);
4102 }
4103 else
4104 {
4105 jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
4106 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4107 JUMPHERE(jump);
4108 }
4109 }
4110 }
4111
4112 if (ket == OP_KETRMIN)
4113 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4114
4115 if (ket == OP_KETRMAX)
4116 {
4117 rmaxlabel = LABEL();
4118 if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA)
4119 FALLBACK_AS(bracket_fallback)->althotpath = rmaxlabel;
4120 }
4121
4122 /* Handling capturing brackets and alternatives. */
4123 if (opcode == OP_ONCE)
4124 {
4125 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4126 {
4127 /* Neither capturing brackets nor recursions are not found in the block. */
4128 if (ket == OP_KETRMIN)
4129 {
4130 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4131 allocate_stack(common, 2);
4132 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4133 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4134 OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
4135 }
4136 else if (ket == OP_KETRMAX || has_alternatives)
4137 {
4138 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4139 allocate_stack(common, 1);
4140 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4141 }
4142 else
4143 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4144 }
4145 else
4146 {
4147 if (ket == OP_KETRMIN || ket == OP_KETRMAX || has_alternatives)
4148 {
4149 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 2);
4150 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4151 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize + 1));
4152 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4153 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4154 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4155 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize + 1, 2, FALSE);
4156 }
4157 else
4158 {
4159 allocate_stack(common, FALLBACK_AS(bracket_fallback)->u.framesize + 1);
4160 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4161 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(FALLBACK_AS(bracket_fallback)->u.framesize));
4162 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4163 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4164 init_frame(common, ccbegin, FALLBACK_AS(bracket_fallback)->u.framesize, 1, FALSE);
4165 }
4166 }
4167 }
4168 else if (opcode == OP_CBRA || opcode == OP_SCBRA)
4169 {
4170 /* Saving the previous values. */
4171 allocate_stack(common, 3);
4172 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4173 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4174 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4175 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4176 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4177 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4178 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0);
4179 }
4180 else if (opcode == OP_SBRA || opcode == OP_SCOND)
4181 {
4182 /* Saving the previous value. */
4183 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4184 allocate_stack(common, 1);
4185 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0);
4186 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
4187 }
4188 else if (has_alternatives)
4189 {
4190 /* Pushing the starting string pointer. */
4191 allocate_stack(common, 1);
4192 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4193 }
4194
4195 /* Generating code for the first alternative. */
4196 if (opcode == OP_COND || opcode == OP_SCOND)
4197 {
4198 if (*hotpath == OP_CREF)
4199 {
4200 SLJIT_ASSERT(has_alternatives);
4201 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed),
4202 CMP(SLJIT_C_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(GET2(hotpath, 1) << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
4203 hotpath += 3;
4204 }
4205 else if (*hotpath == OP_NCREF)
4206 {
4207 SLJIT_ASSERT(has_alternatives);
4208 stacksize = GET2(hotpath, 1);
4209 jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(stacksize << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
4210
4211 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
4212 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
4213 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
4214 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, stacksize);
4215 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0);
4216 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, common->name_table);
4217 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchovector));
4218 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4219 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, 0));
4220
4221 JUMPHERE(jump);
4222 hotpath += 3;
4223 }
4224 else if (*hotpath == OP_RREF || *hotpath == OP_NRREF)
4225 {
4226 /* Never has other case. */
4227 FALLBACK_AS(bracket_fallback)->u.condfailed = NULL;
4228
4229 stacksize = GET2(hotpath, 1);
4230 if (common->currententry == NULL)
4231 stacksize = 0;
4232 else if (stacksize == RREF_ANY)
4233 stacksize = 1;
4234 else if (common->currententry->start == 0)
4235 stacksize = stacksize == 0;
4236 else
4237 stacksize = stacksize == GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
4238
4239 if (*hotpath == OP_RREF || stacksize || common->currententry == NULL)
4240 {
4241 SLJIT_ASSERT(!has_alternatives);
4242 if (stacksize != 0)
4243 hotpath += 3;
4244 else
4245 {
4246 if (*cc == OP_ALT)
4247 {
4248 hotpath = cc + 1 + LINK_SIZE;
4249 cc += GET(cc, 1);
4250 }
4251 else
4252 hotpath = cc;
4253 }
4254 }
4255 else
4256 {
4257 SLJIT_ASSERT(has_alternatives);
4258
4259 stacksize = GET2(hotpath, 1);
4260 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
4261 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
4262 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
4263 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, GET2(common->start, common->currententry->start + 1 + LINK_SIZE));
4264 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, stacksize);
4265 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_LOCALS_REG, 0);
4266 OP1(SLJIT_MOV, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, common->name_table);
4267 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchgroups));
4268 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4269 add_jump(compiler, &(FALLBACK_AS(bracket_fallback)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_TEMPORARY_REG1, 0, SLJIT_IMM, 0));
4270 hotpath += 3;
4271 }
4272 }
4273 else
4274 {
4275 SLJIT_ASSERT(has_alternatives && *hotpath >= OP_ASSERT && *hotpath <= OP_ASSERTBACK_NOT);
4276 /* Similar code as PUSH_FALLBACK macro. */
4277 assert = sljit_alloc_memory(compiler, sizeof(assert_fallback));
4278 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4279 return NULL;
4280 memset(assert, 0, sizeof(assert_fallback));
4281 assert->common.cc = hotpath;
4282 FALLBACK_AS(bracket_fallback)->u.assert = assert;
4283 hotpath = compile_assert_hotpath(common, hotpath, assert, TRUE);
4284 }
4285 }
4286
4287 compile_hotpath(common, hotpath, cc, fallback);
4288 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4289 return NULL;
4290
4291 if (opcode == OP_ONCE)
4292 {
4293 if (FALLBACK_AS(bracket_fallback)->u.framesize < 0)
4294 {
4295 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4296 /* TMP2 which is set here used by OP_KETRMAX below. */
4297 if (ket == OP_KETRMAX)
4298 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
4299 else if (ket == OP_KETRMIN)
4300 {
4301 /* Move the STR_PTR to the localptr. */
4302 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
4303 }
4304 }
4305 else
4306 {
4307 stacksize = (ket == OP_KETRMIN || ket == OP_KETRMAX || has_alternatives) ? 2 : 1;
4308 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (FALLBACK_AS(bracket_fallback)->u.framesize + stacksize) * sizeof(sljit_w));
4309 if (ket == OP_KETRMAX)
4310 {
4311 /* TMP2 which is set here used by OP_KETRMAX below. */
4312 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4313 }
4314 }
4315 }
4316
4317 stacksize = 0;
4318 if (ket != OP_KET || bra != OP_BRA)
4319 stacksize++;
4320 if (has_alternatives && opcode != OP_ONCE)
4321 stacksize++;
4322
4323 if (stacksize > 0)
4324 allocate_stack(common, stacksize);
4325
4326 stacksize = 0;
4327 if (ket != OP_KET)
4328 {
4329 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
4330 stacksize++;
4331 }
4332 else if (bra != OP_BRA)
4333 {
4334 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4335 stacksize++;
4336 }
4337
4338 if (has_alternatives)
4339 {
4340 if (opcode != OP_ONCE)
4341 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
4342 if (ket != OP_KETRMAX)
4343 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4344 }
4345
4346 /* Must be after the hotpath label. */
4347 if (offset != 0)
4348 {
4349 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4350 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4351 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
4352 }
4353
4354 if (ket == OP_KETRMAX)
4355 {
4356 if (opcode == OP_ONCE || opcode >= OP_SBRA)
4357 {
4358 if (has_alternatives)
4359 FALLBACK_AS(bracket_fallback)->althotpath = LABEL();
4360 /* Checking zero-length iteration. */
4361 if (opcode != OP_ONCE)
4362 CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STR_PTR, 0, rmaxlabel);
4363 else
4364 /* TMP2 must contain the starting STR_PTR. */
4365 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmaxlabel);
4366 }
4367 else
4368 JUMPTO(SLJIT_JUMP, rmaxlabel);
4369 FALLBACK_AS(bracket_fallback)->recursivehotpath = LABEL();
4370 }
4371
4372 if (bra == OP_BRAZERO)
4373 FALLBACK_AS(bracket_fallback)->zerohotpath = LABEL();
4374
4375 if (bra == OP_BRAMINZERO)
4376 {
4377 /* This is a fallback path! (From the viewpoint of OP_BRAMINZERO) */
4378 JUMPTO(SLJIT_JUMP, ((braminzero_fallback*)parent)->hotpath);
4379 if (braminzerojump != NULL)
4380 {
4381 JUMPHERE(braminzerojump);
4382 /* We need to release the end pointer to perform the
4383 fallback for the zero-length iteration. When
4384 framesize is < 0, OP_ONCE will do the release itself. */
4385 if (opcode == OP_ONCE && FALLBACK_AS(bracket_fallback)->u.framesize >= 0)
4386 {
4387 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4388 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
4389 }
4390 else if (ket == OP_KETRMIN && opcode != OP_ONCE)
4391 free_stack(common, 1);
4392 }
4393 /* Continue to the normal fallback. */
4394 }
4395
4396 if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO)
4397 decrease_call_count(common);
4398
4399 /* Skip the other alternatives. */
4400 while (*cc == OP_ALT)
4401 cc += GET(cc, 1);
4402 cc += 1 + LINK_SIZE;
4403 return cc;
4404 }
4405
4406 static uschar *compile_bracketpos_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4407 {
4408 DEFINE_COMPILER;
4409 fallback_common *fallback;
4410 uschar opcode;
4411 int localptr;
4412 int cbraprivptr = 0;
4413 int framesize;
4414 int stacksize;
4415 int offset = 0;
4416 BOOL zero = FALSE;
4417 uschar *ccbegin = NULL;
4418 int stack;
4419 struct sljit_label *loop = NULL;
4420 struct jump_list *emptymatch = NULL;
4421
4422 PUSH_FALLBACK(sizeof(bracketpos_fallback), cc, NULL);
4423 if (*cc == OP_BRAPOSZERO)
4424 {
4425 zero = TRUE;
4426 cc++;
4427 }
4428
4429 opcode = *cc;
4430 localptr = PRIV(cc);
4431 SLJIT_ASSERT(localptr != 0);
4432 FALLBACK_AS(bracketpos_fallback)->localptr = localptr;
4433 switch(opcode)
4434 {
4435 case OP_BRAPOS:
4436 case OP_SBRAPOS:
4437 ccbegin = cc + 1 + LINK_SIZE;
4438 break;
4439
4440 case OP_CBRAPOS:
4441 case OP_SCBRAPOS:
4442 offset = GET2(cc, 1 + LINK_SIZE);
4443 cbraprivptr = OVECTOR_PRIV(offset);
4444 offset <<= 1;
4445 ccbegin = cc + 1 + LINK_SIZE + 2;
4446 break;
4447
4448 default:
4449 SLJIT_ASSERT_STOP();
4450 break;
4451 }
4452
4453 framesize = get_framesize(common, cc, FALSE);
4454 FALLBACK_AS(bracketpos_fallback)->framesize = framesize;
4455 if (framesize < 0)
4456 {
4457 stacksize = (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS) ? 2 : 1;
4458 if (!zero)
4459 stacksize++;
4460 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4461 allocate_stack(common, stacksize);
4462 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, STACK_TOP, 0);
4463
4464 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4465 {
4466 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
4467 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
4468 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4469 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
4470 }
4471 else
4472 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4473
4474 if (!zero)
4475 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 1);
4476 }
4477 else
4478 {
4479 stacksize = framesize + 1;
4480 if (!zero)
4481 stacksize++;
4482 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4483 stacksize++;
4484 FALLBACK_AS(bracketpos_fallback)->stacksize = stacksize;
4485 allocate_stack(common, stacksize);
4486
4487 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4488 OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
4489 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP2, 0);
4490 stack = 0;
4491 if (!zero)
4492 {
4493 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 1);
4494 stack++;
4495 }
4496 if (opcode == OP_BRAPOS || opcode == OP_SBRAPOS)
4497 {
4498 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), STR_PTR, 0);
4499 stack++;
4500 }
4501 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP1, 0);
4502 init_frame(common, cc, stacksize - 1, stacksize - framesize, FALSE);
4503 }
4504
4505 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4506 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4507
4508 loop = LABEL();
4509 while (*cc != OP_KETRPOS)
4510 {
4511 fallback->top = NULL;
4512 fallback->topfallbacks = NULL;
4513 cc += GET(cc, 1);
4514
4515 compile_hotpath(common, ccbegin, cc, fallback);
4516 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4517 return NULL;
4518
4519 if (framesize < 0)
4520 {
4521 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4522
4523 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4524 {
4525 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4526 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4527 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4528 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4529 }
4530 else
4531 {
4532 if (opcode == OP_SBRAPOS)
4533 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4534 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4535 }
4536
4537 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4538 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4539
4540 if (!zero)
4541 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4542 }
4543 else
4544 {
4545 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4546 {
4547 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, stacksize * sizeof(sljit_w));
4548 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4549 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4550 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
4551 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4552 }
4553 else
4554 {
4555 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4556 OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_w));
4557 if (opcode == OP_SBRAPOS)
4558 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4559 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w), STR_PTR, 0);
4560 }
4561
4562 if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
4563 add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
4564
4565 if (!zero)
4566 {
4567 if (framesize < 0)
4568 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
4569 else
4570 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4571 }
4572 }
4573 JUMPTO(SLJIT_JUMP, loop);
4574 flush_stubs(common);
4575
4576 compile_fallbackpath(common, fallback->top);
4577 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
4578 return NULL;
4579 set_jumps(fallback->topfallbacks, LABEL());
4580
4581 if (framesize < 0)
4582 {
4583 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4584 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4585 else
4586 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4587 }
4588 else
4589 {
4590 if (opcode == OP_CBRAPOS || opcode == OP_SCBRAPOS)
4591 {
4592 /* Last alternative. */
4593 if (*cc == OP_KETRPOS)
4594 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4595 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
4596 }
4597 else
4598 {
4599 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
4600 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_w));
4601 }
4602 }
4603
4604 if (*cc == OP_KETRPOS)
4605 break;
4606 ccbegin = cc + 1 + LINK_SIZE;
4607 }
4608
4609 fallback->topfallbacks = NULL;
4610 if (!zero)
4611 {
4612 if (framesize < 0)
4613 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
4614 else /* TMP2 is set to [localptr] above. */
4615 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_w), SLJIT_IMM, 0));
4616 }
4617
4618 /* None of them matched. */
4619 set_jumps(emptymatch, LABEL());
4620 decrease_call_count(common);
4621 return cc + 1 + LINK_SIZE;
4622 }
4623
4624 static SLJIT_INLINE uschar *get_iterator_parameters(compiler_common *common, uschar *cc, uschar *opcode, uschar *type, int *arg1, int *arg2, uschar **end)
4625 {
4626 int class_len;
4627
4628 *opcode = *cc;
4629 if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO)
4630 {
4631 cc++;
4632 *type = OP_CHAR;
4633 }
4634 else if (*opcode >= OP_STARI && *opcode <= OP_POSUPTOI)
4635 {
4636 cc++;
4637 *type = OP_CHARI;
4638 *opcode -= OP_STARI - OP_STAR;
4639 }
4640 else if (*opcode >= OP_NOTSTAR && *opcode <= OP_NOTPOSUPTO)
4641 {
4642 cc++;
4643 *type = OP_NOT;
4644 *opcode -= OP_NOTSTAR - OP_STAR;
4645 }
4646 else if (*opcode >= OP_NOTSTARI && *opcode <= OP_NOTPOSUPTOI)
4647 {
4648 cc++;
4649 *type = OP_NOTI;
4650 *opcode -= OP_NOTSTARI - OP_STAR;
4651 }
4652 else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO)
4653 {
4654 cc++;
4655 *opcode -= OP_TYPESTAR - OP_STAR;
4656 *type = 0;
4657 }
4658 else
4659 {
4660 SLJIT_ASSERT(*opcode >= OP_CLASS || *opcode <= OP_XCLASS);
4661 *type = *opcode;
4662 cc++;
4663 class_len = (*type < OP_XCLASS) ? 33 : GET(cc, 0);
4664 *opcode = cc[class_len - 1];
4665 if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY)
4666 {
4667 *opcode -= OP_CRSTAR - OP_STAR;
4668 if (end != NULL)
4669 *end = cc + class_len;
4670 }
4671 else
4672 {
4673 SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE);
4674 *arg1 = GET2(cc, (class_len + 2));
4675 *arg2 = GET2(cc, class_len);
4676
4677 if (*arg2 == 0)
4678 {
4679 SLJIT_ASSERT(*arg1 != 0);
4680 *opcode = (*opcode == OP_CRRANGE) ? OP_UPTO : OP_MINUPTO;
4681 }
4682 if (*arg1 == *arg2)
4683 *opcode = OP_EXACT;
4684
4685 if (end != NULL)
4686 *end = cc + class_len + 4;
4687 }
4688 return cc;
4689 }
4690
4691 if (*opcode == OP_UPTO || *opcode == OP_MINUPTO || *opcode == OP_EXACT || *opcode == OP_POSUPTO)
4692 {
4693 *arg1 = GET2(cc, 0);
4694 cc += 2;
4695 }
4696
4697 if (*type == 0)
4698 {
4699 *type = *cc;
4700 if (end != NULL)
4701 *end = next_opcode(common, cc);
4702 cc++;
4703 return cc;
4704 }
4705
4706 if (end != NULL)
4707 {
4708 *end = cc + 1;
4709 #ifdef SUPPORT_UTF8
4710 if (common->utf8 && *cc >= 0xc0) *end += _pcre_utf8_table4[*cc & 0x3f];
4711 #endif
4712 }
4713 return cc;
4714 }
4715
4716 static uschar *compile_iterator_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4717 {
4718 DEFINE_COMPILER;
4719 fallback_common *fallback;
4720 uschar opcode;
4721 uschar type;
4722 int arg1 = -1, arg2 = -1;
4723 uschar* end;
4724 jump_list *nomatch = NULL;
4725 struct sljit_jump *jump = NULL;
4726 struct sljit_label *label;
4727
4728 PUSH_FALLBACK(sizeof(iterator_fallback), cc, NULL);
4729
4730 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, &end);
4731
4732 switch(opcode)
4733 {
4734 case OP_STAR:
4735 case OP_PLUS:
4736 case OP_UPTO:
4737 case OP_CRRANGE:
4738 if (type == OP_ANYNL || type == OP_EXTUNI)
4739 {
4740 if (opcode == OP_STAR || opcode == OP_UPTO)
4741 {
4742 allocate_stack(common, 2);
4743 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4744 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
4745 }
4746 else
4747 {
4748 allocate_stack(common, 1);
4749 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
4750 }
4751 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4752 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
4753
4754 label = LABEL();
4755 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4756 if (opcode == OP_UPTO || opcode == OP_CRRANGE)
4757 {
4758 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4759 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4760 if (opcode == OP_CRRANGE && arg2 > 0)
4761 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2, label);
4762 if (opcode == OP_UPTO || (opcode == OP_CRRANGE && arg1 > 0))
4763 jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, arg1);
4764 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4765 }
4766
4767 allocate_stack(common, 1);
4768 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4769 JUMPTO(SLJIT_JUMP, label);
4770 if (jump != NULL)
4771 JUMPHERE(jump);
4772 }
4773 else
4774 {
4775 allocate_stack(common, 2);
4776 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4777 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4778 label = LABEL();
4779 compile_char1_hotpath(common, type, cc, &nomatch);
4780 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4781 if (opcode <= OP_PLUS || (opcode == OP_CRRANGE && arg1 == 0))
4782 {
4783 OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4784 JUMPTO(SLJIT_JUMP, label);
4785 }
4786 else
4787 {
4788 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
4789 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4790 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
4791 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4792 }
4793 set_jumps(nomatch, LABEL());
4794 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
4795 add_jump(compiler, &fallback->topfallbacks,
4796 CMP(SLJIT_C_LESS, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, opcode == OP_PLUS ? 2 : arg2 + 1));
4797 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
4798 }
4799 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4800 break;
4801
4802 case OP_MINSTAR:
4803 case OP_MINPLUS:
4804 allocate_stack(common, 1);
4805 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4806 if (opcode == OP_MINPLUS)
4807 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4808 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4809 break;
4810
4811 case OP_MINUPTO:
4812 case OP_CRMINRANGE:
4813 allocate_stack(common, 2);
4814 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4815 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
4816 if (opcode == OP_CRMINRANGE)
4817 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4818 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4819 break;
4820
4821 case OP_QUERY:
4822 case OP_MINQUERY:
4823 allocate_stack(common, 1);
4824 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
4825 if (opcode == OP_QUERY)
4826 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4827 FALLBACK_AS(iterator_fallback)->hotpath = LABEL();
4828 break;
4829
4830 case OP_EXACT:
4831 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4832 label = LABEL();
4833 compile_char1_hotpath(common, type, cc, &fallback->topfallbacks);
4834 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4835 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4836 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4837 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4838 break;
4839
4840 case OP_POSSTAR:
4841 case OP_POSPLUS:
4842 case OP_POSUPTO:
4843 if (opcode != OP_POSSTAR)
4844 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 1);
4845 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4846 label = LABEL();
4847 compile_char1_hotpath(common, type, cc, &nomatch);
4848 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4849 if (opcode != OP_POSUPTO)
4850 {
4851 if (opcode == OP_POSPLUS)
4852 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2);
4853 JUMPTO(SLJIT_JUMP, label);
4854 }
4855 else
4856 {
4857 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
4858 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
4859 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
4860 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
4861 }
4862 set_jumps(nomatch, LABEL());
4863 if (opcode == OP_POSPLUS)
4864 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_LESS, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 2));
4865 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4866 break;
4867
4868 case OP_POSQUERY:
4869 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4870 compile_char1_hotpath(common, type, cc, &nomatch);
4871 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STR_PTR, 0);
4872 set_jumps(nomatch, LABEL());
4873 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
4874 break;
4875
4876 default:
4877 SLJIT_ASSERT_STOP();
4878 break;
4879 }
4880
4881 decrease_call_count(common);
4882 return end;
4883 }
4884
4885 static SLJIT_INLINE uschar *compile_fail_accept_hotpath(compiler_common *common, uschar *cc, fallback_common *parent)
4886 {
4887 DEFINE_COMPILER;
4888 fallback_common *fallback;
4889
4890 PUSH_FALLBACK(sizeof(bracket_fallback), cc, NULL);
4891
4892 if (*cc == OP_FAIL)
4893 {
4894 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4895 return cc + 1;
4896 }
4897
4898 if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL)
4899 {
4900 /* No need to check notempty conditions. */
4901 if (common->acceptlabel == NULL)
4902 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
4903 else
4904 JUMPTO(SLJIT_JUMP, common->acceptlabel);
4905 return cc + 1;
4906 }
4907
4908 if (common->acceptlabel == NULL)
4909 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0)));
4910 else
4911 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), common->acceptlabel);
4912 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
4913 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
4914 add_jump(compiler, &fallback->topfallbacks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4915 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
4916 if (common->acceptlabel == NULL)
4917 add_jump(compiler, &common->accept, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0));
4918 else
4919 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->acceptlabel);
4920 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
4921 if (common->acceptlabel == NULL)
4922 add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0));
4923 else
4924 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->acceptlabel);
4925 add_jump(compiler, &fallback->topfallbacks, JUMP(SLJIT_JUMP));
4926 return cc + 1;
4927 }
4928
4929 static SLJIT_INLINE uschar *compile_close_hotpath(compiler_common *common, uschar *cc)
4930 {
4931 DEFINE_COMPILER;
4932 int offset = GET2(cc, 1);
4933
4934 /* Data will be discarded anyway... */
4935 if (common->currententry != NULL)
4936 return cc + 3;
4937
4938 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR_PRIV(offset));
4939 offset <<= 1;
4940 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
4941 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
4942 return cc + 3;
4943 }
4944
4945 static void compile_hotpath(compiler_common *common, uschar *cc, uschar *ccend, fallback_common *parent)
4946 {
4947 DEFINE_COMPILER;
4948 fallback_common *fallback;
4949
4950 while (cc < ccend)
4951 {
4952 switch(*cc)
4953 {
4954 case OP_SOD:
4955 case OP_SOM:
4956 case OP_NOT_WORD_BOUNDARY:
4957 case OP_WORD_BOUNDARY:
4958 case OP_NOT_DIGIT:
4959 case OP_DIGIT:
4960 case OP_NOT_WHITESPACE:
4961 case OP_WHITESPACE:
4962 case OP_NOT_WORDCHAR:
4963 case OP_WORDCHAR:
4964 case OP_ANY:
4965 case OP_ALLANY:
4966 case OP_ANYBYTE:
4967 case OP_NOTPROP:
4968 case OP_PROP:
4969 case OP_ANYNL:
4970 case OP_NOT_HSPACE:
4971 case OP_HSPACE:
4972 case OP_NOT_VSPACE:
4973 case OP_VSPACE:
4974 case OP_EXTUNI:
4975 case OP_EODN:
4976 case OP_EOD:
4977 case OP_CIRC:
4978 case OP_CIRCM:
4979 case OP_DOLL:
4980 case OP_DOLLM:
4981 case OP_NOT:
4982 case OP_NOTI:
4983 case OP_REVERSE:
4984 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4985 break;
4986
4987 case OP_SET_SOM:
4988 PUSH_FALLBACK_NOVALUE(sizeof(fallback_common), cc);
4989 allocate_stack(common, 1);
4990 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
4991 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
4992 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
4993 cc++;
4994 break;
4995
4996 case OP_CHAR:
4997 case OP_CHARI:
4998 cc = compile_charn_hotpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
4999 break;
5000
5001 case OP_STAR:
5002 case OP_MINSTAR:
5003 case OP_PLUS:
5004 case OP_MINPLUS:
5005 case OP_QUERY:
5006 case OP_MINQUERY:
5007 case OP_UPTO:
5008 case OP_MINUPTO:
5009 case OP_EXACT:
5010 case OP_POSSTAR:
5011 case OP_POSPLUS:
5012 case OP_POSQUERY:
5013 case OP_POSUPTO:
5014 case OP_STARI:
5015 case OP_MINSTARI:
5016 case OP_PLUSI:
5017 case OP_MINPLUSI:
5018 case OP_QUERYI:
5019 case OP_MINQUERYI:
5020 case OP_UPTOI:
5021 case OP_MINUPTOI:
5022 case OP_EXACTI:
5023 case OP_POSSTARI:
5024 case OP_POSPLUSI:
5025 case OP_POSQUERYI:
5026 case OP_POSUPTOI:
5027 case OP_NOTSTAR:
5028 case OP_NOTMINSTAR:
5029 case OP_NOTPLUS:
5030 case OP_NOTMINPLUS:
5031 case OP_NOTQUERY:
5032 case OP_NOTMINQUERY:
5033 case OP_NOTUPTO:
5034 case OP_NOTMINUPTO:
5035 case OP_NOTEXACT:
5036 case OP_NOTPOSSTAR:
5037 case OP_NOTPOSPLUS:
5038 case OP_NOTPOSQUERY:
5039 case OP_NOTPOSUPTO:
5040 case OP_NOTSTARI:
5041 case OP_NOTMINSTARI:
5042 case OP_NOTPLUSI:
5043 case OP_NOTMINPLUSI:
5044 case OP_NOTQUERYI:
5045 case OP_NOTMINQUERYI:
5046 case OP_NOTUPTOI:
5047 case OP_NOTMINUPTOI:
5048 case OP_NOTEXACTI:
5049 case OP_NOTPOSSTARI:
5050 case OP_NOTPOSPLUSI:
5051 case OP_NOTPOSQUERYI:
5052 case OP_NOTPOSUPTOI:
5053 case OP_TYPESTAR:
5054 case OP_TYPEMINSTAR:
5055 case OP_TYPEPLUS:
5056 case OP_TYPEMINPLUS:
5057 case OP_TYPEQUERY:
5058 case OP_TYPEMINQUERY:
5059 case OP_TYPEUPTO:
5060 case OP_TYPEMINUPTO:
5061 case OP_TYPEEXACT:
5062 case OP_TYPEPOSSTAR:
5063 case OP_TYPEPOSPLUS:
5064 case OP_TYPEPOSQUERY:
5065 case OP_TYPEPOSUPTO:
5066 cc = compile_iterator_hotpath(common, cc, parent);
5067 break;
5068
5069 case OP_CLASS:
5070 case OP_NCLASS:
5071 if (cc[33] >= OP_CRSTAR && cc[33] <= OP_CRMINRANGE)
5072 cc = compile_iterator_hotpath(common, cc, parent);
5073 else
5074 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
5075 break;
5076
5077 #ifdef SUPPORT_UTF8
5078 case OP_XCLASS:
5079 if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRMINRANGE)
5080 cc = compile_iterator_hotpath(common, cc, parent);
5081 else
5082 cc = compile_char1_hotpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks);
5083 break;
5084 #endif
5085
5086 case OP_REF:
5087 case OP_REFI:
5088 if (cc[3] >= OP_CRSTAR && cc[3] <= OP_CRMINRANGE)
5089 cc = compile_ref_iterator_hotpath(common, cc, parent);
5090 else
5091 cc = compile_ref_hotpath(common, cc, parent->top != NULL ? &parent->top->nextfallbacks : &parent->topfallbacks, TRUE, FALSE);
5092 break;
5093
5094 case OP_RECURSE:
5095 cc = compile_recurse_hotpath(common, cc, parent);
5096 break;
5097
5098 case OP_ASSERT:
5099 case OP_ASSERT_NOT:
5100 case OP_ASSERTBACK:
5101 case OP_ASSERTBACK_NOT:
5102 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
5103 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
5104 break;
5105
5106 case OP_BRAMINZERO:
5107 PUSH_FALLBACK_NOVALUE(sizeof(braminzero_fallback), cc);
5108 cc = bracketend(cc + 1);
5109 if (*(cc - 1 - LINK_SIZE) != OP_KETRMIN)
5110 {
5111 allocate_stack(common, 1);
5112 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5113 }
5114 else
5115 {
5116 allocate_stack(common, 2);
5117 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5118 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0);
5119 }
5120 FALLBACK_AS(braminzero_fallback)->hotpath = LABEL();
5121 if (cc[1] > OP_ASSERTBACK_NOT)
5122 decrease_call_count(common);
5123 break;
5124
5125 case OP_ONCE:
5126 case OP_ONCE_NC:
5127 case OP_BRA:
5128 case OP_CBRA:
5129 case OP_COND:
5130 case OP_SBRA:
5131 case OP_SCBRA:
5132 case OP_SCOND:
5133 cc = compile_bracket_hotpath(common, cc, parent);
5134 break;
5135
5136 case OP_BRAZERO:
5137 if (cc[1] > OP_ASSERTBACK_NOT)
5138 cc = compile_bracket_hotpath(common, cc, parent);
5139 else
5140 {
5141 PUSH_FALLBACK_NOVALUE(sizeof(assert_fallback), cc);
5142 cc = compile_assert_hotpath(common, cc, FALLBACK_AS(assert_fallback), FALSE);
5143 }
5144 break;
5145
5146 case OP_BRAPOS:
5147 case OP_CBRAPOS:
5148 case OP_SBRAPOS:
5149 case OP_SCBRAPOS:
5150 case OP_BRAPOSZERO:
5151 cc = compile_bracketpos_hotpath(common, cc, parent);
5152 break;
5153
5154 case OP_FAIL:
5155 case OP_ACCEPT:
5156 case OP_ASSERT_ACCEPT:
5157 cc = compile_fail_accept_hotpath(common, cc, parent);
5158 break;
5159
5160 case OP_CLOSE:
5161 cc = compile_close_hotpath(common, cc);
5162 break;
5163
5164 case OP_SKIPZERO:
5165 cc = bracketend(cc + 1);
5166 break;
5167
5168 default:
5169 SLJIT_ASSERT_STOP();
5170 return;
5171 }
5172 if (cc == NULL)
5173 return;
5174 }
5175 SLJIT_ASSERT(cc == ccend);
5176 }
5177
5178 #undef PUSH_FALLBACK
5179 #undef PUSH_FALLBACK_NOVALUE
5180 #undef FALLBACK_AS
5181
5182 #define COMPILE_FALLBACKPATH(current) \
5183 do \
5184 { \
5185 compile_fallbackpath(common, (current)); \
5186 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \
5187 return; \
5188 } \
5189 while (0)
5190
5191 #define CURRENT_AS(type) ((type*)current)
5192
5193 static void compile_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5194 {
5195 DEFINE_COMPILER;
5196 uschar *cc = current->cc;
5197 uschar opcode;
5198 uschar type;
5199 int arg1 = -1, arg2 = -1;
5200 struct sljit_label *label = NULL;
5201 struct sljit_jump *jump = NULL;
5202
5203 cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, NULL);
5204
5205 switch(opcode)
5206 {
5207 case OP_STAR:
5208 case OP_PLUS:
5209 case OP_UPTO:
5210 case OP_CRRANGE:
5211 if (type == OP_ANYNL || type == OP_EXTUNI)
5212 {
5213 set_jumps(current->topfallbacks, LABEL());
5214 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5215 free_stack(common, 1);
5216 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5217 }
5218 else
5219 {
5220 if (opcode == OP_STAR || opcode == OP_UPTO)
5221 arg2 = 0;
5222 else if (opcode == OP_PLUS)
5223 arg2 = 1;
5224 jump = CMP(SLJIT_C_LESS_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, arg2 + 1);
5225 OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
5226 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5227 skip_char_back(common);
5228 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5229 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5230 if (opcode == OP_PLUS || opcode == OP_CRRANGE)
5231 set_jumps(current->topfallbacks, LABEL());
5232 JUMPHERE(jump);
5233 free_stack(common, 2);
5234 }
5235 break;
5236
5237 case OP_MINSTAR:
5238 case OP_MINPLUS:
5239 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5240 if (opcode == OP_MINPLUS)
5241 {
5242 set_jumps(current->topfallbacks, LABEL());
5243 current->topfallbacks = NULL;
5244 }
5245 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5246 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5247 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5248 set_jumps(current->topfallbacks, LABEL());
5249 free_stack(common, 1);
5250 break;
5251
5252 case OP_MINUPTO:
5253 case OP_CRMINRANGE:
5254 if (opcode == OP_CRMINRANGE)
5255 {
5256 set_jumps(current->topfallbacks, LABEL());
5257 current->topfallbacks = NULL;
5258 label = LABEL();
5259 }
5260 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5261 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5262
5263 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5264 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
5265 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
5266 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
5267
5268 if (opcode == OP_CRMINRANGE)
5269 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2 + 1, label);
5270
5271 if (opcode == OP_CRMINRANGE && arg1 == 0)
5272 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5273 else
5274 CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 2, CURRENT_AS(iterator_fallback)->hotpath);
5275
5276 set_jumps(current->topfallbacks, LABEL());
5277 free_stack(common, 2);
5278 break;
5279
5280 case OP_QUERY:
5281 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5282 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5283 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5284 jump = JUMP(SLJIT_JUMP);
5285 set_jumps(current->topfallbacks, LABEL());
5286 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5287 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5288 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5289 JUMPHERE(jump);
5290 free_stack(common, 1);
5291 break;
5292
5293 case OP_MINQUERY:
5294 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5295 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5296 jump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5297 compile_char1_hotpath(common, type, cc, &current->topfallbacks);
5298 JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_fallback)->hotpath);
5299 set_jumps(current->topfallbacks, LABEL());
5300 JUMPHERE(jump);
5301 free_stack(common, 1);
5302 break;
5303
5304 case OP_EXACT:
5305 case OP_POSPLUS:
5306 set_jumps(current->topfallbacks, LABEL());
5307 break;
5308
5309 case OP_POSSTAR:
5310 case OP_POSQUERY:
5311 case OP_POSUPTO:
5312 break;
5313
5314 default:
5315 SLJIT_ASSERT_STOP();
5316 break;
5317 }
5318 }
5319
5320 static void compile_ref_iterator_fallbackpath(compiler_common *common, struct fallback_common *current)
5321 {
5322 DEFINE_COMPILER;
5323 uschar *cc = current->cc;
5324 uschar type;
5325
5326 type = cc[3];
5327 if ((type & 0x1) == 0)
5328 {
5329 set_jumps(current->topfallbacks, LABEL());
5330 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5331 free_stack(common, 1);
5332 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5333 return;
5334 }
5335
5336 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5337 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_fallback)->hotpath);
5338 set_jumps(current->topfallbacks, LABEL());
5339 free_stack(common, 2);
5340 }
5341
5342 static void compile_recurse_fallbackpath(compiler_common *common, struct fallback_common *current)
5343 {
5344 DEFINE_COMPILER;
5345
5346 set_jumps(current->topfallbacks, LABEL());
5347 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5348 free_stack(common, 1);
5349 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
5350 }
5351
5352 static void compile_assert_fallbackpath(compiler_common *common, struct fallback_common *current)
5353 {
5354 DEFINE_COMPILER;
5355 uschar *cc = current->cc;
5356 uschar bra = OP_BRA;
5357 struct sljit_jump *brajump = NULL;
5358
5359 SLJIT_ASSERT(*cc != OP_BRAMINZERO);
5360 if (*cc == OP_BRAZERO)
5361 {
5362 bra = *cc;
5363 cc++;
5364 }
5365
5366 if (bra == OP_BRAZERO)
5367 {
5368 SLJIT_ASSERT(current->topfallbacks == NULL);
5369 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5370 }
5371
5372 if (CURRENT_AS(assert_fallback)->framesize < 0)
5373 {
5374 set_jumps(current->topfallbacks, LABEL());
5375
5376 if (bra == OP_BRAZERO)
5377 {
5378 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5379 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5380 free_stack(common, 1);
5381 }
5382 return;
5383 }
5384
5385 if (bra == OP_BRAZERO)
5386 {
5387 if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT)
5388 {
5389 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5390 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_fallback)->hotpath);
5391 free_stack(common, 1);
5392 return;
5393 }
5394 free_stack(common, 1);
5395 brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
5396 }
5397
5398 if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK)
5399 {
5400 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr);
5401 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5402 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_fallback)->framesize * sizeof(sljit_w));
5403
5404 set_jumps(current->topfallbacks, LABEL());
5405 }
5406 else
5407 set_jumps(current->topfallbacks, LABEL());
5408
5409 if (bra == OP_BRAZERO)
5410 {
5411 /* We know there is enough place on the stack. */
5412 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5413 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
5414 JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_fallback)->hotpath);
5415 JUMPHERE(brajump);
5416 }
5417 }
5418
5419 static void compile_bracket_fallbackpath(compiler_common *common, struct fallback_common *current)
5420 {
5421 DEFINE_COMPILER;
5422 int opcode;
5423 int offset = 0;
5424 int localptr = CURRENT_AS(bracket_fallback)->localptr;
5425 int stacksize;
5426 int count;
5427 uschar *cc = current->cc;
5428 uschar *ccbegin;
5429 uschar *ccprev;
5430 jump_list *jumplist = NULL;
5431 jump_list *jumplistitem = NULL;
5432 uschar bra = OP_BRA;
5433 uschar ket;
5434 assert_fallback *assert;
5435 BOOL has_alternatives;
5436 struct sljit_jump *brazero = NULL;
5437 struct sljit_jump *once = NULL;
5438 struct sljit_jump *cond = NULL;
5439 struct sljit_label *rminlabel = NULL;
5440
5441 if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO)
5442 {
5443 bra = *cc;
5444 cc++;
5445 }
5446
5447 opcode = *cc;
5448 ccbegin = cc;
5449 ket = *(bracketend(ccbegin) - 1 - LINK_SIZE);
5450 cc += GET(cc, 1);
5451 has_alternatives = *cc == OP_ALT;
5452 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5453 has_alternatives = (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) || CURRENT_AS(bracket_fallback)->u.condfailed != NULL;
5454 if (opcode == OP_CBRA || opcode == OP_SCBRA)
5455 offset = (GET2(ccbegin, 1 + LINK_SIZE)) << 1;
5456 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
5457 opcode = OP_SCOND;
5458 if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC))
5459 opcode = OP_ONCE;
5460
5461 if (ket == OP_KETRMAX)
5462 {
5463 if (bra != OP_BRAZERO)
5464 free_stack(common, 1);
5465 else
5466 {
5467 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5468 free_stack(common, 1);
5469 brazero = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5470 }
5471 }
5472 else if (ket == OP_KETRMIN)
5473 {
5474 if (bra != OP_BRAMINZERO)
5475 {
5476 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5477 if (opcode >= OP_SBRA || opcode == OP_ONCE)
5478 {
5479 /* Checking zero-length iteration. */
5480 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize < 0)
5481 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, CURRENT_AS(bracket_fallback)->recursivehotpath);
5482 else
5483 {
5484 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5485 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_fallback)->u.framesize + 1) * sizeof(sljit_w), CURRENT_AS(bracket_fallback)->recursivehotpath);
5486 }
5487 if (opcode != OP_ONCE)
5488 free_stack(common, 1);
5489 }
5490 else
5491 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->recursivehotpath);
5492 }
5493 rminlabel = LABEL();
5494 }
5495 else if (bra == OP_BRAZERO)
5496 {
5497 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5498 free_stack(common, 1);
5499 brazero = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
5500 }
5501
5502 if (SLJIT_UNLIKELY(opcode == OP_ONCE))
5503 {
5504 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5505 {
5506 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5507 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5508 }
5509 once = JUMP(SLJIT_JUMP);
5510 }
5511 else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5512 {
5513 if (has_alternatives)
5514 {
5515 /* Always exactly one alternative. */
5516 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5517 free_stack(common, 1);
5518
5519 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5520 if (SLJIT_UNLIKELY(!jumplistitem))
5521 return;
5522 jumplist = jumplistitem;
5523 jumplistitem->next = NULL;
5524 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 1);
5525 }
5526 }
5527 else if (*cc == OP_ALT)
5528 {
5529 /* Build a jump list. Get the last successfully matched branch index. */
5530 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5531 free_stack(common, 1);
5532 count = 1;
5533 do
5534 {
5535 /* Append as the last item. */
5536 if (jumplist != NULL)
5537 {
5538 jumplistitem->next = sljit_alloc_memory(compiler, sizeof(jump_list));
5539 jumplistitem = jumplistitem->next;
5540 }
5541 else
5542 {
5543 jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
5544 jumplist = jumplistitem;
5545 }
5546
5547 if (SLJIT_UNLIKELY(!jumplistitem))
5548 return;
5549
5550 jumplistitem->next = NULL;
5551 jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, count++);
5552 cc += GET(cc, 1);
5553 }
5554 while (*cc == OP_ALT);
5555
5556 cc = ccbegin + GET(ccbegin, 1);
5557 }
5558
5559 COMPILE_FALLBACKPATH(current->top);
5560 if (current->topfallbacks)
5561 set_jumps(current->topfallbacks, LABEL());
5562
5563 if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
5564 {
5565 /* Conditional block always has at most one alternative. */
5566 if (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT)
5567 {
5568 SLJIT_ASSERT(has_alternatives);
5569 assert = CURRENT_AS(bracket_fallback)->u.assert;
5570 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK))
5571 {
5572 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5573 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5574 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5575 }
5576 cond = JUMP(SLJIT_JUMP);
5577 set_jumps(CURRENT_AS(bracket_fallback)->u.assert->condfailed, LABEL());
5578 }
5579 else if (CURRENT_AS(bracket_fallback)->u.condfailed != NULL)
5580 {
5581 SLJIT_ASSERT(has_alternatives);
5582 cond = JUMP(SLJIT_JUMP);
5583 set_jumps(CURRENT_AS(bracket_fallback)->u.condfailed, LABEL());
5584 }
5585 else
5586 SLJIT_ASSERT(!has_alternatives);
5587 }
5588
5589 if (has_alternatives)
5590 {
5591 count = 1;
5592 do
5593 {
5594 current->top = NULL;
5595 current->topfallbacks = NULL;
5596 current->nextfallbacks = NULL;
5597 if (*cc == OP_ALT)
5598 {
5599 ccprev = cc + 1 + LINK_SIZE;
5600 cc += GET(cc, 1);
5601 if (opcode != OP_COND && opcode != OP_SCOND)
5602 {
5603 if (localptr != 0 && opcode != OP_ONCE)
5604 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5605 else
5606 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5607 }
5608 compile_hotpath(common, ccprev, cc, current);
5609 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
5610 return;
5611 }
5612
5613 /* Instructions after the current alternative is succesfully matched. */
5614 /* There is a similar code in compile_bracket_hotpath. */
5615 if (opcode == OP_ONCE)
5616 {
5617 if (CURRENT_AS(bracket_fallback)->u.framesize < 0)
5618 {
5619 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5620 /* TMP2 which is set here used by OP_KETRMAX below. */
5621 if (ket == OP_KETRMAX)
5622 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
5623 else if (ket == OP_KETRMIN)
5624 {
5625 /* Move the STR_PTR to the localptr. */
5626 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), 0);
5627 }
5628 }
5629 else
5630 {
5631 OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_IMM, (CURRENT_AS(bracket_fallback)->u.framesize + 2) * sizeof(sljit_w));
5632 if (ket == OP_KETRMAX)
5633 {
5634 /* TMP2 which is set here used by OP_KETRMAX below. */
5635 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5636 }
5637 }
5638 }
5639
5640 stacksize = 0;
5641 if (opcode != OP_ONCE)
5642 stacksize++;
5643 if (ket != OP_KET || bra != OP_BRA)
5644 stacksize++;
5645
5646 if (stacksize > 0) {
5647 if (opcode != OP_ONCE || CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5648 allocate_stack(common, stacksize);
5649 else
5650 {
5651 /* We know we have place at least for one item on the top of the stack. */
5652 SLJIT_ASSERT(stacksize == 1);
5653 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_w));
5654 }
5655 }
5656
5657 stacksize = 0;
5658 if (ket != OP_KET || bra != OP_BRA)
5659 {
5660 if (ket != OP_KET)
5661 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
5662 else
5663 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0);
5664 stacksize++;
5665 }
5666
5667 if (opcode != OP_ONCE)
5668 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, count++);
5669
5670 if (offset != 0)
5671 {
5672 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr);
5673 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
5674 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 0), TMP1, 0);
5675 }
5676
5677 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->althotpath);
5678
5679 if (opcode != OP_ONCE)
5680 {
5681 SLJIT_ASSERT(jumplist);
5682 JUMPHERE(jumplist->jump);
5683 jumplist = jumplist->next;
5684 }
5685
5686 COMPILE_FALLBACKPATH(current->top);
5687 if (current->topfallbacks)
5688 set_jumps(current->topfallbacks, LABEL());
5689 SLJIT_ASSERT(!current->nextfallbacks);
5690 }
5691 while (*cc == OP_ALT);
5692 SLJIT_ASSERT(!jumplist);
5693
5694 if (cond != NULL)
5695 {
5696 SLJIT_ASSERT(opcode == OP_COND || opcode == OP_SCOND);
5697 assert = CURRENT_AS(bracket_fallback)->u.assert;
5698 if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT))
5699 {
5700 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr);
5701 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5702 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->localptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_w));
5703 }
5704 JUMPHERE(cond);
5705 }
5706
5707 /* Free the STR_PTR. */
5708 if (localptr == 0)
5709 free_stack(common, 1);
5710 }
5711
5712 if (offset != 0)
5713 {
5714 /* Using both tmp register is better for instruction scheduling. */
5715 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5716 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5717 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5718 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5719 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(2));
5720 free_stack(common, 3);
5721 }
5722 else if (opcode == OP_SBRA || opcode == OP_SCOND)
5723 {
5724 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), STACK(0));
5725 free_stack(common, 1);
5726 }
5727 else if (opcode == OP_ONCE)
5728 {
5729 cc = ccbegin + GET(ccbegin, 1);
5730 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5731 {
5732 /* Reset head and drop saved frame. */
5733 stacksize = (ket == OP_KETRMAX || ket == OP_KETRMIN || *cc == OP_ALT) ? 2 : 1;
5734 free_stack(common, CURRENT_AS(bracket_fallback)->u.framesize + stacksize);
5735 }
5736 else if (ket == OP_KETRMAX || (*cc == OP_ALT && ket != OP_KETRMIN))
5737 {
5738 /* The STR_PTR must be released. */
5739 free_stack(common, 1);
5740 }
5741
5742 JUMPHERE(once);
5743 /* Restore previous localptr */
5744 if (CURRENT_AS(bracket_fallback)->u.framesize >= 0)
5745 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_fallback)->u.framesize * sizeof(sljit_w));
5746 else if (ket == OP_KETRMIN)
5747 {
5748 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5749 /* See the comment below. */
5750 free_stack(common, 2);
5751 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), localptr, TMP1, 0);
5752 }
5753 }
5754
5755 if (ket == OP_KETRMAX)
5756 {
5757 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5758 CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_fallback)->recursivehotpath);
5759 if (bra == OP_BRAZERO)
5760 {
5761 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5762 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5763 JUMPHERE(brazero);
5764 }
5765 free_stack(common, 1);
5766 }
5767 else if (ket == OP_KETRMIN)
5768 {
5769 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5770
5771 /* OP_ONCE removes everything in case of a fallback, so we don't
5772 need to explicitly release the STR_PTR. The extra release would
5773 affect badly the free_stack(2) above. */
5774 if (opcode != OP_ONCE)
5775 free_stack(common, 1);
5776 CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rminlabel);
5777 if (opcode == OP_ONCE)
5778 free_stack(common, bra == OP_BRAMINZERO ? 2 : 1);
5779 else if (bra == OP_BRAMINZERO)
5780 free_stack(common, 1);
5781 }
5782 else if (bra == OP_BRAZERO)
5783 {
5784 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5785 JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_fallback)->zerohotpath);
5786 JUMPHERE(brazero);
5787 }
5788 }
5789
5790 static void compile_bracketpos_fallbackpath(compiler_common *common, struct fallback_common *current)
5791 {
5792 DEFINE_COMPILER;
5793 int offset;
5794 struct sljit_jump *jump;
5795
5796 if (CURRENT_AS(bracketpos_fallback)->framesize < 0)
5797 {
5798 if (*current->cc == OP_CBRAPOS || *current->cc == OP_SCBRAPOS)
5799 {
5800 offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1;
5801 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5802 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
5803 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
5804 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
5805 }
5806 set_jumps(current->topfallbacks, LABEL());
5807 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5808 return;
5809 }
5810
5811 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr);
5812 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
5813
5814 if (current->topfallbacks)
5815 {
5816 jump = JUMP(SLJIT_JUMP);
5817 set_jumps(current->topfallbacks, LABEL());
5818 /* Drop the stack frame. */
5819 free_stack(common, CURRENT_AS(bracketpos_fallback)->stacksize);
5820 JUMPHERE(jump);
5821 }
5822 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_fallback)->localptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_fallback)->framesize * sizeof(sljit_w));
5823 }
5824
5825 static void compile_braminzero_fallbackpath(compiler_common *common, struct fallback_common *current)
5826 {
5827 assert_fallback fallback;
5828
5829 current->top = NULL;
5830 current->topfallbacks = NULL;
5831 current->nextfallbacks = NULL;
5832 if (current->cc[1] > OP_ASSERTBACK_NOT)
5833 {
5834 /* Manual call of compile_bracket_hotpath and compile_bracket_fallbackpath. */
5835 compile_bracket_hotpath(common, current->cc, current);
5836 compile_bracket_fallbackpath(common, current->top);
5837 }
5838 else
5839 {
5840 memset(&fallback, 0, sizeof(fallback));
5841 fallback.common.cc = current->cc;
5842 fallback.hotpath = CURRENT_AS(braminzero_fallback)->hotpath;
5843 /* Manual call of compile_assert_hotpath. */
5844 compile_assert_hotpath(common, current->cc, &fallback, FALSE);
5845 }
5846 SLJIT_ASSERT(!current->nextfallbacks && !current->topfallbacks);
5847 }
5848
5849 static void compile_fallbackpath(compiler_common *common, struct fallback_common *current)
5850 {
5851 DEFINE_COMPILER;
5852
5853 while (current)
5854 {
5855 if (current->nextfallbacks != NULL)
5856 set_jumps(current->nextfallbacks, LABEL());
5857 switch(*current->cc)
5858 {
5859 case OP_SET_SOM:
5860 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
5861 free_stack(common, 1);
5862 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP1, 0);
5863 break;
5864
5865 case OP_STAR:
5866 case OP_MINSTAR:
5867 case OP_PLUS:
5868 case OP_MINPLUS:
5869 case OP_QUERY:
5870 case OP_MINQUERY:
5871 case OP_UPTO:
5872 case OP_MINUPTO:
5873 case OP_EXACT:
5874 case OP_POSSTAR:
5875 case OP_POSPLUS:
5876 case OP_POSQUERY:
5877 case OP_POSUPTO:
5878 case OP_STARI:
5879 case OP_MINSTARI:
5880 case OP_PLUSI:
5881 case OP_MINPLUSI:
5882 case OP_QUERYI:
5883 case OP_MINQUERYI:
5884 case OP_UPTOI:
5885 case OP_MINUPTOI:
5886 case OP_EXACTI:
5887 case OP_POSSTARI:
5888 case OP_POSPLUSI:
5889 case OP_POSQUERYI:
5890 case OP_POSUPTOI:
5891 case OP_NOTSTAR:
5892 case OP_NOTMINSTAR:
5893 case OP_NOTPLUS:
5894 case OP_NOTMINPLUS:
5895 case OP_NOTQUERY:
5896 case OP_NOTMINQUERY:
5897 case OP_NOTUPTO:
5898 case OP_NOTMINUPTO:
5899 case OP_NOTEXACT:
5900 case OP_NOTPOSSTAR:
5901 case OP_NOTPOSPLUS:
5902 case OP_NOTPOSQUERY:
5903 case OP_NOTPOSUPTO:
5904 case OP_NOTSTARI:
5905 case OP_NOTMINSTARI:
5906 case OP_NOTPLUSI:
5907 case OP_NOTMINPLUSI:
5908 case OP_NOTQUERYI:
5909 case OP_NOTMINQUERYI:
5910 case OP_NOTUPTOI:
5911 case OP_NOTMINUPTOI:
5912 case OP_NOTEXACTI:
5913 case OP_NOTPOSSTARI:
5914 case OP_NOTPOSPLUSI:
5915 case OP_NOTPOSQUERYI:
5916 case OP_NOTPOSUPTOI:
5917 case OP_TYPESTAR:
5918 case OP_TYPEMINSTAR:
5919 case OP_TYPEPLUS:
5920 case OP_TYPEMINPLUS:
5921 case OP_TYPEQUERY:
5922 case OP_TYPEMINQUERY:
5923 case OP_TYPEUPTO:
5924 case OP_TYPEMINUPTO:
5925 case OP_TYPEEXACT:
5926 case OP_TYPEPOSSTAR:
5927 case OP_TYPEPOSPLUS:
5928 case OP_TYPEPOSQUERY:
5929 case OP_TYPEPOSUPTO:
5930 case OP_CLASS:
5931 case OP_NCLASS:
5932 case OP_XCLASS:
5933 compile_iterator_fallbackpath(common, current);
5934 break;
5935
5936 case OP_REF:
5937 case OP_REFI:
5938 compile_ref_iterator_fallbackpath(common, current);
5939 break;
5940
5941 case OP_RECURSE:
5942 compile_recurse_fallbackpath(common, current);
5943 break;
5944
5945 case OP_ASSERT:
5946 case OP_ASSERT_NOT:
5947 case OP_ASSERTBACK:
5948 case OP_ASSERTBACK_NOT:
5949 compile_assert_fallbackpath(common, current);
5950 break;
5951
5952 case OP_ONCE:
5953 case OP_ONCE_NC:
5954 case OP_BRA:
5955 case OP_CBRA:
5956 case OP_COND:
5957 case OP_SBRA:
5958 case OP_SCBRA:
5959 case OP_SCOND:
5960 compile_bracket_fallbackpath(common, current);
5961 break;
5962
5963 case OP_BRAZERO:
5964 if (current->cc[1] > OP_ASSERTBACK_NOT)
5965 compile_bracket_fallbackpath(common, current);
5966 else
5967 compile_assert_fallbackpath(common, current);
5968 break;
5969
5970 case OP_BRAPOS:
5971 case OP_CBRAPOS:
5972 case OP_SBRAPOS:
5973 case OP_SCBRAPOS:
5974 case OP_BRAPOSZERO:
5975 compile_bracketpos_fallbackpath(common, current);
5976 break;
5977
5978 case OP_BRAMINZERO:
5979 compile_braminzero_fallbackpath(common, current);
5980 break;
5981
5982 case OP_FAIL:
5983 case OP_ACCEPT:
5984 case OP_ASSERT_ACCEPT:
5985 set_jumps(current->topfallbacks, LABEL());
5986 break;
5987
5988 default:
5989 SLJIT_ASSERT_STOP();
5990 break;
5991 }
5992 current = current->prev;
5993 }
5994 }
5995
5996 static SLJIT_INLINE void compile_recurse(compiler_common *common)
5997 {
5998 DEFINE_COMPILER;
5999 uschar *cc = common->start + common->currententry->start;
6000 uschar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : 2);
6001 uschar *ccend = bracketend(cc);
6002 int localsize = get_localsize(common, ccbegin, ccend);
6003 int framesize = get_framesize(common, cc, TRUE);
6004 int alternativesize;
6005 BOOL needsframe;
6006 fallback_common altfallback;
6007 struct sljit_jump *jump;
6008
6009 SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS);
6010 needsframe = framesize >= 0;
6011 if (!needsframe)
6012 framesize = 0;
6013 alternativesize = *(cc + GET(cc, 1)) == OP_ALT ? 1 : 0;
6014
6015 SLJIT_ASSERT(common->currententry->entry == NULL);
6016 common->currententry->entry = LABEL();
6017 set_jumps(common->currententry->calls, common->currententry->entry);
6018
6019 sljit_emit_fast_enter(compiler, TMP2, 0, 1, 5, 5, common->localsize);
6020 allocate_stack(common, localsize + framesize + alternativesize);
6021 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(localsize + framesize + alternativesize - 1), TMP2, 0);
6022 copy_locals(common, ccbegin, ccend, TRUE, localsize + framesize + alternativesize, framesize + alternativesize);
6023 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, STACK_TOP, 0);
6024 if (needsframe)
6025 init_frame(common, cc, framesize + alternativesize - 1, alternativesize, FALSE);
6026
6027 if (alternativesize > 0)
6028 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
6029
6030 memset(&altfallback, 0, sizeof(fallback_common));
6031 common->acceptlabel = NULL;
6032 common->accept = NULL;
6033 altfallback.cc = ccbegin;
6034 cc += GET(cc, 1);
6035 while (1)
6036 {
6037 altfallback.top = NULL;
6038 altfallback.topfallbacks = NULL;
6039
6040 if (altfallback.cc != ccbegin)
6041 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
6042
6043 compile_hotpath(common, altfallback.cc, cc, &altfallback);
6044 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6045 return;
6046
6047 add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP));
6048
6049 compile_fallbackpath(common, altfallback.top);
6050 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6051 return;
6052 set_jumps(altfallback.topfallbacks, LABEL());
6053
6054 if (*cc != OP_ALT)
6055 break;
6056
6057 altfallback.cc = cc + 1 + LINK_SIZE;
6058 cc += GET(cc, 1);
6059 }
6060 /* None of them matched. */
6061 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0);
6062 jump = JUMP(SLJIT_JUMP);
6063
6064 set_jumps(common->accept, LABEL());
6065 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD);
6066 if (needsframe)
6067 {
6068 OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6069 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_w));
6070 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
6071 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_w));
6072 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP3, 0);
6073 }
6074 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1);
6075
6076 JUMPHERE(jump);
6077 copy_locals(common, ccbegin, ccend, FALSE, localsize + framesize + alternativesize, framesize + alternativesize);
6078 free_stack(common, localsize + framesize + alternativesize);
6079 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_w));
6080 OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
6081 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), RECURSIVE_HEAD, TMP2, 0);
6082 sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0);
6083 }
6084
6085 #undef COMPILE_FALLBACKPATH
6086 #undef CURRENT_AS
6087
6088 void
6089 _pcre_jit_compile(const real_pcre *re, pcre_extra *extra)
6090 {
6091 struct sljit_compiler *compiler;
6092 fallback_common rootfallback;
6093 compiler_common common_data;
6094 compiler_common *common = &common_data;
6095 const uschar *tables = re->tables;
6096 pcre_study_data *study;
6097 uschar *ccend;
6098 executable_function *function;
6099 void *executable_func;
6100 struct sljit_label *leave;
6101 struct sljit_label *mainloop = NULL;
6102 struct sljit_label *empty_match_found;
6103 struct sljit_label *empty_match_fallback;
6104 struct sljit_jump *alloc_error;
6105 struct sljit_jump *reqbyte_notfound = NULL;
6106 struct sljit_jump *empty_match;
6107
6108 SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0);
6109 study = extra->study_data;
6110
6111 if (!tables)
6112 tables = _pcre_default_tables;
6113
6114 memset(&rootfallback, 0, sizeof(fallback_common));
6115 rootfallback.cc = (uschar *)re + re->name_table_offset + re->name_count * re->name_entry_size;
6116
6117 common->compiler = NULL;
6118 common->start = rootfallback.cc;
6119 common->cbraptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_w);
6120 common->fcc = tables + fcc_offset;
6121 common->lcc = (sljit_w)(tables + lcc_offset);
6122 common->nltype = NLTYPE_FIXED;
6123 switch(re->options & PCRE_NEWLINE_BITS)
6124 {
6125 case 0:
6126 /* Compile-time default */
6127 switch (NEWLINE)
6128 {
6129 case -1: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
6130 case -2: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
6131 default: common->newline = NEWLINE; break;
6132 }
6133 break;
6134 case PCRE_NEWLINE_CR: common->newline = CHAR_CR; break;
6135 case PCRE_NEWLINE_LF: common->newline = CHAR_NL; break;
6136 case PCRE_NEWLINE_CR+
6137 PCRE_NEWLINE_LF: common->newline = (CHAR_CR << 8) | CHAR_NL; break;
6138 case PCRE_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break;
6139 case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
6140 default: return;
6141 }
6142 if ((re->options & PCRE_BSR_ANYCRLF) != 0)
6143 common->bsr_nltype = NLTYPE_ANYCRLF;
6144 else if ((re->options & PCRE_BSR_UNICODE) != 0)
6145 common->bsr_nltype = NLTYPE_ANY;
6146 else
6147 {
6148 #ifdef BSR_ANYCRLF
6149 common->bsr_nltype = NLTYPE_ANYCRLF;
6150 #else
6151 common->bsr_nltype = NLTYPE_ANY;
6152 #endif
6153 }
6154 common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0;
6155 common->ctypes = (sljit_w)(tables + ctypes_offset);
6156 common->name_table = (sljit_w)re + re->name_table_offset;
6157 common->name_count = re->name_count;
6158 common->name_entry_size = re->name_entry_size;
6159 common->acceptlabel = NULL;
6160 common->stubs = NULL;
6161 common->entries = NULL;
6162 common->currententry = NULL;
6163 common->accept = NULL;
6164 common->calllimit = NULL;
6165 common->stackalloc = NULL;
6166 common->revertframes = NULL;
6167 common->wordboundary = NULL;
6168 common->anynewline = NULL;
6169 common->hspace = NULL;
6170 common->vspace = NULL;
6171 common->casefulcmp = NULL;
6172 common->caselesscmp = NULL;
6173 common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0;
6174 #ifdef SUPPORT_UTF8
6175 common->utf8 = (re->options & PCRE_UTF8) != 0;
6176 #ifdef SUPPORT_UCP
6177 common->useucp = (re->options & PCRE_UCP) != 0;
6178 #endif
6179 common->utf8readchar = NULL;
6180 common->utf8readtype8 = NULL;
6181 #endif
6182 #ifdef SUPPORT_UCP
6183 common->getucd = NULL;
6184 #endif
6185 ccend = bracketend(rootfallback.cc);
6186 SLJIT_ASSERT(*rootfallback.cc == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET);
6187 common->localsize = get_localspace(common, rootfallback.cc, ccend);
6188 if (common->localsize < 0)
6189 return;
6190 common->localsize += common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w);
6191 if (common->localsize > SLJIT_MAX_LOCAL_SIZE)
6192 return;
6193 common->localptrs = (int*)SLJIT_MALLOC((ccend - rootfallback.cc) * sizeof(int));
6194 if (!common->localptrs)
6195 return;
6196 memset(common->localptrs, 0, (ccend - rootfallback.cc) * sizeof(int));
6197 set_localptrs(common, common->cbraptr + (re->top_bracket + 1) * sizeof(sljit_w), ccend);
6198
6199 compiler = sljit_create_compiler();
6200 if (!compiler)
6201 {
6202 SLJIT_FREE(common->localptrs);
6203 return;
6204 }
6205 common->compiler = compiler;
6206
6207 /* Main pcre_jit_exec entry. */
6208 sljit_emit_enter(compiler, 1, 5, 5, common->localsize);
6209
6210 /* Register init. */
6211 reset_ovector(common, (re->top_bracket + 1) * 2);
6212 if ((re->flags & PCRE_REQCHSET) != 0)
6213 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), REQ_BYTE_PTR, SLJIT_TEMPORARY_REG1, 0);
6214
6215 OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_GENERAL_REG1, 0);
6216 OP1(SLJIT_MOV, TMP1, 0, SLJIT_GENERAL_REG1, 0);
6217 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6218 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end));
6219 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6220 OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, calllimit));
6221 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base));
6222 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit));
6223 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT, TMP1, 0);
6224
6225 /* Main part of the matching */
6226 if ((re->options & PCRE_ANCHORED) == 0)
6227 {
6228 mainloop = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0, (re->options & PCRE_FIRSTLINE) != 0);
6229 /* Forward search if possible. */
6230 if ((re->flags & PCRE_FIRSTSET) != 0)
6231 fast_forward_first_byte(common, re->first_byte, (re->options & PCRE_FIRSTLINE) != 0);
6232 else if ((re->flags & PCRE_STARTLINE) != 0)
6233 fast_forward_newline(common, (re->options & PCRE_FIRSTLINE) != 0);
6234 else if ((re->flags & PCRE_STARTLINE) == 0 && study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)
6235 fast_forward_start_bits(common, (sljit_uw)study->start_bits, (re->options & PCRE_FIRSTLINE) != 0);
6236 }
6237 if ((re->flags & PCRE_REQCHSET) != 0)
6238 reqbyte_notfound = search_requested_char(common, re->req_byte, (re->flags & PCRE_FIRSTSET) != 0);
6239
6240 /* Store the current STR_PTR in OVECTOR(0). */
6241 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
6242 /* Copy the limit of allowed recursions. */
6243 OP1(SLJIT_MOV, CALL_COUNT, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CALL_LIMIT);
6244
6245 compile_hotpath(common, rootfallback.cc, ccend, &rootfallback);
6246 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6247 {
6248 sljit_free_compiler(compiler);
6249 SLJIT_FREE(common->localptrs);
6250 return;
6251 }
6252
6253 empty_match = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6254 empty_match_found = LABEL();
6255
6256 common->acceptlabel = LABEL();
6257 if (common->accept != NULL)
6258 set_jumps(common->accept, common->acceptlabel);
6259
6260 /* This means we have a match. Update the ovector. */
6261 copy_ovector(common, re->top_bracket + 1);
6262 leave = LABEL();
6263 sljit_emit_return(compiler, SLJIT_UNUSED, 0);
6264
6265 empty_match_fallback = LABEL();
6266 compile_fallbackpath(common, rootfallback.top);
6267 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6268 {
6269 sljit_free_compiler(compiler);
6270 SLJIT_FREE(common->localptrs);
6271 return;
6272 }
6273
6274 SLJIT_ASSERT(rootfallback.prev == NULL);
6275
6276 /* Check we have remaining characters. */
6277 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
6278
6279 if ((re->options & PCRE_ANCHORED) == 0)
6280 {
6281 if ((re->options & PCRE_FIRSTLINE) == 0)
6282 {
6283 if (study != NULL && study->minlength > 1)
6284 {
6285 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6286 CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, STR_END, 0, mainloop);
6287 }
6288 else
6289 CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
6290 }
6291 else
6292 {
6293 if (study != NULL && study->minlength > 1)
6294 {
6295 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, study->minlength);
6296 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, STR_END, 0);
6297 COND_VALUE(SLJIT_MOV, TMP2, 0, SLJIT_C_GREATER);
6298 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END);
6299 COND_VALUE(SLJIT_OR | SLJIT_SET_E, TMP2, 0, SLJIT_C_GREATER_EQUAL);
6300 JUMPTO(SLJIT_C_ZERO, mainloop);
6301 }
6302 else
6303 CMPTO(SLJIT_C_LESS, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), FIRSTLINE_END, mainloop);
6304 }
6305 }
6306
6307 if (reqbyte_notfound != NULL)
6308 JUMPHERE(reqbyte_notfound);
6309 /* Copy OVECTOR(1) to OVECTOR(0) */
6310 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
6311 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
6312 JUMPTO(SLJIT_JUMP, leave);
6313
6314 flush_stubs(common);
6315
6316 JUMPHERE(empty_match);
6317 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6318 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
6319 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_fallback);
6320 OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
6321 CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found);
6322 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
6323 CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found);
6324 JUMPTO(SLJIT_JUMP, empty_match_fallback);
6325
6326 common->currententry = common->entries;
6327 while (common->currententry != NULL)
6328 {
6329 /* Might add new entries. */
6330 compile_recurse(common);
6331 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
6332 {
6333 sljit_free_compiler(compiler);
6334 SLJIT_FREE(common->localptrs);
6335 return;
6336 }
6337 flush_stubs(common);
6338 common->currententry = common->currententry->next;
6339 }
6340
6341 /* Allocating stack, returns with PCRE_ERROR_JIT_STACKLIMIT if fails. */
6342 /* This is a (really) rare case. */
6343 set_jumps(common->stackalloc, LABEL());
6344 /* RETURN_ADDR is not a saved register. */
6345 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, 1, 5, 5, common->localsize);
6346 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
6347 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6348 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6349 OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
6350 OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
6351
6352 sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
6353 alloc_error = CMP(SLJIT_C_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
6354 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
6355 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
6356 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top));
6357 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit));
6358 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
6359 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
6360
6361 /* Allocation failed. */
6362 JUMPHERE(alloc_error);
6363 /* We break the return address cache here, but this is a really rare case. */
6364 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_JIT_STACKLIMIT);
6365 JUMPTO(SLJIT_JUMP, leave);
6366
6367 /* Call limit reached. */
6368 set_jumps(common->calllimit, LABEL());
6369 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_MATCHLIMIT);
6370 JUMPTO(SLJIT_JUMP, leave);
6371
6372 if (common->revertframes != NULL)
6373 {
6374 set_jumps(common->revertframes, LABEL());
6375 do_revertframes(common);
6376 }
6377 if (common->wordboundary != NULL)
6378 {
6379 set_jumps(common->wordboundary, LABEL());
6380 check_wordboundary(common);
6381 }
6382 if (common->anynewline != NULL)
6383 {
6384 set_jumps(common->anynewline, LABEL());
6385 check_anynewline(common);
6386 }
6387 if (common->hspace != NULL)
6388 {
6389 set_jumps(common->hspace, LABEL());
6390 check_hspace(common);
6391 }
6392 if (common->vspace != NULL)
6393 {
6394 set_jumps(common->vspace, LABEL());
6395 check_vspace(common);
6396 }
6397 if (common->casefulcmp != NULL)
6398 {
6399 set_jumps(common->casefulcmp, LABEL());
6400 do_casefulcmp(common);
6401 }
6402 if (common->caselesscmp != NULL)
6403 {
6404 set_jumps(common->caselesscmp, LABEL());
6405 do_caselesscmp(common);
6406 }
6407 #ifdef SUPPORT_UTF8
6408 if (common->utf8readchar != NULL)
6409 {
6410 set_jumps(common->utf8readchar, LABEL());
6411 do_utf8readchar(common);
6412 }
6413 if (common->utf8readtype8 != NULL)
6414 {
6415 set_jumps(common->utf8readtype8, LABEL());
6416 do_utf8readtype8(common);
6417 }
6418 #endif
6419 #ifdef SUPPORT_UCP
6420 if (common->getucd != NULL)
6421 {
6422 set_jumps(common->getucd, LABEL());
6423 do_getucd(common);
6424 }
6425 #endif
6426
6427 SLJIT_FREE(common->localptrs);
6428 executable_func = sljit_generate_code(compiler);
6429 sljit_free_compiler(compiler);
6430 if (executable_func == NULL)
6431 return;
6432
6433 function = SLJIT_MALLOC(sizeof(executable_function));
6434 if (function == NULL)
6435 {
6436 /* This case is highly unlikely since we just recently
6437 freed a lot of memory. Although not impossible. */
6438 sljit_free_code(executable_func);
6439 return;
6440 }
6441
6442 function->executable_func = executable_func;
6443 function->callback = NULL;
6444 function->userdata = NULL;
6445 extra->executable_jit = function;
6446 extra->flags |= PCRE_EXTRA_EXECUTABLE_JIT;
6447 }
6448
6449 static int jit_machine_stack_exec(jit_arguments *arguments, executable_function *function)
6450 {
6451 union {
6452 void* executable_func;
6453 jit_function call_executable_func;
6454 } convert_executable_func;
6455 uschar local_area[LOCAL_SPACE_SIZE];
6456 struct sljit_stack local_stack;
6457
6458 local_stack.top = (sljit_w)&local_area;
6459 local_stack.base = local_stack.top;
6460 local_stack.limit = local_stack.base + LOCAL_SPACE_SIZE;
6461 local_stack.max_limit = local_stack.limit;
6462 arguments->stack = &local_stack;
6463 convert_executable_func.executable_func = function->executable_func;
6464 return convert_executable_func.call_executable_func(arguments);
6465 }
6466
6467 int
6468 _pcre_jit_exec(const real_pcre *re, void *executable_func,
6469 PCRE_SPTR subject, int length, int start_offset, int options,
6470 int match_limit, int *offsets, int offsetcount)
6471 {
6472 executable_function *function = (executable_function*)executable_func;
6473 union {
6474 void* executable_func;
6475 jit_function call_executable_func;
6476 } convert_executable_func;
6477 jit_arguments arguments;
6478 int maxoffsetcount;
6479 int retval;
6480
6481 /* Sanity checks should be handled by pcre_exec. */
6482 arguments.stack = NULL;
6483 arguments.str = subject + start_offset;
6484 arguments.begin = subject;
6485 arguments.end = subject + length;
6486 arguments.calllimit = match_limit; /* JIT decreases this value less times. */
6487 arguments.notbol = (options & PCRE_NOTBOL) != 0;
6488 arguments.noteol = (options & PCRE_NOTEOL) != 0;
6489 arguments.notempty = (options & PCRE_NOTEMPTY) != 0;
6490 arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0;
6491 arguments.offsets = offsets;
6492
6493 /* pcre_exec() rounds offsetcount to a multiple of 3, and then uses only 2/3 of
6494 the output vector for storing captured strings, with the remainder used as
6495 workspace. We don't need the workspace here. For compatibility, we limit the
6496 number of captured strings in the same way as pcre_exec(), so that the user
6497 gets the same result with and without JIT. */
6498
6499 offsetcount = ((offsetcount - (offsetcount % 3)) * 2)/3;
6500 maxoffsetcount = (re->top_bracket + 1) * 2;
6501 if (offsetcount > maxoffsetcount)
6502 offsetcount = maxoffsetcount;
6503 arguments.offsetcount = offsetcount;
6504
6505 if (function->callback)
6506 arguments.stack = (struct sljit_stack*)function->callback(function->userdata);
6507 else
6508 arguments.stack = (struct sljit_stack*)function->userdata;
6509
6510 if (arguments.stack == NULL)
6511 retval = jit_machine_stack_exec(&arguments, function);
6512 else
6513 {
6514 convert_executable_func.executable_func = function->executable_func;
6515 retval = convert_executable_func.call_executable_func(&arguments);
6516 }
6517
6518 if (retval * 2 > offsetcount)
6519 retval = 0;
6520 return retval;
6521 }
6522
6523 void
6524 _pcre_jit_free(void *executable_func)
6525 {
6526 executable_function *function = (executable_function*)executable_func;
6527 sljit_free_code(function->executable_func);
6528 SLJIT_FREE(function);
6529 }
6530
6531 PCRE_EXP_DECL pcre_jit_stack *
6532 pcre_jit_stack_alloc(int startsize, int maxsize)
6533 {
6534 if (startsize < 1 || maxsize < 1)
6535 return NULL;
6536 if (startsize > maxsize)
6537 startsize = maxsize;
6538 startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6539 maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
6540 return (pcre_jit_stack*)sljit_allocate_stack(startsize, maxsize);
6541 }
6542
6543 PCRE_EXP_DECL void
6544 pcre_jit_stack_free(pcre_jit_stack *stack)
6545 {
6546 sljit_free_stack((struct sljit_stack*)stack);
6547 }
6548
6549 PCRE_EXP_DECL void
6550 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6551 {
6552 executable_function *function;
6553 if (extra != NULL &&
6554 (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 &&
6555 extra->executable_jit != NULL)
6556 {
6557 function = (executable_function*)extra->executable_jit;
6558 function->callback = callback;
6559 function->userdata = userdata;
6560 }
6561 }
6562
6563 #else /* SUPPORT_JIT */
6564
6565 /* These are dummy functions to avoid linking errors when JIT support is not
6566 being compiled. */
6567
6568 PCRE_EXP_DECL pcre_jit_stack *
6569 pcre_jit_stack_alloc(int startsize, int maxsize)
6570 {
6571 (void)startsize;
6572 (void)maxsize;
6573 return NULL;
6574 }
6575
6576 PCRE_EXP_DECL void
6577 pcre_jit_stack_free(pcre_jit_stack *stack)
6578 {
6579 (void)stack;
6580 }
6581
6582 PCRE_EXP_DECL void
6583 pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata)
6584 {
6585 (void)extra;
6586 (void)callback;
6587 (void)userdata;
6588 }
6589
6590 #endif
6591
6592 /* End of pcre_jit_compile.c */

  ViewVC Help
Powered by ViewVC 1.1.5