/[pcre]/code/trunk/maint/GenerateUtt.py
ViewVC logotype

Contents of /code/trunk/maint/GenerateUtt.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1490 - (show annotations)
Thu Jun 19 07:51:39 2014 UTC (5 years, 5 months ago) by chpe
File MIME type: text/x-python
File size: 5184 byte(s)
Update to Unicode 7.0.0 release
1 #! /usr/bin/python
2
3 # Generate utt tables. Note: this script is written in Python 2 and is
4 # incompatible with Python 3. However, the 2to3 conversion script has been
5 # successfully tested on it.
6
7 # The source file pcre_tables.c contains (amongst other things), a table that
8 # is indexed by script name. In order to reduce the number of relocations when
9 # loading the library, the names are held as a single large string, with
10 # offsets in the table. This is tedious to maintain by hand. Therefore, this
11 # script is used to generate the table. The output is sent to stdout; usually
12 # that should be directed to a temporary file. Then pcre_tables.c can be edited
13 # by replacing the relevant definitions and table therein with the temporary
14 # file.
15
16 # Modified by PH 17-March-2009 to generate the more verbose form that works
17 # for UTF-support in EBCDIC as well as ASCII environments.
18 # Modified by PH 01-March-2010 to add new scripts for Unicode 5.2.0.
19 # Modified by PH 04-May-2010 to add new "X.." special categories.
20 # Modified by PH 30-April-2011 to add new scripts for Unicode 6.0.0
21 # Modified by ChPe 30-September-2012 to add this note; no other changes were
22 # necessary for Unicode 6.2.0 support.
23 # Modfied by PH 26-February-2013 to add the Xuc special category.
24
25 script_names = ['Arabic', 'Armenian', 'Bengali', 'Bopomofo', 'Braille', 'Buginese', 'Buhid', 'Canadian_Aboriginal', \
26 'Cherokee', 'Common', 'Coptic', 'Cypriot', 'Cyrillic', 'Deseret', 'Devanagari', 'Ethiopic', 'Georgian', \
27 'Glagolitic', 'Gothic', 'Greek', 'Gujarati', 'Gurmukhi', 'Han', 'Hangul', 'Hanunoo', 'Hebrew', 'Hiragana', \
28 'Inherited', 'Kannada', 'Katakana', 'Kharoshthi', 'Khmer', 'Lao', 'Latin', 'Limbu', 'Linear_B', 'Malayalam', \
29 'Mongolian', 'Myanmar', 'New_Tai_Lue', 'Ogham', 'Old_Italic', 'Old_Persian', 'Oriya', 'Osmanya', 'Runic', \
30 'Shavian', 'Sinhala', 'Syloti_Nagri', 'Syriac', 'Tagalog', 'Tagbanwa', 'Tai_Le', 'Tamil', 'Telugu', 'Thaana', \
31 'Thai', 'Tibetan', 'Tifinagh', 'Ugaritic', 'Yi', \
32 # New for Unicode 5.0
33 'Balinese', 'Cuneiform', 'Nko', 'Phags_Pa', 'Phoenician', \
34 # New for Unicode 5.1
35 'Carian', 'Cham', 'Kayah_Li', 'Lepcha', 'Lycian', 'Lydian', 'Ol_Chiki', 'Rejang', 'Saurashtra', 'Sundanese', 'Vai', \
36 # New for Unicode 5.2
37 'Avestan', 'Bamum', 'Egyptian_Hieroglyphs', 'Imperial_Aramaic', \
38 'Inscriptional_Pahlavi', 'Inscriptional_Parthian', \
39 'Javanese', 'Kaithi', 'Lisu', 'Meetei_Mayek', \
40 'Old_South_Arabian', 'Old_Turkic', 'Samaritan', 'Tai_Tham', 'Tai_Viet', \
41 # New for Unicode 6.0.0
42 'Batak', 'Brahmi', 'Mandaic', \
43 # New for Unicode 6.1.0
44 'Chakma', 'Meroitic_Cursive', 'Meroitic_Hieroglyphs', 'Miao', 'Sharada', 'Sora_Sompeng', 'Takri',
45 # New for Unicode 7.0.0
46 'Bassa_Vah', 'Caucasian_Albanian', 'Duployan', 'Elbasan', 'Grantha', 'Khojki', 'Khudawadi',
47 'Linear_A', 'Mahajani', 'Manichaean', 'Mende_Kikakui', 'Modi', 'Mro', 'Nabataean',
48 'Old_North_Arabian', 'Old_Permic', 'Pahawh_Hmong', 'Palmyrene', 'Psalter_Pahlavi',
49 'Pau_Cin_Hau', 'Siddham', 'Tirhuta', 'Warang_Citi'
50 ]
51
52 category_names = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu',
53 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps',
54 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs' ]
55
56 general_category_names = ['C', 'L', 'M', 'N', 'P', 'S', 'Z']
57
58 # First add the Unicode script and category names.
59
60 utt_table = zip(script_names, ['PT_SC'] * len(script_names))
61 utt_table += zip(category_names, ['PT_PC'] * len(category_names))
62 utt_table += zip(general_category_names, ['PT_GC'] * len(general_category_names))
63
64 # Now add our own specials.
65
66 utt_table.append(('Any', 'PT_ANY'))
67 utt_table.append(('L&', 'PT_LAMP'))
68 utt_table.append(('Xan', 'PT_ALNUM'))
69 utt_table.append(('Xps', 'PT_PXSPACE'))
70 utt_table.append(('Xsp', 'PT_SPACE'))
71 utt_table.append(('Xuc', 'PT_UCNC'))
72 utt_table.append(('Xwd', 'PT_WORD'))
73
74 # Sort the table.
75
76 utt_table.sort()
77
78 # We have to use STR_ macros to define the strings so that it all works in
79 # UTF-8 mode on EBCDIC platforms.
80
81 for utt in utt_table:
82 print '#define STRING_%s0' % (utt[0].replace('&', '_AMPERSAND')),
83 for c in utt[0]:
84 if c == '_':
85 print 'STR_UNDERSCORE',
86 elif c == '&':
87 print 'STR_AMPERSAND',
88 else:
89 print 'STR_%s' % c,;
90 print '"\\0"'
91
92 # Print the actual table, using the string names
93
94 print ''
95 print 'const char PRIV(utt_names)[] =';
96 last = ''
97 for utt in utt_table:
98 if utt == utt_table[-1]:
99 last = ';'
100 print ' STRING_%s0%s' % (utt[0].replace('&', '_AMPERSAND'), last)
101 # This was how it was done before the EBCDIC-compatible modification.
102 # print ' "%s\\0"%s' % (utt[0], last)
103
104 print '\nconst ucp_type_table PRIV(utt)[] = {'
105 offset = 0
106 last = ','
107 for utt in utt_table:
108 if utt[1] in ('PT_ANY', 'PT_LAMP', 'PT_ALNUM', 'PT_PXSPACE',
109 'PT_SPACE', 'PT_UCNC', 'PT_WORD'):
110 value = '0'
111 else:
112 value = 'ucp_' + utt[0]
113 if utt == utt_table[-1]:
114 last = ''
115 print ' { %3d, %s, %s }%s' % (offset, utt[1], value, last)
116 offset += len(utt[0]) + 1
117 print '};'

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.5