Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7730f48

Browse files
tglsfdctglsfdc
committed
Teach UtfToLocal/LocalToUtf to support algorithmic encoding conversions.
Until now, these functions have only supported encoding conversions using lookup tables, which is fine as long as there's not too many code points to convert. However, GB18030 expects all 1.1 million Unicode code points to be convertible, which would require a ridiculously-sized lookup table. Fortunately, a large fraction of those conversions can be expressed through arithmetic, ie the conversions are one-to-one in certain defined ranges. To support that, provide a callback function that is used after consulting the lookup tables. (This patch doesn't actually change anything about the GB18030 conversion behavior, just provide infrastructure for fixing it.) Since this requires changing the APIs of UtfToLocal/LocalToUtf anyway, take the opportunity to rearrange their argument lists into what seems to me a saner order. And beautify the call sites by using lengthof() instead of error-prone sizeof() arithmetic. In passing, also mark all the lookup tables used by these calls "const". This moves an impressive amount of stuff into the text segment, at least on my machine, and is safer anyhow.
1 parent 83e176e commit 7730f48

File tree

108 files changed

+538
-408
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+538
-408
lines changed

‎src/backend/utils/mb/Unicode/UCS_to_BIG5.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@
9797

9898
$file = lc("utf8_to_big5.map");
9999
open(FILE, "> $file") || die("cannot open $file");
100-
print FILE "static pg_utf_to_local ULmapBIG5[ $count ] = {\n";
100+
print FILE "static const pg_utf_to_local ULmapBIG5[ $count ] = {\n";
101101

102102
for $index (sort { $a <=> $b } keys(%array))
103103
{
@@ -185,7 +185,7 @@
185185

186186
$file = lc("big5_to_utf8.map");
187187
open(FILE, "> $file") || die("cannot open $file");
188-
print FILE "static pg_local_to_utf LUmapBIG5[ $count ] = {\n";
188+
print FILE "static const pg_local_to_utf LUmapBIG5[ $count ] = {\n";
189189
for $index (sort { $a <=> $b } keys(%array))
190190
{
191191
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_EUC_CN.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555

5656
$file = "utf8_to_euc_cn.map";
5757
open(FILE, "> $file") || die("cannot open $file");
58-
print FILE "static pg_utf_to_local ULmapEUC_CN[ $count ] = {\n";
58+
print FILE "static const pg_utf_to_local ULmapEUC_CN[ $count ] = {\n";
5959

6060
for $index (sort { $a <=> $b } keys(%array))
6161
{
@@ -109,7 +109,7 @@
109109

110110
$file = "euc_cn_to_utf8.map";
111111
open(FILE, "> $file") || die("cannot open $file");
112-
print FILE "static pg_local_to_utf LUmapEUC_CN[ $count ] = {\n";
112+
print FILE "static const pg_local_to_utf LUmapEUC_CN[ $count ] = {\n";
113113
for $index (sort { $a <=> $b } keys(%array))
114114
{
115115
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_EUC_JIS_2004.pl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
print FILE "/*\n";
7373
print FILE " * This file was generated by UCS_to_EUC_JIS_2004.pl\n";
7474
print FILE " */\n";
75-
print FILE "static pg_utf_to_local ULmapEUC_JIS_2004[] = {\n";
75+
print FILE "static const pg_utf_to_local ULmapEUC_JIS_2004[] = {\n";
7676

7777
for $index (sort { $a <=> $b } keys(%array))
7878
{
@@ -133,7 +133,7 @@
133133
print FILE " * This file was generated by UCS_to_EUC_JIS_2004.pl\n";
134134
print FILE " */\n";
135135
print FILE
136-
"static pg_utf_to_local_combined ULmapEUC_JIS_2004_combined[] = {\n";
136+
"static const pg_utf_to_local_combined ULmapEUC_JIS_2004_combined[] = {\n";
137137

138138
for $index (sort { $a cmp $b } keys(%array1))
139139
{
@@ -256,7 +256,7 @@
256256
print FILE "/*\n";
257257
print FILE " * This file was generated by UCS_to_EUC_JIS_2004.pl\n";
258258
print FILE " */\n";
259-
print FILE "static pg_local_to_utf LUmapEUC_JIS_2004[] = {\n";
259+
print FILE "static const pg_local_to_utf LUmapEUC_JIS_2004[] = {\n";
260260

261261
for $index (sort { $a <=> $b } keys(%array))
262262
{
@@ -283,7 +283,7 @@
283283
print FILE " * This file was generated by UCS_to_EUC_JIS_2004.pl\n";
284284
print FILE " */\n";
285285
print FILE
286-
"static pg_local_to_utf_combined LUmapEUC_JIS_2004_combined[] = {\n";
286+
"static const pg_local_to_utf_combined LUmapEUC_JIS_2004_combined[] = {\n";
287287

288288
for $index (sort { $a <=> $b } keys(%array1))
289289
{

‎src/backend/utils/mb/Unicode/UCS_to_EUC_JP.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@
136136

137137
$file = "utf8_to_euc_jp.map";
138138
open(FILE, "> $file") || die("cannot open $file");
139-
print FILE "static pg_utf_to_local ULmapEUC_JP[ $count ] = {\n";
139+
print FILE "static const pg_utf_to_local ULmapEUC_JP[ $count ] = {\n";
140140

141141
for $index (sort { $a <=> $b } keys(%array))
142142
{
@@ -263,7 +263,7 @@
263263

264264
$file = "euc_jp_to_utf8.map";
265265
open(FILE, "> $file") || die("cannot open $file");
266-
print FILE "static pg_local_to_utf LUmapEUC_JP[ $count ] = {\n";
266+
print FILE "static const pg_local_to_utf LUmapEUC_JP[ $count ] = {\n";
267267
for $index (sort { $a <=> $b } keys(%array))
268268
{
269269
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_EUC_KR.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555

5656
$file = "utf8_to_euc_kr.map";
5757
open(FILE, "> $file") || die("cannot open $file");
58-
print FILE "static pg_utf_to_local ULmapEUC_KR[ $count ] = {\n";
58+
print FILE "static const pg_utf_to_local ULmapEUC_KR[ $count ] = {\n";
5959

6060
for $index (sort { $a <=> $b } keys(%array))
6161
{
@@ -109,7 +109,7 @@
109109

110110
$file = "euc_kr_to_utf8.map";
111111
open(FILE, "> $file") || die("cannot open $file");
112-
print FILE "static pg_local_to_utf LUmapEUC_KR[ $count ] = {\n";
112+
print FILE "static const pg_local_to_utf LUmapEUC_KR[ $count ] = {\n";
113113
for $index (sort { $a <=> $b } keys(%array))
114114
{
115115
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_EUC_TW.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@
7171

7272
$file = "utf8_to_euc_tw.map";
7373
open(FILE, "> $file") || die("cannot open $file");
74-
print FILE "static pg_utf_to_local ULmapEUC_TW[ $count ] = {\n";
74+
print FILE "static const pg_utf_to_local ULmapEUC_TW[ $count ] = {\n";
7575

7676
for $index (sort { $a <=> $b } keys(%array))
7777
{
@@ -138,7 +138,7 @@
138138

139139
$file = "euc_tw_to_utf8.map";
140140
open(FILE, "> $file") || die("cannot open $file");
141-
print FILE "static pg_local_to_utf LUmapEUC_TW[ $count ] = {\n";
141+
print FILE "static const pg_local_to_utf LUmapEUC_TW[ $count ] = {\n";
142142
for $index (sort { $a <=> $b } keys(%array))
143143
{
144144
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_GB18030.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252

5353
$file = "utf8_to_gb18030.map";
5454
open(FILE, "> $file") || die("cannot open $file");
55-
print FILE "static pg_utf_to_local ULmapGB18030[ $count ] = {\n";
55+
print FILE "static const pg_utf_to_local ULmapGB18030[ $count ] = {\n";
5656

5757
for $index (sort { $a <=> $b } keys(%array))
5858
{
@@ -106,7 +106,7 @@
106106

107107
$file = "gb18030_to_utf8.map";
108108
open(FILE, "> $file") || die("cannot open $file");
109-
print FILE "static pg_local_to_utf LUmapGB18030[ $count ] = {\n";
109+
print FILE "static const pg_local_to_utf LUmapGB18030[ $count ] = {\n";
110110
for $index (sort { $a <=> $b } keys(%array))
111111
{
112112
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_SHIFT_JIS_2004.pl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
print FILE "/*\n";
7373
print FILE " * This file was generated by UCS_to_SHIFT_JIS_2004.pl\n";
7474
print FILE " */\n";
75-
print FILE "static pg_utf_to_local ULmapSHIFT_JIS_2004[] = {\n";
75+
print FILE "static const pg_utf_to_local ULmapSHIFT_JIS_2004[] = {\n";
7676

7777
for $index (sort { $a <=> $b } keys(%array))
7878
{
@@ -99,7 +99,7 @@
9999
print FILE " * This file was generated by UCS_to_SHIFT_JIS_2004.pl\n";
100100
print FILE " */\n";
101101
print FILE
102-
"static pg_utf_to_local_combined ULmapSHIFT_JIS_2004_combined[] = {\n";
102+
"static const pg_utf_to_local_combined ULmapSHIFT_JIS_2004_combined[] = {\n";
103103

104104
for $index (sort { $a cmp $b } keys(%array1))
105105
{
@@ -185,7 +185,7 @@
185185
print FILE "/*\n";
186186
print FILE " * This file was generated by UCS_to_SHIFTJIS_2004.pl\n";
187187
print FILE " */\n";
188-
print FILE "static pg_local_to_utf LUmapSHIFT_JIS_2004[] = {\n";
188+
print FILE "static const pg_local_to_utf LUmapSHIFT_JIS_2004[] = {\n";
189189

190190
for $index (sort { $a <=> $b } keys(%array))
191191
{
@@ -212,7 +212,7 @@
212212
print FILE " * This file was generated by UCS_to_SHIFT_JIS_2004.pl\n";
213213
print FILE " */\n";
214214
print FILE
215-
"static pg_local_to_utf_combined LUmapSHIFT_JIS_2004_combined[] = {\n";
215+
"static const pg_local_to_utf_combined LUmapSHIFT_JIS_2004_combined[] = {\n";
216216

217217
for $index (sort { $a <=> $b } keys(%array1))
218218
{

‎src/backend/utils/mb/Unicode/UCS_to_SJIS.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272

7373
$file = "utf8_to_sjis.map";
7474
open(FILE, "> $file") || die("cannot open $file");
75-
print FILE "static pg_utf_to_local ULmapSJIS[ $count ] = {\n";
75+
print FILE "static const pg_utf_to_local ULmapSJIS[ $count ] = {\n";
7676

7777
for $index (sort { $a <=> $b } keys(%array))
7878
{
@@ -122,7 +122,7 @@
122122

123123
$file = "sjis_to_utf8.map";
124124
open(FILE, "> $file") || die("cannot open $file");
125-
print FILE "static pg_local_to_utf LUmapSJIS[ $count ] = {\n";
125+
print FILE "static const pg_local_to_utf LUmapSJIS[ $count ] = {\n";
126126
for $index (sort { $a <=> $b } keys(%array))
127127
{
128128
$utf = $array{$index};

‎src/backend/utils/mb/Unicode/UCS_to_most.pl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@
8888

8989
$file = lc("utf8_to_${charset}.map");
9090
open(FILE, "> $file") || die("cannot open $file");
91-
print FILE "static pg_utf_to_local ULmap${charset}[ $count ] = {\n";
91+
print FILE "static const pg_utf_to_local ULmap${charset}[ $count ] = {\n";
9292

9393
for $index (sort { $a <=> $b } keys(%array))
9494
{
@@ -140,7 +140,7 @@
140140

141141
$file = lc("${charset}_to_utf8.map");
142142
open(FILE, "> $file") || die("cannot open $file");
143-
print FILE "static pg_local_to_utf LUmap${charset}[ $count ] = {\n";
143+
print FILE "static const pg_local_to_utf LUmap${charset}[ $count ] = {\n";
144144
for $index (sort { $a <=> $b } keys(%array))
145145
{
146146
$utf = $array{$index};

0 commit comments

Comments
 (0)