Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit c7e428e

Browse files
author
Max Schaefer
committed
JavaScript: Handle E4X/Flow lexical ambiguity.
1 parent d6deefe commit c7e428e

3 files changed

Lines changed: 254 additions & 5 deletions

File tree

‎javascript/extractor/src/com/semmle/jcorn/CustomParser.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -546,12 +546,21 @@ protected XMLAttributeSelector decoratorToAttributeSelector(Decorator d) {
546546

547547
@Override
548548
protected Token readToken(int code) {
549-
// skip XML processing instructions (which are allowed in E4X, but not in JSX)
549+
// skip XML processing instructions (which are allowed in E4X, but not in JSX);
550+
// there is a lexical ambiguity between an XML processing instruction starting a
551+
// chunk of E4X content and a Flow type annotation (both can start with `<?`)
552+
// hence if we can't find the closing `?>` of a putative XML processing instruction
553+
// we backtrack and try lexing as something else
550554
if (this.options.e4x()) {
551555
while (code == '<') {
552556
if (charAt(this.pos+1) == '?') {
557+
int oldPos = this.pos;
553558
this.pos += 2;
554-
jsx_readUntil("?>");
559+
if (!jsx_readUntil("?>")) {
560+
// didn't find a closing `?>`, so backtrack
561+
this.pos = oldPos;
562+
break;
563+
}
555564
} else {
556565
break;
557566
}
@@ -564,7 +573,10 @@ protected Token readToken(int code) {
564573

565574
@Override
566575
protected Either<Integer, Token> jsx_readChunk(StringBuilder out, int chunkStart, int ch) {
567-
// skip XML comments, processing instructions and CDATA (which are allowed in E4X, but not in JSX)
576+
// skip XML comments, processing instructions and CDATA (which are allowed in E4X,
577+
// but not in JSX)
578+
// unlike in `readToken` above, we know that we're inside JSX/E4X code, so there is
579+
// no ambiguity with Flow type annotations
568580
if (this.options.e4x() && ch == '<') {
569581
if (inputSubstring(this.pos+1, this.pos+4).equals("!--")) {
570582
out.append(inputSubstring(chunkStart, this.pos));
@@ -589,15 +601,16 @@ protected Either<Integer, Token> jsx_readChunk(StringBuilder out, int chunkStart
589601
return super.jsx_readChunk(out, chunkStart, ch);
590602
}
591603

592-
private void jsx_readUntil(String terminator) {
604+
private boolean jsx_readUntil(String terminator) {
593605
char fst = terminator.charAt(0);
594606
while (this.pos+terminator.length() <= this.input.length()) {
595607
if (charAt(this.pos) == fst &&
596608
inputSubstring(this.pos, this.pos+terminator.length()).equals(terminator)) {
597609
this.pos += terminator.length();
598-
break;
610+
return true;
599611
}
600612
++this.pos;
601613
}
614+
return false;
602615
}
603616
}

‎javascript/extractor/tests/flow/input/ambig.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
declare module "test" {
2+
declare type Foo = {|
3+
|};
4+
5+
declare type Bar = {|
6+
baz: () => Promise<?Foo>
7+
|};
8+
}

‎javascript/extractor/tests/flow/output/trap/ambig.js.trap‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
#10000=@"/ambig.js;sourcefile"
2+
files(#10000,"/ambig.js","ambig","js",0)
3+
#10001=@"/;folder"
4+
folders(#10001,"/","")
5+
containerparent(#10001,#10000)
6+
#10002=@"loc,{#10000},0,0,0,0"
7+
locations_default(#10002,#10000,0,0,0,0)
8+
hasLocation(#10000,#10002)
9+
#20000=@"global_scope"
10+
scopes(#20000,0)
11+
#20001=@"script;{#10000},1,1"
12+
#20002=*
13+
lines(#20002,#20001,"declare module ""test"" {","
14+
")
15+
#20003=@"loc,{#10000},1,1,1,23"
16+
locations_default(#20003,#10000,1,1,1,23)
17+
hasLocation(#20002,#20003)
18+
#20004=*
19+
lines(#20004,#20001," declare type Foo = {|","
20+
")
21+
#20005=@"loc,{#10000},2,1,2,23"
22+
locations_default(#20005,#10000,2,1,2,23)
23+
hasLocation(#20004,#20005)
24+
indentation(#10000,2," ",2)
25+
#20006=*
26+
lines(#20006,#20001," |};","
27+
")
28+
#20007=@"loc,{#10000},3,1,3,5"
29+
locations_default(#20007,#10000,3,1,3,5)
30+
hasLocation(#20006,#20007)
31+
indentation(#10000,3," ",2)
32+
#20008=*
33+
lines(#20008,#20001,"","
34+
")
35+
#20009=@"loc,{#10000},4,1,4,0"
36+
locations_default(#20009,#10000,4,1,4,0)
37+
hasLocation(#20008,#20009)
38+
#20010=*
39+
lines(#20010,#20001," declare type Bar = {|","
40+
")
41+
#20011=@"loc,{#10000},5,1,5,23"
42+
locations_default(#20011,#10000,5,1,5,23)
43+
hasLocation(#20010,#20011)
44+
indentation(#10000,5," ",2)
45+
#20012=*
46+
lines(#20012,#20001," baz: () => Promise<?Foo>","
47+
")
48+
#20013=@"loc,{#10000},6,1,6,28"
49+
locations_default(#20013,#10000,6,1,6,28)
50+
hasLocation(#20012,#20013)
51+
indentation(#10000,6," ",4)
52+
#20014=*
53+
lines(#20014,#20001," |};","
54+
")
55+
#20015=@"loc,{#10000},7,1,7,5"
56+
locations_default(#20015,#10000,7,1,7,5)
57+
hasLocation(#20014,#20015)
58+
indentation(#10000,7," ",2)
59+
#20016=*
60+
lines(#20016,#20001,"}","
61+
")
62+
#20017=@"loc,{#10000},8,1,8,1"
63+
locations_default(#20017,#10000,8,1,8,1)
64+
hasLocation(#20016,#20017)
65+
numlines(#20001,8,7,0)
66+
#20018=*
67+
tokeninfo(#20018,6,#20001,0,"declare")
68+
#20019=@"loc,{#10000},1,1,1,7"
69+
locations_default(#20019,#10000,1,1,1,7)
70+
hasLocation(#20018,#20019)
71+
#20020=*
72+
tokeninfo(#20020,6,#20001,1,"module")
73+
#20021=@"loc,{#10000},1,9,1,14"
74+
locations_default(#20021,#10000,1,9,1,14)
75+
hasLocation(#20020,#20021)
76+
#20022=*
77+
tokeninfo(#20022,4,#20001,2,"""test""")
78+
#20023=@"loc,{#10000},1,16,1,21"
79+
locations_default(#20023,#10000,1,16,1,21)
80+
hasLocation(#20022,#20023)
81+
#20024=*
82+
tokeninfo(#20024,8,#20001,3,"{")
83+
#20025=@"loc,{#10000},1,23,1,23"
84+
locations_default(#20025,#10000,1,23,1,23)
85+
hasLocation(#20024,#20025)
86+
#20026=*
87+
tokeninfo(#20026,6,#20001,4,"declare")
88+
#20027=@"loc,{#10000},2,3,2,9"
89+
locations_default(#20027,#10000,2,3,2,9)
90+
hasLocation(#20026,#20027)
91+
#20028=*
92+
tokeninfo(#20028,6,#20001,5,"type")
93+
#20029=@"loc,{#10000},2,11,2,14"
94+
locations_default(#20029,#10000,2,11,2,14)
95+
hasLocation(#20028,#20029)
96+
#20030=*
97+
tokeninfo(#20030,6,#20001,6,"Foo")
98+
#20031=@"loc,{#10000},2,16,2,18"
99+
locations_default(#20031,#10000,2,16,2,18)
100+
hasLocation(#20030,#20031)
101+
#20032=*
102+
tokeninfo(#20032,8,#20001,7,"=")
103+
#20033=@"loc,{#10000},2,20,2,20"
104+
locations_default(#20033,#10000,2,20,2,20)
105+
hasLocation(#20032,#20033)
106+
#20034=*
107+
tokeninfo(#20034,8,#20001,8,"{|")
108+
#20035=@"loc,{#10000},2,22,2,23"
109+
locations_default(#20035,#10000,2,22,2,23)
110+
hasLocation(#20034,#20035)
111+
#20036=*
112+
tokeninfo(#20036,8,#20001,9,"|}")
113+
#20037=@"loc,{#10000},3,3,3,4"
114+
locations_default(#20037,#10000,3,3,3,4)
115+
hasLocation(#20036,#20037)
116+
#20038=*
117+
tokeninfo(#20038,8,#20001,10,";")
118+
#20039=@"loc,{#10000},3,5,3,5"
119+
locations_default(#20039,#10000,3,5,3,5)
120+
hasLocation(#20038,#20039)
121+
#20040=*
122+
tokeninfo(#20040,6,#20001,11,"declare")
123+
#20041=@"loc,{#10000},5,3,5,9"
124+
locations_default(#20041,#10000,5,3,5,9)
125+
hasLocation(#20040,#20041)
126+
#20042=*
127+
tokeninfo(#20042,6,#20001,12,"type")
128+
#20043=@"loc,{#10000},5,11,5,14"
129+
locations_default(#20043,#10000,5,11,5,14)
130+
hasLocation(#20042,#20043)
131+
#20044=*
132+
tokeninfo(#20044,6,#20001,13,"Bar")
133+
#20045=@"loc,{#10000},5,16,5,18"
134+
locations_default(#20045,#10000,5,16,5,18)
135+
hasLocation(#20044,#20045)
136+
#20046=*
137+
tokeninfo(#20046,8,#20001,14,"=")
138+
#20047=@"loc,{#10000},5,20,5,20"
139+
locations_default(#20047,#10000,5,20,5,20)
140+
hasLocation(#20046,#20047)
141+
#20048=*
142+
tokeninfo(#20048,8,#20001,15,"{|")
143+
#20049=@"loc,{#10000},5,22,5,23"
144+
locations_default(#20049,#10000,5,22,5,23)
145+
hasLocation(#20048,#20049)
146+
#20050=*
147+
tokeninfo(#20050,6,#20001,16,"baz")
148+
#20051=@"loc,{#10000},6,5,6,7"
149+
locations_default(#20051,#10000,6,5,6,7)
150+
hasLocation(#20050,#20051)
151+
#20052=*
152+
tokeninfo(#20052,8,#20001,17,":")
153+
#20053=@"loc,{#10000},6,8,6,8"
154+
locations_default(#20053,#10000,6,8,6,8)
155+
hasLocation(#20052,#20053)
156+
#20054=*
157+
tokeninfo(#20054,8,#20001,18,"(")
158+
#20055=@"loc,{#10000},6,10,6,10"
159+
locations_default(#20055,#10000,6,10,6,10)
160+
hasLocation(#20054,#20055)
161+
#20056=*
162+
tokeninfo(#20056,8,#20001,19,")")
163+
#20057=@"loc,{#10000},6,11,6,11"
164+
locations_default(#20057,#10000,6,11,6,11)
165+
hasLocation(#20056,#20057)
166+
#20058=*
167+
tokeninfo(#20058,8,#20001,20,"=>")
168+
#20059=@"loc,{#10000},6,13,6,14"
169+
locations_default(#20059,#10000,6,13,6,14)
170+
hasLocation(#20058,#20059)
171+
#20060=*
172+
tokeninfo(#20060,6,#20001,21,"Promise")
173+
#20061=@"loc,{#10000},6,16,6,22"
174+
locations_default(#20061,#10000,6,16,6,22)
175+
hasLocation(#20060,#20061)
176+
#20062=*
177+
tokeninfo(#20062,8,#20001,22,"<")
178+
#20063=@"loc,{#10000},6,23,6,23"
179+
locations_default(#20063,#10000,6,23,6,23)
180+
hasLocation(#20062,#20063)
181+
#20064=*
182+
tokeninfo(#20064,8,#20001,23,"?")
183+
#20065=@"loc,{#10000},6,24,6,24"
184+
locations_default(#20065,#10000,6,24,6,24)
185+
hasLocation(#20064,#20065)
186+
#20066=*
187+
tokeninfo(#20066,6,#20001,24,"Foo")
188+
#20067=@"loc,{#10000},6,25,6,27"
189+
locations_default(#20067,#10000,6,25,6,27)
190+
hasLocation(#20066,#20067)
191+
#20068=*
192+
tokeninfo(#20068,8,#20001,25,">")
193+
#20069=@"loc,{#10000},6,28,6,28"
194+
locations_default(#20069,#10000,6,28,6,28)
195+
hasLocation(#20068,#20069)
196+
#20070=*
197+
tokeninfo(#20070,8,#20001,26,"|}")
198+
#20071=@"loc,{#10000},7,3,7,4"
199+
locations_default(#20071,#10000,7,3,7,4)
200+
hasLocation(#20070,#20071)
201+
#20072=*
202+
tokeninfo(#20072,8,#20001,27,";")
203+
#20073=@"loc,{#10000},7,5,7,5"
204+
locations_default(#20073,#10000,7,5,7,5)
205+
hasLocation(#20072,#20073)
206+
#20074=*
207+
tokeninfo(#20074,8,#20001,28,"}")
208+
hasLocation(#20074,#20017)
209+
#20075=*
210+
tokeninfo(#20075,0,#20001,29,"")
211+
#20076=@"loc,{#10000},9,1,9,0"
212+
locations_default(#20076,#10000,9,1,9,0)
213+
hasLocation(#20075,#20076)
214+
toplevels(#20001,0)
215+
#20077=@"loc,{#10000},1,1,9,0"
216+
locations_default(#20077,#10000,1,1,9,0)
217+
hasLocation(#20001,#20077)
218+
#20078=*
219+
entry_cfg_node(#20078,#20001)
220+
#20079=@"loc,{#10000},1,1,1,0"
221+
locations_default(#20079,#10000,1,1,1,0)
222+
hasLocation(#20078,#20079)
223+
#20080=*
224+
exit_cfg_node(#20080,#20001)
225+
hasLocation(#20080,#20076)
226+
successor(#20078,#20080)
227+
numlines(#10000,8,7,0)
228+
filetype(#10000,"javascript")

0 commit comments

Comments
 (0)