Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7bb8edd

Browse files
semmle-qlcisemmle-qlci
authored
Merge pull request #720 from esben-semmle/js/more-flow-parsing
Approved by xiemaisi
2 parents 9c41b21 + 73af2ad commit 7bb8edd

28 files changed

Lines changed: 3466 additions & 196 deletions

‎change-notes/1.20/extractor-javascript.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,6 @@
1818

1919
## Changes to code extraction
2020

21+
* The extractor now supports additional [Flow](https://flow.org/) syntax.
2122
* The extractor now supports [Nullish Coalescing](https://github.com/tc39/proposal-nullish-coalescing) expressions.
2223
* The TypeScript extractor now handles the control-flow of logical operators and destructuring assignments more accurately.

‎javascript/extractor/src/com/semmle/jcorn/Parser.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,6 +1744,10 @@ protected Expression parseNew() {
17441744
if (isOnOptionalChain(false, callee))
17451745
this.raise(callee, "An optional chain may not be used in a `new` expression.");
17461746

1747+
return parseNewArguments(startLoc, callee);
1748+
}
1749+
1750+
protected Expression parseNewArguments(Position startLoc, Expression callee) {
17471751
List<Expression> arguments;
17481752
if (this.eat(TokenType.parenL))
17491753
arguments = this.parseExprList(TokenType.parenR, this.options.ecmaVersion() >= 8, false, null);

‎javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java‎

Lines changed: 70 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,24 @@
1515
import com.semmle.jcorn.TokenType;
1616
import com.semmle.jcorn.TokenType.Properties;
1717
import com.semmle.jcorn.Whitespace;
18+
import com.semmle.js.ast.BinaryExpression;
1819
import com.semmle.js.ast.ExportDeclaration;
1920
import com.semmle.js.ast.ExportSpecifier;
2021
import com.semmle.js.ast.Expression;
2122
import com.semmle.js.ast.ExpressionStatement;
2223
import com.semmle.js.ast.FieldDefinition;
2324
import com.semmle.js.ast.Identifier;
25+
import com.semmle.js.ast.ImportDeclaration;
2426
import com.semmle.js.ast.ImportSpecifier;
27+
import com.semmle.js.ast.Literal;
2528
import com.semmle.js.ast.MethodDefinition;
2629
import com.semmle.js.ast.Node;
2730
import com.semmle.js.ast.Position;
2831
import com.semmle.js.ast.SourceLocation;
2932
import com.semmle.js.ast.Statement;
3033
import com.semmle.js.ast.Token;
34+
import com.semmle.js.ast.UnaryExpression;
35+
import com.semmle.util.data.Pair;
3136
import com.semmle.util.exception.Exceptions;
3237

3338
/**
@@ -226,30 +231,21 @@ private void flowParseDeclareModule(Position start) {
226231
this.expect(TokenType.braceL);
227232
while (this.type != TokenType.braceR) {
228233
Position stmtStart = startLoc;
229-
230-
if (this.eat(TokenType._import)) {
231-
this.flowParseDeclareImport(stmtStart);
232-
} else {
233-
// todo: declare check
234-
this.next();
235-
234+
if (this.eatContextual("declare")) {
236235
this.flowParseDeclare(stmtStart);
236+
} else if (this.eat(TokenType._import)) {
237+
if (peekAtSpecialFlowImportSpecifier() == null) {
238+
this.raise(stmtStart,
239+
"Imports within a `declare module` body must always be `import type` or `import typeof`.");
240+
}
241+
this.parseImportRest(new SourceLocation(stmtStart));
242+
} else {
243+
unexpected();
237244
}
238245
}
239246
this.expect(TokenType.braceR);
240247
}
241248

242-
private void flowParseDeclareImport(Position stmtStart) {
243-
String kind = flowParseImportSpecifiers();
244-
if (kind == null) {
245-
this.raise(stmtStart, "Imports within a `declare module` body must always be `import type` or `import typeof`.");
246-
}
247-
this.expect(TokenType.name);
248-
this.expectContextual("from");
249-
this.expect(TokenType.string);
250-
this.semicolon();
251-
}
252-
253249
private void flowParseDeclareModuleExports() {
254250
this.expectContextual("module");
255251
this.expect(TokenType.dot);
@@ -919,6 +915,11 @@ protected ExportDeclaration parseExportRest(SourceLocation loc, Set<String> expo
919915
List<ExportSpecifier> specifiers = this.parseExportSpecifiers(exports);
920916
this.parseExportFrom(specifiers, null, false);
921917
return null;
918+
} else if (this.eat(TokenType.star)) {
919+
if (this.eatContextual("as"))
920+
this.parseIdent(true);
921+
this.parseExportFrom(null, null, true);
922+
return null;
922923
} else {
923924
// `export type Foo = Bar;`
924925
this.flowParseTypeAlias(startLoc);
@@ -1004,12 +1005,7 @@ protected Expression processBindingListItem(Expression param) {
10041005
}
10051006

10061007
private String flowParseImportSpecifiers() {
1007-
String kind = null;
1008-
if (this.type == TokenType._typeof) {
1009-
kind = "typeof";
1010-
} else if (this.isContextual("type")) {
1011-
kind = "type";
1012-
}
1008+
String kind = peekAtSpecialFlowImportSpecifier();
10131009
if (kind != null) {
10141010
String lh = lookahead(4);
10151011
if (!lh.isEmpty()) {
@@ -1022,6 +1018,16 @@ private String flowParseImportSpecifiers() {
10221018
return kind;
10231019
}
10241020

1021+
private String peekAtSpecialFlowImportSpecifier() {
1022+
String kind = null;
1023+
if (this.type == TokenType._typeof) {
1024+
kind = "typeof";
1025+
} else if (this.isContextual("type")) {
1026+
kind = "type";
1027+
}
1028+
return kind;
1029+
}
1030+
10251031
@Override
10261032
protected List<ImportSpecifier> parseImportSpecifiers() {
10271033
String kind = null;
@@ -1037,7 +1043,7 @@ protected List<ImportSpecifier> parseImportSpecifiers() {
10371043

10381044
@Override
10391045
protected ImportSpecifier parseImportSpecifier() {
1040-
if (this.type == TokenType._typeof || this.isContextual("type")) {
1046+
if (peekAtSpecialFlowImportSpecifier() != null) {
10411047
String lh = lookahead(2);
10421048
if (lh.charAt(0) == ',' || lh.charAt(0) == '}' || lh.equals("as"))
10431049
return super.parseImportSpecifier();
@@ -1201,4 +1207,43 @@ protected boolean atGetterSetterName(PropertyInfo pi) {
12011207
return super.atGetterSetterName(pi);
12021208
}
12031209

1210+
@Override
1211+
protected Pair<Expression, Boolean> parseSubscript(final Expression base, Position startLoc, boolean noCalls) {
1212+
if (!noCalls) {
1213+
maybeFlowParseTypeParameterInstantiation(base, true);
1214+
}
1215+
return super.parseSubscript(base, startLoc, noCalls);
1216+
}
1217+
1218+
private void maybeFlowParseTypeParameterInstantiation(Expression left, boolean requireParenL) {
1219+
if (flow() && this.isRelational("<")) {
1220+
// Ambiguous case: `e1<e2>(e3)` is parsed differently as JS and Flow code:
1221+
// JS: two relational comparisons: `e1 < e2 > e3`
1222+
// Flow: a call `e1(e3)` with explicit type parameter `e2`
1223+
1224+
// Heuristic: if the left operand of the `<` token is a primitive from a literal or unary/binary expression, then it probably isn't a call, as that would always crash
1225+
left = left.stripParens();
1226+
if (left instanceof Literal || left instanceof UnaryExpression || left instanceof BinaryExpression)
1227+
return;
1228+
1229+
// If it can be parsed as Flow, we use that, otherwise we parse it as JS
1230+
State backup = new State();
1231+
try {
1232+
this.flowParseTypeParameterInstantiation();
1233+
if (requireParenL && this.type != TokenType.parenL) {
1234+
unexpected();
1235+
}
1236+
backup.commit();
1237+
} catch (SyntaxError e) {
1238+
Exceptions.ignore(e, "Backtracking parser.");
1239+
backup.reset();
1240+
}
1241+
}
1242+
}
1243+
1244+
@Override
1245+
protected Expression parseNewArguments(Position startLoc, Expression callee) {
1246+
maybeFlowParseTypeParameterInstantiation(callee, false /* case: new e1<e2>e3 */);
1247+
return super.parseNewArguments(startLoc, callee);
1248+
}
12041249
}

‎javascript/extractor/src/com/semmle/js/extractor/Main.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class Main {
4141
* such a way that it may produce different tuples for the same file under the same
4242
* {@link ExtractorConfig}.
4343
*/
44-
public static final String EXTRACTOR_VERSION = "2018-12-19";
44+
public static final String EXTRACTOR_VERSION = "2019-09-01";
4545

4646
public static final Pattern NEWLINE = Pattern.compile("\n");
4747

‎javascript/extractor/tests/flow/input/boundExplicitTypeParameters.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
var b = ::o.m<T>;

‎javascript/extractor/tests/flow/input/declareExport.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ declare export var foo : boolean;
22
declare export default typeof simpleTextBuffer$TextBuffer;
33
declare export default marked$Marked;
44
declare export type ProcessEnv = { };
5+
declare module "foo" { declare export type * from "bar"; }

‎javascript/extractor/tests/flow/input/explicitTypeParameters.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
new K1<T1>();
2+
f1<T2>();
3+
const f2 = function*(): T3<T4> {
4+
yield* f3<T5>(x);
5+
}
6+
f4<T6>(v1);
7+
f5<_,_,_>();
8+
f6<
9+
|T7
10+
|T8
11+
|T9
12+
>();

‎javascript/extractor/tests/flow/input/explicitTypeParametersForArgumentlessConstructorCall.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
new K1<T1>;

‎javascript/extractor/tests/flow/input/export.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
export { a } from "m";
2+
export * from "m";
3+
export type { b } from "m";
4+
export type * from "m";
5+
export type * as fooTypes from "foo";

‎javascript/extractor/tests/flow/input/importNonTypeInDeclaredModule.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
declare module "m1" {
2+
import {
3+
T1,
4+
T2
5+
} from "m2"
6+
}
7+

0 commit comments

Comments
 (0)