Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0957801

Browse files
redsun82redsun82
authored
Merge pull request #9521 from github/redsun82/swift-qltestgen
Swift: generated extractor tests
2 parents f14a90f + 78deff6 commit 0957801

314 files changed

Lines changed: 2117 additions & 528 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/swift-codegen.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
run: |
2323
bazel run //swift/codegen
2424
git add swift
25-
git diff --exit-code --stat HEAD
25+
git diff --exit-code HEAD
2626
- name: Generate C++ files
2727
run: |
2828
bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers

‎swift/codegen/codegen.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ def _parse_args() -> argparse.Namespace:
3131
p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements",
3232
help="output directory for QL stub/customization files (default %(default)s). Defines also the "
3333
"generated qll file importing every class file")
34+
p.add_argument("--ql-test-output", type=_abspath, default=paths.swift_dir / "ql/test/extractor-tests/generated",
35+
help="output directory for QL generated extractor test files (default %(default)s)")
3436
p.add_argument("--ql-format", action="store_true", default=True,
3537
help="use codeql to autoformat QL files (which is the default)")
3638
p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")

‎swift/codegen/generators/qlgen.py‎

Lines changed: 89 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import logging
44
import pathlib
55
import subprocess
6+
import typing
67

78
import inflection
89

@@ -11,41 +12,48 @@
1112
log = logging.getLogger(__name__)
1213

1314

15+
class FormatError(Exception):
16+
pass
17+
18+
1419
def get_ql_property(cls: schema.Class, prop: schema.Property):
20+
common_args = dict(
21+
type=prop.type if not prop.is_predicate else "predicate",
22+
skip_qltest="no_qltest" in prop.tags,
23+
is_child=prop.is_child,
24+
is_optional=prop.is_optional,
25+
is_predicate=prop.is_predicate,
26+
)
1527
if prop.is_single:
1628
return ql.Property(
29+
**common_args,
1730
singular=inflection.camelize(prop.name),
18-
type=prop.type,
1931
tablename=inflection.tableize(cls.name),
20-
tableparams=["this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
21-
is_child=prop.is_child,
32+
tableparams=[
33+
"this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
2234
)
2335
elif prop.is_repeated:
2436
return ql.Property(
37+
**common_args,
2538
singular=inflection.singularize(inflection.camelize(prop.name)),
2639
plural=inflection.pluralize(inflection.camelize(prop.name)),
27-
type=prop.type,
2840
tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
2941
tableparams=["this", "index", "result"],
30-
is_optional=prop.is_optional,
31-
is_child=prop.is_child,
3242
)
3343
elif prop.is_optional:
3444
return ql.Property(
45+
**common_args,
3546
singular=inflection.camelize(prop.name),
36-
type=prop.type,
3747
tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
3848
tableparams=["this", "result"],
39-
is_optional=True,
40-
is_child=prop.is_child,
4149
)
4250
elif prop.is_predicate:
4351
return ql.Property(
44-
singular=inflection.camelize(prop.name, uppercase_first_letter=False),
45-
type="predicate",
52+
**common_args,
53+
singular=inflection.camelize(
54+
prop.name, uppercase_first_letter=False),
4655
tablename=inflection.underscore(f"{cls.name}_{prop.name}"),
4756
tableparams=["this"],
48-
is_predicate=True,
4957
)
5058

5159

@@ -56,6 +64,7 @@ def get_ql_class(cls: schema.Class):
5664
final=not cls.derived,
5765
properties=[get_ql_property(cls, p) for p in cls.properties],
5866
dir=cls.dir,
67+
skip_qltest="no_qltest" in cls.tags,
5968
)
6069

6170

@@ -77,48 +86,106 @@ def get_classes_used_by(cls: ql.Class):
7786

7887
def is_generated(file):
7988
with open(file) as contents:
80-
return next(contents).startswith("// generated")
89+
for line in contents:
90+
return line.startswith("// generated")
91+
return False
8192

8293

8394
def format(codeql, files):
8495
format_cmd = [codeql, "query", "format", "--in-place", "--"]
85-
format_cmd.extend(str(f) for f in files)
86-
res = subprocess.run(format_cmd, check=True, stderr=subprocess.PIPE, text=True)
96+
format_cmd.extend(str(f) for f in files if f.suffix in (".qll", ".ql"))
97+
res = subprocess.run(format_cmd, stderr=subprocess.PIPE, text=True)
98+
if res.returncode:
99+
for line in res.stderr.splitlines():
100+
log.error(line.strip())
101+
raise FormatError("QL format failed")
87102
for line in res.stderr.splitlines():
88103
log.debug(line.strip())
89104

90105

106+
def _get_all_properties(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
107+
typing.Tuple[ql.Class, ql.Property]]:
108+
for b in cls.bases:
109+
base = lookup[b]
110+
for item in _get_all_properties(base, lookup):
111+
yield item
112+
for p in cls.properties:
113+
yield cls, p
114+
115+
116+
def _get_all_properties_to_be_tested(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
117+
ql.PropertyForTest]:
118+
# deduplicate using id
119+
already_seen = set()
120+
for c, p in _get_all_properties(cls, lookup):
121+
if not (c.skip_qltest or p.skip_qltest or id(p) in already_seen):
122+
already_seen.add(id(p))
123+
yield ql.PropertyForTest(p.getter, p.type, p.is_single, p.is_predicate, p.is_repeated)
124+
125+
126+
def _partition(l, pred):
127+
""" partitions a list according to boolean predicate """
128+
res = ([], [])
129+
for x in l:
130+
res[not pred(x)].append(x)
131+
return res
132+
133+
91134
def generate(opts, renderer):
92135
input = opts.schema
93136
out = opts.ql_output
94137
stub_out = opts.ql_stub_output
138+
test_out = opts.ql_test_output
139+
missing_test_source_filename = "MISSING_SOURCE.txt"
95140
existing = {q for q in out.rglob("*.qll")}
96141
existing |= {q for q in stub_out.rglob("*.qll") if is_generated(q)}
142+
existing |= {q for q in test_out.rglob("*.ql")}
143+
existing |= {q for q in test_out.rglob(missing_test_source_filename)}
97144

98145
data = schema.load(input)
99146

100147
classes = [get_ql_class(cls) for cls in data.classes]
101-
classes.sort(key=lambda cls: cls.name)
148+
lookup = {cls.name: cls for cls in classes}
149+
classes.sort(key=lambda cls: (cls.dir, cls.name))
102150
imports = {}
103151

104152
for c in classes:
105153
imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
106154

107155
for c in classes:
108-
qll = (out / c.path).with_suffix(".qll")
156+
qll = out / c.path.with_suffix(".qll")
109157
c.imports = [imports[t] for t in get_classes_used_by(c)]
110158
renderer.render(c, qll)
111-
stub_file = (stub_out / c.path).with_suffix(".qll")
159+
stub_file = stub_out / c.path.with_suffix(".qll")
112160
if not stub_file.is_file() or is_generated(stub_file):
113-
stub = ql.Stub(name=c.name, base_import=get_import(qll, opts.swift_dir))
161+
stub = ql.Stub(
162+
name=c.name, base_import=get_import(qll, opts.swift_dir))
114163
renderer.render(stub, stub_file)
115164

116165
# for example path/to/elements -> path/to/elements.qll
117166
include_file = stub_out.with_suffix(".qll")
118-
all_imports = ql.ImportList(list(sorted(imports.values())))
119-
renderer.render(all_imports, include_file)
167+
renderer.render(ql.ImportList(list(imports.values())), include_file)
168+
169+
renderer.render(ql.GetParentImplementation(
170+
classes), out / 'GetImmediateParent.qll')
120171

121-
renderer.render(ql.GetParentImplementation(classes), out / 'GetImmediateParent.qll')
172+
for c in classes:
173+
if not c.final or c.skip_qltest:
174+
continue
175+
test_dir = test_out / c.path
176+
test_dir.mkdir(parents=True, exist_ok=True)
177+
if not any(test_dir.glob("*.swift")):
178+
log.warning(f"no test source in {c.path}")
179+
renderer.render(ql.MissingTestInstructions(),
180+
test_dir / missing_test_source_filename)
181+
continue
182+
total_props, partial_props = _partition(_get_all_properties_to_be_tested(c, lookup),
183+
lambda p: p.is_single or p.is_predicate)
184+
renderer.render(ql.ClassTester(class_name=c.name,
185+
properties=total_props), test_dir / f"{c.name}.ql")
186+
for p in partial_props:
187+
renderer.render(ql.PropertyTester(class_name=c.name,
188+
property=p), test_dir / f"{c.name}_{p.getter}.ql")
122189

123190
renderer.cleanup(existing)
124191
if opts.ql_format:

‎swift/codegen/lib/ql.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ class Property:
3737
is_optional: bool = False
3838
is_predicate: bool = False
3939
is_child: bool = False
40+
skip_qltest: bool = False
4041

4142
def __post_init__(self):
4243
if self.tableparams:
@@ -63,6 +64,10 @@ def type_is_class(self):
6364
def is_repeated(self):
6465
return bool(self.plural)
6566

67+
@property
68+
def is_single(self):
69+
return not (self.is_optional or self.is_repeated or self.is_predicate)
70+
6671

6772
@dataclass
6873
class Class:
@@ -74,6 +79,7 @@ class Class:
7479
properties: List[Property] = field(default_factory=list)
7580
dir: pathlib.Path = pathlib.Path()
7681
imports: List[str] = field(default_factory=list)
82+
skip_qltest: bool = False
7783

7884
def __post_init__(self):
7985
self.bases = sorted(self.bases)
@@ -113,3 +119,33 @@ class GetParentImplementation:
113119
template: ClassVar = 'ql_parent'
114120

115121
classes: List[Class] = field(default_factory=list)
122+
123+
124+
@dataclass
125+
class PropertyForTest:
126+
getter: str
127+
type: str = None
128+
is_single: bool = False
129+
is_predicate: bool = False
130+
is_repeated: bool = False
131+
132+
133+
@dataclass
134+
class ClassTester:
135+
template: ClassVar = 'ql_test_class'
136+
137+
class_name: str
138+
properties: List[PropertyForTest] = field(default_factory=list)
139+
140+
141+
@dataclass
142+
class PropertyTester:
143+
template: ClassVar = 'ql_test_property'
144+
145+
class_name: str
146+
property: PropertyForTest
147+
148+
149+
@dataclass
150+
class MissingTestInstructions:
151+
template: ClassVar = 'ql_test_missing'

‎swift/codegen/lib/schema.py‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,17 @@
33
import pathlib
44
import re
55
from dataclasses import dataclass, field
6-
from typing import List, Set, Dict, ClassVar
6+
from typing import List, Set, Union, Dict, ClassVar
77

88
import yaml
99

10+
11+
class Error(Exception):
12+
13+
def __str__(self):
14+
return f"schema.Error{args}"
15+
16+
1017
root_class_name = "Element"
1118

1219

@@ -20,6 +27,7 @@ class Property:
2027
name: str
2128
type: str = None
2229
is_child: bool = False
30+
tags: List[str] = field(default_factory=list)
2331

2432

2533
@dataclass
@@ -55,6 +63,7 @@ class Class:
5563
derived: Set[str] = field(default_factory=set)
5664
properties: List[Property] = field(default_factory=list)
5765
dir: pathlib.Path = pathlib.Path()
66+
tags: List[str] = field(default_factory=list)
5867

5968

6069
@dataclass
@@ -63,18 +72,24 @@ class Schema:
6372
includes: Set[str] = field(default_factory=set)
6473

6574

66-
def _parse_property(name: str, type: str, is_child: bool = False):
67-
assert not (is_child and type[0].islower()), f"children must have class type, got {type} for {name}"
75+
def _parse_property(name: str, type: Union[str, Dict[str, str]], is_child: bool = False):
76+
if isinstance(type, dict):
77+
tags = type.get("_tags", [])
78+
type = type["type"]
79+
else:
80+
tags = []
81+
if is_child and type[0].islower():
82+
raise Error(f"children must have class type, got {type} for {name}")
6883
if type.endswith("?*"):
69-
return RepeatedOptionalProperty(name, type[:-2], is_child=is_child)
84+
return RepeatedOptionalProperty(name, type[:-2], is_child=is_child, tags=tags)
7085
elif type.endswith("*"):
71-
return RepeatedProperty(name, type[:-1], is_child=is_child)
86+
return RepeatedProperty(name, type[:-1], is_child=is_child, tags=tags)
7287
elif type.endswith("?"):
73-
return OptionalProperty(name, type[:-1], is_child=is_child)
88+
return OptionalProperty(name, type[:-1], is_child=is_child, tags=tags)
7489
elif type == "predicate":
75-
return PredicateProperty(name)
90+
return PredicateProperty(name, tags=tags)
7691
else:
77-
return SingleProperty(name, type, is_child=is_child)
92+
return SingleProperty(name, type, is_child=is_child, tags=tags)
7893

7994

8095
class _DirSelector:
@@ -98,7 +113,8 @@ def load(path):
98113
for name, info in data.items():
99114
if name.startswith("_"):
100115
continue
101-
assert name[0].isupper()
116+
if not name[0].isupper():
117+
raise Error(f"keys in the schema file must be capitalized class names or metadata, got {name}")
102118
cls = classes[name]
103119
for k, v in info.items():
104120
if not k.startswith("_"):
@@ -113,6 +129,10 @@ def load(path):
113129
cls.dir = pathlib.Path(v)
114130
elif k == "_children":
115131
cls.properties.extend(_parse_property(kk, vv, is_child=True) for kk, vv in v.items())
132+
elif k == "_tags":
133+
cls.tags = v
134+
else:
135+
raise Error(f"unknown metadata {k} for class {name}")
116136
if not cls.bases and cls.name != root_class_name:
117137
cls.bases.add(root_class_name)
118138
classes[root_class_name].derived.add(name)

‎swift/codegen/schema.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,20 @@ _directories:
1313
stmt: Stmt$
1414

1515
Element:
16-
is_unknown: predicate
16+
is_unknown:
17+
type: predicate
18+
_tags: [ no_qltest ]
1719

1820
File:
1921
name: string
2022

2123
Locatable:
22-
location: Location
24+
location:
25+
type: Location
26+
_tags: [no_qltest]
2327

2428
Location:
29+
_tags: [ no_qltest ]
2530
file: File
2631
start_line: int
2732
start_column: int

0 commit comments

Comments
 (0)