Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ce214cf

Browse files
tamasvajktamasvajk
committed
Split generated stubs to separate files
1 parent 88c97bd commit ce214cf

3 files changed

Lines changed: 148 additions & 23 deletions

File tree

‎csharp/ql/src/Stubs/AllStubsFromReference.ql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ class NonTargetAssembly extends ExcludedAssembly {
1818
}
1919

2020
from Assembly a
21-
select a, generatedCode(a)
21+
select a.getFullName(), a.getName(), a.getVersion().toString(), a.getFile().getAbsolutePath(),
22+
generatedCode(a)

‎csharp/ql/src/Stubs/helpers.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,10 @@ def trim_output_file(file):
3535
f = open(file, "wb")
3636
f.write(contents)
3737
f.close()
38+
39+
40+
# remove all files with extension
41+
def remove_files(path, ext):
42+
for file in os.listdir(path):
43+
if file.endswith(ext):
44+
os.remove(os.path.join(path, file))

‎csharp/ql/src/Stubs/make_stubs_nuget.py‎

Lines changed: 139 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import sys
22
import os
33
import helpers
4+
import json
5+
import shutil
46

57
print('Script to generate stub file from a nuget package')
68
print(' Usage: python ' + sys.argv[0] +
7-
' NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir] [OUTPUT_NAME=stub]')
9+
' NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir]')
810
print(' The script uses the dotnet cli, codeql cli, and dotnet format global tool')
911

1012
if len(sys.argv) < 2:
@@ -15,22 +17,37 @@
1517
thisDir = os.path.abspath(os.path.dirname(thisScript))
1618
nuget = sys.argv[1]
1719

20+
# /input contains a dotnet project that's being extracted
1821
workDir = os.path.abspath(helpers.get_argv(3, "tempDir"))
1922
projectNameIn = "input"
2023
projectDirIn = os.path.join(workDir, projectNameIn)
2124

25+
# /output contains the output of the stub generation
26+
outputDirName = "output"
27+
outputDir = os.path.join(workDir, outputDirName)
28+
29+
# /output/raw contains the bqrs result from the query, the json equivalent
30+
rawOutputDirName = "raw"
31+
rawOutputDir = os.path.join(outputDir, rawOutputDirName)
32+
os.makedirs(rawOutputDir)
33+
34+
# /output/output contains a dotnet project with the generated stubs
2235
projectNameOut = "output"
23-
projectDirOut = os.path.join(workDir, projectNameOut)
36+
projectDirOut = os.path.join(outputDir, projectNameOut)
37+
38+
# /db contains the extracted QL DB
2439
dbName = 'db'
2540
dbDir = os.path.join(workDir, dbName)
26-
outputName = helpers.get_argv(4, "stub")
41+
outputName = "stub"
2742
outputFile = os.path.join(projectDirOut, outputName + '.cs')
28-
bqrsFile = os.path.join(projectDirOut, outputName + '.bqrs')
43+
bqrsFile = os.path.join(rawOutputDir, outputName + '.bqrs')
44+
jsonFile = os.path.join(rawOutputDir, outputName + '.json')
2945
version = helpers.get_argv(2, "latest")
3046

3147
print("\n* Creating new input project")
3248
helpers.run_cmd(['dotnet', 'new', 'classlib', "--language", "C#", '--name',
3349
projectNameIn, '--output', projectDirIn])
50+
helpers.remove_files(projectDirIn, '.cs')
3451

3552
print("\n* Adding reference to package: " + nuget)
3653
cmd = ['dotnet', 'add', projectDirIn, 'package', nuget]
@@ -47,28 +64,128 @@
4764
print("Expected database directory " + dbDir + " not found.")
4865
exit(1)
4966

50-
print("\n* Creating new output project")
51-
helpers.run_cmd(['dotnet', 'new', 'classlib', "--language", "C#", '--name',
52-
projectNameOut, '--output', projectDirOut])
53-
5467
print("\n* Running stubbing CodeQL query")
5568
helpers.run_cmd(['codeql', 'query', 'run', os.path.join(
5669
thisDir, 'AllStubsFromReference.ql'), '--database', dbDir, '--output', bqrsFile])
5770

5871
helpers.run_cmd(['codeql', 'bqrs', 'decode', bqrsFile, '--output',
59-
outputFile, '--format=text', '--no-titles'])
60-
61-
helpers.trim_output_file(outputFile)
62-
63-
print("\n --> Generated output file: " + outputFile)
64-
65-
print("\n* Formatting file")
66-
helpers.run_cmd(['dotnet', 'format', projectDirOut,
67-
'--include', outputName + '.cs'])
68-
69-
print("\n* Building output project")
70-
helpers.run_cmd(['dotnet', 'build', '/t:rebuild', '/p:AllowUnsafeBlocks=true', projectDirOut],
71-
'ERR: Build failed. Script failed to generate a stub that builds')
72+
jsonFile, '--format=json'])
73+
74+
print("\n* Creating new raw output project")
75+
rawSrcOutputDirName = 'src'
76+
rawSrcOutputDir = os.path.join(rawOutputDir, rawSrcOutputDirName)
77+
helpers.run_cmd(['dotnet', 'new', 'classlib', "--language", "C#",
78+
'--name', rawSrcOutputDirName, '--output', rawSrcOutputDir])
79+
helpers.remove_files(rawSrcOutputDir, '.cs')
80+
81+
# load json from file
82+
pathInfos = {}
83+
with open(jsonFile) as json_data:
84+
data = json.load(json_data)
85+
for row in data['#select']['tuples']:
86+
pathInfos[row[3]] = os.path.join(rawSrcOutputDir, row[1] + '.cs')
87+
with open(pathInfos[row[3]], 'a') as f:
88+
f.write(row[4])
89+
90+
print("\n --> Generated stub files: " + rawSrcOutputDir)
91+
92+
print("\n* Formatting files")
93+
helpers.run_cmd(['dotnet', 'format', rawSrcOutputDir])
94+
95+
print("\n --> Generated (formatted) stub files: " + rawSrcOutputDir)
96+
97+
98+
print("\n* Processing project.assets.json to generate folder structure")
99+
stubsDirName = 'stubs'
100+
stubsDir = os.path.join(outputDir, stubsDirName)
101+
os.makedirs(stubsDir)
102+
103+
stubFileName = '_stub.cs'
104+
105+
frameworksDirName = 'frameworks'
106+
frameworksDir = os.path.join(stubsDir, frameworksDirName)
107+
108+
frameworks = set()
109+
copiedFiles = set()
110+
111+
assetsJsonFile = os.path.join(projectDirIn, 'obj', 'project.assets.json')
112+
with open(assetsJsonFile) as json_data:
113+
data = json.load(json_data)
114+
if len(data['targets']) > 1:
115+
print("ERROR: More than one target found in " + assetsJsonFile)
116+
exit(1)
117+
target = list(data['targets'].keys())[0]
118+
print("Found target: " + target)
119+
for package in data['targets'][target].keys():
120+
parts = package.split('/')
121+
name = parts[0]
122+
version = parts[1]
123+
packageDir = os.path.join(stubsDir, name, version)
124+
if not os.path.exists(packageDir):
125+
os.makedirs(packageDir)
126+
print(' * Processing package: ' + name + '/' + version)
127+
with open(os.path.join(packageDir, stubFileName), 'a') as f:
128+
f.write('// Stub for ' + name + ' version ' + version + '\n\n')
129+
130+
dlls = set()
131+
if 'compile' in data['targets'][target][package]:
132+
for dll in data['targets'][target][package]['compile']:
133+
dlls.add((name + '/' + version + '/' + dll).lower())
134+
if 'runtime' in data['targets'][target][package]:
135+
for dll in data['targets'][target][package]['runtime']:
136+
dlls.add((name + '/' + version + '/' + dll).lower())
137+
138+
for pathInfo in pathInfos:
139+
for dll in dlls:
140+
if pathInfo.lower().endswith(dll):
141+
copiedFiles.add(pathInfo)
142+
shutil.copy2(pathInfos[pathInfo], packageDir)
143+
f.write('// semmle-extractor-options: ' +
144+
os.path.basename(pathInfos[pathInfo]) + '\n')
145+
146+
if 'dependencies' in data['targets'][target][package]:
147+
for dependency in data['targets'][target][package]['dependencies'].keys():
148+
depVersion = data['targets'][target][package]['dependencies'][dependency]
149+
f.write('// semmle-extractor-options: ../../' +
150+
dependency + '/' + depVersion + '/' + stubFileName + '\n')
151+
152+
if 'frameworkReferences' in data['targets'][target][package]:
153+
if not os.path.exists(frameworksDir):
154+
os.makedirs(frameworksDir)
155+
for framework in data['targets'][target][package]['frameworkReferences']:
156+
frameworks.add(framework)
157+
frameworkDir = os.path.join(frameworksDir, framework)
158+
if not os.path.exists(frameworkDir):
159+
os.makedirs(frameworkDir)
160+
f.write('// semmle-extractor-options: ../../' + frameworksDirName + '/' +
161+
framework + '/' + stubFileName + '\n')
162+
163+
for framework in frameworks:
164+
with open(os.path.join(frameworksDir, framework, stubFileName), 'a') as f:
165+
f.write('// Stub for ' + framework + '\n\n')
166+
for pathInfo in pathInfos:
167+
if 'packs/' + framework.lower() in pathInfo.lower():
168+
copiedFiles.add(pathInfo)
169+
shutil.copy2(pathInfos[pathInfo], os.path.join(
170+
frameworksDir, framework))
171+
f.write('// semmle-extractor-options: ' +
172+
os.path.basename(pathInfos[pathInfo]) + '\n')
173+
174+
# todo: write not copied files to others folder
175+
for pathInfo in pathInfos:
176+
if pathInfo not in copiedFiles:
177+
print('Not copied to nuget or framework folder: ' + pathInfo)
178+
othersDir = os.path.join(stubsDir, 'others')
179+
if not os.path.exists(othersDir):
180+
os.makedirs(othersDir)
181+
shutil.copy2(pathInfos[pathInfo], othersDir)
182+
183+
print("\n --> Generated structured stub files: " + stubsDir)
184+
185+
print("\n* Building raw output project")
186+
helpers.run_cmd(['dotnet', 'build', '/t:rebuild', '/p:AllowUnsafeBlocks=true', '/p:WarningLevel=0',
187+
rawSrcOutputDir], 'ERR: Build failed. Script failed to generate a stub that builds')
188+
189+
print("\n --> Generated structured stub files: " + stubsDir)
72190

73-
print("\n --> Generated output file: " + outputFile)
74191
exit(0)

0 commit comments

Comments
 (0)