Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dc7f8a6

Browse files
igfooigfoo
committed
Kotlin: Refactor TrapWriter/Logger
It's now Tpossible for TrapWriter to log warnings. This required a little juggling to break the dependency loop between the two classes.
1 parent 1d824a4 commit dc7f8a6

6 files changed

Lines changed: 70 additions & 37 deletions

File tree

‎java/kotlin-extractor/generate_dbscheme.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = N
143143
if colname == kind:
144144
kt.write(str(num))
145145
elif db_type == 'string':
146-
kt.write('\\"${escapeTrapString(truncateString(' + colname + '))}\\"')
146+
kt.write('\\"${this.escapeTrapString(this.truncateString(' + colname + '))}\\"')
147147
elif db_type == 'date':
148148
kt.write('D\\"${' + colname + '}\\"')
149149
else:

‎java/kotlin-extractor/src/main/kotlin/ExternalClassExtractor.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class ExternalClassExtractor(val logger: FileLogger, val invocationTrapFile: Str
4040
GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use { trapFileBW ->
4141
// We want our comments to be the first thing in the file,
4242
// so start off with a mere TrapWriter
43-
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
43+
val tw = TrapWriter(logger, TrapLabelManager(), trapFileBW)
4444
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
4545
tw.writeComment("Part of invocation $invocationTrapFile")
4646
// Now elevate to a SourceFileTrapWriter, and populate the

‎java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import java.io.FileOutputStream
88
import java.nio.file.Files
99
import java.nio.file.Paths
1010
import com.semmle.util.files.FileUtil
11-
import com.semmle.util.unicode.UTF8Util
1211
import kotlin.system.exitProcess
1312

1413
class KotlinExtractorExtension(
@@ -40,15 +39,16 @@ class KotlinExtractorExtension(
4039
val trapDir = File(System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } ?: "kotlin-extractor/trap")
4140
FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW ->
4241
val lm = TrapLabelManager()
43-
val tw = TrapWriter(lm, invocationTrapFileBW)
42+
val logCounter = LogCounter()
43+
val loggerBase = LoggerBase(logCounter)
44+
val tw = TrapWriter(loggerBase, lm, invocationTrapFileBW)
4445
// The interceptor has already defined #compilation = *
4546
val compilation: Label<DbCompilation> = StringLabel("compilation")
4647
tw.writeCompilation_started(compilation)
4748
if (compilationStartTime != null) {
4849
tw.writeCompilation_compiler_times(compilation, -1.0, (System.currentTimeMillis()-compilationStartTime)/1000.0)
4950
}
5051
tw.flush()
51-
val logCounter = LogCounter()
5252
val logger = Logger(logCounter, tw)
5353
logger.info("Extraction started")
5454
logger.flush()
@@ -116,12 +116,6 @@ class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblem
116116
}
117117
}
118118

119-
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
120-
121-
const val MAX_STRLEN = 1.shl(20) // 1 megabyte
122-
123-
fun truncateString(str: String) = str.substring(0, UTF8Util.encodablePrefixLength(str, MAX_STRLEN))
124-
125119
private fun equivalentTrap(f1: File, f2: File): Boolean {
126120
f1.bufferedReader().use { bw1 ->
127121
f2.bufferedReader().use { bw2 ->
@@ -178,7 +172,7 @@ fun doFile(fileExtractionProblems: FileExtractionProblems,
178172
trapTmpFile.bufferedWriter().use { trapFileBW ->
179173
// We want our comments to be the first thing in the file,
180174
// so start off with a mere TrapWriter
181-
val tw = TrapWriter(TrapLabelManager(), trapFileBW)
175+
val tw = TrapWriter(logger, TrapLabelManager(), trapFileBW)
182176
tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
183177
tw.writeComment("Part of invocation $invocationTrapFile")
184178
// Now elevate to a SourceFileTrapWriter, and populate the

‎java/kotlin-extractor/src/main/kotlin/TrapWriter.kt‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
1111
import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
1212

1313
import com.semmle.extractor.java.PopulateFile
14+
import com.semmle.util.unicode.UTF8Util
1415

1516
/**
1617
* Each `.trap` file has a `TrapLabelManager` while we are writing it.
@@ -39,7 +40,7 @@ class TrapLabelManager {
3940
* instances will have different additional state, but they must all
4041
* share the same `TrapLabelManager` and `BufferedWriter`.
4142
*/
42-
open class TrapWriter (protected val lm: TrapLabelManager, private val bw: BufferedWriter) {
43+
open class TrapWriter (protected val loggerBase: LoggerBase, protected val lm: TrapLabelManager, private val bw: BufferedWriter) {
4344
/**
4445
* Returns the label that is defined to be the given key, if such
4546
* a label exists, and `null` otherwise. Most users will want to use
@@ -176,31 +177,49 @@ open class TrapWriter (protected val lm: TrapLabelManager, private val bw: Buffe
176177
bw.flush()
177178
}
178179

180+
fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
181+
182+
val MAX_STRLEN = 1.shl(20) // 1 megabyte
183+
184+
fun truncateString(str: String): String {
185+
val len = str.length
186+
val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
187+
if (newLen < len) {
188+
loggerBase.warn(this,
189+
"Truncated string of length $len",
190+
"Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'")
191+
return str.take(newLen)
192+
} else {
193+
return str
194+
}
195+
}
196+
179197
/**
180198
* Gets a FileTrapWriter like this one (using the same label manager,
181199
* writer etc), but using the given `filePath` for locations.
182200
*/
183201
fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
184-
FileTrapWriter(lm, bw, filePath, populateFileTables)
202+
FileTrapWriter(loggerBase, lm, bw, filePath, populateFileTables)
185203

186204
/**
187205
* Gets a FileTrapWriter like this one (using the same label manager,
188206
* writer etc), but using the given `IrFile` for locations.
189207
*/
190208
fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) =
191-
SourceFileTrapWriter(lm, bw, file, populateFileTables)
209+
SourceFileTrapWriter(loggerBase, lm, bw, file, populateFileTables)
192210
}
193211

194212
/**
195213
* A `FileTrapWriter` is used when we know which file we are extracting
196214
* entities from, so we can at least give the right file as a location.
197215
*/
198216
open class FileTrapWriter (
217+
loggerBase: LoggerBase,
199218
lm: TrapLabelManager,
200219
bw: BufferedWriter,
201220
val filePath: String,
202221
populateFileTables: Boolean
203-
): TrapWriter (lm, bw) {
222+
): TrapWriter (loggerBase, lm, bw) {
204223
val fileId = mkFileId(filePath, populateFileTables)
205224

206225
/**
@@ -244,11 +263,12 @@ open class FileTrapWriter (
244263
* and column numbers.
245264
*/
246265
class SourceFileTrapWriter (
266+
loggerBase: LoggerBase,
247267
lm: TrapLabelManager,
248268
bw: BufferedWriter,
249269
irFile: IrFile,
250270
populateFileTables: Boolean) :
251-
FileTrapWriter(lm, bw, irFile.path, populateFileTables) {
271+
FileTrapWriter(loggerBase, lm, bw, irFile.path, populateFileTables) {
252272

253273
private val fileEntry = irFile.fileEntry
254274

‎java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private v
5656
}
5757

5858
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
59-
tw.writeKtComments(commentLabel, type.value, escapeTrapString(comment.text))
59+
tw.writeKtComments(commentLabel, type.value, tw.escapeTrapString(comment.text))
6060
val locId = tw.getLocation(comment.startOffset, comment.endOffset)
6161
tw.writeHasLocation(commentLabel, locId)
6262

@@ -71,14 +71,14 @@ class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private v
7171

7272
for (sec in comment.getAllSections()) {
7373
val commentSectionLabel = tw.getFreshIdLabel<DbKtcommentsection>()
74-
tw.writeKtCommentSections(commentSectionLabel, commentLabel, escapeTrapString(sec.getContent()))
74+
tw.writeKtCommentSections(commentSectionLabel, commentLabel, tw.escapeTrapString(sec.getContent()))
7575
val name = sec.name
7676
if (name != null) {
77-
tw.writeKtCommentSectionNames(commentSectionLabel, escapeTrapString(name))
77+
tw.writeKtCommentSectionNames(commentSectionLabel, tw.escapeTrapString(name))
7878
}
7979
val subjectName = sec.getSubjectName()
8080
if (subjectName != null) {
81-
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, escapeTrapString(subjectName))
81+
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, tw.escapeTrapString(subjectName))
8282
}
8383
}
8484

‎java/kotlin-extractor/src/main/kotlin/utils/Logger.kt‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ enum class Severity(val sev: Int) {
2828
ErrorGlobal(8)
2929
}
3030

31-
open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
32-
private fun timestamp(): String {
31+
open class LoggerBase(val logCounter: LogCounter) {
32+
protected fun timestamp(): String {
3333
return "[${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())} K]"
3434
}
3535

@@ -47,12 +47,7 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
4747
return null
4848
}
4949

50-
fun flush() {
51-
tw.flush()
52-
System.out.flush()
53-
}
54-
55-
fun diagnostic(severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { tw.unknownLocation }) {
50+
fun diagnostic(tw: TrapWriter, severity: Severity, msg: String, extraInfo: String?, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { tw.unknownLocation }) {
5651
val diagnosticLoc = getDiagnosticLocation()
5752
val diagnosticLocStr = if(diagnosticLoc == null) "<unknown location>" else diagnosticLoc
5853
val extraInfoStr = if (extraInfo == null) "" else (extraInfo + "\n")
@@ -78,6 +73,34 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
7873
print("$ts Diagnostic($diagnosticLocStr): $locStr$fullMsg")
7974
}
8075

76+
fun warn(tw: TrapWriter, msg: String, extraInfo: String?) {
77+
diagnostic(tw, Severity.Warn, msg, extraInfo)
78+
}
79+
fun error(tw: TrapWriter, msg: String, extraInfo: String?) {
80+
diagnostic(tw, Severity.Error, msg, extraInfo)
81+
}
82+
}
83+
84+
open class Logger(logCounter: LogCounter, open val tw: TrapWriter): LoggerBase(logCounter) {
85+
private fun getDiagnosticLocation(): String? {
86+
val st = Exception().stackTrace
87+
for(x in st) {
88+
when(x.className) {
89+
"com.github.codeql.Logger",
90+
"com.github.codeql.FileLogger" -> {}
91+
else -> {
92+
return x.toString()
93+
}
94+
}
95+
}
96+
return null
97+
}
98+
99+
fun flush() {
100+
tw.flush()
101+
System.out.flush()
102+
}
103+
81104
fun info(msg: String) {
82105
val fullMsg = "${timestamp()} $msg"
83106
tw.writeComment(fullMsg)
@@ -98,13 +121,13 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
98121
warn(msg, exn.stackTraceToString())
99122
}
100123
fun warn(msg: String, extraInfo: String?) {
101-
diagnostic(Severity.Warn, msg, extraInfo)
124+
warn(tw, msg, extraInfo)
102125
}
103126
fun warn(msg: String) {
104127
warn(msg, null)
105128
}
106129
fun error(msg: String, extraInfo: String?) {
107-
diagnostic(Severity.Error, msg, extraInfo)
130+
error(tw, msg, extraInfo)
108131
}
109132
fun error(msg: String) {
110133
error(msg, null)
@@ -124,19 +147,15 @@ open class Logger(val logCounter: LogCounter, open val tw: TrapWriter) {
124147
}
125148

126149
class FileLogger(logCounter: LogCounter, override val tw: FileTrapWriter): Logger(logCounter, tw) {
127-
private fun timestamp(): String {
128-
return "[${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())} K]"
129-
}
130-
131150
fun warnElement(msg: String, element: IrElement) {
132151
val locationString = tw.getLocationString(element)
133152
val mkLocationId = { tw.getLocation(element) }
134-
diagnostic(Severity.Warn, msg, null, locationString, mkLocationId)
153+
diagnostic(tw, Severity.Warn, msg, null, locationString, mkLocationId)
135154
}
136155

137156
fun errorElement(msg: String, element: IrElement) {
138157
val locationString = tw.getLocationString(element)
139158
val mkLocationId = { tw.getLocation(element) }
140-
diagnostic(Severity.Error, msg, null, locationString, mkLocationId)
159+
diagnostic(tw, Severity.Error, msg, null, locationString, mkLocationId)
141160
}
142161
}

0 commit comments

Comments
 (0)