From 740484013f28011724025d5a2fe8d549c3311eb0 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Thu, 14 Nov 2019 01:30:24 -0500 Subject: [PATCH 1/2] Use 'raise Foo from bar' where supported --- CppHeaderParser/CppHeaderParser.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/CppHeaderParser/CppHeaderParser.py b/CppHeaderParser/CppHeaderParser.py index 3abc4ea..2e211ce 100644 --- a/CppHeaderParser/CppHeaderParser.py +++ b/CppHeaderParser/CppHeaderParser.py @@ -72,6 +72,14 @@ # Controls trace_print debug_trace = 0 +if sys.version_info >= (3, 3): + # `raise e from src_e` syntax only supported on python 3.3+ + exec("def raise_exc(e, src_e): raise e from src_e", globals()) +else: + + def raise_exc(e, src_e): + raise e + def error_print(arg): if print_errors: @@ -1797,7 +1805,7 @@ def finalize_vars(self): else: trace_print("-" * 80) trace_print(var) - raise NotImplemented + raise NotImplementedError ## need full name space for classes in raw type ## if var["raw_type"].startswith("::"): @@ -2880,12 +2888,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): self.stack = [] self.nameStack = [] - except: + except Exception as e: if debug: raise - raise CppParseError( - 'Not able to parse %s on line %d evaluating "%s"\nError around: %s' - % (self.headerFileName, tok.lineno, tok.value, " ".join(self.nameStack)) + raise_exc( + CppParseError( + 'Not able to parse %s on line %d evaluating "%s"\nError around: %s' + % ( + self.headerFileName, + tok.lineno, + tok.value, + " ".join(self.nameStack), + ) + ), + e, ) self.finalize() From 0c151442037ce6474bfbe360e2fa51429f5c659f Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Thu, 14 Nov 2019 01:47:57 -0500 Subject: [PATCH 2/2] Use #line preprocessor directives to set the line number and filename --- CppHeaderParser/CppHeaderParser.py | 161 +++++++++++-------- CppHeaderParser/lexer.py | 22 ++- CppHeaderParser/test/test_CppHeaderParser.py | 48 ++++++ 3 files changed, 163 insertions(+), 68 deletions(-) diff --git a/CppHeaderParser/CppHeaderParser.py b/CppHeaderParser/CppHeaderParser.py index 2e211ce..ee2d40a 100644 --- a/CppHeaderParser/CppHeaderParser.py +++ b/CppHeaderParser/CppHeaderParser.py @@ -231,16 +231,11 @@ def is_property_namestack(nameStack): return r -def detect_lineno(s): - """Detect the line number for a given token string""" - try: - rtn = s.lineno() - if rtn != -1: - return rtn - except: - pass - global curLine - return curLine +def set_location_info(thing, location): + filename, line_number = location + if filename: + thing["filename"] = filename + thing["line_number"] = line_number def filter_out_attribute_keyword(stack): @@ -328,22 +323,11 @@ def _split_by_comma(namestack): class TagStr(str): """Wrapper for a string that allows us to store the line number associated with it""" - lineno_reg = {} - - def __new__(cls, *args, **kw): - new_obj = str.__new__(cls, *args) - if "lineno" in kw: - TagStr.lineno_reg[id(new_obj)] = kw["lineno"] - return new_obj - - def __del__(self): - try: - del TagStr.lineno_reg[id(self)] - except: - pass - - def lineno(self): - return TagStr.lineno_reg.get(id(self), -1) + def __new__(cls, *args, **kwargs): + location = kwargs.pop("location") + s = str.__new__(cls, *args, **kwargs) + s.location = location + return s class CppParseError(Exception): @@ -428,7 +412,7 @@ def get_pure_virtual_methods(self, type="public"): r[meth["name"]] = meth return r - def __init__(self, nameStack, curTemplate, doxygen): + def __init__(self, nameStack, curTemplate, doxygen, location): #: hm self["nested_classes"] = [] self["parent"] = None @@ -480,7 +464,7 @@ def __init__(self, nameStack, curTemplate, doxygen): pass self["name"] = nameStack[1] - self["line_number"] = detect_lineno(nameStack[0]) + set_location_info(self, location) # Handle template classes if len(nameStack) > 3 and nameStack[2].startswith("<"): @@ -714,8 +698,8 @@ class CppUnion(CppClass): * ``members`` - List of members of the union """ - def __init__(self, nameStack, doxygen): - CppClass.__init__(self, nameStack, None, doxygen) + def __init__(self, nameStack, doxygen, location): + CppClass.__init__(self, nameStack, None, doxygen, location) self["name"] = "union " + self["name"] self["members"] = self["properties"]["public"] @@ -845,7 +829,7 @@ def show(self): r.append("destructor") return "\n\t\t ".join(r) - def __init__(self, nameStack, curClass, methinfo, curTemplate, doxygen): + def __init__(self, nameStack, curClass, methinfo, curTemplate, doxygen, location): debug_print("Method: %s" % nameStack) debug_print("Template: %s" % curTemplate) @@ -911,7 +895,7 @@ def __init__(self, nameStack, curClass, methinfo, curTemplate, doxygen): break self.update(methinfo) - self["line_number"] = detect_lineno(nameStack[0]) + set_location_info(self, location) # Filter out initializer lists used in constructors try: @@ -967,8 +951,12 @@ def __init__(self, nameStack, curClass, methinfo, curTemplate, doxygen): i += 1 if param_separator: + tpstack = paramsStack[0:param_separator] param = CppVariable( - paramsStack[0:param_separator], None, doxyVarDesc=doxyVarDesc + tpstack, + None, + getattr(tpstack[0], "location", location), + doxyVarDesc=doxyVarDesc, ) if len(list(param.keys())): params.append(param) @@ -977,7 +965,12 @@ def __init__(self, nameStack, curClass, methinfo, curTemplate, doxygen): self["vararg"] = True paramsStack = paramsStack[1:] else: - param = CppVariable(paramsStack, None, doxyVarDesc=doxyVarDesc) + param = CppVariable( + paramsStack, + None, + getattr(paramsStack[0], "location", location), + doxyVarDesc=doxyVarDesc, + ) if len(list(param.keys())): params.append(param) break @@ -1045,7 +1038,7 @@ class CppVariable(_CppVariable): Vars = [] - def __init__(self, nameStack, doxygen, **kwargs): + def __init__(self, nameStack, doxygen, location, **kwargs): debug_print("trace %s" % nameStack) if len(nameStack) and nameStack[0] == "extern": self["extern"] = True @@ -1083,7 +1076,7 @@ def __init__(self, nameStack, doxygen, **kwargs): debug_print("Variable: %s" % nameStack) - self["line_number"] = detect_lineno(nameStack[0]) + set_location_info(self, location) self["function_pointer"] = 0 if len(nameStack) < 2: # +++ @@ -1225,7 +1218,7 @@ class CppEnum(_CppEnum): if a value for a given enum value was defined """ - def __init__(self, nameStack, doxygen): + def __init__(self, nameStack, doxygen, location): if doxygen: self["doxygen"] = doxygen if len(nameStack) == 3 and nameStack[0] == "enum": @@ -1237,7 +1230,7 @@ def __init__(self, nameStack, doxygen): debug_print("Bad enum") return valueList = [] - self["line_number"] = detect_lineno(nameStack[0]) + set_location_info(self, location) # Figure out what values it has valueStack = nameStack[nameStack.index("{") + 1 : nameStack.index("}")] while len(valueStack): @@ -1303,15 +1296,14 @@ class CppStruct(dict): Structs = [] - def __init__(self, nameStack): + def __init__(self, nameStack, location): if len(nameStack) >= 2: self["type"] = nameStack[1] else: self["type"] = None self["fields"] = [] + set_location_info(self, location) self.Structs.append(self) - global curLine - self["line_number"] = curLine C99_NONSTANDARD = { @@ -1998,7 +1990,7 @@ def _evaluate_struct_stack(self): """Create a Struct out of the name stack (but not its parts)""" # print( 'eval struct stack', self.nameStack ) # if self.braceDepth != len(self.nameSpaces): return - struct = CppStruct(self.nameStack) + struct = CppStruct(self.nameStack, self._get_location(self.nameStack)) struct["namespace"] = self.cur_namespace() self.structs[struct["type"]] = struct self.structs_order.append(struct) @@ -2176,6 +2168,7 @@ def _evaluate_method_stack(self): info, self.curTemplate, self.lex.get_doxygen(), + self._get_location(self.nameStack), ) klass = self.classes[info["class"]] klass["methods"]["public"].append(newMethod) @@ -2192,6 +2185,7 @@ def _evaluate_method_stack(self): info, self.curTemplate, self.lex.get_doxygen(), + self._get_location(self.nameStack), ) klass = self.classes[self.curClass] klass["methods"][self.curAccessSpecifier].append(newMethod) @@ -2203,7 +2197,12 @@ def _evaluate_method_stack(self): else: # non class functions debug_print("FREE FUNCTION") newMethod = CppMethod( - self.nameStack, None, info, self.curTemplate, self.lex.get_doxygen() + self.nameStack, + None, + info, + self.curTemplate, + self.lex.get_doxygen(), + self._get_location(self.nameStack), ) self.functions.append(newMethod) global parseHistory @@ -2326,7 +2325,11 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None): ) return - newVar = CppVariable(self.nameStack, self.lex.get_doxygen()) + newVar = CppVariable( + self.nameStack, + self.lex.get_doxygen(), + self._get_location(self.nameStack), + ) newVar["namespace"] = self.current_namespace() if self.curStruct: self.curStruct["fields"].append(newVar) @@ -2342,7 +2345,11 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None): newVar.update(addToVar) else: debug_print("Found Global variable") - newVar = CppVariable(self.nameStack, self.lex.get_doxygen()) + newVar = CppVariable( + self.nameStack, + self.lex.get_doxygen(), + self._get_location(self.nameStack), + ) if addToVar: newVar.update(addToVar) self.variables.append(newVar) @@ -2378,7 +2385,11 @@ def _evaluate_class_stack(self): "curAccessSpecifier changed/defaulted to %s" % self.curAccessSpecifier ) if self.nameStack[0] == "union": - newClass = CppUnion(self.nameStack, self.lex.get_doxygen()) + newClass = CppUnion( + self.nameStack, + self.lex.get_doxygen(), + self._get_location(self.nameStack), + ) if newClass["name"] == "union ": self.anon_union_counter = [self.braceDepth, 2] else: @@ -2386,7 +2397,10 @@ def _evaluate_class_stack(self): trace_print("NEW UNION", newClass["name"]) else: newClass = CppClass( - self.nameStack, self.curTemplate, self.lex.get_doxygen() + self.nameStack, + self.curTemplate, + self.lex.get_doxygen(), + self._get_location(self.nameStack), ) trace_print("NEW CLASS", newClass["name"]) newClass["declaration_method"] = self.nameStack[0] @@ -2682,13 +2696,11 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): self.braceDepth = 0 - lex = Lexer() + lex = Lexer(self.headerFileName) lex.input(headerFileStr) self.lex = lex - global curLine - curLine = 0 - + tok = None try: while True: tok = lex.token() @@ -2699,13 +2711,13 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): and self.anon_union_counter[1] ): self.anon_union_counter[1] -= 1 - tok.value = TagStr(tok.value, lineno=tok.lineno) + tok.value = TagStr(tok.value, location=lex.current_location()) # debug_print("TOK: %s"%tok) if tok.type == "NAME" and tok.value in self.IGNORE_NAMES: continue if tok.type != "TEMPLATE_NAME": self.stack.append(tok.value) - curLine = tok.lineno + if tok.type in ("PRECOMP_MACRO", "PRECOMP_MACRO_CONT"): debug_print("PRECOMP: %s" % tok) self._precomp_macro_buf.append(tok.value) @@ -2891,17 +2903,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): except Exception as e: if debug: raise - raise_exc( - CppParseError( + if tok: + filename, lineno = tok.value.location + msg = ( 'Not able to parse %s on line %d evaluating "%s"\nError around: %s' - % ( - self.headerFileName, - tok.lineno, - tok.value, - " ".join(self.nameStack), - ) - ), - e, + % (filename, lineno, tok.value, " ".join(self.nameStack)) + ) + else: + msg = "Error parsing %s\nError around: %s" % ( + self.headerFileName, + " ".join(self.nameStack), + ) + + raise_exc( + CppParseError(msg), e, ) self.finalize() @@ -2934,6 +2949,14 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): ]: del self.__dict__[key] + def _get_location(self, stack): + if stack: + location = getattr(stack[0], "location", None) + if location is not None: + return location + + return self.lex.current_location() + def _evaluate_stack(self, token=None): """Evaluates the current name stack""" @@ -3004,11 +3027,15 @@ def _evaluate_stack(self, token=None): # using foo = ns::bar alias = self.nameStack[1] ns, stack = _split_namespace(self.nameStack[3:]) - atype = CppVariable(stack, self.lex.get_doxygen()) + atype = CppVariable( + stack, self.lex.get_doxygen(), self._get_location(stack) + ) else: # using foo::bar ns, stack = _split_namespace(self.nameStack[1:]) - atype = CppVariable(stack, self.lex.get_doxygen()) + atype = CppVariable( + stack, self.lex.get_doxygen(), self._get_location(stack) + ) alias = atype["type"] atype["namespace"] = ns @@ -3110,7 +3137,9 @@ def _evaluate_stack(self, token=None): def _evaluate_enum_stack(self): """Create an Enum out of the name stack""" debug_print("evaluating enum") - newEnum = CppEnum(self.nameStack, self.lex.get_doxygen()) + newEnum = CppEnum( + self.nameStack, self.lex.get_doxygen(), self._get_location(self.nameStack) + ) if len(list(newEnum.keys())): if len(self.curClass): newEnum["namespace"] = self.cur_namespace(False) diff --git a/CppHeaderParser/lexer.py b/CppHeaderParser/lexer.py index e36ecab..5c7fc1e 100644 --- a/CppHeaderParser/lexer.py +++ b/CppHeaderParser/lexer.py @@ -1,6 +1,8 @@ import ply.lex as lex import re +_line_re = re.compile(r'^#line (\d+) "(.*)"') + class Lexer(object): @@ -62,7 +64,16 @@ class Lexer(object): t_PERCENT = r"%" t_CARET = r"\^" t_EXCLAMATION = r"!" - t_PRECOMP_MACRO = r"\#.*" + + def t_PRECOMP_MACRO(self, t): + r"\#.*" + m = _line_re.match(t.value) + if m: + self.filename = m.group(2) + self.line_offset = 1 + self.lex.lineno - int(m.group(1)) + else: + return t + t_PRECOMP_MACRO_CONT = r".*\\\n" def t_COMMENT_SINGLELINE(self, t): @@ -109,14 +120,21 @@ def t_NEWLINE(self, t): def t_error(self, v): print("Lex error: ", v) - def __init__(self): + def __init__(self, filename): self.lex = lex.lex(module=self) self.input = self.lex.input self.token = self.lex.token + # For tracking current file/line position + self.filename = filename + self.line_offset = 0 + # Doxygen comments self.doxygenCommentCache = "" + def current_location(self): + return self.filename, self.lex.lineno - self.line_offset + def get_doxygen(self): doxygen = self.doxygenCommentCache self.doxygenCommentCache = "" diff --git a/CppHeaderParser/test/test_CppHeaderParser.py b/CppHeaderParser/test/test_CppHeaderParser.py index a9eaea4..9ab0bbf 100644 --- a/CppHeaderParser/test/test_CppHeaderParser.py +++ b/CppHeaderParser/test/test_CppHeaderParser.py @@ -2907,5 +2907,53 @@ def test_fn(self): ) +class MultiFile_TestCase(unittest.TestCase): + def setUp(self): + self.cppHeader = CppHeaderParser.CppHeader( + """ +#line 3 "child.h" +#include +#line 3 "base.h" +void functionInBase(void); + +class Base +{ +public: + virtual void baseFunction(); +}; +#line 7 "child.h" +void functionInChild(void); + +class Child : public Base +{ +public: + void childOnlyFunction(); + void baseFunction() override; +}; + +""", + "string", + ) + + def assertLocation(self, thing, fname, lineno): + self.assertEqual(fname, thing["filename"]) + self.assertEqual(lineno, thing["line_number"]) + + def test_fn(self): + baseFn = self.cppHeader.functions[0] + self.assertEqual("functionInBase", baseFn["name"]) + self.assertLocation(baseFn, "base.h", 3) + + base = self.cppHeader.classes["Base"] + self.assertLocation(base, "base.h", 5) + + childFn = self.cppHeader.functions[1] + self.assertEqual("functionInChild", childFn["name"]) + self.assertLocation(childFn, "child.h", 7) + + child = self.cppHeader.classes["Child"] + self.assertLocation(child, "child.h", 9) + + if __name__ == "__main__": unittest.main()