Add lexer, with tests

kennknowles · kennknowles · commit 9c88079576d8 · 2013-02-09T00:29:55.000-05:00
diff --git a/jsonpath_rw/lexer.py b/jsonpath_rw/lexer.py
@@ -0,0 +1,110 @@
+import sys
+import logging
+
+import ply.lex
+
+logger = logging.getLogger(__name__)
+
+class JsonPathLexer(object):
+    '''
+    A Lexical analyzer for JsonPath. 
+    '''
+    
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    def tokenize(self, string):
+        '''
+        Maps a string to an iterator over tokens. In other words: [char] -> [token]
+        '''
+        
+        new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
+        new_lexer.latest_newline = 0
+        new_lexer.input(string)
+
+        while True:
+            t = new_lexer.token()
+            if t is None: break
+            t.col = t.lexpos - new_lexer.latest_newline
+            yield t
+
+    # ============== PLY Lexer specification ==================
+    #
+    # This probably should be private but:
+    #   - the parser requires access to `tokens` (perhaps they should be defined in a third, shared dependency)
+    #   - things like `literals` might be a legitimate part of the public interface.
+    #
+    # Anyhow, it is pythonic to give some rope to hang oneself with :-)
+
+    literals = ['*', '.', '[', ']', '(', ')', '$', ',', ':', '|', '&', '@']
+    
+    reserved_words = { 'where': 'WHERE' }
+
+    tokens = ['DOUBLEDOT', 'NUMBER', 'ID'] + reserved_words.values()
+
+    states = [ ('singlequote', 'exclusive'),
+               ('doublequote', 'exclusive') ]
+
+    # Normal lexing, rather easy
+    t_DOUBLEDOT = r'\.\.'
+    t_ignore = ' \t'
+
+    def t_ID(self, t):
+        r'[a-zA-Z_][a-zA-Z0-9_]*'
+        t.type = self.reserved_words.get(t.value, 'ID')
+        return t
+
+    def t_NUMBER(self, t):
+        r'\d+'
+        t.value = int(t.value)
+        return t
+
+    # Single-quoted strings
+    t_singlequote_ignore = ''
+    def t_SINGLEQUOTE(self, t):
+        r'\''
+        t.lexer.string_start = t.lexer.lexpos
+        t.lexer.push_state('singlequote')
+
+    def t_singlequote_SINGLEQUOTE(self, t):
+        r"([^']|\\')*'"
+        t.value = t.value[:-1]
+        t.type = 'ID'
+        t.lexer.pop_state()
+        return t
+
+    def t_singlequote_error(self, t):
+        raise Exception('Error on line %s, col %s while lexing singlequoted field: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.latest_newline, t.value[0]))
+
+
+    # Double-quoted strings
+    t_doublequote_ignore = ''
+    def t_DOUBLEQUOTE(self, t):
+        r'"'
+        t.lexer.string_start = t.lexer.lexpos
+        t.lexer.push_state('doublequote')
+
+    def t_doublequote_DOUBLEQUOTE(self, t):
+        r'([^"]|\\")*"'
+        t.value = t.value[:-1]
+        t.type = 'ID'
+        t.lexer.pop_state()
+        return t
+
+    def t_doublequote_error(self, t):
+        raise Exception('Error on line %s, col %s while lexing doublequoted field: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.latest_newline, t.value[0]))
+
+    # Counting lines, handling errors
+    def t_newline(self, t):
+        r'\n'
+        t.lexer.lineno += 1
+        t.lexer.latest_newline = t.lexpos
+
+    def t_error(self, t):
+        raise Exception('Error on line %s, col %s: Unexpected character: %s ' % (t.lexer.lineno, t.lexpos - t.latest_newline, t.value[0]))
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    lexer = JsonPathLexer(debug=True)
+    for token in lexer.tokenize(sys.stdin.read()):
+        print '%-20s%s' % (token.value, token.type)
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
@@ -0,0 +1,46 @@
+import logging
+import unittest
+
+from ply.lex import LexToken
+
+from jsonpath_rw.lexer import JsonPathLexer
+
+class TestLexer(unittest.TestCase):
+
+    def token(self, value, ty=None):
+        t = LexToken()
+        t.type = ty if ty != None else value
+        t.value = value
+        t.lineno = -1
+        t.lexpos = -1
+        return t
+    
+    def assert_lex_equiv(self, s, stream2):
+        # NOTE: lexer fails to reset after call?
+        l = JsonPathLexer(debug=True)
+        stream1 = list(l.tokenize(s)) # Save the stream for debug output when a test fails
+        stream2 = list(stream2)
+        assert len(stream1) == len(stream2)
+        for token1, token2 in zip(stream1, stream2):
+            print token1, token2
+            assert token1.type  == token2.type
+            assert token1.value == token2.value
+
+    @classmethod
+    def setup_class(cls):
+        logging.basicConfig()
+
+    def test_simple_inputs(self):
+        self.assert_lex_equiv('$', [self.token('$', '$')])
+        self.assert_lex_equiv('"hello"', [self.token('hello', 'ID')])
+        self.assert_lex_equiv("'goodbye'", [self.token('goodbye', 'ID')])
+        self.assert_lex_equiv('fuzz', [self.token('fuzz', 'ID')])
+        self.assert_lex_equiv('1', [self.token(1, 'NUMBER')])
+        self.assert_lex_equiv('45', [self.token(45, 'NUMBER')])
+        self.assert_lex_equiv('"fuzz.bang"', [self.token('fuzz.bang', 'ID')])
+        self.assert_lex_equiv('fuzz.bang', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('bang', 'ID')])
+        self.assert_lex_equiv('fuzz.*', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('*', '*')])
+        self.assert_lex_equiv('fuzz..bang', [self.token('fuzz', 'ID'), self.token('..', 'DOUBLEDOT'), self.token('bang', 'ID')])
+        self.assert_lex_equiv('&', [self.token('&', '&')])
+        self.assert_lex_equiv('|', [self.token('|', '|')])
+        self.assert_lex_equiv('where', [self.token('where', 'WHERE')])
