updated interpreter

interpreter/_token.py

@@ -1,12 +1,15 @@
 """Private definitions for Token class used by the Lexer"""
 
+import logging
 import re
 
 from enum import IntEnum
 from typing import Union
 
 NUMERIC_CONSTANT_PATTERN = re.compile("([0-9]+)|(true)|(false)")
-KEYWORD_PATTERN = re.compile("(return)|(continue)|(break)")
+KEYWORD_PATTERN = re.compile("(return)|(continue)|(break)|(new)")
+STRING_LITERAL_PATTERN = re.compile("('|\")(.*)(\"|')")
+MATH_OP_PATTERN = re.compile("\+|-|\*|/")
 
 class Token_type(IntEnum):
     UNKNOWN=-1
@@ -17,20 +20,30 @@ class Token_type(IntEnum):
     LEFT_BRACKET=4,
     RIGHT_BRACKET=5,
     COMMA=6,
-    NUMERIC_CONSTANT=7,
-    IF_STATEMENT=8,
-    WHILE_STATEMENT=9,
-    DO_WHILE_STATEMENT=10,
-    FOR_STATEMENT=11,
-    KEY_WORD=13,
-    STRING_LITERAL=14
+    EQUAL_SIGN=7,
+    SEMICOLON=8
+    MATH_OP=9
+    NUMERIC_CONSTANT=10,
+    IF_STATEMENT=11,
+    ELSE_STATEMENT=12,
+    WHILE_STATEMENT=13,
+    DO_WHILE_STATEMENT=14,
+    FOR_STATEMENT=15,
+    KEY_WORD=16,
+    STRING_LITERAL=17
+    TYPE_NAME=18
 
 class Token:
     def __init__(self, type: Token_type, content: Union[str, None]=None) -> None:
         self.type = type
         self.content = content
 
-def make_token(tag: str) -> Token:
+    def __str__(self) -> str:
+        if self.content:
+            return f"{self.type}: {self.content}"
+        return f"{self.typetype}"
+
+def make_token(tag: str, type_name_pattern:re.Pattern) -> Token:
     if tag == '(':
         return Token(Token_type.LEFT_PAREN)
     elif tag == ')':
@@ -45,14 +58,30 @@ def make_token(tag: str) -> Token:
         return Token(Token_type.RIGHT_BRACKET)
     elif tag == ',':
         return Token(Token_type.COMMA)
+    elif tag == '=':
+        return Token(Token_type.EQUAL_SIGN)
+    elif tag == ';':
+        return Token(Token_type.SEMICOLON)
+    elif MATH_OP_PATTERN.match(tag):
+        return Token(Token_type.MATH_OP)
     elif NUMERIC_CONSTANT_PATTERN.match(tag):
         return Token(Token_type.NUMERIC_CONSTANT, tag)
-    elif tag == 'if':
+    elif tag == "if":
         return Token(Token_type.IF_STATEMENT)
-    elif tag == 'while':
+    elif tag == "else":
+        return Token(Token_type.ELSE_STATEMENT)
+    elif tag == "while":
         return Token(Token_type.WHILE_STATEMENT)
-    elif tag == 'do':
+    elif tag == "do":
         return Token(Token_type.DO_WHILE_STATEMENT)
-    elif tag == 'for':
+    elif tag == "for":
         return Token(Token_type.FOR_STATEMENT)
-    #TODO: finish this
+    elif KEYWORD_PATTERN.match(tag):
+        return Token(Token_type.KEY_WORD, tag)
+    elif STRING_LITERAL_PATTERN.match(tag):
+        return Token(Token_type, tag[1:-1])
+    elif type_name_pattern.match(tag):
+        return Token(Token_type.TYPE_NAME, tag)
+    else:
+        logging.warn(f"unknown token {tag}")
+        return Token(Token_type.UNKNOWN, tag)