plexx-dev/Nassi-Shneiderman-Diagramm-Generator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added instruction generation code to Lexer

Browse Source
This commit is contained in:
weckyy702
2021-04-01 17:00:19 +02:00
parent 49f3c84e60
commit 3bc93fa3a7
4 changed files with 148 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
interpreter/Tokenizer.py
View File

@@ -4,12 +4,12 @@ import logging
import re
from typing import List, Optional
from interpreter._token import Token, make_token
from interpreter._token import Token, make_token, SourceLocation
class Tokenizer:
"""This class will take the provided source file and convert it to a list of tokens"""
TOKEN_MATCH = re.compile(r"""\(|\)|\{|\}|;|(\n)|\+|-|\*|/|<|>|,| """) #TODO: make this modular
TOKEN_MATCH = re.compile(r"""\(|\)|\{|\}|;|(\n)|\+|-|\*|/|<|>|,| """)
def __init__(self, file_name: str) -> None:
with open(file_name) as f:
@@ -17,8 +17,12 @@ class Tokenizer:
self.source_index = 0
self.line_number = 1
self.source_text = re.sub("(private)|(public)|(protected)", "", self.source_text)
self.type_name_pattern = re.compile('(char)|(int)|(void)|(double)|(Pixel)') #TODO: make this modular
self._filename = file_name
def get_tokens(self) -> List[Token]:
tokens = []
@@ -33,7 +37,7 @@ class Tokenizer:
token = self._get_token(char)
logging.debug(f"found token \"{token}\" on line {self.line_number}")
tokens.append(make_token(token, self.type_name_pattern))
tokens.append(make_token(token, SourceLocation(self._filename, self.line_number), self.type_name_pattern))
return tokens