# module to read in SL-LEX formatted tokens class Token: """ Token class will represent a snail token """ def __init__(self, lineno, colno, token_type, lexeme): """ Constructor for a token lineno - line number of the token colno - column number of the token token_type - type of the token lexeme - actual text of the token """ self.type = token_type.upper() self.value = lexeme self.lineno = lineno self.lexerpos = colno # we are hacking this def __repr__(self): tok = f"{self.lineno}:{self.lexerpos} {self.type}" if self.type in ["IDENT", "STRING", "INT"]: tok += f" {self.value}" return tok class TokenReader: """ Class to read in SL-LEX Tokens """ def __init__(self, f): """ f - file-like object containing the SL-LEX token information """ # create a list of all of the lines from f lines = f.readlines() # list of token objects # any member variable with a name beginning with __ is essentially private self.__tokens = [] # keep track of the current line in the SL-LEX file i = 0 while i < len(lines): line_no = int(lines[i]) col_no = int(lines[i+1]) tok_type = lines[i+2].strip() if tok_type in ["ident", "int", "string"]: # FIXME `strip()` may cause a bug lexeme = lines[i+3].strip() i += 1 else: lexeme = tok_type # increment i i += 3 # add token object to list of tokens self.__tokens.append(Token(line_no, col_no, tok_type, lexeme)) # wrap the list of tokens in an iterator self.token_stream = iter(self.__tokens) def token(self): """ return the next token object or None if there are no more """ return next(self.token_stream, None) # main program if __name__ == "__main__": # this will only run when executed from the command line import argparse parser = argparse.ArgumentParser() parser.add_argument("sl_file", help="The SL-LEX file to load") args = parser.parse_args() # args.sl_file will be the file name of the SL-LEX file with open(args.sl_file) as f: reader = TokenReader(f) while (token := reader.token()) is not None: # print out the token print(token)