WqVoon/SimpleTokenParser

编译原理作业，c语言词法分析器

仅支持py3

使用方法

from Tokens import TokenProcessor as TP

with open(<文件名>) as f:
    tp = TP(f)

for token_type, token_id in tp:
    print(tp[token_type][token_id])

或

from Tokens import TokenProcessor as TP

src = """
#include <included_file>

int main(void)
{
    puts("Hello World");    
    return 0;
}
"""

tp = TP(src)

for token_type, token_id in tp:
    print(tp[token_type][token_id])