Improving prefix-string parsing

Vaibhav Pathak · certik · commit daae9dce104a · 2024-02-08T18:15:43.000-07:00
diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy
@@ -179,6 +179,9 @@ void yyerror(YYLTYPE *yyloc, LCompilers::LPython::Parser &p, const std::string &
 %token KW_MATCH
 %token KW_CASE
 
+%token KW_STR_PREFIX
+%type <string> KW_STR_PREFIX
+
 // Nonterminal tokens
 
 %type <ast> script_unit
@@ -1102,9 +1105,9 @@ subscript
 
 string
     : string TK_STRING { $$ = STRING2($1, $2, @$); } // TODO
-    | string id TK_STRING { $$ = STRING4($1, STRING3($2, $3, @$), @$); }
+    | string KW_STR_PREFIX TK_STRING { $$ = STRING4($1, STRING3($2, $3, @$), @$); }
     | TK_STRING { $$ = STRING1($1, @$); }
-    | id TK_STRING { $$ = STRING3($1, $2, @$); }
+    | KW_STR_PREFIX TK_STRING { $$ = STRING3($1, $2, @$); }
     ;
 
 lambda_parameter
diff --git a/src/lpython/parser/semantics.h b/src/lpython/parser/semantics.h
@@ -800,7 +800,7 @@ static inline ast_t* concat_string(Allocator &al, Location &l,
 #define INTEGER(x, l) make_ConstantInt_t(p.m_a, l, x, nullptr)
 #define STRING1(x, l) make_ConstantStr_t(p.m_a, l, str_unescape_c(p.m_a, x), nullptr)
 #define STRING2(x, y, l) concat_string(p.m_a, l, EXPR(x), str_unescape_c(p.m_a, y), nullptr)
-#define STRING3(id, x, l) PREFIX_STRING(p.m_a, l, name2char(id), x.c_str(p.m_a))
+#define STRING3(prefix, x, l) PREFIX_STRING(p.m_a, l, prefix.c_str(p.m_a), x.c_str(p.m_a))
 #define STRING4(x, s, l) concat_string(p.m_a, l, EXPR(x), "", EXPR(s))
 #define FLOAT(x, l) make_ConstantFloat_t(p.m_a, l, x, nullptr)
 #define COMPLEX(x, l) make_ConstantComplex_t(p.m_a, l, 0, x, nullptr)
diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re
@@ -435,6 +435,19 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
                 }
             }
 
+            [rR][bB] | [bB][rR]
+            | [fF][rR] | [rR][fF]
+            | [rR] | [bB] | [fF] | [uU]
+             {
+                if(cur[0] == '\'' || cur[0] == '"'){
+                    KW(STR_PREFIX);
+                }
+                else {
+                    token(yylval.string);
+                    RET(TK_NAME);
+                }
+            }
+
             // Tokens
             newline {
                 if(parenlevel) { continue; }
@@ -763,6 +776,7 @@ std::string token2text(const int token)
 
         T(KW_MATCH, "match")
         T(KW_CASE, "case")
+        T(KW_STR_PREFIX, "string prefix")
 
         default : {
             std::cout << "TOKEN: " << token << std::endl;
