Commit db537fe0 authored by Julien Lopez's avatar Julien Lopez
Browse files

Better approach: now we create a token SQUOTE "'" in the lexer, then we decide

if it is a string or a type variable in the parser. Still some errors in the
parser (see make test).
parent 162b4367
This diff is collapsed.
......@@ -46,11 +46,10 @@ end
type token =
| KEYWORD of string
| IDENT of string
| PVAR of string
| ANY_IN_NS of string
| INT of string
| STRING1 of string
| STRING2 of string
| STRING of string
| SQUOTE
| EOI
module Token = struct
......@@ -65,10 +64,9 @@ module Token = struct
function
| KEYWORD s -> sf "KEYWORD %S" s
| IDENT s -> sf "IDENT %S" s
| PVAR s -> sf "PVAR \"%s\"" s
| INT s -> sf "INT %s" s
| STRING1 s -> sf "STRING \"%s\"" s
| STRING2 s -> sf "STRING \"%s\"" s
| SQUOTE -> sf "'"
| STRING s -> sf "STRING \"%s\"" s
(* here it's not %S since the string is already escaped *)
| ANY_IN_NS s -> sf "ANY_IN_NS %S" s
| EOI -> sf "EOI"
......@@ -82,7 +80,7 @@ module Token = struct
let extract_string =
function
| KEYWORD s | IDENT s | PVAR s | INT s | STRING1 s | STRING2 s |
| KEYWORD s | IDENT s | INT s | STRING s |
ANY_IN_NS s -> s
| tok ->
invalid_arg ("Cannot extract a string from this token: "^
......@@ -212,14 +210,10 @@ let rec token = lexer
return lexbuf (KEYWORD (L.utf8_lexeme lexbuf))
| '"' ->
let start = L.lexeme_start lexbuf in
string_double (L.lexeme_start lexbuf) lexbuf;
string (L.lexeme_start lexbuf) lexbuf;
let s = get_stored_string () in
return_loc start (L.lexeme_end lexbuf) (STRING2 s)
| "'" ->
let start = L.lexeme_start lexbuf in
let b = string_simple (L.lexeme_start lexbuf) 0 lexbuf in
let s = get_stored_string () in
return_loc start (L.lexeme_end lexbuf) (if b then STRING1(s) else PVAR(s))
return_loc start (L.lexeme_end lexbuf) (STRING s)
| "'" -> return lexbuf SQUOTE
| "(*" ->
in_comment := true;
comment (L.lexeme_start lexbuf) lexbuf;
......@@ -241,12 +235,8 @@ and comment start = lexer
comment start lexbuf
| "*)" ->
()
| '"' ->
string_double (L.lexeme_start lexbuf) lexbuf;
clear_buff ();
comment start lexbuf
| "'" ->
ignore(string_simple (L.lexeme_start lexbuf) 0 lexbuf);
| '"' | "'" ->
string (L.lexeme_start lexbuf) lexbuf;
clear_buff ();
comment start lexbuf
| eof ->
......@@ -262,38 +252,20 @@ and tcomment start = lexer
| _ ->
tcomment start lexbuf
and string_double start = lexer
and string start = lexer
| '"' -> ()
| '\\' ['\\' '"' '\''] ->
store_ascii (L.latin1_lexeme_char lexbuf 1); string_double start lexbuf
| "\\n" -> store_ascii '\n'; string_double start lexbuf
| "\\t" -> store_ascii '\t'; string_double start lexbuf
| "\\r" -> store_ascii '\r'; string_double start lexbuf
store_ascii (L.latin1_lexeme_char lexbuf 1); string start lexbuf
| "\\n" -> store_ascii '\n'; string start lexbuf
| "\\t" -> store_ascii '\t'; string start lexbuf
| "\\r" -> store_ascii '\r'; string start lexbuf
| '\\' ['0'-'9']+ ';' ->
store_code (parse_char lexbuf 10 1); string_double start lexbuf
store_code (parse_char lexbuf 10 1); string start lexbuf
| '\\' 'x' ['0'-'9' 'a'-'f' 'A'-'F']+ ';' ->
store_code (parse_char lexbuf 16 2); string_double start lexbuf
store_code (parse_char lexbuf 16 2); string start lexbuf
| '\\' -> illegal lexbuf;
| eof -> error start (start+1) "Unterminated string_double"
| _ -> store_lexeme lexbuf; string_double start lexbuf
and string_simple start nb = lexer
| "'" -> true
| ' ' -> if nb != 0 then false
else (store_lexeme lexbuf; string_simple (nb+1) start lexbuf)
| '\\' ['\\' '"' '\''] -> store_ascii (L.latin1_lexeme_char lexbuf 1);
string_simple (nb+1) start lexbuf
| "\\n" -> store_ascii '\n'; string_simple (nb+1) start lexbuf
| "\\t" -> store_ascii '\t'; string_simple (nb+1) start lexbuf
| "\\r" -> store_ascii '\r'; string_simple (nb+1) start lexbuf
| '\\' ['0'-'9']+ ';' ->
store_code (parse_char lexbuf 10 1); string_simple (nb+1) start lexbuf
| '\\' 'x' ['0'-'9' 'a'-'f' 'A'-'F']+ ';' ->
store_code (parse_char lexbuf 16 2); string_simple (nb+1) start lexbuf
| '\\' -> illegal lexbuf;
| eof -> error start (start+1) "Unterminated string_simple (nb+1)"
| _ -> store_lexeme lexbuf; string_simple (nb+1) start lexbuf
| eof -> error start (start+1) "Unterminated string"
| _ -> store_lexeme lexbuf; string start lexbuf
let lexbuf = ref None
let last_tok = ref (KEYWORD "DUMMY")
......
......@@ -3,11 +3,10 @@ open Camlp4.Sig
type token =
| KEYWORD of string
| IDENT of string
| PVAR of string
| ANY_IN_NS of string
| INT of string
| STRING1 of string
| STRING2 of string
| STRING of string
| SQUOTE
| EOI
exception Error of int * int * string
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment