Simplify jbuild lexer and move types around
The types should exist in the lexer module where they're used Signed-off-by: Rudi Grinberg <rudi.grinberg@gmail.com>
This commit is contained in:
parent
1e5dc322e2
commit
cd6d5dba53
|
@ -1,5 +1,9 @@
|
||||||
{
|
{
|
||||||
open Lexer0
|
open Lexer0
|
||||||
|
|
||||||
|
type block_string_line_kind =
|
||||||
|
| With_escape_sequences
|
||||||
|
| Raw
|
||||||
}
|
}
|
||||||
|
|
||||||
let comment = ';' [^ '\n' '\r']*
|
let comment = ';' [^ '\n' '\r']*
|
||||||
|
|
|
@ -1,5 +1,14 @@
|
||||||
{
|
{
|
||||||
open Lexer0
|
open Lexer0
|
||||||
|
|
||||||
|
(* The difference between the old and new syntax is that the old
|
||||||
|
syntax allows backslash following by any characters other than 'n',
|
||||||
|
'x', ... and interpret it as it. The new syntax is stricter in
|
||||||
|
order to allow introducing new escape sequence in the future if
|
||||||
|
needed. *)
|
||||||
|
type escape_mode =
|
||||||
|
| In_block_comment (* Inside #|...|# comments (old syntax) *)
|
||||||
|
| In_quoted_string
|
||||||
}
|
}
|
||||||
|
|
||||||
let comment = ';' [^ '\n' '\r']*
|
let comment = ';' [^ '\n' '\r']*
|
||||||
|
@ -24,7 +33,7 @@ rule token = parse
|
||||||
| '"'
|
| '"'
|
||||||
{ Buffer.clear escaped_buf;
|
{ Buffer.clear escaped_buf;
|
||||||
let start = Lexing.lexeme_start_p lexbuf in
|
let start = Lexing.lexeme_start_p lexbuf in
|
||||||
let s = quoted_string Old_syntax lexbuf in
|
let s = quoted_string In_quoted_string lexbuf in
|
||||||
lexbuf.lex_start_p <- start;
|
lexbuf.lex_start_p <- start;
|
||||||
Quoted_string s
|
Quoted_string s
|
||||||
}
|
}
|
||||||
|
@ -77,7 +86,7 @@ and quoted_string mode = parse
|
||||||
quoted_string mode lexbuf
|
quoted_string mode lexbuf
|
||||||
}
|
}
|
||||||
| eof
|
| eof
|
||||||
{ if mode <> In_block_comment then
|
{ if mode = In_block_comment then
|
||||||
error lexbuf "unterminated quoted string";
|
error lexbuf "unterminated quoted string";
|
||||||
Buffer.contents escaped_buf
|
Buffer.contents escaped_buf
|
||||||
}
|
}
|
||||||
|
@ -120,14 +129,14 @@ and escape_sequence mode = parse
|
||||||
}
|
}
|
||||||
| (digit as c1) (digit as c2) (digit as c3)
|
| (digit as c1) (digit as c2) (digit as c3)
|
||||||
{ let v = eval_decimal_escape c1 c2 c3 in
|
{ let v = eval_decimal_escape c1 c2 c3 in
|
||||||
if mode <> In_block_comment && v > 255 then
|
if mode = In_quoted_string && v > 255 then
|
||||||
error lexbuf "escape sequence in quoted string out of range"
|
error lexbuf "escape sequence in quoted string out of range"
|
||||||
~delta:(-1);
|
~delta:(-1);
|
||||||
Buffer.add_char escaped_buf (Char.chr v);
|
Buffer.add_char escaped_buf (Char.chr v);
|
||||||
Other
|
Other
|
||||||
}
|
}
|
||||||
| digit* as s
|
| digit* as s
|
||||||
{ if mode <> In_block_comment then
|
{ if mode = In_quoted_string then
|
||||||
error lexbuf "unterminated decimal escape sequence" ~delta:(-1);
|
error lexbuf "unterminated decimal escape sequence" ~delta:(-1);
|
||||||
Buffer.add_char escaped_buf '\\';
|
Buffer.add_char escaped_buf '\\';
|
||||||
Buffer.add_string escaped_buf s;
|
Buffer.add_string escaped_buf s;
|
||||||
|
@ -139,21 +148,19 @@ and escape_sequence mode = parse
|
||||||
Other
|
Other
|
||||||
}
|
}
|
||||||
| 'x' hexdigit* as s
|
| 'x' hexdigit* as s
|
||||||
{ if mode <> In_block_comment then
|
{ if mode = In_quoted_string then
|
||||||
error lexbuf "unterminated hexadecimal escape sequence" ~delta:(-1);
|
error lexbuf "unterminated hexadecimal escape sequence" ~delta:(-1);
|
||||||
Buffer.add_char escaped_buf '\\';
|
Buffer.add_char escaped_buf '\\';
|
||||||
Buffer.add_string escaped_buf s;
|
Buffer.add_string escaped_buf s;
|
||||||
Other
|
Other
|
||||||
}
|
}
|
||||||
| _ as c
|
| _ as c
|
||||||
{ if mode = New_syntax then
|
{ Buffer.add_char escaped_buf '\\';
|
||||||
error lexbuf "unknown escape sequence" ~delta:(-1);
|
|
||||||
Buffer.add_char escaped_buf '\\';
|
|
||||||
Buffer.add_char escaped_buf c;
|
Buffer.add_char escaped_buf c;
|
||||||
Other
|
Other
|
||||||
}
|
}
|
||||||
| eof
|
| eof
|
||||||
{ if mode <> In_block_comment then
|
{ if mode = In_quoted_string then
|
||||||
error lexbuf "unterminated escape sequence" ~delta:(-1);
|
error lexbuf "unterminated escape sequence" ~delta:(-1);
|
||||||
Other
|
Other
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,24 +31,10 @@ let error ?(delta=0) lexbuf message =
|
||||||
|
|
||||||
let escaped_buf = Buffer.create 256
|
let escaped_buf = Buffer.create 256
|
||||||
|
|
||||||
(* The difference between the old and new syntax is that the old
|
|
||||||
syntax allows backslash following by any characters other than 'n',
|
|
||||||
'x', ... and interpret it as it. The new syntax is stricter in
|
|
||||||
order to allow introducing new escape sequence in the future if
|
|
||||||
needed. *)
|
|
||||||
type escape_mode =
|
|
||||||
| In_block_comment (* Inside #|...|# comments (old syntax) *)
|
|
||||||
| Old_syntax
|
|
||||||
| New_syntax
|
|
||||||
|
|
||||||
type escape_sequence =
|
type escape_sequence =
|
||||||
| Newline
|
| Newline
|
||||||
| Other
|
| Other
|
||||||
|
|
||||||
type block_string_line_kind =
|
|
||||||
| With_escape_sequences
|
|
||||||
| Raw
|
|
||||||
|
|
||||||
let eval_decimal_char c = Char.code c - Char.code '0'
|
let eval_decimal_char c = Char.code c - Char.code '0'
|
||||||
|
|
||||||
let eval_decimal_escape c1 c2 c3 =
|
let eval_decimal_escape c1 c2 c3 =
|
||||||
|
|
|
@ -24,19 +24,10 @@ val escaped_buf : Buffer.t
|
||||||
|
|
||||||
exception Error of Error.t
|
exception Error of Error.t
|
||||||
|
|
||||||
type escape_mode =
|
|
||||||
| In_block_comment (* Inside #|...|# comments (old syntax) *)
|
|
||||||
| Old_syntax
|
|
||||||
| New_syntax
|
|
||||||
|
|
||||||
type escape_sequence =
|
type escape_sequence =
|
||||||
| Newline
|
| Newline
|
||||||
| Other
|
| Other
|
||||||
|
|
||||||
type block_string_line_kind =
|
|
||||||
| With_escape_sequences
|
|
||||||
| Raw
|
|
||||||
|
|
||||||
val eval_decimal_char : char -> int
|
val eval_decimal_char : char -> int
|
||||||
|
|
||||||
val eval_decimal_escape : char -> char -> char -> int
|
val eval_decimal_escape : char -> char -> char -> int
|
||||||
|
|
Loading…
Reference in New Issue