diff --git a/src/usexp/lexer.mll b/src/usexp/lexer.mll index cb5d6343..c53bab93 100644 --- a/src/usexp/lexer.mll +++ b/src/usexp/lexer.mll @@ -75,7 +75,8 @@ type block_string_line_kind = let comment = ';' [^ '\n' '\r']* let newline = '\r'? '\n' let blank = [' ' '\t' '\012'] -let atom_char = [^ ';' '(' ')' '"' ' ' '\t' '\r' '\n' '\012'] +let atom_char = + [^ ';' '(' ')' '"' ' ' '\t' '\r' '\n' '\000'-'\032' '\127'-'\255'] let digit = ['0'-'9'] let hexdigit = ['0'-'9' 'a'-'f' 'A'-'F'] diff --git a/src/usexp/usexp.ml b/src/usexp/usexp.ml index 11bfd71d..48a3dd32 100644 --- a/src/usexp/usexp.ml +++ b/src/usexp/usexp.ml @@ -17,7 +17,7 @@ module Atom = struct let rec loop s i len = i = len || match String.unsafe_get s i with - | '"' | '(' | ')' | ';' | ' ' | '\t' | '\n' | '\012' | '\r' -> false + | '"' | '(' | ')' | ';' | '\000'..'\032' | '\127'..'\255' -> false | _ -> loop s (i + 1) len in fun s -> @@ -52,19 +52,8 @@ let atom s = let unsafe_atom_of_string s = Atom(A s) -let should_be_atom = - let rec loop s i len = - i = len || - match String.unsafe_get s i with - | '"' | '(' | ')' | ';' | '\000'..'\032' | '\127'..'\255' -> false - | _ -> loop s (i + 1) len - in - fun s -> - let len = String.length s in - len > 0 && loop s 0 len - let atom_or_quoted_string s = - if should_be_atom s then Atom (A s) + if Atom.is_valid s then Atom (A s) else Quoted_string s let quote_length s = @@ -260,7 +249,7 @@ module Ast = struct | List of Loc.t * t list let atom_or_quoted_string loc s = - if should_be_atom s then Atom (loc, A s) + if Atom.is_valid s then Atom (loc, A s) else Quoted_string (loc, s) let loc (Atom (loc, _) | Quoted_string (loc, _) | List (loc, _)) = loc