diff --git a/src/gsc_tokens.erl b/src/gsc_tokens.erl index d20a819..150bd92 100644 --- a/src/gsc_tokens.erl +++ b/src/gsc_tokens.erl @@ -374,11 +374,8 @@ tokens_from_iolist(S) -> tokens(S). | {error, gsc_err()}, Tokens :: [tk()]. % @doc -% Recursively parse all tokens off the front end of the string. `Rest' is -% the first tail of the string for which no token parser succeeds. -% -% Semantically, `Rest'` being nonempty amounts to the presence of an illegal -% character. +% Recursively parse all tokens off the front end of the +% string. tokens(S) -> % defensive normalization @@ -525,7 +522,7 @@ slurp_token(Pos, SrcStr) -> Rest :: string(). % @doc % grab a single token off the front of the string according to -% `token_shapes_parse_order/0' +% `ParseOrder' slurp_token_shapes([TokenType | TTs], Pos, SrcStr) -> case slurp_token_of_shape(TokenType, Pos, SrcStr) of @@ -562,9 +559,9 @@ slurp_token_of_shape(lcom, Pos, SrcStr) -> case SrcStr of "//" ++ _ -> {Line, Rest} = takeline("", SrcStr), - Token = #tk{shape = lcom, - pos = Pos, - str = Line}, + Token = #tk{shape = lcom, + pos = Pos, + str = Line}, {tokmatch, Token, Rest}; _ -> no_tokmatch