264 lines
7.9 KiB
Erlang
264 lines
7.9 KiB
Erlang
|
|
-spec consume_token(SourceStr) -> {tok, Token, Rest}
|
|
| none
|
|
when SourceStr :: string(),
|
|
Token :: token(),
|
|
Rest :: string().
|
|
|
|
consume_token(SrcStr) ->
|
|
consume_token_II(token_parsers(), SrcStr).
|
|
|
|
|
|
consume_token_II([TokenParser | Rest], SrcStr) ->
|
|
case consume_token(TokenParser, SrcStr) of
|
|
none -> consume_token_II(Rest, SrcStr);
|
|
Tok -> Tok;
|
|
end;
|
|
consume_token_II([], _) ->
|
|
none.
|
|
|
|
|
|
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
|
consume_token(identifier, [LowerUnder | Rest])
|
|
when (($a =< LowerUnder) andalso (LowerUnder =< $z))
|
|
orelse ($_ =:= LowerUnder)
|
|
->
|
|
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
|
TokenStr = [LowerUnder | TokenTail],
|
|
Token = #identifier{str = TokenStr},
|
|
{tok, Token, NewRest};
|
|
consume_token(identifier, _) ->
|
|
none;
|
|
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
|
consume_token(constructor, [Upper | Rest])
|
|
when ($A =< Upper) andalso (Upper =< $Z) ->
|
|
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
|
TokenStr = [Upper | TokenTail],
|
|
Token = #constructor{str = TokenStr},
|
|
{tok, Token, NewRest};
|
|
consume_token(constructor, _) ->
|
|
none;
|
|
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
|
consume_token(qualified_identifier, Str) ->
|
|
case consume_qualifiers(Str) of
|
|
{[], _} ->
|
|
none;
|
|
{Qualifiers, Rest0} ->
|
|
case consume_token(identifier, Rest0) of
|
|
{tok, Identifier, Rest1} ->
|
|
Token = #qualified_identifier{qualifiers = Qualifiers,
|
|
identifier = Identifier},
|
|
{tok, Token, Rest1};
|
|
none ->
|
|
none
|
|
end
|
|
end;
|
|
consume_token(qualified_constructor, Str) ->
|
|
case consume_qualifiers(Str) of
|
|
{[], _} ->
|
|
none;
|
|
{Qualifiers, Rest0} ->
|
|
case consume_token(constructor, Rest0) of
|
|
{tok, Constructor, Rest1} ->
|
|
Token = #qualified_constructor{qualifiers = Qualifiers,
|
|
constructor = Constructor},
|
|
{tok, Token, Rest1};
|
|
none ->
|
|
none
|
|
end
|
|
end;
|
|
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
|
consume_token(type_variable, [$' | Rest0]) ->
|
|
case consume_token(identifier, Rest0) of
|
|
{tok, Identifier, Rest1} ->
|
|
Token = #type_variable{identifier = Identifier},
|
|
{tok, Token, Rest1};
|
|
none ->
|
|
none
|
|
end;
|
|
consume_token(type_variable, _) ->
|
|
none;
|
|
consume_token(int, Str) ->
|
|
case {consume_int16(Str), consume_int10(Str)} of
|
|
{{tok, Int16, Rest}, _} ->
|
|
{tok, #int{inner_int = Int16}, Rest};
|
|
{none, {tok, Int10, Rest}} ->
|
|
{tok, #int{inner_int = Int10}, Rest};
|
|
{none, none} ->
|
|
none
|
|
end;
|
|
consume_token(string, _) ->
|
|
none;
|
|
consume_token(char, _) ->
|
|
none;
|
|
consume_token(akstr, _) ->
|
|
none;
|
|
consume_token(ctstr, _) ->
|
|
none;
|
|
consume_token(sgstr, _) ->
|
|
none.
|
|
|
|
|
|
|
|
-spec consume_int16(SrcStr) -> {tok, #int16{}, Rest} | none
|
|
when SrcStr :: string(),
|
|
Rest :: string().
|
|
|
|
consume_int16("0x" ++ Rest0) ->
|
|
case consume_hex_digits(Rest0) of
|
|
{piece16, Piece0, Rest1} ->
|
|
{Pieces, Rest2} = consume_uhexs(Rest1),
|
|
Token = #int16{hex_pieces = [Pieces0 | Pieces]},
|
|
{tok, Token, Rest2};
|
|
none ->
|
|
none
|
|
end;
|
|
consume_int16(_) ->
|
|
none.
|
|
|
|
|
|
|
|
-spec takewhile(Pred, List) -> {Taken, Rest}
|
|
when Pred :: fun( (Item) -> boolean() ),
|
|
List :: [Item],
|
|
Item :: any(),
|
|
Taken :: [Item],
|
|
Rest :: [Item].
|
|
|
|
takewhile(Pred, List) ->
|
|
takewhile(Pred, List, []).
|
|
|
|
|
|
takewhile(Pred, [X | Xs], Acc) ->
|
|
case Pred(X) of
|
|
true -> takewhile(Pred, Xs, [X | Acc]);
|
|
false -> {lists:reverse(Acc), Xs};
|
|
end;
|
|
takewhile(_, [], Acc) ->
|
|
{lists:reverse(Acc), []}.
|
|
|
|
|
|
-spec is_ulduq(Char) -> boolean()
|
|
when Char :: integer().
|
|
% @doc upper lower digit underscore quote
|
|
|
|
is_ulduq(C) ->
|
|
(($A =< C) andalso (C =< $Z))
|
|
orelse (($a =< C) andalso (C =< $z))
|
|
orelse (C =:= $_)
|
|
orelse (C =:= $').
|
|
|
|
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
|
-record(id,
|
|
{str :: string()}).
|
|
|
|
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
|
-record(con,
|
|
{str :: string()}).
|
|
|
|
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
|
-record(qid,
|
|
{qualifiers :: [#con{}],
|
|
identifier :: #id{}}).
|
|
|
|
% - `QCon = (Con\.)+Con` qualified constructor
|
|
-record(qcon,
|
|
{qualifiers :: [#con{}],
|
|
constructor :: #con{}}).
|
|
|
|
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
|
-record(tvar,
|
|
{identifier :: #identifier{}}).
|
|
|
|
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
|
-record(int16,
|
|
{hex_pieces :: [string()]}).
|
|
|
|
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
|
-record(int10,
|
|
{dec_pieces :: [string()]}).
|
|
|
|
-type int() :: int10() | int16().
|
|
|
|
|
|
% @doc
|
|
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
|
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
|
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
|
% - `QCon = (Con\.)+Con` qualified constructor
|
|
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
|
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
|
% - `Bytes = #[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` byte array literal with optional `_` separators
|
|
% - `String` string literal enclosed in `"` with escape character `\`
|
|
% - `Char` character literal enclosed in `'` with escape character `\`
|
|
% - `AccountAddress` base58-encoded 32 byte account pubkey with `ak_` prefix
|
|
% - `ContractAddress` base58-encoded 32 byte contract address with `ct_` prefix
|
|
% - `Signature` base58-encoded 64 byte cryptographic signature with `sg_` prefix
|
|
-type token() :: #id{}
|
|
| #con{}
|
|
| #qid{}
|
|
| #tvar{}
|
|
| int()
|
|
| {string, string()}
|
|
| {char, string()}
|
|
| {akstr, string()}
|
|
| {ctstr, string()}
|
|
| {sgstr, string()}.
|
|
|
|
-type token_shape() :: id
|
|
| con
|
|
| qid,
|
|
| qcon
|
|
| tvar
|
|
| int
|
|
| string
|
|
| char
|
|
| akstr
|
|
| ctstr
|
|
| sgstr.
|
|
|
|
%-----------------------------------------
|
|
% functions
|
|
%-----------------------------------------
|
|
|
|
-spec token_parsers() -> [atom()].
|
|
% in parse order
|
|
|
|
token_parsers() ->
|
|
[id,
|
|
con,
|
|
qid,
|
|
qcon,
|
|
tvar,
|
|
int,
|
|
string,
|
|
char,
|
|
akstr,
|
|
ctstr,
|
|
sgstr].
|
|
|
|
|
|
-spec token_consume(Matcher, SourceString) -> MaybeToken
|
|
when Matcher :: token_shape(),
|
|
SourceString :: string(),
|
|
MaybeToken :: {match, Token :: token(), Rest :: string()}
|
|
| nomatch.
|
|
|
|
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
|
token_consume(id, [AZU | Src1]) when $_ =:= AZU; $a =< AZU, AZU =< $z ->
|
|
{RestOfId, Src2} = tdw(fun is_ulduq/1, Src1),
|
|
{match, #id{str = [AZU | RestOfId]}, Src2};
|
|
token_consume(id, _) ->
|
|
nomatch;
|
|
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
|
token_consume(con, [AZ | Src1]) -> when $A =< AZU, AZU =< $Z ->
|
|
{RestOfCon, Src2} = tdw(fun is_ulduq/1, Src1),
|
|
{match, #con{str = [AZ | RestOfCon]}, Src2};
|
|
token_consume(con, _) ->
|
|
nomatch;
|
|
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
|
token_consume(qid, Src0) ->
|
|
case one_or_more({seq, con, {char, $.}}, Src0)
|
|
|
|
|