scratch work
This commit is contained in:
Peter Harpending
@@ -0,0 +1,263 @@
|
||||
|
||||
-spec consume_token(SourceStr) -> {tok, Token, Rest}
|
||||
| none
|
||||
when SourceStr :: string(),
|
||||
Token :: token(),
|
||||
Rest :: string().
|
||||
|
||||
consume_token(SrcStr) ->
|
||||
consume_token_II(token_parsers(), SrcStr).
|
||||
|
||||
|
||||
consume_token_II([TokenParser | Rest], SrcStr) ->
|
||||
case consume_token(TokenParser, SrcStr) of
|
||||
none -> consume_token_II(Rest, SrcStr);
|
||||
Tok -> Tok;
|
||||
end;
|
||||
consume_token_II([], _) ->
|
||||
none.
|
||||
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
consume_token(identifier, [LowerUnder | Rest])
|
||||
when (($a =< LowerUnder) andalso (LowerUnder =< $z))
|
||||
orelse ($_ =:= LowerUnder)
|
||||
->
|
||||
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
||||
TokenStr = [LowerUnder | TokenTail],
|
||||
Token = #identifier{str = TokenStr},
|
||||
{tok, Token, NewRest};
|
||||
consume_token(identifier, _) ->
|
||||
none;
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
consume_token(constructor, [Upper | Rest])
|
||||
when ($A =< Upper) andalso (Upper =< $Z) ->
|
||||
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
||||
TokenStr = [Upper | TokenTail],
|
||||
Token = #constructor{str = TokenStr},
|
||||
{tok, Token, NewRest};
|
||||
consume_token(constructor, _) ->
|
||||
none;
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
consume_token(qualified_identifier, Str) ->
|
||||
case consume_qualifiers(Str) of
|
||||
{[], _} ->
|
||||
none;
|
||||
{Qualifiers, Rest0} ->
|
||||
case consume_token(identifier, Rest0) of
|
||||
{tok, Identifier, Rest1} ->
|
||||
Token = #qualified_identifier{qualifiers = Qualifiers,
|
||||
identifier = Identifier},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end
|
||||
end;
|
||||
consume_token(qualified_constructor, Str) ->
|
||||
case consume_qualifiers(Str) of
|
||||
{[], _} ->
|
||||
none;
|
||||
{Qualifiers, Rest0} ->
|
||||
case consume_token(constructor, Rest0) of
|
||||
{tok, Constructor, Rest1} ->
|
||||
Token = #qualified_constructor{qualifiers = Qualifiers,
|
||||
constructor = Constructor},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end
|
||||
end;
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
consume_token(type_variable, [$' | Rest0]) ->
|
||||
case consume_token(identifier, Rest0) of
|
||||
{tok, Identifier, Rest1} ->
|
||||
Token = #type_variable{identifier = Identifier},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end;
|
||||
consume_token(type_variable, _) ->
|
||||
none;
|
||||
consume_token(int, Str) ->
|
||||
case {consume_int16(Str), consume_int10(Str)} of
|
||||
{{tok, Int16, Rest}, _} ->
|
||||
{tok, #int{inner_int = Int16}, Rest};
|
||||
{none, {tok, Int10, Rest}} ->
|
||||
{tok, #int{inner_int = Int10}, Rest};
|
||||
{none, none} ->
|
||||
none
|
||||
end;
|
||||
consume_token(string, _) ->
|
||||
none;
|
||||
consume_token(char, _) ->
|
||||
none;
|
||||
consume_token(akstr, _) ->
|
||||
none;
|
||||
consume_token(ctstr, _) ->
|
||||
none;
|
||||
consume_token(sgstr, _) ->
|
||||
none.
|
||||
|
||||
|
||||
|
||||
-spec consume_int16(SrcStr) -> {tok, #int16{}, Rest} | none
|
||||
when SrcStr :: string(),
|
||||
Rest :: string().
|
||||
|
||||
consume_int16("0x" ++ Rest0) ->
|
||||
case consume_hex_digits(Rest0) of
|
||||
{piece16, Piece0, Rest1} ->
|
||||
{Pieces, Rest2} = consume_uhexs(Rest1),
|
||||
Token = #int16{hex_pieces = [Pieces0 | Pieces]},
|
||||
{tok, Token, Rest2};
|
||||
none ->
|
||||
none
|
||||
end;
|
||||
consume_int16(_) ->
|
||||
none.
|
||||
|
||||
|
||||
|
||||
-spec takewhile(Pred, List) -> {Taken, Rest}
|
||||
when Pred :: fun( (Item) -> boolean() ),
|
||||
List :: [Item],
|
||||
Item :: any(),
|
||||
Taken :: [Item],
|
||||
Rest :: [Item].
|
||||
|
||||
takewhile(Pred, List) ->
|
||||
takewhile(Pred, List, []).
|
||||
|
||||
|
||||
takewhile(Pred, [X | Xs], Acc) ->
|
||||
case Pred(X) of
|
||||
true -> takewhile(Pred, Xs, [X | Acc]);
|
||||
false -> {lists:reverse(Acc), Xs};
|
||||
end;
|
||||
takewhile(_, [], Acc) ->
|
||||
{lists:reverse(Acc), []}.
|
||||
|
||||
|
||||
-spec is_ulduq(Char) -> boolean()
|
||||
when Char :: integer().
|
||||
% @doc upper lower digit underscore quote
|
||||
|
||||
is_ulduq(C) ->
|
||||
(($A =< C) andalso (C =< $Z))
|
||||
orelse (($a =< C) andalso (C =< $z))
|
||||
orelse (C =:= $_)
|
||||
orelse (C =:= $').
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
-record(id,
|
||||
{str :: string()}).
|
||||
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
-record(con,
|
||||
{str :: string()}).
|
||||
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
-record(qid,
|
||||
{qualifiers :: [#con{}],
|
||||
identifier :: #id{}}).
|
||||
|
||||
% - `QCon = (Con\.)+Con` qualified constructor
|
||||
-record(qcon,
|
||||
{qualifiers :: [#con{}],
|
||||
constructor :: #con{}}).
|
||||
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
-record(tvar,
|
||||
{identifier :: #identifier{}}).
|
||||
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
-record(int16,
|
||||
{hex_pieces :: [string()]}).
|
||||
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
-record(int10,
|
||||
{dec_pieces :: [string()]}).
|
||||
|
||||
-type int() :: int10() | int16().
|
||||
|
||||
|
||||
% @doc
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
% - `QCon = (Con\.)+Con` qualified constructor
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
% - `Bytes = #[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` byte array literal with optional `_` separators
|
||||
% - `String` string literal enclosed in `"` with escape character `\`
|
||||
% - `Char` character literal enclosed in `'` with escape character `\`
|
||||
% - `AccountAddress` base58-encoded 32 byte account pubkey with `ak_` prefix
|
||||
% - `ContractAddress` base58-encoded 32 byte contract address with `ct_` prefix
|
||||
% - `Signature` base58-encoded 64 byte cryptographic signature with `sg_` prefix
|
||||
-type token() :: #id{}
|
||||
| #con{}
|
||||
| #qid{}
|
||||
| #tvar{}
|
||||
| int()
|
||||
| {string, string()}
|
||||
| {char, string()}
|
||||
| {akstr, string()}
|
||||
| {ctstr, string()}
|
||||
| {sgstr, string()}.
|
||||
|
||||
-type token_shape() :: id
|
||||
| con
|
||||
| qid,
|
||||
| qcon
|
||||
| tvar
|
||||
| int
|
||||
| string
|
||||
| char
|
||||
| akstr
|
||||
| ctstr
|
||||
| sgstr.
|
||||
|
||||
%-----------------------------------------
|
||||
% functions
|
||||
%-----------------------------------------
|
||||
|
||||
-spec token_parsers() -> [atom()].
|
||||
% in parse order
|
||||
|
||||
token_parsers() ->
|
||||
[id,
|
||||
con,
|
||||
qid,
|
||||
qcon,
|
||||
tvar,
|
||||
int,
|
||||
string,
|
||||
char,
|
||||
akstr,
|
||||
ctstr,
|
||||
sgstr].
|
||||
|
||||
|
||||
-spec token_consume(Matcher, SourceString) -> MaybeToken
|
||||
when Matcher :: token_shape(),
|
||||
SourceString :: string(),
|
||||
MaybeToken :: {match, Token :: token(), Rest :: string()}
|
||||
| nomatch.
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
token_consume(id, [AZU | Src1]) when $_ =:= AZU; $a =< AZU, AZU =< $z ->
|
||||
{RestOfId, Src2} = tdw(fun is_ulduq/1, Src1),
|
||||
{match, #id{str = [AZU | RestOfId]}, Src2};
|
||||
token_consume(id, _) ->
|
||||
nomatch;
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
token_consume(con, [AZ | Src1]) -> when $A =< AZU, AZU =< $Z ->
|
||||
{RestOfCon, Src2} = tdw(fun is_ulduq/1, Src1),
|
||||
{match, #con{str = [AZ | RestOfCon]}, Src2};
|
||||
token_consume(con, _) ->
|
||||
nomatch;
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
token_consume(qid, Src0) ->
|
||||
case one_or_more({seq, con, {char, $.}}, Src0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user