Merge remote-tracking branch 'refs/remotes/origin/master'
This commit is contained in:
@@ -0,0 +1,263 @@
|
||||
|
||||
-spec consume_token(SourceStr) -> {tok, Token, Rest}
|
||||
| none
|
||||
when SourceStr :: string(),
|
||||
Token :: token(),
|
||||
Rest :: string().
|
||||
|
||||
consume_token(SrcStr) ->
|
||||
consume_token_II(token_parsers(), SrcStr).
|
||||
|
||||
|
||||
consume_token_II([TokenParser | Rest], SrcStr) ->
|
||||
case consume_token(TokenParser, SrcStr) of
|
||||
none -> consume_token_II(Rest, SrcStr);
|
||||
Tok -> Tok;
|
||||
end;
|
||||
consume_token_II([], _) ->
|
||||
none.
|
||||
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
consume_token(identifier, [LowerUnder | Rest])
|
||||
when (($a =< LowerUnder) andalso (LowerUnder =< $z))
|
||||
orelse ($_ =:= LowerUnder)
|
||||
->
|
||||
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
||||
TokenStr = [LowerUnder | TokenTail],
|
||||
Token = #identifier{str = TokenStr},
|
||||
{tok, Token, NewRest};
|
||||
consume_token(identifier, _) ->
|
||||
none;
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
consume_token(constructor, [Upper | Rest])
|
||||
when ($A =< Upper) andalso (Upper =< $Z) ->
|
||||
{TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
|
||||
TokenStr = [Upper | TokenTail],
|
||||
Token = #constructor{str = TokenStr},
|
||||
{tok, Token, NewRest};
|
||||
consume_token(constructor, _) ->
|
||||
none;
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
consume_token(qualified_identifier, Str) ->
|
||||
case consume_qualifiers(Str) of
|
||||
{[], _} ->
|
||||
none;
|
||||
{Qualifiers, Rest0} ->
|
||||
case consume_token(identifier, Rest0) of
|
||||
{tok, Identifier, Rest1} ->
|
||||
Token = #qualified_identifier{qualifiers = Qualifiers,
|
||||
identifier = Identifier},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end
|
||||
end;
|
||||
consume_token(qualified_constructor, Str) ->
|
||||
case consume_qualifiers(Str) of
|
||||
{[], _} ->
|
||||
none;
|
||||
{Qualifiers, Rest0} ->
|
||||
case consume_token(constructor, Rest0) of
|
||||
{tok, Constructor, Rest1} ->
|
||||
Token = #qualified_constructor{qualifiers = Qualifiers,
|
||||
constructor = Constructor},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end
|
||||
end;
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
consume_token(type_variable, [$' | Rest0]) ->
|
||||
case consume_token(identifier, Rest0) of
|
||||
{tok, Identifier, Rest1} ->
|
||||
Token = #type_variable{identifier = Identifier},
|
||||
{tok, Token, Rest1};
|
||||
none ->
|
||||
none
|
||||
end;
|
||||
consume_token(type_variable, _) ->
|
||||
none;
|
||||
consume_token(int, Str) ->
|
||||
case {consume_int16(Str), consume_int10(Str)} of
|
||||
{{tok, Int16, Rest}, _} ->
|
||||
{tok, #int{inner_int = Int16}, Rest};
|
||||
{none, {tok, Int10, Rest}} ->
|
||||
{tok, #int{inner_int = Int10}, Rest};
|
||||
{none, none} ->
|
||||
none
|
||||
end;
|
||||
consume_token(string, _) ->
|
||||
none;
|
||||
consume_token(char, _) ->
|
||||
none;
|
||||
consume_token(akstr, _) ->
|
||||
none;
|
||||
consume_token(ctstr, _) ->
|
||||
none;
|
||||
consume_token(sgstr, _) ->
|
||||
none.
|
||||
|
||||
|
||||
|
||||
-spec consume_int16(SrcStr) -> {tok, #int16{}, Rest} | none
|
||||
when SrcStr :: string(),
|
||||
Rest :: string().
|
||||
|
||||
consume_int16("0x" ++ Rest0) ->
|
||||
case consume_hex_digits(Rest0) of
|
||||
{piece16, Piece0, Rest1} ->
|
||||
{Pieces, Rest2} = consume_uhexs(Rest1),
|
||||
Token = #int16{hex_pieces = [Pieces0 | Pieces]},
|
||||
{tok, Token, Rest2};
|
||||
none ->
|
||||
none
|
||||
end;
|
||||
consume_int16(_) ->
|
||||
none.
|
||||
|
||||
|
||||
|
||||
-spec takewhile(Pred, List) -> {Taken, Rest}
|
||||
when Pred :: fun( (Item) -> boolean() ),
|
||||
List :: [Item],
|
||||
Item :: any(),
|
||||
Taken :: [Item],
|
||||
Rest :: [Item].
|
||||
|
||||
takewhile(Pred, List) ->
|
||||
takewhile(Pred, List, []).
|
||||
|
||||
|
||||
takewhile(Pred, [X | Xs], Acc) ->
|
||||
case Pred(X) of
|
||||
true -> takewhile(Pred, Xs, [X | Acc]);
|
||||
false -> {lists:reverse(Acc), Xs};
|
||||
end;
|
||||
takewhile(_, [], Acc) ->
|
||||
{lists:reverse(Acc), []}.
|
||||
|
||||
|
||||
-spec is_ulduq(Char) -> boolean()
|
||||
when Char :: integer().
|
||||
% @doc upper lower digit underscore quote
|
||||
|
||||
is_ulduq(C) ->
|
||||
(($A =< C) andalso (C =< $Z))
|
||||
orelse (($a =< C) andalso (C =< $z))
|
||||
orelse (C =:= $_)
|
||||
orelse (C =:= $').
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
-record(id,
|
||||
{str :: string()}).
|
||||
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
-record(con,
|
||||
{str :: string()}).
|
||||
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
-record(qid,
|
||||
{qualifiers :: [#con{}],
|
||||
identifier :: #id{}}).
|
||||
|
||||
% - `QCon = (Con\.)+Con` qualified constructor
|
||||
-record(qcon,
|
||||
{qualifiers :: [#con{}],
|
||||
constructor :: #con{}}).
|
||||
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
-record(tvar,
|
||||
{identifier :: #identifier{}}).
|
||||
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
-record(int16,
|
||||
{hex_pieces :: [string()]}).
|
||||
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
-record(int10,
|
||||
{dec_pieces :: [string()]}).
|
||||
|
||||
-type int() :: int10() | int16().
|
||||
|
||||
|
||||
% @doc
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
% - `QCon = (Con\.)+Con` qualified constructor
|
||||
% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
|
||||
% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
|
||||
% - `Bytes = #[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` byte array literal with optional `_` separators
|
||||
% - `String` string literal enclosed in `"` with escape character `\`
|
||||
% - `Char` character literal enclosed in `'` with escape character `\`
|
||||
% - `AccountAddress` base58-encoded 32 byte account pubkey with `ak_` prefix
|
||||
% - `ContractAddress` base58-encoded 32 byte contract address with `ct_` prefix
|
||||
% - `Signature` base58-encoded 64 byte cryptographic signature with `sg_` prefix
|
||||
-type token() :: #id{}
|
||||
| #con{}
|
||||
| #qid{}
|
||||
| #tvar{}
|
||||
| int()
|
||||
| {string, string()}
|
||||
| {char, string()}
|
||||
| {akstr, string()}
|
||||
| {ctstr, string()}
|
||||
| {sgstr, string()}.
|
||||
|
||||
-type token_shape() :: id
|
||||
| con
|
||||
| qid,
|
||||
| qcon
|
||||
| tvar
|
||||
| int
|
||||
| string
|
||||
| char
|
||||
| akstr
|
||||
| ctstr
|
||||
| sgstr.
|
||||
|
||||
%-----------------------------------------
|
||||
% functions
|
||||
%-----------------------------------------
|
||||
|
||||
-spec token_parsers() -> [atom()].
|
||||
% in parse order
|
||||
|
||||
token_parsers() ->
|
||||
[id,
|
||||
con,
|
||||
qid,
|
||||
qcon,
|
||||
tvar,
|
||||
int,
|
||||
string,
|
||||
char,
|
||||
akstr,
|
||||
ctstr,
|
||||
sgstr].
|
||||
|
||||
|
||||
-spec token_consume(Matcher, SourceString) -> MaybeToken
|
||||
when Matcher :: token_shape(),
|
||||
SourceString :: string(),
|
||||
MaybeToken :: {match, Token :: token(), Rest :: string()}
|
||||
| nomatch.
|
||||
|
||||
% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
|
||||
token_consume(id, [AZU | Src1]) when $_ =:= AZU; $a =< AZU, AZU =< $z ->
|
||||
{RestOfId, Src2} = tdw(fun is_ulduq/1, Src1),
|
||||
{match, #id{str = [AZU | RestOfId]}, Src2};
|
||||
token_consume(id, _) ->
|
||||
nomatch;
|
||||
% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
|
||||
token_consume(con, [AZ | Src1]) -> when $A =< AZU, AZU =< $Z ->
|
||||
{RestOfCon, Src2} = tdw(fun is_ulduq/1, Src1),
|
||||
{match, #con{str = [AZ | RestOfCon]}, Src2};
|
||||
token_consume(con, _) ->
|
||||
nomatch;
|
||||
% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
|
||||
token_consume(qid, Src0) ->
|
||||
case one_or_more({seq, con, {char, $.}}, Src0)
|
||||
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
% @doc
|
||||
% type for an SFC AST
|
||||
%
|
||||
% semantically:
|
||||
% #pr{} means the parser succeeded
|
||||
% noparse means the parser didn't consume any tokens
|
||||
% {error, _} means the parser can prove the user made an
|
||||
% error (e.g. unterminated open paren)
|
||||
%
|
||||
% based on so_syntax.erl
|
||||
%
|
||||
% this is the second layer in the parsing pipeline after tokens
|
||||
%
|
||||
% quoth `sophia/docs/so_syntax.md`:
|
||||
%
|
||||
% ## Declarations
|
||||
%
|
||||
% A Sophia file consists of a sequence of *declarations* in a layout block.
|
||||
%
|
||||
% ```c
|
||||
% File ::= Block(TopDecl)
|
||||
%
|
||||
% TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
|
||||
% | 'contract' 'interface' Con [Implement] '=' Block(Decl)
|
||||
% | 'namespace' Con '=' Block(Decl)
|
||||
% | '@compiler' PragmaOp Version
|
||||
% | 'include' String
|
||||
% | Using
|
||||
%
|
||||
% Implement ::= ':' Sep1(Con, ',')
|
||||
%
|
||||
% Decl ::= 'type' Id ['(' TVar* ')'] '=' TypeAlias
|
||||
% | 'record' Id ['(' TVar* ')'] '=' RecordType
|
||||
% | 'datatype' Id ['(' TVar* ')'] '=' DataType
|
||||
% | 'let' Id [':' Type] '=' Expr
|
||||
% | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
|
||||
% | Using
|
||||
%
|
||||
% FunDecl ::= Id ':' Type // Type signature
|
||||
% | Id Args [':' Type] '=' Block(Stmt) // Definition
|
||||
% | Id Args [':' Type] Block(GuardedDef) // Guarded definitions
|
||||
%
|
||||
% GuardedDef ::= '|' Sep1(Expr, ',') '=' Block(Stmt)
|
||||
%
|
||||
% Using ::= 'using' Con ['as' Con] [UsingParts]
|
||||
% UsingParts ::= 'for' '[' Sep1(Id, ',') ']'
|
||||
% | 'hiding' '[' Sep1(Id, ',') ']'
|
||||
%
|
||||
% PragmaOp ::= '<' | '=<' | '==' | '>=' | '>'
|
||||
% Version ::= Sep1(Int, '.')
|
||||
%
|
||||
% EModifier ::= 'payable' | 'stateful'
|
||||
% FModifier ::= 'stateful' | 'private'
|
||||
%
|
||||
% Args ::= '(' Sep(Pattern, ',') ')'
|
||||
% ```
|
||||
-module(sfc_ast).
|
||||
|
||||
-export_type([
|
||||
]).
|
||||
|
||||
-export([
|
||||
parse/2
|
||||
]).
|
||||
|
||||
-include("sfc.hrl").
|
||||
|
||||
|
||||
%%-----------------------------
|
||||
%% FUNCTIONS
|
||||
%%-----------------------------
|
||||
|
||||
-spec parse(ParseTarget, SigTokens) -> Result
|
||||
when ParseTarget :: parse_target(),
|
||||
SigTokens :: list(sfc_token()),
|
||||
Result :: perhaps_parsed().
|
||||
% @doc
|
||||
% parse target against list of tokens;
|
||||
%
|
||||
% for MVP, it is assumed the given list of tokens
|
||||
% contains
|
||||
|
||||
parse({ast, AstTarget}, Tokens) -> parse_ast(AstTarget, Tokens);
|
||||
parse({tokens, Target}, Tokens) -> parse_tokens(Target, Tokens);
|
||||
parse({prs, Strategy, Target}, Tokens) -> parse_prs(Strategy, Target, Tokens).
|
||||
|
||||
|
||||
|
||||
-spec parse_ast(AstTarget, SigTokens) -> Perhaps
|
||||
when AstTarget :: ast_target(),
|
||||
SigTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private parse where result is an AST
|
||||
%
|
||||
% happy path: returns
|
||||
%
|
||||
% #pr{type = ast,
|
||||
% result = AST :: ast(),
|
||||
% rest = _ :: [sfc_token()]}
|
||||
|
||||
parse_ast(AstTarget, SrcTokens) ->
|
||||
case AstTarget of
|
||||
file -> parse_ast_file(SrcTokens);
|
||||
top_decl -> parse_ast_top_decl(SrcTokens);
|
||||
_ -> {error, #sfc_err_nyi{}}
|
||||
end.
|
||||
|
||||
|
||||
|
||||
-spec parse_ast_file(SrcTokens) -> Perhaps
|
||||
when SrcTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private parse an #ast_file{} record
|
||||
%
|
||||
% on success returns
|
||||
% #pr{type = ast,
|
||||
% result = #ast_file{},
|
||||
% rest = []}
|
||||
|
||||
parse_ast_file([]) ->
|
||||
noparse;
|
||||
%{error, #sfc_err_empty_file{}};
|
||||
parse_ast_file(SrcTks0) ->
|
||||
case parse({prs, plus, {ast, top_decl}}, SrcTks0) of
|
||||
#pr{type=prs, result=TopDeclPrs, rest=SrcTks1} ->
|
||||
TopDeclAsts = [Ast || #pr{type=ast, result=Ast} <- TopDeclPrs],
|
||||
AstFile =
|
||||
#ast_file{top_decls = TopDeclAsts},
|
||||
ParseResult = #pr{type = ast,
|
||||
result = AstFile,
|
||||
rest = SrcTks1},
|
||||
ParseResult;
|
||||
Nope ->
|
||||
Nope
|
||||
end.
|
||||
|
||||
|
||||
|
||||
-spec parse_ast_top_decl(SrcTokens) -> Perhaps
|
||||
when SrcTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private
|
||||
% parse a TopDecl from tokens
|
||||
%
|
||||
% on success returns
|
||||
%
|
||||
% #pr{type = ast,
|
||||
% result = #ast_top_decl{},
|
||||
% rest = [_ : #sfc_token{}]}
|
||||
% @end
|
||||
|
||||
parse_ast_top_decl(SrcTks0 = [_ | _]) ->
|
||||
PerhapsTks = parse_tokens(block_item, SrcTks0),
|
||||
case PerhapsTks of
|
||||
#pr{type = tokens, result = []} ->
|
||||
noparse;
|
||||
#pr{type=tokens, result=DeclTks, rest=SrcTks1} ->
|
||||
AstResult =
|
||||
#ast_top_decl{tokens = DeclTks},
|
||||
ParseResult = #pr{type = ast,
|
||||
result = AstResult,
|
||||
rest = SrcTks1},
|
||||
ParseResult;
|
||||
_ ->
|
||||
{error, #sfc_err_nyi{}}
|
||||
end;
|
||||
parse_ast_top_decl([]) ->
|
||||
noparse.
|
||||
|
||||
|
||||
|
||||
-spec parse_tokens(TokensTarget, SrcTokens) -> Result
|
||||
when TokensTarget :: tokens_target(),
|
||||
SrcTokens :: [sfc_token()],
|
||||
Result :: perhaps_parsed().
|
||||
% @private this is for parsing a flat list of tokens
|
||||
%
|
||||
% essentially this is "eat until the next token boundary"
|
||||
%
|
||||
% on success returns
|
||||
%
|
||||
% #pr{type = tokens,
|
||||
% result = [sfc_token()],
|
||||
% rest = [sfc_token()]}
|
||||
% @end
|
||||
|
||||
% need to think carefully about this case;
|
||||
% for now parsing against an empty list of tokens
|
||||
% is always a success
|
||||
parse_tokens(_, []) ->
|
||||
#pr{type = tokens, result = [], rest = []};
|
||||
% WMA SrcTokens is nonempty
|
||||
parse_tokens(block, SrcTokens) ->
|
||||
{_, BlkIndent} = start_pos(SrcTokens),
|
||||
TakeToken = fun(Tk) -> BlkIndent =< indent_level(Tk) end,
|
||||
{BlkTokens, Rest} = tw(TakeToken, SrcTokens),
|
||||
#pr{type = tokens,
|
||||
result = BlkTokens,
|
||||
rest = Rest};
|
||||
parse_tokens(block_item, [H | T]) ->
|
||||
BlkIndent = indent_level(H),
|
||||
TakeToken = fun(Tk) -> BlkIndent < indent_level(Tk) end,
|
||||
{BlockTail, Rest} = tw(TakeToken, T),
|
||||
BlkTokens = [H | BlockTail],
|
||||
#pr{type = tokens,
|
||||
result = BlkTokens,
|
||||
rest = Rest};
|
||||
parse_tokens(_, _) ->
|
||||
{error, #sfc_err_nyi{}}.
|
||||
|
||||
|
||||
% takewhile
|
||||
tw(Pred, List) ->
|
||||
tw(Pred, [], List).
|
||||
|
||||
tw(Pred, Stk, [X | Xs]) ->
|
||||
case Pred(X) of
|
||||
true -> tw(Pred, [X | Stk], Xs);
|
||||
false -> {lists:reverse(Stk), [X | Xs]}
|
||||
end;
|
||||
tw(_, Stk, []) ->
|
||||
{lists:reverse(Stk), []}.
|
||||
|
||||
|
||||
|
||||
-spec parse_prs(Strategy, Target, SrcTokens) -> Perhaps
|
||||
when Strategy :: prs_strategy(),
|
||||
Target :: parse_target(),
|
||||
SrcTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private on success returns
|
||||
%
|
||||
% #pr{type = prs,
|
||||
% result = [_ :: #pr{}]
|
||||
% rest = [_ :: #sfc_token{}]}
|
||||
|
||||
parse_prs(Strategy, Target, SrcTokens) ->
|
||||
case Strategy of
|
||||
star -> parse_prs_star(Target, SrcTokens);
|
||||
plus -> parse_prs_plus(Target, SrcTokens);
|
||||
_ -> {error, #sfc_err_nyi{}}
|
||||
end.
|
||||
|
||||
|
||||
|
||||
-spec parse_prs_star(ParseTarget, SrcTokens) -> Perhaps
|
||||
when ParseTarget :: parse_target(),
|
||||
SrcTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private
|
||||
% parse 0 or more of
|
||||
|
||||
parse_prs_star(_ParseTarget, []) ->
|
||||
#pr{type = prs, result = [], rest = []};
|
||||
parse_prs_star(ParseTarget, SrcTokens) ->
|
||||
parse_prs_star(ParseTarget, [], SrcTokens).
|
||||
|
||||
% WMA SrcTokens nonempty
|
||||
parse_prs_star(ParseTarget, Stack, SrcTokens) ->
|
||||
case parse(ParseTarget, SrcTokens) of
|
||||
#pr{result=NewItem, rest=NewSrcTokens} ->
|
||||
NewStack = [NewItem | Stack],
|
||||
parse_prs_star(ParseTarget,
|
||||
NewStack,
|
||||
NewSrcTokens);
|
||||
noparse ->
|
||||
PRs = lists:reverse(Stack),
|
||||
#pr{type = prs,
|
||||
result = PRs,
|
||||
rest = SrcTokens};
|
||||
Error = {error, _} ->
|
||||
Error
|
||||
end.
|
||||
|
||||
|
||||
|
||||
-spec parse_prs_plus(ParseTarget, SrcTokens) -> Perhaps
|
||||
when ParseTarget :: parse_target(),
|
||||
SrcTokens :: [sfc_token()],
|
||||
Perhaps :: perhaps_parsed().
|
||||
% @private
|
||||
% parse 1 or more of
|
||||
|
||||
parse_prs_plus(ParseTgt, SrcTks0) ->
|
||||
case parse(ParseTgt, SrcTks0) of
|
||||
#pr{result = Hd, rest = SrcTks1} ->
|
||||
parse_prs_star(ParseTgt, [Hd], SrcTks1);
|
||||
Nope ->
|
||||
Nope
|
||||
end.
|
||||
|
||||
|
||||
|
||||
-spec start_pos(Tokens) -> Pos
|
||||
when Tokens :: [sfc_token()],
|
||||
Pos :: sfc_pos().
|
||||
% @private
|
||||
% start position of NONEMPTY list of tokens
|
||||
|
||||
start_pos([#sfc_token{pos = Result} | _]) ->
|
||||
Result.
|
||||
|
||||
|
||||
|
||||
-spec end_pos(Tokens) -> Pos
|
||||
when Tokens :: [sfc_token()],
|
||||
Pos :: sfc_pos().
|
||||
% @private
|
||||
% end position of NONEMPTY list of tokens
|
||||
|
||||
end_pos(Tokens) ->
|
||||
end_pos(Tokens, start_pos(Tokens)).
|
||||
|
||||
end_pos([#sfc_token{string = S} | Ts], Pos) ->
|
||||
NewPos = sfc_tokenizer:new_pos(Pos, S),
|
||||
end_pos(Ts, NewPos);
|
||||
end_pos([], Result) ->
|
||||
Result.
|
||||
|
||||
|
||||
|
||||
-spec indent_level(Token) -> Col
|
||||
when Token :: sfc_token(),
|
||||
Col :: pos_integer().
|
||||
% @private
|
||||
% get col val from token
|
||||
|
||||
indent_level(#sfc_token{pos = {_, C}}) ->
|
||||
C.
|
||||
@@ -0,0 +1,94 @@
|
||||
-module(sfc_group_tokens).
|
||||
|
||||
-export_type([
|
||||
]).
|
||||
|
||||
-export([
|
||||
group_tokens/1
|
||||
]).
|
||||
|
||||
-include("sfc.hrl").
|
||||
|
||||
|
||||
-spec group_tokens(Tokens) -> Result
|
||||
when Tokens :: [sfc_token()],
|
||||
Result :: {ok, Groups}
|
||||
| {error, Reason},
|
||||
Groups :: [TokenGroup],
|
||||
TokenGroup :: sfc_token()
|
||||
| #sfc_tg_list{},
|
||||
Reason :: any().
|
||||
|
||||
group_tokens(Tokens) ->
|
||||
group_tokens([], Tokens).
|
||||
|
||||
|
||||
group_tokens(Stack, Tokens = [_ | _]) ->
|
||||
case token_or_group(Tokens) of
|
||||
{push, Item, NewTokens} ->
|
||||
group_tokens([Item | Stack], NewTokens);
|
||||
Error ->
|
||||
Error
|
||||
end;
|
||||
group_tokens(Stack, []) ->
|
||||
{ok, lists:reverse(Stack)}.
|
||||
|
||||
|
||||
% erlang's type language is lacking here...
|
||||
% items are always a list of sfc_token_group_items
|
||||
% placeholder type
|
||||
-type sfc_tg_item_() :: any().
|
||||
|
||||
% tg = token group
|
||||
% {start_line, indent} should always equal the Pos of the first token
|
||||
-record(sfc_tg_block,
|
||||
{start_line = none :: none | pos_integer(),
|
||||
indent = none :: none | pos_integer(),
|
||||
items = none :: none | {stack, [sfc_tg_item_()]} | [sfc_tg_item_()]}).
|
||||
|
||||
|
||||
% delist = delimited list (...) [...], {...}
|
||||
%
|
||||
% don't even comma-separate values, just make sure
|
||||
-record(sfc_tg_delist,
|
||||
{open = none :: none | '(' | '[' | '{',
|
||||
items = none :: none | {stack, [sfc_tg_item()]} | [sfc_tg_item()],
|
||||
close = none :: none | ')' | ']' | '}',
|
||||
|
||||
-type sfc_tg_item() :: sfc_token(),
|
||||
| #sfc_tg_block{}
|
||||
| #sfc_tg_round{}
|
||||
| #sfc_.
|
||||
|
||||
-type sfc_tg() :: [sfc_tg_item()].
|
||||
|
||||
-spec token_or_group(Tokens) -> Result
|
||||
when Tokens :: [sfc_token()],
|
||||
Result :: {push, Item, NewTokens}
|
||||
| {error, Reason}
|
||||
Item :: sfc_token()
|
||||
| {block, IndentLevel :: pos_integer(),
|
||||
|
||||
,
|
||||
|
||||
token_or_group(Tokens) ->
|
||||
|
||||
|
||||
|
||||
groupist_order() ->
|
||||
[round, square, curly, block].
|
||||
|
||||
|
||||
match_groupists([Groupist | Rest], Tokens) ->
|
||||
case groupist(Grupist, Tokens) of
|
||||
Match = {groupmatch, _, _} ->
|
||||
Match
|
||||
no_groupmatch ->
|
||||
match_groupists(Rest, Tokens);
|
||||
Error ->
|
||||
Error
|
||||
end;
|
||||
match_groupists(_, []) ->
|
||||
{ok, []};
|
||||
match_groupists([], [#sfc_token{pos = Pos} | _}) ->
|
||||
{error, {cannot_make_groupist_decision, Pos}}.
|
||||
@@ -0,0 +1,178 @@
|
||||
% @doc
|
||||
% helper functions for grabbing collections of tokens
|
||||
% off the token stream
|
||||
%
|
||||
% generally assume no whitespace/comment tokens in
|
||||
% input stream
|
||||
-module(sfc_token_chunks).
|
||||
|
||||
-export_type([
|
||||
]).
|
||||
|
||||
-export([
|
||||
barf/2
|
||||
]).
|
||||
|
||||
% $sfc_include is so c() works from sfp eshell
|
||||
-include("$sfc_include/sfc.hrl").
|
||||
|
||||
%------------------------------------------
|
||||
% Types
|
||||
%------------------------------------------
|
||||
|
||||
-type chunk_shape()
|
||||
:: block
|
||||
| block_item
|
||||
| {block_item, Level :: pos_integer()}
|
||||
.
|
||||
|
||||
% FIXME
|
||||
-type choke_reason() :: any().
|
||||
|
||||
|
||||
-spec barf(ChunkShape, SigTokens) -> Perhaps
|
||||
when ChunkShape :: chunk_shape(),
|
||||
SigTokens :: Tokens,
|
||||
Perhaps :: {barf, Taken, Rest}
|
||||
| {choke, Reason},
|
||||
Taken :: Tokens,
|
||||
Rest :: Tokens,
|
||||
Reason :: choke_reason(),
|
||||
Tokens :: [sfc_token()].
|
||||
|
||||
% @doc
|
||||
% slurp/barf terminology comes from paredit mode in
|
||||
% emacs
|
||||
%
|
||||
% slurp ~= accepting input
|
||||
% barf ~= separating input
|
||||
%
|
||||
% slurp: (foo bar) baz ~> (foo bar baz)
|
||||
% barf : (foo bar baz) ~> foo (bar baz)
|
||||
|
||||
barf(_, []) ->
|
||||
{barf, [], []};
|
||||
barf(block, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
|
||||
Take =
|
||||
fun(#sfc_token{pos = {_, TkCol}}) ->
|
||||
BlkCol =< TkCol
|
||||
end,
|
||||
{A, B} = tw(Take, T),
|
||||
{barf, [H | A], B};
|
||||
barf(block_item, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
|
||||
Take =
|
||||
fun(#sfc_token{pos = {_, TkCol}}) ->
|
||||
BlkCol < TkCol
|
||||
end,
|
||||
{A, B} = tw(Take, T),
|
||||
{barf, [H | A], B}.
|
||||
barf({block_item, Level}, Tokens = [#sfc_token{pos = {_, StartLevel}} | _]) ->
|
||||
case Level =:= StartLevel of
|
||||
false -> {barf, [], Tokens};
|
||||
true -> barf(block_item, Tokens)
|
||||
end.
|
||||
barf(_, _) ->
|
||||
{choke, #sfc_err_nyi{}}.
|
||||
|
||||
|
||||
% takewhile but returns {Taken, Rest}
|
||||
tw(Pred, List) ->
|
||||
tw(Pred, [], List).
|
||||
|
||||
tw(Pred, Stk, [X | Xs]) ->
|
||||
case Pred(X) of
|
||||
true -> tw(Pred, [X | Stk], Xs);
|
||||
false -> {lists:reverse(Stk), [X | Xs]}
|
||||
end;
|
||||
tw(_, Stk, []) ->
|
||||
{lists:reverse(Stk), []}.
|
||||
|
||||
|
||||
%-spec consume_delist_round(Tokens) -> Result
|
||||
% when Tokens :: [sfc_token()],
|
||||
% Result :: {ok, {Delist, NewTokens}}
|
||||
% | {error, sfc_err()},
|
||||
% Delist :: Tokens,
|
||||
% NewTokens :: Tokens.
|
||||
%
|
||||
%consume_delist_round(Tokens = [#sfc_token{type = punct, string = "("} | _]) ->
|
||||
% consume_delist(Tokens);
|
||||
%consume_delist_round(Tokens) ->
|
||||
% {ok, {[], Tokens}}.
|
||||
%
|
||||
%
|
||||
%
|
||||
%-spec consume_delist_square(Tokens) -> Result
|
||||
% when Tokens :: [sfc_token()],
|
||||
% Result :: {ok, {Delist, NewTokens}}
|
||||
% | {error, sfc_err()},
|
||||
% Delist :: Tokens,
|
||||
% NewTokens :: Tokens.
|
||||
%
|
||||
%consume_delist_square(Tokens = [#sfc_token{type = punct, string = "["} | _]) ->
|
||||
% consume_delist(Tokens);
|
||||
%consume_delist_square(Tokens) ->
|
||||
% {ok, {[], Tokens}}.
|
||||
%
|
||||
%
|
||||
%
|
||||
%-spec consume_delist_curly(Tokens) -> Result
|
||||
% when Tokens :: [sfc_token()],
|
||||
% Result :: {ok, {Delist, NewTokens}}
|
||||
% | {error, sfc_err()},
|
||||
% Delist :: Tokens,
|
||||
% NewTokens :: Tokens.
|
||||
%
|
||||
%consume_delist_curly(Tokens = [#sfc_token{type = punct, string = "{"} | _]) ->
|
||||
% consume_delist(Tokens);
|
||||
%consume_delist_curly(Tokens) ->
|
||||
% {ok, {[], Tokens}}.
|
||||
%
|
||||
%
|
||||
%
|
||||
%-spec consume_delist(Tokens) -> Result
|
||||
% when Tokens :: [sfc_token()],
|
||||
% Result :: {ok, {Delist, NewTokens}}
|
||||
% | {error, sfc_err()},
|
||||
% Delist :: Tokens,
|
||||
% NewTokens :: Tokens.
|
||||
%
|
||||
%consume_delist([Head | Tail]) ->
|
||||
% case Head of
|
||||
% #sfc_token{type = punct, string = OpenStr = "("} ->
|
||||
% CloseStr = ")",
|
||||
% delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
|
||||
% #sfc_token{type = punct, string = OpenStr = "["} ->
|
||||
% CloseStr = "]",
|
||||
% delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
|
||||
% #sfc_token{type = punct, string = OpenStr = "{"} ->
|
||||
% CloseStr = "}",
|
||||
% delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
|
||||
% _ ->
|
||||
% {ok, {[], [Head | Tail]}}
|
||||
% end;
|
||||
%consume_delist([]) ->
|
||||
% {ok, {[], []}}.
|
||||
%
|
||||
%
|
||||
%% happy path: have a token and we are in the depth of the list
|
||||
%delist_verb(Depth, {OpenStr, CloseStr}, Stack, [Tok | Rest]) when Depth >= 1 ->
|
||||
% NewDepth =
|
||||
% case Tok of
|
||||
% % open token -> increase depth
|
||||
% #sfc_token{type = punct, string = Open} when Open =:= OpenStr ->
|
||||
% Depth + 1;
|
||||
% % close token -> decrease depth
|
||||
% #sfc_token{type = punct, string = Close} when Close =:= CloseStr ->
|
||||
% Depth - 1;
|
||||
% _ ->
|
||||
% Depth
|
||||
% end,
|
||||
% delist_verb(NewDepth, {OpenStr, CloseStr}, [Tok | Stack], Rest);
|
||||
%% we've hit end of available input before a close token
|
||||
%delist_verb(Depth, {OpenStr, _CloseStr}, _Stack, []) when Depth >= 1 ->
|
||||
% {error, #sfc_err{atom = unclosed_delimiter,
|
||||
% string = io_lib:format("unclosed open delimiter: ~p", [OpenStr])}};
|
||||
%% we've popped back out of the list, we're done
|
||||
%delist_verb(_Depth = 0, {_OpenStr, _CloseStr}, Stack, Tokens) ->
|
||||
% {ok, {lists:reverse(Stack), Tokens}}.
|
||||
+6
-9
@@ -374,11 +374,8 @@ tokens_from_iolist(S) -> tokens(S).
|
||||
| {error, gsc_err()},
|
||||
Tokens :: [tk()].
|
||||
% @doc
|
||||
% Recursively parse all tokens off the front end of the string. `Rest' is
|
||||
% the first tail of the string for which no token parser succeeds.
|
||||
%
|
||||
% Semantically, `Rest'` being nonempty amounts to the presence of an illegal
|
||||
% character.
|
||||
% Recursively parse all tokens off the front end of the
|
||||
% string.
|
||||
|
||||
tokens(S) ->
|
||||
% defensive normalization
|
||||
@@ -525,7 +522,7 @@ slurp_token(Pos, SrcStr) ->
|
||||
Rest :: string().
|
||||
% @doc
|
||||
% grab a single token off the front of the string according to
|
||||
% `token_shapes_parse_order/0'
|
||||
% `ParseOrder'
|
||||
|
||||
slurp_token_shapes([TokenType | TTs], Pos, SrcStr) ->
|
||||
case slurp_token_of_shape(TokenType, Pos, SrcStr) of
|
||||
@@ -562,9 +559,9 @@ slurp_token_of_shape(lcom, Pos, SrcStr) ->
|
||||
case SrcStr of
|
||||
"//" ++ _ ->
|
||||
{Line, Rest} = takeline("", SrcStr),
|
||||
Token = #tk{shape = lcom,
|
||||
pos = Pos,
|
||||
str = Line},
|
||||
Token = #tk{shape = lcom,
|
||||
pos = Pos,
|
||||
str = Line},
|
||||
{tokmatch, Token, Rest};
|
||||
_ ->
|
||||
no_tokmatch
|
||||
|
||||
Reference in New Issue
Block a user