wip name cleanups

This commit is contained in:
2026-06-01 18:00:37 -07:00
parent f548c7d88d
commit 9da6dbf18d
12 changed files with 804 additions and 281 deletions
+25 -25
View File
@@ -4,15 +4,15 @@
% based on original sophia compiler
%
% parse layers:
% 1. sfc_tokenizer: SrcStr -> (Tokens | SigTokens)
% 1. gsc_tokenizer: SrcStr -> (Tokens | SigTokens)
%
% SigTokens = not comment/whitespace
%
% layers:
% a. sfc_strmatch : matches string shapes
% b. sfc_so_scan : converts to so_scan shapes
% a. gsc_strmatch : matches string shapes
% b. gsc_so_scan : converts to so_scan shapes
%
% 2. sfc_ast: SigTokens -> AST
% 2. gsc_ast: SigTokens -> AST
%
% terminology:
%
@@ -36,8 +36,8 @@
% - too fuzzy right now
% - possibly:
% - rename parser layers sequentially:
% - sfc_
-module(sfc).
% - gsc_
-module(gsc).
-export_type([
@@ -54,13 +54,13 @@
ast_from_tokens/1
]).
-include("$sfc_include/sfc.hrl").
-include("$gsc_include/gsc.hrl").
%-----------------------------------------
% types
%-----------------------------------------
-type token() :: sfc_token().
-type token() :: tk().
%-----------------------------------------
% functions
@@ -68,13 +68,13 @@
sigtokens_from_file(X) ->
case tokens_from_file(X) of
{ok, Y} -> {ok, sfc_tokens:filter_significant(Y)};
{ok, Y} -> {ok, gsc_tokens:filter_significant(Y)};
Err -> Err
end.
sigtokens_from_string(X) ->
case tokens_from_string(X) of
{ok, Y} -> {ok, sfc_tokens:filter_significant(Y)};
{ok, Y} -> {ok, gsc_tokens:filter_significant(Y)};
Err -> Err
end.
@@ -82,8 +82,8 @@ sigtokens_from_string(X) ->
-spec tokens_from_file(FilePath) -> Perhaps
when FilePath :: string(),
Perhaps :: {ok, Tokens}
| {error, sfc_err() | any()},
Tokens :: [sfc_token()].
| {error, gsc_err() | any()},
Tokens :: [tk()].
tokens_from_file(FilePath) ->
case file:read_file(FilePath) of
@@ -97,18 +97,18 @@ tokens_from_file(FilePath) ->
-spec tokens_from_string(SrcStr) -> Result
when SrcStr :: string(),
Result :: {ok, Tokens}
| {error, sfc_err()},
Tokens :: [sfc_token()].
| {error, gsc_err()},
Tokens :: [tk()].
tokens_from_string(SrcStr) ->
sfc_tokens:tokens(SrcStr).
gsc_tokens:tokens(SrcStr).
-spec ast_from_file(FilePath) -> Perhaps
when FilePath :: string(),
Perhaps :: {ok, AST} | {error, sfc_err()},
AST :: sfc_ast().
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_file(FilePath) ->
case file:read_file(FilePath) of
@@ -120,11 +120,11 @@ ast_from_file(FilePath) ->
-spec ast_from_string(SrcStr) -> Perhaps
when SrcStr :: string(),
Perhaps :: {ok, AST} | {error, sfc_err()},
AST :: sfc_ast().
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_string(SrcStr) ->
case sfc_tokens:significant_tokens(SrcStr) of
case gsc_tokens:significant_tokens(SrcStr) of
{ok, SigTks} -> ast_from_tokens(SigTks);
Error -> Error
end.
@@ -132,13 +132,13 @@ ast_from_string(SrcStr) ->
-spec ast_from_tokens(SrcTokens) -> Perhaps
when SrcTokens :: [sfc_token()],
Perhaps :: {ok, AST} | {error, sfc_err()},
AST :: sfc_ast().
when SrcTokens :: [tk()],
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_tokens(Tks) ->
SigTks = sfc_tokens:filter_significant(Tks),
case sfc_ast:gulp_file(SigTks) of
SigTks = gsc_tokens:filter_significant(Tks),
case gsc_ast:gulp_file(SigTks) of
{gulp, AST} -> {ok, AST};
Error -> Error
end.
+54 -54
View File
@@ -56,7 +56,7 @@
% Domain ::= Type // Single argument
% | '(' Sep(Type, ',') ')' // Multiple arguments
% ```
-module(sfc_ast).
-module(gsc_ast).
%-compile([export_all,nowarn_export_all]).
%
@@ -71,16 +71,16 @@
% gulp_file/1
%]).
%
%-include("$sfc_include/sfc.hrl").
%-include("$gsc_include/gsc.hrl").
%
%%%-----------------------------
%%% TYPES: sfc_ast
%%% TYPES: gsc_ast
%%%-----------------------------
%
%
%% % placeholders
%-type ast_() :: any().
%-record(ast_nyi, {tokens = none :: [sfc_token()]}).
%-record(ast_nyi, {tokens = none :: [tk()]}).
%-type ast_nyi() :: #ast_nyi{}.
%
%
@@ -109,8 +109,8 @@
%% ta = type alias
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%-record(ast_ta,
% {alias = none :: none | sfc_token(),
% tvars = none :: none | [sfc_token()],
% {alias = none :: none | tk(),
% tvars = none :: none | [tk()],
% points_to = none :: none | ast_type_expr()}).
%
%% Decl ::= 'type' Id ['(' TVar* ')'] '=' TypeAlias
@@ -126,8 +126,8 @@
%-record(ast_ct,
% {payable = none :: none | boolean(),
% main = none :: none | boolean(),
% name = none :: none | sfc_token(),
% impls = none :: none | [sfc_token()],
% name = none :: none | tk(),
% impls = none :: none | [tk()],
% decls = none :: none | [decl()]}).
%
%-record(ast_td_iface,{}).
@@ -158,7 +158,7 @@
%
%-type parse_error_() :: any().
%-record(parse_error,
% {pos = none :: none | sfc_pos(),
% {pos = none :: none | gsc_pos(),
% msg = "" :: string(),
% subs = [] :: [parse_error_()],
% extra = none :: any()}).
@@ -189,20 +189,20 @@
%%%-----------------------------
%
%-spec gulp_file(Tokens) -> Perhaps
% when Tokens :: [sfc_token()],
% when Tokens :: [tk()],
% Perhaps :: {gulp, #ast_file{}}
% | {error, #parse_error{}}.
%
%gulp_file([]) ->
% {error, empty_file};
%gulp_file(Tokens) ->
% case sfc_tokens:take_block(Tokens) of
% case gsc_tokens:take_block(Tokens) of
% {Tokens, []} ->
% gulp_block(fun gulp_top_decl/1, Tokens);
% %gulp_file2([], [], Tokens);
% {A, B} ->
% StartPos = sfc_tokens:start_pos(A),
% ErrPos = sfc_tokens:start_pos(B),
% StartPos = gsc_tokens:start_pos(A),
% ErrPos = gsc_tokens:start_pos(B),
% Msg = efmt("gulp_file: block starting at ~p ends at ~p instead of EOF",
% [StartPos, ErrPos]),
% {error, #parse_error{pos = ErrPos, msg = Msg}}
@@ -212,7 +212,7 @@
%
%%gulp_file2(AccOks, AccErrs, Tokens = [_ | _]) ->
%% % ItemTokens will be nonempty
%% {ItemTokens, NewTokens} = sfc_tokens:take_block_item(Tokens),
%% {ItemTokens, NewTokens} = gsc_tokens:take_block_item(Tokens),
%% case gulp_top_decl(ItemTokens) of
%% {gulp, Ok} -> gulp_file2([Ok | AccOks], AccErrs, NewTokens);
%% Err -> gulp_file2(AccOks, [Err | AccErrs], NewTokens)
@@ -227,7 +227,7 @@
%
%-spec gulp_block(GulpItem, Tokens) -> GulpedItems
% when GulpItem :: fun((ItemTokens) -> GulpedItem),
% Tokens :: [sfc_token()],
% Tokens :: [tk()],
% ItemTokens :: Tokens,
% GulpedItem :: {gulp, Item} | {error, Reason},
% GulpedItems :: {gulp, Items} | {error, Reason},
@@ -258,7 +258,7 @@
%
%gulp_block(GulpItem, AccOks, AccErrs, Tokens = [_ | _]) ->
% % ItemTokens will be nonempty
% {ItemTokens, NewTokens} = sfc_tokens:take_block_item(Tokens),
% {ItemTokens, NewTokens} = gsc_tokens:take_block_item(Tokens),
% case GulpItem(ItemTokens) of
% {gulp, Ok} -> gulp_block(GulpItem, [Ok | AccOks], AccErrs, NewTokens);
% Err -> gulp_block(GulpItem, AccOks, [Err | AccErrs], NewTokens)
@@ -272,7 +272,7 @@
%
%
%-spec gulp_top_decl(DeclTokens) -> Result
% when DeclTokens :: [sfc_token()],
% when DeclTokens :: [tk()],
% Result :: {gulp, ast()} | {error, any()}.
%
%% @doc
@@ -284,7 +284,7 @@
%% | Using
%% @end
%gulp_top_decl(DeclTokens) ->
% case sfc_tokens:strings(3, DeclTokens) of
% case gsc_tokens:strings(3, DeclTokens) of
% ["payable", "contract", "interface"] ->
% gulp_nyi(DeclTokens);
% ["contract", "interface" | _] ->
@@ -311,7 +311,7 @@
% gulp_nyi(DeclTokens);
% _ ->
% % decl tokens will always be nonempty
% [H = #sfc_token{pos = ErrPos} | _] = DeclTokens,
% [H = #tk{pos = ErrPos} | _] = DeclTokens,
% EMsg = efmt("gulp_top_decl: bad token: ~p; "
% "expecting one of: [payable, main, contract, namespace, @compiler, include, using]",
% [H]),
@@ -320,7 +320,7 @@
% end.
%
%
%gulp_ct2(Ast, Tokens = [#sfc_token{pos = Pos} | _]) ->
%gulp_ct2(Ast, Tokens = [#tk{pos = Pos} | _]) ->
% % need to pass through pos for error messages on premature end of
% % input... for now it's a fixme thing
% gulp_ct3(Ast, Tokens, Pos).
@@ -331,7 +331,7 @@
%gulp_ct3(Ast = #ast_ct{name = none}, Tokens, Pos) ->
% NewPos = Pos, %% fixme
% case Tokens of
% [Name = #sfc_token{type = con} | NewTokens] ->
% [Name = #tk{type = con} | NewTokens] ->
% NewAst = Ast#ast_ct{name = Name},
% gulp_ct3(NewAst, NewTokens, NewPos);
% [Name | _] ->
@@ -355,7 +355,7 @@
%% contract X : Y, Z, W = <block>
%% ^
%gulp_ct3(Ast = #ast_ct{decls = none},
% [#sfc_token{string = "="} | Tokens],
% [#tk{string = "="} | Tokens],
% _Pos) ->
% case gulp_block(fun gulp_decl/1, Tokens) of
% {gulp, Decls} ->
@@ -370,7 +370,7 @@
%
%
%-spec slurp_ct_impls(Tokens) -> Slurped
% when Tokens :: [sfc_token()],
% when Tokens :: [tk()],
% Slurped :: {slurp, Impls, NewTokens}
% | {error, Reason},
% Impls :: Tokens,
@@ -380,11 +380,11 @@
%
%% FIXME: this should be a restructured a tiny bit for better error
%% handling
%slurp_ct_impls([_ = #sfc_token{string = ":"},
% Con = #sfc_token{type = con}
%slurp_ct_impls([_ = #tk{string = ":"},
% Con = #tk{type = con}
% | NewTokens]) ->
% slurp_ct_impls2([Con], NewTokens);
%slurp_ct_impls(NewTokens = [#sfc_token{string = "="} | _]) ->
%slurp_ct_impls(NewTokens = [#tk{string = "="} | _]) ->
% {slurp, [], NewTokens};
%slurp_ct_impls([BadToken | _]) ->
% Msg = efmt("slurp_ct_impls: expecting (: Con | =), got ~p", [BadToken]),
@@ -393,11 +393,11 @@
% {error, {fixme, "expecting : or =, got end-of-input"}}.
%
%
%slurp_ct_impls2(Stk, [_ = #sfc_token{string = ","},
% Con = #sfc_token{type = con}
%slurp_ct_impls2(Stk, [_ = #tk{string = ","},
% Con = #tk{type = con}
% | NewTokens]) ->
% slurp_ct_impls2([Con | Stk], NewTokens);
%slurp_ct_impls2(Stk, NewTokens = [#sfc_token{string = "="} | _]) ->
%slurp_ct_impls2(Stk, NewTokens = [#tk{string = "="} | _]) ->
% {slurp, lists:reverse(Stk), NewTokens};
%slurp_ct_impls2(Stk, BadTokens) ->
% {error, {fixme, nyi, slurp_ct_impls2, [Stk, BadTokens]}}.
@@ -410,7 +410,7 @@
%% | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
%% | Using
%gulp_decl(Tokens) ->
% case sfc_tokens:strings(1, Tokens) of
% case gsc_tokens:strings(1, Tokens) of
% ["type"] -> gulp_type_alias(Tokens);
% _ -> gulp_nyi(Tokens)
% end.
@@ -423,7 +423,7 @@
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% ^
%gulp_ta(Ast = #ast_ta{alias = none},
% _ = [#sfc_token{string = "type"}, Alias = #sfc_token{type = id}
% _ = [#tk{string = "type"}, Alias = #tk{type = id}
% | NewTokens]) ->
% NewAst = Ast#ast_ta{alias = Alias},
% gulp_ta(NewAst, NewTokens);
@@ -440,7 +440,7 @@
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% ^
%gulp_ta(Ast = #ast_ta{points_to = none},
% _ = [#sfc_token{string = "="} | NewTokens]) ->
% _ = [#tk{string = "="} | NewTokens]) ->
% case gulp_type_expr(NewTokens) of
% {gulp, TypeExpr} ->
% Result = Ast#ast_ta{points_to = TypeExpr},
@@ -455,25 +455,25 @@
%% ^
%%
%% "(bar, baz) = ..." ~> {slurp, [bar, baz], "= ..."}
%slurp_ta_tvars(Tks = [#sfc_token{string = "="} | _]) ->
%slurp_ta_tvars(Tks = [#tk{string = "="} | _]) ->
% {slurp, [], Tks};
%slurp_ta_tvars([#sfc_token{string = "("},
% #sfc_token{string = ")"}
%slurp_ta_tvars([#tk{string = "("},
% #tk{string = ")"}
% | NewTokens]) ->
% {slurp, [], NewTokens};
%slurp_ta_tvars([_ = #sfc_token{string = "("},
% TVar = #sfc_token{type = tvar}
%slurp_ta_tvars([_ = #tk{string = "("},
% TVar = #tk{type = tvar}
% | NewTokens]) ->
% slurp_tavars([TVar], NewTokens).
%
%
%slurp_tavars(Stk,
% [_ = #sfc_token{string = ","},
% TVar = #sfc_token{type = tvar}
% [_ = #tk{string = ","},
% TVar = #tk{type = tvar}
% | NewTks]) ->
% slurp_tavars([TVar | Stk], NewTks);
%slurp_tavars(Stk,
% [#sfc_token{string = ")"}
% [#tk{string = ")"}
% | NewTks]) ->
% {slurp, lists:reverse(Stk), NewTks}.
%
@@ -536,7 +536,7 @@
%% (X) // legal, equiv to X
%% (X, Y) // illegal, should be X * Y
%%
%slurp_type_expr(Tks = [#sfc_token{pos = Pos} | _]) ->
%slurp_type_expr(Tks = [#tk{pos = Pos} | _]) ->
% case slurp_type1(Tks) of
% % plist cases
% % function types _ => _
@@ -546,7 +546,7 @@
% % () => _
% % (_) => _
% % (_, _) => _
% {DomainType, [#sfc_token{string = "=>"} | After]} ->
% {DomainType, [#tk{string = "=>"} | After]} ->
% case slurp_type_expr(After) of
% {slurp, CodomainType, NewTks} ->
% TypeExpr = #ast_te_fn{dom = DomainType,
@@ -572,7 +572,7 @@
% {slurp, {token, Tk}, NewTks} ->
% case NewTks of
% % foo => bar is fine, normalized to (foo) => bar
% [#sfc_token{string = "=>"} | After] ->
% [#tk{string = "=>"} | After] ->
% case slurp_type_expr(After) of
% {slurp, Codom, NewAfter} ->
% ArgType = #ast_te_tk{token = ArgTypeTk},
@@ -600,7 +600,7 @@
% % ["("] -> error(nyi);
% % ["*"] -> error(nyi);
% %case After of
% % [#sfc_token{string = "("} | _] ->
% % [#tk{string = "("} | _] ->
% % case slurp_type1_II(After) of
% % {slurp, {plist, ArgTypes}, NewAfter} ->
% % {slurp, #ast_te_ap{fn = FirstType
@@ -609,12 +609,12 @@
%
%
%% Type1 = {plist, Types} () (foo) (foo, bar)
%% | {token, #sfc_token{}} foo Bar.baz 'quux
%% | {token, #tk{}} foo Bar.baz 'quux
%slurp_type1(Tks) ->
% case sfc_tokens:slurp_plist(Tks) of
% case gsc_tokens:slurp_plist(Tks) of
% % head token is NOT open paren -> must be id/qid/tvar
% {slurp, [], [Tk | NewTks]} ->
% TkType = Tk#sfc_token.type,
% TkType = Tk#tk.type,
% case TkType of
% id -> {slurp, {token, Tk}, NewTks};
% qid -> {slurp, {token, Tk}, NewTks};
@@ -633,10 +633,10 @@
%
%
%%slurp_type_expr_plist(Tks) ->
%% case sfc_tokens:slurp_plist(Tks) of
%% case gsc_tokens:slurp_plist(Tks) of
%% % head token is NOT open paren -> must be id/qid/tvar
%% {slurp, [], [Tk | NewTks]} ->
%% TkType = Tk#sfc_token.type,
%% TkType = Tk#tk.type,
%% case TkType of
%% id -> {slurp, {token, Tk}, NewTks};
%% qid -> {slurp, {token, Tk}, NewTks};
@@ -653,23 +653,23 @@
%% Error = {error, _} -> Error
%% end.
%
%gulp_ptype1([#sfc_token{string = "("}, #sfc_token{string = ")"}]) ->
%gulp_ptype1([#tk{string = "("}, #tk{string = ")"}]) ->
% {gulp, []};
%gulp_ptype1([#sfc_token{string = "("} | Tail]) ->
%gulp_ptype1([#tk{string = "("} | Tail]) ->
% gulp_ptype1_II([], Tail).
%
%gulp_ptype1_II(Stk, Tks) ->
% case slurp_type_expr(Tks) of
% {slurp, NewType, [#sfc_token{string = ")"}]} ->
% {slurp, NewType, [#tk{string = ")"}]} ->
% {gulp, lists:reverse([NewType | Stk])};
% {slurp, NewType, [#sfc_token{string = ","} | NewTks]} ->
% {slurp, NewType, [#tk{string = ","} | NewTks]} ->
% gulp_ptype1_II([NewType | Stk], NewTks);
% Error = {error, _} ->
% Error
% end.
%
%
%%gulp_te_tk([Tk = #sfc_token{type = TkType}])
%%gulp_te_tk([Tk = #tk{type = TkType}])
%% when id =:= TkType;
%% qid =:= TkType;
%% tvar =:= TkType ->
+5 -5
View File
@@ -1,5 +1,5 @@
% @doc
% sfc_bst = ast second attempt but prefix so tab complete
% gsc_bst = ast second attempt but prefix so tab complete
%
% from docs/sophia/so_syntax.md:
%
@@ -38,12 +38,12 @@
% FModifier ::= 'stateful' | 'private'
%
% Args ::= '(' Sep(Pattern, ',') ')'
-module(sfc_bst).
-module(gsc_bst).
-compile([export_all, nowarn_export_all]).
-include("$sfc_include/sfc.hrl").
-include("$gsc_include/gsc.hrl").
%-record(bst_nyi, {tokens :: [sfc_token()]).
%-record(bst_nyi, {tokens :: [tk()]).
%
%% ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
%-record(bst_ct,
@@ -73,7 +73,7 @@
%% | '@compiler' PragmaOp Version
%% | 'include' String
%% | Using
%gulp(top_decl, [#sfc_token{string = S} | Rest]) ->
%gulp(top_decl, [#tk{string = S} | Rest]) ->
% case strings(3, Tokens) of
% ["payable", "contract", "interface"] ->
% gulp_ct(#bst_iface{payable = true, main = true}, drop(3, Tokens));
+23 -23
View File
@@ -1,4 +1,4 @@
-module(sfc_parse_type_expr).
-module(gsc_parse_type_expr).
-export_type([
]).
@@ -9,7 +9,7 @@
take_until_ifx_op/1
]).
-include("$sfc_include/sfc.hrl").
-include("$gsc_include/gsc.hrl").
%------------------------------------------------------
@@ -17,18 +17,18 @@
%------------------------------------------------------
-type vtk_ifx_op() :: vtk_apply_to
| {'vtk_*', sfc_token()}
| {'vtk_=>', sfc_token()}.
| {'vtk_*', tk()}
| {'vtk_=>', tk()}.
-type vtk() :: sfc_token()
| {vtk_plist, [sfc_token()]}
-type vtk() :: tk()
| {vtk_plist, [tk()]}
| vtk_ifx_op().
-type gulped(X) :: {gulp, X}
| {error, any()}.
-type slurped(X) :: {slurp, X, Rest :: [sfc_token()]}
-type slurped(X) :: {slurp, X, Rest :: [tk()]}
| {error, any()}.
@@ -43,7 +43,7 @@
% @doc for testing
unsafe_vtks_from_string(S) ->
{ok, SigTks} = sfc_tokens:significant_tokens(S),
{ok, SigTks} = gsc_tokens:significant_tokens(S),
{gulp, Vtks} = gulp_vtks(SigTks),
Vtks.
@@ -79,9 +79,9 @@ unsafe_vtks_from_string(S) ->
-record(ast_parens,
{open = none :: none | sfc_token(),
inner = none :: none | [sfc_token()],
close = none :: none | sfc_token()}).
{open = none :: none | tk(),
inner = none :: none | [tk()],
close = none :: none | tk()}).
chunk_by(Strategy, Tokens) ->
chunk_by(Strategy, [], Tokens).
@@ -91,7 +91,7 @@ chunk_by(Strategy, Tokens) ->
Strategy :: chunk_strategy(),
Oks :: [any()],
Errs :: [{error, Reason :: any()}],
Tokens :: [sfc_token()],
Tokens :: [tk()],
Result :: {ok,
gulp_chunks_by(_, Stk, [], []) ->
@@ -109,8 +109,8 @@ gulp_chunks_by(plist, Stk, Errs, Tokens) ->
gulp_chunks_by(plist, Stk, [Error | Errs], Tokens);
end.
slurp_plist_rec(Tokens = [#sfc_token{string = "(" | _]) ->
case sfc_tokens:slurp_plist(Tokens) of
slurp_plist_rec(Tokens = [#tk{string = "(" | _]) ->
case gsc_tokens:slurp_plist(Tokens) of
{slurp, [], _} ->
barf;
{slurp, PTokens, NewTokens} ->
@@ -118,18 +118,18 @@ slurp_plist_rec(Tokens = [#sfc_token{string = "(" | _]) ->
end;
%-spec gulp_ifx_tree(Tokens) -> gulped(IfxTree) when
% Tokens :: [sfc_token()],
% Tokens :: [tk()],
% IfxTree :: ifx_tree().
%
%-spec chunk_by(ChunkStrategy, Tokens) -> Result when
% ChunkStrategy :: chunk_strategy(),
% Tokens :: [sfc_token()],
% Tokens :: [tk()],
% Result :: {ChunkStrategy,
-spec gulp_vtks(Tokens) -> Result when
Tokens :: [sfc_token()],
Tokens :: [tk()],
Result :: gulped(VirtualTokens),
VirtualTokens :: [vtk()].
@@ -155,8 +155,8 @@ gulp_vtks(Acc, Tks0) ->
% ~> [..., foo, {plist, "(bar, baz)"}, ...]
{_Pfx = Tks1_BeforeOpen,
_Sfx = Tks2_OpenNAfter
= [#sfc_token{string = "("} | _]} ->
case sfc_tokens:slurp_plist(Tks2_OpenNAfter) of
= [#tk{string = "("} | _]} ->
case gsc_tokens:slurp_plist(Tks2_OpenNAfter) of
{slurp, Tks2A_OpenToClose, Tks2B_AfterClose} ->
NewAcc = [Acc,
Tks1_BeforeOpen,
@@ -168,7 +168,7 @@ gulp_vtks(Acc, Tks0) ->
end;
% product
{_Pfx = Tks0_BeforeTimes,
_Sfx = [ Tk1A_Times = #sfc_token{string = "*"}
_Sfx = [ Tk1A_Times = #tk{string = "*"}
| Tks1B_AfterTimes]} ->
NewAcc = [Acc,
Tks0_BeforeTimes,
@@ -176,7 +176,7 @@ gulp_vtks(Acc, Tks0) ->
gulp_vtks(NewAcc, Tks1B_AfterTimes);
% funType
{_Pfx = Tks0_BeforeOp,
_Sfx = [ Tk1A_Op = #sfc_token{string = "=>"}
_Sfx = [ Tk1A_Op = #tk{string = "=>"}
| Tks1B_AfterOp]} ->
NewAcc = [Acc,
Tks0_BeforeOp,
@@ -187,7 +187,7 @@ gulp_vtks(Acc, Tks0) ->
-spec take_until_ifx_op(Tokens) -> Result when
Tokens :: [sfc_token()],
Tokens :: [tk()],
Result :: {Taken, NewTokens},
Taken :: Tokens,
NewTokens :: Tokens.
@@ -200,7 +200,7 @@ take_until_ifx_op(Tks) ->
take_until_ifx_op(Stack, []) ->
{lists:reverse(Stack), []};
take_until_ifx_op(Stack, Tokens = [Token | NewTokens]) ->
TokStr = Token#sfc_token.string,
TokStr = Token#tk.string,
Continue =
case TokStr of
% exit cases
+2 -2
View File
@@ -65,12 +65,12 @@
% KW = string:join(Keywords, "|"),
%
% There is a lot going on in that code. This is purely the part that matches
% strings specifically, . The *tokenizer* (sfc_tokenizer) knows the hierarchy
% strings specifically, . The *tokenizer* (gsc_tokenizer) knows the hierarchy
% of sophia tokens (e.g. it knows to match keywords before identifiers, so that
% `contract` gets tokenized as a keyword and not a variable name), and then
% calls into this module in order to match the string shape it's looking for.
% @end
-module(sfc_strmatch).
-module(gsc_strmatch).
%-compile([export_all, nowarn_export_all]).
+94 -94
View File
@@ -7,16 +7,16 @@
% For MVP it mimics the behavior of so_scan exactly, in terms of like what its
% definition of a token is and so on.
%
% sfc_so_scan.erl contains a compatibility layer that should agree with so_scan
% gsc_so_scan.erl contains a compatibility layer that should agree with so_scan
% exactly. It converts the data types here to the shapes that so_scan outputs.
%
% This is for two reasons:
%
% 1. in order to enable testing the two modules against each other, and
% 2. to future-proof in case we decide to incrementally incorporate the sfc
% 2. to future-proof in case we decide to incrementally incorporate the gsc
% code into the legacy sophia compiler
% @end
-module(sfc_tokens).
-module(gsc_tokens).
% meta
-export([
@@ -46,7 +46,7 @@
new_pos/2
]).
-include("$sfc_include/sfc.hrl").
-include("$gsc_include/gsc.hrl").
%=======================================================
@@ -55,11 +55,11 @@
-spec strings(N, Tokens) -> AtMostNStrings
when N :: non_neg_integer(),
Tokens :: [sfc_token()],
Tokens :: [tk()],
AtMostNStrings :: [string()].
% @doc return the strings of the first N tokens
strings(N, [#sfc_token{string = S} | Rest]) when is_integer(N), N >= 1 ->
strings(N, [#tk{string = S} | Rest]) when is_integer(N), N >= 1 ->
[S | strings(N-1, Rest)];
strings(_, []) ->
[];
@@ -84,16 +84,16 @@ strings(0, _) ->
% ...
-spec take_block(Tokens) -> {BlockTokens, Rest}
when Tokens :: [sfc_token()],
when Tokens :: [tk()],
BlockTokens :: Tokens,
Rest :: Tokens.
% @doc
% takes all tokens whose column position is >= the column position of
% the head token
take_block([H = #sfc_token{pos = {_, BlkCol}} | T]) ->
take_block([H = #tk{pos = {_, BlkCol}} | T]) ->
TokenInBlock =
fun(#sfc_token{pos = {_, TkCol}}) ->
fun(#tk{pos = {_, TkCol}}) ->
BlkCol =< TkCol
end,
take_while(TokenInBlock, [H], T);
@@ -103,16 +103,16 @@ take_block([]) ->
-spec take_block_item(Tokens) -> {ItemTokens, Rest}
when Tokens :: [sfc_token()],
when Tokens :: [tk()],
ItemTokens :: Tokens,
Rest :: Tokens.
% @doc
% takes all tokens whose column position is > the column position of
% the head token
take_block_item([H = #sfc_token{pos = {_, ItemCol}} | T]) ->
take_block_item([H = #tk{pos = {_, ItemCol}} | T]) ->
TokenInItem =
fun(#sfc_token{pos = {_, TkCol}}) ->
fun(#tk{pos = {_, TkCol}}) ->
ItemCol < TkCol
end,
take_while(TokenInItem, [H], T);
@@ -128,7 +128,7 @@ take_block_item([]) ->
Mismatch :: {fixme, mismatch, OpenStack, ClosedBy},
OpenStack :: Tokens,
ClosedBy :: none | {value, Token},
Token :: sfc_token().
Token :: tk().
% @doc
% the verbiage here is `slurp' rather than `take' because we insist on
@@ -159,7 +159,7 @@ take_block_item([]) ->
% counterintuitive to end-users (who are programmers, entirely
% unfamiliar with notions like stacks and open/close delimiters)
slurp_plist([Hd = #sfc_token{string = "("} | Tl]) ->
slurp_plist([Hd = #tk{string = "("} | Tl]) ->
slurp_dlist([Hd], [Hd], Tl);
slurp_plist(Tks) ->
{slurp, [], Tks}.
@@ -170,30 +170,30 @@ slurp_dlist(All, [], NewTokens) ->
{slurp, lists:reverse(All), NewTokens};
% WMA stack is nonempty
% happy cases of opens getting popped
slurp_dlist(All, [#sfc_token{string = "("} | NewOpen],
[#sfc_token{string = ")"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{string = "("} | NewOpen],
[#tk{string = ")"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
slurp_dlist(All, [#sfc_token{string = "["} | NewOpen],
[#sfc_token{string = "]"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{string = "["} | NewOpen],
[#tk{string = "]"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
slurp_dlist(All, [#sfc_token{string = "{"} | NewOpen],
[#sfc_token{string = "}"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{string = "{"} | NewOpen],
[#tk{string = "}"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
% happy: open delimiters getting pushed
slurp_dlist(All, Opens, [#sfc_token{string = "("} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{string = "("} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
slurp_dlist(All, Opens, [#sfc_token{string = "["} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{string = "["} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
slurp_dlist(All, Opens, [#sfc_token{string = "{"} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{string = "{"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
% sad: mismatch cases
slurp_dlist(All, Opens, []) ->
{error, {fixme, mismatch, Opens, none}};
slurp_dlist(All, Opens, [#sfc_token{string = "}"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{string = "}"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
slurp_dlist(All, Opens, [#sfc_token{string = "]"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{string = "]"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
slurp_dlist(All, Opens, [#sfc_token{string = ")"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{string = ")"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
% general case: non-terminal token gets pushed
slurp_dlist(All, Opens, [Tk | NewTks]) ->
@@ -211,7 +211,7 @@ slurp_dlist(All, Opens, [Tk | NewTks]) ->
% ]).
%-------------------------------------------------------
-spec token_types_parse_order() -> [sfc_token_type()].
-spec token_types_parse_order() -> [gsc_token_type()].
% @doc
% list of sophia tokens in parse order (if an earlier type matches, the later
% type isn't even checked)
@@ -288,9 +288,9 @@ kwds() ->
%-------------------------------------------------------
% Token accessors
-spec indent_level(sfc_token()) -> pos_integer().
-spec indent_level(tk()) -> pos_integer().
indent_level(#sfc_token{pos = {_, IndentLevel}}) ->
indent_level(#tk{pos = {_, IndentLevel}}) ->
IndentLevel.
@@ -298,8 +298,8 @@ indent_level(#sfc_token{pos = {_, IndentLevel}}) ->
-spec significant_tokens(SrcStr) -> Result
when SrcStr :: iolist(),
Result :: {ok, Tokens}
| {error, sfc_err()},
Tokens :: [sfc_token()].
| {error, gsc_err()},
Tokens :: [tk()].
significant_tokens(SrcStr) ->
case tokens(SrcStr) of
@@ -312,7 +312,7 @@ significant_tokens(SrcStr) ->
-spec filter_significant(Tokens) -> SignificantTokens
when Tokens :: [sfc_token()],
when Tokens :: [tk()],
SignificantTokens :: Tokens.
filter_significant(Tokens) ->
@@ -321,19 +321,19 @@ filter_significant(Tokens) ->
-spec is_significant(Token) -> boolean()
when Token :: sfc_token().
when Token :: tk().
is_significant(#sfc_token{type = bcom}) -> false;
is_significant(#sfc_token{type = lcom}) -> false;
is_significant(#sfc_token{type = ws}) -> false;
is_significant(#tk{type = bcom}) -> false;
is_significant(#tk{type = lcom}) -> false;
is_significant(#tk{type = ws}) -> false;
is_significant(_) -> true.
-spec tokens(SrcStr) -> Result
when SrcStr :: iolist(),
Result :: {ok, Tokens}
| {error, sfc_err()},
Tokens :: [sfc_token()].
| {error, gsc_err()},
Tokens :: [tk()].
% @doc
% Recursively parse all tokens off the front end of the string. `Rest' is
% the first tail of the string for which no token parser succeeds.
@@ -349,13 +349,13 @@ tokens(Stack, _FinalPos, "") ->
{ok, lists:reverse(Stack)};
tokens(Stack, Pos, SrcStr) ->
case slurp_token(Pos, SrcStr) of
{tokmatch, NewToken = #sfc_token{string = TokStr},
{tokmatch, NewToken = #tk{string = TokStr},
NewSrcStr} ->
NewPos = new_pos(Pos, TokStr),
tokens([NewToken | Stack], NewPos, NewSrcStr);
no_tokmatch ->
PrevTokens = lists:reverse(Stack),
Err = #sfc_err_no_tokmatch{prev_tokens = PrevTokens,
Err = #gsc_err_no_tokmatch{prev_tokens = PrevTokens,
break_pos = Pos,
rest = SrcStr},
{error, Err};
@@ -367,7 +367,7 @@ tokens(Stack, Pos, SrcStr) ->
% for now we're just going to agree with so_scan
{ierr, unterminated_block_comment} ->
PrevTokens = lists:reverse(Stack),
Err = #sfc_err_bcom_unterminated{prev_tokens = PrevTokens,
Err = #gsc_err_bcom_unterminated{prev_tokens = PrevTokens,
break_pos = Pos,
rest = SrcStr},
{error, Err};
@@ -455,13 +455,13 @@ next_tabstop8(Col0) when Col0 >= 0 ->
-spec slurp_token(Pos, SrcStr) -> Result
when Pos :: sfc_pos(),
when Pos :: gsc_pos(),
SrcStr :: string(),
Result :: {tokmatch, Token, Rest}
| no_tokmatch
| {error, sfc_err()}
| {error, gsc_err()}
| {ierr, unterminated_block_comment},
Token :: sfc_token(),
Token :: tk(),
Rest :: string().
% @doc
% grab a single token off the front of the string according to
@@ -474,14 +474,14 @@ slurp_token(Pos, SrcStr) ->
-spec slurp_token_types(ParseOrder, Pos, SrcStr) -> Result
when ParseOrder :: [sfc_token_type()],
Pos :: sfc_pos(),
when ParseOrder :: [gsc_token_type()],
Pos :: gsc_pos(),
SrcStr :: string(),
Result :: {tokmatch, Token, Rest}
| no_tokmatch
| {error, sfc_err()}
| {error, gsc_err()}
| {ierr, unterminated_block_comment},
Token :: sfc_token(),
Token :: tk(),
Rest :: string().
% @doc
% grab a single token off the front of the string according to
@@ -499,14 +499,14 @@ slurp_token_types([], _Pos, _SrcStr) ->
-spec slurp_token_of_type(TokenType, Pos, SrcStr) -> MaybeToken
when TokenType :: sfc_token_type(),
Pos :: sfc_pos(),
when TokenType :: gsc_token_type(),
Pos :: gsc_pos(),
SrcStr :: string(),
MaybeToken :: {tokmatch, Token, Rest}
| no_tokmatch
| {error, sfc_err()}
| {error, gsc_err()}
| {ierr, unterminated_block_comment},
Token :: sfc_token(),
Token :: tk(),
Rest :: string().
% @doc
% match a sophia token of a given type off the front of the string
@@ -522,7 +522,7 @@ slurp_token_of_type(lcom, Pos, SrcStr) ->
case SrcStr of
"//" ++ _ ->
{Line, Rest} = takeline("", SrcStr),
Token = #sfc_token{type = lcom,
Token = #tk{type = lcom,
pos = Pos,
string = Line},
{tokmatch, Token, Rest};
@@ -536,7 +536,7 @@ slurp_token_of_type(bcom, Pos, SrcStr0) ->
"/*" ++ SrcStr1 ->
case bcom("/*", 1, SrcStr1) of
{ok, CommentStr, SrcStr2} ->
Token = #sfc_token{type = bcom,
Token = #tk{type = bcom,
pos = Pos,
string = CommentStr},
{tokmatch, Token, SrcStr2};
@@ -547,12 +547,12 @@ slurp_token_of_type(bcom, Pos, SrcStr0) ->
no_tokmatch
end;
slurp_token_of_type(ws, Pos, SrcStr) ->
WhitespaceMatcher = sfc_strmatch:smr_sf_ws(),
case sfc_strmatch:match(WhitespaceMatcher, SrcStr) of
WhitespaceMatcher = gsc_strmatch:smr_sf_ws(),
case gsc_strmatch:match(WhitespaceMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, WS, Rest} ->
Token = #sfc_token{type = ws,
Token = #tk{type = ws,
pos = Pos,
string = WS},
{tokmatch, Token, Rest}
@@ -570,86 +570,86 @@ slurp_token_of_type(ws, Pos, SrcStr) ->
% of the kwds
slurp_token_of_type(kwd, Pos, SrcStr) ->
case slurp_token_of_type(id, Pos, SrcStr) of
{tokmatch, IdTok = #sfc_token{string = IdStr}, Rest} ->
{tokmatch, IdTok = #tk{string = IdStr}, Rest} ->
case lists:member(IdStr, kwds()) of
false ->
no_tokmatch;
true ->
KwTok = IdTok#sfc_token{type = kwd},
KwTok = IdTok#tk{type = kwd},
{tokmatch, KwTok, Rest}
end;
no_tokmatch ->
no_tokmatch
end;
slurp_token_of_type(op, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_op(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_op(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = op, pos = Pos, string = Str},
Token = #tk{type = op, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(punct, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_punct(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_punct(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = punct, pos = Pos, string = Str},
Token = #tk{type = punct, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
% SOPHIA VARIABLE NAMES: id, con, qid, qcon, tvar
slurp_token_of_type(id, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_id(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_id(), SrcStr) of
{strmatch, IdStr, Rest} ->
Token = #sfc_token{type = id, pos = Pos, string = IdStr},
Token = #tk{type = id, pos = Pos, string = IdStr},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(con, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_con(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_con(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = con, pos = Pos, string = Str},
Token = #tk{type = con, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(qid, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_qid(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_qid(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = qid, pos = Pos, string = Str},
Token = #tk{type = qid, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(qcon, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_qcon(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_qcon(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = qcon, pos = Pos, string = Str},
Token = #tk{type = qcon, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(tvar, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_tvar(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_tvar(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = tvar, pos = Pos, string = Str},
Token = #tk{type = tvar, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(int16, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_int16(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_int16(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = int16, pos = Pos, string = Str},
Token = #tk{type = int16, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(int10, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_int10(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_int10(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #sfc_token{type = int10, pos = Pos, string = Str},
Token = #tk{type = int10, pos = Pos, string = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
@@ -659,60 +659,60 @@ slurp_token_of_type(int10, Pos, SrcStr) ->
%
% char: sophia char literal
slurp_token_of_type(ak, Pos, SrcStr) ->
StringMatcher = sfc_strmatch:smr_sf_ak(),
case sfc_strmatch:match(StringMatcher, SrcStr) of
StringMatcher = gsc_strmatch:smr_sf_ak(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = ak, pos = Pos, string = TokenStr},
Token = #tk{type = ak, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(ct, Pos, SrcStr) ->
StringMatcher = sfc_strmatch:smr_sf_ct(),
case sfc_strmatch:match(StringMatcher, SrcStr) of
StringMatcher = gsc_strmatch:smr_sf_ct(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = ct, pos = Pos, string = TokenStr},
Token = #tk{type = ct, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(sg, Pos, SrcStr) ->
StringMatcher = sfc_strmatch:smr_sf_sg(),
case sfc_strmatch:match(StringMatcher, SrcStr) of
StringMatcher = gsc_strmatch:smr_sf_sg(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = sg, pos = Pos, string = TokenStr},
Token = #tk{type = sg, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(char, Pos, SrcStr) ->
StringMatcher = sfc_strmatch:smr_sf_char(),
case sfc_strmatch:match(StringMatcher, SrcStr) of
StringMatcher = gsc_strmatch:smr_sf_char(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = char, pos = Pos, string = TokenStr},
Token = #tk{type = char, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(string, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_str(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_str(), SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = string, pos = Pos, string = TokenStr},
Token = #tk{type = string, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(bytes, Pos, SrcStr) ->
case sfc_strmatch:match(sfc_strmatch:smr_sf_bytes(), SrcStr) of
case gsc_strmatch:match(gsc_strmatch:smr_sf_bytes(), SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #sfc_token{type = bytes, pos = Pos, string = TokenStr},
Token = #tk{type = bytes, pos = Pos, string = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(NyiType, Pos, SrcStr) ->
Message = io_lib:format("cannot slurp token of type: ~p", [NyiType]),
error(#sfc_err{atom = nyi,
error(#gsc_err{atom = nyi,
string = Message,
extra = [{token_type, NyiType},
{pos, Pos},
+27 -27
View File
@@ -1,9 +1,9 @@
% @doc compatibility layer to test against so_scan
%
% converts sfc_tokens data to so_scan tokens
% converts gsc_tokens data to so_scan tokens
%
% Ref: so_scan.erl
-module(sfc_so_scan).
-module(gso_scan).
-export_type([
so_kwd/0,
@@ -19,7 +19,7 @@
ken_barson_rises/2
]).
-include("$sfc_include/sfc.hrl").
-include("$gsc_include/gsc.hrl").
%================================
% API: types
@@ -76,14 +76,14 @@
-type so_symbol() :: so_kwd() | so_special_char() | atom().
-type so_token2() :: {Symbol :: so_symbol(),
Location :: sfc_pos()}.
Location :: gsc_pos()}.
% FIXME
% this is 'id', 'con', qid
-type so_tk3type() :: char | string | hex | int | bytes | qid | qcon | tvar | id | con.
-type so_token3() :: {TokenType :: so_tk3type(),
Location :: sfc_pos(),
Location :: gsc_pos(),
TokenValue :: term()}.
-type so_token() :: so_token2() | so_token3().
@@ -93,23 +93,23 @@
% API: functions
%================================
-spec scan(SrcStr) -> {ok, SoTokens} | {error, sfc_err()}
-spec scan(SrcStr) -> {ok, SoTokens} | {error, gsc_err()}
when SrcStr :: iolist(),
SoTokens :: [so_token()].
% @doc
% this is meant to agree with so_scan:scan/1 in all cases
%
% this converts sfc's internal representation of tokens into the format that
% this converts gsc's internal representation of tokens into the format that
% so_scan outputs
% @end
scan(SrcStr) ->
case sfc_tokens:tokens(SrcStr) of
case gsc_tokens:tokens(SrcStr) of
{ok, SfLTokens} ->
SoTokens = to_so_tokens(SfLTokens),
{ok, SoTokens};
% fucking stupid
{error, #sfc_err_bcom_unterminated{prev_tokens = SfcTokens}} ->
{error, #gsc_err_bcom_unterminated{prev_tokens = SfcTokens}} ->
{ok, to_so_tokens(SfcTokens)};
Error ->
Error
@@ -118,11 +118,11 @@ scan(SrcStr) ->
-spec to_so_tokens(SfcTokens) -> SoTokens
when SfcTokens :: [sfc_token()],
when SfcTokens :: [tk()],
SoTokens :: [so_token()].
% @doc
% most sfc tokens map 1-to-1 with so_tokens. the
% most gsc tokens map 1-to-1 with so_tokens. the
% exception is ak/ct/sg literals. this is a
% many-to-one-mapping, and therefore ak, sg, ct need to
% be handled at the list level.
@@ -133,7 +133,7 @@ scan(SrcStr) ->
% stage computes the pubkey that corresponds to.
%
% as a result, if we have ak_GHI, I is not a valid
% base58 char, so WE (sfc) end up lexing that as
% base58 char, so WE (gsc) end up lexing that as
%
% [{ak, "ak_GH"}, {con, "I"}]
%
@@ -151,12 +151,12 @@ scan(SrcStr) ->
%
% so if we see an ak/ct/sg token, we summon evil ben
% carson to reconjoin the unconjoined twins
to_so_tokens([ AkTok = #sfc_token{type = AkCtSg, pos = Pos}
to_so_tokens([ AkTok = #tk{type = AkCtSg, pos = Pos}
| Sheeit])
when ak =:= AkCtSg;
ct =:= AkCtSg;
sg =:= AkCtSg ->
{#sfc_token{string = FinalAkStr}, NewSheeit}
{#tk{string = FinalAkStr}, NewSheeit}
= ken_barson_rises(AkTok, Sheeit),
[{id, Pos, FinalAkStr}| to_so_tokens(NewSheeit)];
% this part is just lists:filtermap
@@ -171,8 +171,8 @@ to_so_tokens([]) ->
-spec ken_barson_rises(InitApiToken, SfToks) -> {FinalApiToken, NewSfToks}
when InitApiToken :: sfc_token(),
SfToks :: [sfc_token()],
when InitApiToken :: tk(),
SfToks :: [tk()],
FinalApiToken :: InitApiToken,
NewSfToks :: SfToks.
% @doc
@@ -252,15 +252,15 @@ to_so_tokens([]) ->
%
% When any of these appear AFTER at least one valid
% base58 char in a `ak_`/`ct_`/`sg_` prefixed
% identifier, `sfc` splits what `so_scan` sees as one
% `id` token into 2+ sfc tokens.
% identifier, `gsc` splits what `so_scan` sees as one
% `id` token into 2+ gsc tokens.
%
% **No split if non-base58 char is immediately after
% `_`**: `smr_plus` requires >=1 base58 char to
% match; `ak_I`, `ak_0`, `ak__bar` all fall
% through to `id` and both tokenizers agree.
ken_barson_rises(AkTokAcc = #sfc_token{string = AkStr},
SrcTokens = [#sfc_token{type = CandidateType,
ken_barson_rises(AkTokAcc = #tk{string = AkStr},
SrcTokens = [#tk{type = CandidateType,
string = CandidateString}
| Rest]) ->
% candidate:
@@ -273,7 +273,7 @@ ken_barson_rises(AkTokAcc = #sfc_token{string = AkStr},
Smash ->
% dig out the token from LcTokApi
NewAkStr = AkStr ++ CandidateString,
NewAkTokAcc = AkTokAcc#sfc_token{string = NewAkStr},
NewAkTokAcc = AkTokAcc#tk{string = NewAkStr},
ken_barson_rises(NewAkTokAcc, Rest);
Pass ->
{AkTokAcc, SrcTokens}
@@ -310,7 +310,7 @@ pass_types() ->
-spec to_so_token(SfcToken) -> MaybeSoToken
when SfcToken :: sfc_token(),
when SfcToken :: tk(),
MaybeSoToken :: {true, SoToken}
| false,
SoToken :: so_token().
@@ -320,7 +320,7 @@ pass_types() ->
% follow-on tokens
% @end
to_so_token(#sfc_token{type = SfTokenType,
to_so_token(#tk{type = SfTokenType,
pos = Pos,
string = SfTokenStr}) ->
case SfTokenType of
@@ -369,8 +369,8 @@ to_so_token(#sfc_token{type = SfTokenType,
int10 -> {true, {int, Pos, so_parse_int(SfTokenStr)}};
bytes -> {true, {bytes, Pos, so_parse_bytes(SfTokenStr)}};
NYI ->
Msg = io_lib:format("sfc_so_scan:to_so_token/1: unhandled token shape: ~p", [NYI]),
error(#sfc_err{atom = nyi,
Msg = io_lib:format("gsc_so_scan:to_so_token/1: unhandled token shape: ~p", [NYI]),
error(#gsc_err{atom = nyi,
string = Msg})
end.
@@ -392,7 +392,7 @@ so_parse_char([$' | Chars]) ->
case unicode:characters_to_nfc_list(unescape($', Chars, [])) of
[Char] -> Char;
_Bad ->
error(#sfc_err{atom = bad_token,
error(#gsc_err{atom = bad_token,
string = "Bad character literal: '" ++ Chars})
end.
@@ -434,7 +434,7 @@ unescape(Delim, [$\\, Code | Chars], Acc) ->
$r -> Ok($\r);
$t -> Ok($\t);
$v -> Ok($\v);
_ -> error(#sfc_err{atom = bad_escape_char,
_ -> error(#gsc_err{atom = bad_escape_char,
string = "Bad control sequence: \\" ++ [Code]}) %% TODO
end;
unescape(Delim, [C | Chars], Acc) ->