pharpend/sfc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more mass renaming

Browse Source
This commit is contained in:
Peter Harpending
2026-06-02 01:48:05 -07:00
parent eff77fff6b
commit 270f192f0c
53 changed files with 1264 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gsc_strmatch.erl → src/gs_strmatch.erl
+1 -1
View File
@@ -70,7 +70,7 @@
% `contract` gets tokenized as a keyword and not a variable name), and then
% calls into this module in order to match the string shape it's looking for.
% @end
-module(gsc_strmatch).
-module(gs_strmatch).
%-compile([export_all, nowarn_export_all]).
src/gsc_tokens.erl → src/gs_tokens.erl
+124 -112
View File
@@ -16,11 +16,11 @@
% 2. to future-proof in case we decide to incrementally incorporate the gsc
% code into the legacy sophia compiler
% @end
-module(gsc_tokens).
-module(gs_tokens).
% meta
-export([
token_types_parse_order/0,
token_shapes_parse_order/0,
kwds/0
]).
@@ -39,10 +39,11 @@
is_significant/1,
filter_significant/1,
significant_tokens/1,
tokens_from_iolist/1,
tokens/1,
slurp_token/2,
slurp_token_types/3,
slurp_token_of_type/3,
slurp_token_shapes/3,
slurp_token_of_shape/3,
new_pos/2
]).
@@ -59,7 +60,7 @@
AtMostNStrings :: [string()].
% @doc return the strings of the first N tokens
strings(N, [#tk{string = S} | Rest]) when is_integer(N), N >= 1 ->
strings(N, [#tk{str = S} | Rest]) when is_integer(N), N >= 1 ->
[S | strings(N-1, Rest)];
strings(_, []) ->
[];
@@ -159,7 +160,7 @@ take_block_item([]) ->
% counterintuitive to end-users (who are programmers, entirely
% unfamiliar with notions like stacks and open/close delimiters)
slurp_plist([Hd = #tk{string = "("} | Tl]) ->
slurp_plist([Hd = #tk{str = "("} | Tl]) ->
slurp_dlist([Hd], [Hd], Tl);
slurp_plist(Tks) ->
{slurp, [], Tks}.
@@ -170,30 +171,30 @@ slurp_dlist(All, [], NewTokens) ->
{slurp, lists:reverse(All), NewTokens};
% WMA stack is nonempty
% happy cases of opens getting popped
slurp_dlist(All, [#tk{string = "("} | NewOpen],
[#tk{string = ")"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{str = "("} | NewOpen],
[#tk{str = ")"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
slurp_dlist(All, [#tk{string = "["} | NewOpen],
[#tk{string = "]"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{str = "["} | NewOpen],
[#tk{str = "]"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
slurp_dlist(All, [#tk{string = "{"} | NewOpen],
[#tk{string = "}"} = Tk | NewTks]) ->
slurp_dlist(All, [#tk{str = "{"} | NewOpen],
[#tk{str = "}"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], NewOpen, NewTks);
% happy: open delimiters getting pushed
slurp_dlist(All, Opens, [#tk{string = "("} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{str = "("} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
slurp_dlist(All, Opens, [#tk{string = "["} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{str = "["} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
slurp_dlist(All, Opens, [#tk{string = "{"} = Tk | NewTks]) ->
slurp_dlist(All, Opens, [#tk{str = "{"} = Tk | NewTks]) ->
slurp_dlist([Tk | All], [Tk | Opens], NewTks);
% sad: mismatch cases
slurp_dlist(All, Opens, []) ->
{error, {fixme, mismatch, Opens, none}};
slurp_dlist(All, Opens, [#tk{string = "}"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{str = "}"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
slurp_dlist(All, Opens, [#tk{string = "]"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{str = "]"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
slurp_dlist(All, Opens, [#tk{string = ")"} = BadClose | _]) ->
slurp_dlist(All, Opens, [#tk{str = ")"} = BadClose | _]) ->
{error, {fixme, mismatch, Opens, {value, BadClose}}};
% general case: non-terminal token gets pushed
slurp_dlist(All, Opens, [Tk | NewTks]) ->
@@ -206,15 +207,15 @@ slurp_dlist(All, Opens, [Tk | NewTks]) ->
% This is parse order definition, list of keywords, etc
%
% -export([
% token_types_parse_order/0,
% token_shapes_parse_order/0,
% kwds/0
% ]).
%-------------------------------------------------------
-spec token_types_parse_order() -> [gsc_token_type()].
-spec token_shapes_parse_order() -> [tk_shape()].
% @doc
% list of sophia tokens in parse order (if an earlier type matches, the later
% type isn't even checked)
% list of sophia token shapes in parse order (if an earlier shape matches, the later
% shape isn't even checked)
%
%
% Rules =
@@ -245,7 +246,7 @@ slurp_dlist(All, Opens, [Tk | NewTks]) ->
% ],
% @end
token_types_parse_order() ->
token_shapes_parse_order() ->
% written in this style to be maximally editable
lists:flatten([
% comments and whitespace
@@ -282,8 +283,8 @@ kwds() ->
% -export([
% tokens/1,
% slurp_token/1,
% slurp_token_types/2,
% slurp_token_of_type/2
% slurp_token_shapes/2,
% slurp_token_of_shape/2
% ]).
%-------------------------------------------------------
@@ -323,12 +324,23 @@ filter_significant(Tokens) ->
-spec is_significant(Token) -> boolean()
when Token :: tk().
is_significant(#tk{type = bcom}) -> false;
is_significant(#tk{type = lcom}) -> false;
is_significant(#tk{type = ws}) -> false;
is_significant(#tk{shape = bcom}) -> false;
is_significant(#tk{shape = lcom}) -> false;
is_significant(#tk{shape = ws}) -> false;
is_significant(_) -> true.
-spec tokens_from_iolist(SrcStr) -> Result when
SrcStr :: iolist(),
Result :: {ok, Tokens}
| {error, gsc_err()},
Tokens :: [tk()].
% @doc alias for tokens/1
tokens_from_iolist(S) -> tokens(S).
-spec tokens(SrcStr) -> Result
when SrcStr :: iolist(),
Result :: {ok, Tokens}
@@ -349,7 +361,7 @@ tokens(Stack, _FinalPos, "") ->
{ok, lists:reverse(Stack)};
tokens(Stack, Pos, SrcStr) ->
case slurp_token(Pos, SrcStr) of
{tokmatch, NewToken = #tk{string = TokStr},
{tokmatch, NewToken = #tk{str = TokStr},
NewSrcStr} ->
NewPos = new_pos(Pos, TokStr),
tokens([NewToken | Stack], NewPos, NewSrcStr);
@@ -455,7 +467,7 @@ next_tabstop8(Col0) when Col0 >= 0 ->
-spec slurp_token(Pos, SrcStr) -> Result
when Pos :: gsc_pos(),
when Pos :: tk_pos(),
SrcStr :: string(),
Result :: {tokmatch, Token, Rest}
| no_tokmatch
@@ -465,17 +477,17 @@ next_tabstop8(Col0) when Col0 >= 0 ->
Rest :: string().
% @doc
% grab a single token off the front of the string according to
% `token_types_parse_order/0'
% `token_shapes_parse_order/0'
slurp_token(Pos, SrcStr) ->
% this is the easiest format if i need to fuck with it
slurp_token_types(token_types_parse_order(), Pos, SrcStr).
slurp_token_shapes(token_shapes_parse_order(), Pos, SrcStr).
-spec slurp_token_types(ParseOrder, Pos, SrcStr) -> Result
when ParseOrder :: [gsc_token_type()],
Pos :: gsc_pos(),
-spec slurp_token_shapes(ParseOrder, Pos, SrcStr) -> Result
when ParseOrder :: [tk_shape()],
Pos :: tk_pos(),
SrcStr :: string(),
Result :: {tokmatch, Token, Rest}
| no_tokmatch
@@ -485,22 +497,22 @@ slurp_token(Pos, SrcStr) ->
Rest :: string().
% @doc
% grab a single token off the front of the string according to
% `token_types_parse_order/0'
% `token_shapes_parse_order/0'
slurp_token_types([TokenType | TTs], Pos, SrcStr) ->
case slurp_token_of_type(TokenType, Pos, SrcStr) of
slurp_token_shapes([TokenType | TTs], Pos, SrcStr) ->
case slurp_token_of_shape(TokenType, Pos, SrcStr) of
Match = {tokmatch, _, _} -> Match;
no_tokmatch -> slurp_token_types(TTs, Pos, SrcStr);
no_tokmatch -> slurp_token_shapes(TTs, Pos, SrcStr);
IErr = {ierr, _} -> IErr;
Error = {error, _} -> Error
end;
slurp_token_types([], _Pos, _SrcStr) ->
slurp_token_shapes([], _Pos, _SrcStr) ->
no_tokmatch.
-spec slurp_token_of_type(TokenType, Pos, SrcStr) -> MaybeToken
when TokenType :: gsc_token_type(),
Pos :: gsc_pos(),
-spec slurp_token_of_shape(TokenType, Pos, SrcStr) -> MaybeToken
when TokenType :: tk_shape(),
Pos :: tk_pos(),
SrcStr :: string(),
MaybeToken :: {tokmatch, Token, Rest}
| no_tokmatch
@@ -509,7 +521,7 @@ slurp_token_types([], _Pos, _SrcStr) ->
Token :: tk(),
Rest :: string().
% @doc
% match a sophia token of a given type off the front of the string
% match a sophia token of a given shape off the front of the string
% @end
% COMMENTS AND WHITESPACE: lcom, bcom, ws
@@ -518,27 +530,27 @@ slurp_token_types([], _Pos, _SrcStr) ->
%
% i am not going to bother writing a string matcher thing for this
% FIXME: make a string matcher for line comments
slurp_token_of_type(lcom, Pos, SrcStr) ->
slurp_token_of_shape(lcom, Pos, SrcStr) ->
case SrcStr of
"//" ++ _ ->
{Line, Rest} = takeline("", SrcStr),
Token = #tk{type = lcom,
Token = #tk{shape = lcom,
pos = Pos,
string = Line},
str = Line},
{tokmatch, Token, Rest};
_ ->
no_tokmatch
end;
% Block comments cannot have a string matcher because they have a whole stack
% thing keeping track of depth because of nested block comments
slurp_token_of_type(bcom, Pos, SrcStr0) ->
slurp_token_of_shape(bcom, Pos, SrcStr0) ->
case SrcStr0 of
"/*" ++ SrcStr1 ->
case bcom("/*", 1, SrcStr1) of
{ok, CommentStr, SrcStr2} ->
Token = #tk{type = bcom,
Token = #tk{shape = bcom,
pos = Pos,
string = CommentStr},
str = CommentStr},
{tokmatch, Token, SrcStr2};
Error ->
Error
@@ -546,15 +558,15 @@ slurp_token_of_type(bcom, Pos, SrcStr0) ->
_ ->
no_tokmatch
end;
slurp_token_of_type(ws, Pos, SrcStr) ->
WhitespaceMatcher = gsc_strmatch:smr_sf_ws(),
case gsc_strmatch:match(WhitespaceMatcher, SrcStr) of
slurp_token_of_shape(ws, Pos, SrcStr) ->
WhitespaceMatcher = gs_strmatch:smr_sf_ws(),
case gs_strmatch:match(WhitespaceMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, WS, Rest} ->
Token = #tk{type = ws,
Token = #tk{shape = ws,
pos = Pos,
string = WS},
str = WS},
{tokmatch, Token, Rest}
end;
% KEYWORDS, OPERATORS, PUNCTUATION: kwd, op, punct
@@ -568,88 +580,88 @@ slurp_token_of_type(ws, Pos, SrcStr) ->
%
% we know kwds are always ids, so we parse it as an id and see if it's one
% of the kwds
slurp_token_of_type(kwd, Pos, SrcStr) ->
case slurp_token_of_type(id, Pos, SrcStr) of
{tokmatch, IdTok = #tk{string = IdStr}, Rest} ->
slurp_token_of_shape(kwd, Pos, SrcStr) ->
case slurp_token_of_shape(id, Pos, SrcStr) of
{tokmatch, IdTok = #tk{str = IdStr}, Rest} ->
case lists:member(IdStr, kwds()) of
false ->
no_tokmatch;
true ->
KwTok = IdTok#tk{type = kwd},
KwTok = IdTok#tk{shape = kwd},
{tokmatch, KwTok, Rest}
end;
no_tokmatch ->
no_tokmatch
end;
slurp_token_of_type(op, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_op(), SrcStr) of
slurp_token_of_shape(op, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_op(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = op, pos = Pos, string = Str},
Token = #tk{shape = op, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(punct, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_punct(), SrcStr) of
slurp_token_of_shape(punct, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_punct(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = punct, pos = Pos, string = Str},
Token = #tk{shape = punct, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
% SOPHIA VARIABLE NAMES: id, con, qid, qcon, tvar
slurp_token_of_type(id, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_id(), SrcStr) of
slurp_token_of_shape(id, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_id(), SrcStr) of
{strmatch, IdStr, Rest} ->
Token = #tk{type = id, pos = Pos, string = IdStr},
Token = #tk{shape = id, pos = Pos, str = IdStr},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(con, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_con(), SrcStr) of
slurp_token_of_shape(con, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_con(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = con, pos = Pos, string = Str},
Token = #tk{shape = con, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(qid, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_qid(), SrcStr) of
slurp_token_of_shape(qid, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_qid(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = qid, pos = Pos, string = Str},
Token = #tk{shape = qid, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(qcon, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_qcon(), SrcStr) of
slurp_token_of_shape(qcon, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_qcon(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = qcon, pos = Pos, string = Str},
Token = #tk{shape = qcon, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(tvar, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_tvar(), SrcStr) of
slurp_token_of_shape(tvar, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_tvar(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = tvar, pos = Pos, string = Str},
Token = #tk{shape = tvar, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(int16, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_int16(), SrcStr) of
slurp_token_of_shape(int16, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_int16(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = int16, pos = Pos, string = Str},
Token = #tk{shape = int16, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
end;
slurp_token_of_type(int10, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_int10(), SrcStr) of
slurp_token_of_shape(int10, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_int10(), SrcStr) of
{strmatch, Str, Rest} ->
Token = #tk{type = int10, pos = Pos, string = Str},
Token = #tk{shape = int10, pos = Pos, str = Str},
{tokmatch, Token, Rest};
no_strmatch ->
no_tokmatch
@@ -658,63 +670,63 @@ slurp_token_of_type(int10, Pos, SrcStr) ->
% ak, ct, sg
%
% char: sophia char literal
slurp_token_of_type(ak, Pos, SrcStr) ->
StringMatcher = gsc_strmatch:smr_sf_ak(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
slurp_token_of_shape(ak, Pos, SrcStr) ->
StringMatcher = gs_strmatch:smr_sf_ak(),
case gs_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = ak, pos = Pos, string = TokenStr},
Token = #tk{shape = ak, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(ct, Pos, SrcStr) ->
StringMatcher = gsc_strmatch:smr_sf_ct(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
slurp_token_of_shape(ct, Pos, SrcStr) ->
StringMatcher = gs_strmatch:smr_sf_ct(),
case gs_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = ct, pos = Pos, string = TokenStr},
Token = #tk{shape = ct, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(sg, Pos, SrcStr) ->
StringMatcher = gsc_strmatch:smr_sf_sg(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
slurp_token_of_shape(sg, Pos, SrcStr) ->
StringMatcher = gs_strmatch:smr_sf_sg(),
case gs_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = sg, pos = Pos, string = TokenStr},
Token = #tk{shape = sg, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(char, Pos, SrcStr) ->
StringMatcher = gsc_strmatch:smr_sf_char(),
case gsc_strmatch:match(StringMatcher, SrcStr) of
slurp_token_of_shape(char, Pos, SrcStr) ->
StringMatcher = gs_strmatch:smr_sf_char(),
case gs_strmatch:match(StringMatcher, SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = char, pos = Pos, string = TokenStr},
Token = #tk{shape = char, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(string, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_str(), SrcStr) of
slurp_token_of_shape(string, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_str(), SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = string, pos = Pos, string = TokenStr},
Token = #tk{shape = string, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(bytes, Pos, SrcStr) ->
case gsc_strmatch:match(gsc_strmatch:smr_sf_bytes(), SrcStr) of
slurp_token_of_shape(bytes, Pos, SrcStr) ->
case gs_strmatch:match(gs_strmatch:smr_sf_bytes(), SrcStr) of
no_strmatch ->
no_tokmatch;
{strmatch, TokenStr, Rest} ->
Token = #tk{type = bytes, pos = Pos, string = TokenStr},
Token = #tk{shape = bytes, pos = Pos, str = TokenStr},
{tokmatch, Token, Rest}
end;
slurp_token_of_type(NyiType, Pos, SrcStr) ->
Message = io_lib:format("cannot slurp token of type: ~p", [NyiType]),
slurp_token_of_shape(NyiType, Pos, SrcStr) ->
Message = io_lib:format("cannot slurp token of shape: ~p", [NyiType]),
error(#gsc_err{atom = nyi,
string = Message,
extra = [{token_type, NyiType},
str = Message,
extra = [{token_shape, NyiType},
{pos, Pos},
{rest, SrcStr}]}).
src/gsc.erl
+7 -59
View File
@@ -4,15 +4,14 @@
% based on original sophia compiler
%
% parse layers:
% 1. gsc_tokenizer: SrcStr -> (Tokens | SigTokens)
% 1. gs_tokens: SrcStr -> (Tokens | SigTokens)
%
% SigTokens = not comment/whitespace
%
% layers:
% a. gsc_strmatch : matches string shapes
% b. gsc_so_scan : converts to so_scan shapes
% a. gs_strmatch : matches string shapes
% b. gso_scan : converts to so_scan shapes
%
% 2. gsc_ast: SigTokens -> AST
%
% terminology:
%
@@ -30,16 +29,8 @@
%
% @end
% TODO:
% - barf for outputs, slurp for inputs
% - architecture needs more careful thought but only after something works
% - too fuzzy right now
% - possibly:
% - rename parser layers sequentially:
% - gsc_
-module(gsc).
-export_type([
token/0
]).
@@ -48,10 +39,7 @@
sigtokens_from_file/1,
sigtokens_from_string/1,
tokens_from_file/1,
tokens_from_string/1,
ast_from_file/1,
ast_from_string/1,
ast_from_tokens/1
tokens_from_string/1
]).
-include("$gsc_include/gsc.hrl").
@@ -68,13 +56,13 @@
sigtokens_from_file(X) ->
case tokens_from_file(X) of
{ok, Y} -> {ok, gsc_tokens:filter_significant(Y)};
{ok, Y} -> {ok, gs_tokens:filter_significant(Y)};
Err -> Err
end.
sigtokens_from_string(X) ->
case tokens_from_string(X) of
{ok, Y} -> {ok, gsc_tokens:filter_significant(Y)};
{ok, Y} -> {ok, gs_tokens:filter_significant(Y)};
Err -> Err
end.
@@ -101,44 +89,4 @@ tokens_from_file(FilePath) ->
Tokens :: [tk()].
tokens_from_string(SrcStr) ->
gsc_tokens:tokens(SrcStr).
-spec ast_from_file(FilePath) -> Perhaps
when FilePath :: string(),
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_file(FilePath) ->
case file:read_file(FilePath) of
{ok, FileBytes} -> ast_from_string(FileBytes);
Error -> Error
end.
-spec ast_from_string(SrcStr) -> Perhaps
when SrcStr :: string(),
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_string(SrcStr) ->
case gsc_tokens:significant_tokens(SrcStr) of
{ok, SigTks} -> ast_from_tokens(SigTks);
Error -> Error
end.
-spec ast_from_tokens(SrcTokens) -> Perhaps
when SrcTokens :: [tk()],
Perhaps :: {ok, AST} | {error, gsc_err()},
AST :: gsc_ast().
ast_from_tokens(Tks) ->
SigTks = gsc_tokens:filter_significant(Tks),
case gsc_ast:gulp_file(SigTks) of
{gulp, AST} -> {ok, AST};
Error -> Error
end.
gs_tokens:tokens(SrcStr).
src/gsc_ast.erl
-693
View File
@@ -1,693 +0,0 @@
% @doc
% type for a GSC AST
%
% based on so_syntax.erl
%
% this is the second layer in the parsing pipeline after tokens. From
% so_syntax:
%
% ## Declarations
%
% A Sophia file consists of a sequence of *declarations* in a layout block.
%
% ```c
% File ::= Block(TopDecl)
%
% TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
% | 'contract' 'interface' Con [Implement] '=' Block(Decl)
% | 'namespace' Con '=' Block(Decl)
% | '@compiler' PragmaOp Version
% | 'include' String
% | Using
%
% Implement ::= ':' Sep1(Con, ',')
%
% Decl ::= 'type' Id ['(' TVar* ')'] '=' TypeAlias
% | 'record' Id ['(' TVar* ')'] '=' RecordType
% | 'datatype' Id ['(' TVar* ')'] '=' DataType
% | 'let' Id [':' Type] '=' Expr
% | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
% | Using
%
% FunDecl ::= Id ':' Type // Type signature
% | Id Args [':' Type] '=' Block(Stmt) // Definition
% | Id Args [':' Type] Block(GuardedDef) // Guarded definitions
%
% GuardedDef ::= '|' Sep1(Expr, ',') '=' Block(Stmt)
%
% Using ::= 'using' Con ['as' Con] [UsingParts]
% UsingParts ::= 'for' '[' Sep1(Id, ',') ']'
% | 'hiding' '[' Sep1(Id, ',') ']'
%
% PragmaOp ::= '<' | '=<' | '==' | '>=' | '>'
% Version ::= Sep1(Int, '.')
%
% EModifier ::= 'payable' | 'stateful'
% FModifier ::= 'stateful' | 'private'
%
% Args ::= '(' Sep(Pattern, ',') ')'
%
% Type ::= Domain '=>' Type // Function type
% | Type '(' Sep(Type, ',') ')' // Type application
% | '(' Type ')' // Parens
% | 'unit' | Sep(Type, '*') // Tuples
% | Id | QId | TVar
%
% Domain ::= Type // Single argument
% | '(' Sep(Type, ',') ')' // Multiple arguments
% ```
-module(gsc_ast).
%-compile([export_all,nowarn_export_all]).
%
%%-export_type([
%%]).
%%
%%-export([
%% slurp/2
%%]).
%
%-export([
% gulp_file/1
%]).
%
%-include("$gsc_include/gsc.hrl").
%
%%%-----------------------------
%%% TYPES: gsc_ast
%%%-----------------------------
%
%
%% % placeholders
%-type ast_() :: any().
%-record(ast_nyi, {tokens = none :: [tk()]}).
%-type ast_nyi() :: #ast_nyi{}.
%
%
%% product type: foo * bar * baz
%%
%% stupid weird implication from bad syntax foresight trying to be
%% fancy and overload what parens do is products must always have at
%% least two operands; probably this is because `(foo)` is always the
%% same as `foo`
%%
%% - 0-tuple -> `unit`
%% - 1-tuple -> type itself
%% - 2+ -> here
%-record(ast_te_pi,
% {types = none :: none | [ast_te_()]}).
%
%-type ast_type_expr()
% :: #ast_te_fn{} % function (string, string) => string
% | #ast_te_ap{} % application map(string, int)
% | #ast_te_pi{} % product foo * bar * baz
% | #ast_te_tk{} % token string int 'a Foo.Bar.baz
% | ast_nyi().
%-type ast_te() :: ast_type_expr().
%
%
%% ta = type alias
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%-record(ast_ta,
% {alias = none :: none | tk(),
% tvars = none :: none | [tk()],
% points_to = none :: none | ast_type_expr()}).
%
%% Decl ::= 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% | 'record' Id ['(' TVar* ')'] '=' RecordType
%% | 'datatype' Id ['(' TVar* ')'] '=' DataType
%% | 'let' Id [':' Type] '=' Expr
%% | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
%% | Using
%-type decl() :: #ast_ta{}
% | #ast_nyi{}.
%
%% ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
%-record(ast_ct,
% {payable = none :: none | boolean(),
% main = none :: none | boolean(),
% name = none :: none | tk(),
% impls = none :: none | [tk()],
% decls = none :: none | [decl()]}).
%
%-record(ast_td_iface,{}).
%-record(ast_td_ns,{}).
%-record(ast_td_compiler,{}).
%-record(ast_td_include,{}).
%-record(ast_using,{}).
%
%% @doc
%% TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
%% | 'contract' 'interface' Con [Implement] '=' Block(Decl)
%% | 'namespace' Con '=' Block(Decl)
%% | '@compiler' PragmaOp Version
%% | 'include' String
%% | Using
%-type top_decl() :: #ast_ct{}
% | #ast_td_iface{}
% | #ast_td_ns{}
% | #ast_td_compiler{}
% | #ast_td_include{}
% | #ast_using{}
% .
%
%-record(ast_file,
% {top_decls :: [top_decl()]}).
%-type ast_file() :: #ast_file{}.
%
%
%-type parse_error_() :: any().
%-record(parse_error,
% {pos = none :: none | gsc_pos(),
% msg = "" :: string(),
% subs = [] :: [parse_error_()],
% extra = none :: any()}).
%-type parse_error() :: parse_error().
%
%
%-type ast()
% :: #ast_ct{}
% | #ast_file{}
% | #ast_nyi{}
% | #ast_ta{}
% | #ast_td_compiler{}
% | #ast_td_iface{}
% | #ast_td_include{}
% | #ast_td_ns{}
% | #ast_te_ap{} % application map(string, int)
% | #ast_te_fn{} % function (string, string) => string
% | #ast_te_pi{} % product foo * bar * baz
% | #ast_te_tk{} % token string int 'a Foo.Bar.baz
% | #ast_using{}.
%
%
%
%
%
%%%-----------------------------
%%% FUNCTIONS
%%%-----------------------------
%
%-spec gulp_file(Tokens) -> Perhaps
% when Tokens :: [tk()],
% Perhaps :: {gulp, #ast_file{}}
% | {error, #parse_error{}}.
%
%gulp_file([]) ->
% {error, empty_file};
%gulp_file(Tokens) ->
% case gsc_tokens:take_block(Tokens) of
% {Tokens, []} ->
% gulp_block(fun gulp_top_decl/1, Tokens);
% %gulp_file2([], [], Tokens);
% {A, B} ->
% StartPos = gsc_tokens:start_pos(A),
% ErrPos = gsc_tokens:start_pos(B),
% Msg = efmt("gulp_file: block starting at ~p ends at ~p instead of EOF",
% [StartPos, ErrPos]),
% {error, #parse_error{pos = ErrPos, msg = Msg}}
% end.
%
%
%
%%gulp_file2(AccOks, AccErrs, Tokens = [_ | _]) ->
%% % ItemTokens will be nonempty
%% {ItemTokens, NewTokens} = gsc_tokens:take_block_item(Tokens),
%% case gulp_top_decl(ItemTokens) of
%% {gulp, Ok} -> gulp_file2([Ok | AccOks], AccErrs, NewTokens);
%% Err -> gulp_file2(AccOks, [Err | AccErrs], NewTokens)
%% end;
%%gulp_file2(AccOks, [], []) ->
%% TopDecls = lists:reverse(AccOks),
%% {gulp, #ast_file{top_decls = TopDecls}};
%%gulp_file2(_, AccErrs, []) ->
%% Errs = [#parse_error{pos = Pos} | _]
%% = lists:reverse(AccErrs),
%% {error, #parse_error{pos = Pos, subs = Errs}}.
%
%-spec gulp_block(GulpItem, Tokens) -> GulpedItems
% when GulpItem :: fun((ItemTokens) -> GulpedItem),
% Tokens :: [tk()],
% ItemTokens :: Tokens,
% GulpedItem :: {gulp, Item} | {error, Reason},
% GulpedItems :: {gulp, Items} | {error, Reason},
% Items :: [Item],
% Item :: ast(),
% Reason :: any().
%% @doc
%% this almost is the familiar `traverse' idiom from Haskell
%%
%% ghci> :t traverse
%% traverse
%% :: (Traversable t, Applicative f) => (a -> f b) -> t a -> f (t b)
%% ghci> :t mapM
%% mapM :: (Traversable t, Monad m) => (a -> m b) -> t a -> m (t b)
%%
%% (or `mapM' if youre being sloppy)
%%
%% however in our case our function is actually of
%% signature
%%
%% :: (t a -> f b)
%%
%% so this is more sophisticated
%% @end
%
%gulp_block(GulpItem, Tokens) ->
% gulp_block(GulpItem, [], [], Tokens).
%
%gulp_block(GulpItem, AccOks, AccErrs, Tokens = [_ | _]) ->
% % ItemTokens will be nonempty
% {ItemTokens, NewTokens} = gsc_tokens:take_block_item(Tokens),
% case GulpItem(ItemTokens) of
% {gulp, Ok} -> gulp_block(GulpItem, [Ok | AccOks], AccErrs, NewTokens);
% Err -> gulp_block(GulpItem, AccOks, [Err | AccErrs], NewTokens)
% end;
%gulp_block(_, AccOks, [], []) ->
% Items = lists:reverse(AccOks),
% {gulp, Items};
%gulp_block(_, _, AccErrs, []) ->
% {error, #parse_error{subs = lists:reverse(AccErrs)}}.
%
%
%
%-spec gulp_top_decl(DeclTokens) -> Result
% when DeclTokens :: [tk()],
% Result :: {gulp, ast()} | {error, any()}.
%
%% @doc
%% TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
%% | 'contract' 'interface' Con [Implement] '=' Block(Decl)
%% | 'namespace' Con '=' Block(Decl)
%% | '@compiler' PragmaOp Version
%% | 'include' String
%% | Using
%% @end
%gulp_top_decl(DeclTokens) ->
% case gsc_tokens:strings(3, DeclTokens) of
% ["payable", "contract", "interface"] ->
% gulp_nyi(DeclTokens);
% ["contract", "interface" | _] ->
% gulp_nyi(DeclTokens);
% ["payable", "main", "contract"] ->
% gulp_ct2(#ast_ct{payable = true, main = true},
% drop(3, DeclTokens));
% ["main", "contract" | _] ->
% gulp_ct2(#ast_ct{payable = false, main = true},
% drop(2, DeclTokens));
% ["payable", "contract" | _] ->
% gulp_ct2(#ast_ct{payable = true, main = false},
% drop(2, DeclTokens));
% ["contract" | _] ->
% gulp_ct2(#ast_ct{payable = false, main = false},
% drop(1, DeclTokens));
% ["namespace" | _] ->
% gulp_nyi(DeclTokens);
% ["@", "compiler" | _] ->
% gulp_nyi(DeclTokens);
% ["include" | _] ->
% gulp_nyi(DeclTokens);
% ["using" | _ ] ->
% gulp_nyi(DeclTokens);
% _ ->
% % decl tokens will always be nonempty
% [H = #tk{pos = ErrPos} | _] = DeclTokens,
% EMsg = efmt("gulp_top_decl: bad token: ~p; "
% "expecting one of: [payable, main, contract, namespace, @compiler, include, using]",
% [H]),
% {error, #parse_error{pos = ErrPos,
% msg = EMsg}}
% end.
%
%
%gulp_ct2(Ast, Tokens = [#tk{pos = Pos} | _]) ->
% % need to pass through pos for error messages on premature end of
% % input... for now it's a fixme thing
% gulp_ct3(Ast, Tokens, Pos).
%
%
%% payable main contract X : Y, Z, W = <block>
%% ^
%gulp_ct3(Ast = #ast_ct{name = none}, Tokens, Pos) ->
% NewPos = Pos, %% fixme
% case Tokens of
% [Name = #tk{type = con} | NewTokens] ->
% NewAst = Ast#ast_ct{name = Name},
% gulp_ct3(NewAst, NewTokens, NewPos);
% [Name | _] ->
% {error, {fixme, {bad_contract_name, Name}}};
% [] ->
% {error, {fixme, no_contract_name}}
% end;
%% contract X = <block>
%% contract X : Y, Z, W = <block>
%% ^
%gulp_ct3(Ast = #ast_ct{impls = none}, Tokens, Pos) ->
% NewPos = Pos, %% fixme
% case slurp_ct_impls(Tokens) of
% {slurp, Impls, NewTokens} ->
% NewAst = Ast#ast_ct{impls = Impls},
% gulp_ct3(NewAst, NewTokens, NewPos);
% Error ->
% Error
% end;
%% contract X = <block>
%% contract X : Y, Z, W = <block>
%% ^
%gulp_ct3(Ast = #ast_ct{decls = none},
% [#tk{string = "="} | Tokens],
% _Pos) ->
% case gulp_block(fun gulp_decl/1, Tokens) of
% {gulp, Decls} ->
% Result = Ast#ast_ct{decls = Decls},
% {gulp, Result};
% Error ->
% Error
% end;
%gulp_ct3(Ast, Tokens, Pos) ->
% {error, {fixme, {nyi, gulp_ct3, [Ast, Tokens, Pos]}}}.
%
%
%
%-spec slurp_ct_impls(Tokens) -> Slurped
% when Tokens :: [tk()],
% Slurped :: {slurp, Impls, NewTokens}
% | {error, Reason},
% Impls :: Tokens,
% NewTokens :: Tokens,
% Reason :: any().
%% @doc ": X, Y, Z = ..." ~> {slurp, [X, Y, Z], "= ..."}
%
%% FIXME: this should be a restructured a tiny bit for better error
%% handling
%slurp_ct_impls([_ = #tk{string = ":"},
% Con = #tk{type = con}
% | NewTokens]) ->
% slurp_ct_impls2([Con], NewTokens);
%slurp_ct_impls(NewTokens = [#tk{string = "="} | _]) ->
% {slurp, [], NewTokens};
%slurp_ct_impls([BadToken | _]) ->
% Msg = efmt("slurp_ct_impls: expecting (: Con | =), got ~p", [BadToken]),
% {error, {fixme, Msg}};
%slurp_ct_impls([]) ->
% {error, {fixme, "expecting : or =, got end-of-input"}}.
%
%
%slurp_ct_impls2(Stk, [_ = #tk{string = ","},
% Con = #tk{type = con}
% | NewTokens]) ->
% slurp_ct_impls2([Con | Stk], NewTokens);
%slurp_ct_impls2(Stk, NewTokens = [#tk{string = "="} | _]) ->
% {slurp, lists:reverse(Stk), NewTokens};
%slurp_ct_impls2(Stk, BadTokens) ->
% {error, {fixme, nyi, slurp_ct_impls2, [Stk, BadTokens]}}.
%
%
%% Decl ::= 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% | 'record' Id ['(' TVar* ')'] '=' RecordType
%% | 'datatype' Id ['(' TVar* ')'] '=' DataType
%% | 'let' Id [':' Type] '=' Expr
%% | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
%% | Using
%gulp_decl(Tokens) ->
% case gsc_tokens:strings(1, Tokens) of
% ["type"] -> gulp_type_alias(Tokens);
% _ -> gulp_nyi(Tokens)
% end.
%
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%gulp_type_alias(Tokens) ->
% gulp_ta(#ast_ta{}, Tokens).
%
%
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% ^
%gulp_ta(Ast = #ast_ta{alias = none},
% _ = [#tk{string = "type"}, Alias = #tk{type = id}
% | NewTokens]) ->
% NewAst = Ast#ast_ta{alias = Alias},
% gulp_ta(NewAst, NewTokens);
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% ^
%gulp_ta(Ast = #ast_ta{tvars = none},
% Tokens) ->
% case slurp_ta_tvars(Tokens) of
% {slurp, TVars, NewTokens} ->
% NewAst = Ast#ast_ta{tvars = TVars},
% gulp_ta(NewAst, NewTokens);
% Error -> Error
% end;
%% 'type' Id ['(' TVar* ')'] '=' TypeAlias
%% ^
%gulp_ta(Ast = #ast_ta{points_to = none},
% _ = [#tk{string = "="} | NewTokens]) ->
% case gulp_type_expr(NewTokens) of
% {gulp, TypeExpr} ->
% Result = Ast#ast_ta{points_to = TypeExpr},
% {gulp, Result};
% Error -> Error
% end;
%gulp_ta(A, B) ->
% {error, {fixme, gulp_ta, A, B, ?MODULE, ?LINE}}.
%
%
%% type foo(bar, baz) = quux
%% ^
%%
%% "(bar, baz) = ..." ~> {slurp, [bar, baz], "= ..."}
%slurp_ta_tvars(Tks = [#tk{string = "="} | _]) ->
% {slurp, [], Tks};
%slurp_ta_tvars([#tk{string = "("},
% #tk{string = ")"}
% | NewTokens]) ->
% {slurp, [], NewTokens};
%slurp_ta_tvars([_ = #tk{string = "("},
% TVar = #tk{type = tvar}
% | NewTokens]) ->
% slurp_tavars([TVar], NewTokens).
%
%
%slurp_tavars(Stk,
% [_ = #tk{string = ","},
% TVar = #tk{type = tvar}
% | NewTks]) ->
% slurp_tavars([TVar | Stk], NewTks);
%slurp_tavars(Stk,
% [#tk{string = ")"}
% | NewTks]) ->
% {slurp, lists:reverse(Stk), NewTks}.
%
%
%gulp_type_expr(Tokens) ->
% case slurp_type_expr(Tokens) of
% {slurp, T, []} -> {gulp, T};
% {slurp, _, E} -> {error, {fixme, trailing, E}};
% Error -> Error
% end.
%
%
%% Type ::= Domain '=>' Type // Function type
%% | Type '(' Sep(Type, ',') ')' // Type application
%% | '(' Type ')' // Parens
%% | 'unit'
%% | Sep(Type, '*') // Tuples
%% | Id
%% | QId
%% | TVar
%%
%% Domain ::= Type // Single argument
%% | '(' Sep(Type, ',') ')' // Multiple arguments
%
%% terminals:
%% TkType = id | qid | tvar
%% PList = PList0 | PList1 | PList2
%% ProdType = Type [* Type]+
%% LamType = Type [=> Type]+
%%
%% PList0 = ()
%% PList1 = (Type)
%% PList2 = (Type [, Type]+)
%%
%% Ah ok:
%%
%% Type1 = TkType | PList(Type)
%%
%% Type = Type1 % string, (string), (string, string)
%% | Type1 => Type % function
%% | Type1 PList(Type) % application
%% | Type1 * Type % product
%
%
%% each type either starts with an id/qid/tvar token or an open paren
%%
%% we're going to eliminate open paren cases first:
%%
%% legal cases:
%%
%% 1. Function types can have arbitrary #args:
%%
%% () => ... // legal
%% (_) => ... // legal
%% (_, _) => ... // legal
%%
%% 2. Otherwise it must be a plist of ONLY one type
%%
%% () // illegal, should be "unit"
%% (X) // legal, equiv to X
%% (X, Y) // illegal, should be X * Y
%%
%slurp_type_expr(Tks = [#tk{pos = Pos} | _]) ->
% case slurp_type1(Tks) of
% % plist cases
% % function types _ => _
% {slurp, {plist, PList}, NewTks} ->
% case {PList, NewTks} of
% % ANY plist (including (), (foo)) followed by => is fine
% % () => _
% % (_) => _
% % (_, _) => _
% {DomainType, [#tk{string = "=>"} | After]} ->
% case slurp_type_expr(After) of
% {slurp, CodomainType, NewTks} ->
% TypeExpr = #ast_te_fn{dom = DomainType,
% codom = CodomainType},
% {slurp, TypeExpr, NewTks};
% Error = {error, _} ->
% Error
% end;
% % WMA not a function type, so only plists with 1 type are
% % allowed
% {[One], After} ->
% slurp_type_expr2(One, After);
% % plists of 0 or 2+ not allowed
% {[], _} ->
% Msg = "() is called unit",
% {error, {fixme, ?MODULE, ?LINE, illegal_type, Pos, Msg}};
% {_, _} ->
% Msg = "(_, _, ...) must be followed by =>; "
% "the type for tuples (A, B) is A * B",
% {error, {fixme, ?MODULE, ?LINE, illegal_type, Pos, Msg}}
% end;
% % first token is id|qid|tvar: foo, Foo.bar, 'foo
% {slurp, {token, Tk}, NewTks} ->
% case NewTks of
% % foo => bar is fine, normalized to (foo) => bar
% [#tk{string = "=>"} | After] ->
% case slurp_type_expr(After) of
% {slurp, Codom, NewAfter} ->
% ArgType = #ast_te_tk{token = ArgTypeTk},
% Dom = [ArgType],
% Result = #ast_te_fn{dom = Dom, codom = Codom},
% {slurp, Result, NewAfter};
% Error = {error, _} ->
% Error
% end;
% {_, _} ->
% One = #ast_te_tk{token = Tk},
% slurp_type_expr2(One, After)
% end;
% Error = {error, _} ->
% Error
% end.
%
%
%% parsed the first type, now need to see if this is an application
%% or a product or if we're done
%slurp_type_expr2(FirstType, After) ->
% % fixme: just going to ignore products and applications for now
% {slurp, FirstType, After}.
% %case strings(1, After) of
% % ["("] -> error(nyi);
% % ["*"] -> error(nyi);
% %case After of
% % [#tk{string = "("} | _] ->
% % case slurp_type1_II(After) of
% % {slurp, {plist, ArgTypes}, NewAfter} ->
% % {slurp, #ast_te_ap{fn = FirstType
%
%
%
%
%% Type1 = {plist, Types} () (foo) (foo, bar)
%% | {token, #tk{}} foo Bar.baz 'quux
%slurp_type1(Tks) ->
% case gsc_tokens:slurp_plist(Tks) of
% % head token is NOT open paren -> must be id/qid/tvar
% {slurp, [], [Tk | NewTks]} ->
% TkType = Tk#tk.type,
% case TkType of
% id -> {slurp, {token, Tk}, NewTks};
% qid -> {slurp, {token, Tk}, NewTks};
% tvar -> {slurp, {token, Tk}, NewTks};
% _ -> {error, {fixme, illegal_type, Tk, ?MODULE, ?LINE}}
% end;
% % head token is an open paren, need to recursively consume
% % the inner types
% {slurp, PListTokens, After} ->
% case gulp_ptype1(PListTokens) of
% {gulp, Types} -> {slurp, {plist, Types}, After};
% Error = {error, _} -> Error
% end;
% Error = {error, _} -> Error
% end.
%
%
%%slurp_type_expr_plist(Tks) ->
%% case gsc_tokens:slurp_plist(Tks) of
%% % head token is NOT open paren -> must be id/qid/tvar
%% {slurp, [], [Tk | NewTks]} ->
%% TkType = Tk#tk.type,
%% case TkType of
%% id -> {slurp, {token, Tk}, NewTks};
%% qid -> {slurp, {token, Tk}, NewTks};
%% tvar -> {slurp, {token, Tk}, NewTks};
%% _ -> {error, {fixme, illegal_type, Tk}}
%% end;
%% % head token is an open paren, need to recursively consume
%% % the inner types
%% {slurp, PListTokens, After} ->
%% case gulp_ptype1(PListTokens) of
%% {gulp, Types} -> {slurp, {plist, Types}, After};
%% Error = {error, _} -> Error
%% end;
%% Error = {error, _} -> Error
%% end.
%
%gulp_ptype1([#tk{string = "("}, #tk{string = ")"}]) ->
% {gulp, []};
%gulp_ptype1([#tk{string = "("} | Tail]) ->
% gulp_ptype1_II([], Tail).
%
%gulp_ptype1_II(Stk, Tks) ->
% case slurp_type_expr(Tks) of
% {slurp, NewType, [#tk{string = ")"}]} ->
% {gulp, lists:reverse([NewType | Stk])};
% {slurp, NewType, [#tk{string = ","} | NewTks]} ->
% gulp_ptype1_II([NewType | Stk], NewTks);
% Error = {error, _} ->
% Error
% end.
%
%
%%gulp_te_tk([Tk = #tk{type = TkType}])
%% when id =:= TkType;
%% qid =:= TkType;
%% tvar =:= TkType ->
%% {gulp, #ast_te_tk{token = Tk}};
%%gulp_te_tk(Tks) ->
%% {error, {fixme, nyi, gulp_te_tks, Tks}}.
%
%
%gulp_nyi(Tokens) ->
% {gulp, #ast_nyi{tokens = Tokens}}.
%
%
%drop(N, [_ | Xs]) when is_integer(N), N >= 1 ->
% drop(N-1, Xs);
%drop(_, []) ->
% [];
%drop(0, X) ->
% X.
%
%efmt(Fmt, Args) ->
% unicode:characters_to_list(io_lib:format(Fmt, Args)).
src/gsc_parse_type_expr.erl
-216
View File
@@ -1,216 +0,0 @@
-module(gsc_parse_type_expr).
-export_type([
]).
-export([
unsafe_vtks_from_string/1,
gulp_vtks/1,
take_until_ifx_op/1
]).
-include("$gsc_include/gsc.hrl").
%------------------------------------------------------
% TYPES
%------------------------------------------------------
-type vtk_ifx_op() :: vtk_apply_to
| {'vtk_*', tk()}
| {'vtk_=>', tk()}.
-type vtk() :: tk()
| {vtk_plist, [tk()]}
| vtk_ifx_op().
-type gulped(X) :: {gulp, X}
| {error, any()}.
-type slurped(X) :: {slurp, X, Rest :: [tk()]}
| {error, any()}.
%------------------------------------------------------
% FUNCTIONS
%------------------------------------------------------
-spec unsafe_vtks_from_string(SrcStr) -> Vtks when
SrcStr :: string(),
Vtks :: [vtk()].
% @doc for testing
unsafe_vtks_from_string(S) ->
{ok, SigTks} = gsc_tokens:significant_tokens(S),
{gulp, Vtks} = gulp_vtks(SigTks),
Vtks.
% operators in descending order of exteriority
%
% precedence verbiage confuses me
%
% 1 + 2 * 3 ^ 4
%
% (+ 1 (* 2 (^ 3 4)))
%
% precedence is thinking about the operators as like
% having arms and legs and doing something. how much
% power do they have to bind to their neighbors. it's
% thinking of your operators as verbs in some state
% machine.
%
% i don't like when i have to model the state machine
% in my head when i'm reading code. everything should
% just be there. exteriority and interiority are
% properties of nouns, not properties of verbs.
%
% functional programming is all about offloading as
% much of your reasoning into nouns as possible. verbs
% change things. and change is bad.
-type chunk_strategy()
:: ast_parens % prefix paren (x y z)
| '=>' % infixr
| '*' % infix
| 'apply' % postfix paren f (...)
.
-record(ast_parens,
{open = none :: none | tk(),
inner = none :: none | [tk()],
close = none :: none | tk()}).
chunk_by(Strategy, Tokens) ->
chunk_by(Strategy, [], Tokens).
-spec chunk0(Strategy, Oks, Errs, Tokens) -> Result when
Strategy :: chunk_strategy(),
Oks :: [any()],
Errs :: [{error, Reason :: any()}],
Tokens :: [tk()],
Result :: {ok,
gulp_chunks_by(_, Stk, [], []) ->
{gulp, lists:reverse(Stk)};
gulp_chunks_by(_, _, Errs, []) ->
{error, {fixme, {?MODULE, ?LINE}, Errs}};
gulp_chunks_by(plist, Stk, Errs, Tokens) ->
case slurp_plist_rec(Tokens) of
{slurp, Plist, NewTokens} ->
gulp_chunks_by(plist, [PList | Stk], Errs, NewTokens);
barf ->
[Token | NewTokens] = Tokens,
gulp_chunks_by(plist, [Token | Stk], Errs, NewTokens);
Error ->
gulp_chunks_by(plist, Stk, [Error | Errs], Tokens);
end.
slurp_plist_rec(Tokens = [#tk{string = "(" | _]) ->
case gsc_tokens:slurp_plist(Tokens) of
{slurp, [], _} ->
barf;
{slurp, PTokens, NewTokens} ->
PTokensInner = pt_inner(PTokens),
end;
%-spec gulp_ifx_tree(Tokens) -> gulped(IfxTree) when
% Tokens :: [tk()],
% IfxTree :: ifx_tree().
%
%-spec chunk_by(ChunkStrategy, Tokens) -> Result when
% ChunkStrategy :: chunk_strategy(),
% Tokens :: [tk()],
% Result :: {ChunkStrategy,
-spec gulp_vtks(Tokens) -> Result when
Tokens :: [tk()],
Result :: gulped(VirtualTokens),
VirtualTokens :: [vtk()].
% @private
% for infix precedence/associativity resolution.
%
% better/original name (although misnomer) was
% insert_virtual_tokens. we're using the gulp verbiage
% because we're guarding against mistmatched delimiters
% @end
gulp_vtks(Tokens) ->
gulp_vtks([], Tokens).
gulp_vtks(Acc, Tks0) ->
% scan until next "(" | "*" | "=>"
case take_until_ifx_op(Tks0) of
% no infix op remaining, return arg
{_Pfx = Tks0,
_Sfx = []} ->
{gulp, lists:flatten([Acc, Tks0])};
% application
% "... foo(bar, baz) ..."
% ~> [..., foo, {plist, "(bar, baz)"}, ...]
{_Pfx = Tks1_BeforeOpen,
_Sfx = Tks2_OpenNAfter
= [#tk{string = "("} | _]} ->
case gsc_tokens:slurp_plist(Tks2_OpenNAfter) of
{slurp, Tks2A_OpenToClose, Tks2B_AfterClose} ->
NewAcc = [Acc,
Tks1_BeforeOpen,
vtk_apply_to,
{vtk_plist, Tks2A_OpenToClose}],
gulp_vtks(NewAcc, Tks2B_AfterClose);
Error = {error, _} ->
Error
end;
% product
{_Pfx = Tks0_BeforeTimes,
_Sfx = [ Tk1A_Times = #tk{string = "*"}
| Tks1B_AfterTimes]} ->
NewAcc = [Acc,
Tks0_BeforeTimes,
{'vtk_*', Tk1A_Times}],
gulp_vtks(NewAcc, Tks1B_AfterTimes);
% funType
{_Pfx = Tks0_BeforeOp,
_Sfx = [ Tk1A_Op = #tk{string = "=>"}
| Tks1B_AfterOp]} ->
NewAcc = [Acc,
Tks0_BeforeOp,
{'vtk_=>', Tk1A_Op}],
gulp_vtks(NewAcc, Tks1B_AfterOp)
end.
-spec take_until_ifx_op(Tokens) -> Result when
Tokens :: [tk()],
Result :: {Taken, NewTokens},
Taken :: Tokens,
NewTokens :: Tokens.
% @doc
% consume tokens until one of ["(", "*", "=>"]
take_until_ifx_op(Tks) ->
take_until_ifx_op([], Tks).
take_until_ifx_op(Stack, []) ->
{lists:reverse(Stack), []};
take_until_ifx_op(Stack, Tokens = [Token | NewTokens]) ->
TokStr = Token#tk.string,
Continue =
case TokStr of
% exit cases
"(" -> false;
"*" -> false;
"=>" -> false;
_ -> true
end,
case Continue of
true -> take_until_ifx_op([Token | Stack], NewTokens);
false -> {lists:reverse(Stack), Tokens}
end.
src/gso_scan.erl
+16 -16
View File
@@ -1,6 +1,6 @@
% @doc compatibility layer to test against so_scan
%
% converts gsc_tokens data to so_scan tokens
% converts gs_tokens data to so_scan tokens
%
% Ref: so_scan.erl
-module(gso_scan).
@@ -76,14 +76,14 @@
-type so_symbol() :: so_kwd() | so_special_char() | atom().
-type so_token2() :: {Symbol :: so_symbol(),
Location :: gsc_pos()}.
Location :: tk_pos()}.
% FIXME
% this is 'id', 'con', qid
-type so_tk3type() :: char | string | hex | int | bytes | qid | qcon | tvar | id | con.
-type so_token3() :: {TokenType :: so_tk3type(),
Location :: gsc_pos(),
Location :: tk_pos(),
TokenValue :: term()}.
-type so_token() :: so_token2() | so_token3().
@@ -104,7 +104,7 @@
% @end
scan(SrcStr) ->
case gsc_tokens:tokens(SrcStr) of
case gs_tokens:tokens(SrcStr) of
{ok, SfLTokens} ->
SoTokens = to_so_tokens(SfLTokens),
{ok, SoTokens};
@@ -151,12 +151,12 @@ scan(SrcStr) ->
%
% so if we see an ak/ct/sg token, we summon evil ben
% carson to reconjoin the unconjoined twins
to_so_tokens([ AkTok = #tk{type = AkCtSg, pos = Pos}
to_so_tokens([ AkTok = #tk{shape = AkCtSg, pos = Pos}
| Sheeit])
when ak =:= AkCtSg;
ct =:= AkCtSg;
sg =:= AkCtSg ->
{#tk{string = FinalAkStr}, NewSheeit}
{#tk{str = FinalAkStr}, NewSheeit}
= ken_barson_rises(AkTok, Sheeit),
[{id, Pos, FinalAkStr}| to_so_tokens(NewSheeit)];
% this part is just lists:filtermap
@@ -259,9 +259,9 @@ to_so_tokens([]) ->
% `_`**: `smr_plus` requires >=1 base58 char to
% match; `ak_I`, `ak_0`, `ak__bar` all fall
% through to `id` and both tokenizers agree.
ken_barson_rises(AkTokAcc = #tk{string = AkStr},
SrcTokens = [#tk{type = CandidateType,
string = CandidateString}
ken_barson_rises(AkTokAcc = #tk{str = AkStr},
SrcTokens = [#tk{shape = CandidateType,
str = CandidateString}
| Rest]) ->
% candidate:
% dig out the token type and the string
@@ -273,7 +273,7 @@ ken_barson_rises(AkTokAcc = #tk{string = AkStr},
Smash ->
% dig out the token from LcTokApi
NewAkStr = AkStr ++ CandidateString,
NewAkTokAcc = AkTokAcc#tk{string = NewAkStr},
NewAkTokAcc = AkTokAcc#tk{str = NewAkStr},
ken_barson_rises(NewAkTokAcc, Rest);
Pass ->
{AkTokAcc, SrcTokens}
@@ -320,9 +320,9 @@ pass_types() ->
% follow-on tokens
% @end
to_so_token(#tk{type = SfTokenType,
pos = Pos,
string = SfTokenStr}) ->
to_so_token(#tk{shape = SfTokenType,
pos = Pos,
str = SfTokenStr}) ->
case SfTokenType of
%-----------------
% Ignored
@@ -371,7 +371,7 @@ to_so_token(#tk{type = SfTokenType,
NYI ->
Msg = io_lib:format("gsc_so_scan:to_so_token/1: unhandled token shape: ~p", [NYI]),
error(#gsc_err{atom = nyi,
string = Msg})
str = Msg})
end.
%% ak/ct/sg all tokenize to id
@@ -393,7 +393,7 @@ so_parse_char([$' | Chars]) ->
[Char] -> Char;
_Bad ->
error(#gsc_err{atom = bad_token,
string = "Bad character literal: '" ++ Chars})
str = "Bad character literal: '" ++ Chars})
end.
so_parse_string([$" | Chars]) ->
@@ -435,7 +435,7 @@ unescape(Delim, [$\\, Code | Chars], Acc) ->
$t -> Ok($\t);
$v -> Ok($\v);
_ -> error(#gsc_err{atom = bad_escape_char,
string = "Bad control sequence: \\" ++ [Code]}) %% TODO
str = "Bad control sequence: \\" ++ [Code]}) %% TODO
end;
unescape(Delim, [C | Chars], Acc) ->
unescape(Delim, Chars, [C | Acc]).