gsc_test_ytree: task now fails successfully

i have realized that a lot of the complexity and cognitive overhead
is downstream from trying to prematurely handle errors.

so i'm going to do the erlang thing and just assert/assume everything
worked perfectly and let the errors speak for themselves.

so re "task fails successfully": now instead of getting an NYI that
looks like a success result, we're now properly getting an error.

.gitignore

@@ -4,6 +4,7 @@ deps
 *.beam
 *.plt
 *.swp
+*.swo
 erl_crash.dump
 ebin/*.beam
 doc/*.html

scratch/sfc00.erl

@@ -0,0 +1,263 @@
+
+-spec consume_token(SourceStr) -> {tok, Token, Rest}
+                                | none
+    when SourceStr :: string(),
+         Token     :: token(),
+         Rest      :: string().
+
+consume_token(SrcStr) ->
+    consume_token_II(token_parsers(), SrcStr).
+
+
+consume_token_II([TokenParser | Rest], SrcStr) ->
+    case consume_token(TokenParser, SrcStr) of
+        none -> consume_token_II(Rest, SrcStr);
+        Tok  -> Tok;
+    end;
+consume_token_II([], _) ->
+    none.
+
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+consume_token(identifier, [LowerUnder | Rest])
+        when (($a =< LowerUnder) andalso (LowerUnder =< $z))
+             orelse ($_ =:= LowerUnder)
+        ->
+    {TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
+    TokenStr = [LowerUnder | TokenTail],
+    Token = #identifier{str = TokenStr},
+    {tok, Token, NewRest};
+consume_token(identifier, _) ->
+    none;
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+consume_token(constructor, [Upper | Rest])
+        when ($A =< Upper) andalso (Upper =< $Z) ->
+    {TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
+    TokenStr = [Upper | TokenTail],
+    Token = #constructor{str = TokenStr},
+    {tok, Token, NewRest};
+consume_token(constructor, _) ->
+    none;
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+consume_token(qualified_identifier, Str) ->
+    case consume_qualifiers(Str) of
+        {[], _} ->
+            none;
+        {Qualifiers, Rest0} ->
+            case consume_token(identifier, Rest0) of
+                {tok, Identifier, Rest1} ->
+                    Token = #qualified_identifier{qualifiers = Qualifiers,
+                                                  identifier = Identifier},
+                    {tok, Token, Rest1};
+                none ->
+                    none
+            end
+    end;
+consume_token(qualified_constructor, Str) ->
+    case consume_qualifiers(Str) of
+        {[], _} ->
+            none;
+        {Qualifiers, Rest0} ->
+            case consume_token(constructor, Rest0) of
+                {tok, Constructor, Rest1} ->
+                    Token = #qualified_constructor{qualifiers  = Qualifiers,
+                                                   constructor = Constructor},
+                    {tok, Token, Rest1};
+                none ->
+                    none
+            end
+    end;
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+consume_token(type_variable, [$' | Rest0]) ->
+    case consume_token(identifier, Rest0) of
+        {tok, Identifier, Rest1} ->
+            Token = #type_variable{identifier = Identifier},
+            {tok, Token, Rest1};
+        none ->
+            none
+    end;
+consume_token(type_variable, _) ->
+    none;
+consume_token(int, Str) ->
+    case {consume_int16(Str), consume_int10(Str)} of
+        {{tok, Int16, Rest}, _} ->
+            {tok, #int{inner_int = Int16}, Rest};
+        {none, {tok, Int10, Rest}} ->
+            {tok, #int{inner_int = Int10}, Rest};
+        {none, none} ->
+            none
+    end;
+consume_token(string, _) ->
+    none;
+consume_token(char, _) ->
+    none;
+consume_token(akstr, _) ->
+    none;
+consume_token(ctstr, _) ->
+    none;
+consume_token(sgstr, _) ->
+    none.
+
+
+
+-spec consume_int16(SrcStr) -> {tok, #int16{}, Rest} | none
+    when SrcStr :: string(),
+         Rest   :: string().
+
+consume_int16("0x" ++ Rest0) ->
+    case consume_hex_digits(Rest0) of
+        {piece16, Piece0, Rest1} ->
+            {Pieces, Rest2} = consume_uhexs(Rest1),
+            Token = #int16{hex_pieces = [Pieces0 | Pieces]},
+            {tok, Token, Rest2};
+        none ->
+            none
+    end;
+consume_int16(_) ->
+    none.
+
+
+
+-spec takewhile(Pred, List) -> {Taken, Rest}
+    when Pred  :: fun( (Item) -> boolean() ),
+         List  :: [Item],
+         Item  :: any(),
+         Taken :: [Item],
+         Rest  :: [Item].
+
+takewhile(Pred, List) ->
+    takewhile(Pred, List, []).
+
+
+takewhile(Pred, [X | Xs], Acc) -> 
+    case Pred(X) of
+        true  -> takewhile(Pred, Xs, [X | Acc]);
+        false -> {lists:reverse(Acc), Xs};
+    end;
+takewhile(_, [], Acc) -> 
+    {lists:reverse(Acc), []}.
+
+
+-spec is_ulduq(Char) -> boolean()
+    when Char :: integer().
+% @doc upper lower digit underscore quote
+
+is_ulduq(C) ->
+    (($A =< C) andalso (C =< $Z))
+        orelse (($a =< C) andalso (C =< $z))
+        orelse (C =:= $_)
+        orelse (C =:= $').
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+-record(id,
+        {str :: string()}).
+
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+-record(con,
+        {str :: string()}).
+
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+-record(qid,
+        {qualifiers :: [#con{}],
+         identifier :: #id{}}).
+
+% - `QCon = (Con\.)+Con` qualified constructor
+-record(qcon,
+        {qualifiers  :: [#con{}],
+         constructor :: #con{}}).
+
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+-record(tvar,
+        {identifier :: #identifier{}}).
+
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+-record(int16,
+        {hex_pieces :: [string()]}).
+
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+-record(int10,
+        {dec_pieces :: [string()]}).
+
+-type int() :: int10() | int16().
+
+
+% @doc
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+% - `QCon = (Con\.)+Con` qualified constructor
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+% - `Bytes = #[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` byte array literal with optional `_` separators
+% - `String` string literal enclosed in `"` with escape character `\`
+% - `Char` character literal enclosed in `'` with escape character `\`
+% - `AccountAddress` base58-encoded 32 byte account pubkey with `ak_` prefix
+% - `ContractAddress` base58-encoded 32 byte contract address with `ct_` prefix
+% - `Signature` base58-encoded 64 byte cryptographic signature with `sg_` prefix
+-type token() :: #id{}
+               | #con{}
+               | #qid{}
+               | #tvar{}
+               | int()
+               | {string, string()}
+               | {char, string()}
+               | {akstr, string()}
+               | {ctstr, string()}
+               | {sgstr, string()}.
+
+-type token_shape() :: id
+                     | con
+                     | qid,
+                     | qcon
+                     | tvar
+                     | int
+                     | string
+                     | char
+                     | akstr
+                     | ctstr
+                     | sgstr.
+
+%-----------------------------------------
+% functions
+%-----------------------------------------
+
+-spec token_parsers() -> [atom()].
+% in parse order
+
+token_parsers() ->
+    [id,
+     con,
+     qid,
+     qcon,
+     tvar,
+     int,
+     string,
+     char,
+     akstr,
+     ctstr,
+     sgstr].
+
+
+-spec token_consume(Matcher, SourceString) -> MaybeToken
+    when Matcher      :: token_shape(),
+         SourceString :: string(),
+         MaybeToken   :: {match, Token :: token(), Rest :: string()}
+                       | nomatch.
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+token_consume(id, [AZU | Src1]) when $_ =:= AZU; $a =< AZU, AZU =< $z ->
+    {RestOfId, Src2} = tdw(fun is_ulduq/1, Src1),
+    {match, #id{str = [AZU | RestOfId]}, Src2};
+token_consume(id, _) ->
+    nomatch;
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+token_consume(con, [AZ | Src1]) -> when $A =< AZU, AZU =< $Z ->
+    {RestOfCon, Src2} = tdw(fun is_ulduq/1, Src1),
+    {match, #con{str = [AZ | RestOfCon]}, Src2};
+token_consume(con, _) ->
+    nomatch;
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+token_consume(qid, Src0) ->
+    case one_or_more({seq, con, {char, $.}}, Src0)
+
+

scratch/sfc_ast.erl

@@ -0,0 +1,329 @@
+% @doc
+% type for an SFC AST
+%
+% semantically:
+%       #pr{}       means the parser succeeded
+%       noparse     means the parser didn't consume any tokens
+%       {error, _}  means the parser can prove the user made an
+%                   error (e.g. unterminated open paren)
+%
+% based on so_syntax.erl
+%
+% this is the second layer in the parsing pipeline after tokens
+%
+% quoth `sophia/docs/so_syntax.md`:
+%
+%   ## Declarations
+%
+%   A Sophia file consists of a sequence of *declarations* in a layout block.
+%
+%   ```c
+%   File ::= Block(TopDecl)
+%
+%   TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
+%             | 'contract' 'interface' Con [Implement] '=' Block(Decl)
+%             | 'namespace' Con '=' Block(Decl)
+%             | '@compiler' PragmaOp Version
+%             | 'include' String
+%             | Using
+%
+%   Implement ::= ':' Sep1(Con, ',')
+%
+%   Decl ::= 'type'     Id ['(' TVar* ')'] '=' TypeAlias
+%          | 'record'   Id ['(' TVar* ')'] '=' RecordType
+%          | 'datatype' Id ['(' TVar* ')'] '=' DataType
+%          | 'let'      Id [':' Type]      '=' Expr
+%          | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
+%          | Using
+%
+%   FunDecl ::= Id ':' Type                             // Type signature
+%             | Id Args [':' Type] '=' Block(Stmt)      // Definition
+%             | Id Args [':' Type] Block(GuardedDef)    // Guarded definitions
+%
+%   GuardedDef ::= '|' Sep1(Expr, ',') '=' Block(Stmt)
+%
+%   Using ::= 'using' Con ['as' Con] [UsingParts]
+%   UsingParts ::= 'for' '[' Sep1(Id, ',') ']'
+%                | 'hiding' '[' Sep1(Id, ',') ']'
+%
+%   PragmaOp ::= '<' | '=<' | '==' | '>=' | '>'
+%   Version  ::= Sep1(Int, '.')
+%
+%   EModifier ::= 'payable' | 'stateful'
+%   FModifier ::= 'stateful' | 'private'
+%
+%   Args ::= '(' Sep(Pattern, ',') ')'
+%   ```
+-module(sfc_ast).
+
+-export_type([
+]).
+
+-export([
+    parse/2
+]).
+
+-include("sfc.hrl").
+
+
+%%-----------------------------
+%% FUNCTIONS
+%%-----------------------------
+
+-spec parse(ParseTarget, SigTokens) -> Result
+    when ParseTarget :: parse_target(),
+         SigTokens   :: list(sfc_token()),
+         Result      :: perhaps_parsed().
+% @doc
+% parse target against list of tokens;
+%
+% for MVP, it is assumed the given list of tokens
+% contains
+
+parse({ast, AstTarget}, Tokens)        -> parse_ast(AstTarget, Tokens);
+parse({tokens, Target}, Tokens)        -> parse_tokens(Target, Tokens);
+parse({prs, Strategy, Target}, Tokens) -> parse_prs(Strategy, Target, Tokens).
+
+
+
+-spec parse_ast(AstTarget, SigTokens) -> Perhaps
+    when AstTarget :: ast_target(),
+         SigTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private parse where result is an AST
+%
+% happy path: returns
+%
+%   #pr{type   = ast,
+%       result = AST :: ast(),
+%       rest   = _ :: [sfc_token()]}
+
+parse_ast(AstTarget, SrcTokens) ->
+    case AstTarget of
+        file     -> parse_ast_file(SrcTokens);
+        top_decl -> parse_ast_top_decl(SrcTokens);
+        _        -> {error, #sfc_err_nyi{}}
+    end.
+
+
+
+-spec parse_ast_file(SrcTokens) -> Perhaps
+    when SrcTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private parse an #ast_file{} record
+%
+% on success returns
+%   #pr{type   = ast,
+%       result = #ast_file{},
+%       rest   = []}
+
+parse_ast_file([]) ->
+    noparse;
+    %{error, #sfc_err_empty_file{}};
+parse_ast_file(SrcTks0) ->
+    case parse({prs, plus, {ast, top_decl}}, SrcTks0) of
+        #pr{type=prs, result=TopDeclPrs, rest=SrcTks1} ->
+            TopDeclAsts = [Ast || #pr{type=ast, result=Ast} <- TopDeclPrs],
+            AstFile =
+                #ast_file{top_decls = TopDeclAsts},
+            ParseResult = #pr{type   = ast,
+                              result = AstFile,
+                              rest   = SrcTks1},
+            ParseResult;
+        Nope ->
+            Nope
+    end.
+
+
+
+-spec parse_ast_top_decl(SrcTokens) -> Perhaps
+    when SrcTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private
+% parse a TopDecl from tokens
+%
+% on success returns
+%
+%   #pr{type   = ast,
+%       result = #ast_top_decl{},
+%       rest   = [_ : #sfc_token{}]}
+% @end
+
+parse_ast_top_decl(SrcTks0 = [_ | _]) ->
+    PerhapsTks = parse_tokens(block_item, SrcTks0),
+    case PerhapsTks of
+        #pr{type = tokens, result = []} ->
+            noparse;
+        #pr{type=tokens, result=DeclTks, rest=SrcTks1} ->
+            AstResult =
+                #ast_top_decl{tokens = DeclTks},
+            ParseResult = #pr{type   = ast,
+                              result = AstResult,
+                              rest   = SrcTks1},
+            ParseResult;
+        _ ->
+            {error, #sfc_err_nyi{}}
+    end;
+parse_ast_top_decl([]) ->
+    noparse.
+
+
+
+-spec parse_tokens(TokensTarget, SrcTokens) -> Result
+    when TokensTarget :: tokens_target(),
+         SrcTokens    :: [sfc_token()],
+         Result       :: perhaps_parsed().
+% @private this is for parsing a flat list of tokens
+%
+% essentially this is "eat until the next token boundary"
+%
+% on success returns
+%
+%   #pr{type   = tokens,
+%       result = [sfc_token()],
+%       rest   = [sfc_token()]}
+% @end
+
+% need to think carefully about this case;
+% for now parsing against an empty list of tokens
+% is always a success
+parse_tokens(_, []) ->
+    #pr{type = tokens, result = [], rest = []};
+% WMA SrcTokens is nonempty
+parse_tokens(block, SrcTokens) ->
+    {_, BlkIndent} = start_pos(SrcTokens),
+    TakeToken = fun(Tk) -> BlkIndent =< indent_level(Tk) end,
+    {BlkTokens, Rest} = tw(TakeToken, SrcTokens),
+    #pr{type   = tokens,
+        result = BlkTokens,
+        rest   = Rest};
+parse_tokens(block_item, [H | T]) ->
+    BlkIndent = indent_level(H),
+    TakeToken = fun(Tk) -> BlkIndent < indent_level(Tk) end,
+    {BlockTail, Rest} = tw(TakeToken, T),
+    BlkTokens = [H | BlockTail],
+    #pr{type   = tokens,
+        result = BlkTokens,
+        rest   = Rest};
+parse_tokens(_, _) ->
+    {error, #sfc_err_nyi{}}.
+
+
+% takewhile
+tw(Pred, List) ->
+    tw(Pred, [], List).
+
+tw(Pred, Stk, [X | Xs]) ->
+    case Pred(X) of
+        true  -> tw(Pred, [X | Stk], Xs);
+        false -> {lists:reverse(Stk), [X | Xs]}
+    end;
+tw(_, Stk, []) ->
+    {lists:reverse(Stk), []}.
+
+
+
+-spec parse_prs(Strategy, Target, SrcTokens) -> Perhaps
+    when Strategy   :: prs_strategy(),
+         Target     :: parse_target(),
+         SrcTokens  :: [sfc_token()],
+         Perhaps    :: perhaps_parsed().
+% @private on success returns
+%
+%   #pr{type   = prs,
+%       result = [_ :: #pr{}]
+%       rest   = [_ :: #sfc_token{}]}
+
+parse_prs(Strategy, Target, SrcTokens) ->
+    case Strategy of
+        star -> parse_prs_star(Target, SrcTokens);
+        plus -> parse_prs_plus(Target, SrcTokens);
+        _    -> {error, #sfc_err_nyi{}}
+    end.
+
+
+
+-spec parse_prs_star(ParseTarget, SrcTokens) -> Perhaps
+    when ParseTarget :: parse_target(),
+         SrcTokens   :: [sfc_token()],
+         Perhaps     :: perhaps_parsed().
+% @private
+% parse 0 or more of
+
+parse_prs_star(_ParseTarget, []) ->
+    #pr{type = prs, result = [], rest = []};
+parse_prs_star(ParseTarget, SrcTokens) ->
+    parse_prs_star(ParseTarget, [], SrcTokens).
+
+% WMA SrcTokens nonempty
+parse_prs_star(ParseTarget, Stack, SrcTokens) ->
+    case parse(ParseTarget, SrcTokens) of
+        #pr{result=NewItem, rest=NewSrcTokens} ->
+            NewStack = [NewItem | Stack],
+            parse_prs_star(ParseTarget,
+                           NewStack,
+                           NewSrcTokens);
+        noparse ->
+            PRs = lists:reverse(Stack),
+            #pr{type   = prs,
+                result = PRs,
+                rest   = SrcTokens};
+        Error = {error, _} ->
+            Error
+    end.
+
+
+
+-spec parse_prs_plus(ParseTarget, SrcTokens) -> Perhaps
+    when ParseTarget :: parse_target(),
+         SrcTokens   :: [sfc_token()],
+         Perhaps     :: perhaps_parsed().
+% @private
+% parse 1 or more of
+
+parse_prs_plus(ParseTgt, SrcTks0) ->
+    case parse(ParseTgt, SrcTks0) of
+        #pr{result = Hd, rest = SrcTks1} ->
+            parse_prs_star(ParseTgt, [Hd], SrcTks1);
+        Nope ->
+            Nope
+    end.
+
+
+
+-spec start_pos(Tokens) -> Pos
+    when Tokens :: [sfc_token()],
+         Pos    :: sfc_pos().
+% @private
+% start position of NONEMPTY list of tokens
+
+start_pos([#sfc_token{pos = Result} | _]) ->
+    Result.
+
+
+
+-spec end_pos(Tokens) -> Pos
+    when Tokens :: [sfc_token()],
+         Pos    :: sfc_pos().
+% @private
+% end position of NONEMPTY list of tokens
+
+end_pos(Tokens) ->
+    end_pos(Tokens, start_pos(Tokens)).
+
+end_pos([#sfc_token{string = S} | Ts], Pos) ->
+    NewPos = sfc_tokenizer:new_pos(Pos, S),
+    end_pos(Ts, NewPos);
+end_pos([], Result) ->
+    Result.
+
+
+
+-spec indent_level(Token) -> Col
+    when Token :: sfc_token(),
+         Col   :: pos_integer().
+% @private
+% get col val from token
+
+indent_level(#sfc_token{pos = {_, C}}) ->
+    C.

scratch/sfc_group_tokens.erl

@@ -0,0 +1,94 @@
+-module(sfc_group_tokens).
+
+-export_type([
+]).
+
+-export([
+    group_tokens/1
+]).
+
+-include("sfc.hrl").
+
+
+-spec group_tokens(Tokens) -> Result
+    when Tokens     :: [sfc_token()],
+         Result     :: {ok, Groups}
+                     | {error, Reason},
+         Groups     :: [TokenGroup],
+         TokenGroup :: sfc_token()
+                     | #sfc_tg_list{},
+         Reason     :: any().
+
+group_tokens(Tokens) ->
+    group_tokens([], Tokens).
+
+
+group_tokens(Stack, Tokens = [_ | _]) ->
+    case token_or_group(Tokens) of
+        {push, Item, NewTokens} ->
+            group_tokens([Item | Stack], NewTokens);
+        Error ->
+            Error
+    end;
+group_tokens(Stack, []) ->
+    {ok, lists:reverse(Stack)}.
+
+
+% erlang's type language is lacking here...
+% items are always a list of sfc_token_group_items
+% placeholder type
+-type sfc_tg_item_() :: any().
+
+% tg = token group
+% {start_line, indent} should always equal the Pos of the first token
+-record(sfc_tg_block,
+        {start_line = none :: none | pos_integer(),
+         indent     = none :: none | pos_integer(),
+         items      = none :: none | {stack, [sfc_tg_item_()]} | [sfc_tg_item_()]}).
+
+
+% delist = delimited list (...) [...], {...}
+%
+% don't even comma-separate values, just make sure 
+-record(sfc_tg_delist,
+        {open   = none :: none | '(' | '[' | '{',
+         items  = none :: none | {stack, [sfc_tg_item()]} | [sfc_tg_item()],
+         close  = none :: none | ')' | ']' | '}',
+
+-type sfc_tg_item() :: sfc_token(),
+                     | #sfc_tg_block{}
+                     | #sfc_tg_round{}
+                     | #sfc_.
+
+-type sfc_tg() :: [sfc_tg_item()].
+
+-spec token_or_group(Tokens) -> Result
+    when Tokens :: [sfc_token()],
+         Result :: {push, Item, NewTokens}
+                 | {error, Reason}
+         Item   :: sfc_token()
+                 | {block, IndentLevel :: pos_integer(),
+                           
+                 ,
+
+token_or_group(Tokens) ->
+
+
+
+groupist_order() ->
+    [round, square, curly, block].
+
+
+match_groupists([Groupist | Rest], Tokens) ->
+    case groupist(Grupist, Tokens) of
+        Match = {groupmatch, _, _} ->
+            Match
+        no_groupmatch ->
+            match_groupists(Rest, Tokens);
+        Error ->
+            Error
+    end;
+match_groupists(_, []) ->
+    {ok, []};
+match_groupists([], [#sfc_token{pos = Pos} | _}) ->
+    {error, {cannot_make_groupist_decision, Pos}}.

scratch/sfc_token_chunks.erl

@@ -0,0 +1,178 @@
+% @doc
+% helper functions for grabbing collections of tokens
+% off the token stream
+%
+% generally assume no whitespace/comment tokens in
+% input stream
+-module(sfc_token_chunks).
+
+-export_type([
+]).
+
+-export([
+    barf/2
+]).
+
+% $sfc_include is so c() works from sfp eshell
+-include("$sfc_include/sfc.hrl").
+
+%------------------------------------------
+% Types
+%------------------------------------------
+
+-type chunk_shape()
+    :: block
+     | block_item
+     | {block_item, Level :: pos_integer()}
+     .
+
+% FIXME
+-type choke_reason() :: any().
+
+
+-spec barf(ChunkShape, SigTokens) -> Perhaps
+    when ChunkShape :: chunk_shape(),
+         SigTokens  :: Tokens,
+         Perhaps :: {barf, Taken, Rest}
+                  | {choke, Reason},
+         Taken  :: Tokens,
+         Rest   :: Tokens,
+         Reason :: choke_reason(),
+         Tokens :: [sfc_token()].
+
+% @doc
+% slurp/barf terminology comes from paredit mode in
+% emacs
+%
+% slurp ~= accepting input
+% barf ~= separating input
+%
+%   slurp: (foo bar) baz ~> (foo bar baz)
+%   barf : (foo bar baz) ~> foo (bar baz)
+
+barf(_, []) ->
+    {barf, [], []};
+barf(block, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
+    Take =
+        fun(#sfc_token{pos = {_, TkCol}}) ->
+            BlkCol =< TkCol
+        end,
+    {A, B} = tw(Take, T),
+    {barf, [H | A], B};
+barf(block_item, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
+    Take =
+        fun(#sfc_token{pos = {_, TkCol}}) ->
+            BlkCol < TkCol
+        end,
+    {A, B} = tw(Take, T),
+    {barf, [H | A], B}.
+barf({block_item, Level}, Tokens = [#sfc_token{pos = {_, StartLevel}} | _]) ->
+    case Level =:= StartLevel of
+        false -> {barf, [], Tokens};
+        true  -> barf(block_item, Tokens)
+    end.
+barf(_, _) ->
+    {choke, #sfc_err_nyi{}}.
+
+
+% takewhile but returns {Taken, Rest}
+tw(Pred, List) ->
+    tw(Pred, [], List).
+
+tw(Pred, Stk, [X | Xs]) ->
+    case Pred(X) of
+        true  -> tw(Pred, [X | Stk], Xs);
+        false -> {lists:reverse(Stk), [X | Xs]}
+    end;
+tw(_, Stk, []) ->
+    {lists:reverse(Stk), []}.
+
+
+%-spec consume_delist_round(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_round(Tokens = [#sfc_token{type = punct, string = "("} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_round(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist_square(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_square(Tokens = [#sfc_token{type = punct, string = "["} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_square(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist_curly(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_curly(Tokens = [#sfc_token{type = punct, string = "{"} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_curly(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist([Head | Tail]) ->
+%    case Head of
+%        #sfc_token{type = punct, string = OpenStr = "("} ->
+%            CloseStr = ")",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        #sfc_token{type = punct, string = OpenStr = "["} ->
+%            CloseStr = "]",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        #sfc_token{type = punct, string = OpenStr = "{"} ->
+%            CloseStr = "}",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        _ ->
+%            {ok, {[], [Head | Tail]}}
+%    end;
+%consume_delist([]) ->
+%    {ok, {[], []}}.
+%
+%
+%% happy path: have a token and we are in the depth of the list
+%delist_verb(Depth, {OpenStr, CloseStr}, Stack, [Tok | Rest]) when Depth >= 1 ->
+%    NewDepth =
+%        case Tok of
+%            % open token -> increase depth
+%            #sfc_token{type = punct, string = Open} when Open =:= OpenStr ->
+%                Depth + 1;
+%            % close token -> decrease depth
+%            #sfc_token{type = punct, string = Close} when Close =:= CloseStr ->
+%                Depth - 1;
+%            _ ->
+%                Depth
+%        end,
+%    delist_verb(NewDepth, {OpenStr, CloseStr}, [Tok | Stack], Rest);
+%% we've hit end of available input before a close token
+%delist_verb(Depth, {OpenStr, _CloseStr}, _Stack, []) when Depth >= 1 ->
+%    {error, #sfc_err{atom   = unclosed_delimiter,
+%                     string = io_lib:format("unclosed open delimiter: ~p", [OpenStr])}};
+%% we've popped back out of the list, we're done
+%delist_verb(_Depth = 0, {_OpenStr, _CloseStr}, Stack, Tokens) ->
+%    {ok, {lists:reverse(Stack), Tokens}}.

src/gsc_cli.erl

@@ -13,6 +13,7 @@
 %    gso_tokens/1
 %]).
 -export([start/1]).
+-export([do_help/0]).
 
 -include("$gsc_include/gsc.hrl").
 -include("ansi.hrl").
@@ -21,7 +22,53 @@ do_help() ->
     io:format("~ts", [help_screen()]).
 
 help_screen() ->
-    ["you can't help people who refuse to help themsleves\n"].
+    """
+    Usage: gsc <command> [args]
+    COMMANDS
+      (no args)
+      shell
+      eshell
+          Start an Erlang shell with GSC loaded.
+      tokens <file>
+          Print GSC tokens from <file>, one per line.
+      tokens -c <file>
+      tokens --color <file>
+      tcat <file>
+      ctokens <file>
+      color_tokens <file>
+          Print <file> to stdout with each token highlighted in a
+          different color so token boundaries are visible.
+      gso tokens <file>
+      gso_tokens <file>
+          Print sophia-compatible (so_scan) tokens from <file>,
+          one per line.
+      rmm <file>
+          Compile <file> and call its main/0 with GSC context loaded.
+          Useful for running scratch scripts.
+
+    TEST HARNESS  (gsc test ...)
+      gsc test -l
+      gsc test --list
+          List available named tests.
+      gsc test -a
+      gsc test --all
+          Run all named tests.
+      gsc test so tokens <file>
+      gsc test so_tokens <file>
+          Print sophia-original (so_scan) tokens from <file>,
+          one per line.
+      gsc test tokenizers_agree <file>
+          Check whether gsc and so_scan produce identical tokens
+          for <file>.  Prints 'true' or 'false'.
+          <file> is resolved relative to cwd.
+          Example:
+            gsc test tokenizers_agree foo.aes
+      gsc test <name>
+          Run a named test module.  'gsc test foo' and
+          'gsc test gsc_test_foo' are equivalent.
+          Use 'gsc test --list' to see available names.
+    """.
+
 
 -spec start(ArgV) -> ok
     when ArgV :: [string()].
@@ -42,7 +89,7 @@ start(ArgV) ->
 do(["test" | Tests]) ->
     do_test(Tests);
 % slowly phasing out shitty names like lctokens
-% tokens = native sfc token representation
+% tokens = native gsc token representation
 do(["tokens", Foo]) -> do_tokens(Foo);
 % print source file to screen with token boundaries
 % highlighted

src/gsc_signal.erl

@@ -38,11 +38,10 @@ is_block([#tk{pos = {_, BCol}} | Rest]) ->
 
 
 -spec gulp_block_items(Signal) -> Result when
-        Signal :: [tk()],
+        Signal  :: [tk()],
-        Result :: {slurp, Items, NewSignal}
+        Result  :: {gulp, [Item]}
-                | {error, any()},
+                 | {error, any()},
-        Items  :: [Signal],
+        Item    :: Signal.
-        NewSignal :: Signal.
 
 gulp_block_items(S) ->
     case is_block(S) of

src/gsc_tokens.erl

@@ -29,7 +29,7 @@
     take_while/3,
     take_block/1,
     take_block_item/1,
-    strings/2,
+    strings/2, strs/2,
     slurp_plist/1
 ]).
 
@@ -43,6 +43,7 @@
     very_stable_string/1,
     very_stable_characters/1,
     tokens_from_iolist/1,
+    tokens_from_string/1,
     tokens/1,
     slurp_token/2,
     slurp_token_shapes/3,
@@ -57,18 +58,32 @@
 % API: functions
 %=======================================================
 
--spec strings(N, Tokens) -> AtMostNStrings
+-spec strs(N, Tokens) -> AtMostNStrings
-    when N              :: non_neg_integer(),
+    when N              :: infinity | non_neg_integer(),
          Tokens         :: [tk()],
          AtMostNStrings :: [string()].
-% @doc return the strings of the first N tokens
+%% @equiv `strings/2'
 
-strings(N, [#tk{str = S} | Rest]) when is_integer(N), N >= 1 ->
+strs(N, Tks) -> strings(N, Tks).
-    [S | strings(N-1, Rest)];
+
-strings(_, []) ->
+
-    [];
+
-strings(0, _) ->
+-spec strings(N, Tokens) -> AtMostNStrings
-    [].
+    when N              :: infinity | non_neg_integer(),
+         Tokens         :: [tk()],
+         AtMostNStrings :: [string()].
+%% @doc return the strings of the first N tokens
+
+strings(N, Tks) when is_integer(N) ->
+    strsn(N, Tks);
+strings(infinity, Tks) ->
+    [Str || #tk{str = Str} <- Tks].
+
+
+
+strsn(N, [#tk{str = Str} | Rest]) when N >= 1 -> [Str | strsn(N-1, Rest)];
+strsn(_, [])                                  -> [];
+strsn(0, _)                                   -> [].
 
 
 % used by parser
@@ -77,8 +92,7 @@ strings(0, _) ->
 %
 % BLOCK =
 %   foo
-%       ...
+%       ...  bar
-%   bar
 %       ...
 %   baz
 %       ...
@@ -362,10 +376,20 @@ very_stable_codepoints(S) ->
                  | {error, gsc_err()},
          Tokens :: [tk()].
 
-% @doc alias for tokens/1
+%% @equiv `tokens/1'
 tokens_from_iolist(S) -> tokens(S).
 
 
+-spec tokens_from_string(SrcStr) -> Result when
+        SrcStr  :: iolist(),
+        Result  :: {ok, Tokens}
+                 | {error, gsc_err()},
+         Tokens :: [tk()].
+
+%% @equiv `tokens/1'
+tokens_from_string(S) -> tokens(S).
+
+
 
 
 -spec tokens(SrcStr) -> Result

test/gsc_test_ntree.erl

@@ -70,6 +70,7 @@ f2f_op(OpStr, Fst) ->
 % never saw the op
 f2f_op(_opstr, Stk, []) ->
     lists:reverse(Stk);
+
 % see op
 f2f_op(OpStr, LhsStk, [#tk{str = OpStr} = OpTk | Rest]) ->
     Lhf = lists:reverse(LhsStk),

test/gsc_test_tokens.erl

@@ -121,8 +121,8 @@ concat_property(FileName, FilePath) ->
     {FileName ++ ": file = sum(tokens)",
      fun() ->
         case gsc:tokens_from_file(FileChars) of
-            {ok, SfcTokens} ->
+            {ok, GscTokens} ->
-                ConcatStr = concat_token_strs(SfcTokens, []),
+                ConcatStr = concat_token_strs(GscTokens, []),
                 ?assertEqual(FileChars, ConcatStr);
             _Error ->
                 ok

test/gsc_test_ytree.erl

@@ -0,0 +1,194 @@
+-module(gsc_test_ytree).
+
+-export([main/0]).
+-include("$gsc_include/gsc.hrl").
+
+main() ->
+    Src =
+        """
+        contract Foo =
+            type foo = bar
+        """,
+    handle_src_str(Src).
+
+-type syntax_target() :: signal
+                       | file
+                       | {block_of, syntax_target()}
+                       | top_decl
+                           | contract
+                           | interface
+                           | namespace
+                           | '@compiler'
+                           | include
+                       | using
+                       .
+
+-record(ct, {payable  = none :: none | false | true,
+             main     = none :: none | false | true,
+             contract = none :: none | tk(),
+             con      = none :: none | tk(),
+             impls    = none :: none | false | {true, [tk()]},
+             '='      = none :: none | tk()}).
+
+%% @doc This is all the stuff needed to be consumed before parsin
+%% anything in the contract
+-type contract_meta() :: #ct{}.
+
+
+%% @doc
+%% This is all the stuff that has to be parsed before actual parsing
+%% can begin.
+-type stem_meta() :: none
+                   | contract_meta().
+
+% @doc ys = syntax stem
+-record(ys, {target    = signal  :: syntax_target(),
+             signal              :: [tk()],
+             stem_meta           :: stem_meta(),
+             forest              :: [#ys{}]}).
+
+-type syntax_stem() :: #ys{}.
+
+
+handle_src_str(SrcStr) ->
+    Sig = gsc:unsafe_signal_from_string(SrcStr),
+    Result = gulp(file, Sig),
+    io:format("~tp~n", [Result]).
+
+
+-spec gulp(Target, Signal) -> Result when
+        Target :: syntax_target(),
+        Signal :: [tk()],
+        Result :: {gulp, syntax_stem()} | {error, any()}.
+
+gulp(file, Sig) ->
+    gulp({block_of, top_decl}, Sig);
+gulp({block_of, X}, Sig) ->
+    {gulp, BlockItems} = gsc_signal:gulp_block_items(Sig),
+    #ys{target    = {block_of, X},
+        signal    = Sig,
+        stem_meta = none,
+        forest    = [gulp(X, Item) || Item <- BlockItems]};
+gulp(top_decl, Sig) ->
+    % TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
+    %           | 'contract' 'interface' Con [Implement] '=' Block(Decl)
+    %           | 'namespace' Con '=' Block(Decl)
+    %           | '@compiler' PragmaOp Version
+    %           | 'include' String
+    %           | Using
+    NewTarget =
+        case gsc_tokens:strings(3, Sig) of
+            ["namespace", _,           _]           -> namespace;
+            ["@compiler", _,           _]           -> '@compiler';
+            ["include",   _,           _]           -> include;
+            ["using",     _,           _]           -> using;
+            ["payable",   "main",      "contract"]  -> contract;
+            ["payable",   "contract",  "interface"] -> interface;
+            ["payable",   "contract",  _]           -> contract;
+            ["contract",  "interface", _]           -> interface;
+            ["contract",  _,           _]           -> contract;
+            ["main",      "contract",  _]           -> contract;
+            ["main",      "payable",   "contract"]  -> contract
+        end,
+    gulp(NewTarget, Sig);
+gulp(contract, Sig) ->
+    {slurp, CtMeta, BlockSignal} = slurp(#ct{}, Sig),
+    {gulp, BlockItems} = gsc_signal:gulp_block_items(BlockSignal),
+    Forest = [gulp(decl, Item) || Item <- BlockItems],
+    #ys{target = contract,
+        signal = Sig,
+        stem_meta = CtMeta,
+        forest = Forest};
+% Decl ::= 'type'     Id ['(' TVar* ')'] '=' TypeAlias
+%        | 'record'   Id ['(' TVar* ')'] '=' RecordType
+%        | 'datatype' Id ['(' TVar* ')'] '=' DataType
+%        | 'let'      Id [':' Type]      '=' Expr
+%        | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
+%        | Using
+gulp(decl, Sig) ->
+    NewTarget =
+        case strings(1, Sig) of
+            ["type"]     -> type;
+            ["record"]   -> record;
+            ["datatype"] -> datatype;
+            ["let"]      -> 'let';
+            ["using"]    -> using;
+            _            -> fn
+        end,
+    gulp(NewTarget, Sig);
+gulp(NyiTarget, Sig) ->
+    error({nyi, [?MODULE, gulp, NyiTarget, Sig]}).
+
+
+%% ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
+slurp(Meta = #ct{payable = none, main = none}, Sig) ->
+    %% nato phonetic alphabet
+    {Pa, Mi, No} =
+        case gsc_tokens:strings(2, Sig) of
+            ["payable", "main"] -> {true, true, 2};
+            ["main", "payable"] -> {true, true, 2};
+            ["payable", _]      -> {true, false, 1};
+            ["main", _]         -> {false, true, 1};
+            _                   -> {false, false, 0}
+        end,
+    NewMeta = Meta#ct{payable = Pa, main = Mi},
+    NewSig = take(No, Sig),
+    slurp(NewMeta, NewSig);
+slurp(M = #ct{contract = none}, Sig) ->
+    case Sig of
+        [Tk = #tk{str = "contract"} | Rest] ->
+            slurp(M#ct{contract = Tk}, Rest);
+        _ ->
+            error(no_keyword_contract)
+    end;
+slurp(M = #ct{con = none}, Sig) ->
+    case Sig of
+        [Tk = #tk{shape = con} | Rest] ->
+            slurp(M#ct{con = Tk}, Rest);
+        [#tk{str = Irregar} | _] ->
+            error(illegal_contract_name, Irregar);
+        [] ->
+            error(no_contract_name)
+    end;
+slurp(M = #ct{impls = none}, Sig) ->
+    case Sig of
+        [#tk{str = ":"} | Rest] ->
+            {slurp, Impls, NewRest} = slurp({sep1, con, ","}, Rest),
+            NewM = M#ct{impls = {true, Impls}},
+            slurp(NewM, NewRest);
+        _ ->
+            slurp(M#ct{impls = false}, Sig)
+    end;
+slurp(M = #ct{'=' = none}, Sig) ->
+    case Sig of
+        [Tk = #tk{str = "="} | Rest] ->
+            {slurp, M#ct{'=' = Tk}, Rest};
+        _ ->
+            error("where the fuck my = be at nigga")
+    end;
+slurp({sep1, TkShape, SepByStr}, Tks) ->
+    case Tks of
+        [Tk = #tk{shape = TkShape} | Rest] ->
+            slurp_sep1_ii([Tk], TkShape, SepByStr, Rest);
+        _ ->
+            error(irregar)
+    end;
+slurp(BadMeta, BadTokens) ->
+    error({nyi, [?MODULE, slurp, BadMeta, BadTokens]}).
+
+
+slurp_sep1_ii(Stk, TkShape, SepByStr, Sig) ->
+    case Sig of
+        [#tk{str = SepByStr}, T = #tk{shape = TkShape} | Rest] ->
+            slurp_sep1_ii([T | Stk], TkShape, SepByStr, Rest);
+        _ ->
+            {slurp, lists:reverse(Stk), Sig}
+    end.
+
+strings(N, Tks) ->
+    gsc_tokens:strings(N, Tks).
+
+
+take(N, [_ | Rest]) when N >= 1 -> take(N-1, Rest);
+take(0, List)                   -> List;
+take(_, [])                     -> [].

test/gsc_tests.erl

@@ -24,8 +24,8 @@ cli_args(TestNames) ->
     ts_utils:tidily(fun() -> do_gsc_test(TestNames) end).
 
 
-do_gsc_test(["-h" | _])          -> do_help();
+%do_gsc_test(["-h" | _])          -> do_help();
-do_gsc_test(["--help" | _])      -> do_help();
+%do_gsc_test(["--help" | _])      -> do_help();
 do_gsc_test(["-l" | _])          -> do_list();
 do_gsc_test(["--list" | _])      -> do_list();
 do_gsc_test(["-a" | _])          -> do_all();
@@ -35,14 +35,17 @@ do_gsc_test(["so", "tokens", X]) -> do_so_tokens(X);
 do_gsc_test(["tokenizers_agree", X]) ->
     do_tokenizers_agree(X);
 do_gsc_test([TestName]) ->
-    ts_utils:run_test_by_name(TestName);
+    case ts_utils:test_exists_by_name(TestName) of
+        true  -> ts_utils:run_test_by_name(TestName);
+        false -> gsc_cli:do_help()
+    end;
 do_gsc_test(_) ->
-    do_help().
+    gsc_cli:do_help().
 
 
 
-do_help() ->
+%do_help() ->
-    io:format("go help yourself~n").
+%    io:format("go help yourself~n").
 
 do_list() ->
     Names = ts_utils:runnable_test_names(),

test/ts_utils.erl

@@ -10,6 +10,7 @@
     load_dep/1,
     clean_after/1, tidily/1,
     delete_beams/0, tidy/0,
+    test_exists_by_name/1,
     run_test_by_name/1,
     rmm/1, run_mod_main/1,
     runnable_test_names/0,
@@ -110,6 +111,13 @@ tidy() ->
     delete_beams().
 
 
+test_exists_by_name(Name) when is_list(Name) ->
+    case find_test_by_name(Name) of
+        {good, _} -> true;
+        {bad, _}  -> true;
+        not_found -> false
+    end.
+
 -spec run_test_by_name(Name) -> Result when
         Name :: string(),
         Result :: ok.