scratch work

scratch/sfc00.erl

@@ -0,0 +1,263 @@
+
+-spec consume_token(SourceStr) -> {tok, Token, Rest}
+                                | none
+    when SourceStr :: string(),
+         Token     :: token(),
+         Rest      :: string().
+
+consume_token(SrcStr) ->
+    consume_token_II(token_parsers(), SrcStr).
+
+
+consume_token_II([TokenParser | Rest], SrcStr) ->
+    case consume_token(TokenParser, SrcStr) of
+        none -> consume_token_II(Rest, SrcStr);
+        Tok  -> Tok;
+    end;
+consume_token_II([], _) ->
+    none.
+
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+consume_token(identifier, [LowerUnder | Rest])
+        when (($a =< LowerUnder) andalso (LowerUnder =< $z))
+             orelse ($_ =:= LowerUnder)
+        ->
+    {TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
+    TokenStr = [LowerUnder | TokenTail],
+    Token = #identifier{str = TokenStr},
+    {tok, Token, NewRest};
+consume_token(identifier, _) ->
+    none;
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+consume_token(constructor, [Upper | Rest])
+        when ($A =< Upper) andalso (Upper =< $Z) ->
+    {TokenTail, NewRest} = takewhile(fun is_ulduq/1, Rest),
+    TokenStr = [Upper | TokenTail],
+    Token = #constructor{str = TokenStr},
+    {tok, Token, NewRest};
+consume_token(constructor, _) ->
+    none;
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+consume_token(qualified_identifier, Str) ->
+    case consume_qualifiers(Str) of
+        {[], _} ->
+            none;
+        {Qualifiers, Rest0} ->
+            case consume_token(identifier, Rest0) of
+                {tok, Identifier, Rest1} ->
+                    Token = #qualified_identifier{qualifiers = Qualifiers,
+                                                  identifier = Identifier},
+                    {tok, Token, Rest1};
+                none ->
+                    none
+            end
+    end;
+consume_token(qualified_constructor, Str) ->
+    case consume_qualifiers(Str) of
+        {[], _} ->
+            none;
+        {Qualifiers, Rest0} ->
+            case consume_token(constructor, Rest0) of
+                {tok, Constructor, Rest1} ->
+                    Token = #qualified_constructor{qualifiers  = Qualifiers,
+                                                   constructor = Constructor},
+                    {tok, Token, Rest1};
+                none ->
+                    none
+            end
+    end;
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+consume_token(type_variable, [$' | Rest0]) ->
+    case consume_token(identifier, Rest0) of
+        {tok, Identifier, Rest1} ->
+            Token = #type_variable{identifier = Identifier},
+            {tok, Token, Rest1};
+        none ->
+            none
+    end;
+consume_token(type_variable, _) ->
+    none;
+consume_token(int, Str) ->
+    case {consume_int16(Str), consume_int10(Str)} of
+        {{tok, Int16, Rest}, _} ->
+            {tok, #int{inner_int = Int16}, Rest};
+        {none, {tok, Int10, Rest}} ->
+            {tok, #int{inner_int = Int10}, Rest};
+        {none, none} ->
+            none
+    end;
+consume_token(string, _) ->
+    none;
+consume_token(char, _) ->
+    none;
+consume_token(akstr, _) ->
+    none;
+consume_token(ctstr, _) ->
+    none;
+consume_token(sgstr, _) ->
+    none.
+
+
+
+-spec consume_int16(SrcStr) -> {tok, #int16{}, Rest} | none
+    when SrcStr :: string(),
+         Rest   :: string().
+
+consume_int16("0x" ++ Rest0) ->
+    case consume_hex_digits(Rest0) of
+        {piece16, Piece0, Rest1} ->
+            {Pieces, Rest2} = consume_uhexs(Rest1),
+            Token = #int16{hex_pieces = [Pieces0 | Pieces]},
+            {tok, Token, Rest2};
+        none ->
+            none
+    end;
+consume_int16(_) ->
+    none.
+
+
+
+-spec takewhile(Pred, List) -> {Taken, Rest}
+    when Pred  :: fun( (Item) -> boolean() ),
+         List  :: [Item],
+         Item  :: any(),
+         Taken :: [Item],
+         Rest  :: [Item].
+
+takewhile(Pred, List) ->
+    takewhile(Pred, List, []).
+
+
+takewhile(Pred, [X | Xs], Acc) -> 
+    case Pred(X) of
+        true  -> takewhile(Pred, Xs, [X | Acc]);
+        false -> {lists:reverse(Acc), Xs};
+    end;
+takewhile(_, [], Acc) -> 
+    {lists:reverse(Acc), []}.
+
+
+-spec is_ulduq(Char) -> boolean()
+    when Char :: integer().
+% @doc upper lower digit underscore quote
+
+is_ulduq(C) ->
+    (($A =< C) andalso (C =< $Z))
+        orelse (($a =< C) andalso (C =< $z))
+        orelse (C =:= $_)
+        orelse (C =:= $').
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+-record(id,
+        {str :: string()}).
+
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+-record(con,
+        {str :: string()}).
+
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+-record(qid,
+        {qualifiers :: [#con{}],
+         identifier :: #id{}}).
+
+% - `QCon = (Con\.)+Con` qualified constructor
+-record(qcon,
+        {qualifiers  :: [#con{}],
+         constructor :: #con{}}).
+
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+-record(tvar,
+        {identifier :: #identifier{}}).
+
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+-record(int16,
+        {hex_pieces :: [string()]}).
+
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+-record(int10,
+        {dec_pieces :: [string()]}).
+
+-type int() :: int10() | int16().
+
+
+% @doc
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+% - `QCon = (Con\.)+Con` qualified constructor
+% - `TVar = 'Id` type variable (e.g `'a`, `'b`)
+% - `Int = [0-9]+(_[0-9]+)*|0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` integer literal with optional `_` separators
+% - `Bytes = #[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*` byte array literal with optional `_` separators
+% - `String` string literal enclosed in `"` with escape character `\`
+% - `Char` character literal enclosed in `'` with escape character `\`
+% - `AccountAddress` base58-encoded 32 byte account pubkey with `ak_` prefix
+% - `ContractAddress` base58-encoded 32 byte contract address with `ct_` prefix
+% - `Signature` base58-encoded 64 byte cryptographic signature with `sg_` prefix
+-type token() :: #id{}
+               | #con{}
+               | #qid{}
+               | #tvar{}
+               | int()
+               | {string, string()}
+               | {char, string()}
+               | {akstr, string()}
+               | {ctstr, string()}
+               | {sgstr, string()}.
+
+-type token_shape() :: id
+                     | con
+                     | qid,
+                     | qcon
+                     | tvar
+                     | int
+                     | string
+                     | char
+                     | akstr
+                     | ctstr
+                     | sgstr.
+
+%-----------------------------------------
+% functions
+%-----------------------------------------
+
+-spec token_parsers() -> [atom()].
+% in parse order
+
+token_parsers() ->
+    [id,
+     con,
+     qid,
+     qcon,
+     tvar,
+     int,
+     string,
+     char,
+     akstr,
+     ctstr,
+     sgstr].
+
+
+-spec token_consume(Matcher, SourceString) -> MaybeToken
+    when Matcher      :: token_shape(),
+         SourceString :: string(),
+         MaybeToken   :: {match, Token :: token(), Rest :: string()}
+                       | nomatch.
+
+% - `Id = [a-z_][A-Za-z0-9_']*` identifiers start with a lower case letter.
+token_consume(id, [AZU | Src1]) when $_ =:= AZU; $a =< AZU, AZU =< $z ->
+    {RestOfId, Src2} = tdw(fun is_ulduq/1, Src1),
+    {match, #id{str = [AZU | RestOfId]}, Src2};
+token_consume(id, _) ->
+    nomatch;
+% - `Con = [A-Z][A-Za-z0-9_']*` constructors start with an upper case letter.
+token_consume(con, [AZ | Src1]) -> when $A =< AZU, AZU =< $Z ->
+    {RestOfCon, Src2} = tdw(fun is_ulduq/1, Src1),
+    {match, #con{str = [AZ | RestOfCon]}, Src2};
+token_consume(con, _) ->
+    nomatch;
+% - `QId = (Con\.)+Id` qualified identifiers (e.g. `Map.member`)
+token_consume(qid, Src0) ->
+    case one_or_more({seq, con, {char, $.}}, Src0)
+
+

scratch/sfc_ast.erl

@@ -0,0 +1,329 @@
+% @doc
+% type for an SFC AST
+%
+% semantically:
+%       #pr{}       means the parser succeeded
+%       noparse     means the parser didn't consume any tokens
+%       {error, _}  means the parser can prove the user made an
+%                   error (e.g. unterminated open paren)
+%
+% based on so_syntax.erl
+%
+% this is the second layer in the parsing pipeline after tokens
+%
+% quoth `sophia/docs/so_syntax.md`:
+%
+%   ## Declarations
+%
+%   A Sophia file consists of a sequence of *declarations* in a layout block.
+%
+%   ```c
+%   File ::= Block(TopDecl)
+%
+%   TopDecl ::= ['payable'] ['main'] 'contract' Con [Implement] '=' Block(Decl)
+%             | 'contract' 'interface' Con [Implement] '=' Block(Decl)
+%             | 'namespace' Con '=' Block(Decl)
+%             | '@compiler' PragmaOp Version
+%             | 'include' String
+%             | Using
+%
+%   Implement ::= ':' Sep1(Con, ',')
+%
+%   Decl ::= 'type'     Id ['(' TVar* ')'] '=' TypeAlias
+%          | 'record'   Id ['(' TVar* ')'] '=' RecordType
+%          | 'datatype' Id ['(' TVar* ')'] '=' DataType
+%          | 'let'      Id [':' Type]      '=' Expr
+%          | (EModifier* 'entrypoint' | FModifier* 'function') Block(FunDecl)
+%          | Using
+%
+%   FunDecl ::= Id ':' Type                             // Type signature
+%             | Id Args [':' Type] '=' Block(Stmt)      // Definition
+%             | Id Args [':' Type] Block(GuardedDef)    // Guarded definitions
+%
+%   GuardedDef ::= '|' Sep1(Expr, ',') '=' Block(Stmt)
+%
+%   Using ::= 'using' Con ['as' Con] [UsingParts]
+%   UsingParts ::= 'for' '[' Sep1(Id, ',') ']'
+%                | 'hiding' '[' Sep1(Id, ',') ']'
+%
+%   PragmaOp ::= '<' | '=<' | '==' | '>=' | '>'
+%   Version  ::= Sep1(Int, '.')
+%
+%   EModifier ::= 'payable' | 'stateful'
+%   FModifier ::= 'stateful' | 'private'
+%
+%   Args ::= '(' Sep(Pattern, ',') ')'
+%   ```
+-module(sfc_ast).
+
+-export_type([
+]).
+
+-export([
+    parse/2
+]).
+
+-include("sfc.hrl").
+
+
+%%-----------------------------
+%% FUNCTIONS
+%%-----------------------------
+
+-spec parse(ParseTarget, SigTokens) -> Result
+    when ParseTarget :: parse_target(),
+         SigTokens   :: list(sfc_token()),
+         Result      :: perhaps_parsed().
+% @doc
+% parse target against list of tokens;
+%
+% for MVP, it is assumed the given list of tokens
+% contains
+
+parse({ast, AstTarget}, Tokens)        -> parse_ast(AstTarget, Tokens);
+parse({tokens, Target}, Tokens)        -> parse_tokens(Target, Tokens);
+parse({prs, Strategy, Target}, Tokens) -> parse_prs(Strategy, Target, Tokens).
+
+
+
+-spec parse_ast(AstTarget, SigTokens) -> Perhaps
+    when AstTarget :: ast_target(),
+         SigTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private parse where result is an AST
+%
+% happy path: returns
+%
+%   #pr{type   = ast,
+%       result = AST :: ast(),
+%       rest   = _ :: [sfc_token()]}
+
+parse_ast(AstTarget, SrcTokens) ->
+    case AstTarget of
+        file     -> parse_ast_file(SrcTokens);
+        top_decl -> parse_ast_top_decl(SrcTokens);
+        _        -> {error, #sfc_err_nyi{}}
+    end.
+
+
+
+-spec parse_ast_file(SrcTokens) -> Perhaps
+    when SrcTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private parse an #ast_file{} record
+%
+% on success returns
+%   #pr{type   = ast,
+%       result = #ast_file{},
+%       rest   = []}
+
+parse_ast_file([]) ->
+    noparse;
+    %{error, #sfc_err_empty_file{}};
+parse_ast_file(SrcTks0) ->
+    case parse({prs, plus, {ast, top_decl}}, SrcTks0) of
+        #pr{type=prs, result=TopDeclPrs, rest=SrcTks1} ->
+            TopDeclAsts = [Ast || #pr{type=ast, result=Ast} <- TopDeclPrs],
+            AstFile =
+                #ast_file{top_decls = TopDeclAsts},
+            ParseResult = #pr{type   = ast,
+                              result = AstFile,
+                              rest   = SrcTks1},
+            ParseResult;
+        Nope ->
+            Nope
+    end.
+
+
+
+-spec parse_ast_top_decl(SrcTokens) -> Perhaps
+    when SrcTokens :: [sfc_token()],
+         Perhaps   :: perhaps_parsed().
+% @private
+% parse a TopDecl from tokens
+%
+% on success returns
+%
+%   #pr{type   = ast,
+%       result = #ast_top_decl{},
+%       rest   = [_ : #sfc_token{}]}
+% @end
+
+parse_ast_top_decl(SrcTks0 = [_ | _]) ->
+    PerhapsTks = parse_tokens(block_item, SrcTks0),
+    case PerhapsTks of
+        #pr{type = tokens, result = []} ->
+            noparse;
+        #pr{type=tokens, result=DeclTks, rest=SrcTks1} ->
+            AstResult =
+                #ast_top_decl{tokens = DeclTks},
+            ParseResult = #pr{type   = ast,
+                              result = AstResult,
+                              rest   = SrcTks1},
+            ParseResult;
+        _ ->
+            {error, #sfc_err_nyi{}}
+    end;
+parse_ast_top_decl([]) ->
+    noparse.
+
+
+
+-spec parse_tokens(TokensTarget, SrcTokens) -> Result
+    when TokensTarget :: tokens_target(),
+         SrcTokens    :: [sfc_token()],
+         Result       :: perhaps_parsed().
+% @private this is for parsing a flat list of tokens
+%
+% essentially this is "eat until the next token boundary"
+%
+% on success returns
+%
+%   #pr{type   = tokens,
+%       result = [sfc_token()],
+%       rest   = [sfc_token()]}
+% @end
+
+% need to think carefully about this case;
+% for now parsing against an empty list of tokens
+% is always a success
+parse_tokens(_, []) ->
+    #pr{type = tokens, result = [], rest = []};
+% WMA SrcTokens is nonempty
+parse_tokens(block, SrcTokens) ->
+    {_, BlkIndent} = start_pos(SrcTokens),
+    TakeToken = fun(Tk) -> BlkIndent =< indent_level(Tk) end,
+    {BlkTokens, Rest} = tw(TakeToken, SrcTokens),
+    #pr{type   = tokens,
+        result = BlkTokens,
+        rest   = Rest};
+parse_tokens(block_item, [H | T]) ->
+    BlkIndent = indent_level(H),
+    TakeToken = fun(Tk) -> BlkIndent < indent_level(Tk) end,
+    {BlockTail, Rest} = tw(TakeToken, T),
+    BlkTokens = [H | BlockTail],
+    #pr{type   = tokens,
+        result = BlkTokens,
+        rest   = Rest};
+parse_tokens(_, _) ->
+    {error, #sfc_err_nyi{}}.
+
+
+% takewhile
+tw(Pred, List) ->
+    tw(Pred, [], List).
+
+tw(Pred, Stk, [X | Xs]) ->
+    case Pred(X) of
+        true  -> tw(Pred, [X | Stk], Xs);
+        false -> {lists:reverse(Stk), [X | Xs]}
+    end;
+tw(_, Stk, []) ->
+    {lists:reverse(Stk), []}.
+
+
+
+-spec parse_prs(Strategy, Target, SrcTokens) -> Perhaps
+    when Strategy   :: prs_strategy(),
+         Target     :: parse_target(),
+         SrcTokens  :: [sfc_token()],
+         Perhaps    :: perhaps_parsed().
+% @private on success returns
+%
+%   #pr{type   = prs,
+%       result = [_ :: #pr{}]
+%       rest   = [_ :: #sfc_token{}]}
+
+parse_prs(Strategy, Target, SrcTokens) ->
+    case Strategy of
+        star -> parse_prs_star(Target, SrcTokens);
+        plus -> parse_prs_plus(Target, SrcTokens);
+        _    -> {error, #sfc_err_nyi{}}
+    end.
+
+
+
+-spec parse_prs_star(ParseTarget, SrcTokens) -> Perhaps
+    when ParseTarget :: parse_target(),
+         SrcTokens   :: [sfc_token()],
+         Perhaps     :: perhaps_parsed().
+% @private
+% parse 0 or more of
+
+parse_prs_star(_ParseTarget, []) ->
+    #pr{type = prs, result = [], rest = []};
+parse_prs_star(ParseTarget, SrcTokens) ->
+    parse_prs_star(ParseTarget, [], SrcTokens).
+
+% WMA SrcTokens nonempty
+parse_prs_star(ParseTarget, Stack, SrcTokens) ->
+    case parse(ParseTarget, SrcTokens) of
+        #pr{result=NewItem, rest=NewSrcTokens} ->
+            NewStack = [NewItem | Stack],
+            parse_prs_star(ParseTarget,
+                           NewStack,
+                           NewSrcTokens);
+        noparse ->
+            PRs = lists:reverse(Stack),
+            #pr{type   = prs,
+                result = PRs,
+                rest   = SrcTokens};
+        Error = {error, _} ->
+            Error
+    end.
+
+
+
+-spec parse_prs_plus(ParseTarget, SrcTokens) -> Perhaps
+    when ParseTarget :: parse_target(),
+         SrcTokens   :: [sfc_token()],
+         Perhaps     :: perhaps_parsed().
+% @private
+% parse 1 or more of
+
+parse_prs_plus(ParseTgt, SrcTks0) ->
+    case parse(ParseTgt, SrcTks0) of
+        #pr{result = Hd, rest = SrcTks1} ->
+            parse_prs_star(ParseTgt, [Hd], SrcTks1);
+        Nope ->
+            Nope
+    end.
+
+
+
+-spec start_pos(Tokens) -> Pos
+    when Tokens :: [sfc_token()],
+         Pos    :: sfc_pos().
+% @private
+% start position of NONEMPTY list of tokens
+
+start_pos([#sfc_token{pos = Result} | _]) ->
+    Result.
+
+
+
+-spec end_pos(Tokens) -> Pos
+    when Tokens :: [sfc_token()],
+         Pos    :: sfc_pos().
+% @private
+% end position of NONEMPTY list of tokens
+
+end_pos(Tokens) ->
+    end_pos(Tokens, start_pos(Tokens)).
+
+end_pos([#sfc_token{string = S} | Ts], Pos) ->
+    NewPos = sfc_tokenizer:new_pos(Pos, S),
+    end_pos(Ts, NewPos);
+end_pos([], Result) ->
+    Result.
+
+
+
+-spec indent_level(Token) -> Col
+    when Token :: sfc_token(),
+         Col   :: pos_integer().
+% @private
+% get col val from token
+
+indent_level(#sfc_token{pos = {_, C}}) ->
+    C.

scratch/sfc_group_tokens.erl

@@ -0,0 +1,94 @@
+-module(sfc_group_tokens).
+
+-export_type([
+]).
+
+-export([
+    group_tokens/1
+]).
+
+-include("sfc.hrl").
+
+
+-spec group_tokens(Tokens) -> Result
+    when Tokens     :: [sfc_token()],
+         Result     :: {ok, Groups}
+                     | {error, Reason},
+         Groups     :: [TokenGroup],
+         TokenGroup :: sfc_token()
+                     | #sfc_tg_list{},
+         Reason     :: any().
+
+group_tokens(Tokens) ->
+    group_tokens([], Tokens).
+
+
+group_tokens(Stack, Tokens = [_ | _]) ->
+    case token_or_group(Tokens) of
+        {push, Item, NewTokens} ->
+            group_tokens([Item | Stack], NewTokens);
+        Error ->
+            Error
+    end;
+group_tokens(Stack, []) ->
+    {ok, lists:reverse(Stack)}.
+
+
+% erlang's type language is lacking here...
+% items are always a list of sfc_token_group_items
+% placeholder type
+-type sfc_tg_item_() :: any().
+
+% tg = token group
+% {start_line, indent} should always equal the Pos of the first token
+-record(sfc_tg_block,
+        {start_line = none :: none | pos_integer(),
+         indent     = none :: none | pos_integer(),
+         items      = none :: none | {stack, [sfc_tg_item_()]} | [sfc_tg_item_()]}).
+
+
+% delist = delimited list (...) [...], {...}
+%
+% don't even comma-separate values, just make sure 
+-record(sfc_tg_delist,
+        {open   = none :: none | '(' | '[' | '{',
+         items  = none :: none | {stack, [sfc_tg_item()]} | [sfc_tg_item()],
+         close  = none :: none | ')' | ']' | '}',
+
+-type sfc_tg_item() :: sfc_token(),
+                     | #sfc_tg_block{}
+                     | #sfc_tg_round{}
+                     | #sfc_.
+
+-type sfc_tg() :: [sfc_tg_item()].
+
+-spec token_or_group(Tokens) -> Result
+    when Tokens :: [sfc_token()],
+         Result :: {push, Item, NewTokens}
+                 | {error, Reason}
+         Item   :: sfc_token()
+                 | {block, IndentLevel :: pos_integer(),
+                           
+                 ,
+
+token_or_group(Tokens) ->
+
+
+
+groupist_order() ->
+    [round, square, curly, block].
+
+
+match_groupists([Groupist | Rest], Tokens) ->
+    case groupist(Grupist, Tokens) of
+        Match = {groupmatch, _, _} ->
+            Match
+        no_groupmatch ->
+            match_groupists(Rest, Tokens);
+        Error ->
+            Error
+    end;
+match_groupists(_, []) ->
+    {ok, []};
+match_groupists([], [#sfc_token{pos = Pos} | _}) ->
+    {error, {cannot_make_groupist_decision, Pos}}.

scratch/sfc_token_chunks.erl

@@ -0,0 +1,178 @@
+% @doc
+% helper functions for grabbing collections of tokens
+% off the token stream
+%
+% generally assume no whitespace/comment tokens in
+% input stream
+-module(sfc_token_chunks).
+
+-export_type([
+]).
+
+-export([
+    barf/2
+]).
+
+% $sfc_include is so c() works from sfp eshell
+-include("$sfc_include/sfc.hrl").
+
+%------------------------------------------
+% Types
+%------------------------------------------
+
+-type chunk_shape()
+    :: block
+     | block_item
+     | {block_item, Level :: pos_integer()}
+     .
+
+% FIXME
+-type choke_reason() :: any().
+
+
+-spec barf(ChunkShape, SigTokens) -> Perhaps
+    when ChunkShape :: chunk_shape(),
+         SigTokens  :: Tokens,
+         Perhaps :: {barf, Taken, Rest}
+                  | {choke, Reason},
+         Taken  :: Tokens,
+         Rest   :: Tokens,
+         Reason :: choke_reason(),
+         Tokens :: [sfc_token()].
+
+% @doc
+% slurp/barf terminology comes from paredit mode in
+% emacs
+%
+% slurp ~= accepting input
+% barf ~= separating input
+%
+%   slurp: (foo bar) baz ~> (foo bar baz)
+%   barf : (foo bar baz) ~> foo (bar baz)
+
+barf(_, []) ->
+    {barf, [], []};
+barf(block, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
+    Take =
+        fun(#sfc_token{pos = {_, TkCol}}) ->
+            BlkCol =< TkCol
+        end,
+    {A, B} = tw(Take, T),
+    {barf, [H | A], B};
+barf(block_item, [H = #sfc_token{pos = {_, BlkCol}} | T]) ->
+    Take =
+        fun(#sfc_token{pos = {_, TkCol}}) ->
+            BlkCol < TkCol
+        end,
+    {A, B} = tw(Take, T),
+    {barf, [H | A], B}.
+barf({block_item, Level}, Tokens = [#sfc_token{pos = {_, StartLevel}} | _]) ->
+    case Level =:= StartLevel of
+        false -> {barf, [], Tokens};
+        true  -> barf(block_item, Tokens)
+    end.
+barf(_, _) ->
+    {choke, #sfc_err_nyi{}}.
+
+
+% takewhile but returns {Taken, Rest}
+tw(Pred, List) ->
+    tw(Pred, [], List).
+
+tw(Pred, Stk, [X | Xs]) ->
+    case Pred(X) of
+        true  -> tw(Pred, [X | Stk], Xs);
+        false -> {lists:reverse(Stk), [X | Xs]}
+    end;
+tw(_, Stk, []) ->
+    {lists:reverse(Stk), []}.
+
+
+%-spec consume_delist_round(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_round(Tokens = [#sfc_token{type = punct, string = "("} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_round(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist_square(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_square(Tokens = [#sfc_token{type = punct, string = "["} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_square(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist_curly(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist_curly(Tokens = [#sfc_token{type = punct, string = "{"} | _]) ->
+%    consume_delist(Tokens);
+%consume_delist_curly(Tokens) ->
+%    {ok, {[], Tokens}}.
+%
+%
+%
+%-spec consume_delist(Tokens) -> Result
+%    when Tokens    :: [sfc_token()],
+%         Result    :: {ok, {Delist, NewTokens}}
+%                    | {error, sfc_err()},
+%         Delist    :: Tokens,
+%         NewTokens :: Tokens.
+%
+%consume_delist([Head | Tail]) ->
+%    case Head of
+%        #sfc_token{type = punct, string = OpenStr = "("} ->
+%            CloseStr = ")",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        #sfc_token{type = punct, string = OpenStr = "["} ->
+%            CloseStr = "]",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        #sfc_token{type = punct, string = OpenStr = "{"} ->
+%            CloseStr = "}",
+%            delist_verb(1, {OpenStr, CloseStr}, [Head], Tail);
+%        _ ->
+%            {ok, {[], [Head | Tail]}}
+%    end;
+%consume_delist([]) ->
+%    {ok, {[], []}}.
+%
+%
+%% happy path: have a token and we are in the depth of the list
+%delist_verb(Depth, {OpenStr, CloseStr}, Stack, [Tok | Rest]) when Depth >= 1 ->
+%    NewDepth =
+%        case Tok of
+%            % open token -> increase depth
+%            #sfc_token{type = punct, string = Open} when Open =:= OpenStr ->
+%                Depth + 1;
+%            % close token -> decrease depth
+%            #sfc_token{type = punct, string = Close} when Close =:= CloseStr ->
+%                Depth - 1;
+%            _ ->
+%                Depth
+%        end,
+%    delist_verb(NewDepth, {OpenStr, CloseStr}, [Tok | Stack], Rest);
+%% we've hit end of available input before a close token
+%delist_verb(Depth, {OpenStr, _CloseStr}, _Stack, []) when Depth >= 1 ->
+%    {error, #sfc_err{atom   = unclosed_delimiter,
+%                     string = io_lib:format("unclosed open delimiter: ~p", [OpenStr])}};
+%% we've popped back out of the list, we're done
+%delist_verb(_Depth = 0, {_OpenStr, _CloseStr}, Stack, Tokens) ->
+%    {ok, {lists:reverse(Stack), Tokens}}.