restructuring done I think

Makefile

@@ -1,4 +0,0 @@
-all: 
-
-clean:
-	rm

bin/gsc

@@ -29,7 +29,7 @@ FP_THIS_DIR=$(dirname -- "${FP_THIS_FILE}")
 FP_PRJ_DIR=$(dirname -- "${FP_THIS_DIR}")
 
 # simplified
-zx rundir $FP_PRJ_DIR
+zx rundir $FP_PRJ_DIR $@
 
 
 # commented out legacy code in case need it later:

ebin/gsc.app

@@ -4,4 +4,8 @@
               {included_applications,[]},
               {applications,[stdlib,kernel]},
               {vsn,"0.1.0"},
-              {modules,[gsc]}]}.
+              {modules,['ast-gulp',gsc_ast,gsc_bst,gsc_parse_type_expr,
+                        gsc_token_chunks,ifarith,parse_type_expr,test_ntree,
+                        unicode,gsc,gsc_cli,gsc_ntree,gsc_signal,gsc_strmatch,
+                        gsc_tokens,gso_scan,gsc_test_file,gsc_test_ntree,
+                        gsc_test_tokens,ts_utils]}]}.

scratch/gsc_cli

@@ -0,0 +1,84 @@
+
+do(["test"]) ->
+    do_tests();
+do(["test" | Tests]) ->
+    do_tests(Tests);
+do(["tests"]) ->
+    do_tests();
+
+do_runall_tests() ->
+    lists:foreach(fun run_mod_main/1, test_mods()).
+
+
+do_tests(List) ->
+    lists:foreach(fun run_test/1, List).
+
+% n
+run_test(TestName) ->
+    % we have two candidate atoms
+    C1 = list_to_atom(TestName),
+    C2 = list_to_atom("gsc_test_" ++ TestName),
+    KnownMods = test_mods(),
+    IsC1 = lists:member(C1, KnownMods),
+    IsC2 = lists:member(C2, KnownMods),
+    if
+        IsC1 -> rmm(C1);
+        IsC2 -> rmm(C2);
+        true -> error({no_such_test, TestName})
+    end.
+
+
+rmm(X) -> run_mod_main(X).
+
+% KnownTests = test_mods(),
+% TestMods = ensure_all_known([], List, KnownTests),
+% lists:foreach(fun run_mod_main/1, TestMods).
+
+
+%ensure_all_known(Acc, [], _) ->
+%    lists:sort(Acc);
+%ensure_all_known(Acc, [T | Ts], Knowns) ->
+%    case lists:member(T, Knowns) of
+%
+%    end.
+
+
+test_mods() ->
+    known_modules_with_prefix("gsc_test").
+
+known_modules_with_prefix(Pfx) ->
+    ModsZipBeamsZipLoaded = code:all_available(),
+    kmp(Pfx, ModsZipBeamsZipLoaded, []).
+
+kmp(_Pfx, [], Acc) ->
+    lists:sort(Acc);
+kmp(Pfx, [{ModStr, _BeamPath, _Loaded} | Rest], Acc) ->
+    case lists:prefix(Pfx, ModStr) of
+        false -> kmp(Pfx, Rest, Acc);
+        true  -> kmp(Pfx, Rest, [list_to_atom(ModStr) | Acc])
+    end.
+
+run_mod_main(Mod) ->
+    io:format("========================================\n"
+              "~p:main()\n"
+              "========================================\n",
+              [Mod]),
+    try
+        Mod:main()
+    catch
+        Err:ErrType:Trace ->
+            io:format("~p: ~p~n", [Err, ErrType]),
+            io:format("Trace:~n~p~n", [Trace])
+    end.
+
+do_tlist() ->
+    lists:foreach(
+        fun(ModName) ->
+            io:format("~s~n", [ModName])
+        end,
+        test_mods()
+    ).
+
+
+tokenizers_agree(File) ->
+    gso_tokens(File) =:= so_tokens(File).

src/gsc_cli.erl

@@ -1,5 +1,5 @@
 %%% @doc
-%%% GSC CLI: explorer/harness for sfc iteration
+%%% GSC CLI: explorer/harness for gsc iteration
 %%% @end
 
 -module(gsc_cli).
@@ -37,29 +37,19 @@ start(["eshell"]) ->
     do_eshell(),
     ok;
 start(ArgV) ->
-    %io:format("ArgV: ~p~n", [ArgV]),
     do(ArgV),
     zx:silent_stop().
 
-do(["list"]) ->
-    do_tlist();
-do(["list", "tests"]) ->
-    do_tlist();
-do(["test"]) ->
-    do_tests();
 do(["test" | Tests]) ->
-    do_tests(Tests);
+    do_test(Tests);
-do(["tests"]) ->
-    do_tests();
-do(["run", "tests"]) ->
-    do_tests();
-do(["tokenizers_agree", Foo]) ->
-    io:format("~p~n", [tokenizers_agree(Foo)]);
 % slowly phasing out shitty names like lctokens
 % tokens = native sfc token representation
 do(["tokens", Foo]) -> do_tokens(Foo);
-do(["color_tokens", Foo]) -> do_color_tokens(Foo);
+% print source file to screen with token boundaries
+% highlighted
+do(["tcat", Foo])         -> do_color_tokens(Foo);
 do(["ctokens", Foo])      -> do_color_tokens(Foo);
+do(["color_tokens", Foo]) -> do_color_tokens(Foo);
 do(["colour_tokens" | _]) -> do_doi();
 % so_tokens = so_scan tokens
 do(["so", "tokens", Foo]) -> do_so_tokens(Foo);
@@ -67,7 +57,6 @@ do(["so_tokens", Foo]) -> do_so_tokens(Foo);
 % gso_tokens = our mockery
 do(["gso", "tokens", Foo]) -> do_gso_tokens(Foo);
 do(["gso_tokens", Foo]) -> do_gso_tokens(Foo);
-% print source file to screen with token boundaries highlighted
 % script utility
 do(["rmm", Foo]) ->
     do_rmm(Foo);
@@ -79,6 +68,20 @@ do_doi() ->
     FP = zx:get_home() ++ "/priv/doi.txt",
     page_file(FP).
 
+
+do_test(Args) ->
+    GscTestsMod = gsc_tests_mod(),
+    GscTestsMod:cli_args(Args).
+
+
+gsc_tests_mod() ->
+    FilePath = zx:get_home() ++ "/test/gsc_tests.erl",
+    case compile:file(FilePath) of
+        {ok, Mod} -> Mod;
+        Error     -> error(Error)
+    end.
+
+
 % thank you chatgpt
 % os:cmd didnt do nuffin because that's for running
 % stuff in the background and capturing the output, not
@@ -107,83 +110,6 @@ less_file(Less, FilePath) ->
             error(Reason)
     end.
 
-do_tests() ->
-    io:format("TestModules = ~p~n", [test_mods()]),
-    do_runall_tests().
-
-do_runall_tests() ->
-    lists:foreach(fun run_mod_main/1, test_mods()).
-
-
-do_tests(List) ->
-    lists:foreach(fun run_test/1, List).
-
-% n
-run_test(TestName) ->
-    % we have two candidate atoms
-    C1 = list_to_atom(TestName),
-    C2 = list_to_atom("gsc_test_" ++ TestName),
-    KnownMods = test_mods(),
-    IsC1 = lists:member(C1, KnownMods),
-    IsC2 = lists:member(C2, KnownMods),
-    if
-        IsC1 -> rmm(C1);
-        IsC2 -> rmm(C2);
-        true -> error({no_such_test, TestName})
-    end.
-
-
-rmm(X) -> run_mod_main(X).
-
-% KnownTests = test_mods(),
-% TestMods = ensure_all_known([], List, KnownTests),
-% lists:foreach(fun run_mod_main/1, TestMods).
-
-
-%ensure_all_known(Acc, [], _) ->
-%    lists:sort(Acc);
-%ensure_all_known(Acc, [T | Ts], Knowns) ->
-%    case lists:member(T, Knowns) of
-%
-%    end.
-
-
-test_mods() ->
-    known_modules_with_prefix("gsc_test").
-
-known_modules_with_prefix(Pfx) ->
-    ModsZipBeamsZipLoaded = code:all_available(),
-    kmp(Pfx, ModsZipBeamsZipLoaded, []).
-
-kmp(_Pfx, [], Acc) ->
-    lists:sort(Acc);
-kmp(Pfx, [{ModStr, _BeamPath, _Loaded} | Rest], Acc) ->
-    case lists:prefix(Pfx, ModStr) of
-        false -> kmp(Pfx, Rest, Acc);
-        true  -> kmp(Pfx, Rest, [list_to_atom(ModStr) | Acc])
-    end.
-
-run_mod_main(Mod) ->
-    io:format("========================================\n"
-              "~p:main()\n"
-              "========================================\n",
-              [Mod]),
-    try
-        Mod:main()
-    catch
-        Err:ErrType:Trace ->
-            io:format("~p: ~p~n", [Err, ErrType]),
-            io:format("Trace:~n~p~n", [Trace])
-    end.
-
-do_tlist() ->
-    lists:foreach(
-        fun(ModName) ->
-            io:format("~s~n", [ModName])
-        end,
-        test_mods()
-    ).
-
 
 -spec do_eshell() -> ok.
 % @doc start an erlang shell
@@ -196,8 +122,6 @@ do_eshell() ->
         {error, Reason} -> error(Reason)
     end.
 
-tokenizers_agree(File) ->
-    gso_tokens(File) =:= so_tokens(File).
 
 
 do_tokens(FilePath) ->
@@ -263,14 +187,26 @@ colorize_tokens(_, [], Acc) ->
 rotate([A | Rest]) ->
     {A, Rest ++ [A]}.
 
-colorize_token_str(Color, #tk{str = Str}) ->
+colorize_token_str(Color, T = #tk{str = Str}) ->
-    {Pfx, Sfx} = color_fixes(Color),
+    SN =
+        case T#tk.shape of
+            bcom -> noise;
+            lcom -> noise;
+            ws -> noise;
+            _ -> signal
+        end,
+    {Pfx, Sfx} = color_fixes(SN, Color),
     [Pfx, Str, Sfx].
 
-color_fixes(red)     -> {?ANSI_FG_RED,     ?ANSI_FG_RESET};
+
-color_fixes(green)   -> {?ANSI_FG_GREEN,   ?ANSI_FG_RESET};
+% dim noisy tokens
-color_fixes(yellow)  -> {?ANSI_FG_YELLOW,  ?ANSI_FG_RESET};
+color_fixes(noise, Color) ->
-color_fixes(blue)    -> {?ANSI_FG_BLUE,    ?ANSI_FG_RESET};
+    {P, S} = color_fixes(signal, Color),
-color_fixes(magenta) -> {?ANSI_FG_MAGENTA, ?ANSI_FG_RESET};
+    {[?ANSI_DIM, P], [S, ?ANSI_UNDIM]};
-color_fixes(cyan)    -> {?ANSI_FG_CYAN,    ?ANSI_FG_RESET}.
+color_fixes(signal, red)     -> {?ANSI_FG_RED,     ?ANSI_FG_RESET};
+color_fixes(signal, green)   -> {?ANSI_FG_GREEN,   ?ANSI_FG_RESET};
+color_fixes(signal, yellow)  -> {?ANSI_FG_YELLOW,  ?ANSI_FG_RESET};
+color_fixes(signal, blue)    -> {?ANSI_FG_BLUE,    ?ANSI_FG_RESET};
+color_fixes(signal, magenta) -> {?ANSI_FG_MAGENTA, ?ANSI_FG_RESET};
+color_fixes(signal, cyan)    -> {?ANSI_FG_CYAN,    ?ANSI_FG_RESET}.
 

test/gsc_test_file.erl

@@ -257,7 +257,7 @@ s2s_sm_decl_type(M = #decl_type{id = none}, S0) ->
     end;
 s2s_sm_decl_type(M = #decl_type{params = none}, S0) ->
     case S0 of
-        [#tk{str = "("} = T0 | _] ->
+        [#tk{str = "("} = _T0 | _] ->
             error({fixme, parens_bad});
         _ ->
             s2s_sm_decl_type(M#decl_type{params = []}, S0)

test/gsc_test_ntree.erl

@@ -17,7 +17,7 @@
          | {parens, Open :: tk(), Close :: tk()}
          .
 
--type ast() :: ntree(syntax_meta(), tk()).
+%-type ast() :: ntree(syntax_meta(), tk()).
 -type asf() :: nforest(syntax_meta(), tk()).
 -type asts() :: asf().
 
@@ -59,7 +59,7 @@ parse(Signal) ->
     F1 = f2f_parens(F0),
     F2 = f2f_op("=>", F1),
     F3 = f2f_op("*", F2),
-    Result = F2,
+    Result = F3,
     Result.
 
 

test/gsc_test_tokens.erl

@@ -11,13 +11,14 @@
 main() ->
     %io:format("~p~n", [div_files()]),
     %io:format("MAINNNNN!~n", []),
-    eunit:test(?MODULE, [verbose]).
+    %eunit:test(?MODULE, [verbose]),
-    %eunit:test(?MODULE).
+    eunit:test(?MODULE),
+    ok.
 
 
 % directory containing the tests for the tokenizer
 ct_dir() ->
-    zx_daemon:get_home() ++ "/ct".
+    zx_daemon:get_home() ++ "/test/ct".
 
 agreement_tests_dir() ->
     ct_dir() ++ "/tokenizers_agree".

test/gsc_tests.erl

@@ -0,0 +1,43 @@
+% dynamic hacky module that loads all the tests
+-module(gsc_tests).
+
+-export([
+    main/0,
+    cli_args/1
+]).
+
+
+
+main() ->
+    cli_args([]),
+    ok.
+
+cli_args(TestNames) ->
+    % load ts_utils
+    TsUtils = zx:get_home() ++ "/test/ts_utils.erl",
+    case compile:file(TsUtils) of
+        {ok, ts_utils} -> ok;
+        Error -> error(Error)
+    end,
+    ts_utils:tidily(fun() -> do(TestNames) end).
+
+
+do(["-h" | _])     -> do_help();
+do(["--help" | _]) -> do_help();
+do(["-l" | _])     -> do_list();
+do(["--list" | _]) -> do_list();
+do(["-a" | _])     -> do_all();
+do(["--all" | _])  -> do_all();
+do(TestNames) ->
+    lists:foreach(fun ts_utils:run_test_by_name/1, TestNames).
+
+do_help() ->
+    io:format("go help yourself~n").
+
+do_list() ->
+    Names = ts_utils:runnable_test_names(),
+    [io:format("~p~n", [N]) || N <- Names].
+
+do_all() ->
+    {Gd, _} = ts_utils:runnable_test_mods(),
+    [begin ts_utils:rmm(G), io:format("~n") end || G <- Gd].

test/ts_utils.erl

@@ -1,27 +1,322 @@
-% testing utilities
+% test suite utilities
 -module(ts_utils).
 
 -export([
+    load_test_deps/0,
+    test_deps/0,
+    load_dep/1,
+    clean_after/1, tidily/1,
+    delete_beams/0, tidy/0,
+    run_test_by_name/1,
+    rmm/1, run_mod_main/1,
+    runnable_test_names/0,
+    runnable_test_mods/0,
+    load_test_erls/0,
+    abspath_to_name/1,
+    ls_test_erls/0,
+    ls_test_beams/0,
+    is_erl/1,
+    is_beam/1,
+    ls_test/0,
+    test_dir/0,
     ct_dir/0,
-    ct_file/1, ct_file_abspath/1
+    ct_file/1, ct_file_abspath/1, ct_abspath/1
 ]).
 
 
--spec ct_dir() -> string().
+load_test_deps() ->
+    lists:foreach(fun load_dep/1, test_deps()).
 
+test_deps() ->
+    [{"otpr", "sophia", {9, 0, 0}}].
+
+
+load_dep(D) ->
+    ok =
+        case zx_lib:installed(D) of
+            false ->
+                Id = zx_daemon:fetch(D),
+                ok = zx_daemon:wait_result(Id),
+                ok;
+            true ->
+                ok
+        end,
+    zx_daemon:build(D).
+
+
+-spec clean_after(Fun) -> Result when
+        Fun    :: fun(() -> Result),
+        Result :: any().
+
+% @doc
+% run Fun(), delete gsc/test/*.beam afterward even if
+% Fun() errors
+% @end
+clean_after(Fun) ->
+    try
+        load_test_deps(),
+        Fun()
+    after
+        delete_beams()
+    end.
+
+
+% @doc alias for `clean_after/1'
+tidily(Fun) ->
+    clean_after(Fun).
+
+
+
+-spec delete_beams() -> ok.
+
+delete_beams() ->
+    lists:map(fun file:delete/1, ls_test_beams()).
+
+tidy() ->
+    delete_beams().
+
+
+-spec run_test_by_name(Name) -> Result when
+        Name :: string(),
+        Result :: ok.
+
+run_test_by_name(Name) when is_list(Name) ->
+    case find_test_by_name(Name) of
+        {good, Mod} ->
+            rmm(Mod);
+        {bad, Mod} ->
+            io:format("FATAL: Module ~tp didn't compile~n", [Mod]),
+            ok;
+        not_found ->
+            io:format("FATAL: test not found: ~p~n", [Name]),
+            ok
+    end.
+
+
+run_mod_main(Mod) ->
+    rmm(Mod).
+
+rmm(Mod) ->
+    try
+        io:format("=================================================~n"),
+        io:format("~p:main()~n", [Mod]),
+        io:format("=================================================~n"),
+        Mod:main()
+    catch
+        Cat:Err:Tr ->
+            io:format("~tp:main(): ERROR~n", [Mod]),
+            io:format("~tp: ~tp~n", [Cat, Err]),
+            io:format("Trace: ~tp~n", [Tr]),
+            ok
+    end.
+
+
+
+find_test_by_name(Name) ->
+    C1 = list_to_atom(Name),
+    C2 = list_to_atom("gsc_test_" ++ Name),
+    {Gd, Bd} = runnable_test_mods(),
+    C1Gd = lists:member(C1, Gd),
+    C2Gd = lists:member(C2, Gd),
+    C1Bd = lists:member(C1, Gd),
+    C2Bd = lists:member(C2, Bd),
+    if
+        C1Gd -> {good, C1};
+        C2Gd -> {good, C2};
+        C1Bd -> {bad, C1};
+        C2Bd -> {bad, C2};
+        true -> not_found
+    end.
+
+
+-spec runnable_test_names() -> Result when
+        Result :: [{string(), atom()}].
+
+runnable_test_names() ->
+    {Gd, Bd} = runnable_test_mods(),
+    rtns([], lists:sort(Gd ++ Bd)).
+
+rtns(Acc, []) ->
+    lists:reverse(Acc);
+rtns(Acc, [TestMod | Rest]) ->
+    TestName = test_mod_name(TestMod),
+    rtns([{TestName, TestMod} | Acc], Rest).
+
+test_mod_name(TestModAtom) ->
+    "gsc_test_" ++ Name = atom_to_list(TestModAtom),
+    Name.
+
+
+
+-spec runnable_test_mods() -> Result when
+        Result :: {Good, Bad},
+        Good   :: Mods,
+        Bad    :: Mods,
+        Mods   :: [atom()].
+
+runnable_test_mods() ->
+    {Ld, Bds} = load_test_erls(),
+    Gd = lists:filter(fun is_runnable/1, Ld),
+    Bd = lists:filter(fun is_runnable/1, Bds),
+    {Gd, Bd}.
+
+
+
+is_runnable(ModAtom) ->
+    case atom_to_list(ModAtom) of
+        "gsc_test_" ++ _ -> true;
+        _                -> false
+    end.
+
+
+
+-spec load_test_erls() -> {Loaded, Errs} when
+        Loaded  :: [atom()],
+        Errs    :: [atom()].
+
+load_test_erls() ->
+    ltes([], [], ls_test_erls()).
+
+
+ltes(Ld, Errs, []) ->
+    {lists:reverse(Ld), lists:reverse(Errs)};
+ltes(Ld, Errs, [FP | Rest]) ->
+    FN = abspath_to_name(FP),
+    ModAtom = fp_to_mod_atom(FP),
+    case compile:file(FP) of
+        {ok, Mod} ->
+            ltes([Mod | Ld], Errs, Rest);
+        Err ->
+            io:format("ERROR ~tp: ~tp~n", [FN, Err]),
+            ltes(Ld, [ModAtom | Errs], Rest)
+    end.
+
+
+fp_to_mod_atom(FP) ->
+    FN = abspath_to_name(FP),
+    [ModStr, "erl"] = string:split(FN, ".", trailing),
+    list_to_atom(ModStr).
+
+
+
+-spec abspath_to_name(FilePath) -> FileName when
+        FilePath :: string(),
+        FileName :: string().
+% @doc "/path/to/foo.bar" -> "foo.bar"
+
+abspath_to_name(FP) ->
+    lists:last(string:tokens(FP, "/")).
+
+
+
+-spec ls_test_erls() -> AbsPaths when
+        AbsPaths :: [string()].
+% @doc ["/path/to/gsc/test/foo.erl",
+%       "/path/to/gsc/test/bar.erl",
+%       "/path/to/gsc/test/baz.erl"]
+
+ls_test_erls() ->
+    lists:filter(fun is_erl/1, ls_test()).
+
+
+
+-spec ls_test_beams() -> AbsPaths when
+        AbsPaths :: [string()].
+
+% important: beams get dropped in working dir
+ls_test_beams() ->
+    lists:filter(fun is_beam/1, ls_pwd()).
+
+
+
+-spec is_beam(AbsPath) -> IsBeam when
+        AbsPath :: string(),
+        IsBeam  :: boolean().
+
+% @private
+% "foo.beam" ~> true
+%  _         ~> false
+is_beam(Filename) ->
+    case filename:extension(Filename) of
+        ".beam" -> true;
+        _      -> false
+    end.
+
+
+
+-spec is_erl(AbsPath) -> IsErl when
+        AbsPath :: string(),
+        IsErl   :: boolean().
+% @private
+% "foo.erl" ~> true
+% _         ~> false
+
+is_erl(Filename) ->
+    case filename:extension(Filename) of
+        ".erl" -> true;
+        _      -> false
+    end.
+
+
+
+-spec ls_test() -> Abspaths when
+        Abspaths :: [string()].
+% @doc
+% Includes junk/irrelevant files:
+%
+% ["/path/to/gsc/test/foo.erl",
+%  "/path/to/gsc/test/.foo.erl.swp",
+%  "/path/to/gsc/test/bar.erl"]
+
+ls_test() ->
+    TD = test_dir(),
+    {ok, Names} = file:list_dir(TD),
+    lists:sort([TD ++ "/" ++ Name || Name <- Names]).
+
+
+ls_pwd() ->
+    {ok, D} = file:get_cwd(),
+    {ok, Ns} = file:list_dir(D),
+    lists:sort([D ++ "/" ++ N || N <- Ns]).
+
+
+-spec test_dir() -> AbsPath when
+        AbsPath :: string().
+% @doc "/path/to/gsc/test"
+
+test_dir() ->
+     zx_daemon:get_home() ++ "/test".
+
+
+
+-spec ct_dir() -> AbsPath when
+        AbsPath :: string().
+
+% @doc "/path/to/gsc/test/ct"
+%
 % directory containing the tests for the tokenizer
 ct_dir() ->
-    zx_daemon:get_home() ++ "/ct".
+    test_dir() ++ "/ct".
+
 
 
-ct_file_abspath(Name) ->
-    ct_file(Name).
 
 -spec ct_file(Name) -> AbsPath when
         Name :: string(),
         AbsPath :: string().
 % @doc
-% ct_file("foo.aes") -> "/path/to/ct/foo.aes"
+% "foo.aes" -> "/path/to/ct/foo.aes"
 
 ct_file(Name) ->
     ct_dir() ++ "/" ++ Name.
+
+
+% @doc alias for `ct_file/1'
+%
+% "foo.aes" -> "/path/to/ct/foo.aes"
+ct_file_abspath(Name) -> ct_file(Name).
+
+% @doc alias for `ct_file/1'
+%
+% "foo.aes" -> "/path/to/ct/foo.aes"
+ct_abspath(Name) -> ct_file(Name).

zomp.meta

@@ -1,7 +1,7 @@
 {name,"Gajumaru Sophia Compiler"}.
-{type,lib}.
+{type,cli}.
 {modules,[]}.
-{mod, "gsc_cli"}
+{mod, "gsc_cli"}.
 {author,"Peter Harpending"}.
 {prefix,"gsc"}.
 {desc,"Exploratory sophia compiler rewrite"}.