Merge remote-tracking branch 'refs/remotes/origin/master'

src/gsc_cli.erl

@@ -8,11 +8,10 @@
 -copyright("Peter Harpending <peterharpending@qpq.swiss>").
 -license("GPL-3.0-only").
 
--export([
+%-export([
-    tokens/1,
+%    tokens/1,
-    so_tokens/1,
+%    gso_tokens/1
-    gso_tokens/1
+%]).
-]).
 -export([start/1]).
 
 -include("$gsc_include/gsc.hrl").
@@ -47,13 +46,16 @@ do(["test" | Tests]) ->
 do(["tokens", Foo]) -> do_tokens(Foo);
 % print source file to screen with token boundaries
 % highlighted
+do(["tokens", "-c", Foo])       -> do_color_tokens(Foo);
+do(["tokens", "--color", Foo])  -> do_color_tokens(Foo);
 do(["tcat", Foo])               -> do_color_tokens(Foo);
 do(["ctokens", Foo])            -> do_color_tokens(Foo);
 do(["color_tokens", Foo])       -> do_color_tokens(Foo);
+do(["tokens", "--colour" | _])  -> do_doi();
 do(["colour_tokens" | _])       -> do_doi();
-% so_tokens = so_scan tokens
+%% so_tokens = so_scan tokens
-do(["so", "tokens", Foo]) -> do_so_tokens(Foo);
+%do(["so", "tokens", Foo]) -> do_so_tokens(Foo);
-do(["so_tokens", Foo]) -> do_so_tokens(Foo);
+%do(["so_tokens", Foo])    -> do_so_tokens(Foo);
 % gso_tokens = our mockery
 do(["gso", "tokens", Foo]) -> do_gso_tokens(Foo);
 do(["gso_tokens", Foo])    -> do_gso_tokens(Foo);
@@ -127,9 +129,6 @@ do_eshell() ->
 do_tokens(FilePath) ->
     [io:format("~p~n", [Tk]) || Tk <- tokens(FilePath)].
 
-do_so_tokens(FilePath) ->
-    [io:format("~p~n", [Tk]) || Tk <- so_tokens(FilePath)].
-
 do_gso_tokens(FilePath) ->
     [io:format("~p~n", [Tk]) || Tk <- gso_tokens(FilePath)].
 
@@ -143,12 +142,6 @@ do_rmm(FilePath) ->
     end.
 
 
-so_tokens(FilePath) ->
-    {ok, FileBytes} = file:read_file(FilePath),
-    FileStr = unicode:characters_to_nfc_list(FileBytes),
-    {ok, Tokens} = so_scan:scan(FileStr),
-    Tokens.
-
 gso_tokens(FilePath) ->
     {ok, FileBytes} = file:read_file(FilePath),
     FileStr = unicode:characters_to_nfc_list(FileBytes),
@@ -200,13 +193,13 @@ colorize_token_str(Color, T = #tk{str = Str}) ->
 
 
 % dim noisy tokens
-color_fixes(noise, Color) ->
+%color_fixes(noise, Color) ->
-    {P, S} = color_fixes(signal, Color),
+%    {P, S} = color_fixes(signal, Color),
-    {[?ANSI_DIM, P], [S, ?ANSI_UNDIM]};
+%    {[?ANSI_DIM, P], [S, ?ANSI_UNDIM]};
-color_fixes(signal, red)     -> {?ANSI_FG_RED,     ?ANSI_FG_RESET};
+color_fixes(_, red)     -> {?ANSI_FG_RED,     ?ANSI_FG_RESET};
-color_fixes(signal, green)   -> {?ANSI_FG_GREEN,   ?ANSI_FG_RESET};
+color_fixes(_, green)   -> {?ANSI_FG_GREEN,   ?ANSI_FG_RESET};
-color_fixes(signal, yellow)  -> {?ANSI_FG_YELLOW,  ?ANSI_FG_RESET};
+color_fixes(_, yellow)  -> {?ANSI_FG_YELLOW,  ?ANSI_FG_RESET};
-color_fixes(signal, blue)    -> {?ANSI_FG_BLUE,    ?ANSI_FG_RESET};
+color_fixes(_, blue)    -> {?ANSI_FG_BLUE,    ?ANSI_FG_RESET};
-color_fixes(signal, magenta) -> {?ANSI_FG_MAGENTA, ?ANSI_FG_RESET};
+color_fixes(_, magenta) -> {?ANSI_FG_MAGENTA, ?ANSI_FG_RESET};
-color_fixes(signal, cyan)    -> {?ANSI_FG_CYAN,    ?ANSI_FG_RESET}.
+color_fixes(_, cyan)    -> {?ANSI_FG_CYAN,    ?ANSI_FG_RESET}.
 

test/gsc_tests.erl

@@ -19,17 +19,27 @@ cli_args(TestNames) ->
         {ok, ts_utils} -> ok;
         Error -> error(Error)
     end,
-    ts_utils:tidily(fun() -> do(TestNames) end).
+    % this loads the test deps and then cleans up any
+    % beam files afterwards
+    ts_utils:tidily(fun() -> do_gsc_test(TestNames) end).
+
+
+do_gsc_test(["-h" | _])          -> do_help();
+do_gsc_test(["--help" | _])      -> do_help();
+do_gsc_test(["-l" | _])          -> do_list();
+do_gsc_test(["--list" | _])      -> do_list();
+do_gsc_test(["-a" | _])          -> do_all();
+do_gsc_test(["--all" | _])       -> do_all();
+do_gsc_test(["so_tokens", X])    -> do_so_tokens(X);
+do_gsc_test(["so", "tokens", X]) -> do_so_tokens(X);
+do_gsc_test(["tokenizers_agree", X]) ->
+    do_tokenizers_agree(X);
+do_gsc_test([TestName]) ->
+    ts_utils:run_test_by_name(TestName);
+do_gsc_test(_) ->
+    do_help().
 
 
-do(["-h" | _])     -> do_help();
-do(["--help" | _]) -> do_help();
-do(["-l" | _])     -> do_list();
-do(["--list" | _]) -> do_list();
-do(["-a" | _])     -> do_all();
-do(["--all" | _])  -> do_all();
-do(TestNames) ->
-    lists:foreach(fun ts_utils:run_test_by_name/1, TestNames).
 
 do_help() ->
     io:format("go help yourself~n").
@@ -38,6 +48,17 @@ do_list() ->
     Names = ts_utils:runnable_test_names(),
     [io:format("~p~n", [N]) || N <- Names].
 
+
 do_all() ->
     {Gd, _} = ts_utils:runnable_test_mods(),
     [begin ts_utils:rmm(G), io:format("~n") end || G <- Gd].
+
+
+do_so_tokens(FilePath) ->
+    [io:format("~p~n", [Tk]) || Tk <- ts_utils:so_tokens(FilePath)].
+
+
+
+do_tokenizers_agree(RelPath) ->
+    Result = ts_utils:tokenizers_agree(RelPath),
+    io:format("~tp~n", [Result]).

test/ts_utils.erl

@@ -2,6 +2,9 @@
 -module(ts_utils).
 
 -export([
+    tokenizers_agree/1,
+    absify/1,
+    so_tokens/1,
     load_test_deps/0,
     test_deps/0,
     load_dep/1,
@@ -23,6 +26,29 @@
     ct_file/1, ct_file_abspath/1, ct_abspath/1
 ]).
 
+tokenizers_agree(Relpath) ->
+    FilePath = absify(Relpath),
+    % extracting data to be tested
+    % i hate this so much but lazy and this is test code so who really cares. 
+    SoTokens = so_tokens(FilePath),
+    SfTokens = gsc:gso_tokens_from_file(FilePath),
+    case {SoTokens, SfTokens} of
+        {{ok, So},    {ok, Sf}}  -> So =:= Sf;
+        {{error, _}, {error, _}} -> true;
+        {{ok, _},    {error, _}} -> false;
+        {{error, _}, {ok, _}}    -> false
+    end.
+
+
+absify(RelPath) ->
+    filename:absname(RelPath).
+
+
+so_tokens(FilePath) ->
+    {ok, FileBytes} = file:read_file(FilePath),
+    FileStr = binary_to_list(FileBytes),
+    so_scan:scan(FileStr).
+
 
 load_test_deps() ->
     lists:foreach(fun load_dep/1, test_deps()).