some stuff

src/gsc_cli.erl

@@ -8,11 +8,10 @@
 -copyright("Peter Harpending <peterharpending@qpq.swiss>").
 -license("GPL-3.0-only").
 
--export([
+%-export([
-    tokens/1,
+%    tokens/1,
-    so_tokens/1,
+%    gso_tokens/1
-    gso_tokens/1
+%]).
-]).
 -export([start/1]).
 
 -include("$gsc_include/gsc.hrl").
@@ -54,9 +53,9 @@ do(["ctokens", Foo])            -> do_color_tokens(Foo);
 do(["color_tokens", Foo])       -> do_color_tokens(Foo);
 do(["tokens", "--colour" | _])  -> do_doi();
 do(["colour_tokens" | _])       -> do_doi();
-% so_tokens = so_scan tokens
+%% so_tokens = so_scan tokens
-do(["so", "tokens", Foo]) -> do_so_tokens(Foo);
+%do(["so", "tokens", Foo]) -> do_so_tokens(Foo);
-do(["so_tokens", Foo])    -> do_so_tokens(Foo);
+%do(["so_tokens", Foo])    -> do_so_tokens(Foo);
 % gso_tokens = our mockery
 do(["gso", "tokens", Foo]) -> do_gso_tokens(Foo);
 do(["gso_tokens", Foo])    -> do_gso_tokens(Foo);
@@ -130,9 +129,6 @@ do_eshell() ->
 do_tokens(FilePath) ->
     [io:format("~p~n", [Tk]) || Tk <- tokens(FilePath)].
 
-do_so_tokens(FilePath) ->
-    [io:format("~p~n", [Tk]) || Tk <- so_tokens(FilePath)].
-
 do_gso_tokens(FilePath) ->
     [io:format("~p~n", [Tk]) || Tk <- gso_tokens(FilePath)].
 
@@ -146,12 +142,6 @@ do_rmm(FilePath) ->
     end.
 
 
-so_tokens(FilePath) ->
-    {ok, FileBytes} = file:read_file(FilePath),
-    FileStr = unicode:characters_to_nfc_list(FileBytes),
-    {ok, Tokens} = so_scan:scan(FileStr),
-    Tokens.
-
 gso_tokens(FilePath) ->
     {ok, FileBytes} = file:read_file(FilePath),
     FileStr = unicode:characters_to_nfc_list(FileBytes),

test/gsc_tests.erl

@@ -19,17 +19,27 @@ cli_args(TestNames) ->
         {ok, ts_utils} -> ok;
         Error -> error(Error)
     end,
-    ts_utils:tidily(fun() -> do(TestNames) end).
+    % this loads the test deps and then cleans up any
+    % beam files afterwards
+    ts_utils:tidily(fun() -> do_gsc_test(TestNames) end).
+
+
+do_gsc_test(["-h" | _])          -> do_help();
+do_gsc_test(["--help" | _])      -> do_help();
+do_gsc_test(["-l" | _])          -> do_list();
+do_gsc_test(["--list" | _])      -> do_list();
+do_gsc_test(["-a" | _])          -> do_all();
+do_gsc_test(["--all" | _])       -> do_all();
+do_gsc_test(["so_tokens", X])    -> do_so_tokens(X);
+do_gsc_test(["so", "tokens", X]) -> do_so_tokens(X);
+do_gsc_test(["tokenizers_agree", X]) ->
+    do_tokenizers_agree(X);
+do_gsc_test([TestName]) ->
+    ts_utils:run_test_by_name(TestName);
+do_gsc_test(_) ->
+    do_help().
 
 
-do(["-h" | _])     -> do_help();
-do(["--help" | _]) -> do_help();
-do(["-l" | _])     -> do_list();
-do(["--list" | _]) -> do_list();
-do(["-a" | _])     -> do_all();
-do(["--all" | _])  -> do_all();
-do(TestNames) ->
-    lists:foreach(fun ts_utils:run_test_by_name/1, TestNames).
 
 do_help() ->
     io:format("go help yourself~n").
@@ -38,6 +48,17 @@ do_list() ->
     Names = ts_utils:runnable_test_names(),
     [io:format("~p~n", [N]) || N <- Names].
 
+
 do_all() ->
     {Gd, _} = ts_utils:runnable_test_mods(),
     [begin ts_utils:rmm(G), io:format("~n") end || G <- Gd].
+
+
+do_so_tokens(FilePath) ->
+    [io:format("~p~n", [Tk]) || Tk <- ts_utils:so_tokens(FilePath)].
+
+
+
+do_tokenizers_agree(RelPath) ->
+    Result = ts_utils:tokenizers_agree(RelPath),
+    io:format("~tp~n", [Result]).

test/ts_utils.erl

@@ -2,6 +2,9 @@
 -module(ts_utils).
 
 -export([
+    tokenizers_agree/1,
+    absify/1,
+    so_tokens/1,
     load_test_deps/0,
     test_deps/0,
     load_dep/1,
@@ -23,6 +26,29 @@
     ct_file/1, ct_file_abspath/1, ct_abspath/1
 ]).
 
+tokenizers_agree(Relpath) ->
+    FilePath = absify(Relpath),
+    % extracting data to be tested
+    % i hate this so much but lazy and this is test code so who really cares. 
+    SoTokens = so_tokens(FilePath),
+    SfTokens = gsc:gso_tokens_from_file(FilePath),
+    case {SoTokens, SfTokens} of
+        {{ok, So},    {ok, Sf}}  -> So =:= Sf;
+        {{error, _}, {error, _}} -> true;
+        {{ok, _},    {error, _}} -> false;
+        {{error, _}, {ok, _}}    -> false
+    end.
+
+
+absify(RelPath) ->
+    filename:absname(RelPath).
+
+
+so_tokens(FilePath) ->
+    {ok, FileBytes} = file:read_file(FilePath),
+    FileStr = binary_to_list(FileBytes),
+    so_scan:scan(FileStr).
+
 
 load_test_deps() ->
     lists:foreach(fun load_dep/1, test_deps()).