some stuff

test/gsc_tests.erl

@@ -19,17 +19,27 @@ cli_args(TestNames) ->
         {ok, ts_utils} -> ok;
         Error -> error(Error)
     end,
-    ts_utils:tidily(fun() -> do(TestNames) end).
+    % this loads the test deps and then cleans up any
+    % beam files afterwards
+    ts_utils:tidily(fun() -> do_gsc_test(TestNames) end).
+
+
+do_gsc_test(["-h" | _])          -> do_help();
+do_gsc_test(["--help" | _])      -> do_help();
+do_gsc_test(["-l" | _])          -> do_list();
+do_gsc_test(["--list" | _])      -> do_list();
+do_gsc_test(["-a" | _])          -> do_all();
+do_gsc_test(["--all" | _])       -> do_all();
+do_gsc_test(["so_tokens", X])    -> do_so_tokens(X);
+do_gsc_test(["so", "tokens", X]) -> do_so_tokens(X);
+do_gsc_test(["tokenizers_agree", X]) ->
+    do_tokenizers_agree(X);
+do_gsc_test([TestName]) ->
+    ts_utils:run_test_by_name(TestName);
+do_gsc_test(_) ->
+    do_help().
 
 
-do(["-h" | _])     -> do_help();
-do(["--help" | _]) -> do_help();
-do(["-l" | _])     -> do_list();
-do(["--list" | _]) -> do_list();
-do(["-a" | _])     -> do_all();
-do(["--all" | _])  -> do_all();
-do(TestNames) ->
-    lists:foreach(fun ts_utils:run_test_by_name/1, TestNames).
 
 do_help() ->
     io:format("go help yourself~n").
@@ -38,6 +48,17 @@ do_list() ->
     Names = ts_utils:runnable_test_names(),
     [io:format("~p~n", [N]) || N <- Names].
 
+
 do_all() ->
     {Gd, _} = ts_utils:runnable_test_mods(),
     [begin ts_utils:rmm(G), io:format("~n") end || G <- Gd].
+
+
+do_so_tokens(FilePath) ->
+    [io:format("~p~n", [Tk]) || Tk <- ts_utils:so_tokens(FilePath)].
+
+
+
+do_tokenizers_agree(RelPath) ->
+    Result = ts_utils:tokenizers_agree(RelPath),
+    io:format("~tp~n", [Result]).

test/ts_utils.erl

@@ -2,6 +2,9 @@
 -module(ts_utils).
 
 -export([
+    tokenizers_agree/1,
+    absify/1,
+    so_tokens/1,
     load_test_deps/0,
     test_deps/0,
     load_dep/1,
@@ -23,6 +26,29 @@
     ct_file/1, ct_file_abspath/1, ct_abspath/1
 ]).
 
+tokenizers_agree(Relpath) ->
+    FilePath = absify(Relpath),
+    % extracting data to be tested
+    % i hate this so much but lazy and this is test code so who really cares. 
+    SoTokens = so_tokens(FilePath),
+    SfTokens = gsc:gso_tokens_from_file(FilePath),
+    case {SoTokens, SfTokens} of
+        {{ok, So},    {ok, Sf}}  -> So =:= Sf;
+        {{error, _}, {error, _}} -> true;
+        {{ok, _},    {error, _}} -> false;
+        {{error, _}, {ok, _}}    -> false
+    end.
+
+
+absify(RelPath) ->
+    filename:absname(RelPath).
+
+
+so_tokens(FilePath) ->
+    {ok, FileBytes} = file:read_file(FilePath),
+    FileStr = binary_to_list(FileBytes),
+    so_scan:scan(FileStr).
+
 
 load_test_deps() ->
     lists:foreach(fun load_dep/1, test_deps()).