some stuff

test/ts_utils.erl

@@ -2,6 +2,9 @@
 -module(ts_utils).
 
 -export([
+    tokenizers_agree/1,
+    absify/1,
+    so_tokens/1,
     load_test_deps/0,
     test_deps/0,
     load_dep/1,
@@ -23,6 +26,29 @@
     ct_file/1, ct_file_abspath/1, ct_abspath/1
 ]).
 
+tokenizers_agree(Relpath) ->
+    FilePath = absify(Relpath),
+    % extracting data to be tested
+    % i hate this so much but lazy and this is test code so who really cares. 
+    SoTokens = so_tokens(FilePath),
+    SfTokens = gsc:gso_tokens_from_file(FilePath),
+    case {SoTokens, SfTokens} of
+        {{ok, So},    {ok, Sf}}  -> So =:= Sf;
+        {{error, _}, {error, _}} -> true;
+        {{ok, _},    {error, _}} -> false;
+        {{error, _}, {ok, _}}    -> false
+    end.
+
+
+absify(RelPath) ->
+    filename:absname(RelPath).
+
+
+so_tokens(FilePath) ->
+    {ok, FileBytes} = file:read_file(FilePath),
+    FileStr = binary_to_list(FileBytes),
+    so_scan:scan(FileStr).
+
 
 load_test_deps() ->
     lists:foreach(fun load_dep/1, test_deps()).