add vrlp tests
This commit is contained in:
@@ -2,3 +2,4 @@
|
|||||||
*.swo
|
*.swo
|
||||||
erl_crash.dump
|
erl_crash.dump
|
||||||
*.beam
|
*.beam
|
||||||
|
__pycache__
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3.9
|
||||||
|
|
||||||
|
'''
|
||||||
|
generate rlp tests
|
||||||
|
|
||||||
|
really checking to see if my code matches the ethereum python rlp package which
|
||||||
|
has 85 stars on github
|
||||||
|
|
||||||
|
requires rlp package, which requires python 3.7 or greater
|
||||||
|
|
||||||
|
this checks to see if my Erlang implementation matches python. It works for
|
||||||
|
59,638 randomly generated test cases
|
||||||
|
'''
|
||||||
|
|
||||||
|
# need more cases
|
||||||
|
# single bytes
|
||||||
|
# empty list
|
||||||
|
# list with variable number of elements
|
||||||
|
# element has equal chance of becoming a list or a binary
|
||||||
|
|
||||||
|
import rlp
|
||||||
|
import random as r
|
||||||
|
|
||||||
|
from rlpcases import cases
|
||||||
|
|
||||||
|
def main():
|
||||||
|
all_encode_work = True
|
||||||
|
all_decode_work = True
|
||||||
|
for case in cases:
|
||||||
|
decoded_data = case['decoded']
|
||||||
|
encoded_bytes = case['encoded']
|
||||||
|
real_encoded_bytes = rlp.encode(decoded_data)
|
||||||
|
real_decoded_data = rlp.decode(encoded_bytes)
|
||||||
|
# check if encode/decode works
|
||||||
|
encode_works = (real_encoded_bytes == encoded_bytes)
|
||||||
|
decode_works = (real_decoded_data == decoded_data)
|
||||||
|
print('encode works: %s' % (encode_works))
|
||||||
|
print('decode works: %s' % (decode_works))
|
||||||
|
# update globals
|
||||||
|
all_encode_work = all_encode_work and encode_works
|
||||||
|
all_decode_work = all_decode_work and decode_works
|
||||||
|
# print globals
|
||||||
|
print('all encode work: %s' % (all_encode_work))
|
||||||
|
print('all decode work: %s' % (all_decode_work))
|
||||||
|
#print(format_cases_erl(c))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,157 @@
|
|||||||
|
%%% @doc
|
||||||
|
%%% Vanillae RLP encoder/decoder
|
||||||
|
%%%
|
||||||
|
%%% Reference: https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp/
|
||||||
|
%%%
|
||||||
|
%%% Agrees with Ethereum's Python implementation in randomized tests
|
||||||
|
%%% @end
|
||||||
|
|
||||||
|
-module(vrlp).
|
||||||
|
-vsn("0.1.0").
|
||||||
|
-author("Peter Harpending <peter.harpending@gmail.com>").
|
||||||
|
-copyright("Peter Harpending <peter.harpending@gmail.com>").
|
||||||
|
-license("ISC").
|
||||||
|
|
||||||
|
-export_type([decoded_data/0]).
|
||||||
|
-export([encode/1, decode/1]).
|
||||||
|
|
||||||
|
-type decoded_data() :: binary() | [decoded_data()].
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-spec encode(Data) -> RLP
|
||||||
|
when Data :: decoded_data(),
|
||||||
|
RLP :: binary().
|
||||||
|
%% @doc
|
||||||
|
%% Encode some data. Crashes if input is not either a binary or a list (of
|
||||||
|
%% lists of ...) binaries. Lists can be empty, as can binaries.
|
||||||
|
|
||||||
|
encode(Binary) when is_binary(Binary) ->
|
||||||
|
encode_binary(Binary);
|
||||||
|
encode(List) when is_list(List) ->
|
||||||
|
encode_list(List).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-spec encode_binary(Bytes) -> RLP
|
||||||
|
when Bytes :: binary(),
|
||||||
|
RLP :: binary().
|
||||||
|
%% @private
|
||||||
|
%% encode a binary in rlp
|
||||||
|
%% @end
|
||||||
|
|
||||||
|
% single byte case when the byte is between 0..127
|
||||||
|
% result is the byte itself
|
||||||
|
encode_binary(<<Byte>>) when Byte =< 127 ->
|
||||||
|
<<Byte>>;
|
||||||
|
% if the bytestring is 0..55 items long, the first byte is 128 + Length,
|
||||||
|
% the rest of the string is the string
|
||||||
|
encode_binary(Bytes) when byte_size(Bytes) =< 55 ->
|
||||||
|
Size = byte_size(Bytes),
|
||||||
|
<<(128 + Size), Bytes/binary>>;
|
||||||
|
% more than 55 bytes long, first byte is 183 + ByteLengthOfLength
|
||||||
|
% max byte size is 2^64 - 1
|
||||||
|
encode_binary(Bytes) when 55 < byte_size(Bytes), byte_size(Bytes) < (1 bsl 64) ->
|
||||||
|
SizeInt = byte_size(Bytes),
|
||||||
|
SizeBytes = binary:encode_unsigned(SizeInt, big),
|
||||||
|
SizeOfSizeInt = byte_size(SizeBytes),
|
||||||
|
%% 183 = 128 + 55
|
||||||
|
%% SizeOfSizeInt > 0
|
||||||
|
<<(183 + SizeOfSizeInt),
|
||||||
|
SizeBytes/binary,
|
||||||
|
Bytes/binary>>.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-spec encode_list(List) -> RLP
|
||||||
|
when List :: [decoded_data()],
|
||||||
|
RLP :: binary().
|
||||||
|
%% @private
|
||||||
|
%% encode a list in rlp
|
||||||
|
%% @end
|
||||||
|
|
||||||
|
% first we encode the total payload of the list
|
||||||
|
% depending on how long it is, we then branch
|
||||||
|
encode_list(List) ->
|
||||||
|
Payload = << (encode(Item)) || Item <- List>>,
|
||||||
|
Payload_Size = byte_size(Payload),
|
||||||
|
if
|
||||||
|
Payload_Size =< 55 ->
|
||||||
|
<<(192 + Payload_Size), Payload/binary>>;
|
||||||
|
55 < Payload_Size ->
|
||||||
|
SizeBytes = binary:encode_unsigned(Payload_Size, big),
|
||||||
|
SizeOfSizeInt = byte_size(SizeBytes),
|
||||||
|
%% 247 = 192 + 55
|
||||||
|
%% SizeOfSizeInt > 0
|
||||||
|
<<(247 + SizeOfSizeInt),
|
||||||
|
SizeBytes/binary,
|
||||||
|
Payload/binary>>
|
||||||
|
end.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-spec decode(RLP) -> {Data, Rest}
|
||||||
|
when RLP :: binary(),
|
||||||
|
Data :: decoded_data(),
|
||||||
|
Rest :: binary().
|
||||||
|
%% @doc
|
||||||
|
%% Decode an RLP-encoded bytestring. Returns a tuple containing the
|
||||||
|
%% consumed/parsed data, as well as the remainder of the data. Crashes if
|
||||||
|
%% parsing fails.
|
||||||
|
%% @end
|
||||||
|
|
||||||
|
% if the first byte is between 0 and 127, that is the data
|
||||||
|
decode(<<Byte, Rest/binary>>) when Byte =< 127 ->
|
||||||
|
{<<Byte>>, Rest};
|
||||||
|
% if the first byte is between 128 and 183 = 128 + 55, it is a bytestring and
|
||||||
|
% the length is Byte - 128
|
||||||
|
decode(<<Byte, Rest/binary>>) when Byte =< 183 ->
|
||||||
|
PayloadByteLength = Byte - 128,
|
||||||
|
%PayloadBitLength = 8 * PayloadByteLength,
|
||||||
|
%io:format("Byte : ~p~n"
|
||||||
|
% "Rest : ~w~n"
|
||||||
|
% "PayloadByteLength : ~p~n",
|
||||||
|
% %"PayloadBitLength : ~p~n",
|
||||||
|
% [Byte, Rest, PayloadByteLength]),
|
||||||
|
<<Payload:PayloadByteLength/binary,
|
||||||
|
Rest2/binary>> = Rest,
|
||||||
|
{Payload, Rest2};
|
||||||
|
% If the first byte is between 184 = 183 + 1 and 191 = 183 + 8, it is a
|
||||||
|
% bytestring. The byte length of the byte length of bytestring is FirstByte -
|
||||||
|
% 183. Then pull out the actual data
|
||||||
|
decode(<<Byte, Rest/binary>>) when Byte =< 191 ->
|
||||||
|
ByteLengthOfByteLength = Byte - 183,
|
||||||
|
BitLengthOfByteLength = 8 * ByteLengthOfByteLength,
|
||||||
|
<<ByteLengthInt:BitLengthOfByteLength,
|
||||||
|
Rest2/binary>> = Rest,
|
||||||
|
<<Payload:ByteLengthInt/binary,
|
||||||
|
Rest3/binary>> = Rest2,
|
||||||
|
{Payload, Rest3};
|
||||||
|
% If the first byte is between 192 and 247 = 192 + 55, it is a list. The byte
|
||||||
|
% length of the list-payload is FirstByte - 192. Then the list payload, which
|
||||||
|
% needs to be decoded on its own.
|
||||||
|
decode(<<Byte, Rest/binary>>) when Byte =< 247 ->
|
||||||
|
ByteLengthOfListPayload = Byte - 192,
|
||||||
|
<<ListPayload:ByteLengthOfListPayload/binary,
|
||||||
|
Rest2/binary>> = Rest,
|
||||||
|
List = decode_list(ListPayload),
|
||||||
|
{List, Rest2};
|
||||||
|
% If the first byte is between 248 = 247 + 1 and 255 = 247 + 8, it is a list.
|
||||||
|
% The byte length of the byte length of the list-payload is FirstByte - 247.
|
||||||
|
% Then the byte length of the list. Then the list payload, which needs to be
|
||||||
|
% decoded on its own.
|
||||||
|
decode(<<Byte, Rest/binary>>) ->
|
||||||
|
ByteLengthOfByteLengthOfListPayload_int = Byte - 247,
|
||||||
|
BitLengthOfByteLengthOfListPayload_int = 8 * ByteLengthOfByteLengthOfListPayload_int,
|
||||||
|
<<ByteLengthOfListPayload_int:BitLengthOfByteLengthOfListPayload_int,
|
||||||
|
Rest2/binary>> = Rest,
|
||||||
|
<<ListPayload_bytes:ByteLengthOfListPayload_int/binary,
|
||||||
|
Rest3/binary>> = Rest2,
|
||||||
|
List = decode_list(ListPayload_bytes),
|
||||||
|
{List, Rest3}.
|
||||||
|
|
||||||
|
decode_list(<<>>) ->
|
||||||
|
[];
|
||||||
|
decode_list(Bytes) ->
|
||||||
|
{Item, Rest} = decode(Bytes),
|
||||||
|
[Item | decode_list(Rest)].
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
%-module(rlp_testgen).
|
||||||
|
%-compile(export_all).
|
||||||
|
|
||||||
|
-mode(compile).
|
||||||
|
-compile([nowarn_unused_function]).
|
||||||
|
|
||||||
|
main([]) ->
|
||||||
|
% Decoded cases
|
||||||
|
DecodedCases = rand_decode_datas(10_000),
|
||||||
|
%io:format("~p~n", [DecodedCases]),
|
||||||
|
io:format("~s~n", [format_cases_py(DecodedCases)]),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
%%% JS FORMATTING %%%
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
|
||||||
|
format_cases_js(Cases) ->
|
||||||
|
format_stringlist_js(lists:map(fun format_case_js/1, Cases)).
|
||||||
|
|
||||||
|
% format a list of strings into a python list
|
||||||
|
format_stringlist_js(List) ->
|
||||||
|
% stringlist commas are the same in python/js, so reusing python
|
||||||
|
["import * as rlp from './jex_include/local-vanillae-0.1.0/dist/rlp.js';\n"
|
||||||
|
"\n"
|
||||||
|
"type rlpcases = {decoded: rlp.decoded_data, encoded: Uint8Array};\n"
|
||||||
|
"\n"
|
||||||
|
"// @ts-ignore never mind; jesus christ\n"
|
||||||
|
"export const cases : Array<rlpcases> = [\n", slcommas(List, []), "];"].
|
||||||
|
|
||||||
|
% input: decoded_data
|
||||||
|
% format a case as
|
||||||
|
% {'decoded_data': <js term for rlist>,
|
||||||
|
% 'encoded_bytes': new Uint8Array([B1, B2, B3, ...])}
|
||||||
|
format_case_js(DecodedData_rlist) ->
|
||||||
|
% EncodedData_bytes = rlp:encode(
|
||||||
|
% DD_js = format_data_js(DecodedData_rlist),
|
||||||
|
EncodedData_bytes = vrlp:encode(DecodedData_rlist),
|
||||||
|
EncodedBytes_py = format_bytes_js(EncodedData_bytes),
|
||||||
|
DecodedData_py = format_data_js(DecodedData_rlist),
|
||||||
|
[" {'decoded': ", DecodedData_py, ",\n",
|
||||||
|
" 'encoded': ", EncodedBytes_py, "}"].
|
||||||
|
|
||||||
|
|
||||||
|
format_data_js(List) when is_list(List) ->
|
||||||
|
format_list_js(List);
|
||||||
|
format_data_js(Bytes) when is_binary(Bytes) ->
|
||||||
|
format_bytes_js(Bytes).
|
||||||
|
|
||||||
|
|
||||||
|
format_list_js(List) ->
|
||||||
|
[$[, js_lcommas(List, []), $]].
|
||||||
|
|
||||||
|
|
||||||
|
% similar to commas/2 below but for a list
|
||||||
|
% cases
|
||||||
|
% - list is empty -> special case
|
||||||
|
% - exactly one element -> special case
|
||||||
|
% - two or more elements -> peel off one at a time until terminal case of exactly one
|
||||||
|
% initial input empty, so return empty
|
||||||
|
js_lcommas([], []) ->
|
||||||
|
[];
|
||||||
|
% one item left, do not add comma
|
||||||
|
js_lcommas([Item], Acc) ->
|
||||||
|
[Acc, format_data_js(Item)];
|
||||||
|
% two or more items left, add comma
|
||||||
|
js_lcommas([Item | Rest], Acc) ->
|
||||||
|
js_lcommas(Rest, [Acc, format_data_js(Item), ", "]).
|
||||||
|
|
||||||
|
% format a bytestring as "bytes([Byte1, Byte2, ...])"
|
||||||
|
format_bytes_js(Bytes) ->
|
||||||
|
% commas are the same as in python so reusing that
|
||||||
|
Commas = commas(Bytes, []),
|
||||||
|
["new Uint8Array([", Commas, "])"].
|
||||||
|
|
||||||
|
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
%%% PYTHON FORMATTING %%%
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
|
||||||
|
format_cases_py(Cases) ->
|
||||||
|
["# autogenerated rlp cases from vrlp_testgen.erl\n",
|
||||||
|
format_stringlist_py(lists:map(fun format_case_py/1, Cases))].
|
||||||
|
|
||||||
|
% format a list of strings into a python list
|
||||||
|
format_stringlist_py(List) ->
|
||||||
|
["cases = [\n", slcommas(List, []), $]].
|
||||||
|
|
||||||
|
% similar to commas/2 below but for a list of the dictstrings
|
||||||
|
% adding a comma, a newline, and a space
|
||||||
|
%
|
||||||
|
% this one does no intrnal processing
|
||||||
|
%
|
||||||
|
% cases
|
||||||
|
% - list is empty -> special case
|
||||||
|
% - exactly one element -> special case
|
||||||
|
% - two or more elements -> peel off one at a time until terminal case of exactly one
|
||||||
|
% initial input empty, so return empty
|
||||||
|
slcommas([], []) ->
|
||||||
|
[];
|
||||||
|
% one item left, do not add comma
|
||||||
|
slcommas([Item], Acc) ->
|
||||||
|
[Acc, Item];
|
||||||
|
% two or more items left, add comma
|
||||||
|
slcommas([Item | Rest], Acc) ->
|
||||||
|
slcommas(Rest, [Acc, Item, ",\n"]).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
% input: decoded_data
|
||||||
|
% format a case as
|
||||||
|
% {'decoded_data': <python term for rlist>,
|
||||||
|
% 'encoded_bytes': bytes([B1, B2, B3, ...])}
|
||||||
|
format_case_py(DecodedData_rlist) ->
|
||||||
|
% EncodedData_bytes = rlp:encode(
|
||||||
|
% DD_js = format_data_js(DecodedData_rlist),
|
||||||
|
EncodedData_bytes = vrlp:encode(DecodedData_rlist),
|
||||||
|
EncodedBytes_py = format_bytes_py(EncodedData_bytes),
|
||||||
|
DecodedData_py = format_data_py(DecodedData_rlist),
|
||||||
|
[" {'decoded': ", DecodedData_py, ",\n",
|
||||||
|
" 'encoded': ", EncodedBytes_py, "}"].
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
format_data_py(List) when is_list(List) ->
|
||||||
|
format_list_py(List);
|
||||||
|
format_data_py(Bytes) when is_binary(Bytes) ->
|
||||||
|
format_bytes_py(Bytes).
|
||||||
|
|
||||||
|
format_list_py(List) ->
|
||||||
|
[$[, lcommas(List, []), $]].
|
||||||
|
|
||||||
|
% similar to commas/2 below but for a list
|
||||||
|
% cases
|
||||||
|
% - list is empty -> special case
|
||||||
|
% - exactly one element -> special case
|
||||||
|
% - two or more elements -> peel off one at a time until terminal case of exactly one
|
||||||
|
% initial input empty, so return empty
|
||||||
|
lcommas([], []) ->
|
||||||
|
[];
|
||||||
|
% one item left, do not add comma
|
||||||
|
lcommas([Item], Acc) ->
|
||||||
|
[Acc, format_data_py(Item)];
|
||||||
|
% two or more items left, add comma
|
||||||
|
lcommas([Item | Rest], Acc) ->
|
||||||
|
lcommas(Rest, [Acc, format_data_py(Item), ", "]).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
% format a bytestring as "bytes([Byte1, Byte2, ...])"
|
||||||
|
format_bytes_py(Bytes) ->
|
||||||
|
Commas = commas(Bytes, []),
|
||||||
|
["bytes([", Commas, "])"].
|
||||||
|
|
||||||
|
% cases:
|
||||||
|
% - bytestring is empty -> special case
|
||||||
|
% - exactly one element -> special case
|
||||||
|
% - two or more elements -> peel off one at a time until terminal case of exactly two
|
||||||
|
% empty bytestring
|
||||||
|
commas(<<>>, []) ->
|
||||||
|
[];
|
||||||
|
% only one byte left, do not add comma
|
||||||
|
commas(<<B2>>, Acc) ->
|
||||||
|
[Acc, integer_to_list(B2)];
|
||||||
|
% two or more bytes left: peel off one, add comma
|
||||||
|
commas(<<B, Rest/binary>>, Acc) ->
|
||||||
|
commas(Rest, [Acc, integer_to_list(B), ", "]).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
%%% INVERSE PROPERTY TEST %%%
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
|
||||||
|
% test that the inverse property is correct
|
||||||
|
|
||||||
|
test_inverse() ->
|
||||||
|
% test stuff
|
||||||
|
DecodedCases = rand_decode_datas(20),
|
||||||
|
All = lists:all(% predicate
|
||||||
|
fun(X) -> X end,
|
||||||
|
% data
|
||||||
|
lists:map(fun(DecodedData) ->
|
||||||
|
CI = check_inverse(DecodedData),
|
||||||
|
ok = io:format("~p~n", [CI]),
|
||||||
|
CI
|
||||||
|
end,
|
||||||
|
DecodedCases)),
|
||||||
|
io:format("all: ~p~n", [All]),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
|
||||||
|
check_inverse(DecodedData) ->
|
||||||
|
{DeEncodedData, <<>>} = vrlp:decode(vrlp:encode(DecodedData)),
|
||||||
|
DeEncodedData =:= DecodedData.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
%%% RANDOM CASE GENERATION %%%
|
||||||
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||||
|
|
||||||
|
%mkcase(DecodedData) ->
|
||||||
|
% Encoded = rlp:encode(DecodedData),
|
||||||
|
% {Deencoded, <<>>} = rlp:decode(
|
||||||
|
% %{decoded, DecodedData,
|
||||||
|
% % encoded,
|
||||||
|
|
||||||
|
% check
|
||||||
|
% - encode is correct
|
||||||
|
% - decode is the inverse of decode
|
||||||
|
|
||||||
|
% generate N-ish random decode datas
|
||||||
|
% no duplicates
|
||||||
|
rand_decode_datas(N) ->
|
||||||
|
% hack to remove duplicate cases
|
||||||
|
% this is good enough
|
||||||
|
sets:to_list(sets:from_list(rand_list(N))).
|
||||||
|
|
||||||
|
% generate a list of random datas n items long
|
||||||
|
rand_list(N) ->
|
||||||
|
[rand_data() || _ <- lists:seq(1, N)].
|
||||||
|
|
||||||
|
|
||||||
|
% generate a random bytestring or random list with 50% probability
|
||||||
|
rand_data() ->
|
||||||
|
MkList = rand:uniform() < 0.7,
|
||||||
|
case MkList of
|
||||||
|
true -> rand_list();
|
||||||
|
false -> rand_bytes()
|
||||||
|
end.
|
||||||
|
|
||||||
|
|
||||||
|
% generate a random list of random length between 0 and 10, inclusive
|
||||||
|
rand_list() ->
|
||||||
|
% rand:uniform(N) is between 1 and N
|
||||||
|
NItems = rand:uniform(3) - 1,
|
||||||
|
rand_list(NItems).
|
||||||
|
|
||||||
|
|
||||||
|
% generate a random bytestring between 0 and 100 bytes long
|
||||||
|
rand_bytes() ->
|
||||||
|
%works
|
||||||
|
case rand:uniform() < 0.5 of
|
||||||
|
% either generate between 0 and 5 items or between 5 and 100 items
|
||||||
|
true ->
|
||||||
|
NItems = rand:uniform(6) - 1,
|
||||||
|
crypto:strong_rand_bytes(NItems);
|
||||||
|
false ->
|
||||||
|
% range between 1, 96, add 4
|
||||||
|
NItems = (rand:uniform(96) + 4),
|
||||||
|
crypto:strong_rand_bytes(NItems)
|
||||||
|
end.
|
||||||
|
% syntax error:
|
||||||
|
% they look literally the same
|
||||||
|
%NItems = rand:uniform(11) - 1, crypto:strong_rand:bytes(NItems).
|
||||||
Reference in New Issue
Block a user