From 229ca2b6b847e61ee0faa6d8b42ed9dba70cb995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sun, 24 Feb 2019 22:14:22 +0100 Subject: [PATCH] Add property-based tests for codecs --- .travis.yml | 1 + rebar.config | 7 +- src/mtp_obfuscated.erl | 143 ++++++++++++++++++++++++--------- test/mtp_prop_gen.erl | 56 +++++++++++++ test/prop_mtp_abridged.erl | 43 ++++++++++ test/prop_mtp_aes_cbc.erl | 32 ++++++++ test/prop_mtp_full.erl | 44 ++++++++++ test/prop_mtp_intermediate.erl | 52 ++++++++++++ test/prop_mtp_obfuscated.erl | 104 ++++++++++++++++++++++++ 9 files changed, 441 insertions(+), 41 deletions(-) create mode 100644 test/mtp_prop_gen.erl create mode 100644 test/prop_mtp_abridged.erl create mode 100644 test/prop_mtp_aes_cbc.erl create mode 100644 test/prop_mtp_full.erl create mode 100644 test/prop_mtp_intermediate.erl create mode 100644 test/prop_mtp_obfuscated.erl diff --git a/.travis.yml b/.travis.yml index ac185d6..561b435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,4 @@ script: - ./rebar3 compile - ./rebar3 xref - ./rebar3 eunit + - ./rebar3 proper diff --git a/rebar.config b/rebar.config index 8973ac9..ca1fa8b 100644 --- a/rebar.config +++ b/rebar.config @@ -5,6 +5,7 @@ {deps, [{ranch, "1.7.0"}, {lager, "3.6.3"} ]}. +{plugins, [rebar3_proper]}. {xref_checks, [undefined_function_calls, @@ -34,5 +35,9 @@ {sys_config, "./config/prod-sys.config"}, {vm_args, "./config/prod-vm.args"}, {include_erts, true}]}] - }] + }, + {test, + [{deps, + [{proper, "1.3.0"}]} + ]}] }. diff --git a/src/mtp_obfuscated.erl b/src/mtp_obfuscated.erl index c0df7ed..825cdb9 100644 --- a/src/mtp_obfuscated.erl +++ b/src/mtp_obfuscated.erl @@ -7,9 +7,10 @@ -module(mtp_obfuscated). -behaviour(mtp_layer). --export([create/0, - create/1, +-export([client_create/3, + client_create/4, from_header/2, + from_header/3, new/4, encrypt/2, decrypt/2, @@ -27,51 +28,100 @@ -define(APP, mtproto_proxy). +-define(KEY_LEN, 32). +-define(IV_LEN, 16). + -opaque codec() :: #st{}. -%% @doc Creates new obfuscated stream (usual format) --spec create() -> {ok, Header :: binary(), codec()}. -create() -> - create(crypto:strong_rand_bytes(60)). --spec create(binary()) -> {ok, Header :: binary(), codec()}. -create(<>) -> - DownHeader = <>, - new2(DownHeader). +client_create(Secret, Protocol, DcId) -> + client_create(crypto:strong_rand_bytes(58), + Secret, Protocol, DcId). -new2(<> = DownHeader) -> - {EncKey, EncIV} = init_down_encrypt(DownHeader), - {DecKey, DecIV} = init_down_decrypt(DownHeader), - St = new(EncKey, EncIV, DecKey, DecIV), - {<<_:56/binary, Rep:8/binary, _/binary>>, St1} = encrypt(DownHeader, St), - {ok, - <>, - St1}. +-spec client_create(binary(), binary(), mtp_layer:codec(), integer()) -> + {Packet, + {EncKey, EncIv}, + {DecKey, DecIv}, + CliCodec} when + Packet :: binary(), + EncKey :: binary(), + EncIv :: binary(), + DecKey :: binary(), + DecIv :: binary(), + CliCodec :: codec(). +client_create(Seed, Secret, Protocol, DcId) when byte_size(Seed) == 58, + byte_size(Secret) == 16, + DcId > -10, + DcId < 10, + is_atom(Protocol) -> + <> = Seed, + ProtocolBin = encode_protocol(Protocol), + DcIdBin = encode_dc_id(DcId), + Raw = <>, -init_down_decrypt(<<_:8/binary, ToRev:48/binary, _/binary>>) -> - Reversed = bin_rev(ToRev), - <> = Reversed, - {KeyRev, RevIV}. + %% init_up_encrypt/2 + <<_:8/binary, ToRev:(?KEY_LEN + ?IV_LEN)/binary, _/binary>> = Raw, + <> = bin_rev(ToRev), + DecKey = crypto:hash('sha256', <>), -init_down_encrypt(<<_:8/binary, Key:32/binary, IV:16/binary, _/binary>>) -> - {Key, IV}. + %% init_up_decrypt/2 + <<_:8/binary, EncKeySeed:?KEY_LEN/binary, EncIv:?IV_LEN/binary, _/binary>> = Raw, + EncKey = crypto:hash('sha256', <>), + Codec = new(EncKey, EncIv, DecKey, DecIv), + {<<_:56/binary, Encrypted:8/binary>>, Codec1} = encrypt(Raw, Codec), + <> = Raw, + Packet = <>, + {Packet, + {EncKey, EncIv}, + {DecKey, DecIv}, + Codec1}. + + +%% 4byte +encode_protocol(mtp_abridged) -> + <<16#ef, 16#ef, 16#ef, 16#ef>>; +encode_protocol(mtp_intermediate) -> + <<16#ee, 16#ee, 16#ee, 16#ee>>; +encode_protocol(mtp_secure) -> + <<16#dd, 16#dd, 16#dd, 16#dd>>. + +%% 4byte +encode_dc_id(DcId) -> + <>. %% @doc creates new obfuscated stream (MTProto proxy format) +from_header(Header, Secret) -> + {ok, AllowedProtocols} = application:get_env(?APP, allowed_protocols), + from_header(Header, Secret, AllowedProtocols). + -spec from_header(binary(), binary()) -> {ok, integer(), mtp_layer:codec(), codec()} | {error, unknown_protocol | disabled_protocol}. -from_header(Header, Secret) when byte_size(Header) == 64 -> +from_header(Header, Secret, AllowedProtocols) when byte_size(Header) == 64 -> + %% 1) Encryption key + %% [--- _: 8b ----|---------- b: 48b -------------|-- _: 8b --] = header: 64b + %% b_r: 48b = reverse([---------- b ------------------]) + %% [-- key_seed: 32b --|- iv: 16b -] = b_r + %% key: 32b = sha256( [-- key_seed: 32b --|-- secret: 32b --] ) + %% iv: 16b = iv + %% + %% 2) Decryption key + %% [--- _: 8b ---|-- key_seed: 32b --|- iv: 16b -|-- _: 8b --] = header + %% key: 32b = sha256( [-- key_seed: 32b --|-- secret: 32b --] ) + %% ib: 16b = ib + %% + %% 3) Protocol and datacenter + %% decrypted_header: 64b = decrypt(header) + %% [-------------- _a: 56b ----|-------- b: 6b ---------|- _: 2b -] = decrypted_header + %% [- proto: 4b -|- dc: 2b -] {EncKey, EncIV} = init_up_encrypt(Header, Secret), {DecKey, DecIV} = init_up_decrypt(Header, Secret), St = new(EncKey, EncIV, DecKey, DecIV), - {<<_:56/binary, Bin1:8/binary, _/binary>>, St1} = decrypt(Header, St), + {<<_:56/binary, Bin1:6/binary, _:2/binary>>, St1} = decrypt(Header, St), case get_protocol(Bin1) of {error, unknown_protocol} = Err -> Err; Protocol -> - {ok, AllowedProtocols} = application:get_env(?APP, allowed_protocols), case lists:member(Protocol, AllowedProtocols) of true -> DcId = get_dc(Bin1), @@ -82,28 +132,28 @@ from_header(Header, Secret) when byte_size(Header) == 64 -> end. init_up_encrypt(Bin, Secret) -> - <<_:8/binary, ToRev:48/binary, _/binary>> = Bin, + <<_:8/binary, ToRev:(?KEY_LEN + ?IV_LEN)/binary, _/binary>> = Bin, Rev = bin_rev(ToRev), - <> = Rev, + <> = Rev, %% <<_:32/binary, RevIV:16/binary, _/binary>> = Bin, - KeyRevHash = crypto:hash('sha256', <>), - {KeyRevHash, RevIV}. + Key = crypto:hash('sha256', <>), + {Key, IV}. init_up_decrypt(Bin, Secret) -> - <<_:8/binary, Key:32/binary, IV:16/binary, _/binary>> = Bin, - KeyHash = crypto:hash('sha256', <>), - {KeyHash, IV}. + <<_:8/binary, KeySeed:?KEY_LEN/binary, IV:?IV_LEN/binary, _/binary>> = Bin, + Key = crypto:hash('sha256', <>), + {Key, IV}. -get_protocol(<<16#ef, 16#ef, 16#ef, 16#ef, _/binary>>) -> +get_protocol(<<16#ef, 16#ef, 16#ef, 16#ef, _:2/binary>>) -> mtp_abridged; -get_protocol(<<16#ee, 16#ee, 16#ee, 16#ee, _/binary>>) -> +get_protocol(<<16#ee, 16#ee, 16#ee, 16#ee, _:2/binary>>) -> mtp_intermediate; -get_protocol(<<16#dd, 16#dd, 16#dd, 16#dd, _/binary>>) -> +get_protocol(<<16#dd, 16#dd, 16#dd, 16#dd, _:2/binary>>) -> mtp_secure; get_protocol(_) -> {error, unknown_protocol}. -get_dc(<<_:4/binary, DcId:16/signed-little-integer, _/binary>>) -> +get_dc(<<_:4/binary, DcId:16/signed-little-integer>>) -> DcId. @@ -137,3 +187,16 @@ encode_packet(Msg, S) -> bin_rev(Bin) -> %% binary:encode_unsigned(binary:decode_unsigned(Bin, little)). list_to_binary(lists:reverse(binary_to_list(Bin))). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +client_server_test() -> + Secret = crypto:strong_rand_bytes(16), + DcId = 4, + Protocol = mtp_secure, + {Packet, _, _, _CliCodec} = client_create(Secret, Protocol, DcId), + Srv = from_header(Packet, Secret, [Protocol]), + ?assertMatch({ok, DcId, Protocol, _}, Srv). + +-endif. diff --git a/test/mtp_prop_gen.erl b/test/mtp_prop_gen.erl new file mode 100644 index 0000000..750f02b --- /dev/null +++ b/test/mtp_prop_gen.erl @@ -0,0 +1,56 @@ +%% Common data generators for property-based tests + +-module(mtp_prop_gen). +-include_lib("proper/include/proper.hrl"). + +-export([stream_4b/0, + packet_4b/0, + stream_16b/0, + packet_16b/0, + key/0, + iv/0, + secret/0, + dc_id/0, + codec/0 + ]). + + +%% 4-byte aligned packet: `binary()` +packet_4b() -> + ?LET(IoList, proper_types:non_empty(proper_types:list(proper_types:binary(4))), + iolist_to_binary(IoList)). + +%% List of 4-byte aligned packets: `[binary()]` +stream_4b() -> + proper_types:list(packet_4b()). + +%% 16-byte aligned packet: `binary()` +packet_16b() -> + ?LET(IoList, proper_types:non_empty(proper_types:list(proper_types:binary(16))), + iolist_to_binary(IoList)). + +%% List of 16-byte aligned packets: `[binary()]` +stream_16b() -> + proper_types:list(packet_16b()). + +%% 32-byte encryption key: `binary()` +key() -> + proper_types:binary(32). + +%% 16-byte encryption initialization vector: `binary()` +iv() -> + proper_types:binary(16). + +%% 16-byte secret: `binary()` +secret() -> + proper_types:binary(16). + +%% Datacenter ID: `[-9..9]` +dc_id() -> + proper_types:integer(-9, 9). + +codec() -> + Protocols = [mtp_abridged, mtp_intermediate, mtp_secure], + proper_types:oneof( + [proper_types:exactly(P) + || P <- Protocols]). diff --git a/test/prop_mtp_abridged.erl b/test/prop_mtp_abridged.erl new file mode 100644 index 0000000..b5b1973 --- /dev/null +++ b/test/prop_mtp_abridged.erl @@ -0,0 +1,43 @@ +-module(prop_mtp_abridged). +-include_lib("proper/include/proper.hrl"). + +-export([prop_codec/1, prop_stream/1]). + +prop_codec(doc) -> + "Tests that any 4-byte aligned binary can be encoded and decoded back". + +prop_codec() -> + ?FORALL(Bin, mtp_prop_gen:packet_4b(), codec(Bin)). + +codec(Bin) -> + Codec = mtp_abridged:new(), + {Data, Codec1} = mtp_abridged:encode_packet(Bin, Codec), + {ok, Decoded, _} = mtp_abridged:try_decode_packet(iolist_to_binary(Data), Codec1), + Decoded == Bin. + + +prop_stream(doc) -> + "Tests that any number of packets can be encoded, concatenated and decoded". + +prop_stream() -> + ?FORALL(Stream, mtp_prop_gen:stream_4b(), stream_codec(Stream)). + +stream_codec(Stream) -> + Codec = mtp_abridged:new(), + {BinStream, Codec1} = + lists:foldl( + fun(Bin, {Acc, Codec1}) -> + {Data, Codec2} = mtp_abridged:encode_packet(Bin, Codec1), + {<>, + Codec2} + end, {<<>>, Codec}, Stream), + DecodedStream = decode_stream(BinStream, Codec1, []), + Stream == DecodedStream. + +decode_stream(BinStream, Codec, Acc) -> + case mtp_abridged:try_decode_packet(BinStream, Codec) of + {incomplete, _} -> + lists:reverse(Acc); + {ok, DecPacket, Codec1} -> + decode_stream(<<>>, Codec1, [DecPacket | Acc]) + end. diff --git a/test/prop_mtp_aes_cbc.erl b/test/prop_mtp_aes_cbc.erl new file mode 100644 index 0000000..f53f242 --- /dev/null +++ b/test/prop_mtp_aes_cbc.erl @@ -0,0 +1,32 @@ +-module(prop_mtp_aes_cbc). +-include_lib("proper/include/proper.hrl"). + +-export([prop_stream/1]). + +prop_stream(doc) -> + "Tests that any number of packets can be encoded, concatenated and decoded" + " as a stream using the same key for encoding and decoding". + +prop_stream() -> + ?FORALL({Key, Iv, Stream}, arg_set(), stream_codec(Key, Iv, Stream)). + + +arg_set() -> + proper_types:tuple( + [mtp_prop_gen:key(), + mtp_prop_gen:iv(), + mtp_prop_gen:stream_16b() + ]). + +stream_codec(Key, Iv, Stream) -> + Codec = mtp_aes_cbc:new(Key, Iv, Key, Iv, 16), + {BinStream, Codec2} = + lists:foldl( + fun(Bin, {Acc, Codec1}) -> + {Data, Codec2} = mtp_aes_cbc:encrypt(Bin, Codec1), + {<>, + Codec2} + end, {<<>>, Codec}, Stream), + {Decrypted, _Codec3} = mtp_aes_cbc:decrypt(BinStream, Codec2), + %% io:format("Dec: ~p~nOrig: ~p~nCodec: ~p~n", [Decrypted, Stream, _Codec3]), + Decrypted == iolist_to_binary(Stream). diff --git a/test/prop_mtp_full.erl b/test/prop_mtp_full.erl new file mode 100644 index 0000000..77887f1 --- /dev/null +++ b/test/prop_mtp_full.erl @@ -0,0 +1,44 @@ +-module(prop_mtp_full). +-include_lib("proper/include/proper.hrl"). + +-export([prop_codec/1, prop_stream/1]). + + +prop_codec(doc) -> + "Tests that any 4-byte aligned binary can be encoded and decoded back". + +prop_codec() -> + ?FORALL(Bin, mtp_prop_gen:packet_4b(), codec(Bin)). + +codec(Bin) -> + Codec = mtp_full:new(), + {Data, Codec1} = mtp_full:encode_packet(Bin, Codec), + {ok, Decoded, _} = mtp_full:try_decode_packet(iolist_to_binary(Data), Codec1), + Decoded == Bin. + + +prop_stream(doc) -> + "Tests that any number of packets can be encoded, concatenated and decoded". + +prop_stream() -> + ?FORALL(Stream, mtp_prop_gen:stream_4b(), stream_codec(Stream)). + +stream_codec(Stream) -> + Codec = mtp_full:new(), + {BinStream, Codec1} = + lists:foldl( + fun(Bin, {Acc, Codec1}) -> + {Data, Codec2} = mtp_full:encode_packet(Bin, Codec1), + {<>, + Codec2} + end, {<<>>, Codec}, Stream), + DecodedStream = decode_stream(BinStream, Codec1, []), + Stream == DecodedStream. + +decode_stream(BinStream, Codec, Acc) -> + case mtp_full:try_decode_packet(BinStream, Codec) of + {incomplete, _} -> + lists:reverse(Acc); + {ok, DecPacket, Codec1} -> + decode_stream(<<>>, Codec1, [DecPacket | Acc]) + end. diff --git a/test/prop_mtp_intermediate.erl b/test/prop_mtp_intermediate.erl new file mode 100644 index 0000000..82d1ee5 --- /dev/null +++ b/test/prop_mtp_intermediate.erl @@ -0,0 +1,52 @@ +-module(prop_mtp_intermediate). +-include_lib("proper/include/proper.hrl"). + +-export([prop_codec/1, prop_stream/1, prop_stream_padding/1]). + + +prop_codec(doc) -> + "Tests that any 4-byte aligned binary can be encoded and decoded back". + +prop_codec() -> + ?FORALL(Bin, mtp_prop_gen:packet_4b(), codec(Bin)). + +codec(Bin) -> + Codec = mtp_intermediate:new(), + {Data, Codec1} = mtp_intermediate:encode_packet(Bin, Codec), + {ok, Decoded, _} = mtp_intermediate:try_decode_packet(iolist_to_binary(Data), Codec1), + Decoded == Bin. + + +prop_stream(doc) -> + "Tests that any number of packets can be encoded, concatenated and decoded". + +prop_stream() -> + ?FORALL(Stream, mtp_prop_gen:stream_4b(), stream_codec(Stream, false)). + +stream_codec(Stream, Padding) -> + Codec = mtp_intermediate:new(#{padding => Padding}), + {BinStream, Codec1} = + lists:foldl( + fun(Bin, {Acc, Codec1}) -> + {Data, Codec2} = mtp_intermediate:encode_packet(Bin, Codec1), + {<>, + Codec2} + end, {<<>>, Codec}, Stream), + DecodedStream = decode_stream(BinStream, Codec1, []), + Stream == DecodedStream. + +decode_stream(BinStream, Codec, Acc) -> + case mtp_intermediate:try_decode_packet(BinStream, Codec) of + {incomplete, _} -> + lists:reverse(Acc); + {ok, DecPacket, Codec1} -> + decode_stream(<<>>, Codec1, [DecPacket | Acc]) + end. + + +prop_stream_padding(doc) -> + "Tests that any number of packets can be encoded, concatenated and decoded" + " using encoder with random padding enabled". + +prop_stream_padding() -> + ?FORALL(Stream, mtp_prop_gen:stream_4b(), stream_codec(Stream, true)). diff --git a/test/prop_mtp_obfuscated.erl b/test/prop_mtp_obfuscated.erl new file mode 100644 index 0000000..4ccba44 --- /dev/null +++ b/test/prop_mtp_obfuscated.erl @@ -0,0 +1,104 @@ +-module(prop_mtp_obfuscated). +-include_lib("proper/include/proper.hrl"). + +-export([prop_stream/1, + prop_client_server_handshake/1, + prop_client_server_stream/1]). + +prop_stream(doc) -> + "Tests that any number of packets can be encrypted with mtp_obfuscatedcoded," + " concatenated and decoded as a stream using the same key for encoding and decoding". + +prop_stream() -> + ?FORALL({Key, Iv, Stream}, stream_arg_set(), stream_codec(Key, Iv, Stream)). + + +stream_arg_set() -> + proper_types:tuple( + [mtp_prop_gen:key(), + mtp_prop_gen:iv(), + mtp_prop_gen:stream_4b() + ]). + +stream_codec(Key, Iv, Stream) -> + Codec = mtp_obfuscated:new(Key, Iv, Key, Iv), + {BinStream, Codec2} = + lists:foldl( + fun(Bin, {Acc, Codec1}) -> + {Data, Codec2} = mtp_obfuscated:encrypt(Bin, Codec1), + {<>, + Codec2} + end, {<<>>, Codec}, Stream), + {Decrypted, _Codec3} = mtp_obfuscated:decrypt(BinStream, Codec2), + %% io:format("Dec: ~p~nOrig: ~p~nCodec: ~p~n", [Decrypted, Stream, _Codec3]), + Decrypted == iolist_to_binary(Stream). + + +prop_client_server_handshake(doc) -> + "Tests that for any secret, protocol and dc_id, it's possible to perform handshake". + +prop_client_server_handshake() -> + ?FORALL({Secret, DcId, Protocol}, cs_hs_arg_set(), + cs_hs_exchange(Secret, DcId, Protocol)). + +cs_hs_arg_set() -> + proper_types:tuple( + [mtp_prop_gen:secret(), + mtp_prop_gen:dc_id(), + mtp_prop_gen:codec()]). + +cs_hs_exchange(Secret, DcId, Protocol) -> + %% io:format("Secret: ~p; DcId: ~p, Protocol: ~p~n", + %% [Secret, DcId, Protocol]), + {Packet, _, _, _CliCodec} = mtp_obfuscated:client_create(Secret, Protocol, DcId), + case mtp_obfuscated:from_header(Packet, Secret, [Protocol]) of + {ok, DcId, Protocol, _SrvCodec} -> + true; + _ -> + false + end. + +prop_client_server_stream(doc) -> + "Tests that for any secret, protocol and dc_id, it's possible to perform" + " handshake/key exchange and then do bi-directional encode/decode stream of data". + +prop_client_server_stream() -> + ?FORALL({Secret, DcId, Protocol, Stream}, cs_stream_arg_set(), + cs_stream_exchange(Secret, DcId, Protocol, Stream)). + +cs_stream_arg_set() -> + proper_types:tuple( + [mtp_prop_gen:secret(), + mtp_prop_gen:dc_id(), + mtp_prop_gen:codec(), + mtp_prop_gen:stream_4b()]). + +cs_stream_exchange(Secret, DcId, Protocol, Stream) -> + %% io:format("Secret: ~p; DcId: ~p, Protocol: ~p~n", + %% [Secret, DcId, Protocol]), + {Header, _, _, CliCodec} = mtp_obfuscated:client_create(Secret, Protocol, DcId), + {ok, DcId, Protocol, SrvCodec} = mtp_obfuscated:from_header(Header, Secret, [Protocol]), + + %% Client to server + {CliCodec1, + SrvCodec1, + Cli2SrvTransmitted} = transmit_stream(CliCodec, SrvCodec, Stream), + {_CliCodec2, + _SrvCodec2, + Srv2CliTransmitted} = transmit_stream(SrvCodec1, CliCodec1, Stream), + BinStream = iolist_to_binary(Stream), + (Cli2SrvTransmitted == BinStream) + andalso (Srv2CliTransmitted == BinStream). + +transmit_stream(EncCodec, DecCodec, Stream) -> + {EncStream, EncCodec3} = + lists:foldl( + fun(Packet, {Acc, CliCodec1}) -> + {Data, CliCodec2} = mtp_obfuscated:encrypt(Packet, CliCodec1), + {<>, + CliCodec2} + end, {<<>>, EncCodec}, Stream), + {Decrypted, DecCodec2} = mtp_obfuscated:decrypt(EncStream, DecCodec), + {EncCodec3, + DecCodec2, + Decrypted}.