diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index 84cbe05..d8d046f 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -15,10 +15,16 @@ jobs: runs-on: ubuntu-latest + strategy: + matrix: + otp-version: [26, 27, 28] + container: - image: erlang:26 + image: erlang:${{ matrix.otp-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run tests run: make eunit && make cover + - name: Run Dialyzer + run: make dialyzer diff --git a/changelog.md b/changelog.md index 4cc736d..5a7f0c6 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,9 @@ +* 2.11.0 + - Deleted jsone as dependency. + When OTP release is 27 or later, the default JSON provider module is `json`, otherwise `jsone`. + For OTP release 26 or earlier version, you must add `jsone-1.8.1` or newer in your project dependency. + For OTP release 27 or later, you can choose to continue using jsone by calling `avro:set_json_provider(jsone)`. + * 2.10.3 - Allow union type to have zero member types. * 2.10.2 diff --git a/elvis.config b/elvis.config index e7c7063..91c2848 100644 --- a/elvis.config +++ b/elvis.config @@ -7,12 +7,8 @@ [ #{dirs => ["src"], filter => "*.erl", - rules => [ {elvis_style, line_length, - #{ limit => 80, - skip_comments => false - }} - , {elvis_style, no_tabs} - , {elvis_style, no_trailing_whitespace} + rules => [ {elvis_text_style, line_length, #{ limit => 80 }} + , {elvis_text_style, no_trailing_whitespace} , {elvis_style, operator_spaces, #{ rules => [ {right,","} , {right,"+"} @@ -54,16 +50,15 @@ , {elvis_style, dont_repeat_yourself, #{ min_complexity => 15 }} - , {elvis_style, no_debug_call} + , {elvis_style, no_debug_call, + #{ignore => [avro_decoder_hooks]}} ] }, #{dirs => ["test"], filter => "*.erl", - rules => [ {elvis_style, line_length, - #{ limit => 80, - skip_comments => false - }} + rules => [ {elvis_text_style, line_length, #{ limit => 80 }} + , {elvis_text_style, no_trailing_whitespace} ] } ] diff --git a/include/avro_internal.hrl b/include/avro_internal.hrl index 1fb0b3c..5c7d914 100644 --- a/include/avro_internal.hrl +++ b/include/avro_internal.hrl @@ -64,7 +64,7 @@ -type type_or_name() :: avro_type() | name_raw(). -type custom_prop_name() :: binary(). --type custom_prop_value() :: jsone:json_value(). +-type custom_prop_value() :: avro_json_compat:json_value(). -type custom_prop() :: {custom_prop_name(), custom_prop_value()}. -define(ASSIGNED_NAME, <<"_erlavro_assigned">>). @@ -96,7 +96,7 @@ -type type_prop_value() :: namespace() | typedoc() | [name()] | custom_prop_value(). -type type_props() :: [{type_prop_name(), type_prop_value()}]. --type avro_json() :: jsone:json_object(). +-type avro_json() :: avro_json_compat:json_value(). -type avro_binary() :: iolist(). -type avro_codec() :: null | deflate | snappy. diff --git a/rebar.config b/rebar.config index 4dc682f..dfe628a 100644 --- a/rebar.config +++ b/rebar.config @@ -9,10 +9,15 @@ ]}. {edoc_opts, [{preprocess, true}]}. {deps, - [ {jsone, "1.8.1"}, + [ {snappyer, "1.2.9"} ]}. +{profiles, + [{test, + [{deps, [{jsone, "1.8.1"}]}] + }]}. + {cover_opts, [verbose]}. {cover_enabled, true}. {cover_export_enabled, true}. diff --git a/rebar.config.script b/rebar.config.script deleted file mode 100644 index 3c88c0c..0000000 --- a/rebar.config.script +++ /dev/null @@ -1,10 +0,0 @@ -case os:getenv("TRAVIS") of - "true" -> - JobId = os:getenv("TRAVIS_JOB_ID"), - [{coveralls_service_job_id, JobId}, - {plugins, [coveralls]}, - {coveralls_coverdata, "_build/test/cover/eunit.coverdata"}, - {coveralls_service_name , "travis-ci"} | CONFIG]; - _ -> - CONFIG -end. diff --git a/src/avro.erl b/src/avro.erl index 3ae15a5..8106b23 100644 --- a/src/avro.erl +++ b/src/avro.erl @@ -68,6 +68,8 @@ , canonical_form_fingerprint/1 ]). +-export([set_json_provider/1, get_json_provider/0]). + -export_type([ array_type/0 , avro_type/0 , avro_value/0 @@ -143,6 +145,14 @@ -type schema_all() :: avro_type() | binary() | lkup_fun() | schema_store(). -type crc64_fingerprint() :: avro_fingerprint:crc64(). +%% @doc Set JSON library module. +-spec set_json_provider(json | jsone) -> ok. +set_json_provider(Module) -> avro_json_compat:set_provider(Module). + +%% @doc Set JSON library module. +-spec get_json_provider() -> json | jsone. +get_json_provider() -> avro_json_compat:get_provider(). + %% @doc Decode JSON format avro schema into `erlavro' internals. -spec decode_schema(binary()) -> avro_type(). decode_schema(JSON) -> avro_json_decoder:decode_schema(JSON). @@ -541,7 +551,8 @@ is_same_type(T1, T2) -> {Aliases1, Aliases2} -> %% Check if [Fullname1 | Aliases1] and [Fullname2 | Aliases2] have a %% non-empty intersection. These lists are always very short. - is_nonempty_intersection([Fullname1 | Aliases1], [Fullname2 | Aliases2]) + is_nonempty_intersection([Fullname1 | Aliases1], + [Fullname2 | Aliases2]) end end. diff --git a/src/avro_json_compat.erl b/src/avro_json_compat.erl new file mode 100644 index 0000000..6adcf6f --- /dev/null +++ b/src/avro_json_compat.erl @@ -0,0 +1,224 @@ +%%%----------------------------------------------------------------------------- +%%% Copyright (c) 2025 Klarna AB +%%% +%%% This file is provided to you under the Apache License, +%%% Version 2.0 (the "License"); you may not use this file +%%% except in compliance with the License. You may obtain +%%% a copy of the License at +%%% +%%% http://www.apache.org/licenses/LICENSE-2.0 +%%% +%%% Unless required by applicable law or agreed to in writing, +%%% software distributed under the License is distributed on an +%%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%%% KIND, either express or implied. See the License for the +%%% specific language governing permissions and limitations +%%% under the License. +%%% +%%% @doc Compatibility layer for JSON encoding/decoding. +%%% Uses Erlang's native json module on OTP 27+ +%%% falls back to jsone on older versions. +%%% @end +%%%----------------------------------------------------------------------------- + +-module(avro_json_compat). + +%% Suppress Dialyzer warnings about json module functions on OTP < 27 +-if(?OTP_RELEASE < 27). +-dialyzer({nowarn_function, [encode/2, decode/2, encoder/2]}). +-endif. + +-export([set_provider/1, get_provider/0]). +-export([encode/1, encode/2, decode/1, decode/2]). +-export([inline/1]). +-export_type([json_value/0]). + +%% Type definitions compatible with both jsone and native json +-type json_obj() :: #{binary() => json_value()} | + [{binary(), json_value()}] | + {[{binary(), json_value()}]}. + +%% NB: atom() only for encoder inputs. +%% No atom created when decode. +-type string_value() :: atom() | binary(). + +-type json_value() :: null + | boolean() + | integer() + | float() + | string_value() + | [json_value()] + | json_obj(). + +-type encode_option() :: native_utf8. +-type decode_option() :: {object_format, tuple | map | proplist}. + +%% Suppress warnings about json module functions on OTP < 27 +-if(?OTP_RELEASE < 27). +-compile(nowarn_undefined_function). +-endif. + +-define(PROVIDER_PTP_KEY, erlavro_json_provider). + +%%%_* APIs ===================================================================== + +%% @doc Set `json' or `jsone' as the JSON encode/decode provider. +-spec set_provider(json | jsone) -> ok. +set_provider(Module) -> + persistent_term:put(?PROVIDER_PTP_KEY, Module). + +%% @doc Get current JSON encode/decode provider. +-spec get_provider() -> json | jsone. +get_provider() -> + case persistent_term:get(?PROVIDER_PTP_KEY, undefined) of + undefined -> + Module = get_available_provider([json, jsone]), + persistent_term:put(?PROVIDER_PTP_KEY, Module), + Module; + Module -> + Module + end. + +%% @doc Make a inline (already encoded JSON) JSON value. +%% This is a compatible format for jsone. +inline(JSON) -> {{json, JSON}}. + +%% @doc Encode Erlang term to JSON. +%% Equivalent to jsone:encode/1 +%% Returns iodata() (binary() or iolist()) +-spec encode(json_value()) -> iodata(). +encode(Value) -> encode(Value, []). + +%% @doc Encode Erlang term to JSON with options. +-spec encode(json_value(), [encode_option()]) -> iodata(). +encode(Value, Options) -> + case get_provider() of + json -> + iolist_to_binary(json:encode(Value, fun encoder/2)); + Module -> + apply(Module, encode, [Value, Options]) + end. + +%% Custom encoder callback for json:encode/2 +%% Handles conversion from jsone format (lists of tuples) to +%% native json format inline +encoder(Value, Encode) when is_list(Value) -> + %% Check if it's a list of key-value pairs (object) or a plain list (array) + case is_key_value_list(Value) of + true -> + %% It's an object: [{Key, Value}] -> encode as key-value list + %% Convert atom keys to binary keys for native json + ConvertedList = [{convert_key(K), V} || {K, V} <- Value], + json:encode_key_value_list(ConvertedList, Encode); + false -> + %% It's an array: [Value] -> encode as array + json:encode_value(Value, Encode) + end; +encoder({{json, JSON}}, _Encode) -> + iolist_to_binary(JSON); +encoder({[]}, _Encode) -> + <<"{}">>; +encoder({[{_, _} | _] = KvList}, Encode) -> + encoder(KvList, Encode); +encoder(Value, Encode) -> + %% Primitive value or map - use default encoding + json:encode_value(Value, Encode). + +%% @doc Decode JSON binary to Erlang term. +%% Default object format is tuple (compatible with jsone default). +-spec decode(binary()) -> json_value(). +decode(JSON) -> + decode(JSON, [{object_format, tuple}]). + +%% @doc Decode JSON binary to Erlang term with options. +%% Options: +%% {object_format, tuple} - Return objects as {[{Key, Value}]} tuples +%% (default, jsone format) +%% {object_format, map} - Return objects as #{Key => Value} maps +%% {object_format, proplist} - Return objects as [{Key, Value}] proplist +-spec decode(binary(), [decode_option()]) -> json_value(). +decode(JSON, Options) -> + case get_provider() of + json -> + JsonBinary = iolist_to_binary(JSON), + ObjectFormat = proplists:get_value(object_format, Options, tuple), + Decoded = json:decode(JsonBinary), + convert_object_format(Decoded, ObjectFormat); + Module -> + apply(Module, decode, [JSON, Options]) + end. + +%%%_* Internal functions ======================================================= + +%% @private Check if a list represents a key-value object (proplist) +%% Accepts both binary keys and atom keys +-spec is_key_value_list([term()]) -> boolean(). +is_key_value_list([{K, _V} | _]) when is_binary(K) orelse is_atom(K) -> + true; +is_key_value_list(_) -> + false. + +%% @private Convert key to binary format (native json requires binary keys) +-spec convert_key(atom() | binary()) -> binary(). +convert_key(Key) when is_atom(Key) -> + atom_to_binary(Key, utf8); +convert_key(Key) when is_binary(Key) -> + Key. + +%% @private Convert decoded JSON value's object format +%% jsone with {object_format, tuple} returns objects as {Fields} +%% where Fields is [{Key, Value}] +%% native json returns objects as #{Key => Value} maps +-spec convert_object_format(json_value(), tuple | map | proplist) -> + json_value(). +convert_object_format(Value, Format) when is_map(Value) -> + case Format of + tuple -> + %% Convert map to jsone format: {[{Key, Value}]} + %% Recursively convert nested maps in values + %% Note: maps don't preserve JSON key order, so the resulting tuple order + %% may differ from the original JSON order + Fields = [{Key, convert_object_format(V, Format)} + || {Key, V} <- maps:to_list(Value)], + {Fields}; + map -> + Value; + proplist -> + %% Convert to proplist, recursively convert nested maps + [{Key, convert_object_format(V, Format)} + || {Key, V} <- maps:to_list(Value)] + end; +convert_object_format({Fields}, Format) when is_list(Fields) -> + %% Already in jsone tuple format, but need to recurse into + %% nested objects/arrays + {[{Key, convert_object_format(Value, Format)} + || {Key, Value} <- Fields]}; +convert_object_format(List, Format) when is_list(List) -> + %% Check if it's a list of key-value pairs (proplist) or a JSON array + case is_key_value_list(List) of + true -> + %% It's a proplist, recurse into values + [{Key, convert_object_format(Value, Format)} || {Key, Value} <- List]; + false -> + %% It's a JSON array, recurse into elements + [convert_object_format(Item, Format) || Item <- List] + end; +convert_object_format(Value, _Format) -> + %% Primitive value (null, boolean, number, binary) + Value. + +get_available_provider([]) -> + error(erlavro_no_json_provider); +get_available_provider([Module | Rest]) -> + try + apply(Module, module_info, [module]) + catch + _:_ -> + get_available_provider(Rest) + end. + +%%%_* Emacs ==================================================================== +%%% Local Variables: +%%% allout-layout: t +%%% erlang-indent-level: 2 +%%% End: diff --git a/src/avro_json_decoder.erl b/src/avro_json_decoder.erl index d233990..37fbc41 100644 --- a/src/avro_json_decoder.erl +++ b/src/avro_json_decoder.erl @@ -38,7 +38,7 @@ %% Exported for test -export([ parse_schema/1 ]). --type json_value() :: jsone:json_value(). +-type json_value() :: avro_json_compat:json_value(). -type sc_opts() :: avro:schema_opts(). -type default_parse_fun() :: fun((type_or_name(), json_value()) -> avro:out()). @@ -514,7 +514,7 @@ do_parse_union_ex(ValueTypeName, Value, UnionType, %% 'proplist' is not an option because otherwise there is no way to tell %% apart 'object' and 'array'. -spec decode_json(binary()) -> json_value(). -decode_json(JSON) -> jsone:decode(JSON, [{object_format, tuple}]). +decode_json(JSON) -> avro_json_compat:decode(JSON, [{object_format, tuple}]). %% Filter out non-custom properties. -spec filter_custom_props([{binary(), json_value()}], [name()]) -> diff --git a/src/avro_json_encoder.erl b/src/avro_json_encoder.erl index 4cd668c..2a7fc13 100644 --- a/src/avro_json_encoder.erl +++ b/src/avro_json_encoder.erl @@ -37,8 +37,9 @@ -include("avro_internal.hrl"). --type json_value() :: jsone:json_value(). --define(INLINE(JSON), {{json, JSON}}). +-type json_value() :: avro_json_compat:json_value(). + +-define(INLINE(JSON), avro_json_compat:inline(JSON)). %%%_* APIs ===================================================================== @@ -83,7 +84,7 @@ encode(Sc, TypeOrName, Value) -> %% @private -spec encode_json(json_value()) -> iodata(). -encode_json(Input) -> jsone:encode(Input, [native_utf8]). +encode_json(Input) -> avro_json_compat:encode(Input, [native_utf8]). %% @private -spec do_encode(lkup_fun(), type_or_name(), avro_value() | avro:in()) -> @@ -383,13 +384,13 @@ encode_field_with_value({FieldName, Value}) -> %% encode per avro spec. %% @end encode_binary(Bin) -> - [$", encode_binary_body(Bin), $"]. + erlang:iolist_to_binary([$", encode_binary_body(Bin), $"]). %% @private encode_binary_body(<<>>) -> ""; encode_binary_body(<>) -> - [$\\, $u, $0, $0, to_hex(H1), to_hex(H2) |encode_binary_body(Rest)]. + [$\\, $u, $0, $0, to_hex(H1), to_hex(H2) | encode_binary_body(Rest)]. %% @private to_hex(D) when D >= 0 andalso D =< 9 -> diff --git a/src/avro_schema_store.erl b/src/avro_schema_store.erl index cc24639..b5f2e34 100644 --- a/src/avro_schema_store.erl +++ b/src/avro_schema_store.erl @@ -301,7 +301,8 @@ get_type_from_store(NameRef, Store) -> end. %% @private --spec do_get_type_from_store(fullname(), store()) -> false | {ok, fullname() | avro_type()}. +-spec do_get_type_from_store(fullname(), store()) -> + false | {ok, fullname() | avro_type()}. do_get_type_from_store(Name, {dict, Dict}) -> case dict:find(Name, Dict) of error -> false; diff --git a/src/avro_util.erl b/src/avro_util.erl index 2d0977d..1efe2e4 100644 --- a/src/avro_util.erl +++ b/src/avro_util.erl @@ -158,8 +158,8 @@ delete_opts(KvList, Keys) -> canonicalize_custom_props(Props0) -> %% Filter out all type_prop_name() keys first Props = delete_opts(Props0, [namespace, doc, aliases]), - JSON = jsone:encode(make_jsone_input(Props)), - jsone:decode(JSON, [{object_format, proplist}]). + JSON = avro_json_compat:encode(make_jsone_input(Props)), + avro_json_compat:decode(JSON, [{object_format, proplist}]). %% @doc Assert validity of a list of names. -spec verify_names([name_raw()]) -> ok | no_return(). @@ -319,7 +319,8 @@ do_is_compatible(Reader, Writer, RPath, WPath) erlang:throw({ reader_missing_defalut_value , [{field, FieldName} | WPath] }); - [{_, WriterType}] -> % let's not allow several fields to have same alias + [{_, WriterType}] -> + % let's not allow several fields to have same alias FieldDesc = {field, FieldName}, do_is_compatible_next(FieldType, WriterType, [FieldDesc | RPath], @@ -544,7 +545,7 @@ is_valid_name([H | T]) -> is_valid_dotted_name(DottedName) when ?IS_NAME_RAW(DottedName) -> NameStr = binary_to_list(?NAME(DottedName)), Names = tokens_ex(NameStr, $.), - Names =/= [] andalso lists:all(fun is_valid_name/1, Names). + lists:all(fun is_valid_name/1, Names). %% @private -spec reserved_type_names() -> [name()]. diff --git a/src/erlavro.app.src b/src/erlavro.app.src index d1363d9..86266ea 100644 --- a/src/erlavro.app.src +++ b/src/erlavro.app.src @@ -6,7 +6,6 @@ {applications, [ kernel, stdlib, - jsone, snappyer ]}, {env, []}, diff --git a/test/avro_json_decoder_tests.erl b/test/avro_json_decoder_tests.erl index eb09bfd..9d9401b 100644 --- a/test/avro_json_decoder_tests.erl +++ b/test/avro_json_decoder_tests.erl @@ -75,7 +75,7 @@ allow_null_as_string_for_default_test() -> ]} ]), - JSON = iolist_to_binary(jsone:encode(Schema, [native_utf8])), + JSON = iolist_to_binary(avro_json_compat:encode(Schema, [native_utf8])), ExpectedUnion = avro_union:type([ avro_primitive:null_type() , avro_primitive:int_type() ]), @@ -116,7 +116,7 @@ decode_with_default_values_test() -> ]} ]), - JSON = iolist_to_binary(jsone:encode(Schema, [native_utf8])), + JSON = iolist_to_binary(avro_json_compat:encode(Schema, [native_utf8])), ExpectedUnion = avro_union:type([ avro_primitive:boolean_type() , avro_primitive:int_type() ]), @@ -236,7 +236,8 @@ parse_fixed_type_test() -> parse_bytes_value_test() -> RawJson = <<"{\"a\":\"\\u0010\\u0000\\u00FF\"}">>, - #{<<"a">> := Bytes} = jsone:decode(RawJson), + #{<<"a">> := Bytes} = + avro_json_compat:decode(RawJson, [{object_format, map}]), ?assertEqual([16,0,255], unicode:characters_to_list(Bytes, utf8)), Value = parse_value(Bytes, avro_primitive:bytes_type(), none), ?assertEqual(avro_primitive:bytes(<<16,0,255>>), Value). @@ -352,7 +353,8 @@ parse_map_value_test() -> parse_fixed_value_test() -> Type = avro_fixed:type("FooBar", 2), RawJson = <<"{\"a\":\"\\u0001\\u007f\"}">>, - #{<<"a">> := Bytes} = jsone:decode(RawJson), + #{<<"a">> := Bytes} = + avro_json_compat:decode(RawJson, [{object_format, map}]), ?assertEqual([1,127], unicode:characters_to_list(Bytes, utf8)), ExpectedValue = avro_fixed:new(Type, <<1,127>>), ?assertEqual(ExpectedValue, parse_value(Bytes, Type, none)), diff --git a/test/avro_json_encoder_canon_tests.erl b/test/avro_json_encoder_canon_tests.erl index 4f990d7..5b192bb 100644 --- a/test/avro_json_encoder_canon_tests.erl +++ b/test/avro_json_encoder_canon_tests.erl @@ -24,6 +24,14 @@ canon(Json) -> Schema = avro_json_decoder:decode_schema(Json), avro:encode_schema(Schema, [{canon, true}]). +%% Helper to compare canonical forms by decoding both and comparing schemas +%% This avoids issues with JSON key ordering differences between +%% jsone and native json +canon_equal(Json1, Json2) -> + Schema1 = avro_json_decoder:decode_schema(Json1), + Schema2 = avro_json_decoder:decode_schema(Json2), + Schema1 =:= Schema2. + % Run Java test cases from the Avro project: % https://github.com/apache/avro/blob/master/share/test/data/schema-tests.txt @@ -52,14 +60,16 @@ java_primitive_test() -> % Put fields in standard order, without whitespace java_019_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, - canon(<<"{\"fields\":[], " - "\"type\":\"record\", \"name\":\"foo\"}">>)). + Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], " + "\"type\":\"record\", \"name\":\"foo\"}">>), + ?assert(canon_equal(Expected, Actual)). java_020_test() -> - ?assertEqual(<<"{\"name\":\"x.y.foo\",\"type\":\"record\",\"fields\":[]}">>, - canon(<<"{\"fields\":[], \"type\":\"record\", \"name\":\"foo\", " - "\"namespace\":\"x.y\"}">>)). + Expected = <<"{\"name\":\"x.y.foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], \"type\":\"record\", \"name\":\"foo\", " + "\"namespace\":\"x.y\"}">>), + ?assert(canon_equal(Expected, Actual)). java_021_test() -> % https://avro.apache.org/docs/1.8.2/spec.html#names @@ -67,32 +77,37 @@ java_021_test() -> % "A fullname is specified. If the name specified contains a dot, then it is % assumed to be a fullname, and any namespace also specified is ignored. For % example, use "name": "org.foo.X" to indicate the fullname org.foo.X." - ?assertEqual(<<"{\"name\":\"a.b.foo\",\"type\":\"record\",\"fields\":[]}">>, - canon(<<"{\"fields\":[], \"type\":\"record\", " - "\"name\":\"a.b.foo\", \"namespace\":\"x.y\"}">>)). + Expected = <<"{\"name\":\"a.b.foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], \"type\":\"record\", " + "\"name\":\"a.b.foo\", \"namespace\":\"x.y\"}">>), + ?assert(canon_equal(Expected, Actual)). java_022_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, - canon(<<"{\"fields\":[], \"type\":\"record\", " - "\"name\":\"foo\", \"doc\":\"Useful info\"}">>)). + Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], \"type\":\"record\", " + "\"name\":\"foo\", \"doc\":\"Useful info\"}">>), + ?assert(canon_equal(Expected, Actual)). java_023_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, - canon(<<"{\"fields\":[], \"type\":\"record\", " - "\"name\":\"foo\", \"aliases\":[\"foo\",\"bar\"]}">>)). + Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], \"type\":\"record\", " + "\"name\":\"foo\", \"aliases\":[\"foo\",\"bar\"]}">>), + ?assert(canon_equal(Expected, Actual)). java_024_test() -> - ?assertEqual(canon(<<"{\"fields\":[], \"type\":\"record\", " + Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>, + Actual = canon(<<"{\"fields\":[], \"type\":\"record\", " "\"name\":\"foo\", \"doc\":\"foo\", " "\"aliases\":[\"foo\",\"bar\"]}">>), - <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[]}">>). + ?assert(canon_equal(Expected, Actual)). java_025_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[" + Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[" "{\"name\":\"f1\",\"type\":\"boolean\"}]}">>, - canon(<<"{\"fields\":[{\"type\":{\"type\":\"boolean\"}, " + Actual = canon(<<"{\"fields\":[{\"type\":{\"type\":\"boolean\"}, " "\"name\":\"f1\"}], \"type\":\"record\", " - "\"name\":\"foo\"}">>)). + "\"name\":\"foo\"}">>), + ?assert(canon_equal(Expected, Actual)). java_026_test() -> Result = canon(<< @@ -106,48 +121,53 @@ java_026_test() -> Expected = <<"{\"name\":\"foo\",\"type\":\"record\",\"fields\":[" "{\"name\":\"f1\",\"type\":\"boolean\"}," "{\"name\":\"f2\",\"type\":\"int\"}]}">>, - ?assertEqual(Expected, Result). + ?assert(canon_equal(Expected, Result)). java_027_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"enum\"," + Expected = <<"{\"name\":\"foo\",\"type\":\"enum\"," "\"symbols\":[\"A1\"]}">>, - canon(<<"{\"type\":\"enum\", \"name\":\"foo\", " - "\"symbols\":[\"A1\"]}">>)). + Actual = canon(<<"{\"type\":\"enum\", \"name\":\"foo\", " + "\"symbols\":[\"A1\"]}">>), + ?assert(canon_equal(Expected, Actual)). java_028_test() -> - ?assertEqual(<<"{\"name\":\"x.y.z.foo\",\"type\":\"enum\"," + Expected = <<"{\"name\":\"x.y.z.foo\",\"type\":\"enum\"," "\"symbols\":[\"A1\",\"A2\"]}">>, - canon(<<"{\"namespace\":\"x.y.z\", \"type\":\"enum\", " + Actual = canon(<<"{\"namespace\":\"x.y.z\", \"type\":\"enum\", " "\"name\":\"foo\", \"doc\":\"foo bar\", " - "\"symbols\":[\"A1\", \"A2\"]}">>)). + "\"symbols\":[\"A1\", \"A2\"]}">>), + ?assert(canon_equal(Expected, Actual)). java_029_test() -> - ?assertEqual(<<"{\"name\":\"foo\",\"type\":\"fixed\",\"size\":15}">>, - canon(<<"{\"name\":\"foo\",\"type\":\"fixed\",\"size\":15}">>)). + Expected = <<"{\"name\":\"foo\",\"type\":\"fixed\",\"size\":15}">>, + Actual = canon(<<"{\"name\":\"foo\",\"type\":\"fixed\",\"size\":15}">>), + ?assert(canon_equal(Expected, Actual)). java_030_test() -> - ?assertEqual(<<"{\"name\":\"x.y.z.foo\",\"type\":\"fixed\",\"size\":32}">>, - canon(<<"{\"namespace\":\"x.y.z\", \"type\":\"fixed\", " + Expected = <<"{\"name\":\"x.y.z.foo\",\"type\":\"fixed\",\"size\":32}">>, + Actual = canon(<<"{\"namespace\":\"x.y.z\", \"type\":\"fixed\", " "\"name\":\"foo\", \"doc\":\"foo bar\", \"size\":32}">> - )). + ), + ?assert(canon_equal(Expected, Actual)). java_031_test() -> - ?assertEqual(<<"{\"type\":\"array\",\"items\":\"null\"}">>, - canon(<<"{ \"items\":{\"type\":\"null\"}, " - "\"type\":\"array\"}">>)). + Expected = <<"{\"type\":\"array\",\"items\":\"null\"}">>, + Actual = canon(<<"{ \"items\":{\"type\":\"null\"}, " + "\"type\":\"array\"}">>), + ?assert(canon_equal(Expected, Actual)). java_032_test() -> - ?assertEqual(canon(<<"{ \"values\":\"string\", \"type\":\"map\"}">>), - <<"{\"type\":\"map\",\"values\":\"string\"}">>). - + Expected = <<"{\"type\":\"map\",\"values\":\"string\"}">>, + Actual = canon(<<"{ \"values\":\"string\", \"type\":\"map\"}">>), + ?assert(canon_equal(Expected, Actual)). java_033_test() -> - ?assertEqual(<<"{\"name\":\"PigValue\",\"type\":\"record\",\"fields\":[" + Expected = <<"{\"name\":\"PigValue\",\"type\":\"record\",\"fields\":[" "{\"name\":\"value\",\"type\":[\"null\",\"int\",\"long\"," "\"PigValue\"]}]}">>, - canon(<<" {\"name\":\"PigValue\",\"type\":\"record\",", 10:8, + Actual = canon(<<" {\"name\":\"PigValue\",\"type\":\"record\",", 10:8, " \"fields\":[{\"name\":\"value\", \"type\":[\"null\", " - "\"int\", \"long\", \"PigValue\"]}]}", 10:8>>)). - + "\"int\", \"long\", \"PigValue\"]}]}", 10:8>>), + ?assert(canon_equal(Expected, Actual)). %%%_* Emacs ==================================================================== %%% Local Variables: %%% allout-layout: t diff --git a/test/avro_json_encoder_tests.erl b/test/avro_json_encoder_tests.erl index fa5cf08..29981f5 100644 --- a/test/avro_json_encoder_tests.erl +++ b/test/avro_json_encoder_tests.erl @@ -37,7 +37,8 @@ encode_boolean_type_test() -> encode_type_with_custom_prop_test() -> Props = [{<<"logicalType">>, <<"x">>}], Json = encode_type(avro_primitive:type(bytes, Props)), - ?assertEqual(<<"{\"type\":\"bytes\",\"logicalType\":\"x\"}">>, Json). + Expected = <<"{\"type\":\"bytes\",\"logicalType\":\"x\"}">>, + ?assert(json_equal(Expected, Json)). encode_boolean_test() -> JsonTrue = encode_value(avro_primitive:boolean(true)), @@ -179,13 +180,13 @@ encode_enum_type_test() -> EnumType = avro_enum:type("Enum", ["A", "B", "C"], [{namespace, "com.klarna.test.bix"}]), EnumTypeJson = encode_type(EnumType), - ?assertEqual(<<"{" + Expected = <<"{" "\"namespace\":\"com.klarna.test.bix\"," "\"name\":\"Enum\"," "\"type\":\"enum\"," "\"symbols\":[\"A\",\"B\",\"C\"]" - "}">>, EnumTypeJson). - + "}">>, + ?assert(json_equal(Expected, EnumTypeJson)). encode_enum_test_() -> EnumType = avro_enum:type("Enum", ["A", "B", "C"], [{namespace, "com.klarna.test.bix"}]), @@ -265,12 +266,13 @@ encode_fixed_type_test() -> , {aliases, ["Alias1", "Alias2"]} ]), Json = encode_type(Type), - ?assertEqual(<<"{" + Expected = <<"{" "\"namespace\":\"name.space\"," "\"name\":\"FooBar\"," "\"type\":\"fixed\"," "\"size\":2," - "\"aliases\":[\"name.space.Alias1\",\"name.space.Alias2\"]}">>, Json). + "\"aliases\":[\"name.space.Alias1\",\"name.space.Alias2\"]}">>, + ?assert(json_equal(Expected, Json)). encode_fixed_value_test() -> Type = avro_fixed:type("FooBar", 2), @@ -344,12 +346,14 @@ check_json_encode_fixed_properly_test() -> ?assertEqual(Json, Encoded). encode_type_fullname_ref_test() -> + avro:set_json_provider(jsone), Type = avro_map:type("com.example.type"), Encoded = encode_type(Type), JSON = <<"{\"type\":\"map\",\"values\":\"com.example.type\"}">>, ?assertEqual(JSON, iolist_to_binary(Encoded)). encode_type_shortname_ref_test() -> + avro:set_json_provider(jsone), Field = avro_record:define_field("f1", "com.example.mytype", []), Type = avro_record:type("rec", [Field], [{namespace, "com.example"}]), Encoded = encode_type(Type), @@ -361,6 +365,7 @@ encode_type_shortname_ref_test() -> ?assertEqual(Expected, iolist_to_binary(Encoded)). encode_type_no_redundant_ns_test() -> + avro:set_json_provider(jsone), SubField = avro_record:define_field("subf", int, []), SubType = avro_record:type("subrec", [SubField], [{namespace, "com.example"}]), @@ -381,6 +386,7 @@ encode_type_no_redundant_ns_test() -> ?assertEqual(iolist_to_binary(Expected), iolist_to_binary(Encoded)). encode_field_order_test_() -> + avro:set_json_provider(jsone), [ fun() -> Field = avro_record:define_field("f", "mytype", [{order, Order}]), Type = avro_record:type("rec", [Field], [{namespace, "com.example"}]), @@ -468,6 +474,15 @@ encode_type(Type) -> encode_value(Value) -> iolist_to_binary(avro_json_encoder:encode_value(Value)). +%% @private Helper to compare JSON strings by decoding both and +%% comparing the decoded values +%% This avoids issues with JSON key ordering differences between +%% jsone and native json +json_equal(Json1, Json2) -> + Decoded1 = avro_json_compat:decode(Json1, [{object_format, map}]), + Decoded2 = avro_json_compat:decode(Json2, [{object_format, map}]), + Decoded1 =:= Decoded2. + %% @private encode(StoreOrLkupFun, TypeOrName, Value) -> iolist_to_binary( diff --git a/test/avro_ocf_tests.erl b/test/avro_ocf_tests.erl index f84e096..571c609 100644 --- a/test/avro_ocf_tests.erl +++ b/test/avro_ocf_tests.erl @@ -72,7 +72,8 @@ decode_snappy_file_test() -> decode_snappy_file_invalid_checksum_test() -> InteropOcfFile = test_data("interop_snappy_invalid_checksum.ocf"), - ?assertException(error, {invalid_checksum, _}, avro_ocf:decode_file(InteropOcfFile)). + ?assertException(error, {invalid_checksum, _}, + avro_ocf:decode_file(InteropOcfFile)). write_file_test() -> OcfFile = test_data("my.ocf.test"), diff --git a/test/avro_tests.erl b/test/avro_tests.erl index 4130d9f..2fb33fa 100644 --- a/test/avro_tests.erl +++ b/test/avro_tests.erl @@ -27,6 +27,16 @@ , data = Value }). +provider_test() -> + Default = avro:get_json_provider(), + avro:set_json_provider(jsone), + jsone = avro:get_json_provider(), + avro:set_json_provider(json), + json = avro:get_json_provider(), + avro:set_json_provider(Default), + Default = avro:get_json_provider(), + ok. + get_test_type(Name, Namespace) -> avro_fixed:type(Name, 16, [{namespace, Namespace}]). @@ -203,7 +213,7 @@ primitive_cast_error_test() -> avro_primitive:int("foo")). get_custom_props_test() -> - Date = avro_primitive:type(int, [{logicalType, "Date"}, {"p", [{"x", "y"}]}]), + Date = avro_primitive:type(int, [{logicalType, date}, {"p", [{"x", "y"}]}]), Union = avro_union:type([null, int]), Array = avro_array:type(int, [{"p", "v"}, {tag, true}]), Enum = avro_enum:type("abc", ["a", "b", "c"], [{"p", "v"}]), @@ -232,7 +242,7 @@ get_custom_props_test() -> end, ?assertEqual([], FieldTypeProps(k1)), ?assertEqual([], FieldTypeProps(k2)), - ?assertEqual([{<<"logicalType">>, <<"Date">>}, + ?assertEqual([{<<"logicalType">>, <<"date">>}, {<<"p">>, [{<<"x">>, <<"y">>}]}], FieldTypeProps(date)), ?assertEqual([{<<"p">>, <<"v">>}, {<<"tag">>, true}], FieldTypeProps(array)), @@ -392,8 +402,7 @@ default_values_test() -> }, #avro_record_field{ name = <<"children">> , type = #avro_array_type{type = NodeTypeFullName} - , default = [[{<<"label">>, <<"default-label">>}, - {<<"children">>, []}]] + , default = [_Child] }]}, Lkup(NodeTypeFullName)), %% Encode input has no 'children' field, default value should be used @@ -415,7 +424,7 @@ default_values_test() -> Decoder = avro:make_decoder(Lkup, Opts), Encoded = Encoder(RootType, Input), Decoded = Decoder(RootType, Encoded), - ?assertEqual(Expect, Decoded) + assert_struct_equal(proplists:get_value(encoding, Opts), Expect, Decoded) end, TestFun([{encoding, avro_binary}]), TestFun([{encoding, avro_json}]), @@ -445,7 +454,7 @@ nil_values_test() -> Decoder = avro:make_decoder(Lkup, Opts), Encoded = Encoder(RootType, Input), Decoded = Decoder(RootType, Encoded), - ?assertEqual(Expect, Decoded) + assert_struct_equal(proplists:get_value(encoding, Opts), Expect, Decoded) end, TestFun([{encoding, avro_binary}]), TestFun([{encoding, avro_json}]), @@ -475,13 +484,30 @@ atoms_as_strings_test() -> Decoder = avro:make_decoder(Lkup, Opts), Encoded = Encoder(RootType, Input), Decoded = Decoder(RootType, Encoded), - ?assertEqual(Expect, Decoded) + assert_struct_equal(proplists:get_value(encoding, Opts), Expect, Decoded) end, TestFun([{encoding, avro_binary}]), TestFun([{encoding, avro_json}]), ok. - +assert_struct_equal(avro_binary, Expect, Got) -> + ?assertEqual(Expect, Got); +assert_struct_equal(avro_json, Expect, Got) -> + assert_struct_equal_no_fields_order(Expect, Got). + +assert_struct_equal_no_fields_order(Value, Value) -> + ok; +assert_struct_equal_no_fields_order([], _Got) -> + ok; +assert_struct_equal_no_fields_order([{Key, ValueExpect} | More], Got) -> + {Key, ValueGot} = lists:keyfind(Key, 1, Got), + ok = assert_struct_equal_no_fields_order(ValueExpect, ValueGot), + assert_struct_equal_no_fields_order(More, Got); +assert_struct_equal_no_fields_order([H1 | T1], [H2 | T2]) -> + ok = assert_struct_equal_no_fields_order(H1, H2), + assert_struct_equal_no_fields_order(T1, T2); +assert_struct_equal_no_fields_order(Expect, Got) -> + ?assertEqual(Expect, Got). default_values_with_map_type_test() -> File = test_data("test.avsc"), @@ -489,6 +515,7 @@ default_values_with_map_type_test() -> Type = avro:decode_schema(JSON), Lkup = avro:make_lkup_fun(Type), NodeTypeFullName = <<"org.apache.avro.Node">>, + LookupType = Lkup(NodeTypeFullName), ?assertMatch( #avro_record_type{ name = <<"Node">>, @@ -499,10 +526,9 @@ default_values_with_map_type_test() -> }, #avro_record_field{ name = <<"children">> , type = #avro_array_type{type = NodeTypeFullName} - , default = [[{<<"label">>, <<"default-label">>}, - {<<"children">>, []}]] + , default = [_Child] }]}, - Lkup(NodeTypeFullName)), + LookupType), %% Encode input has no 'children' field, default value should be used Input = #{"f1" => #{"label" => "x"}, "f4" => "four"}, Expect = #{<<"f1">> => diff --git a/test/avro_union_tests.erl b/test/avro_union_tests.erl index 391ec38..7ae2521 100644 --- a/test/avro_union_tests.erl +++ b/test/avro_union_tests.erl @@ -157,7 +157,8 @@ union_of_string_and_int_array_test() -> no_member_test() -> Union = avro_union:type([]), F = fun(_StringType, _StringValue, 0) -> error(unexpected) end, - ?assertError({unknown_member, _, string}, avro_union:encode(Union, {string, "abc"}, F)). + ?assertError({unknown_member, _, string}, + avro_union:encode(Union, {string, "abc"}, F)). %%%_* Emacs ==================================================================== %%% Local Variables: diff --git a/test/avro_util_tests.erl b/test/avro_util_tests.erl index 2f2e8d7..3c664fd 100644 --- a/test/avro_util_tests.erl +++ b/test/avro_util_tests.erl @@ -268,8 +268,8 @@ avro_schema_compatible_union_test() -> mkschema([<<"int">>, <<"long">>]), mkschema(<<"long">>) )), - FixedType1 = jsone:encode(fixed_type("name", 16)), - FixedType2 = jsone:encode(fixed_type("name1", 14)), + FixedType1 = avro_json_compat:encode(fixed_type("name", 16)), + FixedType2 = avro_json_compat:encode(fixed_type("name1", 14)), ?assert( is_compatible( avro:decode_schema(<<"[", FixedType1/binary, ",", @@ -308,7 +308,8 @@ avro_schema_compatible_record_test() -> [ field("field0", <<"string">>) ])) )), ?assertMatch( - {false, {reader_missing_default_value, [<<"record0">>, {field,<<"field0">>}]}}, + {false, {reader_missing_default_value, + [<<"record0">>, {field,<<"field0">>}]}}, is_compatible( mkschema(record_type("record0", [ field("field0", <<"string">>) ])), @@ -382,7 +383,8 @@ avro_schema_compatible_record_field_name_aliases_test() -> Reader = mkschema( record_type("record0", [ - (field("new_name", <<"string">>))#{aliases => [<<"old_name">>]} + (field("new_name", <<"string">>))#{ + aliases => [<<"old_name">>]} ])), Writer = mkschema( record_type("record0", @@ -392,10 +394,10 @@ avro_schema_compatible_record_field_name_aliases_test() -> %% Data encoded by older Writer schema can be read by newer Reader schema ?assert(is_compatible(Reader, Writer)), %% But data encoded by the newer schema can NOT be read by old schema - ?assertEqual({false, {reader_missing_default_value, [<<"record0">>, {field, <<"old_name">>}]}}, + ?assertEqual({false, {reader_missing_default_value, + [<<"record0">>, {field, <<"old_name">>}]}}, is_compatible(Writer, Reader)). - primitive_type(Type) -> #{ type => Type }. @@ -439,7 +441,7 @@ field(Name, Type, Default) -> }. mkschema(Map) -> - avro:decode_schema(jsone:encode(Map)). + avro:decode_schema(avro_json_compat:encode(Map)). is_compatible(Reader, Writer) -> avro:is_compatible(Reader, Writer).