13778598045

Committed 11 Mar 2025 01:50AM UTC coverage: 83.201%. First build

Build # 13778598045

Build Type

Pull #14794

github

Committed by

web-flow

Commit Message

Merge 1637a2587 into de1195604

Pull Request Pull Request #14794: feat: add payload_limit param for creating a log tracer

Run Details

40 of 42 new or added lines in 5 files covered. (95.24%)

61214 of 73574 relevant lines covered (83.2%)

16687.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.73

/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl

%%--------------------------------------------------------------------
%% Copyright (c) 2020-2025 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_trace_json_formatter).

-include("emqx_mqtt.hrl").
-include("emqx_trace.hrl").

-export([format/2]).

%% logger_formatter:config/0 is not exported.
-type config() :: map().

-define(DEFAULT_FORMATTER, fun logger:format_otp_report/1).

%%%-----------------------------------------------------------------
%%% Callback Function
%%%-----------------------------------------------------------------

-spec format(LogEvent, Config) -> unicode:chardata() when
    LogEvent :: logger:log_event(),
    Config :: config().
format(
    LogMap,
    #{payload_fmt_opts := PayloadFmtOpts} = Config
) ->
    LogMap0 = emqx_logger_textfmt:evaluate_lazy_values(LogMap),
    LogMap1 = maybe_format_msg(LogMap0, Config),
    %% We just make some basic transformations on the input LogMap and then do
    %% an external call to create the JSON text
    Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
    LogMap2 = LogMap1#{time => Time},
    LogMap3 = prepare_log_data(LogMap2, PayloadFmtOpts),
    [emqx_logger_jsonfmt:best_effort_json(LogMap3, [force_utf8]), "\n"].

%%%-----------------------------------------------------------------
%%% Helper Functions
%%%-----------------------------------------------------------------

maybe_format_msg(#{msg := Msg, meta := Meta} = LogMap, Config) ->
    try do_maybe_format_msg(Msg, Meta, Config) of
        Map when is_map(Map) ->
            LogMap#{meta => maps:merge(Meta, Map), msg => maps:get(msg, Map, "no_message")};
        Bin when is_binary(Bin) ->
            LogMap#{msg => Bin}
    catch
        C:R:S ->
            Meta#{
                msg => "emqx_logger_jsonfmt_format_error",
                fmt_raw_input => Msg,
                fmt_error => C,
                fmt_reason => R,
                fmt_stacktrace => S,
                more => #{
                    original_log_entry => LogMap,
                    config => Config
                }
            }
    end.

do_maybe_format_msg(String, _Meta, _Config) when is_list(String); is_binary(String) ->
    unicode:characters_to_binary(String);
do_maybe_format_msg(undefined, _Meta, _Config) ->
    #{};
do_maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
    case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
        true ->
            %% reporting a map without a customised format function
            Report;
        false ->
            emqx_logger_jsonfmt:format_msg(Msg, Meta, Config)
    end;
do_maybe_format_msg(Msg, Meta, Config) ->
    emqx_logger_jsonfmt:format_msg(Msg, Meta, Config).

prepare_log_data(LogMap, PayloadFmtOpts) when is_map(LogMap) ->
    NewKeyValuePairs = lists:flatmap(
        fun({K, V}) ->
            case prepare_key_value(K, V, PayloadFmtOpts) of
                KV when is_tuple(KV) ->
                    [KV];
                KVs when is_list(KVs) ->
                    KVs
            end
        end,
        maps:to_list(LogMap)
    ),
    maps:from_list(NewKeyValuePairs);
prepare_log_data(V, PayloadFmtOpts) when is_list(V) ->
    [prepare_log_data(Item, PayloadFmtOpts) || Item <- V];
prepare_log_data(V, PayloadFmtOpts) when is_tuple(V) ->
    List = erlang:tuple_to_list(V),
    PreparedList = [prepare_log_data(Item, PayloadFmtOpts) || Item <- List],
    erlang:list_to_tuple(PreparedList);
prepare_log_data(V, _PayloadFmtOpts) ->
    V.

prepare_key_value(host, {I1, I2, I3, I4} = IP, _PayloadFmtOpts) when
    is_integer(I1),
    is_integer(I2),
    is_integer(I3),
    is_integer(I4)
->
    %% We assume this is an IP address
    {host, unicode:characters_to_binary(inet:ntoa(IP))};
prepare_key_value(host, {I1, I2, I3, I4, I5, I6, I7, I8} = IP, _PayloadFmtOpts) when
    is_integer(I1),
    is_integer(I2),
    is_integer(I3),
    is_integer(I4),
    is_integer(I5),
    is_integer(I6),
    is_integer(I7),
    is_integer(I8)
->
    %% We assume this is an IP address
    {host, unicode:characters_to_binary(inet:ntoa(IP))};
prepare_key_value(payload = K, V, PayloadFmtOpts) ->
    {NewV, PEncode1} =
        try
            format_payload(V, PayloadFmtOpts)
        catch
            _:_ ->
                V
        end,
    [{K, NewV}, {payload_encode, PEncode1}];
prepare_key_value(<<"payload">>, V, PayloadFmtOpts) ->
    prepare_key_value(payload, V, PayloadFmtOpts);
prepare_key_value(packet = K, V, PayloadFmtOpts) ->
    NewV =
        try
            format_packet(V, PayloadFmtOpts)
        catch
            _:_ ->
                V
        end,
    {K, NewV};
prepare_key_value(K, {recoverable_error, Msg} = OrgV, PayloadFmtOpts) ->
    try
        prepare_key_value(
            K,
            #{
                error_type => recoverable_error,
                msg => Msg,
                additional_info => <<"The operation may be retried.">>
            },
            PayloadFmtOpts
        )
    catch
        _:_ ->
            {K, OrgV}
    end;
prepare_key_value(rule_ids = K, V, _PayloadFmtOpts) ->
    NewV =
        try
            format_map_set_to_list(V)
        catch
            _:_ ->
                V
        end,
    {K, NewV};
prepare_key_value(client_ids = K, V, _PayloadFmtOpts) ->
    NewV =
        try
            format_map_set_to_list(V)
        catch
            _:_ ->
                V
        end,
    {K, NewV};
prepare_key_value(action_id = K, V, _PayloadFmtOpts) ->
    try
        {action_info, format_action_info(V)}
    catch
        _:_ ->
            {K, V}
    end;
prepare_key_value(K, V, PayloadFmtOpts) ->
    {K, prepare_log_data(V, PayloadFmtOpts)}.

format_packet(undefined, _) -> "";
format_packet(Packet, PayloadFmtOpts) -> emqx_packet:format(Packet, PayloadFmtOpts).

format_payload(undefined, #{payload_encode := Type}) ->
    {"", Type};
format_payload(Payload, PayloadFmtOpts) when is_binary(Payload) ->
    emqx_packet:format_payload(Payload, PayloadFmtOpts);
format_payload(Payload, #{payload_encode := Type}) ->
    {Payload, Type}.

format_map_set_to_list(Map) ->
    Items = [
        begin
            %% Assert that it is really a map set
            true = V,
            %% Assert that the keys have the expected type
            true = is_binary(K),
            K
        end
     || {K, V} <- maps:to_list(Map)
    ],
    lists:sort(Items).

format_action_info(#{mod := _Mod, func := _Func} = FuncCall) ->
    FuncCall;
format_action_info(V) ->
    [<<"action">>, Type, Name | _] = binary:split(V, <<":">>, [global]),
    #{
        type => Type,
        name => Name
    }.

1	%%--------------------------------------------------------------------
2	%% Copyright (c) 2020-2025 EMQ Technologies Co., Ltd. All Rights Reserved.
3	%%
4	%% Licensed under the Apache License, Version 2.0 (the "License");
5	%% you may not use this file except in compliance with the License.
6	%% You may obtain a copy of the License at
7	%%
8	%% http://www.apache.org/licenses/LICENSE-2.0
9	%%
10	%% Unless required by applicable law or agreed to in writing, software
11	%% distributed under the License is distributed on an "AS IS" BASIS,
12	%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	%% See the License for the specific language governing permissions and
14	%% limitations under the License.
15	%%--------------------------------------------------------------------
16	-module(emqx_trace_json_formatter).
17
18	-include("emqx_mqtt.hrl").
19	-include("emqx_trace.hrl").
20
21	-export([format/2]).
22
23	%% logger_formatter:config/0 is not exported.
24	-type config() :: map().
25
26	-define(DEFAULT_FORMATTER, fun logger:format_otp_report/1).
27
28	%%%-----------------------------------------------------------------
29	%%% Callback Function
30	%%%-----------------------------------------------------------------
31
32	-spec format(LogEvent, Config) -> unicode:chardata() when
33	LogEvent :: logger:log_event(),
34	Config :: config().
35	format(
36	LogMap,
37	#{payload_fmt_opts := PayloadFmtOpts} = Config
38	) ->
39	LogMap0 = emqx_logger_textfmt:evaluate_lazy_values(LogMap),	1,845✔
40	LogMap1 = maybe_format_msg(LogMap0, Config),	1,845✔
41	%% We just make some basic transformations on the input LogMap and then do
42	%% an external call to create the JSON text
43	Time = emqx_utils_calendar:now_to_rfc3339(microsecond),	1,845✔
44	LogMap2 = LogMap1#{time => Time},	1,845✔
45	LogMap3 = prepare_log_data(LogMap2, PayloadFmtOpts),	1,845✔
46	[emqx_logger_jsonfmt:best_effort_json(LogMap3, [force_utf8]), "\n"].	1,845✔
47
48	%%%-----------------------------------------------------------------
49	%%% Helper Functions
50	%%%-----------------------------------------------------------------
51
52	maybe_format_msg(#{msg := Msg, meta := Meta} = LogMap, Config) ->
53	try do_maybe_format_msg(Msg, Meta, Config) of	1,845✔
54	Map when is_map(Map) ->
55	LogMap#{meta => maps:merge(Meta, Map), msg => maps:get(msg, Map, "no_message")};	1✔
56	Bin when is_binary(Bin) ->
57	LogMap#{msg => Bin}	1,844✔
58	catch
59	C:R:S ->
60	Meta#{	×
61	msg => "emqx_logger_jsonfmt_format_error",
62	fmt_raw_input => Msg,
63	fmt_error => C,
64	fmt_reason => R,
65	fmt_stacktrace => S,
66	more => #{
67	original_log_entry => LogMap,
68	config => Config
69	}
70	}
71	end.
72
73	do_maybe_format_msg(String, _Meta, _Config) when is_list(String); is_binary(String) ->
74	unicode:characters_to_binary(String);	1,844✔
75	do_maybe_format_msg(undefined, _Meta, _Config) ->
76	#{};	×
77	do_maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
78	case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of	1✔
79	true ->
80	%% reporting a map without a customised format function
81	Report;	1✔
82	false ->
83	emqx_logger_jsonfmt:format_msg(Msg, Meta, Config)	×
84	end;
85	do_maybe_format_msg(Msg, Meta, Config) ->
86	emqx_logger_jsonfmt:format_msg(Msg, Meta, Config).	×
87
88	prepare_log_data(LogMap, PayloadFmtOpts) when is_map(LogMap) ->
89	NewKeyValuePairs = lists:flatmap(	8,496✔
90	fun({K, V}) ->
91	case prepare_key_value(K, V, PayloadFmtOpts) of	30,957✔
92	KV when is_tuple(KV) ->
93	[KV];	30,503✔
94	KVs when is_list(KVs) ->
95	KVs	454✔
96	end
97	end,
98	maps:to_list(LogMap)
99	),
100	maps:from_list(NewKeyValuePairs);	8,496✔
101	prepare_log_data(V, PayloadFmtOpts) when is_list(V) ->
102	[prepare_log_data(Item, PayloadFmtOpts) \|\| Item <- V];	4,208✔
103	prepare_log_data(V, PayloadFmtOpts) when is_tuple(V) ->
104	List = erlang:tuple_to_list(V),	106✔
105	PreparedList = [prepare_log_data(Item, PayloadFmtOpts) \|\| Item <- List],	106✔
106	erlang:list_to_tuple(PreparedList);	106✔
107	prepare_log_data(V, _PayloadFmtOpts) ->
108	V.	40,573✔
109
110	prepare_key_value(host, {I1, I2, I3, I4} = IP, _PayloadFmtOpts) when
111	is_integer(I1),
112	is_integer(I2),
113	is_integer(I3),
114	is_integer(I4)
115	->
116	%% We assume this is an IP address
117	{host, unicode:characters_to_binary(inet:ntoa(IP))};	×
118	prepare_key_value(host, {I1, I2, I3, I4, I5, I6, I7, I8} = IP, _PayloadFmtOpts) when
119	is_integer(I1),
120	is_integer(I2),
121	is_integer(I3),
122	is_integer(I4),
123	is_integer(I5),
124	is_integer(I6),
125	is_integer(I7),
126	is_integer(I8)
127	->
128	%% We assume this is an IP address
129	{host, unicode:characters_to_binary(inet:ntoa(IP))};	×
130	prepare_key_value(payload = K, V, PayloadFmtOpts) ->
131	{NewV, PEncode1} =	454✔
132	try
133	format_payload(V, PayloadFmtOpts)	454✔
134	catch
135	_:_ ->
136	V	×
137	end,
138	[{K, NewV}, {payload_encode, PEncode1}];	454✔
139	prepare_key_value(<<"payload">>, V, PayloadFmtOpts) ->
140	prepare_key_value(payload, V, PayloadFmtOpts);	13✔
141	prepare_key_value(packet = K, V, PayloadFmtOpts) ->
142	NewV =	×
143	try
NEW 144	format_packet(V, PayloadFmtOpts)	×
145	catch
146	_:_ ->
147	V	×
148	end,
149	{K, NewV};	×
150	prepare_key_value(K, {recoverable_error, Msg} = OrgV, PayloadFmtOpts) ->
151	try	12✔
152	prepare_key_value(	12✔
153	K,
154	#{
155	error_type => recoverable_error,
156	msg => Msg,
157	additional_info => <<"The operation may be retried.">>
158	},
159	PayloadFmtOpts
160	)
161	catch
162	_:_ ->
163	{K, OrgV}	×
164	end;
165	prepare_key_value(rule_ids = K, V, _PayloadFmtOpts) ->
166	NewV =	138✔
167	try
168	format_map_set_to_list(V)	138✔
169	catch
170	_:_ ->
171	V	×
172	end,
173	{K, NewV};	138✔
174	prepare_key_value(client_ids = K, V, _PayloadFmtOpts) ->
175	NewV =	138✔
176	try
177	format_map_set_to_list(V)	138✔
178	catch
179	_:_ ->
180	V	×
181	end,
182	{K, NewV};	138✔
183	prepare_key_value(action_id = K, V, _PayloadFmtOpts) ->
184	try	144✔
185	{action_info, format_action_info(V)}	144✔
186	catch
187	_:_ ->
188	{K, V}	×
189	end;
190	prepare_key_value(K, V, PayloadFmtOpts) ->
191	{K, prepare_log_data(V, PayloadFmtOpts)}.	30,083✔
192
193	format_packet(undefined, _) -> "";	×
NEW 194	format_packet(Packet, PayloadFmtOpts) -> emqx_packet:format(Packet, PayloadFmtOpts).	×
195
196	format_payload(undefined, #{payload_encode := Type}) ->
197	{"", Type};	×
198	format_payload(Payload, PayloadFmtOpts) when is_binary(Payload) ->
199	emqx_packet:format_payload(Payload, PayloadFmtOpts);	448✔
200	format_payload(Payload, #{payload_encode := Type}) ->
201	{Payload, Type}.	6✔
202
203	format_map_set_to_list(Map) ->
204	Items = [	276✔
205	begin
206	%% Assert that it is really a map set
207	true = V,	280✔
208	%% Assert that the keys have the expected type
209	true = is_binary(K),	280✔
210	K	280✔
211	end
212	\|\| {K, V} <- maps:to_list(Map)	276✔
213	],
214	lists:sort(Items).	276✔
215
216	format_action_info(#{mod := _Mod, func := _Func} = FuncCall) ->
217	FuncCall;	10✔
218	format_action_info(V) ->
219	[<<"action">>, Type, Name \| _] = binary:split(V, <<":">>, [global]),	134✔
220	#{	134✔
221	type => Type,
222	name => Name
223	}.

emqx / emqx / 13778598045

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous