8000 feat(emqx_trace): add JSON trace log entry formatter by kjellwinblad · Pull Request #12863 · emqx/emqx · GitHub
[go: up one dir, main page]
Skip to content

feat(emqx_trace): add JSON trace log entry formatter #12863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Apr 22, 2024
Merged

feat(emqx_trace): add JSON trace log entry formatter #12863

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
a655874
feat(emqx_trace): add JSON trace log entry formatter
kjellwinblad Apr 11, 2024
3c5c3eb
docs: add change log entry
kjellwinblad Apr 11, 2024
6fd7a06
fix: problems reported by dialyzer
kjellwinblad Apr 11, 2024
aa39835
feat(emqx_cli): add parameter for trace format (plain or json)
kjellwinblad Apr 11, 2024
968dc2c
fix(json trace format): use best_effort_json instead of new encoder
kjellwinblad Apr 12, 2024
6c0ee8b
test(emqx_trace_SUITE): fix failure due to new field
kjellwinblad Apr 12, 2024
ef9f8a8
fix: unreachable clause found by dialyzer
kjellwinblad Apr 14, 2024
10957e7
fix: change name of the default trace log format from plain to text
kjellwinblad Apr 18, 2024
aa388ad
fix(json trace format): format client_ids and rule_ids as lists
kjellwinblad Apr 18, 2024
3232ab5
fix(rule tracing): unset trace meta data in try-after-end
kjellwinblad Apr 18, 2024
8f1486f 8000
fix: clean up trace messages to make it easier to interpret
kjellwinblad Apr 18, 2024
f5b0439
fix(emqx_rule_engine_api_rule_apply_SUITE): flaky test case
kjellwinblad Apr 18, 2024
7be1873
test(emqx_rule_engine_api_rule_apply_SUITE): remove unnecessary code
kjellwinblad Apr 18, 2024
285bfa9
fix: improve rendering of action_template_rendered trace
kjellwinblad Apr 18, 2024
09b414f
test: add necessary application to test suites
kjellwinblad Apr 18, 2024
c163956
fix(trace formatter): remove record field to enable rolling upgrade
kjellwinblad Apr 19, 2024
9e46c18
fix(emqx_trace_SUITE:t_base_create_delete): broken test case
kjellwinblad Apr 19, 2024
2890bc2
fix(tracing): remove internal extra field from the trace config
kjellwinblad Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(json trace format): use best_effort_json instead of new encoder
  • Loading branch information
@kjellwinblad
kjellwinblad committed Apr 18, 2024
commit 968dc2ccdafd78128dbedc3dacc732138567efaa
3 changes: 0 additions & 3 deletions apps/emqx/src/emqx_trace/emqx_trace_handler.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,6 @@ filters(#{type := ruleid, filter := Filter, name := Name}) ->

formatter(#{type := _Type, payload_encode := PayloadEncode, formatter := json}) ->
{emqx_trace_json_formatter, #{
single_line => true,
max_size => unlimited,
depth => unlimited,
payload_encode => PayloadEncode
}};
formatter(#{type := _Type, payload_encode := PayloadEncode}) ->
Expand Down
95 changes: 31 additions & 64 deletions apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,78 +33,45 @@ format(
LogMap,
#{payload_encode := PEncode}
) ->
%% We just make some basic transformations on the input LogMap and then do
%% an external call to create the JSON text
Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
LogMap1 = LogMap#{time => Time},
[format_log_map(LogMap1, PEncode), "\n"].
LogMap2 = prepare_log_map(LogMap1, PEncode),
[emqx_logger_jsonfmt:best_effort_json(LogMap2, [force_utf8]), "\n"].

%%%-----------------------------------------------------------------
%%% Helper Functions
%%%-----------------------------------------------------------------

format_log_map(Map, PEncode) ->
KeyValuePairs = format_key_value_pairs(maps:to_list(Map), PEncode, []),
["{", KeyValuePairs, "}"].
prepare_log_map(LogMap, PEncode) when is_map(LogMap) ->
NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)],
maps:from_list(NewKeyValuePairs);
prepare_log_map(Term, _PEncode) ->
Term.

format_key_value_pairs([], _PEncode, Acc) ->
lists:join(",", Acc);
format_key_value_pairs([{payload, Value} | Rest], PEncode, Acc) ->
FormattedPayload = format_payload(Value, PEncode),
FormattedPayloadEscaped = escape(FormattedPayload),
Pair = ["\"payload\": \"", FormattedPayloadEscaped, "\""],
format_key_value_pairs(Rest, PEncode, [Pair | Acc]);
format_key_value_pairs([{packet, Value} | Rest], PEncode, Acc) ->
Formatted = format_packet(Value, PEncode),
FormattedEscaped = escape(Formatted),
Pair = ["\"packet\": \"", FormattedEscaped, "\""],
format_key_value_pairs(Rest, PEncode, [Pair | Acc]);
format_key_value_pairs([{Key, Value} | Rest], PEncode, Acc) ->
FormattedKey = format_key(Key),
FormattedValue = format_value(Value, PEncode),
Pair = ["\"", FormattedKey, "\":", FormattedValue],
format_key_value_pairs(Rest, PEncode, [Pair | Acc]).

format_key(Term) ->
%% Keys must be strings
String = try_format_unicode(Term),
escape(String).

format_value(Map, PEncode) when is_map(Map) ->
format_log_map(Map, PEncode);
format_value(V, _PEncode) when is_integer(V) ->
integer_to_list(V);
format_value(V, _PEncode) when is_float(V) ->
float_to_list(V, [{decimals, 2}]);
format_value(true, _PEncode) ->
"true";
format_value(false, _PEncode) ->
"false";
format_value(V, _PEncode) ->
String = try_format_unicode(V),
["\"", escape(String), "\""].

try_format_unicode(undefined) ->
%% emqx_logger_textfmt:try_format_unicode converts the atom undefined to
%% the atom undefined
"undefined";
try_format_unicode(V) ->
emqx_logger_textfmt:try_format_unicode(V).

escape(IOList) ->
Bin = iolist_to_binary(IOList),
List = binary_to_list(Bin),
escape_list(List).

escape_list([]) ->
[];
escape_list([$\n | Rest]) ->
%% 92 is backslash
[92, $n | escape_list(Rest)];
escape_list([$" | Rest]) ->
[92, $" | escape_list(Rest)];
escape_list([92 | Rest]) ->
[92, 92 | escape_list(Rest)];
escape_list([X | Rest]) ->
[X | escape_list(Rest)].
prepare_key_value(payload = K, V, PEncode) ->
NewV =
try
format_payload(V, PEncode)
catch
_:_:_ ->
V
end,
{K, NewV};
prepare_key_value(packet = K, V, PEncode) ->
NewV =
try
format_packet(V, PEncode)
catch
_:_:_ ->
V
end,
{K, NewV};
prepare_key_value(K, V, PEncode) when is_map(V) ->
{K, prepare_log_map(V, PEncode)};
prepare_key_value(K, V, _PEncode) ->
{K, V}.

format_packet(undefined, _) -> "";
format_packet(Packet, Encode) -> emqx_packet:format(Packet, Encode).
Expand Down
0