12583614476

Committed 02 Jan 2025 02:04PM UTC coverage: 82.082%. First build

Build # 12583614476

Build Type

Pull #14286

github

Committed by

web-flow

Commit Message

Merge c9df97f1e into d5a56c20b

Pull Request Pull Request #14286: Implement node-level authentication/authorization cache

Run Details

305 of 338 new or added lines in 29 files covered. (90.24%)

56808 of 69209 relevant lines covered (82.08%)

15216.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.01

/apps/emqx_auth/src/emqx_auth_template.erl

%%--------------------------------------------------------------------
%% Copyright (c) 2024-2025 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------

-module(emqx_auth_template).

-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("snabbkaffe/include/trace.hrl").

%% Template parsing/rendering
-export([
    parse_deep/2,
    parse_str/2,
    parse_sql/3,
    cache_key_template/1,
    cache_key/3,
    cache_key/2,
    placeholder_vars_from_str/1,
    render_deep_for_json/2,
    render_deep_for_url/2,
    render_deep_for_raw/2,
    render_str/2,
    render_urlencoded_str/2,
    render_sql_params/2,
    render_strict/2,
    escape_disallowed_placeholders_str/2
]).

-type allowed_var() :: emqx_template:varname() | {var_namespace, emqx_template:varname()}.
-type used_var() :: emqx_template:varname().
-type allowed_vars() :: [allowed_var()].
-type used_vars() :: [used_var()].

%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------

%%--------------------------------------------------------------------
%% Template parsing/rendering

-spec parse_deep(term(), allowed_vars()) -> {used_vars(), emqx_template:t()}.
parse_deep(Template, AllowedVars) ->
    Result = emqx_template:parse_deep(Template),
    handle_disallowed_placeholders(Result, AllowedVars, {deep, Template}).

-spec parse_str(unicode:chardata(), allowed_vars()) -> {used_vars(), emqx_template:t()}.
parse_str(Template, AllowedVars) ->
    Result = emqx_template:parse(Template),
    handle_disallowed_placeholders(Result, AllowedVars, {string, Template}).

-spec parse_sql(
    emqx_template_sql:raw_statement_template(), emqx_template_sql:sql_parameters(), allowed_vars()
) ->
    {
        used_vars(),
        emqx_template_sql:statement(),
        emqx_template_sql:row_template()
    }.
parse_sql(Template, ReplaceWith, AllowedVars) ->
    {Statement, Result} = emqx_template_sql:parse_prepstmt(
        Template,
        #{parameters => ReplaceWith, strip_double_quote => true}
    ),
    {UsedVars, TemplateWithAllowedVars} = handle_disallowed_placeholders(
        Result, AllowedVars, {string, Template}
    ),
    {UsedVars, Statement, TemplateWithAllowedVars}.

-spec cache_key_template(allowed_vars()) -> emqx_template:t().
cache_key_template(Vars) ->
    emqx_template:parse_deep(
        [
            list_to_binary(emqx_utils:gen_id())
            | lists:map(
                fun(Var) ->
                    list_to_binary("${" ++ Var ++ "}")
                end,
                Vars
            )
        ]
    ).

cache_key(Values, TemplatePart, ExtraUniqnessKey) ->
    fun() ->
        {render_deep_for_raw(TemplatePart, Values), ExtraUniqnessKey}
    end.

cache_key(Values, TemplatePart) ->
    fun() ->
        {render_deep_for_raw(TemplatePart, Values)}
    end.

placeholder_vars_from_str(Str) ->
    emqx_template:placeholders(emqx_template:parse(Str)).

escape_disallowed_placeholders_str(Template, AllowedVars) ->
    ParsedTemplate = emqx_template:parse(Template),
    prerender_disallowed_placeholders(ParsedTemplate, AllowedVars).

handle_disallowed_placeholders(Template, AllowedVars, Source) ->
    {UsedAllowedVars, UsedDisallowedVars} = emqx_template:placeholders(AllowedVars, Template),
    TemplateWithAllowedVars =
        case UsedDisallowedVars of
            [] ->
                Template;
            Disallowed ->
                ?tp(warning, "auth_template_invalid", #{
                    template => Source,
                    reason => Disallowed,
                    allowed => #{placeholders => AllowedVars},
                    notice =>
                        "Disallowed placeholders will be rendered as is."
                        " However, consider using `${$}` escaping for literal `$` where"
                        " needed to avoid unexpected results."
                }),
                Result = prerender_disallowed_placeholders(Template, AllowedVars),
                case Source of
                    {string, _} ->
                        emqx_template:parse(Result);
                    {deep, _} ->
                        emqx_template:parse_deep(Result)
                end
        end,
    {UsedAllowedVars, TemplateWithAllowedVars}.

prerender_disallowed_placeholders(Template, AllowedVars) ->
    {Result, _} = emqx_template:render(Template, #{}, #{
        var_trans => fun(Name, _) ->
            % NOTE
            % Rendering disallowed placeholders in escaped form, which will then
            % parse as a literal string.
            case lists:member(Name, AllowedVars) of
                true -> "${" ++ Name ++ "}";
                false -> "${$}{" ++ Name ++ "}"
            end
        end
    }),
    Result.

render_deep_for_json(Template, Credential) ->
    % NOTE
    % Ignoring errors here, undefined bindings will be replaced with empty string.
    {Term, _Errors} = emqx_template:render(
        Template,
        rename_client_info_vars(Credential),
        #{var_trans => fun to_string_for_json/2}
    ),
    Term.

render_deep_for_raw(Template, Credential) ->
    % NOTE
    % Ignoring errors here, undefined bindings will be replaced with empty string.
    {Term, _Errors} = emqx_template:render(
        Template,
        rename_client_info_vars(Credential),
        #{var_trans => fun to_string_for_raw/2}
    ),
    Term.

render_deep_for_url(Template, Credential) ->
    render_deep_for_raw(Template, Credential).

render_str(Template, Credential) ->
    % NOTE
    % Ignoring errors here, undefined bindings will be replaced with empty string.
    {String, _Errors} = emqx_template:render(
        Template,
        rename_client_info_vars(Credential),
        #{var_trans => fun to_string/2}
    ),
    unicode:characters_to_binary(String).

render_urlencoded_str(Template, Credential) ->
    % NOTE
    % Ignoring errors here, undefined bindings will be replaced with empty string.
    {String, _Errors} = emqx_template:render(
        Template,
        rename_client_info_vars(Credential),
        #{var_trans => fun to_urlencoded_string/2}
    ),
    unicode:characters_to_binary(String).

render_sql_params(ParamList, Credential) ->
    % NOTE
    % Ignoring errors here, undefined bindings will be replaced with empty string.
    {Row, _Errors} = emqx_template:render(
        ParamList,
        rename_client_info_vars(Credential),
        #{var_trans => fun to_sql_value/2}
    ),
    Row.

to_urlencoded_string(Name, Value) ->
    case uri_string:compose_query([{<<"q">>, to_string(Name, Value)}]) of
        <<"q=", EncodedBin/binary>> ->
            EncodedBin;
        "q=" ++ EncodedStr ->
            list_to_binary(EncodedStr)
    end.

to_string(Name, Value) ->
    emqx_template:to_string(render_var(Name, Value)).

%% This converter is to generate data structure possibly with non-utf8 strings.
%% It converts to unicode only strings (character lists).

to_string_for_raw(Name, Value) ->
    strings_to_unicode(Name, render_var(Name, Value)).

%% This converter is to generate data structure suitable for JSON serialization.
%% JSON strings are sequences of unicode characters, not bytes.
%% So we force all rendered data to be unicode, not only character lists.

to_string_for_json(Name, Value) ->
    all_to_unicode(Name, render_var(Name, Value)).

strings_to_unicode(_Name, Value) when is_binary(Value) ->
    Value;
strings_to_unicode(Name, Value) when is_list(Value) ->
    to_unicode_binary(Name, Value);
strings_to_unicode(_Name, Value) ->
    emqx_template:to_string(Value).

all_to_unicode(Name, Value) when is_list(Value) orelse is_binary(Value) ->
    to_unicode_binary(Name, Value);
all_to_unicode(_Name, Value) ->
    emqx_template:to_string(Value).

to_unicode_binary(Name, Value) when is_list(Value) orelse is_binary(Value) ->
    try unicode:characters_to_binary(Value) of
        Encoded when is_binary(Encoded) ->
            Encoded;
        _ ->
            error({encode_error, {non_unicode_data, Name}})
    catch
        error:badarg ->
            error({encode_error, {non_unicode_data, Name}})
    end.

to_sql_value(Name, Value) ->
    emqx_utils_sql:to_sql_value(render_var(Name, Value)).

render_var(_, undefined) ->
    % NOTE
    % Any allowed but undefined binding will be replaced with empty string, even when
    % rendering SQL values.
    <<>>;
render_var(?VAR_CERT_PEM, Value) ->
    base64:encode(Value);
render_var(?VAR_PEERHOST, Value) ->
    inet:ntoa(Value);
render_var(?VAR_PASSWORD, Value) ->
    iolist_to_binary(Value);
render_var(?VAR_PEERPORT, Value) ->
    integer_to_binary(Value);
render_var(_Name, Value) ->
    Value.

render_strict(Topic, ClientInfo) ->
    emqx_template:render_strict(Topic, rename_client_info_vars(ClientInfo)).

rename_client_info_vars(ClientInfo) ->
    Renames = [
        {cn, cert_common_name},
        {dn, cert_subject},
        {protocol, proto_name}
    ],
    lists:foldl(
        fun({Old, New}, Acc) ->
            emqx_utils_maps:rename(Old, New, Acc)
        end,
        ClientInfo,
        Renames
    ).

1	%%--------------------------------------------------------------------
2	%% Copyright (c) 2024-2025 EMQ Technologies Co., Ltd. All Rights Reserved.
3	%%
4	%% Licensed under the Apache License, Version 2.0 (the "License");
5	%% you may not use this file except in compliance with the License.
6	%% You may obtain a copy of the License at
7	%%
8	%% http://www.apache.org/licenses/LICENSE-2.0
9	%%
10	%% Unless required by applicable law or agreed to in writing, software
11	%% distributed under the License is distributed on an "AS IS" BASIS,
12	%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	%% See the License for the specific language governing permissions and
14	%% limitations under the License.
15	%%--------------------------------------------------------------------
16
17	-module(emqx_auth_template).
18
19	-include_lib("emqx/include/emqx_placeholder.hrl").
20	-include_lib("snabbkaffe/include/trace.hrl").
21
22	%% Template parsing/rendering
23	-export([
24	parse_deep/2,
25	parse_str/2,
26	parse_sql/3,
27	cache_key_template/1,
28	cache_key/3,
29	cache_key/2,
30	placeholder_vars_from_str/1,
31	render_deep_for_json/2,
32	render_deep_for_url/2,
33	render_deep_for_raw/2,
34	render_str/2,
35	render_urlencoded_str/2,
36	render_sql_params/2,
37	render_strict/2,
38	escape_disallowed_placeholders_str/2
39	]).
40
41	-type allowed_var() :: emqx_template:varname() \| {var_namespace, emqx_template:varname()}.
42	-type used_var() :: emqx_template:varname().
43	-type allowed_vars() :: [allowed_var()].
44	-type used_vars() :: [used_var()].
45
46	%%--------------------------------------------------------------------
47	%% API
48	%%--------------------------------------------------------------------
49
50	%%--------------------------------------------------------------------
51	%% Template parsing/rendering
52
53	-spec parse_deep(term(), allowed_vars()) -> {used_vars(), emqx_template:t()}.
54	parse_deep(Template, AllowedVars) ->
55	Result = emqx_template:parse_deep(Template),	362✔
56	handle_disallowed_placeholders(Result, AllowedVars, {deep, Template}).	362✔
57
58	-spec parse_str(unicode:chardata(), allowed_vars()) -> {used_vars(), emqx_template:t()}.
59	parse_str(Template, AllowedVars) ->
60	Result = emqx_template:parse(Template),	7,766✔
61	handle_disallowed_placeholders(Result, AllowedVars, {string, Template}).	7,766✔
62
63	-spec parse_sql(
64	emqx_template_sql:raw_statement_template(), emqx_template_sql:sql_parameters(), allowed_vars()
65	) ->
66	{
67	used_vars(),
68	emqx_template_sql:statement(),
69	emqx_template_sql:row_template()
70	}.
71	parse_sql(Template, ReplaceWith, AllowedVars) ->
72	{Statement, Result} = emqx_template_sql:parse_prepstmt(	120✔
73	Template,
74	#{parameters => ReplaceWith, strip_double_quote => true}
75	),
76	{UsedVars, TemplateWithAllowedVars} = handle_disallowed_placeholders(	120✔
77	Result, AllowedVars, {string, Template}
78	),
79	{UsedVars, Statement, TemplateWithAllowedVars}.	120✔
80
81	-spec cache_key_template(allowed_vars()) -> emqx_template:t().
82	cache_key_template(Vars) ->
83	emqx_template:parse_deep(	422✔
84	[
85	list_to_binary(emqx_utils:gen_id())
86	\| lists:map(
87	fun(Var) ->
88	list_to_binary("${" ++ Var ++ "}")	673✔
89	end,
90	Vars
91	)
92	]
93	).
94
95	cache_key(Values, TemplatePart, ExtraUniqnessKey) ->
96	fun() ->	15✔
NEW 97	{render_deep_for_raw(TemplatePart, Values), ExtraUniqnessKey}	×
98	end.
99
100	cache_key(Values, TemplatePart) ->
101	fun() ->	548✔
102	{render_deep_for_raw(TemplatePart, Values)}	140✔
103	end.
104
105	placeholder_vars_from_str(Str) ->
106	emqx_template:placeholders(emqx_template:parse(Str)).	101✔
107
108	escape_disallowed_placeholders_str(Template, AllowedVars) ->
109	ParsedTemplate = emqx_template:parse(Template),	94✔
110	prerender_disallowed_placeholders(ParsedTemplate, AllowedVars).	94✔
111
112	handle_disallowed_placeholders(Template, AllowedVars, Source) ->
113	{UsedAllowedVars, UsedDisallowedVars} = emqx_template:placeholders(AllowedVars, Template),	8,248✔
114	TemplateWithAllowedVars =	8,248✔
115	case UsedDisallowedVars of
116	[] ->
117	Template;	8,236✔
118	Disallowed ->
119	?tp(warning, "auth_template_invalid", #{	12✔
120	template => Source,
121	reason => Disallowed,
122	allowed => #{placeholders => AllowedVars},
123	notice =>
124	"Disallowed placeholders will be rendered as is."
125	" However, consider using `${$}` escaping for literal `$` where"
126	" needed to avoid unexpected results."
127	}),
128	Result = prerender_disallowed_placeholders(Template, AllowedVars),	12✔
129	case Source of	12✔
130	{string, _} ->
131	emqx_template:parse(Result);	4✔
132	{deep, _} ->
133	emqx_template:parse_deep(Result)	8✔
134	end
135	end,
136	{UsedAllowedVars, TemplateWithAllowedVars}.	8,248✔
137
138	prerender_disallowed_placeholders(Template, AllowedVars) ->
139	{Result, _} = emqx_template:render(Template, #{}, #{	106✔
140	var_trans => fun(Name, _) ->
141	% NOTE
142	% Rendering disallowed placeholders in escaped form, which will then
143	% parse as a literal string.
144	case lists:member(Name, AllowedVars) of	76✔
145	true -> "${" ++ Name ++ "}";	62✔
146	false -> "${$}{" ++ Name ++ "}"	14✔
147	end
148	end
149	}),
150	Result.	106✔
151
152	render_deep_for_json(Template, Credential) ->
153	% NOTE
154	% Ignoring errors here, undefined bindings will be replaced with empty string.
155	{Term, _Errors} = emqx_template:render(	171✔
156	Template,
157	rename_client_info_vars(Credential),
158	#{var_trans => fun to_string_for_json/2}
159	),
160	Term.	169✔
161
162	render_deep_for_raw(Template, Credential) ->
163	% NOTE
164	% Ignoring errors here, undefined bindings will be replaced with empty string.
165	{Term, _Errors} = emqx_template:render(	759✔
166	Template,
167	rename_client_info_vars(Credential),
168	#{var_trans => fun to_string_for_raw/2}
169	),
170	Term.	759✔
171
172	render_deep_for_url(Template, Credential) ->
173	render_deep_for_raw(Template, Credential).	584✔
174
175	render_str(Template, Credential) ->
176	% NOTE
177	% Ignoring errors here, undefined bindings will be replaced with empty string.
178	{String, _Errors} = emqx_template:render(	59✔
179	Template,
180	rename_client_info_vars(Credential),
181	#{var_trans => fun to_string/2}
182	),
183	unicode:characters_to_binary(String).	59✔
184
185	render_urlencoded_str(Template, Credential) ->
186	% NOTE
187	% Ignoring errors here, undefined bindings will be replaced with empty string.
188	{String, _Errors} = emqx_template:render(	311✔
189	Template,
190	rename_client_info_vars(Credential),
191	#{var_trans => fun to_urlencoded_string/2}
192	),
193	unicode:characters_to_binary(String).	311✔
194
195	render_sql_params(ParamList, Credential) ->
196	% NOTE
197	% Ignoring errors here, undefined bindings will be replaced with empty string.
198	{Row, _Errors} = emqx_template:render(	143✔
199	ParamList,
200	rename_client_info_vars(Credential),
201	#{var_trans => fun to_sql_value/2}
202	),
203	Row.	143✔
204
205	to_urlencoded_string(Name, Value) ->
206	case uri_string:compose_query([{<<"q">>, to_string(Name, Value)}]) of	24✔
207	<<"q=", EncodedBin/binary>> ->
208	EncodedBin;	22✔
209	"q=" ++ EncodedStr ->
210	list_to_binary(EncodedStr)	2✔
211	end.
212
213	to_string(Name, Value) ->
214	emqx_template:to_string(render_var(Name, Value)).	84✔
215
216	%% This converter is to generate data structure possibly with non-utf8 strings.
217	%% It converts to unicode only strings (character lists).
218
219	to_string_for_raw(Name, Value) ->
220	strings_to_unicode(Name, render_var(Name, Value)).	925✔
221
222	%% This converter is to generate data structure suitable for JSON serialization.
223	%% JSON strings are sequences of unicode characters, not bytes.
224	%% So we force all rendered data to be unicode, not only character lists.
225
226	to_string_for_json(Name, Value) ->
227	all_to_unicode(Name, render_var(Name, Value)).	270✔
228
229	strings_to_unicode(_Name, Value) when is_binary(Value) ->
230	Value;	858✔
231	strings_to_unicode(Name, Value) when is_list(Value) ->
232	to_unicode_binary(Name, Value);	8✔
233	strings_to_unicode(_Name, Value) ->
234	emqx_template:to_string(Value).	59✔
235
236	all_to_unicode(Name, Value) when is_list(Value) orelse is_binary(Value) ->
237	to_unicode_binary(Name, Value);	255✔
238	all_to_unicode(_Name, Value) ->
239	emqx_template:to_string(Value).	15✔
240
241	to_unicode_binary(Name, Value) when is_list(Value) orelse is_binary(Value) ->
242	try unicode:characters_to_binary(Value) of	263✔
243	Encoded when is_binary(Encoded) ->
244	Encoded;	261✔
245	_ ->
246	error({encode_error, {non_unicode_data, Name}})	2✔
247	catch
248	error:badarg ->
249	error({encode_error, {non_unicode_data, Name}})	×
250	end.
251
252	to_sql_value(Name, Value) ->
253	emqx_utils_sql:to_sql_value(render_var(Name, Value)).	153✔
254
255	render_var(_, undefined) ->
256	% NOTE
257	% Any allowed but undefined binding will be replaced with empty string, even when
258	% rendering SQL values.
259	<<>>;	31✔
260	render_var(?VAR_CERT_PEM, Value) ->
261	base64:encode(Value);	6✔
262	render_var(?VAR_PEERHOST, Value) ->
263	inet:ntoa(Value);	24✔
264	render_var(?VAR_PASSWORD, Value) ->
265	iolist_to_binary(Value);	378✔
266	render_var(?VAR_PEERPORT, Value) ->
267	integer_to_binary(Value);	6✔
268	render_var(_Name, Value) ->
269	Value.	987✔
270
271	render_strict(Topic, ClientInfo) ->
272	emqx_template:render_strict(Topic, rename_client_info_vars(ClientInfo)).	188✔
273
274	rename_client_info_vars(ClientInfo) ->
275	Renames = [	1,631✔
276	{cn, cert_common_name},
277	{dn, cert_subject},
278	{protocol, proto_name}
279	],
280	lists:foldl(	1,631✔
281	fun({Old, New}, Acc) ->
282	emqx_utils_maps:rename(Old, New, Acc)	4,893✔
283	end,
284	ClientInfo,
285	Renames
286	).

emqx / emqx / 12583614476

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous