20428555173

Committed 22 Dec 2025 10:02AM UTC coverage: 94.693% (-0.1%) from 94.814%

Build # 20428555173

Build Type

Pull #1436

github

Committed by

web-flow

Commit Message

Merge 0ffb48a5d into 57a75b09e

Pull Request Pull Request #1436: Draft: add `rdmo.config` app for `Plugin` model (plugin managament)

Run Details

2191 of 2304 branches covered (95.1%)

23411 of 24723 relevant lines covered (94.69%)

3.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.5

rdmo/core/utils.py

import json
import logging
import os
import re
from datetime import datetime
from pathlib import Path
from urllib.parse import urlparse

from django.conf import settings
from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.template.loader import get_template, render_to_string
from django.utils.dateparse import parse_date
from django.utils.encoding import force_str
from django.utils.formats import get_format
from django.utils.module_loading import import_string
from django.utils.translation import gettext_lazy as _

from defusedcsv import csv
from markdown import markdown

from .constants import HUMAN2BYTES_MAPPER
from .pandoc import get_pandoc_content, get_pandoc_content_disposition

log = logging.getLogger(__name__)


def get_script_alias(request):
    return request.path[:-len(request.path_info)]


def get_referer(request, default=None):
    return request.META.get('HTTP_REFERER', default)


def get_referer_path_info(request, default=''):
    referer = request.META.get('HTTP_REFERER', None)
    if not referer:
        return default

    script_alias = get_script_alias(request)
    return urlparse(referer).path[len(script_alias):]


def get_next(request):
    next = request.POST.get('next')
    current = request.path_info

    if next in (current, None):
        return get_script_alias(request) + '/'
    else:
        return get_script_alias(request) + next


def get_uri_prefix(obj):
    # needs to stay, is part of a migration
    r = settings.DEFAULT_URI_PREFIX
    if bool(obj.uri_prefix) is True:
        r = obj.uri_prefix.rstrip('/')
    return r


def join_url(base, *args) -> str:
    url = base
    for arg in args:
        url = url.rstrip('/') + '/' + arg.lstrip('/')
    return url


def get_model_field_meta(model):
    meta = {}

    for field in model._meta.get_fields():
        match = re.search(r'lang(\d)$', field.name)
        if match:
            lang_index = int(match.group(1))

            try:
                lang_code, lang = settings.LANGUAGES[lang_index - 1]
            except IndexError:
                continue

            field_name = field.name.replace(f'_lang{lang_index}', f'_{lang_code}')

            meta[field_name] = {}
            if hasattr(field, 'verbose_name'):
                # remove the "(primary)" part
                meta[field_name]['verbose_name'] = re.sub(r'\(.*\)$', f'({lang})', str(field.verbose_name))
            if hasattr(field, 'help_text'):
                # remove the "in the primary language" part
                meta[field_name]['help_text'] = re.sub(r' \(.*\).', '.', str(field.help_text))
        else:
            meta[field.name] = {}
            if hasattr(field, 'verbose_name'):
                meta[field.name]['verbose_name'] = field.verbose_name
            if hasattr(field, 'help_text'):
                meta[field.name]['help_text'] = field.help_text

    if model._meta.verbose_name == 'Page':
        meta['elements'] = {
            'verbose_name': _('Elements'),
            'help_text': _('The questions and question sets for this page.')
        }
    elif model._meta.verbose_name == 'QuestionSet':
        meta['elements'] = {
            'verbose_name': _('Elements'),
            'help_text': _('The questions and question sets for this question set.')
        }
    elif model._meta.verbose_name == 'Plugin':
        meta['python_path'] = {
            **meta.get('python_path', {}),
            'choices': [(python_path, python_path) for python_path in get_plugin_python_paths()]
        }

    return meta


def get_languages() -> list[tuple]:
    languages = []
    for i in range(5):
        try:
            language = (settings.LANGUAGES[i][0], settings.LANGUAGES[i][1], f"lang{i + 1}")
            languages.append(language)
        except IndexError:
            pass
    return languages


def get_language_fields(field_name):
    return [
        field_name + '_' + lang_field for lang_code,
        lang_string, lang_field in get_languages()
        ]


def get_language_warning(obj, field):
    for _lang_code, _lang_string, lang_field in get_languages():
        if not getattr(obj, f'{field}_{lang_field}'):
            return True
    return False


def render_to_format(request, export_format, title, template_src, context):
    if export_format not in dict(settings.EXPORT_FORMATS):
        return HttpResponseBadRequest(_('This format is not supported.'))

    # render the template to a html string
    template = get_template(template_src)
    html = template.render(context)
    metadata, html = parse_metadata(html)

    # remove empty lines
    html = os.linesep.join([line for line in html.splitlines() if line.strip()])

    if export_format == 'html':
        # create the response object
        response = HttpResponse(html)
        response['Content-Disposition'] = f'filename="{title}.{export_format}"'

    else:
        pandoc_content = get_pandoc_content(html, metadata, export_format, context)
        pandoc_content_disposition = get_pandoc_content_disposition(export_format, title)

        response = HttpResponse(pandoc_content, content_type=f'application/{export_format}')
        response['Content-Disposition'] = pandoc_content_disposition.encode('utf-8')

    return response


def render_to_csv(title, rows, delimiter=','):
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = f'attachment; filename="{title}.csv"'

    writer = csv.writer(response, delimiter=delimiter)
    for row in rows:
        writer.writerow(
            ['' if x is None else str(x) for x in row]
        )
    return response


def render_to_json(title, data, delimiter=','):
    response = HttpResponse(json.dumps(data, indent=2), content_type='text/json')
    response['Content-Disposition'] = f'attachment; filename="{title}.json"'
    return response


def return_file_response(file_path, content_type):
    file_abspath = Path(settings.MEDIA_ROOT) / file_path
    if file_abspath.exists():
        with file_abspath.open('rb') as fp:
            response = HttpResponse(fp.read(), content_type=content_type)
            response['Content-Disposition'] = 'attachment; filename=' + file_abspath.name
            return response
    else:
        raise Http404


def sanitize_url(s):
    # is used in the rdmo-app
    try:
        m = re.search('[a-z0-9-_]', s)
    except TypeError:
        s = ''
    else:
        if bool(m) is False:
            s = ''
        else:
            s = re.sub(r'/+', '/', s)
    return s


def get_plugin_python_paths(raise_exception=False):
    plugin_paths = []
    for python_path in settings.PLUGINS:
        try:
            import_string(python_path)
        except (ImportError, ValueError) as e:
            if raise_exception:
                raise e from e
        else:
            plugin_paths.append(python_path)
    return plugin_paths


def human2bytes(string):
    if not string or string == '0':
        return 0

    m = re.match(r'([0-9.]+)\s*([A-Za-z]+)', string)
    number, unit = float(m.group(1)), m.group(2).strip().lower()

    conversion = HUMAN2BYTES_MAPPER[unit]
    number = number*conversion['base']**(conversion['power'])
    return number


def is_truthy(value):
    return value is not None and (value is True or value.lower() in ['1', 't', 'true'])


def markdown2html(markdown_string):
    # adoption of the normal markdown function
    html = markdown(force_str(markdown_string)).strip()

    # strip the outer paragraph
    html = re.sub(r'^<p>(.*?)</p>$',r'\1', html)

    # convert `[<string>]{<title>}` to <span title="<title>"><string></span> to allow for underlined tooltips
    html = re.sub(
        r'\[(.*?)\]\{(.*?)\}',
        r'<span data-toggle="tooltip" data-placement="bottom" data-html="true" title="\2">\1</span>',
        html
    )

    # convert everything after `{more}` to <span class="more"><string></span> to be shown/hidden on user input
    show_string = _('show more')
    hide_string = _('show less')
    html = re.sub(
        r'(\{more\})(.*?)$',
        f'<span class="show-more" onclick="showMore(this)">... ({show_string})</span>'
        r'<span class="more">\2</span>'
        f'<span class="show-less" onclick="showLess(this)"> ({hide_string})</span>',
        html
    )

    # textblocks (e.g. for help texts) can be injected into free text fields as small templates via Markdown
    html = inject_textblocks(html)

    return html


def inject_textblocks(html):
    # loop over all strings between curly brackets, e.g. {{ test }}
    for template_code in re.findall(r'{{(.*?)}}', html):
        template_name = settings.MARKDOWN_TEMPLATES.get(template_code.strip())
        if template_name:
            html = re.sub('{{' + template_code + '}}', render_to_string(template_name), html)
    return html


def parse_metadata(html):
    metadata = None
    pattern = re.compile(
        '(<metadata>)(.*)(</metadata>)', re.MULTILINE | re.DOTALL
    )
    m = re.search(pattern, html)
    if bool(m) is True:
        try:
            metadata = json.loads(m.group(2))
        except json.JSONDecodeError:
            pass
        else:
            html = html.replace(m.group(0), '')
    return metadata, html


def remove_double_newlines(string):
    return re.sub(r'[\n]{2,}', '\n\n', string)


def remove_html_special_characters(string):
    return re.sub(r'[<>&"\']', '', string)


def parse_date_from_string(date: str) -> datetime.date:
    if not isinstance(date, str):
        raise TypeError("date must be provided as string")

    try:
        # First, try standard ISO format (YYYY-MM-DD)
        parsed_date = parse_date(date)
    except ValueError as exc:
        raise exc from exc

    # If ISO parsing fails, try localized DATE_INPUT_FORMATS formats
    if parsed_date is None:
        for fmt in get_format('DATE_INPUT_FORMATS'):
            try:
                parsed_date = datetime.strptime(date, fmt).date()
                break  # Stop if parsing succeeds
            except ValueError:
                continue  # Try the next format

    # If still not parsed, raise an error
    if not parsed_date:
        raise ValueError(
            f"Invalid date format for: {date}. Valid formats {get_format('DATE_INPUT_FORMATS')}"
        )
    return parsed_date

1	import json	4✔
2	import logging	4✔
3	import os	4✔
4	import re	4✔
5	from datetime import datetime	4✔
6	from pathlib import Path	4✔
7	from urllib.parse import urlparse	4✔
8
9	from django.conf import settings	4✔
10	from django.http import Http404, HttpResponse, HttpResponseBadRequest	4✔
11	from django.template.loader import get_template, render_to_string	4✔
12	from django.utils.dateparse import parse_date	4✔
13	from django.utils.encoding import force_str	4✔
14	from django.utils.formats import get_format	4✔
15	from django.utils.module_loading import import_string	4✔
16	from django.utils.translation import gettext_lazy as _	4✔
17
18	from defusedcsv import csv	4✔
19	from markdown import markdown	4✔
20
21	from .constants import HUMAN2BYTES_MAPPER	4✔
22	from .pandoc import get_pandoc_content, get_pandoc_content_disposition	4✔
23
24	log = logging.getLogger(__name__)	4✔
25
26
27	def get_script_alias(request):	4✔
28	return request.path[:-len(request.path_info)]	4✔
29
30
31	def get_referer(request, default=None):	4✔
32	return request.META.get('HTTP_REFERER', default)	4✔
33
34
35	def get_referer_path_info(request, default=''):	4✔
36	referer = request.META.get('HTTP_REFERER', None)	4✔
37	if not referer:	4✔
38	return default	4✔
39
40	script_alias = get_script_alias(request)	×
41	return urlparse(referer).path[len(script_alias):]	×
42
43
44	def get_next(request):	4✔
45	next = request.POST.get('next')	4✔
46	current = request.path_info	4✔
47
48	if next in (current, None):	4✔
49	return get_script_alias(request) + '/'	4✔
50	else:
51	return get_script_alias(request) + next	4✔
52
53
54	def get_uri_prefix(obj):	4✔
55	# needs to stay, is part of a migration
56	r = settings.DEFAULT_URI_PREFIX	×
57	if bool(obj.uri_prefix) is True:	×
58	r = obj.uri_prefix.rstrip('/')	×
59	return r	×
60
61
62	def join_url(base, *args) -> str:	4✔
63	url = base	4✔
64	for arg in args:	4✔
65	url = url.rstrip('/') + '/' + arg.lstrip('/')	4✔
66	return url	4✔
67
68
69	def get_model_field_meta(model):	4✔
70	meta = {}	4✔
71
72	for field in model._meta.get_fields():	4✔
73	match = re.search(r'lang(\d)$', field.name)	4✔
74	if match:	4✔
75	lang_index = int(match.group(1))	4✔
76
77	try:	4✔
78	lang_code, lang = settings.LANGUAGES[lang_index - 1]	4✔
79	except IndexError:	4✔
80	continue	4✔
81
82	field_name = field.name.replace(f'_lang{lang_index}', f'_{lang_code}')	4✔
83
84	meta[field_name] = {}	4✔
85	if hasattr(field, 'verbose_name'):	4✔
86	# remove the "(primary)" part
87	meta[field_name]['verbose_name'] = re.sub(r'\(.*\)$', f'({lang})', str(field.verbose_name))	4✔
88	if hasattr(field, 'help_text'):	4✔
89	# remove the "in the primary language" part
90	meta[field_name]['help_text'] = re.sub(r' \(.*\).', '.', str(field.help_text))	4✔
91	else:
92	meta[field.name] = {}	4✔
93	if hasattr(field, 'verbose_name'):	4✔
94	meta[field.name]['verbose_name'] = field.verbose_name	4✔
95	if hasattr(field, 'help_text'):	4✔
96	meta[field.name]['help_text'] = field.help_text	4✔
97
98	if model._meta.verbose_name == 'Page':	4✔
99	meta['elements'] = {	4✔
100	'verbose_name': _('Elements'),
101	'help_text': _('The questions and question sets for this page.')
102	}
103	elif model._meta.verbose_name == 'QuestionSet':	4✔
104	meta['elements'] = {	×
105	'verbose_name': _('Elements'),
106	'help_text': _('The questions and question sets for this question set.')
107	}
108	elif model._meta.verbose_name == 'Plugin':	4✔
109	meta['python_path'] = {	4✔
110	**meta.get('python_path', {}),
111	'choices': [(python_path, python_path) for python_path in get_plugin_python_paths()]
112	}
113
114	return meta	4✔
115
116
117	def get_languages() -> list[tuple]:	4✔
118	languages = []	4✔
119	for i in range(5):	4✔
120	try:	4✔
121	language = (settings.LANGUAGES[i][0], settings.LANGUAGES[i][1], f"lang{i + 1}")	4✔
122	languages.append(language)	4✔
123	except IndexError:	4✔
124	pass	4✔
125	return languages	4✔
126
127
128	def get_language_fields(field_name):	4✔
129	return [	4✔
130	field_name + '_' + lang_field for lang_code,
131	lang_string, lang_field in get_languages()
132	]
133
134
135	def get_language_warning(obj, field):	4✔
136	for _lang_code, _lang_string, lang_field in get_languages():	4✔
137	if not getattr(obj, f'{field}_{lang_field}'):	4✔
138	return True	4✔
139	return False	4✔
140
141
142	def render_to_format(request, export_format, title, template_src, context):	4✔
143	if export_format not in dict(settings.EXPORT_FORMATS):	4✔
144	return HttpResponseBadRequest(_('This format is not supported.'))	×
145
146	# render the template to a html string
147	template = get_template(template_src)	4✔
148	html = template.render(context)	4✔
149	metadata, html = parse_metadata(html)	4✔
150
151	# remove empty lines
152	html = os.linesep.join([line for line in html.splitlines() if line.strip()])	4✔
153
154	if export_format == 'html':	4✔
155	# create the response object
156	response = HttpResponse(html)	4✔
157	response['Content-Disposition'] = f'filename="{title}.{export_format}"'	4✔
158
159	else:
160	pandoc_content = get_pandoc_content(html, metadata, export_format, context)	×
161	pandoc_content_disposition = get_pandoc_content_disposition(export_format, title)	×
162
163	response = HttpResponse(pandoc_content, content_type=f'application/{export_format}')	×
164	response['Content-Disposition'] = pandoc_content_disposition.encode('utf-8')	×
165
166	return response	4✔
167
168
169	def render_to_csv(title, rows, delimiter=','):	4✔
170	response = HttpResponse(content_type='text/csv')	4✔
171	response['Content-Disposition'] = f'attachment; filename="{title}.csv"'	4✔
172
173	writer = csv.writer(response, delimiter=delimiter)	4✔
174	for row in rows:	4✔
175	writer.writerow(	4✔
176	['' if x is None else str(x) for x in row]
177	)
178	return response	4✔
179
180
181	def render_to_json(title, data, delimiter=','):	4✔
182	response = HttpResponse(json.dumps(data, indent=2), content_type='text/json')	4✔
183	response['Content-Disposition'] = f'attachment; filename="{title}.json"'	4✔
184	return response	4✔
185
186
187	def return_file_response(file_path, content_type):	4✔
188	file_abspath = Path(settings.MEDIA_ROOT) / file_path	4✔
189	if file_abspath.exists():	4✔
190	with file_abspath.open('rb') as fp:	4✔
191	response = HttpResponse(fp.read(), content_type=content_type)	4✔
192	response['Content-Disposition'] = 'attachment; filename=' + file_abspath.name	4✔
193	return response	4✔
194	else:
195	raise Http404	×
196
197
198	def sanitize_url(s):	4✔
199	# is used in the rdmo-app
200	try:	4✔
201	m = re.search('[a-z0-9-_]', s)	4✔
202	except TypeError:	4✔
203	s = ''	4✔
204	else:
205	if bool(m) is False:	4✔
206	s = ''	4✔
207	else:
208	s = re.sub(r'/+', '/', s)	4✔
209	return s	4✔
210
211
212	def get_plugin_python_paths(raise_exception=False):	4✔
213	plugin_paths = []	4✔
214	for python_path in settings.PLUGINS:	4✔
215	try:	4✔
216	import_string(python_path)	4✔
217	except (ImportError, ValueError) as e:	×
218	if raise_exception:	×
219	raise e from e	×
220	else:
221	plugin_paths.append(python_path)	4✔
222	return plugin_paths	4✔
223
224
225	def human2bytes(string):	4✔
226	if not string or string == '0':	4✔
227	return 0	4✔
228
229	m = re.match(r'([0-9.]+)\s*([A-Za-z]+)', string)	4✔
230	number, unit = float(m.group(1)), m.group(2).strip().lower()	4✔
231
232	conversion = HUMAN2BYTES_MAPPER[unit]	4✔
233	number = numberconversion['base']*(conversion['power'])	4✔
234	return number	4✔
235
236
237	def is_truthy(value):	4✔
238	return value is not None and (value is True or value.lower() in ['1', 't', 'true'])	4✔
239
240
241	def markdown2html(markdown_string):	4✔
242	# adoption of the normal markdown function
243	html = markdown(force_str(markdown_string)).strip()	4✔
244
245	# strip the outer paragraph
246	html = re.sub(r'^<p>(.*?)</p>$',r'\1', html)	4✔
247
248	# convert `[<string>]{<title>}` to <span title="<title>"><string></span> to allow for underlined tooltips
249	html = re.sub(	4✔
250	r'\[(.?)\]\{(.?)\}',
251	r'<span data-toggle="tooltip" data-placement="bottom" data-html="true" title="\2">\1</span>',
252	html
253	)
254
255	# convert everything after `{more}` to <span class="more"><string></span> to be shown/hidden on user input
256	show_string = _('show more')	4✔
257	hide_string = _('show less')	4✔
258	html = re.sub(	4✔
259	r'(\{more\})(.*?)$',
260	f'<span class="show-more" onclick="showMore(this)">... ({show_string})</span>'
261	r'<span class="more">\2</span>'
262	f'<span class="show-less" onclick="showLess(this)"> ({hide_string})</span>',
263	html
264	)
265
266	# textblocks (e.g. for help texts) can be injected into free text fields as small templates via Markdown
267	html = inject_textblocks(html)	4✔
268
269	return html	4✔
270
271
272	def inject_textblocks(html):	4✔
273	# loop over all strings between curly brackets, e.g. {{ test }}
274	for template_code in re.findall(r'{{(.*?)}}', html):	4✔
275	template_name = settings.MARKDOWN_TEMPLATES.get(template_code.strip())	×
276	if template_name:	×
277	html = re.sub('{{' + template_code + '}}', render_to_string(template_name), html)	×
278	return html	4✔
279
280
281	def parse_metadata(html):	4✔
282	metadata = None	4✔
283	pattern = re.compile(	4✔
284	'(<metadata>)(.*)(</metadata>)', re.MULTILINE \| re.DOTALL
285	)
286	m = re.search(pattern, html)	4✔
287	if bool(m) is True:	4✔
288	try:	4✔
289	metadata = json.loads(m.group(2))	4✔
290	except json.JSONDecodeError:	4✔
291	pass	4✔
292	else:
293	html = html.replace(m.group(0), '')	4✔
294	return metadata, html	4✔
295
296
297	def remove_double_newlines(string):	4✔
298	return re.sub(r'[\n]{2,}', '\n\n', string)	4✔
299
300
301	def remove_html_special_characters(string):	4✔
302	return re.sub(r'[<>&"\']', '', string)	4✔
303
304
305	def parse_date_from_string(date: str) -> datetime.date:	4✔
306	if not isinstance(date, str):	4✔
307	raise TypeError("date must be provided as string")	4✔
308
309	try:	4✔
310	# First, try standard ISO format (YYYY-MM-DD)
311	parsed_date = parse_date(date)	4✔
312	except ValueError as exc:	4✔
313	raise exc from exc	4✔
314
315	# If ISO parsing fails, try localized DATE_INPUT_FORMATS formats
316	if parsed_date is None:	4✔
317	for fmt in get_format('DATE_INPUT_FORMATS'):	4✔
318	try:	4✔
319	parsed_date = datetime.strptime(date, fmt).date()	4✔
320	break # Stop if parsing succeeds	4✔
321	except ValueError:	4✔
322	continue # Try the next format	4✔
323
324	# If still not parsed, raise an error
325	if not parsed_date:	4✔
326	raise ValueError(	4✔
327	f"Invalid date format for: {date}. Valid formats {get_format('DATE_INPUT_FORMATS')}"
328	)
329	return parsed_date	4✔

rdmorganiser / rdmo / 20428555173

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous