8702385459

Committed 16 Apr 2024 08:26AM UTC coverage: 70.485%. First build

Build # 8702385459

Build Type

Pull #1085

github

Committed by

web-flow

Commit Message

bad link videojs lang file in claim_record page (#1105)

Co-authored-by: Charneau Franck <franck.charneau@univ-lr.fr>

Pull Request Pull Request #1085: [DONE - FREEZE] Develop #3.6.0

Run Details

739 of 989 new or added lines in 37 files covered. (74.72%)

10570 of 14996 relevant lines covered (70.49%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.55

/pod/completion/admin.py

"""Admin pages for Esup-Pod Completion items."""

from django.conf import settings
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from pod.completion.models import Contributor
from pod.completion.models import Document
from pod.completion.models import Overlay
from pod.completion.models import Track
from pod.completion.models import EnrichModelQueue
from pod.completion.forms import DocumentAdminForm
from pod.completion.forms import TrackAdminForm
from django.contrib.sites.shortcuts import get_current_site
from django.contrib.sites.models import Site
from pod.video.models import Video

import subprocess
import webvtt
import threading
import os
import shutil

__FILEPICKER__ = False
if getattr(settings, "USE_PODFILE", False):
    __FILEPICKER__ = True

DEBUG = getattr(settings, "DEBUG", True)
TRANSCRIPTION_TYPE = getattr(settings, "TRANSCRIPTION_TYPE", "STT")
TRANSCRIPTION_MODEL_PARAM = getattr(settings, "TRANSCRIPTION_MODEL_PARAM", {})
MODEL_COMPILE_DIR = getattr(settings, "MODEL_COMPILE_DIR", "")


class ContributorInline(admin.TabularInline):
    model = Contributor
    readonly_fields = (
        "video",
        "name",
        "email_address",
        "role",
        "weblink",
    )
    extra = 0

    def has_add_permission(self, request, obj=None):
        return False


class ContributorAdmin(admin.ModelAdmin):
    list_display = (
        "name",
        "role",
        "video",
    )
    list_display_links = ("name",)
    list_filter = ("role",)
    search_fields = ["id", "name", "role", "video__title"]
    autocomplete_fields = ["video"]

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if (db_field.name) == "video":
            kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())
        return super().formfield_for_foreignkey(db_field, request, **kwargs)

    # class Media:
    #     css = {"all": ("css/pod.css",)}

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        if not request.user.is_superuser:
            qs = qs.filter(video__sites=get_current_site(request))
        return qs


admin.site.register(Contributor, ContributorAdmin)


class DocumentInline(admin.TabularInline):
    model = Document
    readonly_fields = (
        "video",
        "document",
    )
    extra = 0

    def has_add_permission(self, request, obj=None):
        return False


class DocumentAdmin(admin.ModelAdmin):
    if __FILEPICKER__:
        form = DocumentAdminForm
    list_display = (
        "document",
        "video",
    )
    list_display_links = ("document",)
    search_fields = ["id", "document__name", "video__title"]
    autocomplete_fields = ["video"]

    def get_queryset(self, request):
        """Get the queryset based on the request."""
        queryset = super().get_queryset(request)
        if not request.user.is_superuser:
            queryset = queryset.filter(video__sites=get_current_site(request))
        return queryset

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if (db_field.name) == "video":
            kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())
        return super().formfield_for_foreignkey(db_field, request, **kwargs)

    class Media:
        css = {
            "all": (
                # "bootstrap/dist/css/bootstrap.min.css",
                # "bootstrap/dist/css/bootstrap-grid.min.css",
                # "css/pod.css",
            )
        }
        js = (
            "podfile/js/filewidget.js",
            "js/main.js",
            "bootstrap/dist/js/bootstrap.min.js",
        )


admin.site.register(Document, DocumentAdmin)


class TrackInline(admin.TabularInline):
    model = Track
    readonly_fields = (
        "video",
        "kind",
        "lang",
        "src",
        "enrich_ready",
    )
    extra = 0

    def has_add_permission(self, request, obj=None):
        return False


class EnrichModelQueueAdmin(admin.ModelAdmin):
    list_display = (
        "title",
        "in_treatment",
    )
    list_filter = ("in_treatment",)


admin.site.register(EnrichModelQueue, EnrichModelQueueAdmin)


class TrackAdmin(admin.ModelAdmin):
    def debug(text):
        if DEBUG:
            print(text)

    def check_if_treatment_in_progress() -> bool:
        return EnrichModelQueue.objects.filter(in_treatment=True).exists()

    def write_into_kaldi_file(enrich_model_queue: EnrichModelQueue):
        with open(
            MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/db/extra.txt", "w"
        ) as f:
            f.write(enrich_model_queue.text)
        subprocess.call(
            [
                "docker",
                "run",
                "-v",
                MODEL_COMPILE_DIR + ":/kaldi/compile-model",
                "-it",
                "kaldi",
                enrich_model_queue.lang,
            ]
        )

    def copy_result_into_current_model(enrich_model_queue: EnrichModelQueue):
        from_path: str = (
            MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/exp/chain/tdnn/graph"
        )
        to_path: str = (
            TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
                enrich_model_queue.lang
            ]["model"]
            + "/graph"
        )
        if os.path.exists(to_path):
            shutil.rmtree(to_path)
        shutil.copytree(from_path, to_path)

        from_path: str = (
            MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/data/lang_test_rescore"
        )
        to_path: str = (
            TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
                enrich_model_queue.lang
            ]["model"]
            + "/rescore/"
        )
        if os.path.isfile(from_path + "/G.fst") and os.path.isfile(
            from_path + "/G.carpa"
        ):
            shutil.copy(from_path + "/G.fst", to_path)
            shutil.copy(from_path + "/G.carpa", to_path)

        from_path: str = (
            MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/exp/rnnlm_out"
        )
        to_path: str = (
            TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
                enrich_model_queue.lang
            ]["model"]
            + "/rnnlm/"
        )
        if os.path.exists(from_path):
            shutil.copy(from_path, to_path)

    def enrich_kaldi_model_launch():
        TrackAdmin.debug("enrich_kaldi_model")
        enrich_model_queue = EnrichModelQueue.objects.filter(model_type="VOSK").first()
        if enrich_model_queue is not None:
            enrich_model_queue.in_treatment = True
            enrich_model_queue.save()
            TrackAdmin.debug("start subprocess")
            TrackAdmin.write_into_kaldi_file(enrich_model_queue)
            TrackAdmin.debug("finish subprocess")
            TrackAdmin.debug("start copy result")
            TrackAdmin.copy_result_into_current_model(enrich_model_queue)
            TrackAdmin.debug("finish copy result")
            enrich_model_queue.delete()
            TrackAdmin.enrich_kaldi_model_launch()
            return
        else:
            TrackAdmin.debug("All queues have been completed !")
            return

    @admin.action(description=_("Enrich with selected subtitles"))
    def enrich_model(modeladmin, request, queryset):
        text = ""
        title = ""
        for query in list(queryset.all()):
            if title != "":
                title += " /-/ "
            title += query.video.title
            file = query.src.file
            for caption in webvtt.read(file.path):
                text += caption.text + " \n"
            query.enrich_ready = False
            query.save()

        EnrichModelQueue(
            title=title, text=text, lang=query.lang, model_type=TRANSCRIPTION_TYPE
        ).save()

        if not TrackAdmin.check_if_treatment_in_progress():
            if TRANSCRIPTION_TYPE == "VOSK":
                t = threading.Thread(target=TrackAdmin.enrich_kaldi_model_launch, args=[])
                t.setDaemon(True)
                t.start()

    if __FILEPICKER__:
        form = TrackAdminForm
    list_display = (
        "src",
        "kind",
        "video",
        "enrich_ready",
    )
    list_display_links = ("src",)
    list_filter = (
        "kind",
        "enrich_ready",
    )
    search_fields = [
        "id",
        "src__name",
        "kind",
        "video__title",
    ]
    autocomplete_fields = ["video"]
    actions = [enrich_model]

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        if not request.user.is_superuser:
            qs = qs.filter(video__sites=get_current_site(request))
        return qs

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if (db_field.name) == "video":
            kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())
        return super().formfield_for_foreignkey(db_field, request, **kwargs)

    class Media:
        css = {
            "all": (
                # "bootstrap/dist/css/bootstrap.min.css",
                # "bootstrap/dist/css/bootstrap-grid.min.css",
                # "css/pod.css",
            )
        }
        js = (
            "js/main.js",
            "bootstrap/dist/js/bootstrap.min.js",
        )


admin.site.register(Track, TrackAdmin)


class OverlayInline(admin.TabularInline):
    model = Overlay
    readonly_fields = (
        "video",
        "title",
        "time_start",
        "time_end",
        "content",
        "position",
        "background",
    )
    exclude = ("slug",)
    extra = 0

    def has_add_permission(self, request, obj=None):
        return False


class OverlayAdmin(admin.ModelAdmin):
    list_display = (
        "title",
        "video",
    )
    list_display_links = ("title",)
    search_fields = ["id", "title", "video__title"]
    autocomplete_fields = ["video"]

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        if not request.user.is_superuser:
            qs = qs.filter(video__sites=get_current_site(request))
        return qs

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if (db_field.name) == "video":
            kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())
        return super().formfield_for_foreignkey(db_field, request, **kwargs)

    # class Media:
    #     css = {"all": ("css/pod.css",)}


admin.site.register(Overlay, OverlayAdmin)

1	"""Admin pages for Esup-Pod Completion items."""
2
3	from django.conf import settings	1✔
4	from django.contrib import admin	1✔
5	from django.utils.translation import ugettext_lazy as _	1✔
6	from pod.completion.models import Contributor	1✔
7	from pod.completion.models import Document	1✔
8	from pod.completion.models import Overlay	1✔
9	from pod.completion.models import Track	1✔
10	from pod.completion.models import EnrichModelQueue	1✔
11	from pod.completion.forms import DocumentAdminForm	1✔
12	from pod.completion.forms import TrackAdminForm	1✔
13	from django.contrib.sites.shortcuts import get_current_site	1✔
14	from django.contrib.sites.models import Site	1✔
15	from pod.video.models import Video	1✔
16
17	import subprocess	1✔
18	import webvtt	1✔
19	import threading	1✔
20	import os	1✔
21	import shutil	1✔
22
23	__FILEPICKER__ = False	1✔
24	if getattr(settings, "USE_PODFILE", False):	1✔
25	__FILEPICKER__ = True	1✔
26
27	DEBUG = getattr(settings, "DEBUG", True)	1✔
28	TRANSCRIPTION_TYPE = getattr(settings, "TRANSCRIPTION_TYPE", "STT")	1✔
29	TRANSCRIPTION_MODEL_PARAM = getattr(settings, "TRANSCRIPTION_MODEL_PARAM", {})	1✔
30	MODEL_COMPILE_DIR = getattr(settings, "MODEL_COMPILE_DIR", "")	1✔
31
32
33	class ContributorInline(admin.TabularInline):	1✔
34	model = Contributor	1✔
35	readonly_fields = (	1✔
36	"video",
37	"name",
38	"email_address",
39	"role",
40	"weblink",
41	)
42	extra = 0	1✔
43
44	def has_add_permission(self, request, obj=None):	1✔
45	return False	×
46
47
48	class ContributorAdmin(admin.ModelAdmin):	1✔
49	list_display = (	1✔
50	"name",
51	"role",
52	"video",
53	)
54	list_display_links = ("name",)	1✔
55	list_filter = ("role",)	1✔
56	search_fields = ["id", "name", "role", "video__title"]	1✔
57	autocomplete_fields = ["video"]	1✔
58
59	def formfield_for_foreignkey(self, db_field, request, **kwargs):	1✔
60	if (db_field.name) == "video":	×
61	kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())	×
62	return super().formfield_for_foreignkey(db_field, request, **kwargs)	×
63
64	# class Media:
65	# css = {"all": ("css/pod.css",)}
66
67	def get_queryset(self, request):	1✔
68	qs = super().get_queryset(request)	×
69	if not request.user.is_superuser:	×
70	qs = qs.filter(video__sites=get_current_site(request))	×
71	return qs	×
72
73
74	admin.site.register(Contributor, ContributorAdmin)	1✔
75
76
77	class DocumentInline(admin.TabularInline):	1✔
78	model = Document	1✔
79	readonly_fields = (	1✔
80	"video",
81	"document",
82	)
83	extra = 0	1✔
84
85	def has_add_permission(self, request, obj=None):	1✔
86	return False	×
87
88
89	class DocumentAdmin(admin.ModelAdmin):	1✔
90	if __FILEPICKER__:	1✔
91	form = DocumentAdminForm	1✔
92	list_display = (	1✔
93	"document",
94	"video",
95	)
96	list_display_links = ("document",)	1✔
97	search_fields = ["id", "document__name", "video__title"]	1✔
98	autocomplete_fields = ["video"]	1✔
99
100	def get_queryset(self, request):	1✔
101	"""Get the queryset based on the request."""
102	queryset = super().get_queryset(request)	×
103	if not request.user.is_superuser:	×
104	queryset = queryset.filter(video__sites=get_current_site(request))	×
105	return queryset	×
106
107	def formfield_for_foreignkey(self, db_field, request, **kwargs):	1✔
108	if (db_field.name) == "video":	×
109	kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())	×
110	return super().formfield_for_foreignkey(db_field, request, **kwargs)	×
111
112	class Media:	1✔
113	css = {	1✔
114	"all": (
115	# "bootstrap/dist/css/bootstrap.min.css",
116	# "bootstrap/dist/css/bootstrap-grid.min.css",
117	# "css/pod.css",
118	)
119	}
120	js = (	1✔
121	"podfile/js/filewidget.js",
122	"js/main.js",
123	"bootstrap/dist/js/bootstrap.min.js",
124	)
125
126
127	admin.site.register(Document, DocumentAdmin)	1✔
128
129
130	class TrackInline(admin.TabularInline):	1✔
131	model = Track	1✔
132	readonly_fields = (	1✔
133	"video",
134	"kind",
135	"lang",
136	"src",
137	"enrich_ready",
138	)
139	extra = 0	1✔
140
141	def has_add_permission(self, request, obj=None):	1✔
142	return False	×
143
144
145	class EnrichModelQueueAdmin(admin.ModelAdmin):	1✔
146	list_display = (	1✔
147	"title",
148	"in_treatment",
149	)
150	list_filter = ("in_treatment",)	1✔
151
152
153	admin.site.register(EnrichModelQueue, EnrichModelQueueAdmin)	1✔
154
155
156	class TrackAdmin(admin.ModelAdmin):	1✔
157	def debug(text):	1✔
158	if DEBUG:	×
159	print(text)	×
160
161	def check_if_treatment_in_progress() -> bool:	1✔
162	return EnrichModelQueue.objects.filter(in_treatment=True).exists()	×
163
164	def write_into_kaldi_file(enrich_model_queue: EnrichModelQueue):	1✔
165	with open(	×
166	MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/db/extra.txt", "w"
167	) as f:
NEW 168	f.write(enrich_model_queue.text)	×
169	subprocess.call(	×
170	[
171	"docker",
172	"run",
173	"-v",
174	MODEL_COMPILE_DIR + ":/kaldi/compile-model",
175	"-it",
176	"kaldi",
177	enrich_model_queue.lang,
178	]
179	)
180
181	def copy_result_into_current_model(enrich_model_queue: EnrichModelQueue):	1✔
182	from_path: str = (	×
183	MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/exp/chain/tdnn/graph"
184	)
185	to_path: str = (	×
186	TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
187	enrich_model_queue.lang
188	]["model"]
189	+ "/graph"
190	)
191	if os.path.exists(to_path):	×
192	shutil.rmtree(to_path)	×
193	shutil.copytree(from_path, to_path)	×
194
195	from_path: str = (	×
196	MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/data/lang_test_rescore"
197	)
198	to_path: str = (	×
199	TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
200	enrich_model_queue.lang
201	]["model"]
202	+ "/rescore/"
203	)
204	if os.path.isfile(from_path + "/G.fst") and os.path.isfile(	×
205	from_path + "/G.carpa"
206	):
207	shutil.copy(from_path + "/G.fst", to_path)	×
208	shutil.copy(from_path + "/G.carpa", to_path)	×
209
210	from_path: str = (	×
211	MODEL_COMPILE_DIR + "/" + enrich_model_queue.lang + "/exp/rnnlm_out"
212	)
213	to_path: str = (	×
214	TRANSCRIPTION_MODEL_PARAM[enrich_model_queue.model_type][
215	enrich_model_queue.lang
216	]["model"]
217	+ "/rnnlm/"
218	)
219	if os.path.exists(from_path):	×
220	shutil.copy(from_path, to_path)	×
221
222	def enrich_kaldi_model_launch():	1✔
223	TrackAdmin.debug("enrich_kaldi_model")	×
NEW 224	enrich_model_queue = EnrichModelQueue.objects.filter(model_type="VOSK").first()	×
NEW 225	if enrich_model_queue is not None:	×
NEW 226	enrich_model_queue.in_treatment = True	×
NEW 227	enrich_model_queue.save()	×
228	TrackAdmin.debug("start subprocess")	×
NEW 229	TrackAdmin.write_into_kaldi_file(enrich_model_queue)	×
230	TrackAdmin.debug("finish subprocess")	×
231	TrackAdmin.debug("start copy result")	×
NEW 232	TrackAdmin.copy_result_into_current_model(enrich_model_queue)	×
233	TrackAdmin.debug("finish copy result")	×
NEW 234	enrich_model_queue.delete()	×
235	TrackAdmin.enrich_kaldi_model_launch()	×
236	return	×
237	else:
238	TrackAdmin.debug("All queues have been completed !")	×
239	return	×
240
241	@admin.action(description=_("Enrich with selected subtitles"))	1✔
242	def enrich_model(modeladmin, request, queryset):	1✔
243	text = ""	×
244	title = ""	×
245	for query in list(queryset.all()):	×
246	if title != "":	×
247	title += " /-/ "	×
248	title += query.video.title	×
249	file = query.src.file	×
250	for caption in webvtt.read(file.path):	×
251	text += caption.text + " \n"	×
252	query.enrich_ready = False	×
253	query.save()	×
254
255	EnrichModelQueue(	×
256	title=title, text=text, lang=query.lang, model_type=TRANSCRIPTION_TYPE
257	).save()
258
259	if not TrackAdmin.check_if_treatment_in_progress():	×
260	if TRANSCRIPTION_TYPE == "VOSK":	×
261	t = threading.Thread(target=TrackAdmin.enrich_kaldi_model_launch, args=[])	×
262	t.setDaemon(True)	×
263	t.start()	×
264
265	if __FILEPICKER__:	1✔
266	form = TrackAdminForm	1✔
267	list_display = (	1✔
268	"src",
269	"kind",
270	"video",
271	"enrich_ready",
272	)
273	list_display_links = ("src",)	1✔
274	list_filter = (	1✔
275	"kind",
276	"enrich_ready",
277	)
278	search_fields = [	1✔
279	"id",
280	"src__name",
281	"kind",
282	"video__title",
283	]
284	autocomplete_fields = ["video"]	1✔
285	actions = [enrich_model]	1✔
286
287	def get_queryset(self, request):	1✔
288	qs = super().get_queryset(request)	×
289	if not request.user.is_superuser:	×
290	qs = qs.filter(video__sites=get_current_site(request))	×
291	return qs	×
292
293	def formfield_for_foreignkey(self, db_field, request, **kwargs):	1✔
294	if (db_field.name) == "video":	×
295	kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())	×
296	return super().formfield_for_foreignkey(db_field, request, **kwargs)	×
297
298	class Media:	1✔
299	css = {	1✔
300	"all": (
301	# "bootstrap/dist/css/bootstrap.min.css",
302	# "bootstrap/dist/css/bootstrap-grid.min.css",
303	# "css/pod.css",
304	)
305	}
306	js = (	1✔
307	"js/main.js",
308	"bootstrap/dist/js/bootstrap.min.js",
309	)
310
311
312	admin.site.register(Track, TrackAdmin)	1✔
313
314
315	class OverlayInline(admin.TabularInline):	1✔
316	model = Overlay	1✔
317	readonly_fields = (	1✔
318	"video",
319	"title",
320	"time_start",
321	"time_end",
322	"content",
323	"position",
324	"background",
325	)
326	exclude = ("slug",)	1✔
327	extra = 0	1✔
328
329	def has_add_permission(self, request, obj=None):	1✔
330	return False	×
331
332
333	class OverlayAdmin(admin.ModelAdmin):	1✔
334	list_display = (	1✔
335	"title",
336	"video",
337	)
338	list_display_links = ("title",)	1✔
339	search_fields = ["id", "title", "video__title"]	1✔
340	autocomplete_fields = ["video"]	1✔
341
342	def get_queryset(self, request):	1✔
343	qs = super().get_queryset(request)	×
344	if not request.user.is_superuser:	×
345	qs = qs.filter(video__sites=get_current_site(request))	×
346	return qs	×
347
348	def formfield_for_foreignkey(self, db_field, request, **kwargs):	1✔
349	if (db_field.name) == "video":	×
350	kwargs["queryset"] = Video.objects.filter(sites=Site.objects.get_current())	×
351	return super().formfield_for_foreignkey(db_field, request, **kwargs)	×
352
353	# class Media:
354	# css = {"all": ("css/pod.css",)}
355
356
357	admin.site.register(Overlay, OverlayAdmin)	1✔

EsupPortail / Esup-Pod / 8702385459

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous