5423251035

pending completion

Build # 5423251035

Build Type

Pull #899

github

Committed by

web-flow

Commit Message

Merge 01a450543 into b8d45a116

Pull Request Pull Request #899: [WIP] Ptitloup/feature new encoding

Run Details

335 of 335 new or added lines in 8 files covered. (100.0%)

9029 of 12629 relevant lines covered (71.49%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.48

/pod/video_encode_transcript/transcript.py

from django.conf import settings
from django.core.files import File
from pod.completion.models import Track
from pod.main.tasks import task_start_transcript

from .utils import (
    send_email,
    send_email_transcript,
    change_encoding_step,
    add_encoding_log,
)
from ..video.models import Video
from .transcript_model import start_transcripting

import os
import time

from tempfile import NamedTemporaryFile

import threading
import logging

DEBUG = getattr(settings, "DEBUG", False)

if getattr(settings, "USE_PODFILE", False):
    __FILEPICKER__ = True
    from pod.podfile.models import CustomFileModel
    from pod.podfile.models import UserFolder
else:
    __FILEPICKER__ = False
    from pod.main.models import CustomFileModel

EMAIL_ON_TRANSCRIPTING_COMPLETION = getattr(
    settings, "EMAIL_ON_TRANSCRIPTING_COMPLETION", True
)
TRANSCRIPTION_MODEL_PARAM = getattr(settings, "TRANSCRIPTION_MODEL_PARAM", False)
USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False)
if USE_TRANSCRIPTION:
    TRANSCRIPTION_TYPE = getattr(settings, "TRANSCRIPTION_TYPE", "VOSK")
TRANSCRIPTION_NORMALIZE = getattr(settings, "TRANSCRIPTION_NORMALIZE", False)
CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False)

USE_DISTANT_ENCODING_TRANSCODING = getattr(
    settings,
    "USE_DISTANT_ENCODING_TRANSCODING",
    False
)
if USE_DISTANT_ENCODING_TRANSCODING:
    from .transcripting_tasks import start_transcripting_task

log = logging.getLogger(__name__)

"""
TO TEST IN THE SHELL -->
from pod.video.transcript import *
stt_model = get_model("fr")
msg, webvtt, all_text = main_stt_transcript(
    "/test/audio_192k_pod.mp3", # file
    177, # file duration
    stt_model # model stt loaded
)
print(webvtt)
"""


# ##########################################################################
# TRANSCRIPT VIDEO: THREAD TO LAUNCH TRANSCRIPT
# ##########################################################################
def start_transcript(video_id, threaded=True):
    if threaded:
        if CELERY_TO_ENCODE:
            task_start_transcript.delay(video_id)
        else:
            log.info("START TRANSCRIPT VIDEO %s" % video_id)
            t = threading.Thread(target=main_threaded_transcript, args=[video_id])
            t.setDaemon(True)
            t.start()
    else:
        main_threaded_transcript(video_id)


def main_threaded_transcript(video_to_encode_id):
    change_encoding_step(video_to_encode_id, 5, "transcripting audio")

    video_to_encode = Video.objects.get(id=video_to_encode_id)

    msg = ""
    lang = video_to_encode.transcript
    # check if TRANSCRIPTION_MODEL_PARAM [lang] exist
    if not TRANSCRIPTION_MODEL_PARAM[TRANSCRIPTION_TYPE].get(lang):
        msg += "\n no stt model found for lang:%s." % lang
        msg += "Please add it in TRANSCRIPTION_MODEL_PARAM."
        change_encoding_step(video_to_encode.id, -1, msg)
        send_email(msg, video_to_encode.id)
    else:
        mp3file = (
            video_to_encode.get_video_mp3().source_file
            if video_to_encode.get_video_mp3()
            else None
        )
        if mp3file is None:
            msg += "\n no mp3 file found for video :%s." % video_to_encode.id
            change_encoding_step(video_to_encode.id, -1, msg)
            send_email(msg, video_to_encode.id)
        else:
            mp3filepath = mp3file.path
            if USE_DISTANT_ENCODING_TRANSCODING:
                start_transcripting_task(
                    video_to_encode.id,
                    mp3filepath,
                    video_to_encode.duration,
                    lang
                )
            else:
                msg, webvtt = start_transcripting(
                    mp3filepath,
                    video_to_encode.duration,
                    lang
                )
                save_vtt_and_notify(video_to_encode, msg, webvtt)
    add_encoding_log(video_to_encode.id, msg)


def save_vtt_and_notify(video_to_encode, msg, webvtt):
    msg += saveVTT(video_to_encode, webvtt)
    change_encoding_step(video_to_encode.id, 0, "done")
    # envois mail fin transcription
    if EMAIL_ON_TRANSCRIPTING_COMPLETION:
        send_email_transcript(video_to_encode)
    add_encoding_log(video_to_encode.id, msg)


def saveVTT(video, webvtt):
    msg = "\nSAVE TRANSCRIPT WEBVTT : %s" % time.ctime()
    lang = video.main_lang
    temp_vtt_file = NamedTemporaryFile(suffix=".vtt")
    webvtt.save(temp_vtt_file.name)
    if webvtt.captions:
        msg += "\nstore vtt file in bdd with CustomFileModel model file field"
        if __FILEPICKER__:
            videodir, created = UserFolder.objects.get_or_create(
                name="%s" % video.slug, owner=video.owner
            )
            """
            previousSubtitleFile = CustomFileModel.objects.filter(
                name__startswith="subtitle_%s" % lang,
                folder=videodir,
                created_by=video.owner
            )
            """
            # for subt in previousSubtitleFile:
            #     subt.delete()
            subtitleFile, created = CustomFileModel.objects.get_or_create(
                name="subtitle_%s_%s" % (lang, time.strftime("%Y%m%d-%H%M%S")),
                folder=videodir,
                created_by=video.owner,
            )
            if subtitleFile.file and os.path.isfile(subtitleFile.file.path):
                os.remove(subtitleFile.file.path)
        else:
            subtitleFile, created = CustomFileModel.objects.get_or_create()

        subtitleFile.file.save(
            "subtitle_%s_%s.vtt" % (lang, time.strftime("%Y%m%d-%H%M%S")),
            File(temp_vtt_file),
        )
        msg += "\nstore vtt file in bdd with Track model src field"

        subtitleVtt, created = Track.objects.get_or_create(video=video, lang=lang)
        subtitleVtt.src = subtitleFile
        subtitleVtt.lang = lang
        subtitleVtt.save()
    else:
        msg += "\nERROR SUBTITLES Output size is 0"
    return msg

1	from django.conf import settings	1✔
2	from django.core.files import File	1✔
3	from pod.completion.models import Track	1✔
4	from pod.main.tasks import task_start_transcript	1✔
5
6	from .utils import (	1✔
7	send_email,
8	send_email_transcript,
9	change_encoding_step,
10	add_encoding_log,
11	)
12	from ..video.models import Video	1✔
13	from .transcript_model import start_transcripting	1✔
14
15	import os	1✔
16	import time	1✔
17
18	from tempfile import NamedTemporaryFile	1✔
19
20	import threading	1✔
21	import logging	1✔
22
23	DEBUG = getattr(settings, "DEBUG", False)	1✔
24
25	if getattr(settings, "USE_PODFILE", False):	1✔
26	__FILEPICKER__ = True	1✔
27	from pod.podfile.models import CustomFileModel	1✔
28	from pod.podfile.models import UserFolder	1✔
29	else:
30	__FILEPICKER__ = False	×
31	from pod.main.models import CustomFileModel	×
32
33	EMAIL_ON_TRANSCRIPTING_COMPLETION = getattr(	1✔
34	settings, "EMAIL_ON_TRANSCRIPTING_COMPLETION", True
35	)
36	TRANSCRIPTION_MODEL_PARAM = getattr(settings, "TRANSCRIPTION_MODEL_PARAM", False)	1✔
37	USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False)	1✔
38	if USE_TRANSCRIPTION:	1✔
39	TRANSCRIPTION_TYPE = getattr(settings, "TRANSCRIPTION_TYPE", "VOSK")	×
40	TRANSCRIPTION_NORMALIZE = getattr(settings, "TRANSCRIPTION_NORMALIZE", False)	1✔
41	CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False)	1✔
42
43	USE_DISTANT_ENCODING_TRANSCODING = getattr(	1✔
44	settings,
45	"USE_DISTANT_ENCODING_TRANSCODING",
46	False
47	)
48	if USE_DISTANT_ENCODING_TRANSCODING:	1✔
49	from .transcripting_tasks import start_transcripting_task	×
50
51	log = logging.getLogger(__name__)	1✔
52
53	"""
54	TO TEST IN THE SHELL -->
55	from pod.video.transcript import *
56	stt_model = get_model("fr")
57	msg, webvtt, all_text = main_stt_transcript(
58	"/test/audio_192k_pod.mp3", # file
59	177, # file duration
60	stt_model # model stt loaded
61	)
62	print(webvtt)
63	"""
64
65
66	# ##########################################################################
67	# TRANSCRIPT VIDEO: THREAD TO LAUNCH TRANSCRIPT
68	# ##########################################################################
69	def start_transcript(video_id, threaded=True):	1✔
70	if threaded:	×
71	if CELERY_TO_ENCODE:	×
72	task_start_transcript.delay(video_id)	×
73	else:
74	log.info("START TRANSCRIPT VIDEO %s" % video_id)	×
75	t = threading.Thread(target=main_threaded_transcript, args=[video_id])	×
76	t.setDaemon(True)	×
77	t.start()	×
78	else:
79	main_threaded_transcript(video_id)	×
80
81
82	def main_threaded_transcript(video_to_encode_id):	1✔
83	change_encoding_step(video_to_encode_id, 5, "transcripting audio")	×
84
85	video_to_encode = Video.objects.get(id=video_to_encode_id)	×
86
87	msg = ""	×
88	lang = video_to_encode.transcript	×
89	# check if TRANSCRIPTION_MODEL_PARAM [lang] exist
90	if not TRANSCRIPTION_MODEL_PARAM[TRANSCRIPTION_TYPE].get(lang):	×
91	msg += "\n no stt model found for lang:%s." % lang	×
92	msg += "Please add it in TRANSCRIPTION_MODEL_PARAM."	×
93	change_encoding_step(video_to_encode.id, -1, msg)	×
94	send_email(msg, video_to_encode.id)	×
95	else:
96	mp3file = (	×
97	video_to_encode.get_video_mp3().source_file
98	if video_to_encode.get_video_mp3()
99	else None
100	)
101	if mp3file is None:	×
102	msg += "\n no mp3 file found for video :%s." % video_to_encode.id	×
103	change_encoding_step(video_to_encode.id, -1, msg)	×
104	send_email(msg, video_to_encode.id)	×
105	else:
106	mp3filepath = mp3file.path	×
107	if USE_DISTANT_ENCODING_TRANSCODING:	×
108	start_transcripting_task(	×
109	video_to_encode.id,
110	mp3filepath,
111	video_to_encode.duration,
112	lang
113	)
114	else:
115	msg, webvtt = start_transcripting(	×
116	mp3filepath,
117	video_to_encode.duration,
118	lang
119	)
120	save_vtt_and_notify(video_to_encode, msg, webvtt)	×
121	add_encoding_log(video_to_encode.id, msg)	×
122
123
124	def save_vtt_and_notify(video_to_encode, msg, webvtt):	1✔
125	msg += saveVTT(video_to_encode, webvtt)	×
126	change_encoding_step(video_to_encode.id, 0, "done")	×
127	# envois mail fin transcription
128	if EMAIL_ON_TRANSCRIPTING_COMPLETION:	×
129	send_email_transcript(video_to_encode)	×
130	add_encoding_log(video_to_encode.id, msg)	×
131
132
133	def saveVTT(video, webvtt):	1✔
134	msg = "\nSAVE TRANSCRIPT WEBVTT : %s" % time.ctime()	×
135	lang = video.main_lang	×
136	temp_vtt_file = NamedTemporaryFile(suffix=".vtt")	×
137	webvtt.save(temp_vtt_file.name)	×
138	if webvtt.captions:	×
139	msg += "\nstore vtt file in bdd with CustomFileModel model file field"	×
140	if __FILEPICKER__:	×
141	videodir, created = UserFolder.objects.get_or_create(	×
142	name="%s" % video.slug, owner=video.owner
143	)
144	"""
145	previousSubtitleFile = CustomFileModel.objects.filter(
146	name__startswith="subtitle_%s" % lang,
147	folder=videodir,
148	created_by=video.owner
149	)
150	"""
151	# for subt in previousSubtitleFile:
152	# subt.delete()
153	subtitleFile, created = CustomFileModel.objects.get_or_create(	×
154	name="subtitle_%s_%s" % (lang, time.strftime("%Y%m%d-%H%M%S")),
155	folder=videodir,
156	created_by=video.owner,
157	)
158	if subtitleFile.file and os.path.isfile(subtitleFile.file.path):	×
159	os.remove(subtitleFile.file.path)	×
160	else:
161	subtitleFile, created = CustomFileModel.objects.get_or_create()	×
162
163	subtitleFile.file.save(	×
164	"subtitle_%s_%s.vtt" % (lang, time.strftime("%Y%m%d-%H%M%S")),
165	File(temp_vtt_file),
166	)
167	msg += "\nstore vtt file in bdd with Track model src field"	×
168
169	subtitleVtt, created = Track.objects.get_or_create(video=video, lang=lang)	×
170	subtitleVtt.src = subtitleFile	×
171	subtitleVtt.lang = lang	×
172	subtitleVtt.save()	×
173	else:
174	msg += "\nERROR SUBTITLES Output size is 0"	×
175	return msg	×

EsupPortail / Esup-Pod / 5423251035

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous