8017784387

Committed 23 Feb 2024 10:21AM UTC coverage: 70.319% (-0.2%) from 70.49%

Build # 8017784387

Build Type

Pull #1056

github

Committed by

web-flow

Commit Message

Merge 1c868b2a3 into f8e2c5049

Pull Request Pull Request #1056: [WIP] Fix remote encoding and other stuff

Run Details

46 of 156 new or added lines in 9 files covered. (29.49%)

6 existing lines in 6 files now uncovered.

9884 of 14056 relevant lines covered (70.32%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.21

/pod/video_encode_transcript/encode.py

"""Esup-Pod module to handle video encoding with CPU."""

from django.conf import settings
from webpush.models import PushInformation

from pod.video.models import Video
from .Encoding_video_model import Encoding_video_model
from .encoding_studio import encode_video_studio
from .models import EncodingLog

from pod.cut.models import CutVideo
from pod.dressing.models import Dressing
from pod.dressing.utils import get_dressing_input
from pod.main.tasks import task_start_encode, task_start_encode_studio
from .encoding_settings import FFMPEG_DRESSING_INPUT
from .utils import (
    change_encoding_step,
    check_file,
    add_encoding_log,
    send_email,
    send_email_encoding,
    send_notification_encoding,
    time_to_seconds,
)
import logging
import time
import threading

__license__ = "LGPL v3"
log = logging.getLogger(__name__)

USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False)
USE_NOTIFICATIONS = getattr(settings, "USE_NOTIFICATIONS", True)
if USE_TRANSCRIPTION:
    from . import transcript

    TRANSCRIPT_VIDEO = getattr(settings, "TRANSCRIPT_VIDEO", "start_transcript")

CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False)
EMAIL_ON_ENCODING_COMPLETION = getattr(settings, "EMAIL_ON_ENCODING_COMPLETION", True)

USE_REMOTE_ENCODING_TRANSCODING = getattr(
    settings, "USE_REMOTE_ENCODING_TRANSCODING", False
)
if USE_REMOTE_ENCODING_TRANSCODING:
    from .encoding_tasks import start_encoding_task

FFMPEG_DRESSING_INPUT = getattr(
    settings, "FFMPEG_DRESSING_INPUT", FFMPEG_DRESSING_INPUT
)

# ##########################################################################
# ENCODE VIDEO: THREAD TO LAUNCH ENCODE
# ##########################################################################

# Disable for the moment, will be reactivated in future version
"""
def start_remote_encode(video_id: int):
    # load module here to prevent circular import
    from .remote_encode import remote_encode_video

    log.info("START ENCODE VIDEO ID %s" % video_id)
    t = threading.Thread(target=remote_encode_video, args=[video_id])
    t.setDaemon(True)
    t.start()
"""


def start_encode(video_id: int, threaded=True):
    """Start local encoding."""
    if threaded:
        if CELERY_TO_ENCODE:
            task_start_encode.delay(video_id)
        else:
            log.info("START ENCODE VIDEO ID %s" % video_id)
            t = threading.Thread(target=encode_video, args=[video_id])
            t.setDaemon(True)
            t.start()
    else:
        encode_video(video_id)


def start_encode_studio(recording_id, video_output, videos, subtime, presenter):
    """Start local encoding."""
    if CELERY_TO_ENCODE:
        task_start_encode_studio.delay(
            recording_id, video_output, videos, subtime, presenter
        )
    else:
        log.info("START ENCODE VIDEO ID %s" % recording_id)
        t = threading.Thread(
            target=encode_video_studio,
            args=[recording_id, video_output, videos, subtime, presenter],
        )
        t.setDaemon(True)
        t.start()


"""
def start_studio_remote_encode(recording_id, video_output, videos, subtime, presenter):
    # load module here to prevent circular import
    from .remote_encode import remote_encode_studio

    log.info("START ENCODE RECORDING ID %s" % recording_id)
    t = threading.Thread(
        target=remote_encode_studio,
        args=[recording_id, video_output, videos, subtime, presenter],
    )
    t.setDaemon(True)
    t.start()
"""


def encode_video(video_id: int) -> None:
    """ENCODE VIDEO: MAIN FUNCTION."""
    start = "Start at: %s" % time.ctime()

    video_to_encode = Video.objects.get(id=video_id)
    video_to_encode.encoding_in_progress = True
    video_to_encode.save()

    if not check_file(video_to_encode.video.path):
        msg = "Wrong file or path:\n%s" % video_to_encode.video.path
        add_encoding_log(video_id, msg)
        change_encoding_step(video_id, -1, msg)
        send_email(msg, video_id)
        return

    change_encoding_step(video_id, 0, "start")
    # start and stop cut ?
    encoding_video = get_encoding_video(video_to_encode)
    encoding_video.add_encoding_log("start_time", "", True, start)
    change_encoding_step(video_id, 1, "remove old data")
    encoding_video.remove_old_data()

    change_encoding_step(video_id, 2, "start encoding")
    if USE_REMOTE_ENCODING_TRANSCODING:
        dressing = None
        dressing_input = ""
        if Dressing.objects.filter(videos=video_to_encode).exists():
            dressing = Dressing.objects.get(videos=video_to_encode).to_json()
            if dressing:
                dressing_input = get_dressing_input(
                    dressing,
                    FFMPEG_DRESSING_INPUT
                )
        start_encoding_task.delay(
            encoding_video.id,
            encoding_video.video_file,
            encoding_video.cutting_start,
            encoding_video.cutting_stop,
            json_dressing=dressing,
            dressing_input=dressing_input
        )
    else:
        encoding_video.start_encode()
        final_video = store_encoding_info(video_id, encoding_video)

        if encoding_video.error_encoding:
            enc_log, created = EncodingLog.objects.get_or_create(video=final_video)
            msg = "Error during video `%s` encoding." % video_id
            if created is False:
                msg += " See log at:\n%s" % enc_log.logfile.url

            send_email(msg, video_id)
        else:
            end_of_encoding(final_video)


def store_encoding_info(video_id: int, encoding_video: Encoding_video_model) -> Video:
    """Store all encoding file and informations from encoding tasks."""
    change_encoding_step(video_id, 3, "store encoding info")
    final_video = encoding_video.store_json_info()
    final_video.is_video = final_video.get_video_m4a() is None
    final_video.encoding_in_progress = False
    final_video.save()
    return final_video


def get_encoding_video(video_to_encode: Video) -> Encoding_video_model:
    """Get the encoding video object from video."""
    dressing = None
    dressing_input = ""
    if Dressing.objects.filter(videos=video_to_encode).exists():
        dressing = Dressing.objects.get(videos=video_to_encode).to_json()
        if dressing:
            dressing_input = get_dressing_input(
                dressing,
                FFMPEG_DRESSING_INPUT
            )

    if CutVideo.objects.filter(video=video_to_encode).exists():
        cut = CutVideo.objects.get(video=video_to_encode)
        cut_start = time_to_seconds(cut.start)
        cut_end = time_to_seconds(cut.end)
        encoding_video = Encoding_video_model(
            video_to_encode.id,
            video_to_encode.video.path,
            cut_start, cut_end,
            json_dressing=dressing,
            dressing_input=dressing_input
        )
        return encoding_video

    return Encoding_video_model(
        video_to_encode.id,
        video_to_encode.video.path,
        0,
        0,
        json_dressing=dressing,
        dressing_input=dressing_input
    )


def end_of_encoding(video: Video):
    """Notify user at the end of encoding & call transcription."""
    if (
        USE_NOTIFICATIONS
        and video.owner.owner.accepts_notifications
        and PushInformation.objects.filter(user=video.owner).exists()
    ):
        send_notification_encoding(video)
    if EMAIL_ON_ENCODING_COMPLETION:
        send_email_encoding(video)

    transcript_video(video.id)
    change_encoding_step(video.id, 0, "end of encoding")


def transcript_video(video_id: int):
    """Transcript video audio to text."""
    video = Video.objects.get(id=video_id)
    if USE_TRANSCRIPTION and video.transcript not in ["", "0", "1"]:
        change_encoding_step(video_id, 4, "transcript video")
        start_transcript_video = getattr(transcript, TRANSCRIPT_VIDEO)
        start_transcript_video(video_id, False)

1	"""Esup-Pod module to handle video encoding with CPU."""
2
3	from django.conf import settings	1✔
4	from webpush.models import PushInformation	1✔
5
6	from pod.video.models import Video	1✔
7	from .Encoding_video_model import Encoding_video_model	1✔
8	from .encoding_studio import encode_video_studio	1✔
9	from .models import EncodingLog	1✔
10
11	from pod.cut.models import CutVideo	1✔
12	from pod.dressing.models import Dressing	1✔
13	from pod.dressing.utils import get_dressing_input	1✔
14	from pod.main.tasks import task_start_encode, task_start_encode_studio	1✔
15	from .encoding_settings import FFMPEG_DRESSING_INPUT	1✔
16	from .utils import (	1✔
17	change_encoding_step,
18	check_file,
19	add_encoding_log,
20	send_email,
21	send_email_encoding,
22	send_notification_encoding,
23	time_to_seconds,
24	)
25	import logging	1✔
26	import time	1✔
27	import threading	1✔
28
29	__license__ = "LGPL v3"	1✔
30	log = logging.getLogger(__name__)	1✔
31
32	USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False)	1✔
33	USE_NOTIFICATIONS = getattr(settings, "USE_NOTIFICATIONS", True)	1✔
34	if USE_TRANSCRIPTION:	1✔
35	from . import transcript	×
36
37	TRANSCRIPT_VIDEO = getattr(settings, "TRANSCRIPT_VIDEO", "start_transcript")	×
38
39	CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False)	1✔
40	EMAIL_ON_ENCODING_COMPLETION = getattr(settings, "EMAIL_ON_ENCODING_COMPLETION", True)	1✔
41
42	USE_REMOTE_ENCODING_TRANSCODING = getattr(	1✔
43	settings, "USE_REMOTE_ENCODING_TRANSCODING", False
44	)
45	if USE_REMOTE_ENCODING_TRANSCODING:	1✔
46	from .encoding_tasks import start_encoding_task	×
47
48	FFMPEG_DRESSING_INPUT = getattr(	1✔
49	settings, "FFMPEG_DRESSING_INPUT", FFMPEG_DRESSING_INPUT
50	)
51
52	# ##########################################################################
53	# ENCODE VIDEO: THREAD TO LAUNCH ENCODE
54	# ##########################################################################
55
56	# Disable for the moment, will be reactivated in future version
57	"""
58	def start_remote_encode(video_id: int):
59	# load module here to prevent circular import
60	from .remote_encode import remote_encode_video
61
62	log.info("START ENCODE VIDEO ID %s" % video_id)
63	t = threading.Thread(target=remote_encode_video, args=[video_id])
64	t.setDaemon(True)
65	t.start()
66	"""
67
68
69	def start_encode(video_id: int, threaded=True):	1✔
70	"""Start local encoding."""
71	if threaded:	1✔
72	if CELERY_TO_ENCODE:	1✔
73	task_start_encode.delay(video_id)	×
74	else:
75	log.info("START ENCODE VIDEO ID %s" % video_id)	1✔
76	t = threading.Thread(target=encode_video, args=[video_id])	1✔
77	t.setDaemon(True)	1✔
78	t.start()	1✔
79	else:
80	encode_video(video_id)	×
81
82
83	def start_encode_studio(recording_id, video_output, videos, subtime, presenter):	1✔
84	"""Start local encoding."""
85	if CELERY_TO_ENCODE:	×
86	task_start_encode_studio.delay(	×
87	recording_id, video_output, videos, subtime, presenter
88	)
89	else:
90	log.info("START ENCODE VIDEO ID %s" % recording_id)	×
91	t = threading.Thread(	×
92	target=encode_video_studio,
93	args=[recording_id, video_output, videos, subtime, presenter],
94	)
95	t.setDaemon(True)	×
96	t.start()	×
97
98
99	"""
100	def start_studio_remote_encode(recording_id, video_output, videos, subtime, presenter):
101	# load module here to prevent circular import
102	from .remote_encode import remote_encode_studio
103
104	log.info("START ENCODE RECORDING ID %s" % recording_id)
105	t = threading.Thread(
106	target=remote_encode_studio,
107	args=[recording_id, video_output, videos, subtime, presenter],
108	)
109	t.setDaemon(True)
110	t.start()
111	"""
112
113
114	def encode_video(video_id: int) -> None:	1✔
115	"""ENCODE VIDEO: MAIN FUNCTION."""
116	start = "Start at: %s" % time.ctime()	1✔
117
118	video_to_encode = Video.objects.get(id=video_id)	1✔
119	video_to_encode.encoding_in_progress = True	1✔
120	video_to_encode.save()	1✔
121
122	if not check_file(video_to_encode.video.path):	1✔
123	msg = "Wrong file or path:\n%s" % video_to_encode.video.path	1✔
124	add_encoding_log(video_id, msg)	1✔
125	change_encoding_step(video_id, -1, msg)	1✔
126	send_email(msg, video_id)	1✔
127	return	1✔
128
129	change_encoding_step(video_id, 0, "start")	1✔
130	# start and stop cut ?
131	encoding_video = get_encoding_video(video_to_encode)	1✔
132	encoding_video.add_encoding_log("start_time", "", True, start)	1✔
133	change_encoding_step(video_id, 1, "remove old data")	1✔
134	encoding_video.remove_old_data()	1✔
135
136	change_encoding_step(video_id, 2, "start encoding")	1✔
137	if USE_REMOTE_ENCODING_TRANSCODING:	1✔
NEW 138	dressing = None	×
NEW 139	dressing_input = ""	×
NEW 140	if Dressing.objects.filter(videos=video_to_encode).exists():	×
NEW 141	dressing = Dressing.objects.get(videos=video_to_encode).to_json()	×
NEW 142	if dressing:	×
NEW 143	dressing_input = get_dressing_input(	×
144	dressing,
145	FFMPEG_DRESSING_INPUT
146	)
UNCOV 147	start_encoding_task.delay(	×
148	encoding_video.id,
149	encoding_video.video_file,
150	encoding_video.cutting_start,
151	encoding_video.cutting_stop,
152	json_dressing=dressing,
153	dressing_input=dressing_input
154	)
155	else:
156	encoding_video.start_encode()	1✔
157	final_video = store_encoding_info(video_id, encoding_video)	1✔
158
159	if encoding_video.error_encoding:	1✔
160	enc_log, created = EncodingLog.objects.get_or_create(video=final_video)	1✔
161	msg = "Error during video `%s` encoding." % video_id	1✔
162	if created is False:	1✔
163	msg += " See log at:\n%s" % enc_log.logfile.url	1✔
164
165	send_email(msg, video_id)	1✔
166	else:
167	end_of_encoding(final_video)	1✔
168
169
170	def store_encoding_info(video_id: int, encoding_video: Encoding_video_model) -> Video:	1✔
171	"""Store all encoding file and informations from encoding tasks."""
172	change_encoding_step(video_id, 3, "store encoding info")	1✔
173	final_video = encoding_video.store_json_info()	1✔
174	final_video.is_video = final_video.get_video_m4a() is None	1✔
175	final_video.encoding_in_progress = False	1✔
176	final_video.save()	1✔
177	return final_video	1✔
178
179
180	def get_encoding_video(video_to_encode: Video) -> Encoding_video_model:	1✔
181	"""Get the encoding video object from video."""
182	dressing = None	1✔
183	dressing_input = ""	1✔
184	if Dressing.objects.filter(videos=video_to_encode).exists():	1✔
NEW 185	dressing = Dressing.objects.get(videos=video_to_encode).to_json()	×
NEW 186	if dressing:	×
NEW 187	dressing_input = get_dressing_input(	×
188	dressing,
189	FFMPEG_DRESSING_INPUT
190	)
191
192	if CutVideo.objects.filter(video=video_to_encode).exists():	1✔
193	cut = CutVideo.objects.get(video=video_to_encode)	×
194	cut_start = time_to_seconds(cut.start)	×
195	cut_end = time_to_seconds(cut.end)	×
196	encoding_video = Encoding_video_model(	×
197	video_to_encode.id,
198	video_to_encode.video.path,
199	cut_start, cut_end,
200	json_dressing=dressing,
201	dressing_input=dressing_input
202	)
203	return encoding_video	×
204
205	return Encoding_video_model(	1✔
206	video_to_encode.id,
207	video_to_encode.video.path,
208	0,
209	0,
210	json_dressing=dressing,
211	dressing_input=dressing_input
212	)
213
214
215	def end_of_encoding(video: Video):	1✔
216	"""Notify user at the end of encoding & call transcription."""
217	if (	1✔
218	USE_NOTIFICATIONS
219	and video.owner.owner.accepts_notifications
220	and PushInformation.objects.filter(user=video.owner).exists()
221	):
222	send_notification_encoding(video)	×
223	if EMAIL_ON_ENCODING_COMPLETION:	1✔
224	send_email_encoding(video)	1✔
225
226	transcript_video(video.id)	1✔
227	change_encoding_step(video.id, 0, "end of encoding")	1✔
228
229
230	def transcript_video(video_id: int):	1✔
231	"""Transcript video audio to text."""
232	video = Video.objects.get(id=video_id)	1✔
233	if USE_TRANSCRIPTION and video.transcript not in ["", "0", "1"]:	1✔
234	change_encoding_step(video_id, 4, "transcript video")	×
235	start_transcript_video = getattr(transcript, TRANSCRIPT_VIDEO)	×
236	start_transcript_video(video_id, False)	×

EsupPortail / Esup-Pod / 8017784387

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous