30161608849

Committed 25 Jul 2026 02:25PM UTC coverage: 15.303% (-0.008%) from 15.311%

Build # 30161608849

Build Type

Pull #2148

github

Committed by

web-flow

Commit Message

Merge d2b82a62f into 13afd939a

Pull Request Pull Request #2148: wasm: stream large media from Blob URLs instead of copying into RAM (Tier B)

Coverage Stats

4 of 179 new or added lines in 10 files covered. (2.23%)

7 existing lines in 5 files now uncovered.

30396 of 198627 relevant lines covered (15.3%)

983.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/plugins/score-plugin-media/Video/Thumbnailer.cpp

#include <Media/Libav.hpp>
#if SCORE_HAS_LIBAV

#include <Video/Thumbnailer.hpp>
#include <Video/VideoDecoder.hpp>

#include <ossia/detail/flicks.hpp>

#include <ossia-qt/invoke.hpp>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/pixdesc.h>
#include <libswscale/swscale.h>
}
#include <score/tools/Debug.hpp>

#include <ossia/detail/libav.hpp>

#include <QDebug>

#include <wobjectimpl.h>

W_OBJECT_IMPL(Video::VideoThumbnailer)
namespace Video
{

VideoThumbnailer::VideoThumbnailer(QString path)
{
  connect(
      this, &VideoThumbnailer::requestThumbnails, this, &VideoThumbnailer::onRequest,
      Qt::QueuedConnection);

  auto inputFile = path.toUtf8();
  if(auto hook = ossia::libav_custom_open())
    m_formatContext = hook(inputFile.data(), m_ioOwner, m_ioFree);

  if(!m_formatContext)
  {
    if(int err
       = avformat_open_input(&m_formatContext, inputFile.data(), nullptr, nullptr);
       err != 0)
    {
      char str[1500];
      qDebug() << "VideoThumbnailer: avformat_open_input failed"
               << av_make_error_string(str, 1499, err);

      SCORE_ASSERT(m_formatContext == nullptr);
      return;
    }
  }

  if(int err = avformat_find_stream_info(m_formatContext, nullptr); err < 0)
  {
    char str[1500];
    qDebug() << "VideoThumbnailer: avformat_find_stream_info failed"
             << av_make_error_string(str, 1499, err);

    avformat_close_input(&m_formatContext);
    m_formatContext = nullptr;
    if(m_ioOwner)
    {
      m_ioFree(m_ioOwner);
      m_ioOwner = nullptr;
      m_ioFree = nullptr;
    }
    return;
  }

  for(unsigned int i = 0; i < m_formatContext->nb_streams; i++)
  {
    if(m_formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
    {
      if(m_stream == -1)
      {
        m_stream = i;
        continue;
      }
    }
    m_formatContext->streams[i]->discard = AVDISCARD_ALL;
  }

  bool res = false;
  if(m_stream != -1)
  {
    const AVStream* stream = m_formatContext->streams[m_stream];
    const AVRational tb = m_formatContext->streams[m_stream]->time_base;
    dts_per_flicks = (tb.den / (tb.num * ossia::flicks_per_second<double>));
    flicks_per_dts = (tb.num * ossia::flicks_per_second<double>) / tb.den;

    m_codec = avcodec_find_decoder(stream->codecpar->codec_id);

    // Ensure we use a software decoder for thumbnails
    if(m_codec && (m_codec->capabilities & AV_CODEC_CAP_HARDWARE))
    {
      const AVCodec* sw = nullptr;
      void* iter = nullptr;
      while(auto* c = av_codec_iterate(&iter))
      {
        if(av_codec_is_decoder(c) && c->id == stream->codecpar->codec_id
           && !(c->capabilities & AV_CODEC_CAP_HARDWARE))
        {
          sw = c;
          break;
        }
      }
      if(sw)
        m_codec = sw;
    }

    if(m_codec)
    {
      pixel_format = (AVPixelFormat)stream->codecpar->format;
      width = stream->codecpar->width;
      height = stream->codecpar->height;
      fps = av_q2d(stream->avg_frame_rate);

      m_codecContext = avcodec_alloc_context3(nullptr);
      avcodec_parameters_to_context(m_codecContext, stream->codecpar);

      m_codecContext->pkt_timebase = stream->time_base;
      m_codecContext->codec_id = m_codec->id;
      int err = avcodec_open2(m_codecContext, m_codec, nullptr);
      res = !(err < 0);
      if(!res)
      {
        char str[1500];
        qDebug() << "VideoThumbnailer: avcodec_open2 failed"
                 << av_make_error_string(str, 1499, err);
        return;
      }

      width = m_codecContext->coded_width;
      height = m_codecContext->coded_height;
      if(height > 0 && width > 0)
      {
        m_aspect = double(width) / height;
      }
      else
      {
        qDebug() << "VideoThumbnailer: invalid video: width or height is 0";
        return;
      }

      smallHeight = 55;
      smallWidth = smallHeight * m_aspect;

      // Allocate a rescale context
      m_rescale = sws_getContext(
          this->width, this->height, this->pixel_format, smallWidth, smallHeight,
          AV_PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL);

      // Allocate a frame to do the RGB conversion
      m_rgb = av_frame_alloc();
      m_rgb->width = smallWidth;
      m_rgb->height = smallHeight;
      m_rgb->format = AV_PIX_FMT_RGB24;
      av_frame_get_buffer(m_rgb, 0);

      fps = av_q2d(stream->avg_frame_rate);
    }
  }
}

VideoThumbnailer::~VideoThumbnailer()
{
  if(m_rgb)
  {
    av_frame_free(&m_rgb);
    m_rgb = nullptr;
  }

  if(m_rescale)
  {
    sws_freeContext(m_rescale);
    m_rescale = nullptr;
  }

  if(m_codecContext)
  {
    avcodec_free_context(&m_codecContext);
    m_codecContext = nullptr;
    m_codec = nullptr;
  }

  if(m_formatContext)
  {
    avio_flush(m_formatContext->pb);
    avformat_flush(m_formatContext);
    avformat_close_input(&m_formatContext);
    m_formatContext = nullptr;
  }
  if(m_ioOwner)
  {
    m_ioFree(m_ioOwner);
    m_ioOwner = nullptr;
    m_ioFree = nullptr;
  }
}

void VideoThumbnailer::onRequest(int64_t req, QVector<int64_t> flicks)
{
  if(!m_codecContext)
    return;

  m_requests = std::move(flicks);
  m_requestIndex = req;
  m_currentIndex = 0;

  if(m_currentIndex < m_requests.size())
  {
    ossia::qt::run_async(this, [this] { processNext(); });
  }
}

QImage VideoThumbnailer::process(int64_t flicks)
{
  AVFramePointer res;
  // 1. Seek and decode
  {
    // Always seek backward to the nearest keyframe before the target.
    // Forward-only seeking fails when there is no keyframe at the exact target.
    if(!ossia::seek_to_flick(
           m_formatContext, m_codecContext, m_formatContext->streams[m_stream], flicks,
           AVSEEK_FLAG_BACKWARD | ossia::OSSIA_LIBAV_SEEK_ROUGH))
    {
      return {};
    }

    AVFramePointer frame{av_frame_alloc()};
    AVPacket* packet = av_packet_alloc();

    int attempts = 0;
    while(av_read_frame(m_formatContext, packet) >= 0 && attempts < 64)
    {
      if(packet->stream_index != m_stream)
      {
        av_packet_unref(packet);
        continue;
      }

      attempts++;
      int ret = avcodec_send_packet(m_codecContext, packet);
      av_packet_unref(packet);
      if(ret < 0 && ret != AVERROR(EAGAIN))
        break;

      ret = avcodec_receive_frame(m_codecContext, frame.get());
      if(ret == 0)
      {
        // For thumbnails, accept the first successfully decoded frame.
        // It will be at or near the requested position (close enough
        // for a 55px-high thumbnail).
        res = std::move(frame);
        m_last_dts = res->pkt_dts;
        break;
      }
      else if(ret != AVERROR(EAGAIN))
      {
        break;
      }
    }

    av_packet_free(&packet);
    if(!res)
      return {};
  }

  // 2. Resize
  QImage img{QSize(m_rgb->linesize[0] / 3, smallHeight), QImage::Format_RGB888};
  uint8_t* data[1] = {(uint8_t*)img.bits()};
  sws_scale(m_rescale, res->data, res->linesize, 0, this->height, data, m_rgb->linesize);

  return img;
}

void VideoThumbnailer::processNext()
{
  if(!m_codecContext)
    return;
  if(m_currentIndex >= m_requests.size())
    return;

  const auto flicks = m_requests[m_currentIndex];

  if(auto img = process(flicks); !img.isNull())
    thumbnailReady(m_requestIndex, flicks, std::move(img));

  m_currentIndex++;
  if(m_currentIndex < m_requests.size())
  {
    ossia::qt::run_async(this, [this] { processNext(); });
  }
}
}
#endif

1	#include <Media/Libav.hpp>
2	#if SCORE_HAS_LIBAV
3
4	#include <Video/Thumbnailer.hpp>
5	#include <Video/VideoDecoder.hpp>
6
7	#include <ossia/detail/flicks.hpp>
8
9	#include <ossia-qt/invoke.hpp>
10	extern "C" {
11	#include <libavcodec/avcodec.h>
12	#include <libavformat/avformat.h>
13	#include <libavutil/pixdesc.h>
14	#include <libswscale/swscale.h>
15	}
16	#include <score/tools/Debug.hpp>
17
18	#include <ossia/detail/libav.hpp>
19
20	#include <QDebug>
21
22	#include <wobjectimpl.h>
23
24	W_OBJECT_IMPL(Video::VideoThumbnailer)	×
25	namespace Video
26	{
27
28	VideoThumbnailer::VideoThumbnailer(QString path)	×
29	{	×
30	connect(	×
31	this, &VideoThumbnailer::requestThumbnails, this, &VideoThumbnailer::onRequest,
32	Qt::QueuedConnection);
33
34	auto inputFile = path.toUtf8();	×
NEW 35	if(auto hook = ossia::libav_custom_open())	×
NEW 36	m_formatContext = hook(inputFile.data(), m_ioOwner, m_ioFree);	×
37
NEW 38	if(!m_formatContext)	×
39	{
NEW 40	if(int err	×
NEW 41	= avformat_open_input(&m_formatContext, inputFile.data(), nullptr, nullptr);	×
NEW 42	err != 0)	×
43	{
44	char str[1500];
NEW 45	qDebug() << "VideoThumbnailer: avformat_open_input failed"	×
NEW 46	<< av_make_error_string(str, 1499, err);	×
47
NEW 48	SCORE_ASSERT(m_formatContext == nullptr);	×
NEW 49	return;	×
50	}
UNCOV 51	}	×
52
53	if(int err = avformat_find_stream_info(m_formatContext, nullptr); err < 0)	×
54	{
55	char str[1500];
56	qDebug() << "VideoThumbnailer: avformat_find_stream_info failed"	×
57	<< av_make_error_string(str, 1499, err);	×
58
59	avformat_close_input(&m_formatContext);	×
60	m_formatContext = nullptr;	×
NEW 61	if(m_ioOwner)	×
62	{
NEW 63	m_ioFree(m_ioOwner);	×
NEW 64	m_ioOwner = nullptr;	×
NEW 65	m_ioFree = nullptr;	×
NEW 66	}	×
UNCOV 67	return;	×
68	}
69
70	for(unsigned int i = 0; i < m_formatContext->nb_streams; i++)	×
71	{
72	if(m_formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)	×
73	{
74	if(m_stream == -1)	×
75	{
76	m_stream = i;	×
77	continue;	×
78	}
79	}	×
80	m_formatContext->streams[i]->discard = AVDISCARD_ALL;	×
81	}	×
82
83	bool res = false;	×
84	if(m_stream != -1)	×
85	{
86	const AVStream* stream = m_formatContext->streams[m_stream];	×
87	const AVRational tb = m_formatContext->streams[m_stream]->time_base;	×
88	dts_per_flicks = (tb.den / (tb.num * ossia::flicks_per_second<double>));	×
89	flicks_per_dts = (tb.num * ossia::flicks_per_second<double>) / tb.den;	×
90
91	m_codec = avcodec_find_decoder(stream->codecpar->codec_id);	×
92
93	// Ensure we use a software decoder for thumbnails
94	if(m_codec && (m_codec->capabilities & AV_CODEC_CAP_HARDWARE))	×
95	{
96	const AVCodec* sw = nullptr;	×
97	void* iter = nullptr;	×
98	while(auto* c = av_codec_iterate(&iter))	×
99	{
100	if(av_codec_is_decoder(c) && c->id == stream->codecpar->codec_id	×
101	&& !(c->capabilities & AV_CODEC_CAP_HARDWARE))	×
102	{
103	sw = c;	×
104	break;	×
105	}
106	}
107	if(sw)	×
108	m_codec = sw;	×
109	}	×
110
111	if(m_codec)	×
112	{
113	pixel_format = (AVPixelFormat)stream->codecpar->format;	×
114	width = stream->codecpar->width;	×
115	height = stream->codecpar->height;	×
116	fps = av_q2d(stream->avg_frame_rate);	×
117
118	m_codecContext = avcodec_alloc_context3(nullptr);	×
119	avcodec_parameters_to_context(m_codecContext, stream->codecpar);	×
120
121	m_codecContext->pkt_timebase = stream->time_base;	×
122	m_codecContext->codec_id = m_codec->id;	×
123	int err = avcodec_open2(m_codecContext, m_codec, nullptr);	×
124	res = !(err < 0);	×
125	if(!res)	×
126	{
127	char str[1500];
128	qDebug() << "VideoThumbnailer: avcodec_open2 failed"	×
129	<< av_make_error_string(str, 1499, err);	×
130	return;	×
131	}
132
133	width = m_codecContext->coded_width;	×
134	height = m_codecContext->coded_height;	×
135	if(height > 0 && width > 0)	×
136	{
137	m_aspect = double(width) / height;	×
138	}	×
139	else
140	{
141	qDebug() << "VideoThumbnailer: invalid video: width or height is 0";	×
142	return;	×
143	}
144
145	smallHeight = 55;	×
146	smallWidth = smallHeight * m_aspect;	×
147
148	// Allocate a rescale context
149	m_rescale = sws_getContext(	×
150	this->width, this->height, this->pixel_format, smallWidth, smallHeight,	×
151	AV_PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL);
152
153	// Allocate a frame to do the RGB conversion
154	m_rgb = av_frame_alloc();	×
155	m_rgb->width = smallWidth;	×
156	m_rgb->height = smallHeight;	×
157	m_rgb->format = AV_PIX_FMT_RGB24;	×
158	av_frame_get_buffer(m_rgb, 0);	×
159
160	fps = av_q2d(stream->avg_frame_rate);	×
161	}	×
162	}	×
163	}	×
164
165	VideoThumbnailer::~VideoThumbnailer()	×
166	{	×
167	if(m_rgb)	×
168	{
169	av_frame_free(&m_rgb);	×
170	m_rgb = nullptr;	×
171	}	×
172
173	if(m_rescale)	×
174	{
175	sws_freeContext(m_rescale);	×
176	m_rescale = nullptr;	×
177	}	×
178
179	if(m_codecContext)	×
180	{
181	avcodec_free_context(&m_codecContext);	×
182	m_codecContext = nullptr;	×
183	m_codec = nullptr;	×
184	}	×
185
186	if(m_formatContext)	×
187	{
188	avio_flush(m_formatContext->pb);	×
189	avformat_flush(m_formatContext);	×
190	avformat_close_input(&m_formatContext);	×
191	m_formatContext = nullptr;	×
192	}	×
NEW 193	if(m_ioOwner)	×
194	{
NEW 195	m_ioFree(m_ioOwner);	×
NEW 196	m_ioOwner = nullptr;	×
NEW 197	m_ioFree = nullptr;	×
NEW 198	}	×
UNCOV 199	}	×
200
201	void VideoThumbnailer::onRequest(int64_t req, QVector<int64_t> flicks)	×
202	{
203	if(!m_codecContext)	×
204	return;	×
205
206	m_requests = std::move(flicks);	×
207	m_requestIndex = req;	×
208	m_currentIndex = 0;	×
209
210	if(m_currentIndex < m_requests.size())	×
211	{
212	ossia::qt::run_async(this, [this] { processNext(); });	×
213	}	×
214	}	×
215
216	QImage VideoThumbnailer::process(int64_t flicks)	×
217	{
218	AVFramePointer res;	×
219	// 1. Seek and decode
220	{
221	// Always seek backward to the nearest keyframe before the target.
222	// Forward-only seeking fails when there is no keyframe at the exact target.
223	if(!ossia::seek_to_flick(	×
224	m_formatContext, m_codecContext, m_formatContext->streams[m_stream], flicks,	×
225	AVSEEK_FLAG_BACKWARD \| ossia::OSSIA_LIBAV_SEEK_ROUGH))
226	{
227	return {};	×
228	}
229
230	AVFramePointer frame{av_frame_alloc()};	×
231	AVPacket* packet = av_packet_alloc();	×
232
233	int attempts = 0;	×
234	while(av_read_frame(m_formatContext, packet) >= 0 && attempts < 64)	×
235	{
236	if(packet->stream_index != m_stream)	×
237	{
238	av_packet_unref(packet);	×
239	continue;	×
240	}
241
242	attempts++;	×
243	int ret = avcodec_send_packet(m_codecContext, packet);	×
244	av_packet_unref(packet);	×
245	if(ret < 0 && ret != AVERROR(EAGAIN))	×
246	break;	×
247
248	ret = avcodec_receive_frame(m_codecContext, frame.get());	×
249	if(ret == 0)	×
250	{
251	// For thumbnails, accept the first successfully decoded frame.
252	// It will be at or near the requested position (close enough
253	// for a 55px-high thumbnail).
254	res = std::move(frame);	×
255	m_last_dts = res->pkt_dts;	×
256	break;	×
257	}
258	else if(ret != AVERROR(EAGAIN))	×
259	{
260	break;	×
261	}
262	}
263
264	av_packet_free(&packet);	×
265	if(!res)	×
266	return {};	×
267	}	×
268
269	// 2. Resize
270	QImage img{QSize(m_rgb->linesize[0] / 3, smallHeight), QImage::Format_RGB888};	×
271	uint8_t* data[1] = {(uint8_t*)img.bits()};	×
272	sws_scale(m_rescale, res->data, res->linesize, 0, this->height, data, m_rgb->linesize);	×
273
274	return img;	×
275	}	×
276
277	void VideoThumbnailer::processNext()	×
278	{
279	if(!m_codecContext)	×
280	return;	×
281	if(m_currentIndex >= m_requests.size())	×
282	return;	×
283
284	const auto flicks = m_requests[m_currentIndex];	×
285
286	if(auto img = process(flicks); !img.isNull())	×
287	thumbnailReady(m_requestIndex, flicks, std::move(img));	×
288
289	m_currentIndex++;	×
290	if(m_currentIndex < m_requests.size())	×
291	{
292	ossia::qt::run_async(this, [this] { processNext(); });	×
293	}	×
294	}	×
295	}
296	#endif

ossia / score / 30161608849

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous