• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

ossia / score / 30203585589

26 Jul 2026 01:12PM UTC coverage: 15.311%. Remained the same
30203585589

Pull #2150

github

web-flow
Merge 0c47fe9f6 into 13afd939a
Pull Request #2150: wasm: camera input through getUserMedia and WebCodecs

30392 of 198496 relevant lines covered (15.31%)

984.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

21.61
/src/plugins/score-plugin-gfx/Gfx/CameraDevice.cpp
1
#include "CameraDevice.hpp"
2

3
#include <Gfx/Graph/VideoNode.hpp>
4
#include <Video/CameraInput.hpp>
5
#include <Video/WebCameraInput.hpp>
6

7
#include <State/MessageListSerialization.hpp>
8
#include <State/Widgets/AddressFragmentLineEdit.hpp>
9

10
#include <Gfx/CameraDeviceEnumerator.hpp>
11
#include <Gfx/GfxApplicationPlugin.hpp>
12
#include <Media/LibavIntrospection.hpp>
13

14
#include <score/serialization/MimeVisitor.hpp>
15

16
#include <ossia-qt/name_utils.hpp>
17

18
#include <QComboBox>
19
#include <QFormLayout>
20
#include <QMenu>
21
#include <QMimeData>
22

23
#include <wobjectimpl.h>
24

25
extern "C" {
26
#include <libavcodec/avcodec.h>
27
#include <libavutil/pixdesc.h>
28
}
29

30
namespace Gfx
31
{
32
class CameraDevice final : public GfxInputDevice
33
{
34
  W_OBJECT(CameraDevice)
35
public:
36
  using GfxInputDevice::GfxInputDevice;
×
37
  ~CameraDevice();
38

39
private:
40
  void disconnect() override
×
41
  {
42
    Gfx::GfxInputDevice::disconnect();
×
43
    auto prev = std::move(m_dev);
×
44
    m_dev = {};
×
45
    deviceChanged(prev.get(), nullptr);
×
46
  }
×
47
  bool reconnect() override;
48
  ossia::net::device_base* getDevice() const override { return m_dev.get(); }
×
49

50
  Gfx::video_texture_input_protocol* m_protocol{};
×
51
  mutable std::unique_ptr<Gfx::video_texture_input_device> m_dev;
52
};
53
}
54

55
W_OBJECT_IMPL(Gfx::CameraDevice)
×
56

57
SCORE_SERALIZE_DATASTREAM_DEFINE(Gfx::CameraSettings);
×
58
namespace Gfx
59
{
60
void enumerateCameraDevices(std::function<void(CameraSettings, QString)> func);
61

62
CameraSettings findBestCameraMode()
×
63
{
64
  std::vector<CameraSettings> candidates;
×
65
  candidates.reserve(200);
×
66

67
  // 1. Collect all modes
68
  enumerateCameraDevices(
×
69
      [&](CameraSettings s, const auto&) { candidates.push_back(std::move(s)); });
×
70

71
  if(candidates.empty())
×
72
    return {};
×
73

74
  // 2. Define the scoring logic
75
  auto compute_score = [](const CameraSettings& s) {
×
76
    bool viable = true;
×
77

78
    // Not a very good default webcam
79
    if(s.device.contains("NDI Webcam", Qt::CaseInsensitive))
×
80
      viable = false;
×
81

82
    if(s.device.contains("leapmotion", Qt::CaseInsensitive))
×
83
      viable = false;
×
84

85
#if defined(_WIN32)
86
    // Win32 does not support depth well
87
    if(s.device.contains("depth", Qt::CaseInsensitive))
88
      viable = false;
89
#endif
90

91
    bool has_color = true;
×
92

93
    // If it's a raw format, check pixel format descriptors
94
    if(s.codec == AV_CODEC_ID_NONE || s.codec == AV_CODEC_ID_RAWVIDEO)
×
95
    {
96
      if(const AVPixFmtDescriptor* desc
×
97
         = av_pix_fmt_desc_get((AVPixelFormat)s.pixelformat))
×
98
      {
99
        // Reject if it explicitly looks like a bayer format,
100
        // has < 3 components, or is bitstream/paletted.
101
        if(desc->nb_components < 3 || (desc->flags & AV_PIX_FMT_FLAG_PAL))
×
102
          has_color = false;
×
103

104
        // Specific check for 1-bit/Monoblack
105
        if(desc->comp[0].depth <= 1)
×
106
          has_color = false;
×
107
      }
×
108
    }
×
109
    else
110
    {
111
      // compressed codec: likely to support color.
112
      has_color = true;
×
113
    }
114

115
    bool good_enough_fps = s.fps >= 30.0;
×
116
    int area = s.size.width() * s.size.height();
×
117
    double rawFps = s.fps;
×
118

119
    return std::make_tuple(
×
120
        viable,
121
        has_color,       // Priority 1: Must be color
122
        good_enough_fps, // Priority 2: Must be >= 30Hz
123
        area,            // Priority 3: Maximize Resolution
124
        rawFps           // Priority 4: Maximize Framerate
125
    );
126
  };
×
127

128
  // 3. Find the element with the highest score
129
  auto bestIt = std::max_element(
×
130
      candidates.begin(), candidates.end(),
×
131
      [&](const CameraSettings& a, const CameraSettings& b) {
×
132
    return compute_score(a) < compute_score(b);
×
133
  });
134

135
  return *bestIt;
×
136
}
×
137

138
CameraDevice::~CameraDevice() { }
×
139

140
bool CameraDevice::reconnect()
×
141
{
142
  disconnect();
×
143

144
  try
145
  {
146
    auto set = this->settings().deviceSpecificSettings.value<CameraSettings>();
×
147
    auto plug = m_ctx.findPlugin<DocumentPlugin>();
×
148
    if(plug)
×
149
    {
150
#if defined(__EMSCRIPTEN__)
151
      // "default" means: let the browser pick, which is also the only thing that
152
      // works before the user has granted camera permission (until then
153
      // enumerateDevices() hands out neither ids nor labels).
154
      auto cam = std::make_shared<::Video::WebCameraInput>();
155
      const auto device
156
          = (set.device == "default") ? std::string{} : set.device.toStdString();
157
      cam->load(device, set.size.width(), set.size.height(), set.fps);
158
#else
159
      auto cam = std::make_shared<::Video::CameraInput>();
×
160

161
      if(set.input == "default" && set.device == "default")
×
162
      {
163
        set = findBestCameraMode();
×
164
      }
×
165

166
      if(set.input.isEmpty() && set.device.isEmpty())
×
167
        return false;
×
168

169
      cam->load(
×
170
          set.input.toStdString(), set.device.toStdString(), set.size.width(),
×
171
          set.size.height(), set.fps, set.codec, set.pixelformat);
×
172
#endif
173

174
      m_protocol = new video_texture_input_protocol{std::move(cam), plug->exec};
×
175
      m_dev = std::make_unique<video_texture_input_device>(
×
176
          std::unique_ptr<ossia::net::protocol_base>(m_protocol),
×
177
          this->settings().name.toStdString());
×
178
      deviceChanged(nullptr, m_dev.get());
×
179
    }
×
180
    // TODOengine->reload(&proto);
181

182
    // setLogging_impl(Device::get_cur_logging(isLogging()));
183
  }
×
184
  catch(std::exception& e)
185
  {
186
    qDebug() << "Could not connect: " << e.what();
×
187
  }
×
188
  catch(...)
189
  {
190
    // TODO save the reason of the non-connection.
191
  }
×
192

193
  return connected();
×
194
}
×
195

196
class DefaultCameraEnumerator : public Device::DeviceEnumerator
197
{
198
public:
199
  void enumerate(std::function<void(const QString&, const Device::DeviceSettings&)> f)
×
200
      const override
201
  {
202
    Device::DeviceSettings s;
×
203
    s.name = "Camera";
×
204
    s.protocol = CameraProtocolFactory::static_concreteKey();
×
205
    CameraSettings set;
×
206
    set.input = "default";
×
207
    set.device = "default";
×
208

209
    s.deviceSpecificSettings = QVariant::fromValue(set);
×
210
    f("Default Camera", s);
×
211
  }
×
212
};
213

214
class CameraEnumerator : public Device::DeviceEnumerator
215
{
216
public:
217
  void enumerate(std::function<void(const QString&, const Device::DeviceSettings&)> f)
218
      const override
219
  {
220
    enumerateCameraDevices([&](const CameraSettings& set, QString name) {
221
      Device::DeviceSettings s;
222
      s.name = name;
223
      s.protocol = CameraProtocolFactory::static_concreteKey();
224
      s.deviceSpecificSettings = QVariant::fromValue(set);
225
      f(name, s);
226
    });
227
  }
228
};
229

230
class CustomCameraEnumerator : public Device::DeviceEnumerator
231
{
232
public:
233
  void enumerate(std::function<void(const QString&, const Device::DeviceSettings&)> f)
×
234
      const override
235
  {
236
    CameraSettings set;
×
237
    set.input = "";
×
238
    set.device = "";
×
239
    set.size = {};
×
240
    set.fps = {};
×
241

242
    set.codec = 0;
×
243
    set.pixelformat = -1;
×
244
    set.colorRange = 0;
×
245
    set.custom = true;
×
246

247
    Device::DeviceSettings s;
×
248
    s.name = "Custom";
×
249
    s.protocol = CameraProtocolFactory::static_concreteKey();
×
250
    s.deviceSpecificSettings = QVariant::fromValue(set);
×
251
    f(s.name, s);
×
252
  }
×
253
};
254

255
QString CameraProtocolFactory::prettyName() const noexcept
1✔
256
{
257
  return QObject::tr("Camera input");
1✔
258
}
259

260
QUrl CameraProtocolFactory::manual() const noexcept
×
261
{
262
  return QUrl("https://ossia.io/score-docs/devices/camera-device.html");
×
263
}
264

265
QString CameraProtocolFactory::category() const noexcept
×
266
{
267
  return StandardCategories::video_in;
×
268
}
269

270
Device::DeviceEnumerators
271
CameraProtocolFactory::getEnumerators(const score::DocumentContext& ctx) const
×
272
{
273
  Device::DeviceEnumerators enums;
×
274
  enums.push_back({"Default", new DefaultCameraEnumerator});
×
275
#if defined(__EMSCRIPTEN__) || defined(__linux__) || defined(__APPLE__) \
276
    || defined(_WIN32)
277
  auto devices = Gfx::make_camera_enumerator();
×
278
  devices->registerAllEnumerators(enums);
×
279
#else
280
  enums.push_back({"Cameras", new CameraEnumerator});
281
#endif
282
#if !defined(__EMSCRIPTEN__)
283
  enums.push_back({"Custom", new CustomCameraEnumerator});
×
284
#endif
285
  return enums;
×
286
}
×
287

288
Device::DeviceInterface* CameraProtocolFactory::makeDevice(
×
289
    const Device::DeviceSettings& settings, const Explorer::DeviceDocumentPlugin& plugin,
290
    const score::DocumentContext& ctx)
291
{
292
  return new CameraDevice(settings, ctx);
×
293
}
×
294

295
const Device::DeviceSettings& CameraProtocolFactory::defaultSettings() const noexcept
1✔
296
{
297
  static const Device::DeviceSettings settings = [&]() {
2✔
298
    Device::DeviceSettings s;
1✔
299
    s.protocol = concreteKey();
1✔
300
    s.name = "Camera";
1✔
301
    CameraSettings specif;
1✔
302
    s.deviceSpecificSettings = QVariant::fromValue(specif);
1✔
303
    return s;
1✔
304
  }();
1✔
305
  return settings;
1✔
306
}
307

308
Device::AddressDialog* CameraProtocolFactory::makeAddAddressDialog(
×
309
    const Device::DeviceInterface& dev, const score::DocumentContext& ctx,
310
    QWidget* parent)
311
{
312
  return nullptr;
×
313
}
314

315
Device::AddressDialog* CameraProtocolFactory::makeEditAddressDialog(
×
316
    const Device::AddressSettings& set, const Device::DeviceInterface& dev,
317
    const score::DocumentContext& ctx, QWidget* parent)
318
{
319
  return nullptr;
×
320
}
321

322
QVariant
323
CameraProtocolFactory::makeProtocolSpecificSettings(const VisitorVariant& visitor) const
2✔
324
{
325
  return makeProtocolSpecificSettings_T<CameraSettings>(visitor);
2✔
326
}
327

328
void CameraProtocolFactory::serializeProtocolSpecificSettings(
4✔
329
    const QVariant& data, const VisitorVariant& visitor) const
330
{
331
  serializeProtocolSpecificSettings_T<CameraSettings>(data, visitor);
4✔
332
}
4✔
333

334
bool CameraProtocolFactory::checkCompatibility(
×
335
    const Device::DeviceSettings& a, const Device::DeviceSettings& b) const noexcept
336
{
337
  return true;
×
338
}
339

340
class CameraSettingsWidget final : public Device::ProtocolSettingsWidget
341
{
342
public:
343
  explicit CameraSettingsWidget(QWidget* parent = nullptr);
344

345
  Device::DeviceSettings getSettings() const override;
346
  void setSettings(const Device::DeviceSettings& settings) override;
347

348
private:
349
  void setDefaults();
350
  QLineEdit* m_deviceNameEdit{};
×
351
  QLineEdit* m_device{};
×
352
  QComboBox* m_input{};
×
353

354
  QFormLayout* m_layout{};
×
355

356
  Device::DeviceSettings m_settings;
357
};
358

359
Device::ProtocolSettingsWidget* CameraProtocolFactory::makeSettingsWidget()
×
360
{
361
  return new CameraSettingsWidget;
×
362
}
×
363

364
CameraSettingsWidget::CameraSettingsWidget(QWidget* parent)
×
365
    : ProtocolSettingsWidget(parent)
×
366
{
×
367
  m_deviceNameEdit = new State::AddressFragmentLineEdit{this};
×
368
  checkForChanges(m_deviceNameEdit);
×
369

370
  m_device = new QLineEdit{this};
×
371
  m_input = new QComboBox{this};
×
372

373
  m_layout = new QFormLayout;
×
374
  m_layout->addRow(tr("Device Name"), m_deviceNameEdit);
×
375
  m_layout->addRow(tr("Device"), m_device);
×
376
  m_layout->addRow(tr("Input"), m_input);
×
377

378
#if QT_VERSION > QT_VERSION_CHECK(6, 4, 0)
379
  m_layout->setRowVisible(1, false);
×
380
  m_layout->setRowVisible(2, false);
×
381
#endif
382

383
  const auto& info = LibavIntrospection::instance();
×
384
  for(auto& demux : info.demuxers)
×
385
  {
386
    QString name = demux.format->name;
×
387
    if(demux.format->long_name && strlen(demux.format->long_name) > 0)
×
388
    {
389
      name += " (";
×
390
      name += demux.format->long_name;
×
391
      name += ")";
×
392
    }
×
393
    m_input->addItem(name, QVariant::fromValue((void*)demux.format));
×
394
  }
×
395

396
  setLayout(m_layout);
×
397

398
  setDefaults();
×
399
}
×
400

401
void CameraSettingsWidget::setDefaults()
×
402
{
403
  m_deviceNameEdit->setText("camera");
×
404
}
×
405

406
Device::DeviceSettings CameraSettingsWidget::getSettings() const
×
407
{
408
  Device::DeviceSettings s = m_settings;
×
409
  s.name = m_deviceNameEdit->text();
×
410
  s.protocol = CameraProtocolFactory::static_concreteKey();
×
411
  CameraSettings specif = s.deviceSpecificSettings.value<CameraSettings>();
×
412
  if(specif.custom)
×
413
  {
414
    specif.device = m_device->text();
×
415
    specif.input = m_input->currentText();
×
416
  }
×
417
  s.deviceSpecificSettings = QVariant::fromValue(std::move(specif));
×
418
  return s;
×
419
}
×
420

421
void CameraSettingsWidget::setSettings(const Device::DeviceSettings& settings)
×
422
{
423
  m_settings = settings;
×
424

425
  // Clean up the name a bit
426
  auto prettyName = settings.name;
×
427
  if(!prettyName.isEmpty())
×
428
  {
429
    prettyName = prettyName.split(':').front();
×
430
    prettyName = prettyName.split('(').front();
×
431
    prettyName.remove("/dev/");
×
432
    prettyName = prettyName.trimmed();
×
433
    ossia::net::sanitize_device_name(prettyName);
×
434
  }
×
435
  m_deviceNameEdit->setText(prettyName);
×
436

437
  const CameraSettings& set = settings.deviceSpecificSettings.value<CameraSettings>();
×
438
  m_device->setText(set.device);
×
439
  m_input->setCurrentText(set.input);
×
440

441
#if QT_VERSION > QT_VERSION_CHECK(6, 4, 0)
442
  m_layout->setRowVisible(1, set.custom);
×
443
  m_layout->setRowVisible(2, set.custom);
×
444
#endif
445
}
×
446

447
}
448

449
template <>
450
void DataStreamReader::read(const Gfx::CameraSettings& n)
2✔
451
{
452
  m_stream << n.input << n.device << n.size.width() << n.size.height() << n.fps
2✔
453
           << n.codec << n.pixelformat << n.colorRange << n.custom;
2✔
454
  insertDelimiter();
2✔
455
}
2✔
456

457
template <>
458
void DataStreamWriter::write(Gfx::CameraSettings& n)
1✔
459
{
460
  m_stream >> n.input >> n.device >> n.size.rwidth() >> n.size.rheight() >> n.fps
1✔
461
      >> n.codec >> n.pixelformat >> n.colorRange >> n.custom;
1✔
462
  checkDelimiter();
1✔
463
}
1✔
464

465
template <>
466
void JSONReader::read(const Gfx::CameraSettings& n)
2✔
467
{
468
  obj["Input"] = n.input;
2✔
469
  obj["Device"] = n.device;
2✔
470
  obj["Size"] = n.size;
2✔
471
  obj["FPS"] = n.fps;
2✔
472
  obj["Codec"] = n.codec;
2✔
473
  obj["PixelFormat"] = n.pixelformat;
2✔
474
  obj["ColorRange"] = n.colorRange;
2✔
475
  obj["Custom"] = n.custom;
2✔
476
}
2✔
477

478
template <>
479
void JSONWriter::write(Gfx::CameraSettings& n)
1✔
480
{
481
  n.input = obj["Input"].toString();
1✔
482
  n.device = obj["Device"].toString();
1✔
483
  n.size <<= obj["Size"];
1✔
484
  n.fps = obj["FPS"].toDouble();
1✔
485
  if(auto codec = obj.tryGet("Codec"))
1✔
486
    n.codec = codec->toInt();
1✔
487
  if(auto format = obj.tryGet("PixelFormat"))
1✔
488
    n.pixelformat = format->toInt();
1✔
489
  if(auto range = obj.tryGet("ColorRange"))
1✔
490
    n.colorRange = range->toInt();
1✔
491
  if(auto custom = obj.tryGet("Custom"))
1✔
492
    n.custom = custom->toBool();
1✔
493
}
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc