• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Open-Sn / opensn / 21158617767

15 Jan 2026 05:46AM UTC coverage: 74.437% (-0.03%) from 74.47%
21158617767

push

github

web-flow
Merge pull request #884 from wdhawkins/td_advance_hooks

Simplifications to time dependent source solver. Adding pre-advance and post-advance hooks

25 of 40 new or added lines in 3 files covered. (62.5%)

1 existing line in 1 file now uncovered.

18741 of 25177 relevant lines covered (74.44%)

66673796.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.65
/modules/linear_boltzmann_solvers/lbs_problem/lbs_problem.cc
1
// SPDX-FileCopyrightText: 2024 The OpenSn Authors <https://open-sn.github.io/opensn/>
2
// SPDX-License-Identifier: MIT
3

4
#include "modules/linear_boltzmann_solvers/lbs_problem/lbs_problem.h"
5
#include "modules/linear_boltzmann_solvers/lbs_problem/iterative_methods/wgs_context.h"
6
#include "modules/linear_boltzmann_solvers/lbs_problem/iterative_methods/ags_linear_solver.h"
7
#include "modules/linear_boltzmann_solvers/lbs_problem/point_source/point_source.h"
8
#include "modules/linear_boltzmann_solvers/lbs_problem/groupset/lbs_groupset.h"
9
#include "framework/math/spatial_discretization/finite_element/piecewise_linear/piecewise_linear_discontinuous.h"
10
#include "framework/field_functions/field_function_grid_based.h"
11
#include "framework/materials/multi_group_xs/multi_group_xs.h"
12
#include "framework/mesh/mesh_continuum/mesh_continuum.h"
13
#include "framework/utils/hdf_utils.h"
14
#include "framework/object_factory.h"
15
#include "framework/logging/log.h"
16
#include "framework/runtime.h"
17
#include "framework/data_types/allowable_range.h"
18
#include "caliper/cali.h"
19
#include <algorithm>
20
#include <iomanip>
21
#include <fstream>
22
#include <cstring>
23
#include <cassert>
24
#include <memory>
25
#include <stdexcept>
26
#include <sys/stat.h>
27

28
namespace opensn
29
{
30

31
LBSProblem::LBSProblem(std::string name, std::shared_ptr<MeshContinuum> grid)
×
32
  : Problem(std::move(name)), grid_(std::move(grid)), use_gpus_(false)
×
33
{
34
}
×
35

36
InputParameters
37
LBSProblem::GetInputParameters()
362✔
38
{
39
  InputParameters params = Problem::GetInputParameters();
362✔
40

41
  params.ChangeExistingParamToOptional("name", "LBSProblem");
724✔
42

43
  params.AddRequiredParameter<std::shared_ptr<MeshContinuum>>("mesh", "Mesh");
724✔
44

45
  params.AddRequiredParameter<size_t>("num_groups", "The total number of groups within the solver");
724✔
46

47
  params.AddRequiredParameterArray("groupsets",
724✔
48
                                   "An array of blocks each specifying the input parameters for a "
49
                                   "<TT>LBSGroupset</TT>.");
50
  params.LinkParameterToBlock("groupsets", "LBSGroupset");
724✔
51

52
  params.AddRequiredParameterArray("xs_map",
724✔
53
                                   "Cross-section map from block IDs to cross-section objects.");
54

55
  params.AddOptionalParameterArray<std::shared_ptr<VolumetricSource>>(
724✔
56
    "volumetric_sources", {}, "An array of handles to volumetric sources.");
57

58
  params.AddOptionalParameterArray<std::shared_ptr<PointSource>>(
724✔
59
    "point_sources", {}, "An array of point sources.");
60

61
  params.AddOptionalParameterArray(
724✔
62
    "boundary_conditions", {}, "An array containing tables for each boundary specification.");
63
  params.LinkParameterToBlock("boundary_conditions", "BoundaryOptionsBlock");
724✔
64

65
  params.AddOptionalParameterBlock(
724✔
66
    "options", ParameterBlock(), "Block of options. See <TT>OptionsBlock</TT>.");
724✔
67
  params.LinkParameterToBlock("options", "OptionsBlock");
724✔
68

69
  params.AddOptionalParameter("use_gpus", false, "Offload the sweep computation to GPUs.");
724✔
70

71
  params.AddOptionalParameter(
724✔
72
    "time_dependent", false, "Flag indicating whether the problem is time dependent.");
73
  return params;
362✔
74
}
×
75

76
LBSProblem::LBSProblem(const InputParameters& params)
362✔
77
  : Problem(params),
78
    time_dependent_(params.GetParamValue<bool>("time_dependent")),
362✔
79
    num_groups_(params.GetParamValue<size_t>("num_groups")),
362✔
80
    grid_(params.GetSharedPtrParam<MeshContinuum>("mesh")),
362✔
81
    use_gpus_(params.GetParamValue<bool>("use_gpus"))
1,448✔
82
{
83
  // Check system for GPU acceleration
84
  if (use_gpus_)
362✔
85
  {
86
    if (time_dependent_)
×
87
      throw std::invalid_argument(GetName() +
×
88
                                  ": Time dependent problems are not supported on GPUs.");
×
89
#ifdef __OPENSN_USE_CUDA__
90
    CheckCapableDevices();
91
#else
92
    throw std::invalid_argument(
×
93
      GetName() + ": GPU support was requested, but OpenSn was built without CUDA enabled.");
×
94
#endif // __OPENSN_USE_CUDA__
95
  }
96

97
  // Initialize options
98
  if (params.IsParameterValid("options"))
362✔
99
  {
100
    auto options_params = LBSProblem::GetOptionsBlock();
202✔
101
    options_params.AssignParameters(params.GetParam("options"));
204✔
102
    SetOptions(options_params);
200✔
103
  }
202✔
104

105
  // Set geometry type
106
  geometry_type_ = grid_->GetGeometryType();
360✔
107
  if (geometry_type_ == GeometryType::INVALID)
360✔
108
    throw std::runtime_error(GetName() + ": Invalid geometry type.");
×
109

110
  // Set boundary conditions
111
  if (params.Has("boundary_conditions"))
360✔
112
  {
113
    const auto& bcs = params.GetParam("boundary_conditions");
360✔
114
    bcs.RequireBlockTypeIs(ParameterBlockType::ARRAY);
360✔
115
    for (size_t b = 0; b < bcs.GetNumParameters(); ++b)
1,098✔
116
    {
117
      auto bndry_params = GetBoundaryOptionsBlock();
738✔
118
      bndry_params.AssignParameters(bcs.GetParam(b));
738✔
119
      SetBoundaryOptions(bndry_params);
738✔
120
    }
738✔
121
  }
122

123
  InitializeGroupsets(params);
360✔
124
  InitializeSources(params);
360✔
125
  InitializeXSmapAndDensities(params);
360✔
126
  InitializeMaterials();
360✔
127
}
380✔
128

129
LBSOptions&
130
LBSProblem::GetOptions()
20,071✔
131
{
132
  return options_;
20,071✔
133
}
134

135
const LBSOptions&
136
LBSProblem::GetOptions() const
576,490,930✔
137
{
138
  return options_;
576,490,930✔
139
}
140

141
double
142
LBSProblem::GetTime() const
110,944✔
143
{
144
  return time_;
110,944✔
145
}
146

147
void
148
LBSProblem::SetTime(double time)
1,500✔
149
{
150
  time_ = time;
1,500✔
151
}
1,500✔
152

153
void
154
LBSProblem::SetTimeStep(double dt)
892✔
155
{
156
  if (dt <= 0.0)
892✔
NEW
157
    throw std::runtime_error(GetName() + " dt must be greater than zero.");
×
158
  dt_ = dt;
892✔
159
}
892✔
160

161
double
162
LBSProblem::GetTimeStep() const
1,709,899,124✔
163
{
164
  return dt_;
1,709,899,124✔
165
}
166

167
bool
168
LBSProblem::IsTimeDependent() const
37,695✔
169
{
170
  return time_dependent_;
37,695✔
171
}
172

173
void
174
LBSProblem::SetTheta(double theta)
40✔
175
{
176
  if (theta < 0.0 or theta > 1.0)
40✔
NEW
177
    throw std::runtime_error(GetName() + " theta must be between 0.0 and 1.0.");
×
178
  theta_ = theta;
40✔
179
}
40✔
180

181
double
182
LBSProblem::GetTheta() const
1,709,899,100✔
183
{
184
  return theta_;
1,709,899,100✔
185
}
186

187
GeometryType
188
LBSProblem::GetGeometryType() const
4✔
189
{
190
  return geometry_type_;
4✔
191
}
192

193
size_t
194
LBSProblem::GetNumMoments() const
187,497✔
195
{
196
  return num_moments_;
187,497✔
197
}
198

199
unsigned int
200
LBSProblem::GetMaxCellDOFCount() const
419✔
201
{
202
  return max_cell_dof_count_;
419✔
203
}
204

205
unsigned int
206
LBSProblem::GetMinCellDOFCount() const
419✔
207
{
208
  return min_cell_dof_count_;
419✔
209
}
210

211
bool
212
LBSProblem::UseGPUs() const
374✔
213
{
214
  return use_gpus_;
374✔
215
}
216

217
size_t
218
LBSProblem::GetNumGroups() const
74,237✔
219
{
220
  return num_groups_;
74,237✔
221
}
222

223
unsigned int
224
LBSProblem::GetScatteringOrder() const
4✔
225
{
226
  return scattering_order_;
4✔
227
}
228

229
size_t
230
LBSProblem::GetNumPrecursors() const
×
231
{
232
  return num_precursors_;
×
233
}
234

235
size_t
236
LBSProblem::GetMaxPrecursorsPerMaterial() const
8✔
237
{
238
  return max_precursors_per_material_;
8✔
239
}
240

241
const std::vector<LBSGroup>&
242
LBSProblem::GetGroups() const
194,753✔
243
{
244
  return groups_;
194,753✔
245
}
246

247
std::vector<LBSGroupset>&
248
LBSProblem::GetGroupsets()
7,312,291✔
249
{
250
  return groupsets_;
7,312,291✔
251
}
252

253
const std::vector<LBSGroupset>&
254
LBSProblem::GetGroupsets() const
×
255
{
256
  return groupsets_;
×
257
}
258

259
void
260
LBSProblem::AddPointSource(std::shared_ptr<PointSource> point_source)
×
261
{
262
  point_sources_.push_back(point_source);
×
263
  if (discretization_)
×
264
    point_sources_.back()->Initialize(*this);
×
265
}
×
266

267
void
268
LBSProblem::ClearPointSources()
×
269
{
270
  point_sources_.clear();
×
271
}
×
272

273
const std::vector<std::shared_ptr<PointSource>>&
274
LBSProblem::GetPointSources() const
8,332✔
275
{
276
  return point_sources_;
8,332✔
277
}
278

279
void
280
LBSProblem::AddVolumetricSource(std::shared_ptr<VolumetricSource> volumetric_source)
16✔
281
{
282
  volumetric_sources_.push_back(volumetric_source);
16✔
283
  if (discretization_)
16✔
284
    volumetric_sources_.back()->Initialize(*this);
16✔
285
}
16✔
286

287
void
288
LBSProblem::ClearVolumetricSources()
4✔
289
{
290
  volumetric_sources_.clear();
4✔
291
}
4✔
292

293
const std::vector<std::shared_ptr<VolumetricSource>>&
294
LBSProblem::GetVolumetricSources() const
8,332✔
295
{
296
  return volumetric_sources_;
8,332✔
297
}
298

299
void
300
LBSProblem::ClearBoundaries()
×
301
{
302
  boundary_preferences_.clear();
×
303
}
×
304

305
const BlockID2XSMap&
306
LBSProblem::GetBlockID2XSMap() const
7,040✔
307
{
308
  return block_id_to_xs_map_;
7,040✔
309
}
310

311
std::shared_ptr<MeshContinuum>
312
LBSProblem::GetGrid() const
255,589✔
313
{
314
  return grid_;
255,589✔
315
}
316

317
const SpatialDiscretization&
318
LBSProblem::GetSpatialDiscretization() const
83,028✔
319
{
320
  return *discretization_;
83,028✔
321
}
322

323
const std::vector<UnitCellMatrices>&
324
LBSProblem::GetUnitCellMatrices() const
7,790✔
325
{
326
  return unit_cell_matrices_;
7,790✔
327
}
328

329
const std::map<uint64_t, UnitCellMatrices>&
330
LBSProblem::GetUnitGhostCellMatrices() const
17✔
331
{
332
  return unit_ghost_cell_matrices_;
17✔
333
}
334

335
std::vector<CellLBSView>&
336
LBSProblem::GetCellTransportViews()
116,494✔
337
{
338
  return cell_transport_views_;
116,494✔
339
}
340

341
const std::vector<CellLBSView>&
342
LBSProblem::GetCellTransportViews() const
166,416✔
343
{
344
  return cell_transport_views_;
166,416✔
345
}
346

347
const UnknownManager&
348
LBSProblem::GetUnknownManager() const
26,927✔
349
{
350
  return flux_moments_uk_man_;
26,927✔
351
}
352

353
size_t
354
LBSProblem::GetLocalNodeCount() const
3,088✔
355
{
356
  return local_node_count_;
3,088✔
357
}
358

359
size_t
360
LBSProblem::GetGlobalNodeCount() const
2,256✔
361
{
362
  return global_node_count_;
2,256✔
363
}
364

365
std::vector<double>&
366
LBSProblem::GetQMomentsLocal()
58,813✔
367
{
368
  return q_moments_local_;
58,813✔
369
}
370

371
const std::vector<double>&
372
LBSProblem::GetQMomentsLocal() const
×
373
{
374
  return q_moments_local_;
×
375
}
376

377
std::vector<double>&
378
LBSProblem::GetExtSrcMomentsLocal()
4✔
379
{
380
  return ext_src_moments_local_;
4✔
381
}
382

383
const std::vector<double>&
384
LBSProblem::GetExtSrcMomentsLocal() const
55,472✔
385
{
386
  return ext_src_moments_local_;
55,472✔
387
}
388

389
std::vector<double>&
390
LBSProblem::GetPhiOldLocal()
94,103✔
391
{
392
  return phi_old_local_;
94,103✔
393
}
394

395
const std::vector<double>&
396
LBSProblem::GetPhiOldLocal() const
×
397
{
398
  return phi_old_local_;
×
399
}
400

401
std::vector<double>&
402
LBSProblem::GetPhiNewLocal()
79,722✔
403
{
404
  return phi_new_local_;
79,722✔
405
}
406

407
const std::vector<double>&
408
LBSProblem::GetPhiNewLocal() const
×
409
{
410
  return phi_new_local_;
×
411
}
412

413
std::vector<double>&
414
LBSProblem::GetPrecursorsNewLocal()
16✔
415
{
416
  return precursor_new_local_;
16✔
417
}
418

419
const std::vector<double>&
420
LBSProblem::GetPrecursorsNewLocal() const
×
421
{
422
  return precursor_new_local_;
×
423
}
424

425
std::vector<double>&
426
LBSProblem::GetDensitiesLocal()
669✔
427
{
428
  return densities_local_;
669✔
429
}
430

431
const std::vector<double>&
432
LBSProblem::GetDensitiesLocal() const
55,472✔
433
{
434
  return densities_local_;
55,472✔
435
}
436

437
SetSourceFunction
438
LBSProblem::GetActiveSetSourceFunction() const
4,162✔
439
{
440
  return active_set_source_function_;
4,162✔
441
}
442

443
std::shared_ptr<AGSLinearSolver>
444
LBSProblem::GetAGSSolver()
344✔
445
{
446
  return ags_solver_;
344✔
447
}
448

449
std::vector<std::shared_ptr<LinearSolver>>&
450
LBSProblem::GetWGSSolvers()
113✔
451
{
452
  return wgs_solvers_;
113✔
453
}
454

455
WGSContext&
456
LBSProblem::GetWGSContext(int groupset_id)
11,532✔
457
{
458
  auto& wgs_solver = wgs_solvers_[groupset_id];
11,532✔
459
  auto raw_context = wgs_solver->GetContext();
11,532✔
460
  auto wgs_context_ptr = std::dynamic_pointer_cast<WGSContext>(raw_context);
11,532✔
461
  OpenSnLogicalErrorIf(not wgs_context_ptr, "Failed to cast WGSContext");
11,532✔
462
  return *wgs_context_ptr;
11,532✔
463
}
23,064✔
464

465
std::map<uint64_t, BoundaryPreference>&
466
LBSProblem::GetBoundaryPreferences()
4✔
467
{
468
  return boundary_preferences_;
4✔
469
}
470

471
std::pair<size_t, size_t>
472
LBSProblem::GetNumPhiIterativeUnknowns()
×
473
{
474
  const auto& sdm = *discretization_;
×
475
  const size_t num_local_phi_dofs = sdm.GetNumLocalDOFs(flux_moments_uk_man_);
×
476
  const size_t num_global_phi_dofs = sdm.GetNumGlobalDOFs(flux_moments_uk_man_);
×
477

478
  return {num_local_phi_dofs, num_global_phi_dofs};
×
479
}
480

481
size_t
482
LBSProblem::MapPhiFieldFunction(size_t g, size_t m) const
27,636✔
483
{
484
  OpenSnLogicalErrorIf(phi_field_functions_local_map_.count({g, m}) == 0,
27,636✔
485
                       std::string("Failure to map phi field function g") + std::to_string(g) +
486
                         " m" + std::to_string(m));
487

488
  return phi_field_functions_local_map_.at({g, m});
27,636✔
489
}
490

491
std::shared_ptr<FieldFunctionGridBased>
492
LBSProblem::GetPowerFieldFunction() const
×
493
{
494
  OpenSnLogicalErrorIf(not options_.power_field_function_on,
×
495
                       "Called when options_.power_field_function_on == false");
496

497
  return field_functions_[power_gen_fieldfunc_local_handle_];
×
498
}
499

500
InputParameters
501
LBSProblem::GetOptionsBlock()
426✔
502
{
503
  InputParameters params;
426✔
504

505
  params.SetGeneralDescription("Set options from a large list of parameters");
852✔
506
  params.AddOptionalParameter("max_mpi_message_size",
852✔
507
                              32768,
508
                              "The maximum MPI message size used during sweep initialization.");
509
  params.AddOptionalParameter(
852✔
510
    "restart_writes_enabled", false, "Flag that controls writing of restart dumps");
511
  params.AddOptionalParameter("write_delayed_psi_to_restart",
852✔
512
                              true,
513
                              "Flag that controls writing of delayed angular fluxes to restarts.");
514
  params.AddOptionalParameter(
852✔
515
    "read_restart_path", "", "Full path for reading restart dumps including file stem.");
516
  params.AddOptionalParameter(
852✔
517
    "write_restart_path", "", "Full path for writing restart dumps including file stem.");
518
  params.AddOptionalParameter("write_restart_time_interval",
852✔
519
                              0,
520
                              "Time interval in seconds at which restart data is to be written.");
521
  params.AddOptionalParameter(
852✔
522
    "use_precursors", false, "Flag for using delayed neutron precursors.");
523
  params.AddOptionalParameter("use_source_moments",
852✔
524
                              false,
525
                              "Flag for ignoring fixed sources and selectively using source "
526
                              "moments obtained elsewhere.");
527
  params.AddOptionalParameter(
852✔
528
    "save_angular_flux", false, "Flag indicating whether angular fluxes are to be stored or not.");
529
  params.AddOptionalParameter(
852✔
530
    "adjoint", false, "Flag for toggling whether the solver is in adjoint mode.");
531
  params.AddOptionalParameter(
852✔
532
    "verbose_inner_iterations", true, "Flag to control verbosity of inner iterations.");
533
  params.AddOptionalParameter(
852✔
534
    "verbose_outer_iterations", true, "Flag to control verbosity of across-groupset iterations.");
535
  params.AddOptionalParameter(
852✔
536
    "max_ags_iterations", 100, "Maximum number of across-groupset iterations.");
537
  params.AddOptionalParameter("ags_tolerance", 1.0e-6, "Across-groupset iterations tolerance.");
852✔
538
  params.AddOptionalParameter("ags_convergence_check",
852✔
539
                              "l2",
540
                              "Type of convergence check for AGS iterations. Valid values are "
541
                              "`\"l2\"` and '\"pointwise\"'");
542
  params.AddOptionalParameter(
852✔
543
    "verbose_ags_iterations", true, "Flag to control verbosity of across-groupset iterations.");
544
  params.AddOptionalParameter("power_field_function_on",
852✔
545
                              false,
546
                              "Flag to control the creation of the power generation field "
547
                              "function. If set to `true` then a field function will be created "
548
                              "with the general name <solver_name>_power_generation`.");
549
  params.AddOptionalParameter("power_default_kappa",
852✔
550
                              3.20435e-11,
551
                              "Default `kappa` value (Energy released per fission) to use for "
552
                              "power generation when cross sections do not have `kappa` values. "
553
                              "Default: 3.20435e-11 Joule (corresponding to 200 MeV per fission).");
554
  params.AddOptionalParameter("power_normalization",
852✔
555
                              -1.0,
556
                              "Power normalization factor to use. Supply a negative or zero number "
557
                              "to turn this off.");
558
  params.AddOptionalParameter("field_function_prefix_option",
852✔
559
                              "prefix",
560
                              "Prefix option on field function names. Default: `\"prefix\"`. Can "
561
                              "be `\"prefix\"` or `\"solver_name\"`. By default this option uses "
562
                              "the value of the `field_function_prefix` parameter. If this "
563
                              "parameter is not set, flux field functions will be exported as "
564
                              "`phi_gXXX_mYYY` where `XXX` is the zero padded 3 digit group number "
565
                              "and `YYY` is the zero padded 3 digit moment.");
566
  params.AddOptionalParameter("field_function_prefix",
852✔
567
                              "",
568
                              "Prefix to use on all field functions. Default: `\"\"`. By default "
569
                              "this option is empty. Ff specified, flux moments will be exported "
570
                              "as `prefix_phi_gXXX_mYYY` where `XXX` is the zero padded 3 digit "
571
                              "group number and `YYY` is the zero padded 3 digit moment. The "
572
                              "underscore after \"prefix\" is added automatically.");
573
  params.ConstrainParameterRange("ags_convergence_check",
1,278✔
574
                                 AllowableRangeList::New({"l2", "pointwise"}));
426✔
575
  params.ConstrainParameterRange("field_function_prefix_option",
1,278✔
576
                                 AllowableRangeList::New({"prefix", "solver_name"}));
426✔
577

578
  return params;
426✔
579
}
×
580

581
InputParameters
582
LBSProblem::GetBoundaryOptionsBlock()
738✔
583
{
584
  InputParameters params;
738✔
585

586
  params.SetGeneralDescription("Set options for boundary conditions.");
1,476✔
587
  params.AddRequiredParameter<std::string>("name",
1,476✔
588
                                           "Boundary name that identifies the specific boundary");
589
  params.AddRequiredParameter<std::string>("type", "Boundary type specification.");
1,476✔
590
  params.AddOptionalParameterArray<double>("group_strength",
1,476✔
591
                                           {},
592
                                           "Required only if \"type\" is \"isotropic\". An array "
593
                                           "of isotropic strength per group");
594
  params.AddOptionalParameter(
1,476✔
595
    "function_name", "", "Text name of the function to be called for this boundary condition.");
596
  params.AddOptionalParameter<std::shared_ptr<AngularFluxFunction>>(
1,476✔
597
    "function",
598
    std::shared_ptr<AngularFluxFunction>{},
1,476✔
599
    "Angular flux function to be used for arbitrary boundary conditions. The function takes an "
600
    "energy group index and a direction index and returns the incoming angular flux value.");
601
  params.ConstrainParameterRange(
2,214✔
602
    "type", AllowableRangeList::New({"vacuum", "isotropic", "reflecting", "arbitrary"}));
738✔
603

604
  return params;
738✔
605
}
×
606

607
InputParameters
608
LBSProblem::GetXSMapEntryBlock()
617✔
609
{
610
  InputParameters params;
617✔
611
  params.SetGeneralDescription("Set the cross-section map for the solver.");
1,234✔
612
  params.AddRequiredParameterArray("block_ids", "Mesh block IDs");
1,234✔
613
  params.AddRequiredParameter<std::shared_ptr<MultiGroupXS>>("xs", "Cross-section object");
1,234✔
614
  return params;
617✔
615
}
×
616

617
void
618
LBSProblem::SetOptions(const InputParameters& input)
212✔
619
{
620
  auto params = LBSProblem::GetOptionsBlock();
212✔
621
  params.AssignParameters(input);
212✔
622

623
  // Handle order insensitive options
624
  for (size_t p = 0; p < params.GetNumParameters(); ++p)
4,664✔
625
  {
626
    const auto& spec = params.GetParam(p);
4,452✔
627

628
    if (spec.GetName() == "max_mpi_message_size")
4,452✔
629
      options_.max_mpi_message_size = spec.GetValue<int>();
212✔
630

631
    else if (spec.GetName() == "restart_writes_enabled")
4,240✔
632
      options_.restart_writes_enabled = spec.GetValue<bool>();
212✔
633

634
    else if (spec.GetName() == "write_delayed_psi_to_restart")
4,028✔
635
      options_.write_delayed_psi_to_restart = spec.GetValue<bool>();
212✔
636

637
    else if (spec.GetName() == "read_restart_path")
3,816✔
638
    {
639
      options_.read_restart_path = spec.GetValue<std::string>();
212✔
640
      if (not options_.read_restart_path.empty())
212✔
641
        options_.read_restart_path += std::to_string(opensn::mpi_comm.rank()) + ".restart.h5";
36✔
642
    }
643

644
    else if (spec.GetName() == "write_restart_path")
3,604✔
645
    {
646
      options_.write_restart_path = spec.GetValue<std::string>();
212✔
647
      if (not options_.write_restart_path.empty())
212✔
648
        options_.write_restart_path += std::to_string(opensn::mpi_comm.rank()) + ".restart.h5";
×
649
    }
650

651
    else if (spec.GetName() == "write_restart_time_interval")
3,392✔
652
      options_.write_restart_time_interval = std::chrono::seconds(spec.GetValue<int>());
212✔
653

654
    else if (spec.GetName() == "use_precursors")
3,180✔
655
      options_.use_precursors = spec.GetValue<bool>();
212✔
656

657
    else if (spec.GetName() == "use_source_moments")
2,968✔
658
      options_.use_src_moments = spec.GetValue<bool>();
212✔
659

660
    else if (spec.GetName() == "save_angular_flux")
2,756✔
661
      options_.save_angular_flux = spec.GetValue<bool>();
212✔
662

663
    else if (spec.GetName() == "verbose_inner_iterations")
2,544✔
664
      options_.verbose_inner_iterations = spec.GetValue<bool>();
212✔
665

666
    else if (spec.GetName() == "max_ags_iterations")
2,332✔
667
      options_.max_ags_iterations = spec.GetValue<int>();
212✔
668

669
    else if (spec.GetName() == "ags_tolerance")
2,120✔
670
      options_.ags_tolerance = spec.GetValue<double>();
212✔
671

672
    else if (spec.GetName() == "ags_convergence_check")
1,908✔
673
    {
674
      auto check = spec.GetValue<std::string>();
212✔
675
      if (check == "pointwise")
212✔
676
        options_.ags_pointwise_convergence = true;
×
677
    }
212✔
678

679
    else if (spec.GetName() == "verbose_ags_iterations")
1,696✔
680
      options_.verbose_ags_iterations = spec.GetValue<bool>();
212✔
681

682
    else if (spec.GetName() == "verbose_outer_iterations")
1,484✔
683
      options_.verbose_outer_iterations = spec.GetValue<bool>();
212✔
684

685
    else if (spec.GetName() == "power_field_function_on")
1,272✔
686
      options_.power_field_function_on = spec.GetValue<bool>();
212✔
687

688
    else if (spec.GetName() == "power_default_kappa")
1,060✔
689
      options_.power_default_kappa = spec.GetValue<double>();
212✔
690

691
    else if (spec.GetName() == "power_normalization")
848✔
692
      options_.power_normalization = spec.GetValue<double>();
212✔
693

694
    else if (spec.GetName() == "field_function_prefix_option")
636✔
695
    {
696
      options_.field_function_prefix_option = spec.GetValue<std::string>();
212✔
697
    }
698

699
    else if (spec.GetName() == "field_function_prefix")
424✔
700
      options_.field_function_prefix = spec.GetValue<std::string>();
212✔
701

702
    else if (spec.GetName() == "adjoint")
212✔
703
      options_.adjoint = spec.GetValue<bool>();
212✔
704

705
  } // for p
706

707
  if (options_.restart_writes_enabled)
212✔
708
  {
709
    // Create restart directory if necessary
710
    auto dir = options_.write_restart_path.parent_path();
×
711
    if (opensn::mpi_comm.rank() == 0)
×
712
    {
713
      if (not std::filesystem::exists(dir))
×
714
      {
715
        if (not std::filesystem::create_directories(dir))
×
716
          throw std::runtime_error(GetName() + ": Failed to create restart directory " +
×
717
                                   dir.string());
×
718
      }
719
      else if (not std::filesystem::is_directory(dir))
×
720
        throw std::runtime_error(GetName() + ": Restart path exists but is not a directory " +
×
721
                                 dir.string());
×
722
    }
723
    opensn::mpi_comm.barrier();
×
724
    UpdateRestartWriteTime();
×
725
  }
×
726
}
212✔
727

728
void
729
LBSProblem::SetBoundaryOptions(const InputParameters& params)
738✔
730
{
731
  const auto boundary_name = params.GetParamValue<std::string>("name");
738✔
732
  const auto bndry_type = params.GetParamValue<std::string>("type");
738✔
733

734
  auto grid = GetGrid();
738✔
735
  const auto bnd_map = grid->GetBoundaryIDMap();
738✔
736
  const auto bnd_name_map = grid->GetBoundaryNameMap();
738✔
737
  auto it = bnd_name_map.find(boundary_name);
738✔
738
  if (it == bnd_name_map.end())
738✔
739
    throw std::runtime_error(std::format("Could not find the specified boundary '{}' - please "
×
740
                                         "check that the 'name' parameter is spelled correctly.",
741
                                         boundary_name));
×
742
  const auto bid = it->second;
738✔
743
  const std::map<std::string, LBSBoundaryType> type_list = {
738✔
744
    {"vacuum", LBSBoundaryType::VACUUM},
738✔
745
    {"isotropic", LBSBoundaryType::ISOTROPIC},
738✔
746
    {"reflecting", LBSBoundaryType::REFLECTING},
738✔
747
    {"arbitrary", LBSBoundaryType::ARBITRARY}};
3,690✔
748

749
  const auto type = type_list.at(bndry_type);
738✔
750
  switch (type)
738✔
751
  {
752
    case LBSBoundaryType::VACUUM:
634✔
753
    case LBSBoundaryType::REFLECTING:
634✔
754
    {
634✔
755
      boundary_preferences_[bid] = {type, {}, "", nullptr};
634✔
756
      break;
634✔
757
    }
758
    case LBSBoundaryType::ISOTROPIC:
100✔
759
    {
100✔
760
      if (not params.Has("group_strength"))
100✔
761
        throw std::runtime_error("Boundary conditions with type=\"isotropic\" require parameter "
×
762
                                 "\"group_strength\"");
×
763
      params.RequireParameterBlockTypeIs("group_strength", ParameterBlockType::ARRAY);
100✔
764
      const auto group_strength = params.GetParamVectorValue<double>("group_strength");
100✔
765
      boundary_preferences_[bid] = {type, group_strength, "", nullptr};
100✔
766
      break;
100✔
767
    }
100✔
768
    case LBSBoundaryType::ARBITRARY:
4✔
769
    {
4✔
770
      if (not params.Has("group_strength"))
4✔
771
        throw std::runtime_error("Boundary conditions with type=\"arbitrary\" require parameter "
×
772
                                 "\"function\"");
×
773
      auto angular_flux_function = params.GetSharedPtrParam<AngularFluxFunction>("function", false);
4✔
774
      if (not angular_flux_function)
4✔
775
        throw std::runtime_error(
×
776
          "Boundary conditions with type=\"arbitrary\" require a non-null "
777
          "AngularFluxFunction object passed via the \"function\" parameter.");
×
778
      boundary_preferences_[bid] = {type, {}, "", angular_flux_function};
4✔
779
      break;
4✔
780
    }
4✔
781
  }
782
}
2,956✔
783

784
void
785
LBSProblem::Initialize()
360✔
786
{
787
  CALI_CXX_MARK_SCOPE("LBSProblem::Initialize");
360✔
788

789
  PrintSimHeader();
360✔
790
  mpi_comm.barrier();
360✔
791

792
  InitializeSpatialDiscretization();
360✔
793
  InitializeParrays();
360✔
794
  InitializeBoundaries();
360✔
795
  InitializeGPUExtras();
360✔
796
  SetAdjoint(options_.adjoint);
360✔
797

798
  // Initialize point sources
799
  for (auto& point_source : point_sources_)
369✔
800
    point_source->Initialize(*this);
9✔
801

802
  // Initialize volumetric sources
803
  for (auto& volumetric_source : volumetric_sources_)
743✔
804
    volumetric_source->Initialize(*this);
383✔
805
}
360✔
806

807
void
808
LBSProblem::PrintSimHeader()
×
809
{
810
  if (opensn::mpi_comm.rank() == 0)
×
811
  {
812
    std::stringstream outstr;
×
813
    outstr << "\n"
×
814
           << "Initializing " << GetName() << "\n\n"
×
815
           << "Scattering order    : " << scattering_order_ << "\n"
×
816
           << "Number of moments   : " << num_moments_ << "\n"
×
817
           << "Number of groups    : " << groups_.size() << "\n"
×
818
           << "Number of groupsets : " << groupsets_.size() << "\n\n";
×
819

820
    for (const auto& groupset : groupsets_)
×
821
    {
822
      outstr << "***** Groupset " << groupset.id << " *****\n"
×
823
             << "Groups:\n";
×
824
      const auto& groups = groupset.groups;
825
      constexpr int groups_per_line = 12;
826
      for (size_t i = 0; i < groups.size(); ++i)
×
827
      {
828
        outstr << std::setw(5) << groups[i].id << ' ';
×
829
        if ((i + 1) % groups_per_line == 0)
×
830
          outstr << '\n';
×
831
      }
832
      if (!groups.empty() && groups.size() % groups_per_line != 0)
×
833
        outstr << '\n';
×
834
    }
835

836
    log.Log() << outstr.str() << '\n';
×
837
  }
×
838
}
×
839

840
void
841
LBSProblem::InitializeSources(const InputParameters& params)
360✔
842
{
843
  if (params.Has("volumetric_sources"))
360✔
844
  {
845
    const auto& vol_srcs = params.GetParam("volumetric_sources");
360✔
846
    vol_srcs.RequireBlockTypeIs(ParameterBlockType::ARRAY);
360✔
847
    for (const auto& src : vol_srcs)
743✔
848
      volumetric_sources_.push_back(src.GetValue<std::shared_ptr<VolumetricSource>>());
766✔
849
  }
850

851
  if (params.Has("point_sources"))
360✔
852
  {
853
    const auto& pt_srcs = params.GetParam("point_sources");
360✔
854
    pt_srcs.RequireBlockTypeIs(ParameterBlockType::ARRAY);
360✔
855
    for (const auto& src : pt_srcs)
369✔
856
      point_sources_.push_back(src.GetValue<std::shared_ptr<PointSource>>());
18✔
857
  }
858
}
360✔
859

860
void
861
LBSProblem::InitializeGroupsets(const InputParameters& params)
360✔
862
{
863
  // Initialize groups
864
  if (num_groups_ == 0)
360✔
865
    throw std::invalid_argument(GetName() + ": Number of groups must be > 0");
×
866
  for (size_t g = 0; g < num_groups_; ++g)
21,553✔
867
    groups_.emplace_back(static_cast<int>(g));
21,193✔
868

869
  // Initialize groupsets
870
  const auto& groupsets_array = params.GetParam("groupsets");
360✔
871
  const size_t num_gs = groupsets_array.GetNumParameters();
360✔
872
  if (num_gs == 0)
360✔
873
    throw std::invalid_argument(GetName() + ": At least one groupset must be specified");
×
874
  for (size_t gs = 0; gs < num_gs; ++gs)
779✔
875
  {
876
    const auto& groupset_params = groupsets_array.GetParam(gs);
419✔
877
    InputParameters gs_input_params = LBSGroupset::GetInputParameters();
419✔
878
    gs_input_params.SetObjectType("LBSProblem:LBSGroupset");
419✔
879
    gs_input_params.AssignParameters(groupset_params);
419✔
880
    groupsets_.emplace_back(gs_input_params, gs, *this);
419✔
881
    if (groupsets_.back().groups.empty())
419✔
882
    {
883
      std::stringstream oss;
×
884
      oss << GetName() << ": No groups added to groupset " << groupsets_.back().id;
×
885
      throw std::runtime_error(oss.str());
×
886
    }
×
887
  }
419✔
888
}
360✔
889

890
void
891
LBSProblem::InitializeXSmapAndDensities(const InputParameters& params)
360✔
892
{
893
  // Build XS map
894
  const auto& xs_array = params.GetParam("xs_map");
360✔
895
  const size_t num_xs = xs_array.GetNumParameters();
360✔
896
  for (size_t i = 0; i < num_xs; ++i)
977✔
897
  {
898
    const auto& item_params = xs_array.GetParam(i);
617✔
899
    InputParameters xs_entry_pars = GetXSMapEntryBlock();
617✔
900
    xs_entry_pars.AssignParameters(item_params);
617✔
901

902
    const auto& block_ids_param = xs_entry_pars.GetParam("block_ids");
617✔
903
    block_ids_param.RequireBlockTypeIs(ParameterBlockType::ARRAY);
617✔
904
    const auto& block_ids = block_ids_param.GetVectorValue<unsigned int>();
617✔
905
    auto xs = xs_entry_pars.GetSharedPtrParam<MultiGroupXS>("xs");
617✔
906
    for (const auto& block_id : block_ids)
1,330✔
907
      block_id_to_xs_map_[block_id] = xs;
713✔
908
  }
617✔
909

910
  // Assign placeholder unit densities
911
  densities_local_.assign(grid_->local_cells.size(), 1.0);
360✔
912
}
360✔
913

914
void
915
LBSProblem::InitializeMaterials()
372✔
916
{
917
  CALI_CXX_MARK_SCOPE("LBSProblem::InitializeMaterials");
372✔
918

919
  log.Log0Verbose1() << "Initializing Materials";
744✔
920

921
  // Create set of material ids locally relevant
922
  int invalid_mat_cell_count = 0;
372✔
923
  std::set<unsigned int> unique_block_ids;
372✔
924
  for (auto& cell : grid_->local_cells)
394,952✔
925
  {
926
    unique_block_ids.insert(cell.block_id);
394,580✔
927
    if (cell.block_id == std::numeric_limits<unsigned int>::max() or
394,580✔
928
        (block_id_to_xs_map_.find(cell.block_id) == block_id_to_xs_map_.end()))
394,580✔
929
      ++invalid_mat_cell_count;
×
930
  }
931
  const auto& ghost_cell_ids = grid_->cells.GetGhostGlobalIDs();
372✔
932
  for (uint64_t cell_id : ghost_cell_ids)
71,746✔
933
  {
934
    const auto& cell = grid_->cells[cell_id];
71,374✔
935
    unique_block_ids.insert(cell.block_id);
71,374✔
936
    if (cell.block_id == std::numeric_limits<unsigned int>::max() or
71,374✔
937
        (block_id_to_xs_map_.find(cell.block_id) == block_id_to_xs_map_.end()))
71,374✔
938
      ++invalid_mat_cell_count;
×
939
  }
940
  OpenSnLogicalErrorIf(invalid_mat_cell_count > 0,
372✔
941
                       std::to_string(invalid_mat_cell_count) +
942
                         " cells encountered with an invalid material id.");
943

944
  // Get ready for processing
945
  for (const auto& [blk_id, mat] : block_id_to_xs_map_)
1,113✔
946
  {
947
    mat->SetAdjointMode(options_.adjoint);
741✔
948

949
    OpenSnLogicalErrorIf(mat->GetNumGroups() < groups_.size(),
741✔
950
                         "Cross-sections for block \"" + std::to_string(blk_id) +
951
                           "\" have fewer groups (" + std::to_string(mat->GetNumGroups()) +
952
                           ") than the simulation (" + std::to_string(groups_.size()) + "). " +
953
                           "Cross-sections must have at least as many groups as the simulation.");
954
  }
955

956
  // Initialize precursor properties
957
  num_precursors_ = 0;
372✔
958
  max_precursors_per_material_ = 0;
372✔
959
  for (const auto& mat_id_xs : block_id_to_xs_map_)
1,113✔
960
  {
961
    const auto& xs = mat_id_xs.second;
741✔
962
    num_precursors_ += xs->GetNumPrecursors();
741✔
963
    max_precursors_per_material_ = std::max(xs->GetNumPrecursors(), max_precursors_per_material_);
741✔
964
  }
965

966
  // if no precursors, turn off precursors
967
  if (num_precursors_ == 0)
372✔
968
    options_.use_precursors = false;
364✔
969

970
  // check compatibility when precursors are on
971
  if (options_.use_precursors)
372✔
972
  {
973
    for (const auto& [mat_id, xs] : block_id_to_xs_map_)
16✔
974
    {
975
      OpenSnLogicalErrorIf(xs->IsFissionable() and num_precursors_ == 0,
8✔
976
                           "Incompatible cross-section data encountered for material id " +
977
                             std::to_string(mat_id) + ". When delayed neutron data is present " +
978
                             "for one fissionable matrial, it must be present for all fissionable "
979
                             "materials.");
980
    }
981
  }
982

983
  // Update transport views if available
984
  if (grid_->local_cells.size() == cell_transport_views_.size())
372✔
985
    for (const auto& cell : grid_->local_cells)
10,812✔
986
    {
987
      const auto& xs_ptr = block_id_to_xs_map_[cell.block_id];
10,800✔
988
      auto& transport_view = cell_transport_views_[cell.local_id];
10,800✔
989
      transport_view.ReassignXS(*xs_ptr);
10,800✔
990
    }
991

992
  mpi_comm.barrier();
372✔
993
}
372✔
994

995
void
996
LBSProblem::InitializeSpatialDiscretization()
352✔
997
{
998
  CALI_CXX_MARK_SCOPE("LBSProblem::InitializeSpatialDiscretization");
352✔
999

1000
  log.Log() << "Initializing spatial discretization.\n";
704✔
1001
  discretization_ = PieceWiseLinearDiscontinuous::New(grid_);
352✔
1002

1003
  ComputeUnitIntegrals();
352✔
1004
}
352✔
1005

1006
void
1007
LBSProblem::ComputeUnitIntegrals()
360✔
1008
{
1009
  CALI_CXX_MARK_SCOPE("LBSProblem::ComputeUnitIntegrals");
360✔
1010

1011
  log.Log() << "Computing unit integrals.\n";
720✔
1012
  const auto& sdm = *discretization_;
360✔
1013

1014
  const size_t num_local_cells = grid_->local_cells.size();
360✔
1015
  unit_cell_matrices_.resize(num_local_cells);
360✔
1016

1017
  for (const auto& cell : grid_->local_cells)
384,140✔
1018
    unit_cell_matrices_[cell.local_id] =
383,780✔
1019
      ComputeUnitCellIntegrals(sdm, cell, grid_->GetCoordinateSystem());
383,780✔
1020

1021
  const auto ghost_ids = grid_->cells.GetGhostGlobalIDs();
360✔
1022
  for (auto ghost_id : ghost_ids)
70,978✔
1023
    unit_ghost_cell_matrices_[ghost_id] =
70,618✔
1024
      ComputeUnitCellIntegrals(sdm, grid_->cells[ghost_id], grid_->GetCoordinateSystem());
141,236✔
1025

1026
  // Assessing global unit cell matrix storage
1027
  std::array<size_t, 2> num_local_ucms = {unit_cell_matrices_.size(),
360✔
1028
                                          unit_ghost_cell_matrices_.size()};
360✔
1029
  std::array<size_t, 2> num_global_ucms = {0, 0};
360✔
1030

1031
  mpi_comm.all_reduce(num_local_ucms.data(), 2, num_global_ucms.data(), mpi::op::sum<size_t>());
360✔
1032

1033
  opensn::mpi_comm.barrier();
360✔
1034
  log.Log() << "Ghost cell unit cell-matrix ratio: "
360✔
1035
            << (double)num_global_ucms[1] * 100 / (double)num_global_ucms[0] << "%";
720✔
1036
  log.Log() << "Cell matrices computed.";
720✔
1037
}
360✔
1038

1039
void
1040
LBSProblem::InitializeParrays()
360✔
1041
{
1042
  CALI_CXX_MARK_SCOPE("LBSProblem::InitializeParrays");
360✔
1043

1044
  log.Log() << "Initializing parallel arrays."
720✔
1045
            << " G=" << num_groups_ << " M=" << num_moments_ << std::endl;
360✔
1046

1047
  // Initialize unknown
1048
  // structure
1049
  flux_moments_uk_man_.unknowns.clear();
360✔
1050
  for (size_t m = 0; m < num_moments_; ++m)
1,294✔
1051
  {
1052
    flux_moments_uk_man_.AddUnknown(UnknownType::VECTOR_N, groups_.size());
934✔
1053
    flux_moments_uk_man_.unknowns.back().name = "m" + std::to_string(m);
934✔
1054
  }
1055

1056
  // Compute local # of dof
1057
  local_node_count_ = discretization_->GetNumLocalNodes();
360✔
1058
  global_node_count_ = discretization_->GetNumGlobalNodes();
360✔
1059

1060
  // Compute num of unknowns
1061
  size_t num_grps = groups_.size();
360✔
1062
  size_t local_unknown_count = local_node_count_ * num_grps * num_moments_;
360✔
1063

1064
  log.LogAllVerbose1() << "LBS Number of phi unknowns: " << local_unknown_count;
720✔
1065

1066
  // Size local vectors
1067
  q_moments_local_.assign(local_unknown_count, 0.0);
360✔
1068
  phi_old_local_.assign(local_unknown_count, 0.0);
360✔
1069
  phi_new_local_.assign(local_unknown_count, 0.0);
360✔
1070

1071
  // Setup precursor vector
1072
  if (options_.use_precursors)
360✔
1073
  {
1074
    size_t num_precursor_dofs = grid_->local_cells.size() * max_precursors_per_material_;
8✔
1075
    precursor_new_local_.assign(num_precursor_dofs, 0.0);
8✔
1076
  }
1077

1078
  // Initialize transport views
1079
  // Transport views act as a data structure to store information
1080
  // related to the transport simulation. The most prominent function
1081
  // here is that it holds the means to know where a given cell's
1082
  // transport quantities are located in the unknown vectors (i.e. phi)
1083
  //
1084
  // Also, for a given cell, within a given sweep chunk,
1085
  // we need to solve a matrix which square size is the
1086
  // amount of nodes on the cell. max_cell_dof_count is
1087
  // initialized here.
1088
  //
1089
  size_t block_MG_counter = 0; // Counts the strides of moment and group
360✔
1090

1091
  const Vector3 ihat(1.0, 0.0, 0.0);
360✔
1092
  const Vector3 jhat(0.0, 1.0, 0.0);
360✔
1093
  const Vector3 khat(0.0, 0.0, 1.0);
360✔
1094

1095
  min_cell_dof_count_ = std::numeric_limits<unsigned int>::max();
360✔
1096
  max_cell_dof_count_ = 0;
360✔
1097
  cell_transport_views_.clear();
360✔
1098
  cell_transport_views_.reserve(grid_->local_cells.size());
360✔
1099
  for (auto& cell : grid_->local_cells)
384,140✔
1100
  {
1101
    size_t num_nodes = discretization_->GetCellNumNodes(cell);
383,780✔
1102

1103
    // compute cell volumes
1104
    double cell_volume = 0.0;
383,780✔
1105
    const auto& IntV_shapeI = unit_cell_matrices_[cell.local_id].intV_shapeI;
383,780✔
1106
    for (size_t i = 0; i < num_nodes; ++i)
3,032,868✔
1107
      cell_volume += IntV_shapeI(i);
2,649,088✔
1108

1109
    size_t cell_phi_address = block_MG_counter;
383,780✔
1110

1111
    const size_t num_faces = cell.faces.size();
383,780✔
1112
    std::vector<bool> face_local_flags(num_faces, true);
383,780✔
1113
    std::vector<int> face_locality(num_faces, opensn::mpi_comm.rank());
383,780✔
1114
    std::vector<const Cell*> neighbor_cell_ptrs(num_faces, nullptr);
383,780✔
1115
    bool cell_on_boundary = false;
383,780✔
1116
    int f = 0;
383,780✔
1117
    for (auto& face : cell.faces)
2,467,052✔
1118
    {
1119
      if (not face.has_neighbor)
2,083,272✔
1120
      {
1121
        cell_on_boundary = true;
77,698✔
1122
        face_local_flags[f] = false;
77,698✔
1123
        face_locality[f] = -1;
77,698✔
1124
      } // if bndry
1125
      else
1126
      {
1127
        const int neighbor_partition = face.GetNeighborPartitionID(grid_.get());
2,005,574✔
1128
        face_local_flags[f] = (neighbor_partition == opensn::mpi_comm.rank());
2,005,574✔
1129
        face_locality[f] = neighbor_partition;
2,005,574✔
1130
        neighbor_cell_ptrs[f] = &grid_->cells[face.neighbor_id];
2,005,574✔
1131
      }
1132

1133
      ++f;
2,083,272✔
1134
    } // for f
1135

1136
    max_cell_dof_count_ = std::max(max_cell_dof_count_, static_cast<unsigned int>(num_nodes));
383,780✔
1137
    min_cell_dof_count_ = std::min(min_cell_dof_count_, static_cast<unsigned int>(num_nodes));
383,780✔
1138
    cell_transport_views_.emplace_back(cell_phi_address,
767,560✔
1139
                                       num_nodes,
1140
                                       num_grps,
1141
                                       num_moments_,
383,780✔
1142
                                       num_faces,
1143
                                       *block_id_to_xs_map_[cell.block_id],
383,780✔
1144
                                       cell_volume,
1145
                                       face_local_flags,
1146
                                       face_locality,
1147
                                       neighbor_cell_ptrs,
1148
                                       cell_on_boundary);
1149
    block_MG_counter += num_nodes * num_grps * num_moments_;
383,780✔
1150
  } // for local cell
383,780✔
1151

1152
  // Populate grid nodal mappings
1153
  // This is used in the Flux Data Structures (FLUDS)
1154
  grid_nodal_mappings_.clear();
360✔
1155
  grid_nodal_mappings_.reserve(grid_->local_cells.size());
360✔
1156
  for (auto& cell : grid_->local_cells)
384,140✔
1157
  {
1158
    CellFaceNodalMapping cell_nodal_mapping;
383,780✔
1159
    cell_nodal_mapping.reserve(cell.faces.size());
383,780✔
1160

1161
    for (auto& face : cell.faces)
2,467,052✔
1162
    {
1163
      std::vector<short> face_node_mapping;
2,083,272✔
1164
      std::vector<short> cell_node_mapping;
2,083,272✔
1165
      int adj_face_idx = -1;
2,083,272✔
1166

1167
      if (face.has_neighbor)
2,083,272✔
1168
      {
1169
        grid_->FindAssociatedVertices(face, face_node_mapping);
2,005,574✔
1170
        grid_->FindAssociatedCellVertices(face, cell_node_mapping);
2,005,574✔
1171
        adj_face_idx = face.GetNeighborAdjacentFaceIndex(grid_.get());
2,005,574✔
1172
      }
1173

1174
      cell_nodal_mapping.emplace_back(adj_face_idx, face_node_mapping, cell_node_mapping);
2,083,272✔
1175
    } // for f
2,083,272✔
1176

1177
    grid_nodal_mappings_.push_back(cell_nodal_mapping);
383,780✔
1178
  } // for local cell
383,780✔
1179

1180
  // Get grid localized communicator set
1181
  grid_local_comm_set_ = grid_->MakeMPILocalCommunicatorSet();
360✔
1182

1183
  // Initialize Field Functions
1184
  InitializeFieldFunctions();
360✔
1185

1186
  opensn::mpi_comm.barrier();
360✔
1187
  log.Log() << "Done with parallel arrays." << std::endl;
720✔
1188
}
360✔
1189

1190
void
1191
LBSProblem::InitializeFieldFunctions()
360✔
1192
{
1193
  CALI_CXX_MARK_SCOPE("LBSProblem::InitializeFieldFunctions");
360✔
1194

1195
  if (not field_functions_.empty())
360✔
1196
    return;
×
1197

1198
  // Initialize Field Functions for flux moments
1199
  phi_field_functions_local_map_.clear();
360✔
1200

1201
  for (size_t g = 0; g < groups_.size(); ++g)
21,553✔
1202
  {
1203
    for (size_t m = 0; m < num_moments_; ++m)
97,704✔
1204
    {
1205
      std::string prefix;
76,511✔
1206
      if (options_.field_function_prefix_option == "prefix")
76,511✔
1207
      {
1208
        prefix = options_.field_function_prefix;
76,511✔
1209
        if (not prefix.empty())
76,511✔
1210
          prefix += "_";
1✔
1211
      }
1212
      if (options_.field_function_prefix_option == "solver_name")
76,511✔
1213
        prefix = GetName() + "_";
×
1214

1215
      std::ostringstream oss;
76,511✔
1216
      oss << prefix << "phi_g" << std::setw(3) << std::setfill('0') << static_cast<int>(g) << "_m"
76,511✔
1217
          << std::setw(2) << std::setfill('0') << static_cast<int>(m);
76,511✔
1218
      const std::string name = oss.str();
76,511✔
1219

1220
      auto group_ff = std::make_shared<FieldFunctionGridBased>(
76,511✔
1221
        name, discretization_, Unknown(UnknownType::SCALAR));
76,511✔
1222

1223
      field_function_stack.push_back(group_ff);
153,022✔
1224
      field_functions_.push_back(group_ff);
76,511✔
1225

1226
      phi_field_functions_local_map_[{g, m}] = field_functions_.size() - 1;
76,511✔
1227
    } // for m
76,511✔
1228
  } // for g
1229

1230
  // Initialize power generation field function
1231
  if (options_.power_field_function_on)
360✔
1232
  {
1233
    std::string prefix;
4✔
1234
    if (options_.field_function_prefix_option == "prefix")
4✔
1235
    {
1236
      prefix = options_.field_function_prefix;
4✔
1237
      if (not prefix.empty())
4✔
1238
        prefix += "_";
×
1239
    }
1240
    if (options_.field_function_prefix_option == "solver_name")
4✔
1241
      prefix = GetName() + "_";
×
1242

1243
    auto power_ff = std::make_shared<FieldFunctionGridBased>(
4✔
1244
      prefix + "power_generation", discretization_, Unknown(UnknownType::SCALAR));
8✔
1245

1246
    field_function_stack.push_back(power_ff);
8✔
1247
    field_functions_.push_back(power_ff);
4✔
1248

1249
    power_gen_fieldfunc_local_handle_ = field_functions_.size() - 1;
4✔
1250
  }
4✔
1251
}
360✔
1252

1253
void
1254
LBSProblem::InitializeSolverSchemes()
360✔
1255
{
1256
  CALI_CXX_MARK_SCOPE("LBSProblem::InitializeSolverSchemes");
360✔
1257

1258
  log.Log() << "Initializing WGS and AGS solvers";
720✔
1259

1260
  InitializeWGSSolvers();
360✔
1261

1262
  ags_solver_ = std::make_shared<AGSLinearSolver>(*this, wgs_solvers_);
360✔
1263
  if (groupsets_.size() == 1)
360✔
1264
  {
1265
    ags_solver_->SetMaxIterations(1);
305✔
1266
    ags_solver_->SetVerbosity(false);
305✔
1267
  }
1268
  else
1269
  {
1270
    ags_solver_->SetMaxIterations(options_.max_ags_iterations);
55✔
1271
    ags_solver_->SetVerbosity(options_.verbose_ags_iterations);
55✔
1272
  }
1273
  ags_solver_->SetTolerance(options_.ags_tolerance);
360✔
1274
}
360✔
1275

1276
#ifndef __OPENSN_USE_CUDA__
1277
void
1278
LBSProblem::InitializeGPUExtras()
360✔
1279
{
1280
}
360✔
1281

1282
void
1283
LBSProblem::ResetGPUCarriers()
352✔
1284
{
1285
}
352✔
1286

1287
void
1288
LBSProblem::CheckCapableDevices()
×
1289
{
1290
}
×
1291
#endif // __OPENSN_USE_CUDA__
1292

1293
std::vector<double>
1294
LBSProblem::MakeSourceMomentsFromPhi()
4✔
1295
{
1296
  CALI_CXX_MARK_SCOPE("LBSProblem::MakeSourceMomentsFromPhi");
4✔
1297

1298
  size_t num_local_dofs = discretization_->GetNumLocalDOFs(flux_moments_uk_man_);
4✔
1299

1300
  std::vector<double> source_moments(num_local_dofs, 0.0);
4✔
1301
  for (auto& groupset : groupsets_)
8✔
1302
  {
1303
    active_set_source_function_(groupset,
4✔
1304
                                source_moments,
1305
                                phi_new_local_,
4✔
1306
                                APPLY_AGS_SCATTER_SOURCES | APPLY_WGS_SCATTER_SOURCES |
1307
                                  APPLY_AGS_FISSION_SOURCES | APPLY_WGS_FISSION_SOURCES);
4✔
1308
  }
1309

1310
  return source_moments;
4✔
1311
}
4✔
1312

1313
void
1314
LBSProblem::UpdateFieldFunctions()
1,200✔
1315
{
1316
  CALI_CXX_MARK_SCOPE("LBSProblem::UpdateFieldFunctions");
1,200✔
1317

1318
  const auto& sdm = *discretization_;
1,200✔
1319
  const auto& phi_uk_man = flux_moments_uk_man_;
1,200✔
1320

1321
  // Update flux moments
1322
  for (const auto& [g_and_m, ff_index] : phi_field_functions_local_map_)
83,239✔
1323
  {
1324
    const size_t g = g_and_m.first;
82,039✔
1325
    const size_t m = g_and_m.second;
82,039✔
1326

1327
    std::vector<double> data_vector_local(local_node_count_, 0.0);
82,039✔
1328

1329
    for (const auto& cell : grid_->local_cells)
21,615,610✔
1330
    {
1331
      const auto& cell_mapping = sdm.GetCellMapping(cell);
21,533,571✔
1332
      const size_t num_nodes = cell_mapping.GetNumNodes();
21,533,571✔
1333

1334
      for (size_t i = 0; i < num_nodes; ++i)
150,504,935✔
1335
      {
1336
        const auto imapA = sdm.MapDOFLocal(cell, i, phi_uk_man, m, g);
128,971,364✔
1337
        const auto imapB = sdm.MapDOFLocal(cell, i);
128,971,364✔
1338

1339
        data_vector_local[imapB] = phi_new_local_[imapA];
128,971,364✔
1340
      } // for node
1341
    } // for cell
1342

1343
    auto& ff_ptr = field_functions_.at(ff_index);
82,039✔
1344
    ff_ptr->UpdateFieldVector(data_vector_local);
82,039✔
1345
  }
82,039✔
1346

1347
  // Update power generation and scalar flux
1348
  if (options_.power_field_function_on)
1,200✔
1349
  {
1350
    std::vector<double> data_vector_power_local(local_node_count_, 0.0);
4✔
1351

1352
    double local_total_power = 0.0;
4✔
1353
    for (const auto& cell : grid_->local_cells)
83,268✔
1354
    {
1355
      const auto& cell_mapping = sdm.GetCellMapping(cell);
83,264✔
1356
      const size_t num_nodes = cell_mapping.GetNumNodes();
83,264✔
1357

1358
      const auto& Vi = unit_cell_matrices_[cell.local_id].intV_shapeI;
83,264✔
1359

1360
      const auto& xs = block_id_to_xs_map_.at(cell.block_id);
83,264✔
1361

1362
      if (not xs->IsFissionable())
83,264✔
1363
        continue;
56,360✔
1364

1365
      for (size_t i = 0; i < num_nodes; ++i)
134,520✔
1366
      {
1367
        const auto imapA = sdm.MapDOFLocal(cell, i);
107,616✔
1368
        const auto imapB = sdm.MapDOFLocal(cell, i, phi_uk_man, 0, 0);
107,616✔
1369

1370
        double nodal_power = 0.0;
1371
        for (size_t g = 0; g < groups_.size(); ++g)
860,928✔
1372
        {
1373
          const double sigma_fg = xs->GetSigmaFission()[g];
753,312✔
1374
          // const double kappa_g = xs->Kappa()[g];
1375
          const double kappa_g = options_.power_default_kappa;
753,312✔
1376

1377
          nodal_power += kappa_g * sigma_fg * phi_new_local_[imapB + g];
753,312✔
1378
        } // for g
1379

1380
        data_vector_power_local[imapA] = nodal_power;
107,616✔
1381
        local_total_power += nodal_power * Vi(i);
107,616✔
1382
      } // for node
1383
    } // for cell
1384

1385
    double scale_factor = 1.0;
4✔
1386
    if (options_.power_normalization > 0.0)
4✔
1387
    {
1388
      double global_total_power = 0.0;
4✔
1389
      mpi_comm.all_reduce(local_total_power, global_total_power, mpi::op::sum<double>());
4✔
1390
      scale_factor = options_.power_normalization / global_total_power;
4✔
1391
      Scale(data_vector_power_local, scale_factor);
4✔
1392
    }
1393

1394
    const size_t ff_index = power_gen_fieldfunc_local_handle_;
4✔
1395

1396
    auto& ff_ptr = field_functions_.at(ff_index);
4✔
1397
    ff_ptr->UpdateFieldVector(data_vector_power_local);
4✔
1398

1399
    // scale scalar flux if neccessary
1400
    if (scale_factor != 1.0)
4✔
1401
    {
1402
      for (size_t g = 0; g < groups_.size(); ++g)
32✔
1403
      {
1404
        const size_t phi_ff_index = phi_field_functions_local_map_.at({g, size_t{0}});
28✔
1405
        auto& phi_ff_ptr = field_functions_.at(phi_ff_index);
28✔
1406
        const auto& phi_vec = phi_ff_ptr->GetLocalFieldVector();
28✔
1407
        std::vector<double> phi_scaled(phi_vec.begin(), phi_vec.end());
28✔
1408
        Scale(phi_scaled, scale_factor);
28✔
1409
        phi_ff_ptr->UpdateFieldVector(phi_scaled);
28✔
1410
      }
28✔
1411
    }
1412
  } // if power enabled
4✔
1413
}
1,200✔
1414

1415
void
1416
LBSProblem::SetPhiFromFieldFunctions(PhiSTLOption which_phi,
×
1417
                                     const std::vector<size_t>& m_indices,
1418
                                     const std::vector<size_t>& g_indices)
1419
{
1420
  CALI_CXX_MARK_SCOPE("LBSProblem::SetPhiFromFieldFunctions");
×
1421

1422
  std::vector<size_t> m_ids_to_copy = m_indices;
×
1423
  std::vector<size_t> g_ids_to_copy = g_indices;
×
1424
  if (m_indices.empty())
×
1425
    for (size_t m = 0; m < num_moments_; ++m)
×
1426
      m_ids_to_copy.push_back(m);
×
1427
  if (g_ids_to_copy.empty())
×
1428
    for (size_t g = 0; g < num_groups_; ++g)
×
1429
      g_ids_to_copy.push_back(g);
×
1430

1431
  const auto& sdm = *discretization_;
×
1432
  const auto& phi_uk_man = flux_moments_uk_man_;
×
1433

1434
  for (const size_t m : m_ids_to_copy)
×
1435
  {
1436
    for (const size_t g : g_ids_to_copy)
×
1437
    {
1438
      const size_t ff_index = phi_field_functions_local_map_.at({g, m});
×
1439
      const auto& ff_ptr = field_functions_.at(ff_index);
×
1440
      const auto& ff_data = ff_ptr->GetLocalFieldVector();
×
1441

1442
      for (const auto& cell : grid_->local_cells)
×
1443
      {
1444
        const auto& cell_mapping = sdm.GetCellMapping(cell);
×
1445
        const size_t num_nodes = cell_mapping.GetNumNodes();
×
1446

1447
        for (size_t i = 0; i < num_nodes; ++i)
×
1448
        {
1449
          const auto imapA = sdm.MapDOFLocal(cell, i);
×
1450
          const auto imapB = sdm.MapDOFLocal(cell, i, phi_uk_man, m, g);
×
1451

1452
          if (which_phi == PhiSTLOption::PHI_OLD)
×
1453
            phi_old_local_[imapB] = ff_data[imapA];
×
1454
          else if (which_phi == PhiSTLOption::PHI_NEW)
×
1455
            phi_new_local_[imapB] = ff_data[imapA];
×
1456
        } // for node
1457
      } // for cell
1458
    } // for g
1459
  } // for m
1460
}
×
1461

1462
LBSProblem::~LBSProblem()
352✔
1463
{
1464
  ResetGPUCarriers();
1465
}
1,760✔
1466

352✔
1467
void
1468
LBSProblem::SetAdjoint(bool adjoint)
372✔
1469
{
1470
  if (adjoint and time_dependent_)
372✔
1471
    throw std::invalid_argument(GetName() + ": Time-dependent adjoint problems are not supported.");
×
1472

1473
  if (adjoint != options_.adjoint)
372✔
1474
  {
1475
    options_.adjoint = adjoint;
12✔
1476

1477
    // If a discretization exists, the solver has already been initialized.
1478
    // Reinitialize the materials to obtain the appropriate xs and clear the
1479
    // sources to prepare for defining the adjoint problem
1480
    if (discretization_)
12✔
1481
    {
1482
      // The materials are reinitialized here to ensure that the proper cross sections
1483
      // are available to the solver. Because an adjoint solve requires volumetric or
1484
      // point sources, the material-based sources are not set within the initialize routine.
1485
      InitializeMaterials();
12✔
1486

1487
      // Forward and adjoint sources are fundamentally different, so any existing sources
1488
      // should be cleared and reset through options upon changing modes.
1489
      point_sources_.clear();
12✔
1490
      volumetric_sources_.clear();
12✔
1491
      boundary_preferences_.clear();
12✔
1492

1493
      // Set all solutions to zero.
1494
      phi_old_local_.assign(phi_old_local_.size(), 0.0);
12✔
1495
      phi_new_local_.assign(phi_new_local_.size(), 0.0);
12✔
1496
      ZeroSolutions();
12✔
1497
      precursor_new_local_.assign(precursor_new_local_.size(), 0.0);
12✔
1498
    }
1499
  }
1500
}
372✔
1501

1502
} // namespace opensn
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc