29619881679

Committed 17 Jul 2026 11:03PM UTC coverage: 86.477% (+0.4%) from 86.054%

Build # 29619881679

Build Type

Pull #63

github

Committed by

MuellerSeb

Commit Message

Allow defaults to satisfy required values

Separate declared requiredness from effective input requirements across validation, namelist evaluation, and generated APIs. Support scalar derived object defaults and broadcast item defaults with deterministic coverage and precedence rules.

Update Fortran, f2py/Python, Markdown, templates, documentation, tests, and regenerated derived-type example artifacts.

Pull Request Pull Request #63: Allow defaults to satisfy required values

Coverage Stats

270 of 287 new or added lines in 7 files covered. (94.08%)

3 existing lines in 1 file now uncovered.

5039 of 5827 relevant lines covered (86.48%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.21

/src/nml_tools/codegen_template.py

"""Template namelist generation."""

from __future__ import annotations

from pathlib import Path
from typing import Any, Iterable

from ._utils import (
    normalize_constant_values,
    normalize_runtime_dimensions,
    reject_constant_dimension_overlap,
    validate_user_fortran_identifier,
)
from .codegen_fortran import (
    FieldTypeInfo,
    _array_default_value,
    _collect_dimension_constants,
    _derived_schema,
    _enum_values,
    _field_type_info,
    _format_scalar_default,
    _parse_default_dimensions,
    _reject_runtime_dimension_lengths,
)
from .schema import _is_simple_derived_schema
from .validate import (
    analyze_property_requirement,
    derived_object_default,
    validate_schema_defaults,
)

_MISSING = object()


def generate_template(
    schemas: Iterable[dict[str, Any]],
    output: str | Path,
    *,
    doc_mode: str = "plain",
    value_mode: str = "empty",
    simple_derived_mode: str = "components",
    title: str | None = None,
    description: str | None = None,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    kind_map: dict[str, str] | None = None,
    kind_allowlist: set[str] | None = None,
    values: dict[str, dict[str, Any]] | None = None,
) -> None:
    """Generate a template namelist file for *schemas* at *output*."""
    rendered = render_template(
        schemas,
        doc_mode=doc_mode,
        value_mode=value_mode,
        simple_derived_mode=simple_derived_mode,
        title=title,
        description=description,
        constants=constants,
        dimensions=dimensions,
        kind_map=kind_map,
        kind_allowlist=kind_allowlist,
        values=values,
    )
    output_path = Path(output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(rendered, encoding="ascii")


def render_template(
    schemas: Iterable[dict[str, Any]],
    *,
    doc_mode: str = "plain",
    value_mode: str = "empty",
    simple_derived_mode: str = "components",
    title: str | None = None,
    description: str | None = None,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    kind_map: dict[str, str] | None = None,
    kind_allowlist: set[str] | None = None,
    values: dict[str, dict[str, Any]] | None = None,
) -> str:
    """Render a template namelist file for *schemas*."""
    doc_mode = doc_mode.strip().lower()
    value_mode = value_mode.strip().lower()
    simple_derived_mode = simple_derived_mode.strip().lower()
    if doc_mode not in {"plain", "documented"}:
        raise ValueError("template doc_mode must be 'plain' or 'documented'")
    if value_mode not in {"empty", "filled", "minimal-empty", "minimal-filled"}:
        raise ValueError(
            "template value_mode must be one of: empty, filled, minimal-empty, minimal-filled"
        )
    if simple_derived_mode not in {"components", "buffer"}:
        raise ValueError("template simple_derived_mode must be 'components' or 'buffer'")

    return _render_template(
        schemas,
        doc_mode=doc_mode,
        value_mode=value_mode,
        simple_derived_mode=simple_derived_mode,
        title=title,
        description=description,
        constants=constants,
        dimensions=dimensions,
        kind_map=kind_map,
        kind_allowlist=kind_allowlist,
        values=values,
    )


def _render_template(
    schemas: Iterable[dict[str, Any]],
    *,
    doc_mode: str,
    value_mode: str,
    simple_derived_mode: str,
    title: str | None,
    description: str | None,
    constants: dict[str, int] | None,
    dimensions: dict[str, int] | None,
    kind_map: dict[str, str] | None,
    kind_allowlist: set[str] | None,
    values: dict[str, dict[str, Any]] | None,
) -> str:
    lines: list[str] = []
    schemas_list = list(schemas)
    values_map = values or {}
    if not isinstance(values_map, dict):
        raise ValueError("template values must be a table of namelist tables")
    constants = normalize_constant_values(constants)
    dimensions = normalize_runtime_dimensions(dimensions)
    reject_constant_dimension_overlap(constants, dimensions)
    shape_constants: dict[str, int] = {
        **constants,
        **dimensions,
    }

    schema_by_name: dict[str, dict[str, Any]] = {}
    for schema in schemas_list:
        validate_schema_defaults(schema, constants=constants, dimensions=dimensions)
        namelist_name = schema.get("x-fortran-namelist")
        if not isinstance(namelist_name, str) or not namelist_name.strip():
            raise ValueError("schema must define 'x-fortran-namelist'")
        validate_user_fortran_identifier(namelist_name, label="'x-fortran-namelist'")
        schema_by_name[namelist_name] = schema

    for namelist_name, namelist_values in values_map.items():
        if not isinstance(namelist_name, str):
            raise ValueError("template values namelist names must be strings")
        if namelist_name not in schema_by_name:
            raise ValueError(f"template values namelist '{namelist_name}' not found in schemas")
        if not isinstance(namelist_values, dict):
            raise ValueError(f"template values for namelist '{namelist_name}' must be a table")
        properties = schema_by_name[namelist_name].get("properties")
        if not isinstance(properties, dict) or not properties:
            raise ValueError("schema must define object 'properties'")
        for key in namelist_values:
            if not isinstance(key, str):
                raise ValueError(
                    f"template values for namelist '{namelist_name}' must use string keys"
                )
            if key not in properties:
                raise ValueError(
                    f"template values for namelist '{namelist_name}' include unknown field '{key}'"
                )

    if doc_mode == "documented":
        _append_template_comment(lines, title)
        has_title = isinstance(title, str) and bool(title.strip())
        has_description = isinstance(description, str) and bool(description.strip())
        if has_title and has_description:
            lines.append("!")
        _append_template_comment(lines, description)
        if lines:
            lines.append("")

    for schema in schemas_list:
        namelist_name = schema.get("x-fortran-namelist")
        if not isinstance(namelist_name, str) or not namelist_name.strip():
            raise ValueError("schema must define 'x-fortran-namelist'")
        validate_user_fortran_identifier(namelist_name, label="'x-fortran-namelist'")
        if schema.get("type") != "object":
            raise ValueError("schema root must be of type 'object'")
        properties = schema.get("properties")
        if not isinstance(properties, dict) or not properties:
            raise ValueError("schema must define object 'properties'")
        required_raw = schema.get("required", [])
        if not isinstance(required_raw, list):
            raise ValueError("schema 'required' must be a list")
        required_names = {
            value.lower() for value in required_raw if isinstance(value, str)
        }
        override_values = (
            values_map.get(namelist_name, {}) if value_mode in {"filled", "minimal-filled"} else {}
        )

        if doc_mode == "documented":
            title = schema.get("title")
            if isinstance(title, str) and title.strip():
                lines.append(f"! {title.strip()}")

        lines.append(f"&{namelist_name}")

        current_property: str | None = None
        try:
            for name, prop in properties.items():
                current_property = name
                if not isinstance(prop, dict):
                    raise ValueError(f"property '{name}' must be an object")
                _reject_runtime_dimension_lengths(prop, dimensions)
                type_info = _field_type_info(prop, constants)
                _collect_dimension_constants(type_info.dimensions, shape_constants)
                _validate_kind_allowlist(type_info, kind_map, kind_allowlist)

                requirement = analyze_property_requirement(
                    name,
                    prop,
                    declared_required=name.lower() in required_names,
                )
                has_default = requirement.fully_default_initialized
                has_override = name in override_values
                if value_mode in {"minimal-empty", "minimal-filled"} and has_default:
                    if value_mode == "minimal-filled" and has_override:
                        pass
                    else:
                        continue

                if doc_mode == "documented":
                    title = prop.get("title")
                    if isinstance(title, str) and title.strip():
                        lines.append(f"  ! {title.strip()}")

                entries = _value_entries(
                    name,
                    prop,
                    type_info,
                    value_mode=value_mode,
                    simple_derived_mode=simple_derived_mode,
                    override=override_values.get(name, _MISSING),
                    constants=shape_constants,
                )
                derived = _derived_schema(prop)
                if (
                    doc_mode == "documented"
                    and simple_derived_mode == "buffer"
                    and derived is not None
                    and _is_simple_derived_schema(derived)
                    and entries
                ):
                    component_names = ", ".join(str(key) for key in derived["properties"])
                    lines.append(f"  ! Component order: {component_names}")
                for entry_name, value_text in entries:
                    if value_text is None:
                        lines.append(f"  {entry_name} =")
                    else:
                        lines.append(f"  {entry_name} = {value_text}")
        except ValueError as exc:
            if current_property is None:
                raise
            msg = str(exc)
            if f"property '{current_property}'" in msg:
                raise
            raise ValueError(f"property '{current_property}': {msg}") from exc

        lines.append("/")
        lines.append("")

    if lines and lines[-1] == "":
        lines.pop()
    return "\n".join(lines) + "\n"


def _append_template_comment(lines: list[str], value: str | None) -> None:
    if not isinstance(value, str) or not value.strip():
        return
    for line in value.strip().splitlines():
        lines.append(f"! {line.strip()}" if line.strip() else "!")


def _array_slice(rank: int) -> str:
    return "(" + ",".join(":" for _ in range(rank)) + ")"


def _value_entries(
    name: str,
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    *,
    value_mode: str,
    simple_derived_mode: str,
    override: Any,
    constants: dict[str, int] | None,
    fallback_default: Any = _MISSING,
) -> list[tuple[str, str | None]]:
    derived = _derived_schema(prop)
    if derived is not None:
        if simple_derived_mode == "buffer" and _is_simple_derived_schema(derived):
            return _derived_buffer_entries(
                name,
                derived,
                type_info,
                value_mode=value_mode,
                override=override,
                constants=constants,
            )
        return _derived_value_entries(
            name,
            prop,
            derived,
            type_info,
            value_mode=value_mode,
            override=override,
            constants=constants,
        )
    if value_mode in {"empty", "minimal-empty"}:
        entry_name = name
        if type_info.category == "array":
            entry_name = f"{name}{_array_slice(len(type_info.dimensions))}"
        return [(entry_name, None)]

    enum_values = _enum_values(prop, type_info, constants)

    if override is not _MISSING:
        return _entries_from_value(name, override, type_info, prop, constants)

    example_value = _get_first_example(prop, type_info)
    if example_value is not None:
        return _entries_from_value(name, example_value, type_info, prop, constants)

    if fallback_default is not _MISSING:
        return _entries_from_value(name, fallback_default, type_info, prop, constants)

    if type_info.category == "array":
        array_default = _array_default_value(prop)
        if array_default is not None:
            default_value, default_from_items = array_default
            if default_from_items and isinstance(default_value, list):
                raise ValueError("array items default must be a scalar")
            if isinstance(default_value, list):
                return _array_list_entries(name, default_value, type_info, prop, constants)
            scalar = _format_scalar_default(
                default_value,
                None,
                type_info.element_category,
            )
            entry_name = f"{name}{_array_slice(len(type_info.dimensions))}"
            return [(entry_name, scalar)]
    elif "default" in prop:
        default_value = prop["default"]
        scalar = _format_scalar_default(default_value, None, type_info.category)
        return [(name, scalar)]

    if enum_values:
        return _entries_from_value(name, enum_values[0], type_info, prop, constants)

    if type_info.category == "array":
        scalar = _format_scalar_default(
            _fallback_scalar_value(type_info),
            None,
            type_info.element_category,
        )
        entry_name = f"{name}{_array_slice(len(type_info.dimensions))}"
        return [(entry_name, scalar)]

    scalar = _format_scalar_default(
        _fallback_scalar_value(type_info),
        None,
        type_info.category,
    )
    return [(name, scalar)]


def _derived_value_entries(
    name: str,
    prop: dict[str, Any],
    derived: dict[str, Any],
    type_info: FieldTypeInfo,
    *,
    value_mode: str,
    override: Any,
    constants: dict[str, int] | None,
) -> list[tuple[str, str | None]]:
    properties = derived.get("properties")
    if not isinstance(properties, dict) or not properties:
        raise ValueError(f"derived property '{name}' must define properties")
    is_array = type_info.category == "array"
    object_default = derived_object_default(name, prop)
    entries: list[tuple[str, str | None]] = []
    if is_array and override is not _MISSING:
        if not isinstance(override, list):
            raise ValueError(f"derived array template value '{name}' must be an array of tables")
        for index, value in enumerate(override, start=1):
            if not isinstance(value, dict):
                raise ValueError(
                    f"derived array template value '{name}[{index}]' must be a table"
                )
            value_map = _as_str_keyed_table(
                value,
                f"derived array template value '{name}[{index}]'",
            )
            entries.extend(
                _derived_component_entries(
                    f"{name}({index})",
                    properties,
                    value_mode=value_mode,
                    overrides=value_map,
                    object_default=object_default,
                    constants=constants,
                )
            )
        return entries
    if override is not _MISSING and not isinstance(override, dict):
        raise ValueError(f"derived template value '{name}' must be a table")
    override_map: dict[str, Any] = {}
    if override is not _MISSING:
        override_map = _as_str_keyed_table(override, f"derived template value '{name}'")
    prefix = name
    if is_array:
        prefix = f"{name}{_array_slice(len(type_info.dimensions))}"
    return _derived_component_entries(
        prefix,
        properties,
        value_mode=value_mode,
        overrides=override_map,
        object_default=object_default,
        constants=constants,
    )


def _as_str_keyed_table(value: object, context: str) -> dict[str, Any]:
    if not isinstance(value, dict):
        raise ValueError(f"{context} must be a table")
    table: dict[str, Any] = {}
    for key, entry in value.items():
        if not isinstance(key, str):
            raise ValueError(f"{context} must use string keys")
        table[key] = entry
    return table


def _derived_component_entries(
    prefix: str,
    properties: dict[str, Any],
    *,
    value_mode: str,
    overrides: dict[str, Any],
    object_default: dict[str, Any],
    constants: dict[str, int] | None,
) -> list[tuple[str, str | None]]:
    for key in overrides:
        if key not in properties:
            raise ValueError(f"derived template value '{prefix}%{key}' is unknown")
    entries: list[tuple[str, str | None]] = []
    for child_name, child in properties.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            raise ValueError(f"derived property '{prefix}' components must be schema objects")
        child_type = _field_type_info(child, constants)
        entries.extend(
            _value_entries(
                f"{prefix}%{child_name}",
                child,
                child_type,
                value_mode=value_mode,
                simple_derived_mode="components",
                override=overrides.get(child_name, _MISSING),
                constants=constants,
                fallback_default=object_default.get(child_name.lower(), _MISSING),
            )
        )
    return entries


def _derived_buffer_entries(
    name: str,
    derived: dict[str, Any],
    type_info: FieldTypeInfo,
    *,
    value_mode: str,
    override: Any,
    constants: dict[str, int] | None,
) -> list[tuple[str, str | None]]:
    properties = derived.get("properties")
    if not isinstance(properties, dict) or not properties:
        raise ValueError(f"derived property '{name}' must define properties")

    is_array = type_info.category == "array"
    object_default = derived_object_default(name, derived)
    coordinates: list[list[int]] = [[]]
    if is_array:
        dimensions = _parse_default_dimensions(type_info.dimensions, constants)
        coordinates = [
            [index + 1 for index in coordinate]
            for coordinate in _iter_fixed_indices(dimensions, "F")
        ]

    if value_mode in {"empty", "minimal-empty"}:
        return [(_derived_element_name(name, coordinate), None) for coordinate in coordinates]

    override_maps: list[dict[str, Any]]
    if is_array and override is not _MISSING:
        if not isinstance(override, list):
            raise ValueError(f"derived array template value '{name}' must be an array of tables")
        if len(override) > len(coordinates):
            raise ValueError(f"derived array template value '{name}' is longer than its shape")
        coordinates = coordinates[: len(override)]
        override_maps = []
        for index, value in enumerate(override, start=1):
            override_maps.append(
                _as_str_keyed_table(
                    value,
                    f"derived array template value '{name}[{index}]'",
                )
            )
    else:
        if override is not _MISSING and not isinstance(override, dict):
            raise ValueError(f"derived template value '{name}' must be a table")
        override_map = (
            _as_str_keyed_table(override, f"derived template value '{name}'")
            if override is not _MISSING
            else {}
        )
        override_maps = [override_map for _ in coordinates]

    entries: list[tuple[str, str | None]] = []
    for coordinate, overrides in zip(coordinates, override_maps):
        prefix = _derived_element_name(name, coordinate)
        entries.append(
            (
                prefix,
                _derived_buffer_value(
                    prefix,
                    properties,
                    overrides,
                    value_mode,
                    constants,
                    object_default,
                ),
            )
        )
    return entries


def _derived_element_name(name: str, coordinate: list[int]) -> str:
    if not coordinate:
        return name
    return f"{name}({','.join(str(index) for index in coordinate)})"


def _derived_buffer_value(
    prefix: str,
    properties: dict[str, Any],
    overrides: dict[str, Any],
    value_mode: str,
    constants: dict[str, int] | None,
    object_default: dict[str, Any] | None = None,
) -> str | None:
    for key in overrides:
        if key not in properties:
            raise ValueError(f"derived template value '{prefix}%{key}' is unknown")
    slots: list[str | None] = []
    selected_default = object_default or {}
    for child_name, child in properties.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            raise ValueError(f"derived property '{prefix}' components must be schema objects")
        child_entries = _value_entries(
            child_name,
            child,
            _field_type_info(child, constants),
            value_mode=value_mode,
            simple_derived_mode="components",
            override=overrides.get(child_name, _MISSING),
            constants=constants,
            fallback_default=selected_default.get(child_name.lower(), _MISSING),
        )
        if len(child_entries) != 1:
            raise ValueError(f"derived property '{prefix}%{child_name}' must be scalar")
        slots.append(child_entries[0][1])
    while slots and slots[-1] is None:
        slots.pop()
    if not slots:
        return None
    return ", ".join("" if slot is None else slot for slot in slots)


def _array_list_entries(
    name: str,
    values: list[Any],
    type_info: FieldTypeInfo,
    prop: dict[str, Any],
    constants: dict[str, int] | None,
) -> list[tuple[str, str | None]]:
    rank = len(type_info.dimensions)
    if rank <= 1:
        entry_name = f"{name}{_array_slice(rank)}"
        return [(entry_name, _format_value_list(values, type_info))]

    dims = _parse_default_dimensions(type_info.dimensions, constants)
    order = _resolve_default_order(prop)
    slice_dim = _slice_dim_for_order(rank, order)
    slice_size = dims[slice_dim]
    fixed_dims = [dims[idx] for idx in range(rank) if idx != slice_dim]
    entries: list[tuple[str, str | None]] = []
    offset = 0
    for fixed_indices in _iter_fixed_indices(fixed_dims, order):
        if offset >= len(values):
            break
        chunk = values[offset : offset + slice_size]
        if not chunk:
            break
        entry_name = _slice_entry_name(
            name,
            rank,
            slice_dim,
            [index + 1 for index in fixed_indices],
        )
        entries.append((entry_name, _format_value_list(chunk, type_info)))
        offset += slice_size
    if not entries:
        entry_name = f"{name}{_array_slice(rank)}"
        entries.append((entry_name, _format_value_list(values, type_info)))
    return entries


def _slice_dim_for_order(rank: int, order: str) -> int:
    if order == "F":
        return 0
    return rank - 1


def _iter_fixed_indices(dims: list[int], order: str) -> Iterable[list[int]]:
    if not dims:
        yield []
        return
    indices = [0] * len(dims)
    while True:
        yield indices.copy()
        positions = range(len(dims)) if order == "F" else range(len(dims) - 1, -1, -1)
        for pos in positions:
            indices[pos] += 1
            if indices[pos] < dims[pos]:
                break
            indices[pos] = 0
        else:
            break


def _slice_entry_name(
    name: str,
    rank: int,
    slice_dim: int,
    fixed_indices: list[int],
) -> str:
    indices: list[str] = []
    fixed_iter = iter(fixed_indices)
    for idx in range(rank):
        if idx == slice_dim:
            indices.append(":")
        else:
            indices.append(str(next(fixed_iter)))
    return f"{name}({','.join(indices)})"


def _format_value_list(values: list[Any], type_info: FieldTypeInfo) -> str:
    formatted = [
        _format_scalar_default(value, None, type_info.element_category) for value in values
    ]
    return ", ".join(formatted)


def _entries_from_value(
    name: str,
    value: Any,
    type_info: FieldTypeInfo,
    prop: dict[str, Any],
    constants: dict[str, int] | None,
) -> list[tuple[str, str | None]]:
    if type_info.category == "array":
        if isinstance(value, list):
            _validate_example_list(value)
            return _array_list_entries(name, value, type_info, prop, constants)
        scalar = _format_scalar_default(value, None, type_info.element_category)
        entry_name = f"{name}{_array_slice(len(type_info.dimensions))}"
        return [(entry_name, scalar)]
    if isinstance(value, list):
        raise ValueError("scalar template values must not be lists")
    scalar = _format_scalar_default(value, None, type_info.category)
    return [(name, scalar)]


def _get_first_example(prop: dict[str, Any], type_info: FieldTypeInfo) -> Any | None:
    examples = prop.get("examples")
    if examples is None:
        return None
    if not isinstance(examples, list):
        raise ValueError("property examples must be a list")
    if not examples:
        return None
    first = examples[0]
    if type_info.category == "array":
        if isinstance(first, list):
            _validate_example_list(first)
        return first
    if isinstance(first, list):
        raise ValueError("scalar examples must not be lists")
    return first


def _validate_example_list(values: list[Any]) -> None:
    for value in values:
        if isinstance(value, list):
            raise ValueError("array examples must be flat lists")


def _fallback_scalar_value(type_info: FieldTypeInfo) -> Any:
    category = type_info.element_category if type_info.category == "array" else type_info.category
    if category == "integer":
        return 0
    if category == "real":
        return 0.0
    if category == "boolean":
        return False
    if category == "string":
        return ""
    raise ValueError(f"unsupported template category '{category}'")


def _validate_kind_allowlist(
    type_info: FieldTypeInfo,
    kind_map: dict[str, str] | None,
    allowlist: set[str] | None,
) -> None:
    if type_info.kind is None or allowlist is None:
        return
    kind_id = type_info.kind
    mapped = None
    if kind_map is not None and kind_id in kind_map:
        mapped = kind_map[kind_id]
        if not isinstance(mapped, str) or not mapped:
            raise ValueError(f"kind map target for '{kind_id}' must be a string")
        if mapped not in allowlist:
            raise ValueError(
                f"kind map target '{mapped}' for '{kind_id}' not present in kind module list"
            )
        return
    if kind_id not in allowlist:
        raise ValueError(f"kind '{kind_id}' not present in kind module list")


def _resolve_default_order(prop: dict[str, Any]) -> str:
    order_raw = prop.get("x-fortran-default-order", "F")
    if not isinstance(order_raw, str):
        raise ValueError("array default order must be 'F' or 'C'")
    order = order_raw.upper()
    if order not in {"F", "C"}:
        raise ValueError("array default order must be 'F' or 'C'")
    return order

MuellerSeb / nml-tools / 29619881679

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous