29619881679

Committed 17 Jul 2026 11:03PM UTC coverage: 86.477% (+0.4%) from 86.054%

Build # 29619881679

Build Type

Pull #63

github

Committed by

MuellerSeb

Commit Message

Allow defaults to satisfy required values

Separate declared requiredness from effective input requirements across validation, namelist evaluation, and generated APIs. Support scalar derived object defaults and broadcast item defaults with deterministic coverage and precedence rules.

Update Fortran, f2py/Python, Markdown, templates, documentation, tests, and regenerated derived-type example artifacts.

Pull Request Pull Request #63: Allow defaults to satisfy required values

Coverage Stats

270 of 287 new or added lines in 7 files covered. (94.08%)

3 existing lines in 1 file now uncovered.

5039 of 5827 relevant lines covered (86.48%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.65

/src/nml_tools/codegen_markdown.py

"""Markdown documentation generation."""

from __future__ import annotations

import re
import string
from pathlib import Path
from typing import Any

from ._utils import (
    normalize_constant_values,
    normalize_runtime_dimensions,
    reject_constant_dimension_overlap,
    validate_user_fortran_identifier,
)
from .codegen_fortran import (
    FieldTypeInfo,
    _array_default_value,
    _bounds_spec,
    _collect_dimension_constants,
    _derived_origin,
    _derived_schema,
    _enum_category,
    _enum_values,
    _field_type_info,
    _format_scalar_default,
    _parse_default_dimensions,
    _prepare_array_default,
    _reject_runtime_dimension_lengths,
)
from .codegen_template import render_template
from .schema import _is_simple_derived_schema, get_string_format
from .validate import (
    analyze_property_requirement,
    derived_component_defaults,
    derived_object_default,
    validate_schema_defaults,
)

_DEFAULT_MISSING = object()


def generate_docs(
    schema: dict[str, Any],
    output: str | Path,
    *,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    md_doxygen_id_from_name: bool = False,
    md_add_toc_statement: bool = False,
) -> None:
    """Generate Markdown docs for *schema* at *output*."""
    rendered = render_docs(
        schema,
        constants=constants,
        dimensions=dimensions,
        md_doxygen_id_from_name=md_doxygen_id_from_name,
        md_add_toc_statement=md_add_toc_statement,
    )
    output_path = Path(output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(rendered, encoding="ascii")


def render_docs(
    schema: dict[str, Any],
    *,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    md_doxygen_id_from_name: bool = False,
    md_add_toc_statement: bool = False,
) -> str:
    """Render Markdown docs for *schema*."""
    namelist_name = schema.get("x-fortran-namelist")
    if not isinstance(namelist_name, str) or not namelist_name.strip():
        raise ValueError("schema must define 'x-fortran-namelist'")
    validate_user_fortran_identifier(namelist_name, label="'x-fortran-namelist'")

    if schema.get("type") != "object":
        raise ValueError("schema root must be of type 'object'")

    properties = schema.get("properties")
    if not isinstance(properties, dict) or not properties:
        raise ValueError("schema must define object 'properties'")

    title = schema.get("title", namelist_name)
    if not isinstance(title, str):
        raise ValueError("schema title must be a string")
    description = schema.get("description")
    if description is not None and not isinstance(description, str):
        raise ValueError("schema description must be a string")

    required_raw = schema.get("required", [])
    if required_raw is None:
        required_raw = []
    if not isinstance(required_raw, list):
        raise ValueError("schema 'required' must be a list")
    required_set = _validate_required(required_raw)
    constants = normalize_constant_values(constants)
    dimensions = normalize_runtime_dimensions(dimensions)
    reject_constant_dimension_overlap(constants, dimensions)
    validate_schema_defaults(schema, constants=constants, dimensions=dimensions)
    shape_constants: dict[str, int] = {
        **constants,
        **dimensions,
    }

    title_line = f"# {title}"
    if md_doxygen_id_from_name:
        title_line = f"{title_line} {{#{namelist_name}}}"
    lines = [title_line, ""]
    if md_add_toc_statement:
        lines.append("[TOC]")
        lines.append("")
    if description:
        lines.append(description)
        lines.append("")
    lines.append(f"**Namelist**: `{namelist_name}`")
    lines.append("")
    lines.append("## Fields")
    lines.append("")

    header = ["Name", "Type", "Declared required", "Input required", "Info"]
    lines.append(f"| {' | '.join(header)} |")
    lines.append(f"| {' | '.join('---' for _ in header)} |")

    current_property: str | None = None
    try:
        for name, prop in properties.items():
            current_property = name
            if not isinstance(prop, dict):
                raise ValueError(f"property '{name}' must be an object")
            _reject_runtime_dimension_lengths(prop, dimensions)
            type_info = _field_type_info(prop, constants)
            _collect_dimension_constants(type_info.dimensions, shape_constants)
            type_label = _format_table_type(type_info)
            info_label = _format_info(prop)
            declared_required = name.lower() in required_set
            requirement = analyze_property_requirement(
                name,
                prop,
                declared_required=declared_required,
            )
            row = [
                f"[{name}](#{_github_section_id(name)})",
                type_label,
                "yes" if declared_required else "no",
                "yes" if requirement.requires_input else "no",
                info_label,
            ]
            lines.append(f"| {' | '.join(_escape_table_cell(cell) for cell in row)} |")
    except ValueError as exc:
        if current_property is None:
            raise
        msg = str(exc)
        if f"property '{current_property}'" in msg:
            raise
        raise ValueError(f"property '{current_property}': {msg}") from exc

    lines.append("")

    lines.append("## Field details")
    lines.append("")

    current_property = None
    try:
        for name, prop in properties.items():
            current_property = name
            if not isinstance(prop, dict):
                raise ValueError(f"property '{name}' must be an object")
            _reject_runtime_dimension_lengths(prop, dimensions)
            type_info = _field_type_info(prop, constants)
            declared_required = name.lower() in required_set
            requirement = analyze_property_requirement(
                name,
                prop,
                declared_required=declared_required,
            )
            default_label = _get_default_value(prop, type_info, shape_constants)
            enum_label = _get_enum_values(prop, type_info, shape_constants)
            example_values = _get_example_values(prop, type_info)
            flex_tail_dims = _get_flex_tail_dims(prop, type_info)
            title = _get_title(prop)
            description_text = _get_description(prop)

            # if title:
            #     lines.append(f"### `{name}` - {title}")
            # else:
            #     lines.append(f"### `{name}`")
            lines.append(f"### {name}")
            lines.append("")
            if title:
                lines.append(f"{title} `{name}`")
            else:
                lines.append(f"`{name}`")
            lines.append("")
            if description_text:
                lines.append(description_text)
                lines.append("")

            lines.append("Summary:")
            lines.append(f"- Type: `{_format_specific_type(type_info)}`")
            format_label = _get_format_label(prop, type_info)
            if format_label is not None:
                lines.append(format_label)
            if type_info.category == "array" and flex_tail_dims > 0:
                lines.append(f"- Flexible tail dims: {flex_tail_dims}")
            lines.append(f"- Declared required: {'yes' if declared_required else 'no'}")
            lines.append(f"- Input required: {'yes' if requirement.requires_input else 'no'}")
            if default_label is not None:
                if isinstance(default_label, tuple):
                    base, note = default_label
                    if note:
                        lines.append(f"- Default: `{base}` {note}")
                    else:
                        lines.append(f"- Default: `{base}`")
                else:
                    lines.append(f"- Default: `{default_label}`")
            for bounds_label in _get_bounds_labels(prop, type_info):
                lines.append(bounds_label)
            if enum_label is not None:
                lines.append(f"- Allowed values: {enum_label}")
            if example_values is not None:
                examples_text = ", ".join(f"`{value}`" for value in example_values)
                lines.append(f"- Examples: {examples_text}")
            lines.append("")
            derived = _derived_schema(prop)
            if derived is not None:
                _append_derived_field_components(
                    lines,
                    name,
                    prop,
                    derived,
                    shape_constants,
                )
    except ValueError as exc:
        if current_property is None:
            raise
        msg = str(exc)
        if f"property '{current_property}'" in msg:
            raise
        raise ValueError(f"property '{current_property}': {msg}") from exc

    derived_types = _collect_documented_derived_types(properties)
    if derived_types:
        lines.append("## Derived types")
        lines.append("")
        for derived in derived_types:
            _append_derived_type_documentation(lines, derived, shape_constants)

    lines.append("## Example")
    lines.append("")
    lines.append("```fortran")
    filled_template = render_template(
        [schema],
        doc_mode="plain",
        value_mode="filled",
        constants=shape_constants,
        kind_map=None,
        kind_allowlist=None,
    )
    lines.extend(filled_template.rstrip("\n").splitlines())
    lines.append("```")
    lines.append("")

    return "\n".join(lines) + "\n"


def _validate_required(values: list[Any]) -> set[str]:
    required: set[str] = set()
    for value in values:
        if not isinstance(value, str):
            raise ValueError("schema 'required' entries must be strings")
        required.add(value.lower())
    return required


def _format_table_type(type_info: FieldTypeInfo) -> str:
    if type_info.category == "array":
        if type_info.element_category == "derived":
            return f"{type_info.type_spec} array"
        element = _format_scalar_type_name(type_info.element_category)
        return f"{element} array"
    if type_info.category == "derived":
        return type_info.type_spec
    return _format_scalar_type_name(type_info.category)


def _format_scalar_type_name(category: str | None) -> str:
    if category == "boolean":
        return "logical"
    if category == "string":
        return "string"
    if category == "integer":
        return "integer"
    if category == "real":
        return "real"
    raise ValueError(f"unsupported type category '{category}'")


def _format_specific_type(type_info: FieldTypeInfo) -> str:
    if type_info.category != "array":
        return type_info.type_spec
    dimensions = ", ".join(type_info.dimensions)
    return f"{type_info.type_spec}, dimension({dimensions})"


def _append_derived_field_components(
    lines: list[str],
    field_name: str,
    prop: dict[str, Any],
    derived: dict[str, Any],
    constants: dict[str, int] | None,
) -> None:
    components = derived.get("properties")
    if not isinstance(components, dict):
        return
    required = {
        value.lower()
        for value in derived.get("required", [])
        if isinstance(value, str)
    }
    effective_defaults = derived_component_defaults(field_name, prop)
    object_defaults = derived_object_default(field_name, prop)
    object_source = "item default" if prop.get("type") == "array" else "object default"
    lines.append("Components:")
    for child_name, child in components.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            continue
        type_info = _field_type_info(child, constants)
        path = f"{field_name}%{child_name}"
        child_key = child_name.lower()
        declared_required = child_key in required
        input_required = declared_required and child_key not in effective_defaults
        details = [
            f"`{_format_specific_type(type_info)}`",
            f"declared required {'yes' if declared_required else 'no'}",
            f"input required {'yes' if input_required else 'no'}",
        ]
        if child_key in effective_defaults:
            default_text = _format_scalar_default(
                effective_defaults[child_key],
                None,
                type_info.category,
            )
            source = object_source if child_key in object_defaults else "component default"
            details.append(f"default `{default_text}` ({source})")
        format_label = _get_format_label(child, type_info)
        if format_label is not None:
            details.append(format_label[2:] if format_label.startswith("- ") else format_label)
        bounds = _get_bounds_labels(child, type_info)
        details.extend(label[2:] if label.startswith("- ") else label for label in bounds)
        enum = _get_enum_values(child, type_info, constants)
        if enum is not None:
            details.append(f"allowed values {enum}")
        lines.append(f"- `{path}`: " + "; ".join(details))
    lines.append("")


def _collect_documented_derived_types(properties: dict[str, Any]) -> list[dict[str, Any]]:
    seen: set[tuple[str, str]] = set()
    collected: list[dict[str, Any]] = []
    for prop in properties.values():
        if not isinstance(prop, dict):
            continue
        derived = _derived_schema(prop)
        if derived is None:
            continue
        origin = _derived_origin(derived)
        identity = origin["identity"]
        if identity in seen:
            continue
        seen.add(identity)
        collected.append(origin["definition"])
    return collected


def _append_derived_type_documentation(
    lines: list[str],
    derived: dict[str, Any],
    constants: dict[str, int] | None,
) -> None:
    type_name = derived.get("x-fortran-type")
    if not isinstance(type_name, str):
        return
    title = derived.get("title", type_name)
    lines.append(f"### `{type_name}`")
    lines.append("")
    if isinstance(title, str) and title != type_name:
        lines.append(title)
        lines.append("")
    description = derived.get("description")
    if isinstance(description, str) and description.strip():
        lines.append(description.strip())
        lines.append("")
    module = derived.get("x-fortran-module")
    ownership = f"imported from `{module}`" if isinstance(module, str) else "`nml_helper`"
    lines.append(f"- Ownership: {ownership}")
    components = derived.get("properties")
    if isinstance(components, dict):
        required = {
            value.lower()
            for value in derived.get("required", [])
            if isinstance(value, str)
        }
        effective_defaults = derived_component_defaults(type_name, derived)
        object_defaults = derived_object_default(type_name, derived)
        lines.append(
            "- Buffer-compatible: " + ("yes" if _is_simple_derived_schema(derived) else "no")
        )
        lines.append(f"- Component order: {', '.join(str(name) for name in components)}")
        if isinstance(module, str):
            lines.append(
                "- **Declaration-order contract:** the imported Fortran type must declare "
                "components in the resolved schema order shown above."
            )
        for child_name, child in components.items():
            if not isinstance(child_name, str) or not isinstance(child, dict):
                continue
            info = _field_type_info(child, constants)
            child_key = child_name.lower()
            declared_required = child_key in required
            input_required = declared_required and child_key not in effective_defaults
            details = [
                f"`{_format_specific_type(info)}`",
                f"declared required {'yes' if declared_required else 'no'}",
                f"input required {'yes' if input_required else 'no'}",
            ]
            if child_key in effective_defaults:
                default_text = _format_scalar_default(
                    effective_defaults[child_key],
                    None,
                    info.category,
                )
                source = (
                    "object default" if child_key in object_defaults else "component default"
                )
                details.append(f"default `{default_text}` ({source})")
            format_label = _get_format_label(child, info)
            if format_label is not None:
                details.append(format_label[2:] if format_label.startswith("- ") else format_label)
            lines.append(f"- `{child_name}`: " + "; ".join(details))
    lines.append("")


def _get_default_value(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
) -> str | tuple[str, str | None] | None:
    derived = _derived_schema(prop)
    if derived is not None:
        object_default = derived_object_default("field", prop)
        if not object_default:
            return None
        base = _format_derived_default_mapping(derived, object_default, constants)
        if type_info.category == "array":
            return base, "(broadcast item default)"
        return base
    if type_info.category == "array":
        array_default = _array_default_value(prop)
        if array_default is None:
            return None
        default_value, default_from_items = array_default
        return _format_array_default_display(
            prop,
            type_info,
            constants,
            default_value=default_value,
            from_items=default_from_items,
        )
    if "default" not in prop:
        return None
    return _format_default_plain(prop["default"], type_info, prop, constants)


def _format_derived_default_mapping(
    derived: dict[str, Any],
    default: dict[str, Any],
    constants: dict[str, int] | None,
) -> str:
    components = derived.get("properties")
    if not isinstance(components, dict):
        raise ValueError("derived object must define properties")
    entries: list[str] = []
    for child_name, child in components.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            raise ValueError("derived object components must be schema objects")
        key = child_name.lower()
        if key not in default:
            continue
        info = _field_type_info(child, constants)
        value = _format_scalar_default(default[key], None, info.category)
        entries.append(f"{child_name}: {value}")
    return "{" + ", ".join(entries) + "}"


def _format_info(prop: dict[str, Any]) -> str:
    title = _get_title(prop)
    return title or "n/a"


def _get_title(prop: dict[str, Any]) -> str | None:
    title = prop.get("title")
    if title is None:
        return None
    if not isinstance(title, str):
        raise ValueError("property title must be a string")
    title = title.strip()
    return title or None


def _get_description(prop: dict[str, Any]) -> str | None:
    description = prop.get("description")
    if description is None:
        return None
    if not isinstance(description, str):
        raise ValueError("property description must be a string")
    description = description.strip()
    return description or None


def _get_format_label(prop: dict[str, Any], type_info: FieldTypeInfo) -> str | None:
    if type_info.category == "array":
        items = prop.get("items")
        if not isinstance(items, dict):
            return None
        value = get_string_format(items)
        if value is None:
            return None
        return f"- Item format: `{value}`"
    value = get_string_format(prop)
    if value is None:
        return None
    return f"- Format: `{value}`"


def _get_enum_values(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
) -> str | None:
    enum_values = _enum_values(prop, type_info, constants)
    if enum_values is None:
        return None
    category = _enum_category(type_info)
    values = [_format_scalar_default(value, None, category) for value in enum_values]
    return ", ".join(f"`{value}`" for value in values)


def _get_bounds_labels(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> list[str]:
    bounds = _bounds_spec(prop, type_info)
    if bounds is None:
        return []
    category = bounds["category"]
    labels: list[str] = []
    min_value = bounds["min_value"]
    max_value = bounds["max_value"]
    if min_value is not None:
        op = ">" if bounds["min_exclusive"] else ">="
        literal = _format_scalar_default(min_value, None, category)
        labels.append(f"- Minimum: `{op} {literal}`")
    if max_value is not None:
        op = "<" if bounds["max_exclusive"] else "<="
        literal = _format_scalar_default(max_value, None, category)
        labels.append(f"- Maximum: `{op} {literal}`")
    return labels


def _get_flex_tail_dims(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> int:
    flex_raw = prop.get("x-fortran-flex-tail-dims")
    if flex_raw is None:
        flex_value = 0
    else:
        if isinstance(flex_raw, bool) or not isinstance(flex_raw, int):
            raise ValueError("property flex tail dims must be an integer")
        flex_value = flex_raw
    if flex_value < 0:
        raise ValueError("property flex tail dims must be >= 0")
    if flex_value == 0:
        return 0
    if type_info.category != "array":
        raise ValueError("flex tail dims only apply to arrays")
    if flex_value > len(type_info.dimensions):
        raise ValueError("flex tail dims must not exceed array rank")
    return flex_value


def _escape_table_cell(value: str) -> str:
    escaped = value.replace("|", "\\|").replace("\n", " ").strip()
    return escaped or "n/a"


def _get_example_values(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> list[str] | None:
    examples = prop.get("examples")
    if examples is None:
        return None
    if not isinstance(examples, list):
        raise ValueError("property examples must be a list")
    if not examples:
        return None
    return [_format_example_value(value, type_info) for value in examples]


def _format_example_value(value: Any, type_info: FieldTypeInfo) -> str:
    if type_info.category == "array":
        if isinstance(value, list):
            formatted = [
                _format_scalar_default(item, None, type_info.element_category) for item in value
            ]
            return f"[{', '.join(formatted)}]"
        return _format_scalar_default(value, None, type_info.element_category)
    if isinstance(value, list):
        raise ValueError("scalar examples must not be lists")
    return _format_scalar_default(value, None, type_info.category)


def _format_default_plain(
    value: Any,
    type_info: FieldTypeInfo,
    prop: dict[str, Any],
    constants: dict[str, int] | None,
) -> str:
    if type_info.category == "array":
        if not isinstance(value, list):
            raise ValueError("array default must be a list")
        parsed_dims = _parse_default_dimensions(type_info.dimensions, constants)
        array_default = _prepare_array_default(value, parsed_dims, prop)
        elements = [
            _format_scalar_default(element, None, type_info.element_category)
            for element in array_default.source_values
        ]
        if (
            len(type_info.dimensions) == 1
            and array_default.order_values is None
            and array_default.pad_values is None
        ):
            return f"[{', '.join(elements)}]"

        shape_literal = ", ".join(type_info.dimensions)
        arguments = [f"[{', '.join(elements)}]", f"shape=[{shape_literal}]"]

        if array_default.order_values is not None:
            order_literal = ", ".join(str(index) for index in array_default.order_values)
            arguments.append(f"order=[{order_literal}]")

        if array_default.pad_values is not None:
            pad_elements = [
                _format_scalar_default(element, None, type_info.element_category)
                for element in array_default.pad_values
            ]
            arguments.append(f"pad=[{', '.join(pad_elements)}]")

        return f"reshape({', '.join(arguments)})"

    return _format_scalar_default(value, None, type_info.category)


def _format_array_default_display(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
    *,
    default_value: Any = _DEFAULT_MISSING,
    from_items: bool = False,
) -> tuple[str, str | None]:
    if default_value is _DEFAULT_MISSING:
        default_value = prop["default"]
    if not from_items and not isinstance(default_value, list):
        raise ValueError("array default must be a list")
    default_is_list = isinstance(default_value, list)
    default_list = default_value if default_is_list else [default_value]

    parsed_dims = _parse_default_dimensions(type_info.dimensions, constants)
    default_prop = prop
    if from_items:
        default_prop = dict(prop)
        default_prop["x-fortran-default-repeat"] = True
    _prepare_array_default(default_list, parsed_dims, default_prop)

    if default_is_list:
        elements = [
            _format_scalar_default(value, None, type_info.element_category)
            for value in default_list
        ]
        base = f"[{', '.join(elements)}]"
    else:
        base = _format_scalar_default(default_value, None, type_info.element_category)

    repeat_raw = prop.get("x-fortran-default-repeat", False)
    if not isinstance(repeat_raw, bool):
        raise ValueError("array default repeat must be a boolean")
    notes: list[str] = []
    if repeat_raw:
        notes.append("repeated")

    order_raw = prop.get("x-fortran-default-order", "F")
    if not isinstance(order_raw, str):
        raise ValueError("array default order must be 'F' or 'C'")
    order = order_raw.upper()
    if order not in {"F", "C"}:
        raise ValueError("array default order must be 'F' or 'C'")
    if order == "C":
        notes.append("order: C")

    pad_raw = prop.get("x-fortran-default-pad")
    if pad_raw is not None:
        pad_list = pad_raw if isinstance(pad_raw, list) else [pad_raw]
        pad_elements = [
            _format_scalar_default(value, None, type_info.element_category) for value in pad_list
        ]
        if isinstance(pad_raw, list):
            pad_text = f"[{', '.join(pad_elements)}]"
        else:
            pad_text = pad_elements[0]
        notes.append(f"pad: `{pad_text}`")

    if notes:
        return base, f"({', '.join(notes)})"
    return base, None


def _github_section_id(title: str) -> str:
    # lowercase
    s = title.strip().lower()
    # remove punctuation characters
    punctuation = string.punctuation.replace("_", "")
    s = re.sub(f"[{re.escape(punctuation)}]", "", s)
    # replace any whitespace run with '-'
    s = re.sub(r"\s+", "-", s)
    return s

MuellerSeb / nml-tools / 29619881679

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous