29620967587

Committed 17 Jul 2026 11:30PM UTC coverage: 86.477% (+0.4%) from 86.054%

Build # 29620967587

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #63 from MuellerSeb/update_derived_type_handling

Allow defaults to satisfy required values

Coverage Stats

270 of 287 new or added lines in 7 files covered. (94.08%)

3 existing lines in 1 file now uncovered.

5039 of 5827 relevant lines covered (86.48%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.65

/src/nml_tools/codegen_markdown.py

"""Markdown documentation generation."""

from __future__ import annotations

import re
import string
from pathlib import Path
from typing import Any

from ._utils import (
    normalize_constant_values,
    normalize_runtime_dimensions,
    reject_constant_dimension_overlap,
    validate_user_fortran_identifier,
)
from .codegen_fortran import (
    FieldTypeInfo,
    _array_default_value,
    _bounds_spec,
    _collect_dimension_constants,
    _derived_origin,
    _derived_schema,
    _enum_category,
    _enum_values,
    _field_type_info,
    _format_scalar_default,
    _parse_default_dimensions,
    _prepare_array_default,
    _reject_runtime_dimension_lengths,
)
from .codegen_template import render_template
from .schema import _is_simple_derived_schema, get_string_format
from .validate import (
    analyze_property_requirement,
    derived_component_defaults,
    derived_object_default,
    validate_schema_defaults,
)

_DEFAULT_MISSING = object()


def generate_docs(
    schema: dict[str, Any],
    output: str | Path,
    *,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    md_doxygen_id_from_name: bool = False,
    md_add_toc_statement: bool = False,
) -> None:
    """Generate Markdown docs for *schema* at *output*."""
    rendered = render_docs(
        schema,
        constants=constants,
        dimensions=dimensions,
        md_doxygen_id_from_name=md_doxygen_id_from_name,
        md_add_toc_statement=md_add_toc_statement,
    )
    output_path = Path(output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(rendered, encoding="ascii")


def render_docs(
    schema: dict[str, Any],
    *,
    constants: dict[str, int] | None = None,
    dimensions: dict[str, int] | None = None,
    md_doxygen_id_from_name: bool = False,
    md_add_toc_statement: bool = False,
) -> str:
    """Render Markdown docs for *schema*."""
    namelist_name = schema.get("x-fortran-namelist")
    if not isinstance(namelist_name, str) or not namelist_name.strip():
        raise ValueError("schema must define 'x-fortran-namelist'")
    validate_user_fortran_identifier(namelist_name, label="'x-fortran-namelist'")

    if schema.get("type") != "object":
        raise ValueError("schema root must be of type 'object'")

    properties = schema.get("properties")
    if not isinstance(properties, dict) or not properties:
        raise ValueError("schema must define object 'properties'")

    title = schema.get("title", namelist_name)
    if not isinstance(title, str):
        raise ValueError("schema title must be a string")
    description = schema.get("description")
    if description is not None and not isinstance(description, str):
        raise ValueError("schema description must be a string")

    required_raw = schema.get("required", [])
    if required_raw is None:
        required_raw = []
    if not isinstance(required_raw, list):
        raise ValueError("schema 'required' must be a list")
    required_set = _validate_required(required_raw)
    constants = normalize_constant_values(constants)
    dimensions = normalize_runtime_dimensions(dimensions)
    reject_constant_dimension_overlap(constants, dimensions)
    validate_schema_defaults(schema, constants=constants, dimensions=dimensions)
    shape_constants: dict[str, int] = {
        **constants,
        **dimensions,
    }

    title_line = f"# {title}"
    if md_doxygen_id_from_name:
        title_line = f"{title_line} {{#{namelist_name}}}"
    lines = [title_line, ""]
    if md_add_toc_statement:
        lines.append("[TOC]")
        lines.append("")
    if description:
        lines.append(description)
        lines.append("")
    lines.append(f"**Namelist**: `{namelist_name}`")
    lines.append("")
    lines.append("## Fields")
    lines.append("")

    header = ["Name", "Type", "Declared required", "Input required", "Info"]
    lines.append(f"| {' | '.join(header)} |")
    lines.append(f"| {' | '.join('---' for _ in header)} |")

    current_property: str | None = None
    try:
        for name, prop in properties.items():
            current_property = name
            if not isinstance(prop, dict):
                raise ValueError(f"property '{name}' must be an object")
            _reject_runtime_dimension_lengths(prop, dimensions)
            type_info = _field_type_info(prop, constants)
            _collect_dimension_constants(type_info.dimensions, shape_constants)
            type_label = _format_table_type(type_info)
            info_label = _format_info(prop)
            declared_required = name.lower() in required_set
            requirement = analyze_property_requirement(
                name,
                prop,
                declared_required=declared_required,
            )
            row = [
                f"[{name}](#{_github_section_id(name)})",
                type_label,
                "yes" if declared_required else "no",
                "yes" if requirement.requires_input else "no",
                info_label,
            ]
            lines.append(f"| {' | '.join(_escape_table_cell(cell) for cell in row)} |")
    except ValueError as exc:
        if current_property is None:
            raise
        msg = str(exc)
        if f"property '{current_property}'" in msg:
            raise
        raise ValueError(f"property '{current_property}': {msg}") from exc

    lines.append("")

    lines.append("## Field details")
    lines.append("")

    current_property = None
    try:
        for name, prop in properties.items():
            current_property = name
            if not isinstance(prop, dict):
                raise ValueError(f"property '{name}' must be an object")
            _reject_runtime_dimension_lengths(prop, dimensions)
            type_info = _field_type_info(prop, constants)
            declared_required = name.lower() in required_set
            requirement = analyze_property_requirement(
                name,
                prop,
                declared_required=declared_required,
            )
            default_label = _get_default_value(prop, type_info, shape_constants)
            enum_label = _get_enum_values(prop, type_info, shape_constants)
            example_values = _get_example_values(prop, type_info)
            flex_tail_dims = _get_flex_tail_dims(prop, type_info)
            title = _get_title(prop)
            description_text = _get_description(prop)

            # if title:
            #     lines.append(f"### `{name}` - {title}")
            # else:
            #     lines.append(f"### `{name}`")
            lines.append(f"### {name}")
            lines.append("")
            if title:
                lines.append(f"{title} `{name}`")
            else:
                lines.append(f"`{name}`")
            lines.append("")
            if description_text:
                lines.append(description_text)
                lines.append("")

            lines.append("Summary:")
            lines.append(f"- Type: `{_format_specific_type(type_info)}`")
            format_label = _get_format_label(prop, type_info)
            if format_label is not None:
                lines.append(format_label)
            if type_info.category == "array" and flex_tail_dims > 0:
                lines.append(f"- Flexible tail dims: {flex_tail_dims}")
            lines.append(f"- Declared required: {'yes' if declared_required else 'no'}")
            lines.append(f"- Input required: {'yes' if requirement.requires_input else 'no'}")
            if default_label is not None:
                if isinstance(default_label, tuple):
                    base, note = default_label
                    if note:
                        lines.append(f"- Default: `{base}` {note}")
                    else:
                        lines.append(f"- Default: `{base}`")
                else:
                    lines.append(f"- Default: `{default_label}`")
            for bounds_label in _get_bounds_labels(prop, type_info):
                lines.append(bounds_label)
            if enum_label is not None:
                lines.append(f"- Allowed values: {enum_label}")
            if example_values is not None:
                examples_text = ", ".join(f"`{value}`" for value in example_values)
                lines.append(f"- Examples: {examples_text}")
            lines.append("")
            derived = _derived_schema(prop)
            if derived is not None:
                _append_derived_field_components(
                    lines,
                    name,
                    prop,
                    derived,
                    shape_constants,
                )
    except ValueError as exc:
        if current_property is None:
            raise
        msg = str(exc)
        if f"property '{current_property}'" in msg:
            raise
        raise ValueError(f"property '{current_property}': {msg}") from exc

    derived_types = _collect_documented_derived_types(properties)
    if derived_types:
        lines.append("## Derived types")
        lines.append("")
        for derived in derived_types:
            _append_derived_type_documentation(lines, derived, shape_constants)

    lines.append("## Example")
    lines.append("")
    lines.append("```fortran")
    filled_template = render_template(
        [schema],
        doc_mode="plain",
        value_mode="filled",
        constants=shape_constants,
        kind_map=None,
        kind_allowlist=None,
    )
    lines.extend(filled_template.rstrip("\n").splitlines())
    lines.append("```")
    lines.append("")

    return "\n".join(lines) + "\n"


def _validate_required(values: list[Any]) -> set[str]:
    required: set[str] = set()
    for value in values:
        if not isinstance(value, str):
            raise ValueError("schema 'required' entries must be strings")
        required.add(value.lower())
    return required


def _format_table_type(type_info: FieldTypeInfo) -> str:
    if type_info.category == "array":
        if type_info.element_category == "derived":
            return f"{type_info.type_spec} array"
        element = _format_scalar_type_name(type_info.element_category)
        return f"{element} array"
    if type_info.category == "derived":
        return type_info.type_spec
    return _format_scalar_type_name(type_info.category)


def _format_scalar_type_name(category: str | None) -> str:
    if category == "boolean":
        return "logical"
    if category == "string":
        return "string"
    if category == "integer":
        return "integer"
    if category == "real":
        return "real"
    raise ValueError(f"unsupported type category '{category}'")


def _format_specific_type(type_info: FieldTypeInfo) -> str:
    if type_info.category != "array":
        return type_info.type_spec
    dimensions = ", ".join(type_info.dimensions)
    return f"{type_info.type_spec}, dimension({dimensions})"


def _append_derived_field_components(
    lines: list[str],
    field_name: str,
    prop: dict[str, Any],
    derived: dict[str, Any],
    constants: dict[str, int] | None,
) -> None:
    components = derived.get("properties")
    if not isinstance(components, dict):
        return
    required = {
        value.lower()
        for value in derived.get("required", [])
        if isinstance(value, str)
    }
    effective_defaults = derived_component_defaults(field_name, prop)
    object_defaults = derived_object_default(field_name, prop)
    object_source = "item default" if prop.get("type") == "array" else "object default"
    lines.append("Components:")
    for child_name, child in components.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            continue
        type_info = _field_type_info(child, constants)
        path = f"{field_name}%{child_name}"
        child_key = child_name.lower()
        declared_required = child_key in required
        input_required = declared_required and child_key not in effective_defaults
        details = [
            f"`{_format_specific_type(type_info)}`",
            f"declared required {'yes' if declared_required else 'no'}",
            f"input required {'yes' if input_required else 'no'}",
        ]
        if child_key in effective_defaults:
            default_text = _format_scalar_default(
                effective_defaults[child_key],
                None,
                type_info.category,
            )
            source = object_source if child_key in object_defaults else "component default"
            details.append(f"default `{default_text}` ({source})")
        format_label = _get_format_label(child, type_info)
        if format_label is not None:
            details.append(format_label[2:] if format_label.startswith("- ") else format_label)
        bounds = _get_bounds_labels(child, type_info)
        details.extend(label[2:] if label.startswith("- ") else label for label in bounds)
        enum = _get_enum_values(child, type_info, constants)
        if enum is not None:
            details.append(f"allowed values {enum}")
        lines.append(f"- `{path}`: " + "; ".join(details))
    lines.append("")


def _collect_documented_derived_types(properties: dict[str, Any]) -> list[dict[str, Any]]:
    seen: set[tuple[str, str]] = set()
    collected: list[dict[str, Any]] = []
    for prop in properties.values():
        if not isinstance(prop, dict):
            continue
        derived = _derived_schema(prop)
        if derived is None:
            continue
        origin = _derived_origin(derived)
        identity = origin["identity"]
        if identity in seen:
            continue
        seen.add(identity)
        collected.append(origin["definition"])
    return collected


def _append_derived_type_documentation(
    lines: list[str],
    derived: dict[str, Any],
    constants: dict[str, int] | None,
) -> None:
    type_name = derived.get("x-fortran-type")
    if not isinstance(type_name, str):
        return
    title = derived.get("title", type_name)
    lines.append(f"### `{type_name}`")
    lines.append("")
    if isinstance(title, str) and title != type_name:
        lines.append(title)
        lines.append("")
    description = derived.get("description")
    if isinstance(description, str) and description.strip():
        lines.append(description.strip())
        lines.append("")
    module = derived.get("x-fortran-module")
    ownership = f"imported from `{module}`" if isinstance(module, str) else "`nml_helper`"
    lines.append(f"- Ownership: {ownership}")
    components = derived.get("properties")
    if isinstance(components, dict):
        required = {
            value.lower()
            for value in derived.get("required", [])
            if isinstance(value, str)
        }
        effective_defaults = derived_component_defaults(type_name, derived)
        object_defaults = derived_object_default(type_name, derived)
        lines.append(
            "- Buffer-compatible: " + ("yes" if _is_simple_derived_schema(derived) else "no")
        )
        lines.append(f"- Component order: {', '.join(str(name) for name in components)}")
        if isinstance(module, str):
            lines.append(
                "- **Declaration-order contract:** the imported Fortran type must declare "
                "components in the resolved schema order shown above."
            )
        for child_name, child in components.items():
            if not isinstance(child_name, str) or not isinstance(child, dict):
                continue
            info = _field_type_info(child, constants)
            child_key = child_name.lower()
            declared_required = child_key in required
            input_required = declared_required and child_key not in effective_defaults
            details = [
                f"`{_format_specific_type(info)}`",
                f"declared required {'yes' if declared_required else 'no'}",
                f"input required {'yes' if input_required else 'no'}",
            ]
            if child_key in effective_defaults:
                default_text = _format_scalar_default(
                    effective_defaults[child_key],
                    None,
                    info.category,
                )
                source = (
                    "object default" if child_key in object_defaults else "component default"
                )
                details.append(f"default `{default_text}` ({source})")
            format_label = _get_format_label(child, info)
            if format_label is not None:
                details.append(format_label[2:] if format_label.startswith("- ") else format_label)
            lines.append(f"- `{child_name}`: " + "; ".join(details))
    lines.append("")


def _get_default_value(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
) -> str | tuple[str, str | None] | None:
    derived = _derived_schema(prop)
    if derived is not None:
        object_default = derived_object_default("field", prop)
        if not object_default:
            return None
        base = _format_derived_default_mapping(derived, object_default, constants)
        if type_info.category == "array":
            return base, "(broadcast item default)"
        return base
    if type_info.category == "array":
        array_default = _array_default_value(prop)
        if array_default is None:
            return None
        default_value, default_from_items = array_default
        return _format_array_default_display(
            prop,
            type_info,
            constants,
            default_value=default_value,
            from_items=default_from_items,
        )
    if "default" not in prop:
        return None
    return _format_default_plain(prop["default"], type_info, prop, constants)


def _format_derived_default_mapping(
    derived: dict[str, Any],
    default: dict[str, Any],
    constants: dict[str, int] | None,
) -> str:
    components = derived.get("properties")
    if not isinstance(components, dict):
        raise ValueError("derived object must define properties")
    entries: list[str] = []
    for child_name, child in components.items():
        if not isinstance(child_name, str) or not isinstance(child, dict):
            raise ValueError("derived object components must be schema objects")
        key = child_name.lower()
        if key not in default:
            continue
        info = _field_type_info(child, constants)
        value = _format_scalar_default(default[key], None, info.category)
        entries.append(f"{child_name}: {value}")
    return "{" + ", ".join(entries) + "}"


def _format_info(prop: dict[str, Any]) -> str:
    title = _get_title(prop)
    return title or "n/a"


def _get_title(prop: dict[str, Any]) -> str | None:
    title = prop.get("title")
    if title is None:
        return None
    if not isinstance(title, str):
        raise ValueError("property title must be a string")
    title = title.strip()
    return title or None


def _get_description(prop: dict[str, Any]) -> str | None:
    description = prop.get("description")
    if description is None:
        return None
    if not isinstance(description, str):
        raise ValueError("property description must be a string")
    description = description.strip()
    return description or None


def _get_format_label(prop: dict[str, Any], type_info: FieldTypeInfo) -> str | None:
    if type_info.category == "array":
        items = prop.get("items")
        if not isinstance(items, dict):
            return None
        value = get_string_format(items)
        if value is None:
            return None
        return f"- Item format: `{value}`"
    value = get_string_format(prop)
    if value is None:
        return None
    return f"- Format: `{value}`"


def _get_enum_values(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
) -> str | None:
    enum_values = _enum_values(prop, type_info, constants)
    if enum_values is None:
        return None
    category = _enum_category(type_info)
    values = [_format_scalar_default(value, None, category) for value in enum_values]
    return ", ".join(f"`{value}`" for value in values)


def _get_bounds_labels(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> list[str]:
    bounds = _bounds_spec(prop, type_info)
    if bounds is None:
        return []
    category = bounds["category"]
    labels: list[str] = []
    min_value = bounds["min_value"]
    max_value = bounds["max_value"]
    if min_value is not None:
        op = ">" if bounds["min_exclusive"] else ">="
        literal = _format_scalar_default(min_value, None, category)
        labels.append(f"- Minimum: `{op} {literal}`")
    if max_value is not None:
        op = "<" if bounds["max_exclusive"] else "<="
        literal = _format_scalar_default(max_value, None, category)
        labels.append(f"- Maximum: `{op} {literal}`")
    return labels


def _get_flex_tail_dims(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> int:
    flex_raw = prop.get("x-fortran-flex-tail-dims")
    if flex_raw is None:
        flex_value = 0
    else:
        if isinstance(flex_raw, bool) or not isinstance(flex_raw, int):
            raise ValueError("property flex tail dims must be an integer")
        flex_value = flex_raw
    if flex_value < 0:
        raise ValueError("property flex tail dims must be >= 0")
    if flex_value == 0:
        return 0
    if type_info.category != "array":
        raise ValueError("flex tail dims only apply to arrays")
    if flex_value > len(type_info.dimensions):
        raise ValueError("flex tail dims must not exceed array rank")
    return flex_value


def _escape_table_cell(value: str) -> str:
    escaped = value.replace("|", "\\|").replace("\n", " ").strip()
    return escaped or "n/a"


def _get_example_values(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
) -> list[str] | None:
    examples = prop.get("examples")
    if examples is None:
        return None
    if not isinstance(examples, list):
        raise ValueError("property examples must be a list")
    if not examples:
        return None
    return [_format_example_value(value, type_info) for value in examples]


def _format_example_value(value: Any, type_info: FieldTypeInfo) -> str:
    if type_info.category == "array":
        if isinstance(value, list):
            formatted = [
                _format_scalar_default(item, None, type_info.element_category) for item in value
            ]
            return f"[{', '.join(formatted)}]"
        return _format_scalar_default(value, None, type_info.element_category)
    if isinstance(value, list):
        raise ValueError("scalar examples must not be lists")
    return _format_scalar_default(value, None, type_info.category)


def _format_default_plain(
    value: Any,
    type_info: FieldTypeInfo,
    prop: dict[str, Any],
    constants: dict[str, int] | None,
) -> str:
    if type_info.category == "array":
        if not isinstance(value, list):
            raise ValueError("array default must be a list")
        parsed_dims = _parse_default_dimensions(type_info.dimensions, constants)
        array_default = _prepare_array_default(value, parsed_dims, prop)
        elements = [
            _format_scalar_default(element, None, type_info.element_category)
            for element in array_default.source_values
        ]
        if (
            len(type_info.dimensions) == 1
            and array_default.order_values is None
            and array_default.pad_values is None
        ):
            return f"[{', '.join(elements)}]"

        shape_literal = ", ".join(type_info.dimensions)
        arguments = [f"[{', '.join(elements)}]", f"shape=[{shape_literal}]"]

        if array_default.order_values is not None:
            order_literal = ", ".join(str(index) for index in array_default.order_values)
            arguments.append(f"order=[{order_literal}]")

        if array_default.pad_values is not None:
            pad_elements = [
                _format_scalar_default(element, None, type_info.element_category)
                for element in array_default.pad_values
            ]
            arguments.append(f"pad=[{', '.join(pad_elements)}]")

        return f"reshape({', '.join(arguments)})"

    return _format_scalar_default(value, None, type_info.category)


def _format_array_default_display(
    prop: dict[str, Any],
    type_info: FieldTypeInfo,
    constants: dict[str, int] | None,
    *,
    default_value: Any = _DEFAULT_MISSING,
    from_items: bool = False,
) -> tuple[str, str | None]:
    if default_value is _DEFAULT_MISSING:
        default_value = prop["default"]
    if not from_items and not isinstance(default_value, list):
        raise ValueError("array default must be a list")
    default_is_list = isinstance(default_value, list)
    default_list = default_value if default_is_list else [default_value]

    parsed_dims = _parse_default_dimensions(type_info.dimensions, constants)
    default_prop = prop
    if from_items:
        default_prop = dict(prop)
        default_prop["x-fortran-default-repeat"] = True
    _prepare_array_default(default_list, parsed_dims, default_prop)

    if default_is_list:
        elements = [
            _format_scalar_default(value, None, type_info.element_category)
            for value in default_list
        ]
        base = f"[{', '.join(elements)}]"
    else:
        base = _format_scalar_default(default_value, None, type_info.element_category)

    repeat_raw = prop.get("x-fortran-default-repeat", False)
    if not isinstance(repeat_raw, bool):
        raise ValueError("array default repeat must be a boolean")
    notes: list[str] = []
    if repeat_raw:
        notes.append("repeated")

    order_raw = prop.get("x-fortran-default-order", "F")
    if not isinstance(order_raw, str):
        raise ValueError("array default order must be 'F' or 'C'")
    order = order_raw.upper()
    if order not in {"F", "C"}:
        raise ValueError("array default order must be 'F' or 'C'")
    if order == "C":
        notes.append("order: C")

    pad_raw = prop.get("x-fortran-default-pad")
    if pad_raw is not None:
        pad_list = pad_raw if isinstance(pad_raw, list) else [pad_raw]
        pad_elements = [
            _format_scalar_default(value, None, type_info.element_category) for value in pad_list
        ]
        if isinstance(pad_raw, list):
            pad_text = f"[{', '.join(pad_elements)}]"
        else:
            pad_text = pad_elements[0]
        notes.append(f"pad: `{pad_text}`")

    if notes:
        return base, f"({', '.join(notes)})"
    return base, None


def _github_section_id(title: str) -> str:
    # lowercase
    s = title.strip().lower()
    # remove punctuation characters
    punctuation = string.punctuation.replace("_", "")
    s = re.sub(f"[{re.escape(punctuation)}]", "", s)
    # replace any whitespace run with '-'
    s = re.sub(r"\s+", "-", s)
    return s

MuellerSeb / nml-tools / 29620967587

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous