f8b130ec5f30e99c791d82de015c1b375ad4c767

Committed 20 Jan 2026 04:19PM UTC coverage: 54.957% (+9.7%) from 45.277%

Build # f8b130ec5f30e99c791d82de015c1b375ad4c767

Build Type

push

github

Committed by

web-flow

Commit Message

fix(decoder): handle nested list-format arrays correctly (#7) (#9)

The decoder was incorrectly parsing nested list-format arrays like
`[[[1]]]` because it couldn't distinguish between inline arrays
(`[N]: val1,val2`) and list-format array headers (`[N]:` with nested
content below).

Changes:
- Updated regex patterns to require content after `: ` for inline arrays
- Added new case for list-format array headers ending with `:$`
- Added `parse_nested_list_array` function using existing helpers
- Consolidated edge_cases_test.exs into roundtrip_test.exs
- Renamed test file to roundtrip_test.exs (more idiomatic name)

Coverage Stats

9 of 9 new or added lines in 1 file covered. (100.0%)

2 existing lines in 2 files now uncovered.

510 of 928 relevant lines covered (54.96%)

16.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.76

/lib/toon/encode/encode.ex

defmodule Toon.Encode do
  @moduledoc """
  Main encoder for TOON format.

  This module coordinates the encoding process, dispatching to specialized
  encoders based on the type of value being encoded.
  """

  alias Toon.{Constants, EncodeError, Utils}
  alias Toon.Encode.{Objects, Options, Primitives, Strings}

  @doc """
  Encodes Elixir data to TOON format string.

  ## Options

    * `:indent` - Number of spaces for indentation (default: 2)
    * `:delimiter` - Delimiter for array values: "," | "\\t" | "|" (default: ",")
    * `:length_marker` - Prefix for array length marker (default: nil)

  ## Examples

      iex> Toon.Encode.encode(%{"name" => "Alice", "age" => 30})
      {:ok, "age: 30\\nname: Alice"}

      iex> Toon.Encode.encode(%{"tags" => ["elixir", "toon"]})
      {:ok, "tags[2]: elixir,toon"}

      iex> Toon.Encode.encode(nil)
      {:ok, "null"}

      iex> Toon.Encode.encode(%{"name" => "Alice"}, indent: 4)
      {:ok, "name: Alice"}
  """
  @spec encode(Toon.Types.encodable(), keyword()) ::
          {:ok, String.t()} | {:error, EncodeError.t()}
  def encode(data, opts \\ []) do
    start_time = System.monotonic_time()
    metadata = %{data_type: data_type(data)}

    :telemetry.execute([:toon, :encode, :start], %{system_time: System.system_time()}, metadata)

    result =
      with {:ok, validated_opts} <- Options.validate(opts),
           {:ok, normalized} <- normalize(data) do
        try do
          encoded = do_encode(normalized, 0, validated_opts)
          {:ok, IO.iodata_to_binary(encoded)}
        rescue
          e in EncodeError -> {:error, e}
          e -> {:error, EncodeError.exception(message: Exception.message(e), value: data)}
        end
      else
        {:error, error} ->
          {:error,
           EncodeError.exception(
             message: "Invalid options: #{Exception.message(error)}",
             reason: error
           )}
      end

    duration = System.monotonic_time() - start_time

    case result do
      {:ok, encoded} ->
        :telemetry.execute(
          [:toon, :encode, :stop],
          %{duration: duration, size: byte_size(encoded)},
          metadata
        )

      {:error, error} ->
        :telemetry.execute(
          [:toon, :encode, :exception],
          %{duration: duration},
          Map.put(metadata, :error, error)
        )
    end

    result
  end

  defp data_type(data) when is_map(data), do: :map
  defp data_type(data) when is_list(data), do: :list
  defp data_type(nil), do: :null
  defp data_type(data) when is_boolean(data), do: :boolean
  defp data_type(data) when is_number(data), do: :number
  defp data_type(data) when is_binary(data), do: :string
  defp data_type(_), do: :unknown

  @doc """
  Encodes Elixir data to TOON format string, raising on error.

  ## Examples

      iex> Toon.Encode.encode!(%{"name" => "Alice"})
      "name: Alice"

      iex> Toon.Encode.encode!(%{"tags" => ["a", "b"]})
      "tags[2]: a,b"
  """
  @spec encode!(Toon.Types.encodable(), keyword()) :: String.t()
  def encode!(data, opts \\ []) do
    case encode(data, opts) do
      {:ok, result} -> result
      {:error, error} -> raise error
    end
  end

  # Private functions

  @spec normalize(term()) :: {:ok, Toon.Types.encodable()} | {:error, EncodeError.t()}
  defp normalize(data) do
    {:ok, Utils.normalize(data)}
  rescue
    e ->
      {:error,
       EncodeError.exception(message: "Failed to normalize data: #{Exception.message(e)}")}
  end

  @spec do_encode(Toon.Types.encodable(), non_neg_integer(), map()) :: iodata()
  @doc false
  def do_encode(data, depth, opts) do
    cond do
      Utils.primitive?(data) ->
        Primitives.encode(data, opts.delimiter)

      # Check if this is an ordered list (list of {key, value} tuples)
      is_list(data) and tuple_list?(data) ->
        # Convert to map and encode with key order preserved
        map = Map.new(data)
        key_order = Enum.map(data, fn {k, _v} -> k end)
        Objects.encode(map, depth, Map.put(opts, :key_order, key_order))

      Utils.map?(data) ->
        Objects.encode(data, depth, opts)

      Utils.list?(data) ->
        # Root-level arrays per TOON spec Section 5
        encode_root_array(data, depth, opts)

      true ->
        raise EncodeError,
          message: "Cannot encode value of type #{inspect(data.__struct__ || :unknown)}",
          value: data
    end
  end

  # Check if a list is a tuple list (key-value pairs)
  defp tuple_list?([]), do: false
  defp tuple_list?([{k, _v} | _rest]) when is_binary(k), do: true
  defp tuple_list?(_), do: false

  # Encode root-level array per TOON spec Section 5
  defp encode_root_array(data, depth, opts) do
    length_marker = format_length_marker(length(data), opts.length_marker)
    delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""

    cond do
      # Empty array
      Enum.empty?(data) ->
        length_marker = format_length_marker(0, opts.length_marker)
        ["[", length_marker, "]:"]

      # Inline array (all primitives)
      Utils.all_primitives?(data) ->
        values =
          data
          |> Enum.map(&Primitives.encode(&1, opts.delimiter))
          |> Enum.intersperse(opts.delimiter)

        ["[", length_marker, delimiter_marker, "]: ", values]

      # Tabular array (all maps with same keys and primitive values only)
      Utils.all_maps?(data) and Utils.same_keys?(data) and Utils.all_primitive_values?(data) ->
        encode_root_tabular_array(data, length_marker, delimiter_marker, opts)

      # List format (mixed or non-uniform)
      true ->
        encode_root_list_array(data, length_marker, delimiter_marker, depth, opts)
    end
  end

  # Encode root tabular array
  defp encode_root_tabular_array(data, length_marker, delimiter_marker, opts) do
    # Get keys from first object and use provided key order or sort alphabetically
    keys =
      case data do
        [first | _] ->
          map_keys = Map.keys(first)

          key_order = Map.get(opts, :key_order)

          # Use key_order if provided and matches all keys
          if is_list(key_order) and not Enum.empty?(key_order) do
            ordered = Enum.filter(key_order, &(&1 in map_keys))

            if length(ordered) == length(map_keys) do
              ordered
            else
              Enum.sort(map_keys)
            end
          else
            Enum.sort(map_keys)
          end

        [] ->
          []
      end

    # Format header
    fields = Enum.map(keys, &Strings.encode_key/1) |> Enum.intersperse(opts.delimiter)

    header = [
      "[",
      length_marker,
      delimiter_marker,
      "]",
      Constants.open_brace(),
      fields,
      Constants.close_brace(),
      Constants.colon()
    ]

    # Format data rows
    rows =
      Enum.map(data, fn obj ->
        values =
          keys
          |> Enum.map(fn k -> Map.get(obj, k) end)
          |> Enum.map(&Primitives.encode(&1, opts.delimiter))
          |> Enum.intersperse(opts.delimiter)

        [opts.indent_string, values]
      end)

    [header | rows]
    |> Enum.map_join("\n", &IO.iodata_to_binary/1)
  end

  # Encode root list array
  defp encode_root_list_array(data, length_marker, delimiter_marker, depth, opts) do
    header = ["[", length_marker, delimiter_marker, "]:"]

    items =
      Enum.flat_map(data, fn item ->
        encode_root_list_item(item, depth, opts)
      end)

    # Don't add extra indentation - items already have their indentation
    [
      IO.iodata_to_binary(header)
      | Enum.map(items, fn line ->
          [opts.indent_string, line]
        end)
    ]
    |> Enum.map_join("\n", &IO.iodata_to_binary/1)
  end

  # Encode a single root list item
  defp encode_root_list_item(item, depth, opts) when is_map(item) do
    entries =
      item
      |> Enum.with_index()
      |> Enum.flat_map(fn {{k, v}, index} ->
        encode_root_list_entry(k, v, index, depth, opts)
      end)

    entries
  end

  defp encode_root_list_item(item, _depth, opts) when is_list(item) do
    # Array item - encode as inline array if all primitives
    cond do
      Enum.empty?(item) ->
        [[Constants.list_item_marker(), Constants.space(), "[0]:"]]

      Utils.all_primitives?(item) ->
        length_marker = format_length_marker(length(item), opts.length_marker)
        delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""

        values =
          item
          |> Enum.map(&Primitives.encode(&1, opts.delimiter))
          |> Enum.intersperse(opts.delimiter)

        [
          [
            Constants.list_item_marker(),
            Constants.space(),
            "[",
            length_marker,
            delimiter_marker,
            "]",
            Constants.colon(),
            Constants.space(),
            values
          ]
        ]

      true ->
        # Complex nested array
        length_marker = format_length_marker(length(item), opts.length_marker)
        delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""

        header = [
          Constants.list_item_marker(),
          Constants.space(),
          "[",
          length_marker,
          delimiter_marker,
          "]",
          Constants.colon()
        ]

        # Recursively encode nested items
        nested_items =
          Enum.flat_map(item, fn nested_item ->
            nested = encode_root_list_item(nested_item, 0, opts)

            Enum.map(nested, fn line ->
              [opts.indent_string | line]
            end)
          end)

        [header | nested_items]
    end
  end

  defp encode_root_list_item(item, _depth, opts) do
    # Primitive item
    [[Constants.list_item_marker(), Constants.space(), Primitives.encode(item, opts.delimiter)]]
  end

  # Encode a single entry in root list item
  defp encode_root_list_entry(k, v, index, _depth, opts) do
    if Utils.primitive?(v) do
      encoded_key = Strings.encode_key(k)
      needs_marker = index == 0

      line = [
        encoded_key,
        Constants.colon(),
        Constants.space(),
        Primitives.encode(v, opts.delimiter)
      ]

      if needs_marker do
        [[Constants.list_item_marker(), Constants.space() | line]]
      else
        [[opts.indent_string | line]]
      end
    else
      # Complex structures are handled by the array encoder
      []
    end
  end

  # Format length marker
  defp format_length_marker(length, nil), do: Integer.to_string(length)
  defp format_length_marker(length, marker), do: marker <> Integer.to_string(length)
end

1	defmodule Toon.Encode do
2	@moduledoc """
3	Main encoder for TOON format.
4
5	This module coordinates the encoding process, dispatching to specialized
6	encoders based on the type of value being encoded.
7	"""
8
9	alias Toon.{Constants, EncodeError, Utils}
10	alias Toon.Encode.{Objects, Options, Primitives, Strings}
11
12	@doc """
13	Encodes Elixir data to TOON format string.
14
15	## Options
16
17	* `:indent` - Number of spaces for indentation (default: 2)
18	* `:delimiter` - Delimiter for array values: "," \| "\\t" \| "\|" (default: ",")
19	* `:length_marker` - Prefix for array length marker (default: nil)
20
21	## Examples
22
23	iex> Toon.Encode.encode(%{"name" => "Alice", "age" => 30})
24	{:ok, "age: 30\\nname: Alice"}
25
26	iex> Toon.Encode.encode(%{"tags" => ["elixir", "toon"]})
27	{:ok, "tags[2]: elixir,toon"}
28
29	iex> Toon.Encode.encode(nil)
30	{:ok, "null"}
31
32	iex> Toon.Encode.encode(%{"name" => "Alice"}, indent: 4)
33	{:ok, "name: Alice"}
34	"""
35	@spec encode(Toon.Types.encodable(), keyword()) ::
36	{:ok, String.t()} \| {:error, EncodeError.t()}
37	def encode(data, opts \\ []) do
38	start_time = System.monotonic_time()	62✔
39	metadata = %{data_type: data_type(data)}	62✔
40
41	:telemetry.execute([:toon, :encode, :start], %{system_time: System.system_time()}, metadata)	62✔
42
43	result =	62✔
44	with {:ok, validated_opts} <- Options.validate(opts),	62✔
45	{:ok, normalized} <- normalize(data) do	62✔
46	try do	62✔
47	encoded = do_encode(normalized, 0, validated_opts)	62✔
48	{:ok, IO.iodata_to_binary(encoded)}
49	rescue
50	e in EncodeError -> {:error, e}	×
51	e -> {:error, EncodeError.exception(message: Exception.message(e), value: data)}	×
52	end
53	else
54	{:error, error} ->	×
55	{:error,
56	EncodeError.exception(
57	message: "Invalid options: #{Exception.message(error)}",	×
58	reason: error
59	)}
60	end
61
62	duration = System.monotonic_time() - start_time	62✔
63
64	case result do	62✔
65	{:ok, encoded} ->
66	:telemetry.execute(	62✔
67	[:toon, :encode, :stop],
68	%{duration: duration, size: byte_size(encoded)},
69	metadata
70	)
71
72	{:error, error} ->
73	:telemetry.execute(	×
74	[:toon, :encode, :exception],
75	%{duration: duration},
76	Map.put(metadata, :error, error)
77	)
78	end
79
80	result	62✔
81	end
82
83	defp data_type(data) when is_map(data), do: :map	25✔
84	defp data_type(data) when is_list(data), do: :list	22✔
85	defp data_type(nil), do: :null	1✔
86	defp data_type(data) when is_boolean(data), do: :boolean	2✔
87	defp data_type(data) when is_number(data), do: :number	7✔
88	defp data_type(data) when is_binary(data), do: :string	4✔
89	defp data_type(_), do: :unknown	1✔
90
91	@doc """
92	Encodes Elixir data to TOON format string, raising on error.
93
94	## Examples
95
96	iex> Toon.Encode.encode!(%{"name" => "Alice"})
97	"name: Alice"
98
99	iex> Toon.Encode.encode!(%{"tags" => ["a", "b"]})
100	"tags[2]: a,b"
101	"""
102	@spec encode!(Toon.Types.encodable(), keyword()) :: String.t()
103	def encode!(data, opts \\ []) do
104	case encode(data, opts) do	62✔
105	{:ok, result} -> result	62✔
106	{:error, error} -> raise error	×
107	end
108	end
109
110	# Private functions
111
112	@spec normalize(term()) :: {:ok, Toon.Types.encodable()} \| {:error, EncodeError.t()}
113	defp normalize(data) do	62✔
114	{:ok, Utils.normalize(data)}
115	rescue
116	e ->	×
117	{:error,
118	EncodeError.exception(message: "Failed to normalize data: #{Exception.message(e)}")}	×
119	end
120
121	@spec do_encode(Toon.Types.encodable(), non_neg_integer(), map()) :: iodata()
122	@doc false
123	def do_encode(data, depth, opts) do
124	cond do	64✔
125	Utils.primitive?(data) ->
126	Primitives.encode(data, opts.delimiter)	16✔
127
128	# Check if this is an ordered list (list of {key, value} tuples)
129	is_list(data) and tuple_list?(data) ->	48✔
130	# Convert to map and encode with key order preserved
131	map = Map.new(data)	×
132	key_order = Enum.map(data, fn {k, _v} -> k end)	×
133	Objects.encode(map, depth, Map.put(opts, :key_order, key_order))	×
134
135	Utils.map?(data) ->	48✔
136	Objects.encode(data, depth, opts)	26✔
137
138	Utils.list?(data) ->	22✔
139	# Root-level arrays per TOON spec Section 5
140	encode_root_array(data, depth, opts)	22✔
141
142	true ->	×
143	raise EncodeError,	×
144	message: "Cannot encode value of type #{inspect(data.__struct__ \|\| :unknown)}",	×
145	value: data
146	end
147	end
148
149	# Check if a list is a tuple list (key-value pairs)
150	defp tuple_list?([]), do: false	1✔
151	defp tuple_list?([{k, _v} \| _rest]) when is_binary(k), do: true	×
152	defp tuple_list?(_), do: false	21✔
153
154	# Encode root-level array per TOON spec Section 5
155	defp encode_root_array(data, depth, opts) do
156	length_marker = format_length_marker(length(data), opts.length_marker)	22✔
157	delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""	22✔
158
159	cond do	22✔
160	# Empty array
161	Enum.empty?(data) ->
162	length_marker = format_length_marker(0, opts.length_marker)	1✔
163	["[", length_marker, "]:"]
164
165	# Inline array (all primitives)
166	Utils.all_primitives?(data) ->	21✔
167	values =	6✔
168	data
169	\|> Enum.map(&Primitives.encode(&1, opts.delimiter))	18✔
170	\|> Enum.intersperse(opts.delimiter)	6✔
171
172	["[", length_marker, delimiter_marker, "]: ", values]
173
174	# Tabular array (all maps with same keys and primitive values only)
175	Utils.all_maps?(data) and Utils.same_keys?(data) and Utils.all_primitive_values?(data) ->	15✔
176	encode_root_tabular_array(data, length_marker, delimiter_marker, opts)	5✔
177
178	# List format (mixed or non-uniform)
179	true ->	10✔
180	encode_root_list_array(data, length_marker, delimiter_marker, depth, opts)	10✔
181	end
182	end
183
184	# Encode root tabular array
185	defp encode_root_tabular_array(data, length_marker, delimiter_marker, opts) do
186	# Get keys from first object and use provided key order or sort alphabetically
187	keys =	5✔
188	case data do
189	[first \| _] ->
190	map_keys = Map.keys(first)	5✔
191
192	key_order = Map.get(opts, :key_order)	5✔
193
194	# Use key_order if provided and matches all keys
195	if is_list(key_order) and not Enum.empty?(key_order) do	5✔
196	ordered = Enum.filter(key_order, &(&1 in map_keys))	×
197
198	if length(ordered) == length(map_keys) do	×
199	ordered	×
200	else
201	Enum.sort(map_keys)	×
202	end
203	else
204	Enum.sort(map_keys)	5✔
205	end
206
207	[] ->	×
208	[]
209	end
210
211	# Format header
212	fields = Enum.map(keys, &Strings.encode_key/1) \|> Enum.intersperse(opts.delimiter)	5✔
213
214	header = [	5✔
215	"[",
216	length_marker,
217	delimiter_marker,
218	"]",
219	Constants.open_brace(),
220	fields,
221	Constants.close_brace(),
222	Constants.colon()
223	]
224
225	# Format data rows
226	rows =	5✔
227	Enum.map(data, fn obj ->
228	values =	10✔
229	keys
230	\|> Enum.map(fn k -> Map.get(obj, k) end)	16✔
231	\|> Enum.map(&Primitives.encode(&1, opts.delimiter))	16✔
232	\|> Enum.intersperse(opts.delimiter)	10✔
233
234	[opts.indent_string, values]	10✔
235	end)
236
237	[header \| rows]
238	\|> Enum.map_join("\n", &IO.iodata_to_binary/1)	5✔
239	end
240
241	# Encode root list array
242	defp encode_root_list_array(data, length_marker, delimiter_marker, depth, opts) do
243	header = ["[", length_marker, delimiter_marker, "]:"]	10✔
244
245	items =	10✔
246	Enum.flat_map(data, fn item ->
247	encode_root_list_item(item, depth, opts)	15✔
248	end)
249
250	# Don't add extra indentation - items already have their indentation
251	[
252	IO.iodata_to_binary(header)
253	\| Enum.map(items, fn line ->	27✔
254	[opts.indent_string, line]	27✔
255	end)
256	]
257	\|> Enum.map_join("\n", &IO.iodata_to_binary/1)	10✔
258	end
259
260	# Encode a single root list item
261	defp encode_root_list_item(item, depth, opts) when is_map(item) do
262	entries =	2✔
263	item
264	\|> Enum.with_index()
265	\|> Enum.flat_map(fn {{k, v}, index} ->
266	encode_root_list_entry(k, v, index, depth, opts)	4✔
267	end)
268
269	entries	2✔
270	end
271
272	defp encode_root_list_item(item, _depth, opts) when is_list(item) do
273	# Array item - encode as inline array if all primitives
274	cond do	25✔
275	Enum.empty?(item) ->	6✔
276	[[Constants.list_item_marker(), Constants.space(), "[0]:"]]
277
278	Utils.all_primitives?(item) ->	19✔
279	length_marker = format_length_marker(length(item), opts.length_marker)	8✔
280	delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""	8✔
281
282	values =	8✔
283	item
284	\|> Enum.map(&Primitives.encode(&1, opts.delimiter))	10✔
285	\|> Enum.intersperse(opts.delimiter)	8✔
286
287	[
288	[
289	Constants.list_item_marker(),
290	Constants.space(),
291	"[",
292	length_marker,
293	delimiter_marker,
294	"]",
295	Constants.colon(),
296	Constants.space(),
297	values
298	]
299	]
300
301	true ->	11✔
302	# Complex nested array
303	length_marker = format_length_marker(length(item), opts.length_marker)	11✔
304	delimiter_marker = if opts.delimiter != ",", do: opts.delimiter, else: ""	11✔
305
306	header = [	11✔
307	Constants.list_item_marker(),
308	Constants.space(),
309	"[",
310	length_marker,
311	delimiter_marker,
312	"]",
313	Constants.colon()
314	]
315
316	# Recursively encode nested items
317	nested_items =	11✔
318	Enum.flat_map(item, fn nested_item ->
319	nested = encode_root_list_item(nested_item, 0, opts)	12✔
320
321	Enum.map(nested, fn line ->	12✔
322	[opts.indent_string \| line]	17✔
323	end)
324	end)
325
326	[header \| nested_items]
327	end
328	end
329
330	defp encode_root_list_item(item, _depth, opts) do	×
331	# Primitive item
332	[[Constants.list_item_marker(), Constants.space(), Primitives.encode(item, opts.delimiter)]]	×
333	end
334
335	# Encode a single entry in root list item
336	defp encode_root_list_entry(k, v, index, _depth, opts) do
337	if Utils.primitive?(v) do	4✔
338	encoded_key = Strings.encode_key(k)	2✔
339	needs_marker = index == 0	2✔
340
341	line = [	2✔
342	encoded_key,
343	Constants.colon(),
344	Constants.space(),
345	Primitives.encode(v, opts.delimiter)	2✔
346	]
347
348	if needs_marker do	2✔
349	[[Constants.list_item_marker(), Constants.space() \| line]]
350	else
UNCOV 351	[[opts.indent_string \| line]]	×
352	end
353	else
354	# Complex structures are handled by the array encoder
355	[]
356	end
357	end
358
359	# Format length marker
360	defp format_length_marker(length, nil), do: Integer.to_string(length)	42✔
361	defp format_length_marker(length, marker), do: marker <> Integer.to_string(length)	×
362	end

xu-chris / toon_ex / f8b130ec5f30e99c791d82de015c1b375ad4c767

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous