1c8cfbce0d0b9400324380e1ba82321290fe534d

Committed 27 Jul 2025 02:43PM UTC coverage: 81.516% (-0.2%) from 81.695%

Build # 1c8cfbce0d0b9400324380e1ba82321290fe534d

Build Type

Pull #72

github

Committed by

camatcode

Commit Message

perf: introduce reader telemetry

Pull Request Pull Request #72: perf: introduce telemetry

Run Details

53 of 65 new or added lines in 12 files covered. (81.54%)

18 existing lines in 2 files now uncovered.

1151 of 1412 relevant lines covered (81.52%)

490.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.47

/lib/basenji/reader/pdf_reader.ex

defmodule Basenji.Reader.PDFReader do
  @moduledoc false
  use Basenji.TelemetryHelpers

  import Basenji.Reader

  def format, do: :pdf

  def file_extensions, do: ["pdf"]

  def get_magic_numbers, do: [%{offset: 0, magic: [0x25, 0x50, 0x44, 0x46, 0x2D]}]

  def close(_any), do: :ok

  def get_entries(pdf_file_path, _opts \\ []) do
    with {:ok, %{pages: pages}} <- get_metadata(pdf_file_path) do
      padding = String.length("#{pages}")

      file_entries =
        1..pages
        |> Enum.map(fn idx ->
          %{file_name: "#{String.pad_leading("#{idx}", padding, "0")}.jpg"}
        end)

      {:ok, %{entries: file_entries}}
    end
  end

  def get_entry_stream!(pdf_file_path, entry) do
    file_name = entry[:file_name]
    {page_num, _rest} = Integer.parse(file_name)

    create_resource(fn ->
      with {:ok, output} <- exec("pdftoppm", ["-f", "#{page_num}", "-singlefile", "-jpeg", "-q", pdf_file_path]) do
        [output |> :binary.bin_to_list()]
      end
    end)
  end

  def read(pdf_file_path, _opts \\ []) do
    telemetry_wrap [:basenji, :process], %{action: "read_pdf"} do
      with {:ok, %{entries: file_entries}} <- get_entries(pdf_file_path) do
        file_entries =
          file_entries
          |> Enum.map(fn entry ->
            entry
            |> Map.put(:stream_fun, fn -> get_entry_stream!(pdf_file_path, entry) end)
          end)

        {:ok, %{entries: file_entries}}
      end
    end
  end

  def get_metadata(pdf_file_path) do
    with {:ok, output} <- exec("pdfinfo", ["-isodates", pdf_file_path]) do
      metadata =
        String.split(output, "\n")
        |> Map.new(fn line ->
          String.split(line, ":", parts: 2)
          |> case do
            [k, v] -> to_metadata(k, v)
            [v] -> to_metadata("unknown_#{System.monotonic_time()}", v)
          end
        end)

      {:ok, metadata}
    end
  end

  defp to_metadata(k, v) do
    k = k |> String.trim() |> ProperCase.snake_case() |> String.to_atom()
    v = convert_value(k, v |> String.trim())
    {k, v}
  end

  defp convert_value(:creation_date, v), do: DateTimeParser.parse!(v)
  defp convert_value(:mod_date, v), do: DateTimeParser.parse!(v)
  defp convert_value(:pages, v), do: String.to_integer(v)

  defp convert_value(:filesize, v) do
    {first, _rest} = Integer.parse(v)
    first
  end

  defp convert_value(:pagesize, v) do
    String.split(v, " ")
    |> Enum.reduce([], fn part, acc ->
      Integer.parse(part)
      |> case do
        {coord, _rest} -> [coord | acc]
        _ -> acc
      end
    end)
    |> case do
      [x, y] -> {x, y}
      _ -> v
    end
  end

  defp convert_value(_k, "yes"), do: true
  defp convert_value(_k, "no"), do: false
  defp convert_value(_k, "none"), do: nil
  defp convert_value(_k, v), do: v
end

1	defmodule Basenji.Reader.PDFReader do
2	@moduledoc false
3	use Basenji.TelemetryHelpers
4
5	import Basenji.Reader
6
7	def format, do: :pdf	3✔
8
9	def file_extensions, do: ["pdf"]	781✔
10
11	def get_magic_numbers, do: [%{offset: 0, magic: [0x25, 0x50, 0x44, 0x46, 0x2D]}]	779✔
12
13	def close(_any), do: :ok	4✔
14
15	def get_entries(pdf_file_path, _opts \\ []) do
16	with {:ok, %{pages: pages}} <- get_metadata(pdf_file_path) do	782✔
17	padding = String.length("#{pages}")	781✔
18
19	file_entries =	781✔
20	1..pages
21	\|> Enum.map(fn idx ->
22	%{file_name: "#{String.pad_leading("#{idx}", padding, "0")}.jpg"}	3,124✔
23	end)
24
25	{:ok, %{entries: file_entries}}
26	end
27	end
28
29	def get_entry_stream!(pdf_file_path, entry) do
30	file_name = entry[:file_name]	41✔
31	{page_num, _rest} = Integer.parse(file_name)	41✔
32
33	create_resource(fn ->	41✔
34	with {:ok, output} <- exec("pdftoppm", ["-f", "#{page_num}", "-singlefile", "-jpeg", "-q", pdf_file_path]) do	41✔
35	[output \|> :binary.bin_to_list()]
36	end
37	end)
38	end
39
40	def read(pdf_file_path, _opts \\ []) do
41	telemetry_wrap [:basenji, :process], %{action: "read_pdf"} do	780✔
NEW 42	with {:ok, %{entries: file_entries}} <- get_entries(pdf_file_path) do	×
43	file_entries =	780✔
44	file_entries
45	\|> Enum.map(fn entry ->
46	entry
47	\|> Map.put(:stream_fun, fn -> get_entry_stream!(pdf_file_path, entry) end)	3,120✔
48	end)
49
50	{:ok, %{entries: file_entries}}
51	end
52	end
53	end
54
55	def get_metadata(pdf_file_path) do
56	with {:ok, output} <- exec("pdfinfo", ["-isodates", pdf_file_path]) do	782✔
57	metadata =	781✔
58	String.split(output, "\n")
59	\|> Map.new(fn line ->
60	String.split(line, ":", parts: 2)
61	\|> case do	14,058✔
62	[k, v] -> to_metadata(k, v)	14,058✔
63	[v] -> to_metadata("unknown_#{System.monotonic_time()}", v)	×
64	end
65	end)
66
67	{:ok, metadata}
68	end
69	end
70
71	defp to_metadata(k, v) do
72	k = k \|> String.trim() \|> ProperCase.snake_case() \|> String.to_atom()	14,058✔
73	v = convert_value(k, v \|> String.trim())	14,058✔
74	{k, v}
75	end
76
77	defp convert_value(:creation_date, v), do: DateTimeParser.parse!(v)	781✔
78	defp convert_value(:mod_date, v), do: DateTimeParser.parse!(v)	781✔
79	defp convert_value(:pages, v), do: String.to_integer(v)	781✔
80
81	defp convert_value(:filesize, v) do
82	{first, _rest} = Integer.parse(v)	781✔
83	first	781✔
84	end
85
86	defp convert_value(:pagesize, v) do
87	String.split(v, " ")
88	\|> Enum.reduce([], fn part, acc ->
89	Integer.parse(part)
90	\|> case do	3,124✔
91	{coord, _rest} -> [coord \| acc]	1,562✔
92	_ -> acc	1,562✔
93	end
94	end)
95	\|> case do	781✔
96	[x, y] -> {x, y}	781✔
97	_ -> v	×
98	end
99	end
100
101	defp convert_value(_k, "yes"), do: true	×
102	defp convert_value(_k, "no"), do: false	6,248✔
103	defp convert_value(_k, "none"), do: nil	781✔
104	defp convert_value(_k, v), do: v	3,124✔
105	end

camatcode / basenji / 1c8cfbce0d0b9400324380e1ba82321290fe534d

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous