6713098919

Committed 31 Oct 2023 10:21PM UTC coverage: 52.831% (-21.8%) from 74.636%

Build # 6713098919

Build Type

push

github

Committed by

web-flow

Commit Message

Merge d0a2a1ee9 into b185d4972

Run Details

45 of 45 new or added lines in 3 files covered. (100.0%)

8173 of 15470 relevant lines covered (52.83%)

6562.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.33

/packages/bidi/init.lua

local base = require("packages.base")

local package = pl.class(base)
package._name = "bidi"

local icu = require("justenoughicu")

local function reverse_portion (tbl, s, e)
  local rv = {}
  for i = 1, s-1 do rv[#rv+1] = tbl[i] end
  for i = e, s, -1 do rv[#rv+1] = tbl[i] end
  for i = e+1, #tbl do rv[#rv+1] = tbl[i] end
  return rv
end

local function create_matrix (line, base_level)
  -- L2; create a transformation matrix of elements
  -- such that output[matrix[i]] = input[i]
  -- e.g. No reversions required: [1, 2, 3, 4, 5]
  -- Levels [0, 0, 0, 1, 1] -> [1, 2, 3, 5, 4]

  local max_level = 0
  local matrix = {}
  for i, c in next, line do
    if c.level > max_level then max_level = c.level end
    matrix[i] = i
  end

  for level = base_level+1, max_level do
    local level_start
    for i, _ in next, line do
      if line[i].level >= level then
        if not level_start then
          level_start = i
        elseif i == #line then
          local level_end = i
          matrix = reverse_portion(matrix, level_start, level_end)
          level_start = nil
        end
      else
        if level_start then
          local level_end = i-1
          matrix = reverse_portion(matrix, level_start, level_end)
          level_start = nil
        end
      end
    end
  end

  return matrix
end

local function reverse_each_node (nodelist)
  for j = 1, #nodelist do
    if nodelist[j].type =="hbox" then
      if nodelist[j].value.items then SU.flip_in_place(nodelist[j].value.items) end
      SU.flip_in_place(nodelist[j].value.glyphString)
    end
  end
end

local nodeListToText = function (nl)
  local owners, text = {}, {}
  local p = 1
  for i = 1, #nl do local n = nl[i]
    if n.text then
      local utfchars = SU.splitUtf8(n.text)
      for j = 1, #utfchars do
        owners[p] = { node = n, pos = j }
        text[p] = utfchars[j]
        p = p + 1
      end
    else
      owners[p] = { node = n }
      text[p] = luautf8.char(0xFFFC)
      p = p + 1
    end
  end
  return owners, text
end

local splitNodeAtPos = function (n, splitstart, p)
  if n.is_unshaped then
    local utf8chars = SU.splitUtf8(n.text)
    local n2 = SILE.nodefactory.unshaped({ text = "", options = pl.tablex.copy(n.options) })
    local n1 = SILE.nodefactory.unshaped({ text = "", options = pl.tablex.copy(n.options) })
    for i = splitstart, #utf8chars do
      if i <= p then n1.text = n1.text .. utf8chars[i]
      else n2.text = n2.text .. utf8chars[i]
      end
    end
    return n1, n2
  else
    SU.error("Unsure how to split node " .. tostring(n) .. " at position " .. p, true)
  end
end

local splitNodelistIntoBidiRuns = function (typesetter)
  local nl = typesetter.state.nodes
  if #nl == 0 then return nl end
  local owners, text = nodeListToText(nl)
  local base_level = typesetter.frame:writingDirection() == "RTL" and 1 or 0
  local runs = { icu.bidi_runs(table.concat(text), typesetter.frame:writingDirection()) }
  table.sort(runs, function (a, b) return a.start < b.start end)
  -- local newNl = {}
  -- Split nodes on run boundaries
  for i = 1, #runs do
    local run = runs[i]
    local thisOwner = owners[run.start+run.length]
    local nextOwner = owners[run.start+1+run.length]
    -- print(thisOwner, nextOwner)
    if nextOwner and thisOwner.node == nextOwner.node then
      local before, after = splitNodeAtPos(nextOwner.node, 1, nextOwner.pos-1)
      -- print(before, after)
      local start = nil
      for j = run.start+1, run.start+run.length do
        if owners[j].node==nextOwner.node then
          if not start then start = j end
          owners[j]={ node=before ,pos=j-start+1 }
        end
      end
      for j = run.start + 1 + run.length, #owners do
        if owners[j].node==nextOwner.node then
          owners[j] = { node = after, pos = j - (run.start + run.length) }
        end
      end
    end
  end
  -- Assign direction/level to nodes
  for i = 1, #runs do
    local runstart = runs[i].start+1
    local runend   = runstart + runs[i].length-1
    for j= runstart, runend do
      if owners[j].node and owners[j].node.options then
        owners[j].node.options.direction = runs[i].dir
        owners[j].node.options.bidilevel = runs[i].level - base_level
      end
    end
  end
  -- String together nodelist
  nl={}
  for i = 1, #owners do
    if #nl and nl[#nl] ~= owners[i].node then
      nl[#nl+1] = owners[i].node
      -- print(nl[#nl], nl[#nl].options)
    end
  end
  -- for i = 1, #nl do print(i, nl[i]) end
  return nl
end

local bidiBoxupNodes = function (typesetter)
  local allDone = true
  for i = 1, #typesetter.state.nodes do
    if not typesetter.state.nodes[i].bidiDone then allDone = false end
  end
  if allDone then return typesetter:nobidi_boxUpNodes() end
  local newNodeList = splitNodelistIntoBidiRuns(typesetter)
  typesetter:shapeAllNodes(newNodeList)
  typesetter.state.nodes = newNodeList
  local vboxlist = typesetter:nobidi_boxUpNodes()
  -- Scan for out-of-direction material
  for i = 1, #vboxlist do
    local v = vboxlist[i]
    if v.is_vbox then package.reorder(nil, v, typesetter) end
  end
  return vboxlist
end

function package.reorder (_, n, typesetter)
  local nl = n.nodes
  -- local newNl = {}
  -- local matrix = {}
  local levels = {}
  local base_level = typesetter.frame:writingDirection() == "RTL" and 1 or 0
  for i = 1, #nl do
    if nl[i].options and nl[i].options.bidilevel then
      levels[i] = { level = nl[i].options.bidilevel }
    end
  end
  for i = 1, #nl do
    if not levels[i] then
      -- resolve neutrals
      local left_level, right_level
      for left = i - 1, 1, -1 do
        if nl[left].options and nl[left].options.bidilevel then
          left_level = nl[left].options.bidilevel
          break
        end
      end
      for right = i + 1, #nl do
        if nl[right].options and nl[right].options.bidilevel then
          right_level = nl[right].options.bidilevel
          break
        end
      end
      levels[i] = { level = (left_level == right_level and left_level or 0) }
    end
  end
  local matrix = create_matrix(levels, 0)
  local rv = {}
  -- for i = 1, #nl do print(i, nl[i], levels[i]) end
  for i = 1, #nl do
    if nl[i].is_nnode and levels[i].level %2 ~= base_level then
      SU.flip_in_place(nl[i].nodes)
      reverse_each_node(nl[i].nodes)
    elseif nl[i].is_discretionary and levels[i].level %2 ~= base_level and not nl[i].bidiDone then
      for j = 1, #(nl[i].replacement) do
        if nl[i].replacement[j].is_nnode then
          SU.flip_in_place(nl[i].replacement[j].nodes)
          reverse_each_node(nl[i].replacement[j].nodes)
        end
      end
      for j = 1, #(nl[i].prebreak) do
        if nl[i].prebreak[j].is_nnode then
          SU.flip_in_place(nl[i].prebreak[j].nodes)
          reverse_each_node(nl[i].prebreak[j].nodes)
        end
      end
      for j = 1, #(nl[i].postbreak) do
        if nl[i].postbreak[j].is_nnode then
          SU.flip_in_place(nl[i].postbreak[j].nodes)
          reverse_each_node(nl[i].postbreak[j].nodes)
        end
      end

    end
    rv[matrix[i]] = nl[i]
    nl[i].bidiDone = true
    -- rv[i] = nl[i]
  end
  n.nodes = SU.compress(rv)
end

function package:bidiEnableTypesetter (typesetter)
  if typesetter.nobidi_boxUpNodes and self.class._initialized then
    return SU.warn("BiDi already enabled, nothing to turn on")
  end
  typesetter.nobidi_boxUpNodes = typesetter.boxUpNodes
  typesetter.boxUpNodes = bidiBoxupNodes
end

function package:bidiDisableTypesetter (typesetter)
  if not typesetter.nobidi_boxUpNodes and self.class._initialized then
    return SU.warn("BiDi not enabled, nothing to turn off")
  end
  typesetter.boxUpNodes = typesetter.nobidi_boxUpNodes
  typesetter.nobidi_boxUpNodes = nil
end

function package:_init ()
  base._init(self)
  self:deprecatedExport("reorder", self.reorder)
  self:deprecatedExport("bidiEnableTypesetter", self.bidiEnableTypesetter)
  self:deprecatedExport("bidiDisableTypesetter", self.bidiDisableTypesetter)
  if SILE.typesetter then
    self:bidiEnableTypesetter(SILE.typesetter)
  end
  self:bidiEnableTypesetter(SILE.typesetters.base)
end

function package:registerCommands ()

  self:registerCommand("thisframeLTR", function (_, _)
    local direction = "LTR"
    SILE.typesetter.frame.direction = direction
    SILE.settings:set("font.direction", direction)
    SILE.typesetter:leaveHmode()
    SILE.typesetter.frame:newLine()
  end)

  self:registerCommand("thisframedirection", function (options, _)
    local direction = SU.required(options, "direction", "frame direction")
    SILE.typesetter.frame.direction = direction
    SILE.settings:set("font.direction", direction)
    SILE.typesetter:leaveHmode()
    SILE.typesetter.frame:init()
  end)

  self:registerCommand("thisframeRTL", function (_, _)
    local direction = "RTL"
    SILE.typesetter.frame.direction = direction
    SILE.settings:set("font.direction", direction)
    SILE.typesetter:leaveHmode()
    SILE.typesetter.frame:newLine()
  end)

  self:registerCommand("bidi-on", function (_, _)
    self:bidiEnableTypesetter(SILE.typesetter)
  end)

  self:registerCommand("bidi-off", function (_, _)
    self:bidiDisableTypesetter(SILE.typesetter)
  end)

end

package.documentation = [[
\begin{document}
Scripts like the Latin alphabet you are currently reading are normally written left to right (LTR); however, some scripts, such as Arabic and Hebrew, are written right to left (RTL).
The \autodoc:package{bidi} package, which is loaded by default, provides SILE with the ability to correctly typeset right-to-left text and also documents which mix right-to-left and left-to-right typesetting.
Because it is loaded by default, you can use both LTR and RTL text within a paragraph and SILE will ensure that the output characters appear in the correct order.

The \autodoc:package{bidi} package provides two commands, \autodoc:command{\thisframeLTR} and \autodoc:command{\thisframeRTL}, which set the default text direction for the current frame.
If you tell SILE that a frame is RTL, the text will start in the right margin and proceed leftward.
It also provides the commands \autodoc:command{\bidi-off} and \autodoc:command{\bidi-on}, which allow you to trade off bidirectional support for a dubious increase in speed.
\end{document}
]]

return package

1	local base = require("packages.base")	7✔
2
3	local package = pl.class(base)	7✔
4	package._name = "bidi"	7✔
5
6	local icu = require("justenoughicu")	7✔
7
8	local function reverse_portion (tbl, s, e)
9	local rv = {}	×
10	for i = 1, s-1 do rv[#rv+1] = tbl[i] end	×
11	for i = e, s, -1 do rv[#rv+1] = tbl[i] end	×
12	for i = e+1, #tbl do rv[#rv+1] = tbl[i] end	×
13	return rv	×
14	end
15
16	local function create_matrix (line, base_level)
17	-- L2; create a transformation matrix of elements
18	-- such that output[matrix[i]] = input[i]
19	-- e.g. No reversions required: [1, 2, 3, 4, 5]
20	-- Levels [0, 0, 0, 1, 1] -> [1, 2, 3, 5, 4]
21
22	local max_level = 0	49✔
23	local matrix = {}	49✔
24	for i, c in next, line do	913✔
25	if c.level > max_level then max_level = c.level end	864✔
26	matrix[i] = i	864✔
27	end
28
29	for level = base_level+1, max_level do	49✔
30	local level_start
31	for i, _ in next, line do	×
32	if line[i].level >= level then	×
33	if not level_start then	×
34	level_start = i	×
35	elseif i == #line then	×
36	local level_end = i	×
37	matrix = reverse_portion(matrix, level_start, level_end)	×
38	level_start = nil	×
39	end
40	else
41	if level_start then	×
42	local level_end = i-1	×
43	matrix = reverse_portion(matrix, level_start, level_end)	×
44	level_start = nil	×
45	end
46	end
47	end
48	end
49
50	return matrix	49✔
51	end
52
53	local function reverse_each_node (nodelist)
54	for j = 1, #nodelist do	×
55	if nodelist[j].type =="hbox" then	×
56	if nodelist[j].value.items then SU.flip_in_place(nodelist[j].value.items) end	×
57	SU.flip_in_place(nodelist[j].value.glyphString)	×
58	end
59	end
60	end
61
62	local nodeListToText = function (nl)
63	local owners, text = {}, {}	35✔
64	local p = 1	35✔
65	for i = 1, #nl do local n = nl[i]	248✔
66	if n.text then	124✔
67	local utfchars = SU.splitUtf8(n.text)	40✔
68	for j = 1, #utfchars do	1,533✔
69	owners[p] = { node = n, pos = j }	1,493✔
70	text[p] = utfchars[j]	1,493✔
71	p = p + 1	1,493✔
72	end
73	else
74	owners[p] = { node = n }	84✔
75	text[p] = luautf8.char(0xFFFC)	84✔
76	p = p + 1	84✔
77	end
78	end
79	return owners, text	35✔
80	end
81
82	local splitNodeAtPos = function (n, splitstart, p)
83	if n.is_unshaped then	×
84	local utf8chars = SU.splitUtf8(n.text)	×
85	local n2 = SILE.nodefactory.unshaped({ text = "", options = pl.tablex.copy(n.options) })	×
86	local n1 = SILE.nodefactory.unshaped({ text = "", options = pl.tablex.copy(n.options) })	×
87	for i = splitstart, #utf8chars do	×
88	if i <= p then n1.text = n1.text .. utf8chars[i]	×
89	else n2.text = n2.text .. utf8chars[i]	×
90	end
91	end
92	return n1, n2	×
93	else
94	SU.error("Unsure how to split node " .. tostring(n) .. " at position " .. p, true)	×
95	end
96	end
97
98	local splitNodelistIntoBidiRuns = function (typesetter)
99	local nl = typesetter.state.nodes	35✔
100	if #nl == 0 then return nl end	35✔
101	local owners, text = nodeListToText(nl)	35✔
102	local base_level = typesetter.frame:writingDirection() == "RTL" and 1 or 0	70✔
103	local runs = { icu.bidi_runs(table.concat(text), typesetter.frame:writingDirection()) }	70✔
104	table.sort(runs, function (a, b) return a.start < b.start end)	35✔
105	-- local newNl = {}
106	-- Split nodes on run boundaries
107	for i = 1, #runs do	70✔
108	local run = runs[i]	35✔
109	local thisOwner = owners[run.start+run.length]	35✔
110	local nextOwner = owners[run.start+1+run.length]	35✔
111	-- print(thisOwner, nextOwner)
112	if nextOwner and thisOwner.node == nextOwner.node then	35✔
113	local before, after = splitNodeAtPos(nextOwner.node, 1, nextOwner.pos-1)	×
114	-- print(before, after)
115	local start = nil
116	for j = run.start+1, run.start+run.length do	×
117	if owners[j].node==nextOwner.node then	×
118	if not start then start = j end	×
119	owners[j]={ node=before ,pos=j-start+1 }	×
120	end
121	end
122	for j = run.start + 1 + run.length, #owners do	×
123	if owners[j].node==nextOwner.node then	×
124	owners[j] = { node = after, pos = j - (run.start + run.length) }	×
125	end
126	end
127	end
128	end
129	-- Assign direction/level to nodes
130	for i = 1, #runs do	70✔
131	local runstart = runs[i].start+1	35✔
132	local runend = runstart + runs[i].length-1	35✔
133	for j= runstart, runend do	1,612✔
134	if owners[j].node and owners[j].node.options then	1,577✔
135	owners[j].node.options.direction = runs[i].dir	1,493✔
136	owners[j].node.options.bidilevel = runs[i].level - base_level	1,493✔
137	end
138	end
139	end
140	-- String together nodelist
141	nl={}	35✔
142	for i = 1, #owners do	1,612✔
143	if #nl and nl[#nl] ~= owners[i].node then	1,577✔
144	nl[#nl+1] = owners[i].node	124✔
145	-- print(nl[#nl], nl[#nl].options)
146	end
147	end
148	-- for i = 1, #nl do print(i, nl[i]) end
149	return nl	35✔
150	end
151
152	local bidiBoxupNodes = function (typesetter)
153	local allDone = true	95✔
154	for i = 1, #typesetter.state.nodes do	219✔
155	if not typesetter.state.nodes[i].bidiDone then allDone = false end	164✔
156	end
157	if allDone then return typesetter:nobidi_boxUpNodes() end	95✔
158	local newNodeList = splitNodelistIntoBidiRuns(typesetter)	35✔
159	typesetter:shapeAllNodes(newNodeList)	35✔
160	typesetter.state.nodes = newNodeList	35✔
161	local vboxlist = typesetter:nobidi_boxUpNodes()	35✔
162	-- Scan for out-of-direction material
163	for i = 1, #vboxlist do	141✔
164	local v = vboxlist[i]	106✔
165	if v.is_vbox then package.reorder(nil, v, typesetter) end	106✔
166	end
167	return vboxlist	35✔
168	end
169
170	function package.reorder (_, n, typesetter)	7✔
171	local nl = n.nodes	49✔
172	-- local newNl = {}
173	-- local matrix = {}
174	local levels = {}	49✔
175	local base_level = typesetter.frame:writingDirection() == "RTL" and 1 or 0	98✔
176	for i = 1, #nl do	913✔
177	if nl[i].options and nl[i].options.bidilevel then	864✔
178	levels[i] = { level = nl[i].options.bidilevel }	292✔
179	end
180	end
181	for i = 1, #nl do	913✔
182	if not levels[i] then	864✔
183	-- resolve neutrals
184	local left_level, right_level
185	for left = i - 1, 1, -1 do	1,285✔
186	if nl[left].options and nl[left].options.bidilevel then	1,082✔
187	left_level = nl[left].options.bidilevel	369✔
188	break	369✔
189	end
190	end
191	for right = i + 1, #nl do	1,285✔
192	if nl[right].options and nl[right].options.bidilevel then	1,053✔
193	right_level = nl[right].options.bidilevel	340✔
194	break	340✔
195	end
196	end
197	levels[i] = { level = (left_level == right_level and left_level or 0) }	572✔
198	end
199	end
200	local matrix = create_matrix(levels, 0)	49✔
201	local rv = {}	49✔
202	-- for i = 1, #nl do print(i, nl[i], levels[i]) end
203	for i = 1, #nl do	913✔
204	if nl[i].is_nnode and levels[i].level %2 ~= base_level then	864✔
205	SU.flip_in_place(nl[i].nodes)	×
206	reverse_each_node(nl[i].nodes)	×
207	elseif nl[i].is_discretionary and levels[i].level %2 ~= base_level and not nl[i].bidiDone then	864✔
208	for j = 1, #(nl[i].replacement) do	×
209	if nl[i].replacement[j].is_nnode then	×
210	SU.flip_in_place(nl[i].replacement[j].nodes)	×
211	reverse_each_node(nl[i].replacement[j].nodes)	×
212	end
213	end
214	for j = 1, #(nl[i].prebreak) do	×
215	if nl[i].prebreak[j].is_nnode then	×
216	SU.flip_in_place(nl[i].prebreak[j].nodes)	×
217	reverse_each_node(nl[i].prebreak[j].nodes)	×
218	end
219	end
220	for j = 1, #(nl[i].postbreak) do	×
221	if nl[i].postbreak[j].is_nnode then	×
222	SU.flip_in_place(nl[i].postbreak[j].nodes)	×
223	reverse_each_node(nl[i].postbreak[j].nodes)	×
224	end
225	end
226
227	end
228	rv[matrix[i]] = nl[i]	864✔
229	nl[i].bidiDone = true	864✔
230	-- rv[i] = nl[i]
231	end
232	n.nodes = SU.compress(rv)	98✔
233	end
234
235	function package:bidiEnableTypesetter (typesetter)	7✔
236	if typesetter.nobidi_boxUpNodes and self.class._initialized then	7✔
237	return SU.warn("BiDi already enabled, nothing to turn on")	×
238	end
239	typesetter.nobidi_boxUpNodes = typesetter.boxUpNodes	7✔
240	typesetter.boxUpNodes = bidiBoxupNodes	7✔
241	end
242
243	function package:bidiDisableTypesetter (typesetter)	7✔
244	if not typesetter.nobidi_boxUpNodes and self.class._initialized then	×
245	return SU.warn("BiDi not enabled, nothing to turn off")	×
246	end
247	typesetter.boxUpNodes = typesetter.nobidi_boxUpNodes	×
248	typesetter.nobidi_boxUpNodes = nil	×
249	end
250
251	function package:_init ()	7✔
252	base._init(self)	7✔
253	self:deprecatedExport("reorder", self.reorder)	7✔
254	self:deprecatedExport("bidiEnableTypesetter", self.bidiEnableTypesetter)	7✔
255	self:deprecatedExport("bidiDisableTypesetter", self.bidiDisableTypesetter)	7✔
256	if SILE.typesetter then	7✔
257	self:bidiEnableTypesetter(SILE.typesetter)	×
258	end
259	self:bidiEnableTypesetter(SILE.typesetters.base)	14✔
260	end
261
262	function package:registerCommands ()	7✔
263
264	self:registerCommand("thisframeLTR", function (_, _)	14✔
265	local direction = "LTR"	×
266	SILE.typesetter.frame.direction = direction	×
267	SILE.settings:set("font.direction", direction)	×
268	SILE.typesetter:leaveHmode()	×
269	SILE.typesetter.frame:newLine()	×
270	end)
271
272	self:registerCommand("thisframedirection", function (options, _)	14✔
273	local direction = SU.required(options, "direction", "frame direction")	×
274	SILE.typesetter.frame.direction = direction	×
275	SILE.settings:set("font.direction", direction)	×
276	SILE.typesetter:leaveHmode()	×
277	SILE.typesetter.frame:init()	×
278	end)
279
280	self:registerCommand("thisframeRTL", function (_, _)	14✔
281	local direction = "RTL"	×
282	SILE.typesetter.frame.direction = direction	×
283	SILE.settings:set("font.direction", direction)	×
284	SILE.typesetter:leaveHmode()	×
285	SILE.typesetter.frame:newLine()	×
286	end)
287
288	self:registerCommand("bidi-on", function (_, _)	14✔
289	self:bidiEnableTypesetter(SILE.typesetter)	×
290	end)
291
292	self:registerCommand("bidi-off", function (_, _)	14✔
293	self:bidiDisableTypesetter(SILE.typesetter)	×
294	end)
295
296	end
297
298	package.documentation = [[
299	\begin{document}
300	Scripts like the Latin alphabet you are currently reading are normally written left to right (LTR); however, some scripts, such as Arabic and Hebrew, are written right to left (RTL).
301	The \autodoc:package{bidi} package, which is loaded by default, provides SILE with the ability to correctly typeset right-to-left text and also documents which mix right-to-left and left-to-right typesetting.
302	Because it is loaded by default, you can use both LTR and RTL text within a paragraph and SILE will ensure that the output characters appear in the correct order.
303
304	The \autodoc:package{bidi} package provides two commands, \autodoc:command{\thisframeLTR} and \autodoc:command{\thisframeRTL}, which set the default text direction for the current frame.
305	If you tell SILE that a frame is RTL, the text will start in the right margin and proceed leftward.
306	It also provides the commands \autodoc:command{\bidi-off} and \autodoc:command{\bidi-on}, which allow you to trade off bidirectional support for a dubious increase in speed.
307	\end{document}
308	]]	7✔
309
310	return package	7✔

sile-typesetter / sile / 6713098919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous