8288578143

Committed 14 Mar 2024 09:39PM UTC coverage: 64.155% (-10.6%) from 74.718%

Build # 8288578143

Build Type

Pull #1904

github

Committed by

alerque

Commit Message

chore(core): Fixup ec6ed657 which didn't shim old pack styles properly

Pull Request Pull Request #1904: Merge develop into master (commit to next release being breaking)

Run Details

1648 of 2421 new or added lines in 107 files covered. (68.07%)

1843 existing lines in 77 files now uncovered.

10515 of 16390 relevant lines covered (64.15%)

3306.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.8

/languages/unicode.lua

local icu = require("justenoughicu")

local chardata = require("char-def")

SILE.settings:declare({
  parameter = "languages.fixedNbsp",
  type = "boolean",
  default = false,
  help = "Whether to treat U+00A0 (NO-BREAK SPACE) as a fixed-width space"
})

SILE.nodeMakers.base = pl.class({

    _init = function (self, options)
      self.contents = {}
      self.options = options
      self.token = ""
      self.lastnode = false
      self.lasttype = false
    end,

    makeToken = function (self)
      if #self.contents > 0 then
        coroutine.yield(SILE.shaper:formNnode(self.contents, self.token, self.options))
        SU.debug("tokenizer", "Token:", self.token)
        self.contents = {}
        self.token = ""
        self.lastnode = "nnode"
      end
    end,

    addToken = function (self, char, item)
      self.token = self.token .. char
      table.insert(self.contents, item)
    end,

    makeGlue = function (self, item)
      if SILE.settings:get("typesetter.obeyspaces") or self.lastnode ~= "glue" then
        SU.debug("tokenizer", "Space node")
        coroutine.yield(SILE.shaper:makeSpaceNode(self.options, item))
      end
      self.lastnode = "glue"
      self.lasttype = "sp"
    end,

    makePenalty = function (self, p)
      if self.lastnode ~= "penalty" and self.lastnode ~= "glue" then
        coroutine.yield( SILE.types.node.penalty({ penalty = p or 0 }) )
      end
      self.lastnode = "penalty"
    end,

    makeNonBreakingSpace = function (self)
      -- Unicode Line Breaking Algorithm (UAX 14) specifies that U+00A0
      -- (NO-BREAK SPACE) is expanded or compressed like a normal space.
      coroutine.yield(SILE.types.node.kern(SILE.shaper:measureSpace(self.options)))
      self.lastnode = "glue"
      self.lasttype = "sp"
    end,

    iterator = function (_, _)
      SU.error("Abstract function nodemaker:iterator called", true)
    end,

    charData = function (_, char)
      local cp = SU.codepoint(char)
      if not chardata[cp] then return {} end
      return chardata[cp]
    end,

    isActiveNonBreakingSpace = function (self, char)
      return self:isNonBreakingSpace(char) and not SILE.settings:get("languages.fixedNbsp")
    end,

    isBreaking = function (self, char)
      return self.breakingTypes[self:charData(char).linebreak]
    end,

    isNonBreakingSpace = function (self, char)
      local c = self:charData(char)
      return c.contextname and c.contextname == "nobreakspace"
    end,

    isPunctuation = function (self, char)
      return self.puctuationTypes[self:charData(char).category]
    end,

    isSpace = function (self, char)
      return self.spaceTypes[self:charData(char).linebreak]
    end,

    isQuote = function (self, char)
      return self.quoteTypes[self:charData(char).linebreak]
    end,

    isWord = function (self, char)
      return self.wordTypes[self:charData(char).linebreak]
    end,

  })

SILE.nodeMakers.unicode = pl.class(SILE.nodeMakers.base)

SILE.nodeMakers.unicode.breakingTypes = { ba = true, zw = true }
SILE.nodeMakers.unicode.puctuationTypes = { po = true }
SILE.nodeMakers.unicode.quoteTypes = {} -- quote linebreak category is ambiguous depending on the language
SILE.nodeMakers.unicode.spaceTypes = { sp = true }
SILE.nodeMakers.unicode.wordTypes = { cm = true }

function SILE.nodeMakers.unicode:dealWith (item)
  local char = item.text
  local cp = SU.codepoint(char)
  local thistype = chardata[cp] and chardata[cp].linebreak
  if self:isSpace(item.text) then
    self:makeToken()
    self:makeGlue(item)
  elseif self:isActiveNonBreakingSpace(item.text) then
    self:makeToken()
    self:makeNonBreakingSpace()
  elseif self:isBreaking(item.text) then
    self:addToken(char, item)
    self:makeToken()
    self:makePenalty(0)
  elseif self:isQuote(item.text) then
    self:addToken(char, item)
    self:makeToken()
  elseif self.lasttype and (thistype and thistype ~= self.lasttype and not self:isWord(thistype)) then
    self:addToken(char, item)
  else
    self:letterspace()
    self:addToken(char, item)
  end
  self.lasttype = thistype
end

function SILE.nodeMakers.unicode:handleInitialGlue (items)
  local i = 1
  while i <= #items do
    local item = items[i]
    if self:isSpace(item.text) then self:makeGlue(item) else break end
    i = i + 1
  end
  return i, items
end

function SILE.nodeMakers.unicode:letterspace ()
  if not SILE.settings:get("document.letterspaceglue") then return end
  if self.token then self:makeToken() end
  if self.lastnode and self.lastnode ~= "glue" then
    local w = SILE.settings:get("document.letterspaceglue").width
    SU.debug("tokenizer", "Letter space glue:", w)
    coroutine.yield(SILE.types.node.kern({ width = w }))
    self.lastnode = "glue"
    self.lasttype = "sp"
  end
end

function SILE.nodeMakers.unicode.isICUBreakHere (_, chunks, item)
  return chunks[1] and (item.index >= chunks[1].index)
end

function SILE.nodeMakers.unicode:handleICUBreak (chunks, item)
  -- The ICU library has told us there is a breakpoint at
  -- this index. We need to...
  local bp = chunks[1]
  -- ... remove this breakpoint (and any out of order ones)
  -- from the ICU breakpoints array so that chunks[1] is
  -- the next index point for comparison against the string...
  while chunks[1] and item.index >= chunks[1].index do
    table.remove(chunks, 1)
  end
  -- ...decide which kind of breakpoint we have here and
  -- handle it appropriately.
  if bp.type == "word" then
    self:handleWordBreak(item)
  elseif bp.type == "line" then
    self:handleLineBreak(item, bp.subtype)
  end
  return chunks
end

function SILE.nodeMakers.unicode:handleWordBreak (item)
  self:makeToken()
  if self:isSpace(item.text) then
    -- Spacing word break
    self:makeGlue(item)
  elseif self:isActiveNonBreakingSpace(item.text) then
    -- Non-breaking space word break
    self:makeNonBreakingSpace()
  else
     -- a word break which isn't a space
    self:addToken(item.text, item)
  end
end

function SILE.nodeMakers.unicode:_handleWordBreakRepeatHyphen (item)
  -- According to some language rules, when a break occurs at an explicit hyphen,
  -- the hyphen gets repeated at the beginning of the new line
  if item.text == "-" then
    self:addToken(item.text, item)
    self:makeToken()
    if self.lastnode ~= "discretionary" then
      coroutine.yield(SILE.nodefactory.discretionary({
        postbreak = SILE.shaper:createNnodes("-", self.options)
      }))
      self.lastnode = "discretionary"
    end
  else
    SILE.nodeMakers.unicode.handleWordBreak(self, item)
  end
end

function SILE.nodeMakers.unicode:handleLineBreak (item, subtype)
  -- Because we are in charge of paragraphing, we
  -- will override space-type line breaks, and treat
  -- them just as ordinary word spaces.
  if self:isSpace(item.text) or self:isActiveNonBreakingSpace(item.text) then
    self:handleWordBreak(item)
    return
  end
  -- But explicit line breaks we will turn into
  -- soft and hard breaks.
  self:makeToken()
  self:makePenalty(subtype == "soft" and 0 or -1000)
  local char = item.text
  self:addToken(char, item)
  local cp = SU.codepoint(char)
  self.lasttype = chardata[cp] and chardata[cp].linebreak
end

function SILE.nodeMakers.unicode:_handleLineBreakRepeatHyphen (item, subtype)
  if self.lastnode == "discretionary" then
    -- Initial word boundary after a discretionary:
    -- Bypass it and just deal with the token.
    self:dealWith(item)
  else
    SILE.nodeMakers.unicode.handleLineBreak(self, item, subtype)
  end
end

function SILE.nodeMakers.unicode:iterator (items)
  local fulltext = ""
  for i = 1, #items do
    fulltext = fulltext .. items[i].text
  end
  local chunks = { icu.breakpoints(fulltext, self.options.language) }
  table.remove(chunks, 1)
  return coroutine.wrap(function ()
    local i
    i, self.items = self:handleInitialGlue(items)
    for j = i, #items do
      self.i = j
      self.item = self.items[self.i]
      if self:isICUBreakHere(chunks, self.item) then
        chunks = self:handleICUBreak(chunks, self.item)
      else
        self:dealWith(self.item)
      end
    end
    self:makeToken()
  end)
end

1	local icu = require("justenoughicu")	97✔
2
3	local chardata = require("char-def")	97✔
4
5	SILE.settings:declare({	97✔
6	parameter = "languages.fixedNbsp",
7	type = "boolean",
8	default = false,
9	help = "Whether to treat U+00A0 (NO-BREAK SPACE) as a fixed-width space"	×
10	})
11
12	SILE.nodeMakers.base = pl.class({	194✔
13
14	_init = function (self, options)
15	self.contents = {}	1,998✔
16	self.options = options	1,998✔
17	self.token = ""	1,998✔
18	self.lastnode = false	1,998✔
19	self.lasttype = false	1,998✔
20	end,
21
22	makeToken = function (self)
23	if #self.contents > 0 then	5,387✔
24	coroutine.yield(SILE.shaper:formNnode(self.contents, self.token, self.options))	7,544✔
25	SU.debug("tokenizer", "Token:", self.token)	3,772✔
26	self.contents = {}	3,772✔
27	self.token = ""	3,772✔
28	self.lastnode = "nnode"	3,772✔
29	end
30	end,
31
32	addToken = function (self, char, item)
33	self.token = self.token .. char	12,335✔
34	table.insert(self.contents, item)	12,335✔
35	end,
36
37	makeGlue = function (self, item)
38	if SILE.settings:get("typesetter.obeyspaces") or self.lastnode ~= "glue" then	3,244✔
39	SU.debug("tokenizer", "Space node")	1,602✔
40	coroutine.yield(SILE.shaper:makeSpaceNode(self.options, item))	3,204✔
41	end
42	self.lastnode = "glue"	1,622✔
43	self.lasttype = "sp"	1,622✔
44	end,
45
46	makePenalty = function (self, p)
47	if self.lastnode ~= "penalty" and self.lastnode ~= "glue" then	1,505✔
48	coroutine.yield( SILE.types.node.penalty({ penalty = p or 0 }) )	28✔
49	end
50	self.lastnode = "penalty"	1,505✔
51	end,
52
53	makeNonBreakingSpace = function (self)
54	-- Unicode Line Breaking Algorithm (UAX 14) specifies that U+00A0
55	-- (NO-BREAK SPACE) is expanded or compressed like a normal space.
56	coroutine.yield(SILE.types.node.kern(SILE.shaper:measureSpace(self.options)))	21✔
57	self.lastnode = "glue"	7✔
58	self.lasttype = "sp"	7✔
59	end,
60
61	iterator = function (_, _)
62	SU.error("Abstract function nodemaker:iterator called", true)	×
63	end,
64
65	charData = function (_, char)
66	local cp = SU.codepoint(char)	49,476✔
67	if not chardata[cp] then return {} end	49,476✔
68	return chardata[cp]	48,624✔
69	end,
70
71	isActiveNonBreakingSpace = function (self, char)
72	return self:isNonBreakingSpace(char) and not SILE.settings:get("languages.fixedNbsp")	24,691✔
73	end,
74
75	isBreaking = function (self, char)
76	return self.breakingTypes[self:charData(char).linebreak]	21,120✔
77	end,
78
79	isNonBreakingSpace = function (self, char)
80	local c = self:charData(char)	12,342✔
81	return c.contextname and c.contextname == "nobreakspace"	12,342✔
82	end,
83
84	isPunctuation = function (self, char)
85	return self.puctuationTypes[self:charData(char).category]	×
86	end,
87
88	isSpace = function (self, char)
89	return self.spaceTypes[self:charData(char).linebreak]	31,882✔
90	end,
91
92	isQuote = function (self, char)
93	return self.quoteTypes[self:charData(char).linebreak]	21,120✔
94	end,
95
96	isWord = function (self, char)
97	return self.wordTypes[self:charData(char).linebreak]	146✔
98	end,
99
100	})	97✔
101
102	SILE.nodeMakers.unicode = pl.class(SILE.nodeMakers.base)	194✔
103
104	SILE.nodeMakers.unicode.breakingTypes = { ba = true, zw = true }	97✔
105	SILE.nodeMakers.unicode.puctuationTypes = { po = true }	97✔
106	SILE.nodeMakers.unicode.quoteTypes = {} -- quote linebreak category is ambiguous depending on the language	97✔
107	SILE.nodeMakers.unicode.spaceTypes = { sp = true }	97✔
108	SILE.nodeMakers.unicode.wordTypes = { cm = true }	97✔
109
110	function SILE.nodeMakers.unicode:dealWith (item)	194✔
111	local char = item.text	10,562✔
112	local cp = SU.codepoint(char)	10,562✔
113	local thistype = chardata[cp] and chardata[cp].linebreak	10,562✔
114	if self:isSpace(item.text) then	21,124✔
115	self:makeToken()	2✔
116	self:makeGlue(item)	4✔
117	elseif self:isActiveNonBreakingSpace(item.text) then	21,120✔
118	self:makeToken()	×
119	self:makeNonBreakingSpace()	×
120	elseif self:isBreaking(item.text) then	21,120✔
UNCOV 121	self:addToken(char, item)	×
UNCOV 122	self:makeToken()	×
UNCOV 123	self:makePenalty(0)	×
124	elseif self:isQuote(item.text) then	21,120✔
UNCOV 125	self:addToken(char, item)	×
UNCOV 126	self:makeToken()	×
127	elseif self.lasttype and (thistype and thistype ~= self.lasttype and not self:isWord(thistype)) then	10,633✔
128	self:addToken(char, item)	146✔
129	else
130	self:letterspace()	10,487✔
131	self:addToken(char, item)	10,487✔
132	end
133	self.lasttype = thistype	10,562✔
134	end
135
136	function SILE.nodeMakers.unicode:handleInitialGlue (items)	194✔
137	local i = 1	1,996✔
138	while i <= #items do	2,064✔
139	local item = items[i]	2,045✔
140	if self:isSpace(item.text) then self:makeGlue(item) else break end	4,158✔
141	i = i + 1	68✔
142	end
143	return i, items	1,996✔
144	end
145
146	function SILE.nodeMakers.unicode:letterspace ()	194✔
147	if not SILE.settings:get("document.letterspaceglue") then return end	20,974✔
148	if self.token then self:makeToken() end	55✔
149	if self.lastnode and self.lastnode ~= "glue" then	55✔
150	local w = SILE.settings:get("document.letterspaceglue").width	102✔
151	SU.debug("tokenizer", "Letter space glue:", w)	51✔
152	coroutine.yield(SILE.types.node.kern({ width = w }))	102✔
153	self.lastnode = "glue"	51✔
154	self.lasttype = "sp"	51✔
155	end
156	end
157
158	function SILE.nodeMakers.unicode.isICUBreakHere (_, chunks, item)	194✔
159	return chunks[1] and (item.index >= chunks[1].index)	13,896✔
160	end
161
162	function SILE.nodeMakers.unicode:handleICUBreak (chunks, item)	194✔
163	-- The ICU library has told us there is a breakpoint at
164	-- this index. We need to...
165	local bp = chunks[1]	3,334✔
166	-- ... remove this breakpoint (and any out of order ones)
167	-- from the ICU breakpoints array so that chunks[1] is
168	-- the next index point for comparison against the string...
169	while chunks[1] and item.index >= chunks[1].index do	6,668✔
170	table.remove(chunks, 1)	6,668✔
171	end
172	-- ...decide which kind of breakpoint we have here and
173	-- handle it appropriately.
174	if bp.type == "word" then	3,334✔
175	self:handleWordBreak(item)	3,658✔
176	elseif bp.type == "line" then	1,505✔
177	self:handleLineBreak(item, bp.subtype)	1,505✔
178	end
179	return chunks	3,334✔
180	end
181
182	function SILE.nodeMakers.unicode:handleWordBreak (item)	194✔
183	self:makeToken()	1,829✔
184	if self:isSpace(item.text) then	3,658✔
185	-- Spacing word break
186	self:makeGlue(item)	3,104✔
187	elseif self:isActiveNonBreakingSpace(item.text) then	554✔
188	-- Non-breaking space word break
189	self:makeNonBreakingSpace()	14✔
190	else
191	-- a word break which isn't a space
192	self:addToken(item.text, item)	270✔
193	end
194	end
195
196	function SILE.nodeMakers.unicode:_handleWordBreakRepeatHyphen (item)	194✔
197	-- According to some language rules, when a break occurs at an explicit hyphen,
198	-- the hyphen gets repeated at the beginning of the new line
UNCOV 199	if item.text == "-" then	×
UNCOV 200	self:addToken(item.text, item)	×
UNCOV 201	self:makeToken()	×
UNCOV 202	if self.lastnode ~= "discretionary" then	×
UNCOV 203	coroutine.yield(SILE.nodefactory.discretionary({	×
UNCOV 204	postbreak = SILE.shaper:createNnodes("-", self.options)	×
205	}))
UNCOV 206	self.lastnode = "discretionary"	×
207	end
208	else
UNCOV 209	SILE.nodeMakers.unicode.handleWordBreak(self, item)	×
210	end
211	end
212
213	function SILE.nodeMakers.unicode:handleLineBreak (item, subtype)	194✔
214	-- Because we are in charge of paragraphing, we
215	-- will override space-type line breaks, and treat
216	-- them just as ordinary word spaces.
217	if self:isSpace(item.text) or self:isActiveNonBreakingSpace(item.text) then	4,515✔
218	self:handleWordBreak(item)	×
219	return	×
220	end
221	-- But explicit line breaks we will turn into
222	-- soft and hard breaks.
223	self:makeToken()	1,505✔
224	self:makePenalty(subtype == "soft" and 0 or -1000)	1,505✔
225	local char = item.text	1,505✔
226	self:addToken(char, item)	1,505✔
227	local cp = SU.codepoint(char)	1,505✔
228	self.lasttype = chardata[cp] and chardata[cp].linebreak	1,505✔
229	end
230
231	function SILE.nodeMakers.unicode:_handleLineBreakRepeatHyphen (item, subtype)	194✔
232	if self.lastnode == "discretionary" then	×
233	-- Initial word boundary after a discretionary:
234	-- Bypass it and just deal with the token.
235	self:dealWith(item)	×
236	else
237	SILE.nodeMakers.unicode.handleLineBreak(self, item, subtype)	×
238	end
239	end
240
241	function SILE.nodeMakers.unicode:iterator (items)	194✔
242	local fulltext = ""	1,996✔
243	for i = 1, #items do	15,960✔
244	fulltext = fulltext .. items[i].text	13,964✔
245	end
246	local chunks = { icu.breakpoints(fulltext, self.options.language) }	1,996✔
247	table.remove(chunks, 1)	1,996✔
248	return coroutine.wrap(function ()	1,996✔
249	local i
250	i, self.items = self:handleInitialGlue(items)	3,992✔
251	for j = i, #items do	15,892✔
252	self.i = j	13,896✔
253	self.item = self.items[self.i]	13,896✔
254	if self:isICUBreakHere(chunks, self.item) then	27,792✔
255	chunks = self:handleICUBreak(chunks, self.item)	6,668✔
256	else
257	self:dealWith(self.item)	10,562✔
258	end
259	end
260	self:makeToken()	1,996✔
261	end)
262	end

sile-typesetter / sile / 8288578143

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous