6713098919

Committed 31 Oct 2023 10:21PM UTC coverage: 52.831% (-21.8%) from 74.636%

Build # 6713098919

Build Type

push

github

Committed by

web-flow

Commit Message

Merge d0a2a1ee9 into b185d4972

Run Details

45 of 45 new or added lines in 3 files covered. (100.0%)

8173 of 15470 relevant lines covered (52.83%)

6562.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/languages/ja.lua

-- "jlreq" refers to http://www.w3.org/TR/jlreq/
-- "JIS" refers to JIS X 4051

local hiragana = function (c) return c > 0x3040 and c <= 0x309f end
local katakana = function (c) return c > 0x30a0 and c <= 0x30ff end
local kanji = function (c) return c >= 0x4e00 and c <= 0x9fcc end

local classes = { -- from jlreq
  [0x2018] = 1, [0x201C] = 1, [0x0028] = 1, [0x3014] = 1, [0x005B] = 1,
  [0x007B] = 1, [0x3008] = 1, [0x300A] = 1, [0x300C] = 1, [0x300E] = 1,
  [0x3010] = 1, [0x2985] = 1, [0x3018] = 1, [0x3016] = 1, [0x00AB] = 1, [0x301D] = 1,
  [0xFF08] = 1, -- see note 1 of appendix A
  [0x2019] = 2, [0x201D] = 2, [0x0029] = 2, [0x3015] = 2, [0x005D] = 2,
  [0x007D] = 2, [0x3009] = 2, [0x300B] = 2, [0x300D] = 2, [0x300F] = 2,
  [0x3011] = 2, [0x2986] = 2, [0x3019] = 2, [0x3017] = 2, [0x00BB] = 2, [0x301F] = 2,
  [0xFF09] = 2, -- see note 1 of appendix A
  [0x2010] = 3, [0x301C] = 3, [0x30A0] = 3, [0x2013] = 3,
  [0x0021] = 4, [0x003F] = 4, [0x203C] = 4, [0x2047] = 4, [0x2048] = 4, [0x2049] = 4,
  [0x30FB] = 5, [0x003A] = 5, [0x003B] = 5,
  [0x3002] = 6, [0x002E] = 6,
  [0x3001] = 7, [0x002C] = 7,
  [0x2014] = 8, [0x2026] = 8, [0x2025] = 8, [0x3033] = 8, [0x3034] = 8, [0x3035] = 8,
  [0x30FD] = 9, [0x30FE] = 9, [0x309D] = 9, [0x309E] = 9, [0x3005] = 9, [0x303B] = 9,
  [0x30FC] = 10,
  [0x3041] = 11, [0x3043] = 11, [0x3045] = 11, [0x3047] = 11,
  [0x3049] = 11, [0x30A1] = 11, [0x30A3] = 11, [0x30A5] = 11,
  [0x30A7] = 11, [0x30A9] = 11, [0x3063] = 11, [0x3083] = 11,
  [0x3085] = 11, [0x3087] = 11, [0x308E] = 11, [0x3095] = 11,
  [0x3096] = 11, [0x30C3] = 11, [0x30E3] = 11, [0x30E5] = 11,
  [0x30E7] = 11, [0x30EE] = 11, [0x30F5] = 11, [0x30F6] = 11,
  [0x31F0] = 11, [0x31F1] = 11, [0x31F2] = 11, [0x31F3] = 11,
  [0x31F4] = 11, [0x31F5] = 11, [0x31F6] = 11, [0x31F7] = 11,
  [0x31F8] = 11, [0x31F9] = 11, [0x31FA] = 11, [0x31FB] = 11,
  [0x31FC] = 11, [0x31FD] = 11, [0x31FE] = 11, [0x31FF] = 11,
  [0x00A5] = 12, [0x0024] = 12, [0x00A3] = 12, [0x0023] = 12, [0x20AC] = 12, [0x2116] = 12,
  [0x00B0] = 13, [0x2032] = 13, [0x2033] = 13, [0x2103] = 13, [0x00A2] = 13,
  [0x0025] = 13, [0x2030] = 13, [0x33CB] = 13, [0x2113] = 13, [0x3303] = 13,
  [0x330D] = 13, [0x3314] = 13, [0x3318] = 13, [0x3322] = 13, [0x3323] = 13,
  [0x3326] = 13, [0x3327] = 13, [0x332B] = 13, [0x3336] = 13, [0x333B] = 13,
  [0x3349] = 13, [0x334A] = 13, [0x334D] = 13, [0x3351] = 13, [0x3357] = 13,
  [0x338E] = 13, [0x338F] = 13, [0x339C] = 13, [0x339D] = 13, [0x339E] = 13,
  [0x33A1] = 13, [0x33C4] = 13,
  [0x3000] = 14,
}

local jisClass = function (c)
  if c == -1 then return -1 end
  if classes[c] then return classes[c] end
  if hiragana(c) then return 15 end
  if katakana(c) then return 16 end
  if kanji(c) then return 19 end -- Other things are also 19
  return 27 -- this is lazy
end

-- This roughly implements the kinsoku shori given in Appendix C of jlreq
local badBeforeClasses = { [1] = true, [12] = true, [28] = true }
local badAfterClasses = { }
for _, v in ipairs({ 2, 3, 4, 5, 6, 7, 9, 10, 11, 20, 29 }) do badAfterClasses[v] = true end

local function breakAllowed (before, after)
  local bc = jisClass(before)
  local ac = jisClass(after)
  if badBeforeClasses[bc] then return false end
  if badAfterClasses[ac] then return false end
  if bc == 8 and ac == 8 then
    if before == after then return false else return true end
  end
  -- The following rule is not in jlreq but it stops the algorithm from breaking
  -- up Latin words!
  if bc == 27 and ac == 27 then return false end
  return true
end

local function intercharacterspace (before, after)
  local bc = jisClass(before)
  local ac = jisClass(after)
  -- This rule is not in jlreq but it stops situations like 1：2 getting munched
  if (bc == 5 and ac == 27) or (bc == 27 and ac == 5) then return 0 end

  -- jlreq expects yakumono to contain no following space, so recommends half
  -- a space afterwards. However, in real world fonts, yakumono occupies a full
  -- zenkaku width with trailing space built into the glyph. So we do not add
  -- space there. Instead, where no space is stipulated, we add negative space
  -- to counteract the trailing space in the glyph.
  if ac == 5 then return "-0.25zw" end
  if bc == 5 then return "-0.25zw" end
  if bc == 7 and ac == 1 then return "-0.5zw" end
  if bc == 2 and ac == 1 then return "-0.5zw" end
  if bc == -1 and ac == 1 then return "-0.5zw" end
  if bc == 6 or bc == 7 then
    if ac == 2 or ac == 6 or ac == 7 then return "-0.25zw" end
    return 0
  end
  if (bc == 9 or bc == 10 or bc == 11 or bc == 15 or bc == 16 or bc == 19) and
    (ac == 21 or ac == 24 or ac == 25) then
    return "0.25zw"
  end

  return 0
end

local function stretchability (before, after)
  local bc = jisClass(before)
  local ac = jisClass(after)
  -- somewhat simplified from table 6 of jlreq
  if ac == 1 then return "0.25zw" end
  if ac < 8 then return 0 end
  if bc == 4 and (ac == 21 or ac==24 or ac==25 or ac==27) then
    return "0.5zw"
  end
  if bc < 8 then return 0 end
  return "0.25zw" -- somewhat simplified
end

local function shrinkability (before, after)
  local bc = jisClass(before)
  local ac = jisClass(after)
  -- This rule is not in jlreq but it stops situations like 1：2 getting munched
  if (bc == 5 and ac == 27) or (bc == 27 and ac == 5) then return 0 end
  -- somewhat simplified from table 5 of jlreq
  if ac == 1 then return "0.5zw" end
  if ac == 5 then return "0.25zw" end
  if bc == 5 then return "0.25zw" end
  if bc == 7 then return "0.5zw" end
  return 0
end

-- local okbreak = SILE.nodefactory.penalty(0)

SILE.nodeMakers.ja = pl.class(SILE.nodeMakers.base)

function SILE.nodeMakers.ja:iterator (items)
  local options = self.options
  return coroutine.wrap(function ()
    local db
    local lastcp = -1
    local lastchar = ""
    local space = "%s" -- XXX
    for i = 1, #items do
      local item = items[i]
      local uchar = items[i].text
      local thiscp = SU.codepoint(uchar)
      db = lastchar.. "|" .. uchar
      if string.match(uchar, space) then
        db = db .. " S"
        coroutine.yield(SILE.shaper:makeSpaceNode(options, item))
      else
        local length = SILE.length(
          intercharacterspace(lastcp, thiscp),
          stretchability(lastcp, thiscp),
          shrinkability(lastcp, thiscp)
        )
        if breakAllowed(lastcp, thiscp) then
          db = db .." G ".. tostring(length)
          coroutine.yield(SILE.nodefactory.glue(length))
        elseif length.length ~= 0 or length.stretch ~= 0 or length.shrink ~= 0 then
          db = db .." K ".. tostring(length)
          coroutine.yield(SILE.nodefactory.kern(length))
        else db = db .. " N"
        end
        if jisClass(thiscp) == 5 or jisClass(thiscp) == 6 then
          local node = SILE.shaper:formNnode({ item }, uchar, options)
          node.hangable = true
          coroutine.yield(node)
        else
          coroutine.yield(SILE.shaper:formNnode({ item }, uchar, options))
        end
      end
      lastcp =thiscp
      lastchar = uchar
      SU.debug("ja", db)
    end
  end)
end

return {
  init = function ()

    SILE.hyphenator.languages.ja = { patterns={} }

    SILE.registerCommand("book:chapter:post:ja", function (_, _)
      SILE.call("medskip")
    end, nil, nil, true)

  end
}

1	-- "jlreq" refers to http://www.w3.org/TR/jlreq/
2	-- "JIS" refers to JIS X 4051
3
4	local hiragana = function (c) return c > 0x3040 and c <= 0x309f end	×
5	local katakana = function (c) return c > 0x30a0 and c <= 0x30ff end	×
6	local kanji = function (c) return c >= 0x4e00 and c <= 0x9fcc end	×
7
8	local classes = { -- from jlreq	×
9	[0x2018] = 1, [0x201C] = 1, [0x0028] = 1, [0x3014] = 1, [0x005B] = 1,
10	[0x007B] = 1, [0x3008] = 1, [0x300A] = 1, [0x300C] = 1, [0x300E] = 1,
11	[0x3010] = 1, [0x2985] = 1, [0x3018] = 1, [0x3016] = 1, [0x00AB] = 1, [0x301D] = 1,
12	[0xFF08] = 1, -- see note 1 of appendix A
13	[0x2019] = 2, [0x201D] = 2, [0x0029] = 2, [0x3015] = 2, [0x005D] = 2,
14	[0x007D] = 2, [0x3009] = 2, [0x300B] = 2, [0x300D] = 2, [0x300F] = 2,
15	[0x3011] = 2, [0x2986] = 2, [0x3019] = 2, [0x3017] = 2, [0x00BB] = 2, [0x301F] = 2,
16	[0xFF09] = 2, -- see note 1 of appendix A
17	[0x2010] = 3, [0x301C] = 3, [0x30A0] = 3, [0x2013] = 3,
18	[0x0021] = 4, [0x003F] = 4, [0x203C] = 4, [0x2047] = 4, [0x2048] = 4, [0x2049] = 4,
19	[0x30FB] = 5, [0x003A] = 5, [0x003B] = 5,
20	[0x3002] = 6, [0x002E] = 6,
21	[0x3001] = 7, [0x002C] = 7,
22	[0x2014] = 8, [0x2026] = 8, [0x2025] = 8, [0x3033] = 8, [0x3034] = 8, [0x3035] = 8,
23	[0x30FD] = 9, [0x30FE] = 9, [0x309D] = 9, [0x309E] = 9, [0x3005] = 9, [0x303B] = 9,
24	[0x30FC] = 10,
25	[0x3041] = 11, [0x3043] = 11, [0x3045] = 11, [0x3047] = 11,
26	[0x3049] = 11, [0x30A1] = 11, [0x30A3] = 11, [0x30A5] = 11,
27	[0x30A7] = 11, [0x30A9] = 11, [0x3063] = 11, [0x3083] = 11,
28	[0x3085] = 11, [0x3087] = 11, [0x308E] = 11, [0x3095] = 11,
29	[0x3096] = 11, [0x30C3] = 11, [0x30E3] = 11, [0x30E5] = 11,
30	[0x30E7] = 11, [0x30EE] = 11, [0x30F5] = 11, [0x30F6] = 11,
31	[0x31F0] = 11, [0x31F1] = 11, [0x31F2] = 11, [0x31F3] = 11,
32	[0x31F4] = 11, [0x31F5] = 11, [0x31F6] = 11, [0x31F7] = 11,
33	[0x31F8] = 11, [0x31F9] = 11, [0x31FA] = 11, [0x31FB] = 11,
34	[0x31FC] = 11, [0x31FD] = 11, [0x31FE] = 11, [0x31FF] = 11,
35	[0x00A5] = 12, [0x0024] = 12, [0x00A3] = 12, [0x0023] = 12, [0x20AC] = 12, [0x2116] = 12,
36	[0x00B0] = 13, [0x2032] = 13, [0x2033] = 13, [0x2103] = 13, [0x00A2] = 13,
37	[0x0025] = 13, [0x2030] = 13, [0x33CB] = 13, [0x2113] = 13, [0x3303] = 13,
38	[0x330D] = 13, [0x3314] = 13, [0x3318] = 13, [0x3322] = 13, [0x3323] = 13,
39	[0x3326] = 13, [0x3327] = 13, [0x332B] = 13, [0x3336] = 13, [0x333B] = 13,
40	[0x3349] = 13, [0x334A] = 13, [0x334D] = 13, [0x3351] = 13, [0x3357] = 13,
41	[0x338E] = 13, [0x338F] = 13, [0x339C] = 13, [0x339D] = 13, [0x339E] = 13,
42	[0x33A1] = 13, [0x33C4] = 13,
43	[0x3000] = 14,
44	}
45
46	local jisClass = function (c)
47	if c == -1 then return -1 end	×
48	if classes[c] then return classes[c] end	×
49	if hiragana(c) then return 15 end	×
50	if katakana(c) then return 16 end	×
51	if kanji(c) then return 19 end -- Other things are also 19	×
52	return 27 -- this is lazy	×
53	end
54
55	-- This roughly implements the kinsoku shori given in Appendix C of jlreq
56	local badBeforeClasses = { [1] = true, [12] = true, [28] = true }	×
57	local badAfterClasses = { }	×
58	for _, v in ipairs({ 2, 3, 4, 5, 6, 7, 9, 10, 11, 20, 29 }) do badAfterClasses[v] = true end	×
59
60	local function breakAllowed (before, after)
61	local bc = jisClass(before)	×
62	local ac = jisClass(after)	×
63	if badBeforeClasses[bc] then return false end	×
64	if badAfterClasses[ac] then return false end	×
65	if bc == 8 and ac == 8 then	×
66	if before == after then return false else return true end	×
67	end
68	-- The following rule is not in jlreq but it stops the algorithm from breaking
69	-- up Latin words!
70	if bc == 27 and ac == 27 then return false end	×
71	return true	×
72	end
73
74	local function intercharacterspace (before, after)
75	local bc = jisClass(before)	×
76	local ac = jisClass(after)	×
77	-- This rule is not in jlreq but it stops situations like 1：2 getting munched
78	if (bc == 5 and ac == 27) or (bc == 27 and ac == 5) then return 0 end	×
79
80	-- jlreq expects yakumono to contain no following space, so recommends half
81	-- a space afterwards. However, in real world fonts, yakumono occupies a full
82	-- zenkaku width with trailing space built into the glyph. So we do not add
83	-- space there. Instead, where no space is stipulated, we add negative space
84	-- to counteract the trailing space in the glyph.
85	if ac == 5 then return "-0.25zw" end	×
86	if bc == 5 then return "-0.25zw" end	×
87	if bc == 7 and ac == 1 then return "-0.5zw" end	×
88	if bc == 2 and ac == 1 then return "-0.5zw" end	×
89	if bc == -1 and ac == 1 then return "-0.5zw" end	×
90	if bc == 6 or bc == 7 then	×
91	if ac == 2 or ac == 6 or ac == 7 then return "-0.25zw" end	×
92	return 0	×
93	end
94	if (bc == 9 or bc == 10 or bc == 11 or bc == 15 or bc == 16 or bc == 19) and	×
95	(ac == 21 or ac == 24 or ac == 25) then	×
96	return "0.25zw"	×
97	end
98
99	return 0	×
100	end
101
102	local function stretchability (before, after)
103	local bc = jisClass(before)	×
104	local ac = jisClass(after)	×
105	-- somewhat simplified from table 6 of jlreq
106	if ac == 1 then return "0.25zw" end	×
107	if ac < 8 then return 0 end	×
108	if bc == 4 and (ac == 21 or ac==24 or ac==25 or ac==27) then	×
109	return "0.5zw"	×
110	end
111	if bc < 8 then return 0 end	×
112	return "0.25zw" -- somewhat simplified	×
113	end
114
115	local function shrinkability (before, after)
116	local bc = jisClass(before)	×
117	local ac = jisClass(after)	×
118	-- This rule is not in jlreq but it stops situations like 1：2 getting munched
119	if (bc == 5 and ac == 27) or (bc == 27 and ac == 5) then return 0 end	×
120	-- somewhat simplified from table 5 of jlreq
121	if ac == 1 then return "0.5zw" end	×
122	if ac == 5 then return "0.25zw" end	×
123	if bc == 5 then return "0.25zw" end	×
124	if bc == 7 then return "0.5zw" end	×
125	return 0	×
126	end
127
128	-- local okbreak = SILE.nodefactory.penalty(0)
129
130	SILE.nodeMakers.ja = pl.class(SILE.nodeMakers.base)	×
131
132	function SILE.nodeMakers.ja:iterator (items)	×
133	local options = self.options	×
134	return coroutine.wrap(function ()	×
135	local db
136	local lastcp = -1	×
137	local lastchar = ""	×
138	local space = "%s" -- XXX	×
139	for i = 1, #items do	×
140	local item = items[i]	×
141	local uchar = items[i].text	×
142	local thiscp = SU.codepoint(uchar)	×
143	db = lastchar.. "\|" .. uchar	×
144	if string.match(uchar, space) then	×
145	db = db .. " S"	×
146	coroutine.yield(SILE.shaper:makeSpaceNode(options, item))	×
147	else
148	local length = SILE.length(	×
149	intercharacterspace(lastcp, thiscp),	×
150	stretchability(lastcp, thiscp),	×
151	shrinkability(lastcp, thiscp)	×
152	)
153	if breakAllowed(lastcp, thiscp) then	×
154	db = db .." G ".. tostring(length)	×
155	coroutine.yield(SILE.nodefactory.glue(length))	×
156	elseif length.length ~= 0 or length.stretch ~= 0 or length.shrink ~= 0 then	×
157	db = db .." K ".. tostring(length)	×
158	coroutine.yield(SILE.nodefactory.kern(length))	×
159	else db = db .. " N"	×
160	end
161	if jisClass(thiscp) == 5 or jisClass(thiscp) == 6 then	×
162	local node = SILE.shaper:formNnode({ item }, uchar, options)	×
163	node.hangable = true	×
164	coroutine.yield(node)	×
165	else
166	coroutine.yield(SILE.shaper:formNnode({ item }, uchar, options))	×
167	end
168	end
169	lastcp =thiscp	×
170	lastchar = uchar	×
171	SU.debug("ja", db)	×
172	end
173	end)
174	end
175
176	return {	×
177	init = function ()
178
179	SILE.hyphenator.languages.ja = { patterns={} }	×
180
181	SILE.registerCommand("book:chapter:post:ja", function (_, _)	×
182	SILE.call("medskip")	×
183	end, nil, nil, true)	×
184
185	end
186	}

sile-typesetter / sile / 6713098919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous