moonbitlang / x / 374

Committed 27 Feb 2025 08:48AM UTC coverage: 89.592% (+1.6%) from 88.028%

Build # 374

Build Type

push

github

Committed by

Yoorkin

Commit Message

docs(encoding): init README

Run Details

1274 of 1422 relevant lines covered (89.59%)

434.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.03

/encoding/decoding.mbt

// Copyright 2024 International Digital Economy Academy
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

///| The Unicode Replacement Character, which is used to replace invalid or unrecognized sequences during lossy decoding.
/// https://unicode.org/charts/nameslist/n_FFF0.html
pub const U_REP = '\u{FFFD}'

///|
let utf_8_len = [
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4,
  4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
]

///|
/// Create and return a `Decoder` for the specified character encoding.
///
/// The `Decoder` consumes byte sequences and decodes them into the original string format.
///
/// # Parameters
///
/// - `encoding`: The character encoding format to be used for decoding the input byte sequences.
///
/// # Returns
///
/// A `Decoder` instance that can be used to decode byte sequences into strings.
///
/// # Examples
///
/// ```moonbit
/// let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
/// let decoder = decoder(UTF8)
/// inspect!(decoder.consume!(inputs[0]), content="abc")
/// inspect!(decoder.consume!(inputs[1]), content="")
/// inspect!(decoder.consume!(inputs[2]), content="🐰")
/// assert_true!(decoder.finish!().is_empty())
pub fn decoder(encoding : Encoding) -> Decoder {
  let i = FixedArray::default()
  let i_pos = 0
  let t = FixedArray::make(4, Byte::default())
  let t_len = 0
  let t_need = 0
  let k = match encoding {
    UTF8 => decode_utf_8
    UTF16 => decode_utf_16le
    UTF16LE => decode_utf_16le
    UTF16BE => decode_utf_16be
  }
  { i, i_pos, t, t_len, t_need, k }
}

///|
/// Decode the given byte sequence using the specified `Decoder` and return the resulting string.
///
/// This function can work in streaming mode where bytes are consumed incrementally.
/// When `stream` is `false`, it indicates the end of the input and triggers the final decoding step.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to decode the byte sequence.
/// - `input`: The byte sequence to be decoded.
/// - `stream~`: A boolean indicating whether more bytes will be supplied for decoding. It defaults to `false`.
///
/// # Returns
///
/// A `String` representing the decoded content from the input byte sequence.
///
/// # Errors
///
/// `MalformedError`: when the byte sequence is not properly formatted according to the specified encoding.
/// `TruncatedError`: when the byte sequence ends prematurely, implying that more data is expected for complete decoding.
///
/// # Examples
///
/// ```moonbit
/// let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
/// let decoder = @encoding.decoder(UTF8)
/// inspect!(decoder.decode!(inputs[0], stream=true), content="abc")
/// inspect!(decoder.decode!(inputs[1], stream=true), content="")
/// inspect!(decoder.decode!(inputs[2], stream=false), content="🐰")
/// ```
pub fn decode!(self : Decoder, input : Bytes, stream~ : Bool = false) -> String {
  if input.length() > 0 {
    self.i_cont(input)
  }
  if self.i_rem() == 0 {
    return String::default()
  }

  // drive decoder to decode
  let chars = []
  loop self.decode_() {
    Uchar(u) => {
      chars.push(u)
      continue self.decode_()
    }
    Malformed(bs) =>
      if stream && self.t_need > 0 {
        String::from_array(chars)
      } else {
        raise MalformedError(bs)
      }
    End => String::from_array(chars)
    Refill(t) =>
      if stream {
        String::from_array(chars)
      } else {
        raise TruncatedError(t)
      }
  }
}

///|
/// Consume the given byte sequence using the specified `Decoder` and return the resulting string incrementally.
///
/// This function calls `decode!` with the `stream` parameter set to `true`, indicating that more bytes will follow for decoding.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to consume the byte sequence.
/// - `input`: The byte sequence to be consumed and decoded incrementally.
///
/// # Returns
///
/// A `String` representing the partially decoded content from the input byte sequence, as more bytes are expected.
///
/// # Errors
///
/// `MalformedError`: when the byte sequence is not properly formatted according to the specified encoding.
/// `TruncatedError`: when the byte sequence ends prematurely, implying that more data is expected for complete decoding.
pub fn consume!(self : Decoder, input : Bytes) -> String {
  self.decode!(input, stream=true)
}

///|
/// Finalize the decoding process and return the remaining decoded string.
///
/// This function calls `decode!` with the `stream` parameter set to `false`, indicating that no more bytes will be supplied
/// and triggering the final decoding step to produce the remaining output.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to finalize the decoding process.
///
/// # Returns
///
/// A `String` representing the final part of the decoded content, after all byte sequences have been processed.
///
/// # Errors
///
/// `MalformedError`: This error is raised if the remaining byte sequence is not properly formatted according to the specified encoding.
/// `TruncatedError`: This error is raised if the remaining byte sequence ends prematurely, implying that more data was expected for complete decoding.
pub fn finish!(self : Decoder) -> String {
  self.decode!(b"", stream=false)
}

///|
/// Decode the given byte sequence using the specified `Decoder` and return the resulting string, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
///
/// This function can work in streaming mode where bytes are consumed incrementally.
/// When `stream` is `false`, it indicates the end of the input and triggers the final decoding step.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to decode the byte sequence.
/// - `input`: The byte sequence to be decoded.
/// - `stream~`: A boolean indicating whether more bytes will be supplied for decoding. It defaults to `false`.
///
/// # Returns
///
/// A `String` representing the decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`).
pub fn decode_lossy(
  self : Decoder,
  input : Bytes,
  stream~ : Bool = false
) -> String {
  if input.length() > 0 {
    self.i_cont(input)
  }
  if self.i_rem() == 0 {
    return String::default()
  }

  // drive decoder to decode
  let chars = []
  loop self.decode_() {
    Uchar(u) => {
      chars.push(u)
      continue self.decode_()
    }
    Malformed(_) =>
      if stream && self.t_need > 0 {
        String::from_array(chars)
      } else {
        chars.push(U_REP)
        continue self.decode_()
      }
    End => String::from_array(chars)
    Refill(_) =>
      if stream {
        String::from_array(chars)
      } else {
        continue self.decode_()
      }
  }
}

///|
/// Consume the given byte sequence using the specified `Decoder` and return the resulting string incrementally, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
///
/// This function calls `decode_lossy` with the `stream` parameter set to `true`, indicating that more bytes will follow for decoding.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to consume and decode the byte sequence.
/// - `input`: The byte sequence to be consumed and decoded incrementally.
///
/// # Returns
///
/// A `String` representing the partially decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`), as more bytes are expected.
pub fn lossy_consume(self : Decoder, input : Bytes) -> String {
  self.decode_lossy(input, stream=true)
}

///|
/// Finalize the lossy decoding process and return the remaining decoded string, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
///
/// This function calls `decode_lossy` with the `stream` parameter set to `false`, indicating that no more bytes will be supplied
/// and triggering the final decoding step to produce the remaining output.
///
/// # Parameters
///
/// - `self`: The `Decoder` instance used to finalize the lossy decoding process.
///
/// # Returns
///
/// A `String` representing the final part of the decoded content, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`), after all byte sequences have been processed.
pub fn lossy_finish(self : Decoder) -> String {
  self.decode_lossy(b"", stream=false)
}

///|
fn i_cont(self : Decoder, input : Bytes) -> Unit {
  // concat `input` to `i`, drop decoded `i`
  let i_rem = @math.maximum(self.i_rem(), 0)
  let new_len = i_rem + input.length()
  // init a new `i`
  let new_i = FixedArray::make(new_len, Byte::default())
  if i_rem > 0 {
    // copy the remainder of the old `i` into the new `i`
    self.i.blit_to(new_i, len=i_rem, src_offset=self.i_pos)
  }
  // copy all `input` into new `i`, starting at the remainder of the old `i`
  new_i.blit_from_bytes(i_rem, input, 0, input.length())
  self.i = new_i
  // reset position to starting position
  self.i_pos = 0
}

// Implementations

///|
fn decode_(self : Decoder) -> Decode {
  (self.k)(self)
}

///|
fn ret(self : Decoder, k : Cont, v : Decode) -> Decode {
  self.k = k
  v
}

///|
fn i_rem(self : Decoder) -> Int {
  self.i.length() - self.i_pos
}

///|
fn t_need(self : Decoder, need : Int) -> Unit {
  self.t_len = 0
  self.t_need = need
}

///|
fn eoi(self : Decoder) -> Unit {
  self.i = FixedArray::default()
}

///|
fn refill(self : Decoder, k : Cont) -> Decode {
  self.eoi()
  self.ret(k, Decode::Refill(Bytes::from_fixedarray(self.t)))
}

///|
fn t_fill(k : Cont, decoder : Decoder) -> Decode {
  fn blit(decoder : Decoder, l : Int) -> Unit {
    decoder.i.blit_to(
      decoder.t,
      len=l,
      dst_offset=decoder.t_len,
      src_offset=decoder.i_pos,
    )
    decoder.i_pos += l
    decoder.t_len += l
  }

  let rem = decoder.i_rem()
  if rem < 0 { // eoi
    k(decoder)
  } else {
    let need = decoder.t_need - decoder.t_len
    if rem < need {
      blit(decoder, rem)
      decoder.refill(@tuple.curry(t_fill)(k))
    } else {
      blit(decoder, need)
      k(decoder)
    }
  }
}

// UTF8

///|
fn decode_utf_8(self : Decoder) -> Decode {
  let rem = self.i_rem()
  if rem <= 0 {
    Decode::End
  } else {
    let idx = self.i[self.i_pos].to_int()
    let need = utf_8_len[idx]
    if rem < need {
      self.t_need(need)
      t_fill(t_decode_utf_8, self)
    } else {
      let j = self.i_pos
      if need == 0 {
        self.i_pos += 1
        self.ret(decode_utf_8, malformed(self.i, j, 1))
      } else {
        self.i_pos += need
        self.ret(decode_utf_8, r_utf_8(self.i, j, need))
      }
    }
  }
}

///|
fn t_decode_utf_8(self : Decoder) -> Decode {
  if self.t_len < self.t_need {
    self.ret(decode_utf_8, malformed(self.t, 0, self.t_len))
  } else {
    self.ret(decode_utf_8, r_utf_8(self.t, 0, self.t_len))
  }
}

///|
fn r_utf_8(bytes : FixedArray[Byte], offset : Int, length : Int) -> Decode {
  fn uchar(c : Int) {
    Uchar(Char::from_int(c))
  }

  match length {
    1 => uchar(bytes[offset].to_int())
    2 => {
      let b0 = bytes[offset].to_int()
      let b1 = bytes[offset + 1].to_int()
      if (b1 >> 6) != 0b10 {
        malformed(bytes, offset, length)
      } else {
        uchar(((b0 & 0x1F) << 6) | (b1 & 0x3F))
      }
    }
    3 => {
      let b0 = bytes[offset].to_int()
      let b1 = bytes[offset + 1].to_int()
      let b2 = bytes[offset + 2].to_int()
      let c = ((b0 & 0x0F) << 12) | (((b1 & 0x3F) << 6) | (b2 & 0x3F))
      if (b2 >> 6) != 0b10 {
        malformed(bytes, offset, length)
      } else {
        match b0 {
          0xE0 =>
            if b1 < 0xA0 || 0xBF < b1 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
          0xED =>
            if b1 < 0x80 || 0x9F < b1 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
          _ =>
            if (b1 >> 6) != 0b10 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
        }
      }
    }
    4 => {
      let b0 = bytes[offset].to_int()
      let b1 = bytes[offset + 1].to_int()
      let b2 = bytes[offset + 2].to_int()
      let b3 = bytes[offset + 3].to_int()
      let c = ((b0 & 0x07) << 18) |
        ((b1 & 0x3F) << 12) |
        ((b2 & 0x3F) << 6) |
        (b3 & 0x3F)
      if (b3 >> 6) != 0b10 || (b2 >> 6) != 0b10 {
        malformed(bytes, offset, length)
      } else {
        match b0 {
          0xF0 =>
            if b1 < 0x90 || 0xBF < b1 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
          0xF4 =>
            if b1 < 0x80 || 0x8F < b1 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
          _ =>
            if (b1 >> 6) != 0b10 {
              malformed(bytes, offset, length)
            } else {
              uchar(c)
            }
        }
      }
    }
    _ => panic()
  }
}

// UTF16LE

///|
priv enum UTF16Decode {
  Hi(Int)
  UTF16Malformed(Bytes)
  UTF16Uchar(Char)
}

///|
fn decode_utf_16le(self : Decoder) -> Decode {
  let rem = self.i_rem()
  if rem <= 0 {
    Decode::End
  } else if rem < 2 {
    self.t_need(2)
    t_fill(t_decode_utf_16le, self)
  } else {
    let j = self.i_pos
    self.i_pos += 2
    self.decode_utf_16le_lo(r_utf_16(self.i, j + 1, j))
  }
}

///|
fn t_decode_utf_16le(self : Decoder) -> Decode {
  if self.t_len < self.t_need {
    self.ret(decode_utf_16le, malformed(self.t, 0, self.t_len))
  } else {
    self.decode_utf_16le_lo(r_utf_16(self.t, 1, 0))
  }
}

///|
fn decode_utf_16le_lo(self : Decoder, v : UTF16Decode) -> Decode {
  match v {
    UTF16Uchar(u) => self.ret(decode_utf_16le, Uchar(u))
    UTF16Malformed(s) => self.ret(decode_utf_16le, Malformed(s))
    Hi(hi) => {
      let rem = self.i_rem()
      if rem < 2 {
        self.t_need(2)
        t_fill(@tuple.curry(t_decode_utf_16le_lo)(hi), self)
      } else {
        let j = self.i_pos
        let dcd = r_utf_16_lo(hi, self.i, j + 1, j)
        match dcd {
          Uchar(_) => self.i_pos += 2
          _ => ()
        }
        self.ret(decode_utf_16le, dcd)
      }
    }
  }
}

///|
fn t_decode_utf_16le_lo(hi : Int, decoder : Decoder) -> Decode {
  if decoder.t_len < decoder.t_need {
    decoder.ret(
      decode_utf_16le,
      malformed_pair(false, hi, decoder.t, 0, decoder.t_len),
    )
  } else {
    decoder.ret(decode_utf_16le, r_utf_16_lo(hi, decoder.t, 1, 0))
  }
}

///|
fn r_utf_16_lo(
  hi : Int,
  bytes : FixedArray[Byte],
  offset0 : Int,
  offset1 : Int
) -> Decode {
  let b0 = bytes[offset0].to_int()
  let b1 = bytes[offset1].to_int()
  let lo = (b0 << 8) | b1
  if lo < 0xDC00 || lo > 0xDFFF {
    // NOTE(jinser): only hi malformed, skip lo if lo is illegal
    //
    // For example, b"\xD8\x00\x00\x48" (BE)
    // Since \xD8\x00 is *legal* hi, here will try to parse lo next,
    // however the whole \xD8\x00\x00\x48 is *illegal* so the result will be a `Malformed[b"\xD8\x00\x00\x48"]`
    //
    // But \x00\x48 itself is a *legal* UTF16 code point with a value of `H`,
    // the ideal result should be: `[Malformed(b"\xD8\x00"), Uchar('H')]`
    //
    // > printf '\xD8\x00\x00\x48' | uconv --from-code UTF16BE --to-code UTF8 --from-callback substitute
    // �H
    Malformed([bytes[offset0], bytes[offset1]])
  } else {
    Uchar(Char::from_int(((hi & 0x3FF) << 10) | ((lo & 0x3FF) + 0x10000)))
  }
}

///|
fn r_utf_16(
  bytes : FixedArray[Byte],
  offset0 : Int,
  offset1 : Int
) -> UTF16Decode {
  let b0 = bytes[offset0].to_int()
  let b1 = bytes[offset1].to_int()
  let u = (b0 << 8) | b1
  if u < 0xD800 || u > 0xDFFF {
    UTF16Uchar(Char::from_int(u))
  } else if u > 0xDBFF {
    UTF16Malformed(slice(bytes, @math.minimum(offset0, offset1), 2))
  } else {
    Hi(u)
  }
}

// UTF16BE

///|
fn decode_utf_16be(self : Decoder) -> Decode {
  let rem = self.i_rem()
  if rem <= 0 {
    Decode::End
  } else if rem < 2 {
    self.t_need(2)
    t_fill(t_decode_utf_16be, self)
  } else {
    let j = self.i_pos
    self.i_pos += 2
    self.decode_utf_16be_lo(r_utf_16(self.i, j, j + 1))
  }
}

///|
fn t_decode_utf_16be(self : Decoder) -> Decode {
  if self.t_len < self.t_need {
    self.ret(decode_utf_16be, malformed(self.t, 0, self.t_len))
  } else {
    self.decode_utf_16be_lo(r_utf_16(self.t, 0, 1))
  }
}

///|
fn decode_utf_16be_lo(self : Decoder, decode : UTF16Decode) -> Decode {
  match decode {
    UTF16Uchar(x) => self.ret(decode_utf_16be, Uchar(x))
    UTF16Malformed(x) => self.ret(decode_utf_16be, Malformed(x))
    Hi(hi) => {
      let rem = self.i_rem()
      if rem < 2 {
        self.t_need(2)
        t_fill(@tuple.curry(t_decode_utf_16be_lo)(hi), self)
      } else {
        let j = self.i_pos
        let dcd = r_utf_16_lo(hi, self.i, j, j + 1)
        match dcd {
          Uchar(_) => self.i_pos += 2
          _ => ()
        }
        self.ret(decode_utf_16be, dcd)
      }
    }
  }
}

///|
fn t_decode_utf_16be_lo(hi : Int, self : Decoder) -> Decode {
  if self.t_len < self.t_need {
    self.ret(decode_utf_16be, malformed_pair(true, hi, self.t, 0, self.t_len))
  } else {
    self.ret(decode_utf_16be, r_utf_16_lo(hi, self.t, 0, 1))
  }
}

1	// Copyright 2024 International Digital Economy Academy
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	///\| The Unicode Replacement Character, which is used to replace invalid or unrecognized sequences during lossy decoding.
16	/// https://unicode.org/charts/nameslist/n_FFF0.html
17	pub const U_REP = '\u{FFFD}'
18
19	///\|
20	let utf_8_len = [
21	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
22	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
23	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
24	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
25	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,
26	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
27	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
28	0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
29	2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4,
30	4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
31	]
32
33	///\|
34	/// Create and return a `Decoder` for the specified character encoding.
35	///
36	/// The `Decoder` consumes byte sequences and decodes them into the original string format.
37	///
38	/// # Parameters
39	///
40	/// - `encoding`: The character encoding format to be used for decoding the input byte sequences.
41	///
42	/// # Returns
43	///
44	/// A `Decoder` instance that can be used to decode byte sequences into strings.
45	///
46	/// # Examples
47	///
48	/// ```moonbit
49	/// let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
50	/// let decoder = decoder(UTF8)
51	/// inspect!(decoder.consume!(inputs[0]), content="abc")
52	/// inspect!(decoder.consume!(inputs[1]), content="")
53	/// inspect!(decoder.consume!(inputs[2]), content="🐰")
54	/// assert_true!(decoder.finish!().is_empty())
55	pub fn decoder(encoding : Encoding) -> Decoder {
56	let i = FixedArray::default()	32✔
57	let i_pos = 0
58	let t = FixedArray::make(4, Byte::default())
59	let t_len = 0
60	let t_need = 0
61	let k = match encoding {
62	UTF8 => decode_utf_8	10✔
63	UTF16 => decode_utf_16le	2✔
64	UTF16LE => decode_utf_16le	11✔
65	UTF16BE => decode_utf_16be	9✔
66	}
67	{ i, i_pos, t, t_len, t_need, k }
68	}
69
70	///\|
71	/// Decode the given byte sequence using the specified `Decoder` and return the resulting string.
72	///
73	/// This function can work in streaming mode where bytes are consumed incrementally.
74	/// When `stream` is `false`, it indicates the end of the input and triggers the final decoding step.
75	///
76	/// # Parameters
77	///
78	/// - `self`: The `Decoder` instance used to decode the byte sequence.
79	/// - `input`: The byte sequence to be decoded.
80	/// - `stream~`: A boolean indicating whether more bytes will be supplied for decoding. It defaults to `false`.
81	///
82	/// # Returns
83	///
84	/// A `String` representing the decoded content from the input byte sequence.
85	///
86	/// # Errors
87	///
88	/// `MalformedError`: when the byte sequence is not properly formatted according to the specified encoding.
89	/// `TruncatedError`: when the byte sequence ends prematurely, implying that more data is expected for complete decoding.
90	///
91	/// # Examples
92	///
93	/// ```moonbit
94	/// let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
95	/// let decoder = @encoding.decoder(UTF8)
96	/// inspect!(decoder.decode!(inputs[0], stream=true), content="abc")
97	/// inspect!(decoder.decode!(inputs[1], stream=true), content="")
98	/// inspect!(decoder.decode!(inputs[2], stream=false), content="🐰")
99	/// ```
100	pub fn decode!(self : Decoder, input : Bytes, stream~ : Bool = false) -> String {
101	if input.length() > 0 {	76✔
102	self.i_cont(input)	70✔
103	}
104	if self.i_rem() == 0 {
105	return String::default()	6✔
106	}
107
108	// drive decoder to decode
109	let chars = []
110	loop self.decode_() {
111	Uchar(u) => {	278✔
112	chars.push(u)
113	continue self.decode_()
114	}
115	Malformed(bs) =>
116	if stream && self.t_need > 0 {	4✔
117	String::from_array(chars)	×
118	} else {
119	raise MalformedError(bs)	4✔
120	}
121	End => String::from_array(chars)	42✔
122	Refill(t) =>
123	if stream {	24✔
124	String::from_array(chars)	24✔
125	} else {
126	raise TruncatedError(t)	×
127	}
128	}
129	}
130
131	///\|
132	/// Consume the given byte sequence using the specified `Decoder` and return the resulting string incrementally.
133	///
134	/// This function calls `decode!` with the `stream` parameter set to `true`, indicating that more bytes will follow for decoding.
135	///
136	/// # Parameters
137	///
138	/// - `self`: The `Decoder` instance used to consume the byte sequence.
139	/// - `input`: The byte sequence to be consumed and decoded incrementally.
140	///
141	/// # Returns
142	///
143	/// A `String` representing the partially decoded content from the input byte sequence, as more bytes are expected.
144	///
145	/// # Errors
146	///
147	/// `MalformedError`: when the byte sequence is not properly formatted according to the specified encoding.
148	/// `TruncatedError`: when the byte sequence ends prematurely, implying that more data is expected for complete decoding.
149	pub fn consume!(self : Decoder, input : Bytes) -> String {
150	self.decode!(input, stream=true)	50✔
151	}
152
153	///\|
154	/// Finalize the decoding process and return the remaining decoded string.
155	///
156	/// This function calls `decode!` with the `stream` parameter set to `false`, indicating that no more bytes will be supplied
157	/// and triggering the final decoding step to produce the remaining output.
158	///
159	/// # Parameters
160	///
161	/// - `self`: The `Decoder` instance used to finalize the decoding process.
162	///
163	/// # Returns
164	///
165	/// A `String` representing the final part of the decoded content, after all byte sequences have been processed.
166	///
167	/// # Errors
168	///
169	/// `MalformedError`: This error is raised if the remaining byte sequence is not properly formatted according to the specified encoding.
170	/// `TruncatedError`: This error is raised if the remaining byte sequence ends prematurely, implying that more data was expected for complete decoding.
171	pub fn finish!(self : Decoder) -> String {
172	self.decode!(b"", stream=false)	6✔
173	}
174
175	///\|
176	/// Decode the given byte sequence using the specified `Decoder` and return the resulting string, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
177	///
178	/// This function can work in streaming mode where bytes are consumed incrementally.
179	/// When `stream` is `false`, it indicates the end of the input and triggers the final decoding step.
180	///
181	/// # Parameters
182	///
183	/// - `self`: The `Decoder` instance used to decode the byte sequence.
184	/// - `input`: The byte sequence to be decoded.
185	/// - `stream~`: A boolean indicating whether more bytes will be supplied for decoding. It defaults to `false`.
186	///
187	/// # Returns
188	///
189	/// A `String` representing the decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`).
190	pub fn decode_lossy(
191	self : Decoder,
192	input : Bytes,
193	stream~ : Bool = false
194	) -> String {
195	if input.length() > 0 {	6✔
196	self.i_cont(input)	6✔
197	}
198	if self.i_rem() == 0 {
199	return String::default()	×
200	}
201
202	// drive decoder to decode
203	let chars = []
204	loop self.decode_() {
205	Uchar(u) => {	6✔
206	chars.push(u)
207	continue self.decode_()
208	}
209	Malformed(_) =>
210	if stream && self.t_need > 0 {	9✔
211	String::from_array(chars)	×
212	} else {
213	chars.push(U_REP)	9✔
214	continue self.decode_()
215	}
216	End => String::from_array(chars)	6✔
217	Refill(_) =>
218	if stream {	3✔
219	String::from_array(chars)	×
220	} else {
221	continue self.decode_()	3✔
222	}
223	}
224	}
225
226	///\|
227	/// Consume the given byte sequence using the specified `Decoder` and return the resulting string incrementally, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
228	///
229	/// This function calls `decode_lossy` with the `stream` parameter set to `true`, indicating that more bytes will follow for decoding.
230	///
231	/// # Parameters
232	///
233	/// - `self`: The `Decoder` instance used to consume and decode the byte sequence.
234	/// - `input`: The byte sequence to be consumed and decoded incrementally.
235	///
236	/// # Returns
237	///
238	/// A `String` representing the partially decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`), as more bytes are expected.
239	pub fn lossy_consume(self : Decoder, input : Bytes) -> String {
240	self.decode_lossy(input, stream=true)	×
241	}
242
243	///\|
244	/// Finalize the lossy decoding process and return the remaining decoded string, replacing any invalid sequences with the Unicode Replacement Character (`U+FFFD`).
245	///
246	/// This function calls `decode_lossy` with the `stream` parameter set to `false`, indicating that no more bytes will be supplied
247	/// and triggering the final decoding step to produce the remaining output.
248	///
249	/// # Parameters
250	///
251	/// - `self`: The `Decoder` instance used to finalize the lossy decoding process.
252	///
253	/// # Returns
254	///
255	/// A `String` representing the final part of the decoded content, with any invalid sequences replaced by the Unicode Replacement Character (`U+FFFD`), after all byte sequences have been processed.
256	pub fn lossy_finish(self : Decoder) -> String {
257	self.decode_lossy(b"", stream=false)	×
258	}
259
260	///\|
261	fn i_cont(self : Decoder, input : Bytes) -> Unit {
262	// concat `input` to `i`, drop decoded `i`
263	let i_rem = @math.maximum(self.i_rem(), 0)	76✔
264	let new_len = i_rem + input.length()
265	// init a new `i`
266	let new_i = FixedArray::make(new_len, Byte::default())
267	if i_rem > 0 {
268	// copy the remainder of the old `i` into the new `i`
269	self.i.blit_to(new_i, len=i_rem, src_offset=self.i_pos)	×
270	}
271	// copy all `input` into new `i`, starting at the remainder of the old `i`
272	new_i.blit_from_bytes(i_rem, input, 0, input.length())
273	self.i = new_i
274	// reset position to starting position
275	self.i_pos = 0
276	}
277
278	// Implementations
279
280	///\|
281	fn decode_(self : Decoder) -> Decode {
282	(self.k)(self)	372✔
283	}
284
285	///\|
286	fn ret(self : Decoder, k : Cont, v : Decode) -> Decode {
287	self.k = k	324✔
288	v
289	}
290
291	///\|
292	fn i_rem(self : Decoder) -> Int {
293	self.i.length() - self.i_pos	573✔
294	}
295
296	///\|
297	fn t_need(self : Decoder, need : Int) -> Unit {
298	self.t_len = 0	27✔
299	self.t_need = need
300	}
301
302	///\|
303	fn eoi(self : Decoder) -> Unit {
304	self.i = FixedArray::default()	27✔
305	}
306
307	///\|
308	fn refill(self : Decoder, k : Cont) -> Decode {
309	self.eoi()	27✔
310	self.ret(k, Decode::Refill(Bytes::from_fixedarray(self.t)))
311	}
312
313	///\|
314	fn t_fill(k : Cont, decoder : Decoder) -> Decode {
315	fn blit(decoder : Decoder, l : Int) -> Unit {	54✔
316	decoder.i.blit_to(	51✔
317	decoder.t,
318	len=l,
319	dst_offset=decoder.t_len,
320	src_offset=decoder.i_pos,
321	)
322	decoder.i_pos += l
323	decoder.t_len += l
324	}
325
326	let rem = decoder.i_rem()
327	if rem < 0 { // eoi
328	k(decoder)	3✔
329	} else {
330	let need = decoder.t_need - decoder.t_len	51✔
331	if rem < need {
332	blit(decoder, rem)	27✔
333	decoder.refill(@tuple.curry(t_fill)(k))
334	} else {
335	blit(decoder, need)	24✔
336	k(decoder)
337	}
338	}
339	}
340
341	// UTF8
342
343	///\|
344	fn decode_utf_8(self : Decoder) -> Decode {
345	let rem = self.i_rem()	95✔
346	if rem <= 0 {
347	Decode::End	14✔
348	} else {
349	let idx = self.i[self.i_pos].to_int()	81✔
350	let need = utf_8_len[idx]
351	if rem < need {
352	self.t_need(need)	6✔
353	t_fill(t_decode_utf_8, self)
354	} else {
355	let j = self.i_pos	75✔
356	if need == 0 {
357	self.i_pos += 1	1✔
358	self.ret(decode_utf_8, malformed(self.i, j, 1))
359	} else {
360	self.i_pos += need	74✔
361	self.ret(decode_utf_8, r_utf_8(self.i, j, need))
362	}
363	}
364	}
365	}
366
367	///\|
368	fn t_decode_utf_8(self : Decoder) -> Decode {
369	if self.t_len < self.t_need {	6✔
370	self.ret(decode_utf_8, malformed(self.t, 0, self.t_len))	×
371	} else {
372	self.ret(decode_utf_8, r_utf_8(self.t, 0, self.t_len))	6✔
373	}
374	}
375
376	///\|
377	fn r_utf_8(bytes : FixedArray[Byte], offset : Int, length : Int) -> Decode {
378	fn uchar(c : Int) {	80✔
379	Uchar(Char::from_int(c))	74✔
380	}
381
382	match length {
383	1 => uchar(bytes[offset].to_int())	44✔
384	2 => {	6✔
385	let b0 = bytes[offset].to_int()
386	let b1 = bytes[offset + 1].to_int()
387	if (b1 >> 6) != 0b10 {
388	malformed(bytes, offset, length)	4✔
389	} else {
390	uchar(((b0 & 0x1F) << 6) \| (b1 & 0x3F))	2✔
391	}
392	}
393	3 => {	13✔
394	let b0 = bytes[offset].to_int()
395	let b1 = bytes[offset + 1].to_int()
396	let b2 = bytes[offset + 2].to_int()
397	let c = ((b0 & 0x0F) << 12) \| (((b1 & 0x3F) << 6) \| (b2 & 0x3F))
398	if (b2 >> 6) != 0b10 {
399	malformed(bytes, offset, length)	×
400	} else {
401	match b0 {	13✔
402	0xE0 =>
403	if b1 < 0xA0 \|\| 0xBF < b1 {	1✔
404	malformed(bytes, offset, length)	1✔
405	} else {
406	uchar(c)	×
407	}
408	0xED =>
409	if b1 < 0x80 \|\| 0x9F < b1 {	×
410	malformed(bytes, offset, length)	×
411	} else {
412	uchar(c)	×
413	}
414	_ =>
415	if (b1 >> 6) != 0b10 {	12✔
416	malformed(bytes, offset, length)	×
417	} else {
418	uchar(c)	12✔
419	}
420	}
421	}
422	}
423	4 => {	17✔
424	let b0 = bytes[offset].to_int()
425	let b1 = bytes[offset + 1].to_int()
426	let b2 = bytes[offset + 2].to_int()
427	let b3 = bytes[offset + 3].to_int()
428	let c = ((b0 & 0x07) << 18) \|
429	((b1 & 0x3F) << 12) \|
430	((b2 & 0x3F) << 6) \|
431	(b3 & 0x3F)
432	if (b3 >> 6) != 0b10 \|\| (b2 >> 6) != 0b10 {
433	malformed(bytes, offset, length)	1✔
434	} else {
435	match b0 {	16✔
436	0xF0 =>
437	if b1 < 0x90 \|\| 0xBF < b1 {	16✔
438	malformed(bytes, offset, length)	×
439	} else {
440	uchar(c)	16✔
441	}
442	0xF4 =>
443	if b1 < 0x80 \|\| 0x8F < b1 {	×
444	malformed(bytes, offset, length)	×
445	} else {
446	uchar(c)	×
447	}
448	_ =>
449	if (b1 >> 6) != 0b10 {	×
450	malformed(bytes, offset, length)	×
451	} else {
452	uchar(c)	×
453	}
454	}
455	}
456	}
457	_ => panic()	×
458	}
459	}
460
461	// UTF16LE
462
463	///\|
464	priv enum UTF16Decode {
465	Hi(Int)
466	UTF16Malformed(Bytes)
467	UTF16Uchar(Char)
468	}
469
470	///\|
471	fn decode_utf_16le(self : Decoder) -> Decode {
472	let rem = self.i_rem()	141✔
473	if rem <= 0 {
474	Decode::End	21✔
475	} else if rem < 2 {	120✔
476	self.t_need(2)	9✔
477	t_fill(t_decode_utf_16le, self)
478	} else {
479	let j = self.i_pos	111✔
480	self.i_pos += 2
481	self.decode_utf_16le_lo(r_utf_16(self.i, j + 1, j))
482	}
483	}
484
485	///\|
486	fn t_decode_utf_16le(self : Decoder) -> Decode {
487	if self.t_len < self.t_need {	9✔
488	self.ret(decode_utf_16le, malformed(self.t, 0, self.t_len))	×
489	} else {
490	self.decode_utf_16le_lo(r_utf_16(self.t, 1, 0))	9✔
491	}
492	}
493
494	///\|
495	fn decode_utf_16le_lo(self : Decoder, v : UTF16Decode) -> Decode {
496	match v {	120✔
497	UTF16Uchar(u) => self.ret(decode_utf_16le, Uchar(u))	111✔
498	UTF16Malformed(s) => self.ret(decode_utf_16le, Malformed(s))	×
499	Hi(hi) => {	9✔
500	let rem = self.i_rem()
501	if rem < 2 {
502	self.t_need(2)	4✔
503	t_fill(@tuple.curry(t_decode_utf_16le_lo)(hi), self)
504	} else {
505	let j = self.i_pos	5✔
506	let dcd = r_utf_16_lo(hi, self.i, j + 1, j)
507	match dcd {
508	Uchar(_) => self.i_pos += 2	4✔
509	_ => ()	1✔
510	}
511	self.ret(decode_utf_16le, dcd)
512	}
513	}
514	}
515	}
516
517	///\|
518	fn t_decode_utf_16le_lo(hi : Int, decoder : Decoder) -> Decode {
519	if decoder.t_len < decoder.t_need {	4✔
520	decoder.ret(	2✔
521	decode_utf_16le,
522	malformed_pair(false, hi, decoder.t, 0, decoder.t_len),
523	)
524	} else {
525	decoder.ret(decode_utf_16le, r_utf_16_lo(hi, decoder.t, 1, 0))	2✔
526	}
527	}
528
529	///\|
530	fn r_utf_16_lo(
531	hi : Int,
532	bytes : FixedArray[Byte],
533	offset0 : Int,
534	offset1 : Int
535	) -> Decode {
536	let b0 = bytes[offset0].to_int()	13✔
537	let b1 = bytes[offset1].to_int()
538	let lo = (b0 << 8) \| b1
539	if lo < 0xDC00 \|\| lo > 0xDFFF {
540	// NOTE(jinser): only hi malformed, skip lo if lo is illegal
541	//
542	// For example, b"\xD8\x00\x00\x48" (BE)
543	// Since \xD8\x00 is legal hi, here will try to parse lo next,
544	// however the whole \xD8\x00\x00\x48 is illegal so the result will be a `Malformed[b"\xD8\x00\x00\x48"]`
545	//
546	// But \x00\x48 itself is a legal UTF16 code point with a value of `H`,
547	// the ideal result should be: `[Malformed(b"\xD8\x00"), Uchar('H')]`
548	//
549	// > printf '\xD8\x00\x00\x48' \| uconv --from-code UTF16BE --to-code UTF8 --from-callback substitute
550	// �H
551	Malformed([bytes[offset0], bytes[offset1]])	3✔
552	} else {
553	Uchar(Char::from_int(((hi & 0x3FF) << 10) \| ((lo & 0x3FF) + 0x10000)))	10✔
554	}
555	}
556
557	///\|
558	fn r_utf_16(
559	bytes : FixedArray[Byte],
560	offset0 : Int,
561	offset1 : Int
562	) -> UTF16Decode {
563	let b0 = bytes[offset0].to_int()	216✔
564	let b1 = bytes[offset1].to_int()
565	let u = (b0 << 8) \| b1
566	if u < 0xD800 \|\| u > 0xDFFF {
567	UTF16Uchar(Char::from_int(u))	200✔
568	} else if u > 0xDBFF {	16✔
569	UTF16Malformed(slice(bytes, @math.minimum(offset0, offset1), 2))	×
570	} else {
571	Hi(u)	16✔
572	}
573	}
574
575	// UTF16BE
576
577	///\|
578	fn decode_utf_16be(self : Decoder) -> Decode {
579	let rem = self.i_rem()	109✔
580	if rem <= 0 {
581	Decode::End	13✔
582	} else if rem < 2 {	96✔
583	self.t_need(2)	7✔
584	t_fill(t_decode_utf_16be, self)
585	} else {
586	let j = self.i_pos	89✔
587	self.i_pos += 2
588	self.decode_utf_16be_lo(r_utf_16(self.i, j, j + 1))
589	}
590	}
591
592	///\|
593	fn t_decode_utf_16be(self : Decoder) -> Decode {
594	if self.t_len < self.t_need {	7✔
595	self.ret(decode_utf_16be, malformed(self.t, 0, self.t_len))	×
596	} else {
597	self.decode_utf_16be_lo(r_utf_16(self.t, 0, 1))	7✔
598	}
599	}
600
601	///\|
602	fn decode_utf_16be_lo(self : Decoder, decode : UTF16Decode) -> Decode {
603	match decode {	96✔
604	UTF16Uchar(x) => self.ret(decode_utf_16be, Uchar(x))	89✔
605	UTF16Malformed(x) => self.ret(decode_utf_16be, Malformed(x))	×
606	Hi(hi) => {	7✔
607	let rem = self.i_rem()
608	if rem < 2 {
609	self.t_need(2)	1✔
610	t_fill(@tuple.curry(t_decode_utf_16be_lo)(hi), self)
611	} else {
612	let j = self.i_pos	6✔
613	let dcd = r_utf_16_lo(hi, self.i, j, j + 1)
614	match dcd {
615	Uchar(_) => self.i_pos += 2	4✔
616	_ => ()	2✔
617	}
618	self.ret(decode_utf_16be, dcd)
619	}
620	}
621	}
622	}
623
624	///\|
625	fn t_decode_utf_16be_lo(hi : Int, self : Decoder) -> Decode {
626	if self.t_len < self.t_need {	1✔
627	self.ret(decode_utf_16be, malformed_pair(true, hi, self.t, 0, self.t_len))	1✔
628	} else {
629	self.ret(decode_utf_16be, r_utf_16_lo(hi, self.t, 0, 1))	×
630	}
631	}

moonbitlang / x / 374

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous