• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

saitoha / libsixel / 19211260254

09 Nov 2025 04:26PM UTC coverage: 46.579% (+0.3%) from 46.268%
19211260254

push

github

saitoha
palette: move palette generation into palette module

8397 of 26675 branches covered (31.48%)

294 of 414 new or added lines in 5 files covered. (71.01%)

3 existing lines in 1 file now uncovered.

11960 of 25677 relevant lines covered (46.58%)

1899906.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33
/src/quant.c
1
/*
2
 *
3
 * mediancut algorithm implementation is imported from pnmcolormap.c
4
 * in netpbm library.
5
 * http://netpbm.sourceforge.net/
6
 *
7
 * *******************************************************************************
8
 *                  original license block of pnmcolormap.c
9
 * *******************************************************************************
10
 *
11
 *   Derived from ppmquant, originally by Jef Poskanzer.
12
 *
13
 *   Copyright (C) 1989, 1991 by Jef Poskanzer.
14
 *   Copyright (C) 2001 by Bryan Henderson.
15
 *
16
 *   Permission to use, copy, modify, and distribute this software and its
17
 *   documentation for any purpose and without fee is hereby granted, provided
18
 *   that the above copyright notice appear in all copies and that both that
19
 *   copyright notice and this permission notice appear in supporting
20
 *   documentation.  This software is provided "as is" without express or
21
 *   implied warranty.
22
 *
23
 * ******************************************************************************
24
 *
25
 * Copyright (c) 2014-2018 Hayaki Saito
26
 *
27
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
28
 * this software and associated documentation files (the "Software"), to deal in
29
 * the Software without restriction, including without limitation the rights to
30
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
31
 * the Software, and to permit persons to whom the Software is furnished to do so,
32
 * subject to the following conditions:
33
 *
34
 * The above copyright notice and this permission notice shall be included in all
35
 * copies or substantial portions of the Software.
36
 *
37
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
38
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
39
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
40
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
41
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
42
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
43
 *
44
 *
45
 */
46

47
#include "config.h"
48

49
/* STDC_HEADERS */
50
#include <stdlib.h>
51
#include <stdio.h>
52
#include <time.h>
53

54
#if HAVE_STRING_H
55
# include <string.h>
56
#endif  /* HAVE_STRING_H */
57
#if HAVE_MATH_H
58
#include <math.h>
59
#endif  /* HAVE_MATH_H */
60
#if HAVE_LIMITS_H
61
# include <limits.h>
62
#endif  /* HAVE_MATH_H */
63
#if HAVE_STDINT_H
64
# include <stdint.h>
65
#else
66
# if HAVE_INTTYPES_H
67
#  include <inttypes.h>
68
# endif
69
#endif  /* HAVE_STDINT_H */
70

71
#if defined(SIXEL_USE_SSE2)
72
# include <emmintrin.h>
73
# include <xmmintrin.h>
74
#elif defined(SIXEL_USE_NEON)
75
# include <arm_neon.h>
76
#endif
77

78
#include "quant.h"
79
#include "palette.h"
80
#include "quant-internal.h"
81
#include "compat_stub.h"
82

83
typedef struct {
84
    uint8_t r;
85
    uint8_t g;
86
    uint8_t b;
87
} sixel_color_t;
88

89
static float env_final_merge_target_factor = 1.81;
90

91
static unsigned char const sixel_safe_tones[256] = {
92
    0,   0,   3,   3,   5,   5,   5,   8,   8,   10,  10,  10,  13,  13,  13,
93
    15,  15,  18,  18,  18,  20,  20,  23,  23,  23,  26,  26,  28,  28,  28,
94
    31,  31,  33,  33,  33,  36,  36,  38,  38,  38,  41,  41,  41,  43,  43,
95
    46,  46,  46,  48,  48,  51,  51,  51,  54,  54,  56,  56,  56,  59,  59,
96
    61,  61,  61,  64,  64,  64,  66,  66,  69,  69,  69,  71,  71,  74,  74,
97
    74,  77,  77,  79,  79,  79,  82,  82,  84,  84,  84,  87,  87,  89,  89,
98
    89,  92,  92,  92,  94,  94,  97,  97,  97,  99,  99,  102, 102, 102, 105,
99
    105, 107, 107, 107, 110, 110, 112, 112, 112, 115, 115, 115, 117, 117, 120,
100
    120, 120, 122, 122, 125, 125, 125, 128, 128, 130, 130, 130, 133, 133, 135,
101
    135, 135, 138, 138, 140, 140, 140, 143, 143, 143, 145, 145, 148, 148, 148,
102
    150, 150, 153, 153, 153, 156, 156, 158, 158, 158, 161, 161, 163, 163, 163,
103
    166, 166, 166, 168, 168, 171, 171, 171, 173, 173, 176, 176, 176, 179, 179,
104
    181, 181, 181, 184, 184, 186, 186, 186, 189, 189, 191, 191, 191, 194, 194,
105
    194, 196, 196, 199, 199, 199, 201, 201, 204, 204, 204, 207, 207, 209, 209,
106
    209, 212, 212, 214, 214, 214, 217, 217, 217, 219, 219, 222, 222, 222, 224,
107
    224, 227, 227, 227, 230, 230, 232, 232, 232, 235, 235, 237, 237, 237, 240,
108
    240, 242, 242, 242, 245, 245, 245, 247, 247, 250, 250, 250, 252, 252, 255,
109
    255
110
};
111

112

113
static float mask_a(int x, int y, int c);
114
static float mask_x(int x, int y, int c);
115
static void diffuse_none(unsigned char *data, int width, int height,
116
                         int x, int y, int depth, int error, int direction);
117
static void diffuse_fs(unsigned char *data, int width, int height,
118
                       int x, int y, int depth, int error, int direction);
119
static void diffuse_atkinson(unsigned char *data, int width, int height,
120
                             int x, int y, int depth, int error,
121
                             int direction);
122
static void diffuse_jajuni(unsigned char *data, int width, int height,
123
                           int x, int y, int depth, int error,
124
                           int direction);
125
static void diffuse_stucki(unsigned char *data, int width, int height,
126
                           int x, int y, int depth, int error,
127
                           int direction);
128
static void diffuse_burkes(unsigned char *data, int width, int height,
129
                           int x, int y, int depth, int error,
130
                           int direction);
131
static void diffuse_sierra1(unsigned char *data, int width, int height,
132
                            int x, int y, int depth, int error,
133
                            int direction);
134
static void diffuse_sierra2(unsigned char *data, int width, int height,
135
                            int x, int y, int depth, int error,
136
                            int direction);
137
static void diffuse_sierra3(unsigned char *data, int width, int height,
138
                            int x, int y, int depth, int error,
139
                            int direction);
140
static void diffuse_none_carry(int32_t *carry_curr, int32_t *carry_next,
141
                               int32_t *carry_far, int width, int height,
142
                               int depth, int x, int y, int32_t error,
143
                               int direction, int channel);
144
static void diffuse_fs_carry(int32_t *carry_curr, int32_t *carry_next,
145
                             int32_t *carry_far, int width, int height,
146
                             int depth, int x, int y, int32_t error,
147
                             int direction, int channel);
148
static void diffuse_atkinson_carry(int32_t *carry_curr, int32_t *carry_next,
149
                                   int32_t *carry_far, int width,
150
                                   int height, int depth, int x, int y,
151
                                   int32_t error, int direction,
152
                                   int channel);
153
static void diffuse_jajuni_carry(int32_t *carry_curr, int32_t *carry_next,
154
                                 int32_t *carry_far, int width, int height,
155
                                 int depth, int x, int y, int32_t error,
156
                                 int direction, int channel);
157
static void diffuse_stucki_carry(int32_t *carry_curr, int32_t *carry_next,
158
                                 int32_t *carry_far, int width, int height,
159
                                 int depth, int x, int y, int32_t error,
160
                                 int direction, int channel);
161
static void diffuse_burkes_carry(int32_t *carry_curr, int32_t *carry_next,
162
                                 int32_t *carry_far, int width, int height,
163
                                 int depth, int x, int y, int32_t error,
164
                                 int direction, int channel);
165
static void diffuse_sierra1_carry(int32_t *carry_curr, int32_t *carry_next,
166
                                  int32_t *carry_far, int width, int height,
167
                                  int depth, int x, int y, int32_t error,
168
                                  int direction, int channel);
169
static void diffuse_sierra2_carry(int32_t *carry_curr, int32_t *carry_next,
170
                                  int32_t *carry_far, int width, int height,
171
                                  int depth, int x, int y, int32_t error,
172
                                  int direction, int channel);
173
static void diffuse_sierra3_carry(int32_t *carry_curr, int32_t *carry_next,
174
                                  int32_t *carry_far, int width, int height,
175
                                  int depth, int x, int y, int32_t error,
176
                                  int direction, int channel);
177

178
typedef struct {
179
    int index;
180
    int left;
181
    int right;
182
    unsigned char axis;
183
} sixel_certlut_node_t;
184

185
typedef struct {
186
    uint32_t *level0;
187
    uint8_t *pool;
188
    uint32_t pool_size;
189
    uint32_t pool_capacity;
190
    int wR;
191
    int wG;
192
    int wB;
193
    uint64_t wR2;
194
    uint64_t wG2;
195
    uint64_t wB2;
196
    int32_t wr_scale[256];
197
    int32_t wg_scale[256];
198
    int32_t wb_scale[256];
199
    int32_t *wr_palette;
200
    int32_t *wg_palette;
201
    int32_t *wb_palette;
202
    sixel_color_t const *palette;
203
    int ncolors;
204
    sixel_certlut_node_t *kdnodes;
205
    int kdnodes_count;
206
    int kdtree_root;
207
} sixel_certlut_t;
208

209
#define SIXEL_CERTLUT_BRANCH_FLAG 0x40000000U
210
/* #define DEBUG_CERTLUT_TRACE 1 */
211

212
static void sixel_quant_cell_center(int rmin, int gmin, int bmin, int size,
213
                                    int *cr, int *cg, int *cb);
214
static void sixel_quant_weight_init(sixel_certlut_t *lut, int wR, int wG,
215
                                    int wB);
216
static uint64_t sixel_certlut_distance_precomputed(
217
    sixel_certlut_t const *lut,
218
    int index,
219
    int32_t wr_r,
220
    int32_t wg_g,
221
    int32_t wb_b);
222
static int sixel_certlut_palette_component(sixel_certlut_t const *lut,
223
                                           int index, int axis);
224
static void sixel_certlut_sort_indices(sixel_certlut_t const *lut,
225
                                       int *indices, int count, int axis);
226
static SIXELSTATUS sixel_certlut_kdtree_build(sixel_certlut_t *lut);
227
static int sixel_certlut_kdtree_build_recursive(sixel_certlut_t *lut,
228
                                                int *indices,
229
                                                int count,
230
                                                int depth);
231
static uint64_t sixel_certlut_axis_distance(sixel_certlut_t const *lut,
232
                                            int diff, int axis);
233
static void sixel_certlut_consider_candidate(sixel_certlut_t const *lut,
234
                                             int candidate,
235
                                             int32_t wr_r,
236
                                             int32_t wg_g,
237
                                             int32_t wb_b,
238
                                             int *best_idx,
239
                                             uint64_t *best_dist,
240
                                             int *second_idx,
241
                                             uint64_t *second_dist);
242
static void sixel_certlut_kdtree_search(sixel_certlut_t const *lut,
243
                                        int node_index,
244
                                        int r,
245
                                        int g,
246
                                        int b,
247
                                        int32_t wr_r,
248
                                        int32_t wg_g,
249
                                        int32_t wb_b,
250
                                        int *best_idx,
251
                                        uint64_t *best_dist,
252
                                        int *second_idx,
253
                                        uint64_t *second_dist);
254
static void sixel_quant_distance_pair(sixel_certlut_t const *lut, int r,
255
                                      int g, int b, int *best_idx,
256
                                      int *second_idx, uint64_t *best_dist,
257
                                      uint64_t *second_dist);
258
static int sixel_quant_is_cell_safe(sixel_certlut_t const *lut, int best_idx,
259
                                    int second_idx, int size,
260
                                    uint64_t best_dist,
261
                                    uint64_t second_dist);
262
static uint32_t sixel_quant_pool_alloc(sixel_certlut_t *lut, int *status);
263
static void sixel_quant_assign_leaf(uint32_t *cell, int palette_index);
264
static void sixel_quant_assign_branch(uint32_t *cell, uint32_t offset);
265
static uint8_t sixel_certlut_fallback(sixel_certlut_t const *lut,
266
                                      int r, int g, int b);
267
static int sixel_certlut_build_cell(sixel_certlut_t *lut, uint32_t *cell,
268
                                    int rmin, int gmin, int bmin, int size);
269
static int sixel_certlut_init(sixel_certlut_t *lut);
270
static void sixel_certlut_release(sixel_certlut_t *lut);
271
static int sixel_certlut_prepare_palette_terms(sixel_certlut_t *lut);
272

273
static sixel_certlut_t *certlut_context = NULL;
274

275
static const int (*
276
lso2_table(void))[7]
×
277
{
278
#include "lso2.h"
279
    return var_coefs;
280
}
281

282
#define VARERR_SCALE_SHIFT 12
283
#define VARERR_SCALE       (1 << VARERR_SCALE_SHIFT)
284
#define VARERR_ROUND       (1 << (VARERR_SCALE_SHIFT - 1))
285
#define VARERR_MAX_VALUE   (255 * VARERR_SCALE)
286

287
/*****************************************************************************
288
 *
289
 * quantization
290
 *
291
 *****************************************************************************/
292

293
static unsigned char
UNCOV
294
sixel_quant_reversible_value(unsigned int sample)
×
295
{
296
    if (sample > 255U) {
×
297
        sample = 255U;
×
298
    }
299

300
    return sixel_safe_tones[sample];
×
301
}
302

303
static void
304
sixel_quant_reversible_pixel(unsigned char const *src,
×
305
                             unsigned int depth,
306
                             unsigned char *dst)
307
{
308
    unsigned int plane;
309

310
    for (plane = 0U; plane < depth; ++plane) {
×
311
        dst[plane] = sixel_quant_reversible_value(src[plane]);
×
312
    }
313
}
×
314

315
static void
316
sixel_quant_reversible_tuple(sample *tuple,
×
317
                             unsigned int depth)
318
{
319
    unsigned int plane;
320
    unsigned int sample_value;
321

322
    for (plane = 0U; plane < depth; ++plane) {
×
323
        sample_value = (unsigned int)tuple[plane];
×
324
        tuple[plane] =
×
325
            (sample)sixel_quant_reversible_value(sample_value);
×
326
    }
327
}
×
328

329
void
330
sixel_quant_reversible_palette(unsigned char *palette,
×
331
                               unsigned int colors,
332
                               unsigned int depth)
333
{
334
    unsigned int index;
335
    unsigned int plane;
336
    unsigned int sample_value;
337
    size_t offset;
338

339
    for (index = 0U; index < colors; ++index) {
×
340
        for (plane = 0U; plane < depth; ++plane) {
×
341
            offset = (size_t)index * (size_t)depth + (size_t)plane;
×
342
            sample_value = (unsigned int)palette[offset];
×
343
            palette[offset] =
×
344
                sixel_quant_reversible_value(sample_value);
×
345
        }
346
    }
347
}
×
348

349
typedef struct box* boxVector;
350
struct box {
351
    unsigned int ind;
352
    unsigned int colors;
353
    unsigned int sum;
354
};
355

356
static unsigned int compareplanePlane;
357
static tupletable2 const *force_palette_source;
358
    /* This is a parameter to compareplane().  We use this global variable
359
       so that compareplane() can be called by qsort(), to compare two
360
       tuples.  qsort() doesn't pass any arguments except the two tuples.
361
    */
362
static int
363
compareplane(const void * const arg1,
12,089,356✔
364
             const void * const arg2)
365
{
366
    int lhs, rhs;
367

368
    typedef const struct tupleint * const * const sortarg;
369
    sortarg comparandPP  = (sortarg) arg1;
12,089,356✔
370
    sortarg comparatorPP = (sortarg) arg2;
12,089,356✔
371
    lhs = (int)(*comparandPP)->tuple[compareplanePlane];
12,089,356✔
372
    rhs = (int)(*comparatorPP)->tuple[compareplanePlane];
12,089,356✔
373

374
    return lhs - rhs;
12,089,356✔
375
}
376

377

378
static int
379
sumcompare(const void * const b1, const void * const b2)
27,883,574✔
380
{
381
    return (int)((boxVector)b2)->sum - (int)((boxVector)b1)->sum;
27,883,574✔
382
}
383

384

385
static SIXELSTATUS
386
alloctupletable(
520✔
387
    tupletable          /* out */ *result,
388
    unsigned int const  /* in */  depth,
389
    unsigned int const  /* in */  size,
390
    sixel_allocator_t   /* in */  *allocator)
391
{
392
    SIXELSTATUS status = SIXEL_FALSE;
520✔
393
    enum { message_buffer_size = 256 };
394
    char message[message_buffer_size];
395
    int nwrite;
396
    unsigned int mainTableSize;
397
    unsigned int tupleIntSize;
398
    unsigned int allocSize;
399
    void * pool;
400
    tupletable tbl;
401
    unsigned int i;
402

403
    if (UINT_MAX / sizeof(struct tupleint) < size) {
520!
404
        nwrite = sixel_compat_snprintf(
×
405
            message,
406
            sizeof(message),
407
            "size %u is too big for arithmetic",
408
            size);
409
        if (nwrite > 0) {
×
410
            sixel_helper_set_additional_message(message);
×
411
        }
412
        status = SIXEL_RUNTIME_ERROR;
×
413
        goto end;
×
414
    }
415

416
    mainTableSize = size * sizeof(struct tupleint *);
520✔
417
    tupleIntSize = sizeof(struct tupleint) - sizeof(sample)
520✔
418
        + depth * sizeof(sample);
236✔
419

420
    /* To save the enormous amount of time it could take to allocate
421
       each individual tuple, we do a trick here and allocate everything
422
       as a single malloc block and suballocate internally.
423
    */
424
    if ((UINT_MAX - mainTableSize) / tupleIntSize < size) {
520!
425
        nwrite = sixel_compat_snprintf(
×
426
            message,
427
            sizeof(message),
428
            "size %u is too big for arithmetic",
429
            size);
430
        if (nwrite > 0) {
×
431
            sixel_helper_set_additional_message(message);
×
432
        }
433
        status = SIXEL_RUNTIME_ERROR;
×
434
        goto end;
×
435
    }
436

437
    allocSize = mainTableSize + size * tupleIntSize;
520✔
438

439
    pool = sixel_allocator_malloc(allocator, allocSize);
520✔
440
    if (pool == NULL) {
520!
441
        sixel_compat_snprintf(
×
442
            message,
443
            sizeof(message),
444
            "unable to allocate %u bytes for a %u-entry tuple table",
445
            allocSize,
446
            size);
447
        sixel_helper_set_additional_message(message);
×
448
        status = SIXEL_BAD_ALLOCATION;
×
449
        goto end;
×
450
    }
451
    tbl = (tupletable) pool;
520✔
452

453
    for (i = 0; i < size; ++i)
311,411✔
454
        tbl[i] = (struct tupleint *)
310,891✔
455
            ((char*)pool + mainTableSize + i * tupleIntSize);
310,891✔
456

457
    *result = tbl;
520✔
458

459
    status = SIXEL_OK;
520✔
460

461
end:
284✔
462
    return status;
520✔
463
}
464

465

466
/*
467
** Here is the fun part, the median-cut colormap generator.  This is based
468
** on Paul Heckbert's paper "Color Image Quantization for Frame Buffer
469
** Display", SIGGRAPH '82 Proceedings, page 297.
470
*/
471

472
static tupletable2
473
newColorMap(unsigned int const newcolors, unsigned int const depth, sixel_allocator_t *allocator)
69✔
474
{
475
    SIXELSTATUS status = SIXEL_FALSE;
69✔
476
    tupletable2 colormap;
477
    unsigned int i;
478

479
    colormap.size = 0;
69✔
480
    status = alloctupletable(&colormap.table, depth, newcolors, allocator);
69✔
481
    if (SIXEL_FAILED(status)) {
69!
482
        goto end;
×
483
    }
484
    if (colormap.table) {
92!
485
        for (i = 0; i < newcolors; ++i) {
13,833✔
486
            unsigned int plane;
487
            for (plane = 0; plane < depth; ++plane)
55,056✔
488
                colormap.table[i]->tuple[plane] = 0;
41,292✔
489
        }
4,588✔
490
        colormap.size = newcolors;
69✔
491
    }
23✔
492

493
end:
494
    return colormap;
69✔
495
}
496

497

498
static boxVector
499
newBoxVector(
69✔
500
    unsigned int const  /* in */ colors,
501
    unsigned int const  /* in */ sum,
502
    unsigned int const  /* in */ newcolors,
503
    sixel_allocator_t   /* in */ *allocator)
504
{
505
    boxVector bv;
506

507
    bv = (boxVector)sixel_allocator_malloc(allocator,
92✔
508
                                           sizeof(struct box) * (size_t)newcolors);
69✔
509
    if (bv == NULL) {
69!
510
        quant_trace(stderr, "out of memory allocating box vector table\n");
×
511
        return NULL;
×
512
    }
513

514
    /* Set up the initial box. */
515
    bv[0].ind = 0;
69✔
516
    bv[0].colors = colors;
69✔
517
    bv[0].sum = sum;
69✔
518

519
    return bv;
69✔
520
}
23✔
521

522

523
static void
524
findBoxBoundaries(tupletable2  const colorfreqtable,
24,813✔
525
                  unsigned int const depth,
526
                  unsigned int const boxStart,
527
                  unsigned int const boxSize,
528
                  sample             minval[],
529
                  sample             maxval[])
530
{
531
/*----------------------------------------------------------------------------
532
  Go through the box finding the minimum and maximum of each
533
  component - the boundaries of the box.
534
-----------------------------------------------------------------------------*/
535
    unsigned int plane;
536
    unsigned int i;
537

538
    for (plane = 0; plane < depth; ++plane) {
99,252✔
539
        minval[plane] = colorfreqtable.table[boxStart]->tuple[plane];
74,439✔
540
        maxval[plane] = minval[plane];
74,439✔
541
    }
24,813✔
542

543
    for (i = 1; i < boxSize; ++i) {
2,226,612✔
544
        for (plane = 0; plane < depth; ++plane) {
8,807,196✔
545
            sample const v = colorfreqtable.table[boxStart + i]->tuple[plane];
6,605,397✔
546
            if (v < minval[plane]) minval[plane] = v;
6,605,397✔
547
            if (v > maxval[plane]) maxval[plane] = v;
6,605,397✔
548
        }
2,202,753✔
549
    }
734,251✔
550
}
24,813✔
551

552

553

554
static unsigned int
555
largestByNorm(sample minval[], sample maxval[], unsigned int const depth)
23,427✔
556
{
557

558
    unsigned int largestDimension;
559
    unsigned int plane;
560
    sample largestSpreadSoFar;
561

562
    largestSpreadSoFar = 0;
23,427✔
563
    largestDimension = 0;
23,427✔
564
    for (plane = 0; plane < depth; ++plane) {
93,708✔
565
        sample const spread = maxval[plane]-minval[plane];
70,281✔
566
        if (spread > largestSpreadSoFar) {
70,281✔
567
            largestDimension = plane;
39,144✔
568
            largestSpreadSoFar = spread;
39,144✔
569
        }
12,946✔
570
    }
23,427✔
571
    return largestDimension;
23,427✔
572
}
573

574

575

576
static unsigned int
577
largestByLuminosity(sample minval[], sample maxval[], unsigned int const depth)
1,386✔
578
{
579
/*----------------------------------------------------------------------------
580
   This subroutine presumes that the tuple type is either
581
   BLACKANDWHITE, GRAYSCALE, or RGB (which implies pamP->depth is 1 or 3).
582
   To save time, we don't actually check it.
583
-----------------------------------------------------------------------------*/
584
    unsigned int retval;
585

586
    double lumin_factor[3] = {0.2989, 0.5866, 0.1145};
1,386✔
587

588
    if (depth == 1) {
1,386!
589
        retval = 0;
×
590
    } else {
591
        /* An RGB tuple */
592
        unsigned int largestDimension;
593
        unsigned int plane;
594
        double largestSpreadSoFar;
595

596
        largestSpreadSoFar = 0.0;
1,386✔
597
        largestDimension = 0;
1,386✔
598

599
        for (plane = 0; plane < 3; ++plane) {
5,544✔
600
            double const spread =
4,158✔
601
                lumin_factor[plane] * (maxval[plane]-minval[plane]);
4,158✔
602
            if (spread > largestSpreadSoFar) {
4,158✔
603
                largestDimension = plane;
2,369✔
604
                largestSpreadSoFar = spread;
2,369✔
605
            }
809✔
606
        }
1,386✔
607
        retval = largestDimension;
1,386✔
608
    }
609
    return retval;
1,386✔
610
}
611

612

613

614
static void
615
centerBox(unsigned int const boxStart,
×
616
          unsigned int const boxSize,
617
          tupletable2  const colorfreqtable,
618
          unsigned int const depth,
619
          tuple        const newTuple)
620
{
621

622
    unsigned int plane;
623
    sample minval, maxval;
624
    unsigned int i;
625

626
    for (plane = 0; plane < depth; ++plane) {
×
627
        minval = maxval = colorfreqtable.table[boxStart]->tuple[plane];
×
628

629
        for (i = 1; i < boxSize; ++i) {
×
630
            sample v = colorfreqtable.table[boxStart + i]->tuple[plane];
×
631
            minval = minval < v ? minval: v;
×
632
            maxval = maxval > v ? maxval: v;
×
633
        }
634
        newTuple[plane] = (minval + maxval) / 2;
×
635
    }
636
}
×
637

638

639

640
static void
641
averageColors(unsigned int const boxStart,
×
642
              unsigned int const boxSize,
643
              tupletable2  const colorfreqtable,
644
              unsigned int const depth,
645
              tuple        const newTuple)
646
{
647
    unsigned int plane;
648
    sample sum;
649
    unsigned int i;
650

651
    for (plane = 0; plane < depth; ++plane) {
×
652
        sum = 0;
×
653

654
        for (i = 0; i < boxSize; ++i) {
×
655
            sum += colorfreqtable.table[boxStart + i]->tuple[plane];
×
656
        }
657

658
        newTuple[plane] = sum / boxSize;
×
659
    }
660
}
×
661

662

663

664
static void
665
averagePixels(unsigned int const boxStart,
×
666
              unsigned int const boxSize,
667
              tupletable2 const colorfreqtable,
668
              unsigned int const depth,
669
              tuple const newTuple)
670
{
671

672
    unsigned int n;
673
        /* Number of tuples represented by the box */
674
    unsigned int plane;
675
    unsigned int i;
676

677
    /* Count the tuples in question */
678
    n = 0;  /* initial value */
×
679
    for (i = 0; i < boxSize; ++i) {
×
680
        n += (unsigned int)colorfreqtable.table[boxStart + i]->value;
×
681
    }
682

683
    for (plane = 0; plane < depth; ++plane) {
×
684
        sample sum;
685

686
        sum = 0;
×
687

688
        for (i = 0; i < boxSize; ++i) {
×
689
            sum += colorfreqtable.table[boxStart + i]->tuple[plane]
×
690
                * (unsigned int)colorfreqtable.table[boxStart + i]->value;
×
691
        }
692

693
        newTuple[plane] = sum / n;
×
694
    }
695
}
×
696

697

698

699
static tupletable2
700
colormapFromBv(unsigned int const newcolors,
×
701
               boxVector const bv,
702
               unsigned int const boxes,
703
               tupletable2 const colorfreqtable,
704
               unsigned int const depth,
705
               int const methodForRep,
706
               int const use_reversible,
707
               sixel_allocator_t *allocator)
708
{
709
    /*
710
    ** Ok, we've got enough boxes.  Now choose a representative color for
711
    ** each box.  There are a number of possible ways to make this choice.
712
    ** One would be to choose the center of the box; this ignores any structure
713
    ** within the boxes.  Another method would be to average all the colors in
714
    ** the box - this is the method specified in Heckbert's paper.  A third
715
    ** method is to average all the pixels in the box.
716
    */
717
    tupletable2 colormap;
718
    unsigned int bi;
719

720
    colormap = newColorMap(newcolors, depth, allocator);
×
721
    if (!colormap.size) {
×
722
        return colormap;
×
723
    }
724

725
    for (bi = 0; bi < boxes; ++bi) {
×
726
        switch (methodForRep) {
×
727
        case SIXEL_REP_CENTER_BOX:
728
            centerBox(bv[bi].ind, bv[bi].colors,
×
729
                      colorfreqtable, depth,
730
                      colormap.table[bi]->tuple);
×
731
            break;
×
732
        case SIXEL_REP_AVERAGE_COLORS:
733
            averageColors(bv[bi].ind, bv[bi].colors,
×
734
                          colorfreqtable, depth,
735
                          colormap.table[bi]->tuple);
×
736
            break;
×
737
        case SIXEL_REP_AVERAGE_PIXELS:
738
            averagePixels(bv[bi].ind, bv[bi].colors,
×
739
                          colorfreqtable, depth,
740
                          colormap.table[bi]->tuple);
×
741
            break;
×
742
        default:
743
            quant_trace(stderr, "Internal error: "
×
744
                                "invalid value of methodForRep: %d\n",
745
                        methodForRep);
746
        }
747
        if (use_reversible) {
×
748
            sixel_quant_reversible_tuple(colormap.table[bi]->tuple,
×
749
                                         depth);
750
        }
751
    }
752
    return colormap;
×
753
}
754

755

756
static int
757
force_palette_compare(const void *lhs, const void *rhs)
×
758
{
759
    unsigned int left;
760
    unsigned int right;
761
    unsigned int left_value;
762
    unsigned int right_value;
763

764
    left = *(const unsigned int *)lhs;
×
765
    right = *(const unsigned int *)rhs;
×
766
    left_value = force_palette_source->table[left]->value;
×
767
    right_value = force_palette_source->table[right]->value;
×
768
    if (left_value > right_value) {
×
769
        return -1;
×
770
    }
771
    if (left_value < right_value) {
×
772
        return 1;
×
773
    }
774
    if (left < right) {
×
775
        return -1;
×
776
    }
777
    if (left > right) {
×
778
        return 1;
×
779
    }
780
    return 0;
×
781
}
782

783

784
static SIXELSTATUS
785
force_palette_completion(tupletable2 *colormapP,
×
786
                         unsigned int depth,
787
                         unsigned int reqColors,
788
                         tupletable2 const colorfreqtable,
789
                         sixel_allocator_t *allocator)
790
{
791
    /*
792
     * We enqueue "losers" from the histogram so that we can revive them:
793
     *
794
     *   histogram --> sort by hit count --> append to palette tail
795
     *        ^                             |
796
     *        +-----------------------------+
797
     *
798
     * The ASCII loop shows how discarded colors walk back into the
799
     * palette when the user demands an exact size.
800
     */
801
    SIXELSTATUS status = SIXEL_FALSE;
×
802
    tupletable new_table = NULL;
×
803
    unsigned int *order = NULL;
×
804
    unsigned int current;
805
    unsigned int fill;
806
    unsigned int candidate;
807
    unsigned int plane;
808
    unsigned int source;
809

810
    current = colormapP->size;
×
811
    if (current >= reqColors) {
×
812
        return SIXEL_OK;
×
813
    }
814

815
    status = alloctupletable(&new_table, depth, reqColors, allocator);
×
816
    if (SIXEL_FAILED(status)) {
×
817
        goto end;
×
818
    }
819

820
    if (colorfreqtable.size > 0U) {
×
821
        order = (unsigned int *)sixel_allocator_malloc(
×
822
            allocator, colorfreqtable.size * sizeof(unsigned int));
×
823
        if (order == NULL) {
×
824
            status = SIXEL_BAD_ALLOCATION;
×
825
            goto end;
×
826
        }
827
        for (candidate = 0; candidate < colorfreqtable.size; ++candidate) {
×
828
            order[candidate] = candidate;
×
829
        }
830
        force_palette_source = &colorfreqtable;
×
831
        qsort(order, colorfreqtable.size, sizeof(unsigned int),
×
832
              force_palette_compare);
833
        force_palette_source = NULL;
×
834
    }
835

836
    for (fill = 0; fill < current; ++fill) {
×
837
        new_table[fill]->value = colormapP->table[fill]->value;
×
838
        for (plane = 0; plane < depth; ++plane) {
×
839
            new_table[fill]->tuple[plane] =
×
840
                colormapP->table[fill]->tuple[plane];
×
841
        }
842
    }
843

844
    candidate = 0U;
×
845
    fill = current;
×
846
    if (order != NULL) {
×
847
        while (fill < reqColors && candidate < colorfreqtable.size) {
×
848
            unsigned int index;
849

850
            index = order[candidate];
×
851
            new_table[fill]->value = colorfreqtable.table[index]->value;
×
852
            for (plane = 0; plane < depth; ++plane) {
×
853
                new_table[fill]->tuple[plane] =
×
854
                    colorfreqtable.table[index]->tuple[plane];
×
855
            }
856
            ++fill;
×
857
            ++candidate;
×
858
        }
859
    }
860

861
    if (fill < reqColors && fill == 0U) {
×
862
        new_table[fill]->value = 0U;
×
863
        for (plane = 0; plane < depth; ++plane) {
×
864
            new_table[fill]->tuple[plane] = 0U;
×
865
        }
866
        ++fill;
×
867
    }
868

869
    source = 0U;
×
870
    while (fill < reqColors) {
×
871
        new_table[fill]->value = new_table[source]->value;
×
872
        for (plane = 0; plane < depth; ++plane) {
×
873
            new_table[fill]->tuple[plane] = new_table[source]->tuple[plane];
×
874
        }
875
        ++fill;
×
876
        ++source;
×
877
        if (source >= fill) {
×
878
            source = 0U;
×
879
        }
880
    }
881

882
    sixel_allocator_free(allocator, colormapP->table);
×
883
    colormapP->table = new_table;
×
884
    colormapP->size = reqColors;
×
885
    status = SIXEL_OK;
×
886

887
end:
888
    if (status != SIXEL_OK && new_table != NULL) {
×
889
        sixel_allocator_free(allocator, new_table);
×
890
    }
891
    if (order != NULL) {
×
892
        sixel_allocator_free(allocator, order);
×
893
    }
894
    return status;
×
895
}
896

897

898
static SIXELSTATUS
899
splitBox(boxVector const bv,
24,813✔
900
         unsigned int *const boxesP,
901
         unsigned int const bi,
902
         tupletable2 const colorfreqtable,
903
         unsigned int const depth,
904
         int const methodForLargest)
905
{
906
/*----------------------------------------------------------------------------
907
   Split Box 'bi' in the box vector bv (so that bv contains one more box
908
   than it did as input).  Split it so that each new box represents about
909
   half of the pixels in the distribution given by 'colorfreqtable' for
910
   the colors in the original box, but with distinct colors in each of the
911
   two new boxes.
912

913
   Assume the box contains at least two colors.
914
-----------------------------------------------------------------------------*/
915
    SIXELSTATUS status = SIXEL_FALSE;
24,813✔
916
    unsigned int const boxStart = bv[bi].ind;
24,813✔
917
    unsigned int const boxSize  = bv[bi].colors;
24,813✔
918
    unsigned int const sm       = bv[bi].sum;
24,813✔
919

920
    enum { max_depth= 16 };
921
    sample minval[max_depth];
922
    sample maxval[max_depth];
923

924
    /* assert(max_depth >= depth); */
925

926
    unsigned int largestDimension;
927
    /* number of the plane with the largest spread */
928
    unsigned int medianIndex;
929
    unsigned int lowersum;
930

931
    /* Number of pixels whose value is "less than" the median */
932
    findBoxBoundaries(colorfreqtable, depth, boxStart, boxSize,
33,084✔
933
                      minval, maxval);
8,271✔
934

935
    /* Find the largest dimension, and sort by that component.  I have
936
       included two methods for determining the "largest" dimension;
937
       first by simply comparing the range in RGB space, and second by
938
       transforming into luminosities before the comparison.
939
    */
940
    switch (methodForLargest) {
24,813!
941
    case SIXEL_LARGE_NORM:
15,618✔
942
        largestDimension = largestByNorm(minval, maxval, depth);
23,427✔
943
        break;
23,427✔
944
    case SIXEL_LARGE_LUM:
924✔
945
        largestDimension = largestByLuminosity(minval, maxval, depth);
1,386✔
946
        break;
1,386✔
947
    default:
948
        sixel_helper_set_additional_message(
×
949
            "Internal error: invalid value of methodForLargest.");
950
        status = SIXEL_LOGIC_ERROR;
×
951
        goto end;
×
952
    }
953

954
    /* TODO: I think this sort should go after creating a box,
955
       not before splitting.  Because you need the sort to use
956
       the SIXEL_REP_CENTER_BOX method of choosing a color to
957
       represent the final boxes
958
    */
959

960
    /* Set the gross global variable 'compareplanePlane' as a
961
       parameter to compareplane(), which is called by qsort().
962
    */
963
    compareplanePlane = largestDimension;
24,813✔
964
    qsort((char*) &colorfreqtable.table[boxStart], boxSize,
24,813✔
965
          sizeof(colorfreqtable.table[boxStart]),
966
          compareplane);
967

968
    {
969
        /* Now find the median based on the counts, so that about half
970
           the pixels (not colors, pixels) are in each subdivision.  */
971

972
        unsigned int i;
973

974
        lowersum = colorfreqtable.table[boxStart]->value; /* initial value */
24,813✔
975
        for (i = 1; i < boxSize - 1 && lowersum < sm / 2; ++i) {
1,038,483✔
976
            lowersum += colorfreqtable.table[boxStart + i]->value;
1,013,670✔
977
        }
336,688✔
978
        medianIndex = i;
24,813✔
979
    }
980
    /* Split the box, and sort to bring the biggest boxes to the top.  */
981

982
    bv[bi].colors = medianIndex;
24,813✔
983
    bv[bi].sum = lowersum;
24,813✔
984
    bv[*boxesP].ind = boxStart + medianIndex;
24,813✔
985
    bv[*boxesP].colors = boxSize - medianIndex;
24,813✔
986
    bv[*boxesP].sum = sm - lowersum;
24,813✔
987
    ++(*boxesP);
24,813✔
988
    qsort((char*) bv, *boxesP, sizeof(struct box), sumcompare);
24,813✔
989

990
    status = SIXEL_OK;
24,813✔
991

992
end:
16,542✔
993
    return status;
24,813✔
994
}
995

996

997

998
static unsigned int sixel_final_merge_target(unsigned int reqcolors,
999
                                             int final_merge_mode);
1000
static void sixel_merge_clusters_ward(unsigned long *weights,
1001
                                      unsigned long *sums,
1002
                                      unsigned int depth,
1003
                                      int *cluster_count,
1004
                                      int target);
1005
static SIXELSTATUS sixel_quant_clusters_to_colormap(unsigned long *weights,
1006
                                                    unsigned long *sums,
1007
                                                    unsigned int depth,
1008
                                                    unsigned int cluster_count,
1009
                                                    int use_reversible,
1010
                                                    tupletable2 *colormapP,
1011
                                                    sixel_allocator_t *allocator);
1012

1013
static SIXELSTATUS
1014
mediancut(tupletable2 const colorfreqtable,
69✔
1015
          unsigned int const depth,
1016
          unsigned int const newcolors,
1017
          int const methodForLargest,
1018
          int const methodForRep,
1019
          int const use_reversible,
1020
          int const final_merge_mode,
1021
          tupletable2 *const colormapP,
1022
          sixel_allocator_t *allocator)
1023
{
1024
/*----------------------------------------------------------------------------
1025
   Compute a set of only 'newcolors' colors that best represent an
1026
   image whose pixels are summarized by the histogram
1027
   'colorfreqtable'.  Each tuple in that table has depth 'depth'.
1028
   colorfreqtable.table[i] tells the number of pixels in the subject image
1029
   have a particular color.
1030

1031
   As a side effect, sort 'colorfreqtable'.
1032
-----------------------------------------------------------------------------*/
1033
    boxVector bv;
1034
    unsigned int bi;
1035
    unsigned int boxes;
1036
    int multicolorBoxesExist;
1037
    unsigned int i;
1038
    unsigned int sum;
1039
    unsigned int working_colors;
1040
    int apply_merge;
1041
    unsigned long *cluster_weight;
1042
    unsigned long *cluster_sums;
1043
    int cluster_total;
1044
    unsigned int plane;
1045
    unsigned int offset;
1046
    unsigned int size;
1047
    unsigned long value;
1048
    struct tupleint *entry;
1049
    SIXELSTATUS merge_status;
1050
    SIXELSTATUS status = SIXEL_FALSE;
69✔
1051

1052
    sum = 0;
69✔
1053
    working_colors = newcolors;
69✔
1054
    apply_merge = (final_merge_mode == SIXEL_FINAL_MERGE_AUTO
69✔
1055
                   || final_merge_mode == SIXEL_FINAL_MERGE_WARD);
69!
1056
    bv = NULL;
69✔
1057
    cluster_weight = NULL;
69✔
1058
    cluster_sums = NULL;
69✔
1059
    cluster_total = 0;
69✔
1060
    plane = 0U;
69✔
1061
    offset = 0U;
69✔
1062
    size = 0U;
69✔
1063
    value = 0UL;
69✔
1064
    entry = NULL;
69✔
1065
    merge_status = SIXEL_OK;
69✔
1066

1067
    for (i = 0; i < colorfreqtable.size; ++i) {
291,168✔
1068
        sum += colorfreqtable.table[i]->value;
291,099✔
1069
    }
96,889✔
1070

1071
    if (apply_merge) {
69!
1072
        /* Choose an oversplit target so that the merge stage has slack. */
1073
        working_colors = sixel_final_merge_target(newcolors,
92✔
1074
                                                  final_merge_mode);
23✔
1075
        if (working_colors > colorfreqtable.size) {
69!
1076
            working_colors = colorfreqtable.size;
×
1077
        }
1078
        quant_trace(stderr, "overshoot: %d\n", working_colors);
69✔
1079
    }
23✔
1080
    if (working_colors == 0U) {
69!
1081
        working_colors = 1U;
×
1082
    }
1083

1084
    /* There is at least one box that contains at least 2 colors; ergo,
1085
       there is more splitting we can do.  */
1086
    bv = newBoxVector(colorfreqtable.size, sum, working_colors, allocator);
69✔
1087
    if (bv == NULL) {
69!
1088
        goto end;
×
1089
    }
1090
    boxes = 1;
69✔
1091
    multicolorBoxesExist = (colorfreqtable.size > 1);
69✔
1092

1093
    /* Main loop: split boxes until we have enough. */
1094
    while (boxes < working_colors && multicolorBoxesExist) {
24,882!
1095
        /* Find the first splittable box. */
1096
        for (bi = 0; bi < boxes && bv[bi].colors < 2; ++bi)
442,226!
1097
            ;
1098
        if (bi >= boxes) {
24,813!
1099
            multicolorBoxesExist = 0;
×
1100
        } else {
1101
            status = splitBox(bv, &boxes, bi,
33,084✔
1102
                              colorfreqtable, depth,
8,271✔
1103
                              methodForLargest);
8,271✔
1104
            if (SIXEL_FAILED(status)) {
24,813!
1105
                goto end;
×
1106
            }
1107
        }
1108
    }
1109
    if (apply_merge && boxes > newcolors) {
69!
1110
        /* Capture weight and component sums for each temporary box. */
1111
        cluster_weight = (unsigned long *)sixel_allocator_malloc(
69✔
1112
            allocator, (size_t)boxes * sizeof(unsigned long));
69✔
1113
        cluster_sums = (unsigned long *)sixel_allocator_malloc(
69✔
1114
            allocator, (size_t)boxes * (size_t)depth * sizeof(unsigned long));
69✔
1115
        if (cluster_weight == NULL || cluster_sums == NULL) {
69!
1116
            status = SIXEL_BAD_ALLOCATION;
×
1117
            goto end;
×
1118
        }
1119
        for (bi = 0U; bi < boxes; ++bi) {
24,951✔
1120
            offset = bv[bi].ind;
24,882✔
1121
            size = bv[bi].colors;
24,882✔
1122
            cluster_weight[bi] = 0UL;
24,882✔
1123
            for (plane = 0U; plane < depth; ++plane) {
99,528✔
1124
                cluster_sums[(size_t)bi * (size_t)depth + plane] = 0UL;
74,646✔
1125
            }
24,882✔
1126
            for (i = 0U; i < size; ++i) {
315,981✔
1127
                entry = colorfreqtable.table[offset + i];
291,099✔
1128
                value = (unsigned long)entry->value;
291,099✔
1129
                cluster_weight[bi] += value;
291,099✔
1130
                for (plane = 0U; plane < depth; ++plane) {
1,164,396✔
1131
                    cluster_sums[(size_t)bi * (size_t)depth + plane] +=
873,297✔
1132
                        (unsigned long)entry->tuple[plane] * value;
873,297✔
1133
                }
290,667✔
1134
            }
96,889✔
1135
        }
8,294✔
1136
        cluster_total = (int)boxes;
69✔
1137
        /* Merge clusters greedily using Ward's minimum variance rule. */
1138
        sixel_merge_clusters_ward(cluster_weight, cluster_sums, depth,
92✔
1139
                                  &cluster_total, (int)newcolors);
23✔
1140
        if (cluster_total < 1) {
69!
1141
            cluster_total = 1;
×
1142
        }
1143
        if ((unsigned int)cluster_total > newcolors) {
69!
1144
            cluster_total = (int)newcolors;
×
1145
        }
1146
        /* Rebuild the palette using the merged cluster statistics. */
1147
        merge_status = sixel_quant_clusters_to_colormap(cluster_weight,
92✔
1148
                                                        cluster_sums,
23✔
1149
                                                        depth,
23✔
1150
                                                        (unsigned int)cluster_total,
23✔
1151
                                                        use_reversible,
23✔
1152
                                                        colormapP,
23✔
1153
                                                        allocator);
23✔
1154
        if (SIXEL_FAILED(merge_status)) {
69!
1155
            status = merge_status;
×
1156
            goto end;
×
1157
        }
1158
    } else {
23✔
1159
        *colormapP = colormapFromBv(newcolors, bv, boxes,
×
1160
                                    colorfreqtable, depth,
1161
                                    methodForRep, use_reversible,
1162
                                    allocator);
1163
    }
1164

1165
    status = SIXEL_OK;
69✔
1166

1167
end:
46✔
1168
    if (bv != NULL) {
69!
1169
        sixel_allocator_free(allocator, bv);
69✔
1170
    }
23✔
1171
    if (cluster_sums != NULL) {
69!
1172
        sixel_allocator_free(allocator, cluster_sums);
69✔
1173
    }
23✔
1174
    if (cluster_weight != NULL) {
69!
1175
        sixel_allocator_free(allocator, cluster_weight);
69✔
1176
    }
23✔
1177
    return status;
69✔
1178
}
1179

1180

1181
/* Determine how many clusters to create before the final merge step. */
1182
static unsigned int
1183
sixel_final_merge_target(unsigned int reqcolors,
69✔
1184
                         int final_merge_mode)
1185
{
1186
    double factor;
1187
    unsigned int scaled;
1188

1189
    if (final_merge_mode != SIXEL_FINAL_MERGE_AUTO
69!
1190
        && final_merge_mode != SIXEL_FINAL_MERGE_WARD) {
23!
1191
        return reqcolors;
×
1192
    }
1193
    factor = env_final_merge_target_factor;
69✔
1194
    scaled = (unsigned int)((double)reqcolors * factor);
69✔
1195
    if (scaled <= reqcolors) {
69!
1196
        scaled = reqcolors;
×
1197
    }
1198
    if (scaled < 1U) {
69!
1199
        scaled = 1U;
×
1200
    }
1201

1202
    return scaled;
69✔
1203
}
23✔
1204

1205

1206
/* Merge clusters to the requested size using Ward linkage. */
1207
static void
1208
sixel_merge_clusters_ward(unsigned long *weights,
69✔
1209
                          unsigned long *sums,
1210
                          unsigned int depth,
1211
                          int *cluster_count,
1212
                          int target)
1213
{
1214
    int n;
1215
    int desired;
1216
    int best_i;
1217
    int best_j;
1218
    int i;
1219
    int j;
1220
    int k;
1221
    int channel;
1222
    double best_cost;
1223
    double wi;
1224
    double wj;
1225
    double distance_sq;
1226
    double delta;
1227
    double mean_i;
1228
    double mean_j;
1229
    double diff;
1230
    size_t offset_i;
1231
    size_t offset_j;
1232
    size_t dst;
1233
    size_t src;
1234

1235
    if (cluster_count == NULL) {
69!
1236
        return;
×
1237
    }
1238
    n = *cluster_count;
69✔
1239
    desired = target;
69✔
1240
    if (desired < 1) {
69!
1241
        desired = 1;
×
1242
    }
1243
    while (n > desired) {
11,187✔
1244
        best_i = -1;
11,118✔
1245
        best_j = -1;
11,118✔
1246
        best_cost = 1.0e300;
11,118✔
1247
        for (i = 0; i < n; ++i) {
3,948,603✔
1248
            if (weights[i] == 0UL) {
3,937,485!
1249
                continue;
×
1250
            }
1251
            wi = (double)weights[i];
3,937,485✔
1252
            offset_i = (size_t)i * (size_t)depth;
3,937,485✔
1253
            for (j = i + 1; j < n; ++j) {
724,512,615✔
1254
                if (weights[j] == 0UL) {
720,575,130!
1255
                    continue;
×
1256
                }
1257
                wj = (double)weights[j];
720,575,130✔
1258
                offset_j = (size_t)j * (size_t)depth;
720,575,130✔
1259
                distance_sq = 0.0;
720,575,130✔
1260
                for (channel = 0; channel < (int)depth; ++channel) {
2,147,483,647✔
1261
                    mean_i = (double)sums[offset_i + (size_t)channel] / wi;
2,147,483,647✔
1262
                    mean_j = (double)sums[offset_j + (size_t)channel] / wj;
2,147,483,647✔
1263
                    diff = mean_i - mean_j;
2,147,483,647✔
1264
                    distance_sq += diff * diff;
2,147,483,647✔
1265
                }
720,575,130✔
1266
                delta = (wi * wj) / (wi + wj) * distance_sq;
720,575,130✔
1267
                if (delta < best_cost) {
720,575,130✔
1268
                    best_cost = delta;
124,929✔
1269
                    best_i = i;
124,929✔
1270
                    best_j = j;
124,929✔
1271
                }
41,091✔
1272
            }
240,191,710✔
1273
        }
1,312,495✔
1274
        if (best_i < 0 || best_j < 0) {
11,118!
1275
            break;
1276
        }
1277
        weights[best_i] += weights[best_j];
11,118✔
1278
        offset_i = (size_t)best_i * (size_t)depth;
11,118✔
1279
        offset_j = (size_t)best_j * (size_t)depth;
11,118✔
1280
        for (channel = 0; channel < (int)depth; ++channel) {
44,472✔
1281
            sums[offset_i + (size_t)channel] +=
33,354✔
1282
                sums[offset_j + (size_t)channel];
33,354✔
1283
        }
11,118✔
1284
        for (k = best_j; k < n - 1; ++k) {
941,582✔
1285
            weights[k] = weights[k + 1];
930,464✔
1286
            dst = (size_t)k * (size_t)depth;
930,464✔
1287
            src = (size_t)(k + 1) * (size_t)depth;
930,464✔
1288
            for (channel = 0; channel < (int)depth; ++channel) {
3,721,856✔
1289
                sums[dst + (size_t)channel] = sums[src + (size_t)channel];
2,791,392✔
1290
            }
948,216✔
1291
        }
316,072✔
1292
        --n;
11,118✔
1293
    }
1294
    *cluster_count = n;
69✔
1295
}
23✔
1296

1297

1298
/* Translate merged cluster statistics into a tupletable palette. */
1299
static SIXELSTATUS
1300
sixel_quant_clusters_to_colormap(unsigned long *weights,
69✔
1301
                               unsigned long *sums,
1302
                               unsigned int depth,
1303
                               unsigned int cluster_count,
1304
                               int use_reversible,
1305
                               tupletable2 *colormapP,
1306
                               sixel_allocator_t *allocator)
1307
{
1308
    SIXELSTATUS status;
1309
    tupletable2 colormap;
1310
    unsigned int index;
1311
    unsigned int plane;
1312
    double component;
1313
    unsigned long weight;
1314

1315
    status = SIXEL_BAD_ARGUMENT;
69✔
1316
    if (colormapP == NULL) {
69!
1317
        return status;
×
1318
    }
1319
    colormap = newColorMap(cluster_count, depth, allocator);
69✔
1320
    if (colormap.size == 0U) {
69!
1321
        return SIXEL_BAD_ALLOCATION;
×
1322
    }
1323
    for (index = 0U; index < cluster_count; ++index) {
13,833✔
1324
        weight = weights[index];
13,764✔
1325
        if (weight == 0UL) {
13,764!
1326
            weight = 1UL;
×
1327
        }
1328
        colormap.table[index]->value =
13,764✔
1329
            (unsigned int)((weight > (unsigned long)UINT_MAX)
13,764!
1330
                ? UINT_MAX
1331
                : weight);
4,588✔
1332
        for (plane = 0U; plane < depth; ++plane) {
55,056✔
1333
            component = (double)sums[(size_t)index * (size_t)depth + plane];
41,292✔
1334
            component /= (double)weight;
41,292✔
1335
            if (component < 0.0) {
41,292!
1336
                component = 0.0;
×
1337
            }
1338
            if (component > 255.0) {
41,292!
1339
                component = 255.0;
×
1340
            }
1341
            colormap.table[index]->tuple[plane] =
41,292✔
1342
                (sample)(component + 0.5);
41,292✔
1343
        }
13,764✔
1344
        if (use_reversible) {
13,764!
1345
            sixel_quant_reversible_tuple(colormap.table[index]->tuple,
×
1346
                                         depth);
1347
        }
1348
    }
4,588✔
1349
    *colormapP = colormap;
69✔
1350
    status = SIXEL_OK;
69✔
1351

1352
    return status;
69✔
1353
}
23✔
1354

1355

1356
static int histogram_lut_policy = SIXEL_LUT_POLICY_AUTO;
1357
static int quant_method_for_largest = SIXEL_LARGE_NORM;
1358

1359
void
1360
sixel_quant_set_lut_policy(int lut_policy)
545✔
1361
{
1362
    int normalized;
1363

1364
    normalized = SIXEL_LUT_POLICY_AUTO;
545✔
1365
    if (lut_policy == SIXEL_LUT_POLICY_5BIT
778!
1366
        || lut_policy == SIXEL_LUT_POLICY_6BIT
545!
1367
        || lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
545!
1368
        || lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH
545!
1369
        || lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
545!
1370
        normalized = lut_policy;
×
1371
    }
1372

1373
    histogram_lut_policy = normalized;
545✔
1374
}
545✔
1375

1376
void
1377
sixel_quant_set_method_for_largest(int method)
285✔
1378
{
1379
    int normalized;
1380

1381
    normalized = SIXEL_LARGE_NORM;
285✔
1382
    if (method == SIXEL_LARGE_NORM || method == SIXEL_LARGE_LUM) {
285!
1383
        normalized = method;
285✔
1384
    } else if (method == SIXEL_LARGE_AUTO) {
115!
1385
        normalized = SIXEL_LARGE_NORM;
×
1386
    }
1387

1388
    quant_method_for_largest = normalized;
285✔
1389
}
285✔
1390

1391
size_t
1392
histogram_dense_size(unsigned int depth,
524✔
1393
                     struct histogram_control const *control)
1394
{
1395
    size_t size;
1396
    unsigned int exponent;
1397
    unsigned int i;
1398

1399
    size = 1U;
524✔
1400
    exponent = depth * control->channel_bits;
524✔
1401
    for (i = 0U; i < exponent; ++i) {
9,956✔
1402
        if (size > SIZE_MAX / 2U) {
9,432!
1403
            size = SIZE_MAX;
×
1404
            break;
×
1405
        }
1406
        size <<= 1U;
9,432✔
1407
    }
4,068✔
1408

1409
    return size;
524✔
1410
}
1411

1412
struct histogram_control
1413
histogram_control_make_for_policy(unsigned int depth, int lut_policy)
40,078,482✔
1414
{
1415
    struct histogram_control control;
1416

1417
    control.reversible_rounding = 0;
40,078,482✔
1418
    /*
1419
     * The ASCII ladder below shows how each policy selects bucket width.
1420
     *
1421
     *   auto / 6bit RGB : |--6--|
1422
     *   forced 5bit     : |---5---|
1423
     *   robinhood       : |------8------|
1424
     *   alpha fallback  : |---5---|  (avoids 2^(6*4) buckets)
1425
     */
1426
    control.channel_shift = 2U;
40,078,482✔
1427
    if (depth > 3U) {
40,078,482!
1428
        control.channel_shift = 3U;
×
1429
    }
1430
    if (lut_policy == SIXEL_LUT_POLICY_5BIT) {
40,078,482!
1431
        control.channel_shift = 3U;
×
1432
    } else if (lut_policy == SIXEL_LUT_POLICY_6BIT) {
40,078,482✔
1433
        control.channel_shift = 2U;
264✔
1434
        if (depth > 3U) {
264!
1435
            control.channel_shift = 3U;
×
1436
        }
1437
    } else if (lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
40,078,326!
1438
               || lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
40,078,218!
1439
        control.channel_shift = 0U;
×
1440
    } else if (lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
40,078,218!
1441
        control.channel_shift = 2U;
×
1442
    }
1443
    control.channel_bits = 8U - control.channel_shift;
40,078,482✔
1444
    control.channel_mask = (1U << control.channel_bits) - 1U;
40,078,482✔
1445

1446
    return control;
40,078,482✔
1447
}
1448

1449
unsigned int
1450
histogram_reconstruct(unsigned int quantized,
882,339✔
1451
                      struct histogram_control const *control)
1452
{
1453
    unsigned int value;
1454

1455
    value = quantized << control->channel_shift;
882,339✔
1456
    if (quantized == control->channel_mask) {
882,339✔
1457
        value = 255U;
1,203✔
1458
    } else {
517✔
1459
        if (control->channel_shift > 0U) {
881,136!
1460
            value |= (1U << (control->channel_shift - 1U));
881,136✔
1461
        }
294,374✔
1462
    }
1463
    if (value > 255U) {
882,339!
1464
        value = 255U;
×
1465
    }
1466

1467
    return value;
882,339✔
1468
}
1469

1470
static unsigned int
1471
histogram_quantize(unsigned int sample8,
130,951,272✔
1472
                   struct histogram_control const *control)
1473
{
1474
    unsigned int quantized;
1475
    unsigned int shift;
1476
    unsigned int mask;
1477
    unsigned int rounding;
1478

1479
    /*
1480
     * In reversible mode we already rounded once when snapping to
1481
     * sixel_safe_tones[].  If we rounded to the nearest midpoint
1482
     * again, the second pass would fall back to the lower bucket and break
1483
     * the round-trip.  Biasing towards the upper edge keeps the bucket
1484
     * stable after decoding and encoding again.  Non-reversible runs keep
1485
     * symmetric midpoints to avoid nudging colors upwards.
1486
     *
1487
     *        midpoint bias        upper-edge bias
1488
     *   |----*----|----*----|    |----|----*----|
1489
     *   0         8        16    0    8        16
1490
     *
1491
     * The asterisk marks the midpoint captured by a bucket.  Moving that
1492
     * midpoint to the upper edge keeps reversible tones from drifting.
1493
     */
1494
    shift = control->channel_shift;
130,951,272✔
1495
    mask = control->channel_mask;
130,951,272✔
1496
    if (shift == 0U) {
130,951,272!
1497
        quantized = sample8;
×
1498
    } else {
1499
        if (control->reversible_rounding) {
130,951,272!
1500
            rounding = 1U << shift;
×
1501
        } else {
1502
            rounding = 1U << (shift - 1U);
130,951,272✔
1503
        }
1504
        quantized = (sample8 + rounding) >> shift;
130,951,272✔
1505
        if (quantized > mask) {
130,951,272✔
1506
            quantized = mask;
3,133,447✔
1507
        }
1,500,381✔
1508
    }
1509

1510
    return quantized;
130,951,272✔
1511
}
1512

1513
uint32_t
1514
histogram_pack_color(unsigned char const *data,
43,650,424✔
1515
                     unsigned int const depth,
1516
                     struct histogram_control const *control)
1517
{
1518
    uint32_t packed;
1519
    unsigned int n;
1520
    unsigned int sample8;
1521
    unsigned int bits;
1522

1523
    packed = 0U;
43,650,424✔
1524
    bits = control->channel_bits;
43,650,424✔
1525
    if (control->channel_shift == 0U) {
43,650,424!
1526
        /*
1527
         * The channel shift being zero means each component keeps eight
1528
         * bits.  We therefore pack pixels in RGB order, as illustrated
1529
         * below:
1530
         *
1531
         *      R   G   B
1532
         *     [ ]-[ ]-[ ]
1533
         *      |   |   |
1534
         *      v   v   v
1535
         *     0xRRGGBB
1536
         */
1537
        for (n = 0U; n < depth; ++n) {
×
1538
            packed |= (uint32_t)data[depth - 1U - n] << (n * bits);
×
1539
        }
1540
        return packed;
×
1541
    }
1542

1543
    for (n = 0U; n < depth; ++n) {
174,601,696✔
1544
        sample8 = (unsigned int)data[depth - 1U - n];
130,951,272✔
1545
        packed |= histogram_quantize(sample8, control) << (n * bits);
130,951,272✔
1546
    }
62,349,072✔
1547

1548
    return packed;
43,650,424✔
1549
}
20,783,024✔
1550

1551
uint32_t
1552
histogram_hash_mix(uint32_t key)
40,077,958✔
1553
{
1554
    uint32_t hash;
1555

1556
    /*
1557
     * Multiplicative mixing with two avalanching rounds keeps nearby
1558
     * colors far apart in the hash domain.  The final tweak avoids the
1559
     * 0xffffffff sentinel used by hopscotch slots.
1560
     */
1561
    hash = key * 0x9e3779b9U;
40,077,958✔
1562
    hash ^= hash >> 16;
40,077,958✔
1563
    hash *= 0x7feb352dU;
40,077,958✔
1564
    hash ^= hash >> 15;
40,077,958✔
1565
    hash *= 0x846ca68bU;
40,077,958✔
1566
    hash ^= hash >> 16;
40,077,958✔
1567
    if (hash == 0xffffffffU) {
40,077,958!
1568
        hash ^= 0x632be59bU;
×
1569
    }
1570

1571
    return hash;
40,077,958✔
1572
}
1573

1574
static unsigned int
1575
computeHash(unsigned char const *data,
40,077,958✔
1576
            unsigned int const depth,
1577
            struct histogram_control const *control)
1578
{
1579
    uint32_t packed;
1580

1581
    packed = histogram_pack_color(data, depth, control);
40,077,958✔
1582

1583
    return histogram_hash_mix(packed);
40,077,958✔
1584
}
1585

1586
#define CUCKOO_BUCKET_SIZE 4U
1587
#define CUCKOO_MAX_KICKS 128U
1588
#define CUCKOO_STASH_SIZE 32U
1589
#define CUCKOO_EMPTY_KEY 0xffffffffU
1590

1591
struct cuckoo_bucket32 {
1592
    uint32_t key[CUCKOO_BUCKET_SIZE];
1593
    uint32_t value[CUCKOO_BUCKET_SIZE];
1594
};
1595

1596
struct cuckoo_table32 {
1597
    struct cuckoo_bucket32 *buckets;
1598
    uint32_t stash_key[CUCKOO_STASH_SIZE];
1599
    uint32_t stash_value[CUCKOO_STASH_SIZE];
1600
    size_t bucket_count;
1601
    size_t bucket_mask;
1602
    size_t stash_count;
1603
    size_t entry_count;
1604
    sixel_allocator_t *allocator;
1605
};
1606

1607
static size_t cuckoo_round_buckets(size_t hint);
1608
static size_t cuckoo_hash_primary(uint32_t key, size_t mask);
1609
static size_t cuckoo_hash_secondary(uint32_t key, size_t mask);
1610
static size_t cuckoo_hash_alternate(uint32_t key,
1611
                                    size_t bucket,
1612
                                    size_t mask);
1613
static uint32_t *cuckoo_bucket_find(struct cuckoo_bucket32 *bucket,
1614
                                    uint32_t key);
1615
static int cuckoo_bucket_insert_direct(struct cuckoo_bucket32 *bucket,
1616
                                       uint32_t key,
1617
                                       uint32_t value);
1618
static SIXELSTATUS cuckoo_table32_init(struct cuckoo_table32 *table,
1619
                                       size_t expected,
1620
                                       sixel_allocator_t *allocator);
1621
static void cuckoo_table32_clear(struct cuckoo_table32 *table);
1622
static void cuckoo_table32_fini(struct cuckoo_table32 *table);
1623
static uint32_t *cuckoo_table32_lookup(struct cuckoo_table32 *table,
1624
                                       uint32_t key);
1625
static SIXELSTATUS cuckoo_table32_insert(struct cuckoo_table32 *table,
1626
                                         uint32_t key,
1627
                                         uint32_t value);
1628
static SIXELSTATUS cuckoo_table32_grow(struct cuckoo_table32 *table);
1629

1630
struct robinhood_slot32 {
1631
    uint32_t key;
1632
    uint32_t color;
1633
    uint32_t value;
1634
    uint16_t distance;
1635
    uint16_t pad;
1636
};
1637

1638
struct robinhood_table32 {
1639
    struct robinhood_slot32 *slots;
1640
    size_t capacity;
1641
    size_t count;
1642
    sixel_allocator_t *allocator;
1643
};
1644

1645
static size_t robinhood_round_capacity(size_t hint);
1646
static SIXELSTATUS robinhood_table32_init(struct robinhood_table32 *table,
1647
                                         size_t expected,
1648
                                         sixel_allocator_t *allocator);
1649
static void robinhood_table32_fini(struct robinhood_table32 *table);
1650
static struct robinhood_slot32 *
1651
robinhood_table32_lookup(struct robinhood_table32 *table,
1652
                         uint32_t key,
1653
                         uint32_t color);
1654
static SIXELSTATUS robinhood_table32_insert(struct robinhood_table32 *table,
1655
                                           uint32_t key,
1656
                                           uint32_t color,
1657
                                           uint32_t value);
1658
static SIXELSTATUS robinhood_table32_grow(struct robinhood_table32 *table);
1659
static struct robinhood_slot32 *
1660
robinhood_table32_place(struct robinhood_table32 *table,
1661
                        struct robinhood_slot32 entry);
1662

1663
#define HOPSCOTCH_EMPTY_KEY 0xffffffffU
1664
#define HOPSCOTCH_DEFAULT_NEIGHBORHOOD 32U
1665
#define HOPSCOTCH_INSERT_RANGE 256U
1666

1667
struct hopscotch_slot32 {
1668
    uint32_t key;
1669
    uint32_t color;
1670
    uint32_t value;
1671
};
1672

1673
struct hopscotch_table32 {
1674
    struct hopscotch_slot32 *slots;
1675
    uint32_t *hopinfo;
1676
    size_t capacity;
1677
    size_t count;
1678
    size_t neighborhood;
1679
    sixel_allocator_t *allocator;
1680
};
1681

1682
static SIXELSTATUS hopscotch_table32_init(struct hopscotch_table32 *table,
1683
                                          size_t expected,
1684
                                          sixel_allocator_t *allocator);
1685
static void hopscotch_table32_fini(struct hopscotch_table32 *table);
1686
static struct hopscotch_slot32 *
1687
hopscotch_table32_lookup(struct hopscotch_table32 *table,
1688
                         uint32_t key,
1689
                         uint32_t color);
1690
static SIXELSTATUS hopscotch_table32_insert(struct hopscotch_table32 *table,
1691
                                            uint32_t key,
1692
                                            uint32_t color,
1693
                                            uint32_t value);
1694
static SIXELSTATUS hopscotch_table32_grow(struct hopscotch_table32 *table);
1695

1696
struct histogram_control
1697
histogram_control_make(unsigned int depth)
40,078,218✔
1698
{
1699
    struct histogram_control control;
1700

1701
    control = histogram_control_make_for_policy(depth,
59,084,154✔
1702
                                                histogram_lut_policy);
19,005,936✔
1703

1704
    return control;
40,078,218✔
1705
}
1706

1707
static int
1708
sixel_certlut_init(sixel_certlut_t *lut)
×
1709
{
1710
    int status;
1711

1712
    status = SIXEL_FALSE;
×
1713
    if (lut == NULL) {
×
1714
        goto end;
×
1715
    }
1716

1717
    lut->level0 = NULL;
×
1718
    lut->pool = NULL;
×
1719
    lut->pool_size = 0U;
×
1720
    lut->pool_capacity = 0U;
×
1721
    lut->wR = 1;
×
1722
    lut->wG = 1;
×
1723
    lut->wB = 1;
×
1724
    lut->wR2 = 1U;
×
1725
    lut->wG2 = 1U;
×
1726
    lut->wB2 = 1U;
×
1727
    memset(lut->wr_scale, 0, sizeof(lut->wr_scale));
×
1728
    memset(lut->wg_scale, 0, sizeof(lut->wg_scale));
×
1729
    memset(lut->wb_scale, 0, sizeof(lut->wb_scale));
×
1730
    lut->wr_palette = NULL;
×
1731
    lut->wg_palette = NULL;
×
1732
    lut->wb_palette = NULL;
×
1733
    lut->palette = NULL;
×
1734
    lut->ncolors = 0;
×
1735
    lut->kdnodes = NULL;
×
1736
    lut->kdnodes_count = 0;
×
1737
    lut->kdtree_root = -1;
×
1738
    status = SIXEL_OK;
×
1739

1740
end:
1741
    return status;
×
1742
}
1743

1744
static void
1745
sixel_certlut_release(sixel_certlut_t *lut)
×
1746
{
1747
    if (lut == NULL) {
×
1748
        return;
×
1749
    }
1750
    free(lut->level0);
×
1751
    free(lut->pool);
×
1752
    free(lut->wr_palette);
×
1753
    free(lut->wg_palette);
×
1754
    free(lut->wb_palette);
×
1755
    free(lut->kdnodes);
×
1756
    lut->level0 = NULL;
×
1757
    lut->pool = NULL;
×
1758
    lut->pool_size = 0U;
×
1759
    lut->pool_capacity = 0U;
×
1760
    lut->wr_palette = NULL;
×
1761
    lut->wg_palette = NULL;
×
1762
    lut->wb_palette = NULL;
×
1763
    lut->kdnodes = NULL;
×
1764
    lut->kdnodes_count = 0;
×
1765
    lut->kdtree_root = -1;
×
1766
}
1767

1768
static int
1769
sixel_certlut_prepare_palette_terms(sixel_certlut_t *lut)
×
1770
{
1771
    int status;
1772
    size_t count;
1773
    int index;
1774
    int32_t *wr_terms;
1775
    int32_t *wg_terms;
1776
    int32_t *wb_terms;
1777

1778
    status = SIXEL_FALSE;
×
1779
    wr_terms = NULL;
×
1780
    wg_terms = NULL;
×
1781
    wb_terms = NULL;
×
1782
    if (lut == NULL) {
×
1783
        goto end;
×
1784
    }
1785
    if (lut->ncolors <= 0) {
×
1786
        status = SIXEL_OK;
×
1787
        goto end;
×
1788
    }
1789
    count = (size_t)lut->ncolors;
×
1790
    wr_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
1791
    if (wr_terms == NULL) {
×
1792
        goto end;
×
1793
    }
1794
    wg_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
1795
    if (wg_terms == NULL) {
×
1796
        goto end;
×
1797
    }
1798
    wb_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
1799
    if (wb_terms == NULL) {
×
1800
        goto end;
×
1801
    }
1802
    for (index = 0; index < lut->ncolors; ++index) {
×
1803
        wr_terms[index] = lut->wR * (int)lut->palette[index].r;
×
1804
        wg_terms[index] = lut->wG * (int)lut->palette[index].g;
×
1805
        wb_terms[index] = lut->wB * (int)lut->palette[index].b;
×
1806
    }
1807
    free(lut->wr_palette);
×
1808
    free(lut->wg_palette);
×
1809
    free(lut->wb_palette);
×
1810
    lut->wr_palette = wr_terms;
×
1811
    lut->wg_palette = wg_terms;
×
1812
    lut->wb_palette = wb_terms;
×
1813
    wr_terms = NULL;
×
1814
    wg_terms = NULL;
×
1815
    wb_terms = NULL;
×
1816
    status = SIXEL_OK;
×
1817

1818
end:
1819
    free(wr_terms);
×
1820
    free(wg_terms);
×
1821
    free(wb_terms);
×
1822
    return status;
×
1823
}
1824

1825
static void
1826
sixel_quant_cell_center(int rmin, int gmin, int bmin, int size,
×
1827
                        int *cr, int *cg, int *cb)
1828
{
1829
    int half;
1830

1831
    half = size / 2;
×
1832
    *cr = rmin + half;
×
1833
    *cg = gmin + half;
×
1834
    *cb = bmin + half;
×
1835
    if (size == 1) {
×
1836
        *cr = rmin;
×
1837
        *cg = gmin;
×
1838
        *cb = bmin;
×
1839
    }
1840
}
×
1841

1842
static void
1843
sixel_quant_weight_init(sixel_certlut_t *lut, int wR, int wG, int wB)
×
1844
{
1845
    int i;
1846

1847
    lut->wR = wR;
×
1848
    lut->wG = wG;
×
1849
    lut->wB = wB;
×
1850
    lut->wR2 = (uint64_t)wR * (uint64_t)wR;
×
1851
    lut->wG2 = (uint64_t)wG * (uint64_t)wG;
×
1852
    lut->wB2 = (uint64_t)wB * (uint64_t)wB;
×
1853
    for (i = 0; i < 256; ++i) {
×
1854
        lut->wr_scale[i] = wR * i;
×
1855
        lut->wg_scale[i] = wG * i;
×
1856
        lut->wb_scale[i] = wB * i;
×
1857
    }
1858
}
×
1859

1860
static uint64_t
1861
sixel_certlut_distance_precomputed(sixel_certlut_t const *lut,
×
1862
                                   int index,
1863
                                   int32_t wr_r,
1864
                                   int32_t wg_g,
1865
                                   int32_t wb_b)
1866
{
1867
    uint64_t distance;
1868
    int64_t diff;
1869

1870
    diff = (int64_t)wr_r - (int64_t)lut->wr_palette[index];
×
1871
    distance = (uint64_t)(diff * diff);
×
1872
    diff = (int64_t)wg_g - (int64_t)lut->wg_palette[index];
×
1873
    distance += (uint64_t)(diff * diff);
×
1874
    diff = (int64_t)wb_b - (int64_t)lut->wb_palette[index];
×
1875
    distance += (uint64_t)(diff * diff);
×
1876

1877
    return distance;
×
1878
}
1879

1880
static void
1881
sixel_quant_distance_pair(sixel_certlut_t const *lut, int r, int g, int b,
×
1882
                          int *best_idx, int *second_idx,
1883
                          uint64_t *best_dist, uint64_t *second_dist)
1884
{
1885
    int i;
1886
    int best_candidate;
1887
    int second_candidate;
1888
    uint64_t best_value;
1889
    uint64_t second_value;
1890
    uint64_t distance;
1891
    int rr;
1892
    int gg;
1893
    int bb;
1894
    int32_t wr_r;
1895
    int32_t wg_g;
1896
    int32_t wb_b;
1897

1898
    best_candidate = (-1);
×
1899
    second_candidate = (-1);
×
1900
    best_value = UINT64_MAX;
×
1901
    second_value = UINT64_MAX;
×
1902
    rr = r;
×
1903
    gg = g;
×
1904
    bb = b;
×
1905
    if (rr < 0) {
×
1906
        rr = 0;
×
1907
    } else if (rr > 255) {
×
1908
        rr = 255;
×
1909
    }
1910
    if (gg < 0) {
×
1911
        gg = 0;
×
1912
    } else if (gg > 255) {
×
1913
        gg = 255;
×
1914
    }
1915
    if (bb < 0) {
×
1916
        bb = 0;
×
1917
    } else if (bb > 255) {
×
1918
        bb = 255;
×
1919
    }
1920
    wr_r = lut->wr_scale[rr];
×
1921
    wg_g = lut->wg_scale[gg];
×
1922
    wb_b = lut->wb_scale[bb];
×
1923
    if (lut->kdnodes != NULL && lut->kdtree_root >= 0) {
×
1924
        sixel_certlut_kdtree_search(lut,
×
1925
                                    lut->kdtree_root,
×
1926
                                    r,
1927
                                    g,
1928
                                    b,
1929
                                    wr_r,
1930
                                    wg_g,
1931
                                    wb_b,
1932
                                    &best_candidate,
1933
                                    &best_value,
1934
                                    &second_candidate,
1935
                                    &second_value);
1936
    } else {
1937
        for (i = 0; i < lut->ncolors; ++i) {
×
1938
            distance = sixel_certlut_distance_precomputed(lut,
×
1939
                                                          i,
1940
                                                          wr_r,
1941
                                                          wg_g,
1942
                                                          wb_b);
1943
            if (distance < best_value) {
×
1944
                second_value = best_value;
×
1945
                second_candidate = best_candidate;
×
1946
                best_value = distance;
×
1947
                best_candidate = i;
×
1948
            } else if (distance < second_value) {
×
1949
                second_value = distance;
×
1950
                second_candidate = i;
×
1951
            }
1952
        }
1953
    }
1954
    if (second_candidate < 0) {
×
1955
        second_candidate = best_candidate;
×
1956
        second_value = best_value;
×
1957
    }
1958
    *best_idx = best_candidate;
×
1959
    *second_idx = second_candidate;
×
1960
    *best_dist = best_value;
×
1961
    *second_dist = second_value;
×
1962
}
×
1963

1964
static int
1965
sixel_quant_is_cell_safe(sixel_certlut_t const *lut, int best_idx,
×
1966
                         int second_idx, int size, uint64_t best_dist,
1967
                         uint64_t second_dist)
1968
{
1969
    uint64_t delta_sq;
1970
    uint64_t rhs;
1971
    uint64_t weight_term;
1972
    int64_t wr_delta;
1973
    int64_t wg_delta;
1974
    int64_t wb_delta;
1975

1976
    if (best_idx < 0 || second_idx < 0) {
×
1977
        return 1;
×
1978
    }
1979

1980
    /*
1981
     * The certification bound compares the squared distance gap against the
1982
     * palette separation scaled by the cube diameter.  If the gap wins the
1983
     * entire cube maps to the current best palette entry.
1984
     */
1985
    delta_sq = second_dist - best_dist;
×
1986
    wr_delta = (int64_t)lut->wr_palette[second_idx]
×
1987
        - (int64_t)lut->wr_palette[best_idx];
×
1988
    wg_delta = (int64_t)lut->wg_palette[second_idx]
×
1989
        - (int64_t)lut->wg_palette[best_idx];
×
1990
    wb_delta = (int64_t)lut->wb_palette[second_idx]
×
1991
        - (int64_t)lut->wb_palette[best_idx];
×
1992
    weight_term = (uint64_t)(wr_delta * wr_delta);
×
1993
    weight_term += (uint64_t)(wg_delta * wg_delta);
×
1994
    weight_term += (uint64_t)(wb_delta * wb_delta);
×
1995
    rhs = (uint64_t)3 * (uint64_t)size * (uint64_t)size * weight_term;
×
1996

1997
    return delta_sq * delta_sq > rhs;
×
1998
}
1999

2000
static uint32_t
2001
sixel_quant_pool_alloc(sixel_certlut_t *lut, int *status)
×
2002
{
2003
    uint32_t required;
2004
    uint32_t next_capacity;
2005
    uint32_t offset;
2006
    uint8_t *resized;
2007

2008
    offset = 0U;
×
2009
    if (status != NULL) {
×
2010
        *status = SIXEL_FALSE;
×
2011
    }
2012
    required = lut->pool_size + (uint32_t)(8 * sizeof(uint32_t));
×
2013
    if (required > lut->pool_capacity) {
×
2014
        next_capacity = lut->pool_capacity;
×
2015
        if (next_capacity == 0U) {
×
2016
            next_capacity = (uint32_t)(8 * sizeof(uint32_t));
×
2017
        }
2018
        while (next_capacity < required) {
×
2019
            if (next_capacity > UINT32_MAX / 2U) {
×
2020
                return 0U;
×
2021
            }
2022
            next_capacity *= 2U;
×
2023
        }
2024
        resized = (uint8_t *)realloc(lut->pool, next_capacity);
×
2025
        if (resized == NULL) {
×
2026
            return 0U;
×
2027
        }
2028
        lut->pool = resized;
×
2029
        lut->pool_capacity = next_capacity;
×
2030
    }
2031
    offset = lut->pool_size;
×
2032
    memset(lut->pool + offset, 0, 8 * sizeof(uint32_t));
×
2033
    lut->pool_size = required;
×
2034
    if (status != NULL) {
×
2035
        *status = SIXEL_OK;
×
2036
    }
2037

2038
    return offset;
×
2039
}
2040

2041
static void
2042
sixel_quant_assign_leaf(uint32_t *cell, int palette_index)
×
2043
{
2044
    *cell = 0x80000000U | (uint32_t)(palette_index & 0xff);
×
2045
}
×
2046

2047
static void
2048
sixel_quant_assign_branch(uint32_t *cell, uint32_t offset)
×
2049
{
2050
    *cell = SIXEL_CERTLUT_BRANCH_FLAG | (offset & 0x3fffffffU);
×
2051
}
×
2052

2053
static int
2054
sixel_certlut_palette_component(sixel_certlut_t const *lut,
×
2055
                                int index, int axis)
2056
{
2057
    sixel_color_t const *color;
2058

2059
    color = &lut->palette[index];
×
2060
    if (axis == 0) {
×
2061
        return (int)color->r;
×
2062
    }
2063
    if (axis == 1) {
×
2064
        return (int)color->g;
×
2065
    }
2066
    return (int)color->b;
×
2067
}
2068

2069
static void
2070
sixel_certlut_sort_indices(sixel_certlut_t const *lut,
×
2071
                           int *indices, int count, int axis)
2072
{
2073
    int i;
2074
    int j;
2075
    int key;
2076
    int key_value;
2077
    int current_value;
2078

2079
    for (i = 1; i < count; ++i) {
×
2080
        key = indices[i];
×
2081
        key_value = sixel_certlut_palette_component(lut, key, axis);
×
2082
        j = i - 1;
×
2083
        while (j >= 0) {
×
2084
            current_value = sixel_certlut_palette_component(lut,
×
2085
                                                            indices[j],
×
2086
                                                            axis);
2087
            if (current_value <= key_value) {
×
2088
                break;
×
2089
            }
2090
            indices[j + 1] = indices[j];
×
2091
            --j;
×
2092
        }
2093
        indices[j + 1] = key;
×
2094
    }
2095
}
×
2096

2097
static int
2098
sixel_certlut_kdtree_build_recursive(sixel_certlut_t *lut,
×
2099
                                     int *indices,
2100
                                     int count,
2101
                                     int depth)
2102
{
2103
    int axis;
2104
    int median;
2105
    int node_index;
2106

2107
    if (count <= 0) {
×
2108
        return -1;
×
2109
    }
2110

2111
    axis = depth % 3;
×
2112
    sixel_certlut_sort_indices(lut, indices, count, axis);
×
2113
    median = count / 2;
×
2114
    node_index = lut->kdnodes_count;
×
2115
    if (node_index >= lut->ncolors) {
×
2116
        return -1;
×
2117
    }
2118
    lut->kdnodes_count++;
×
2119
    lut->kdnodes[node_index].index = indices[median];
×
2120
    lut->kdnodes[node_index].axis = (unsigned char)axis;
×
2121
    lut->kdnodes[node_index].left =
×
2122
        sixel_certlut_kdtree_build_recursive(lut,
×
2123
                                             indices,
2124
                                             median,
2125
                                             depth + 1);
2126
    lut->kdnodes[node_index].right =
×
2127
        sixel_certlut_kdtree_build_recursive(lut,
×
2128
                                             indices + median + 1,
×
2129
                                             count - median - 1,
×
2130
                                             depth + 1);
2131

2132
    return node_index;
×
2133
}
2134

2135
static SIXELSTATUS
2136
sixel_certlut_kdtree_build(sixel_certlut_t *lut)
×
2137
{
2138
    SIXELSTATUS status;
2139
    int *indices;
2140
    int i;
2141

2142
    status = SIXEL_FALSE;
×
2143
    indices = NULL;
×
2144
    lut->kdnodes = NULL;
×
2145
    lut->kdnodes_count = 0;
×
2146
    lut->kdtree_root = -1;
×
2147
    if (lut->ncolors <= 0) {
×
2148
        status = SIXEL_OK;
×
2149
        goto end;
×
2150
    }
2151
    lut->kdnodes = (sixel_certlut_node_t *)
×
2152
        calloc((size_t)lut->ncolors, sizeof(sixel_certlut_node_t));
×
2153
    if (lut->kdnodes == NULL) {
×
2154
        goto end;
×
2155
    }
2156
    indices = (int *)malloc((size_t)lut->ncolors * sizeof(int));
×
2157
    if (indices == NULL) {
×
2158
        goto end;
×
2159
    }
2160
    for (i = 0; i < lut->ncolors; ++i) {
×
2161
        indices[i] = i;
×
2162
    }
2163
    lut->kdnodes_count = 0;
×
2164
    lut->kdtree_root = sixel_certlut_kdtree_build_recursive(lut,
×
2165
                                                            indices,
2166
                                                            lut->ncolors,
2167
                                                            0);
2168
    if (lut->kdtree_root < 0) {
×
2169
        goto end;
×
2170
    }
2171
    status = SIXEL_OK;
×
2172

2173
end:
2174
    free(indices);
×
2175
    if (SIXEL_FAILED(status)) {
×
2176
        free(lut->kdnodes);
×
2177
        lut->kdnodes = NULL;
×
2178
        lut->kdnodes_count = 0;
×
2179
        lut->kdtree_root = -1;
×
2180
    }
2181

2182
    return status;
×
2183
}
2184

2185
static uint64_t
2186
sixel_certlut_axis_distance(sixel_certlut_t const *lut, int diff, int axis)
×
2187
{
2188
    uint64_t weight;
2189
    uint64_t abs_diff;
2190

2191
    abs_diff = (uint64_t)(diff < 0 ? -diff : diff);
×
2192
    if (axis == 0) {
×
2193
        weight = lut->wR2;
×
2194
    } else if (axis == 1) {
×
2195
        weight = lut->wG2;
×
2196
    } else {
2197
        weight = lut->wB2;
×
2198
    }
2199

2200
    return weight * abs_diff * abs_diff;
×
2201
}
2202

2203
static void
2204
sixel_certlut_consider_candidate(sixel_certlut_t const *lut,
×
2205
                                 int candidate,
2206
                                 int32_t wr_r,
2207
                                 int32_t wg_g,
2208
                                 int32_t wb_b,
2209
                                 int *best_idx,
2210
                                 uint64_t *best_dist,
2211
                                 int *second_idx,
2212
                                 uint64_t *second_dist)
2213
{
2214
    uint64_t distance;
2215

2216
    distance = sixel_certlut_distance_precomputed(lut,
×
2217
                                                  candidate,
2218
                                                  wr_r,
2219
                                                  wg_g,
2220
                                                  wb_b);
2221
    if (distance < *best_dist) {
×
2222
        *second_dist = *best_dist;
×
2223
        *second_idx = *best_idx;
×
2224
        *best_dist = distance;
×
2225
        *best_idx = candidate;
×
2226
    } else if (distance < *second_dist) {
×
2227
        *second_dist = distance;
×
2228
        *second_idx = candidate;
×
2229
    }
2230
}
×
2231

2232
static void
2233
sixel_certlut_kdtree_search(sixel_certlut_t const *lut,
×
2234
                            int node_index,
2235
                            int r,
2236
                            int g,
2237
                            int b,
2238
                            int32_t wr_r,
2239
                            int32_t wg_g,
2240
                            int32_t wb_b,
2241
                            int *best_idx,
2242
                            uint64_t *best_dist,
2243
                            int *second_idx,
2244
                            uint64_t *second_dist)
2245
{
2246
    sixel_certlut_node_t const *node;
2247
    int axis;
2248
    int value;
2249
    int diff;
2250
    int near_child;
2251
    int far_child;
2252
    uint64_t axis_bound;
2253
    int component;
2254

2255
    if (node_index < 0) {
×
2256
        return;
×
2257
    }
2258
    node = &lut->kdnodes[node_index];
×
2259
    sixel_certlut_consider_candidate(lut,
×
2260
                                     node->index,
×
2261
                                     wr_r,
2262
                                     wg_g,
2263
                                     wb_b,
2264
                                     best_idx,
2265
                                     best_dist,
2266
                                     second_idx,
2267
                                     second_dist);
2268

2269
    axis = (int)node->axis;
×
2270
    value = sixel_certlut_palette_component(lut, node->index, axis);
×
2271
    if (axis == 0) {
×
2272
        component = r;
×
2273
    } else if (axis == 1) {
×
2274
        component = g;
×
2275
    } else {
2276
        component = b;
×
2277
    }
2278
    diff = component - value;
×
2279
    if (diff <= 0) {
×
2280
        near_child = node->left;
×
2281
        far_child = node->right;
×
2282
    } else {
2283
        near_child = node->right;
×
2284
        far_child = node->left;
×
2285
    }
2286
    if (near_child >= 0) {
×
2287
        sixel_certlut_kdtree_search(lut,
×
2288
                                    near_child,
2289
                                    r,
2290
                                    g,
2291
                                    b,
2292
                                    wr_r,
2293
                                    wg_g,
2294
                                    wb_b,
2295
                                    best_idx,
2296
                                    best_dist,
2297
                                    second_idx,
2298
                                    second_dist);
2299
    }
2300
    axis_bound = sixel_certlut_axis_distance(lut, diff, axis);
×
2301
    if (far_child >= 0 && axis_bound <= *second_dist) {
×
2302
        sixel_certlut_kdtree_search(lut,
×
2303
                                    far_child,
2304
                                    r,
2305
                                    g,
2306
                                    b,
2307
                                    wr_r,
2308
                                    wg_g,
2309
                                    wb_b,
2310
                                    best_idx,
2311
                                    best_dist,
2312
                                    second_idx,
2313
                                    second_dist);
2314
    }
2315
}
2316

2317
static uint8_t
2318
sixel_certlut_fallback(sixel_certlut_t const *lut, int r, int g, int b)
×
2319
{
2320
    int best_idx;
2321
    int second_idx;
2322
    uint64_t best_dist;
2323
    uint64_t second_dist;
2324

2325
    best_idx = -1;
×
2326
    second_idx = -1;
×
2327
    best_dist = 0U;
×
2328
    second_dist = 0U;
×
2329
    if (lut == NULL) {
×
2330
        return 0U;
×
2331
    }
2332
    /*
2333
     * The lazy builder may fail when allocations run out.  Fall back to a
2334
     * direct brute-force palette search so lookups still succeed even in low
2335
     * memory conditions.
2336
     */
2337
    sixel_quant_distance_pair(lut,
×
2338
                              r,
2339
                              g,
2340
                              b,
2341
                              &best_idx,
2342
                              &second_idx,
2343
                              &best_dist,
2344
                              &second_dist);
2345
    if (best_idx < 0) {
×
2346
        return 0U;
×
2347
    }
2348

2349
    return (uint8_t)best_idx;
×
2350
}
2351

2352
static int
2353
sixel_certlut_build_cell(sixel_certlut_t *lut, uint32_t *cell,
×
2354
                         int rmin, int gmin, int bmin, int size)
2355
{
2356
    SIXELSTATUS status;
2357
    int cr;
2358
    int cg;
2359
    int cb;
2360
    int best_idx;
2361
    int second_idx;
2362
    uint64_t best_dist;
2363
    uint64_t second_dist;
2364
    uint32_t offset;
2365
    int branch_status;
2366
    uint8_t *pool_before;
2367
    size_t pool_size_before;
2368
    uint32_t cell_offset;
2369
    int cell_in_pool;
2370

2371
    if (cell == NULL || lut == NULL) {
×
2372
        status = SIXEL_BAD_ARGUMENT;
×
2373
        goto end;
×
2374
    }
2375
    if (*cell == 0U) {
×
2376
#ifdef DEBUG_CERTLUT_TRACE
2377
        fprintf(stderr,
2378
                "build_cell rmin=%d gmin=%d bmin=%d size=%d\n",
2379
                rmin,
2380
                gmin,
2381
                bmin,
2382
                size);
2383
#endif
2384
    }
2385
    if (*cell != 0U) {
×
2386
        status = SIXEL_OK;
×
2387
        goto end;
×
2388
    }
2389

2390
    /*
2391
     * Each node represents an axis-aligned cube in RGB space.  The builder
2392
     * certifies the dominant palette index by checking the distance gap at
2393
     * the cell center.  When certification fails the cube is split into eight
2394
     * octants backed by a pool block.  Children remain unbuilt until lookups
2395
     * descend into them, keeping the workload proportional to actual queries.
2396
     */
2397
    status = SIXEL_FALSE;
×
2398
    sixel_quant_cell_center(rmin, gmin, bmin, size, &cr, &cg, &cb);
×
2399
    sixel_quant_distance_pair(lut, cr, cg, cb, &best_idx, &second_idx,
×
2400
                              &best_dist, &second_dist);
2401
    if (best_idx < 0) {
×
2402
        best_idx = 0;
×
2403
    }
2404
    if (size == 1) {
×
2405
        sixel_quant_assign_leaf(cell, best_idx);
×
2406
#ifdef DEBUG_CERTLUT_TRACE
2407
        fprintf(stderr,
2408
                "  leaf idx=%d\n",
2409
                best_idx);
2410
#endif
2411
        status = SIXEL_OK;
×
2412
        goto end;
×
2413
    }
2414
    if (sixel_quant_is_cell_safe(lut, best_idx, second_idx, size,
×
2415
                                 best_dist, second_dist)) {
2416
        sixel_quant_assign_leaf(cell, best_idx);
×
2417
#ifdef DEBUG_CERTLUT_TRACE
2418
        fprintf(stderr,
2419
                "  safe leaf idx=%d\n",
2420
                best_idx);
2421
#endif
2422
        status = SIXEL_OK;
×
2423
        goto end;
×
2424
    }
2425
    pool_before = lut->pool;
×
2426
    pool_size_before = lut->pool_size;
×
2427
    cell_in_pool = 0;
×
2428
    cell_offset = 0U;
×
2429
    /*
2430
     * The pool may grow while building descendants.  Remember the caller's
2431
     * offset so the cell pointer can be refreshed after realloc moves the
2432
     * backing storage.
2433
     */
2434
    if (pool_before != NULL) {
×
2435
        if ((uint8_t *)(void *)cell >= pool_before
×
2436
                && (size_t)((uint8_t *)(void *)cell - pool_before)
×
2437
                        < pool_size_before) {
2438
            cell_in_pool = 1;
×
2439
            cell_offset = (uint32_t)((uint8_t *)(void *)cell - pool_before);
×
2440
        }
2441
    }
2442
    offset = sixel_quant_pool_alloc(lut, &branch_status);
×
2443
    if (branch_status != SIXEL_OK) {
×
2444
        goto end;
×
2445
    }
2446
    if (cell_in_pool != 0) {
×
2447
        cell = (uint32_t *)(void *)(lut->pool + cell_offset);
×
2448
    }
2449
    sixel_quant_assign_branch(cell, offset);
×
2450
#ifdef DEBUG_CERTLUT_TRACE
2451
    fprintf(stderr,
2452
            "  branch offset=%u\n",
2453
            offset);
2454
#endif
2455
    status = SIXEL_OK;
×
2456

2457
end:
2458
    return status;
×
2459
}
2460

2461
static int
2462
sixel_certlut_build(sixel_certlut_t *lut, sixel_color_t const *palette,
×
2463
                    int ncolors, int wR, int wG, int wB)
2464
{
2465
    SIXELSTATUS status;
2466
    int initialized;
2467
    size_t level0_count;
2468
    status = SIXEL_FALSE;
×
2469
    initialized = sixel_certlut_init(lut);
×
2470
    if (SIXEL_FAILED(initialized)) {
×
2471
        goto end;
×
2472
    }
2473
    lut->palette = palette;
×
2474
    lut->ncolors = ncolors;
×
2475
    sixel_quant_weight_init(lut, wR, wG, wB);
×
2476
    status = sixel_certlut_prepare_palette_terms(lut);
×
2477
    if (SIXEL_FAILED(status)) {
×
2478
        goto end;
×
2479
    }
2480
    status = sixel_certlut_kdtree_build(lut);
×
2481
    if (SIXEL_FAILED(status)) {
×
2482
        goto end;
×
2483
    }
2484
    level0_count = (size_t)64 * (size_t)64 * (size_t)64;
×
2485
    lut->level0 = (uint32_t *)calloc(level0_count, sizeof(uint32_t));
×
2486
    if (lut->level0 == NULL) {
×
2487
        goto end;
×
2488
    }
2489
    /*
2490
     * Level 0 cells start uninitialized.  The lookup routine materializes
2491
     * individual subtrees on demand so we avoid evaluating the entire
2492
     * 64x64x64 grid upfront.
2493
     */
2494
    status = SIXEL_OK;
×
2495

2496
end:
2497
    if (SIXEL_FAILED(status)) {
×
2498
        sixel_certlut_release(lut);
×
2499
    }
2500
    return status;
×
2501
}
2502

2503
static uint8_t
2504
sixel_certlut_lookup(sixel_certlut_t *lut, uint8_t r, uint8_t g, uint8_t b)
×
2505
{
2506
    uint32_t entry;
2507
    uint32_t offset;
2508
    uint32_t index;
2509
    uint32_t *children;
2510
    uint32_t *cell;
2511
    int shift;
2512
    int child;
2513
    int status;
2514
    int size;
2515
    int rmin;
2516
    int gmin;
2517
    int bmin;
2518
    int step;
2519
    if (lut == NULL || lut->level0 == NULL) {
×
2520
        return 0U;
×
2521
    }
2522
    /*
2523
     * Cells are created lazily.  A zero entry indicates an uninitialized
2524
     * subtree, so the builder is invoked with the cube bounds of the current
2525
     * traversal.  Should allocation fail we fall back to a direct brute-force
2526
     * palette search for the queried pixel.
2527
     */
2528
    index = ((uint32_t)(r >> 2) << 12)
×
2529
          | ((uint32_t)(g >> 2) << 6)
×
2530
          | (uint32_t)(b >> 2);
×
2531
    cell = lut->level0 + index;
×
2532
    size = 4;
×
2533
    rmin = (int)(r & 0xfc);
×
2534
    gmin = (int)(g & 0xfc);
×
2535
    bmin = (int)(b & 0xfc);
×
2536
    entry = *cell;
×
2537
    if (entry == 0U) {
×
2538
#ifdef DEBUG_CERTLUT_TRACE
2539
        fprintf(stderr,
2540
                "lookup build level0 r=%u g=%u b=%u\n",
2541
                (unsigned int)r,
2542
                (unsigned int)g,
2543
                (unsigned int)b);
2544
#endif
2545
        status = sixel_certlut_build_cell(lut, cell, rmin, gmin, bmin, size);
×
2546
        if (SIXEL_FAILED(status)) {
×
2547
            return sixel_certlut_fallback(lut,
×
2548
                                          (int)r,
2549
                                          (int)g,
2550
                                          (int)b);
2551
        }
2552
        entry = *cell;
×
2553
    }
2554
    shift = 1;
×
2555
    while ((entry & 0x80000000U) == 0U) {
×
2556
        offset = entry & 0x3fffffffU;
×
2557
        children = (uint32_t *)(void *)(lut->pool + offset);
×
2558
        child = (((int)(r >> shift) & 1) << 2)
×
2559
              | (((int)(g >> shift) & 1) << 1)
×
2560
              | ((int)(b >> shift) & 1);
×
2561
#ifdef DEBUG_CERTLUT_TRACE
2562
        fprintf(stderr,
2563
                "descend child=%d size=%d offset=%u\n",
2564
                child,
2565
                size,
2566
                offset);
2567
#endif
2568
        step = size / 2;
×
2569
        if (step <= 0) {
×
2570
            step = 1;
×
2571
        }
2572
        rmin += step * ((child >> 2) & 1);
×
2573
        gmin += step * ((child >> 1) & 1);
×
2574
        bmin += step * (child & 1);
×
2575
        size = step;
×
2576
        cell = children + (size_t)child;
×
2577
        entry = *cell;
×
2578
        if (entry == 0U) {
×
2579
#ifdef DEBUG_CERTLUT_TRACE
2580
            fprintf(stderr,
2581
                    "lookup build child size=%d rmin=%d gmin=%d bmin=%d\n",
2582
                    size,
2583
                    rmin,
2584
                    gmin,
2585
                    bmin);
2586
#endif
2587
            status = sixel_certlut_build_cell(lut,
×
2588
                                              cell,
2589
                                              rmin,
2590
                                              gmin,
2591
                                              bmin,
2592
                                              size);
2593
            if (SIXEL_FAILED(status)) {
×
2594
                return sixel_certlut_fallback(lut,
×
2595
                                              (int)r,
2596
                                              (int)g,
2597
                                              (int)b);
2598
            }
2599
            children = (uint32_t *)(void *)(lut->pool + offset);
×
2600
            cell = children + (size_t)child;
×
2601
            entry = *cell;
×
2602
        }
2603
        if (size == 1) {
×
2604
            break;
×
2605
        }
2606
        if (shift == 0) {
×
2607
            break;
×
2608
        }
2609
        --shift;
×
2610
    }
2611

2612
    return (uint8_t)(entry & 0xffU);
×
2613
}
2614

2615
static void
2616
sixel_certlut_free(sixel_certlut_t *lut)
×
2617
{
2618
    sixel_certlut_release(lut);
×
2619
    if (lut != NULL) {
×
2620
        lut->palette = NULL;
×
2621
        lut->ncolors = 0;
×
2622
    }
2623
}
×
2624

2625
static size_t
2626
robinhood_round_capacity(size_t hint)
×
2627
{
2628
    size_t capacity;
2629

2630
    capacity = 16U;
×
2631
    if (hint < capacity) {
×
2632
        return capacity;
×
2633
    }
2634

2635
    capacity = hint - 1U;
×
2636
    capacity |= capacity >> 1;
×
2637
    capacity |= capacity >> 2;
×
2638
    capacity |= capacity >> 4;
×
2639
    capacity |= capacity >> 8;
×
2640
    capacity |= capacity >> 16;
×
2641
#if SIZE_MAX > UINT32_MAX
2642
    capacity |= capacity >> 32;
×
2643
#endif
2644
    if (capacity == SIZE_MAX) {
×
2645
        return SIZE_MAX;
×
2646
    }
2647
    capacity++;
×
2648
    if (capacity < 16U) {
×
2649
        capacity = 16U;
×
2650
    }
2651

2652
    return capacity;
×
2653
}
2654

2655
static SIXELSTATUS
2656
robinhood_table32_init(struct robinhood_table32 *table,
×
2657
                       size_t expected,
2658
                       sixel_allocator_t *allocator)
2659
{
2660
    size_t hint;
2661
    size_t capacity;
2662

2663
    table->slots = NULL;
×
2664
    table->capacity = 0U;
×
2665
    table->count = 0U;
×
2666
    table->allocator = allocator;
×
2667

2668
    if (expected < 16U) {
×
2669
        expected = 16U;
×
2670
    }
2671
    if (expected > SIZE_MAX / 2U) {
×
2672
        hint = SIZE_MAX / 2U;
×
2673
    } else {
2674
        hint = expected * 2U;
×
2675
    }
2676
    capacity = robinhood_round_capacity(hint);
×
2677
    if (capacity == SIZE_MAX && hint != SIZE_MAX) {
×
2678
        return SIXEL_BAD_ALLOCATION;
×
2679
    }
2680

2681
    table->slots = (struct robinhood_slot32 *)
×
2682
        sixel_allocator_calloc(allocator,
×
2683
                               capacity,
2684
                               sizeof(struct robinhood_slot32));
2685
    if (table->slots == NULL) {
×
2686
        table->capacity = 0U;
×
2687
        table->count = 0U;
×
2688
        return SIXEL_BAD_ALLOCATION;
×
2689
    }
2690
    table->capacity = capacity;
×
2691
    table->count = 0U;
×
2692

2693
    return SIXEL_OK;
×
2694
}
2695

2696
static void
2697
robinhood_table32_fini(struct robinhood_table32 *table)
×
2698
{
2699
    if (table->slots != NULL) {
×
2700
        sixel_allocator_free(table->allocator, table->slots);
×
2701
        table->slots = NULL;
×
2702
    }
2703
    table->capacity = 0U;
×
2704
    table->count = 0U;
×
2705
    table->allocator = NULL;
×
2706
}
×
2707

2708
static struct robinhood_slot32 *
2709
robinhood_table32_lookup(struct robinhood_table32 *table,
×
2710
                         uint32_t key,
2711
                         uint32_t color)
2712
{
2713
    size_t mask;
2714
    size_t index;
2715
    uint16_t distance;
2716
    struct robinhood_slot32 *slot;
2717

2718
    if (table->capacity == 0U || table->slots == NULL) {
×
2719
        return NULL;
×
2720
    }
2721

2722
    mask = table->capacity - 1U;
×
2723
    index = (size_t)(key & mask);
×
2724
    distance = 0U;
×
2725

2726
    for (;;) {
2727
        slot = &table->slots[index];
×
2728
        if (slot->value == 0U) {
×
2729
            return NULL;
×
2730
        }
2731
        if (slot->key == key && slot->color == color) {
×
2732
            return slot;
×
2733
        }
2734
        if (slot->distance < distance) {
×
2735
            return NULL;
×
2736
        }
2737
        index = (index + 1U) & mask;
×
2738
        distance++;
×
2739
    }
2740
}
2741

2742
static struct robinhood_slot32 *
2743
robinhood_table32_place(struct robinhood_table32 *table,
×
2744
                        struct robinhood_slot32 entry)
2745
{
2746
    size_t mask;
2747
    size_t index;
2748
    struct robinhood_slot32 *slot;
2749
    struct robinhood_slot32 tmp;
2750

2751
    mask = table->capacity - 1U;
×
2752
    index = (size_t)(entry.key & mask);
×
2753

2754
    for (;;) {
2755
        slot = &table->slots[index];
×
2756
        if (slot->value == 0U) {
×
2757
            *slot = entry;
×
2758
            table->count++;
×
2759
            return slot;
×
2760
        }
2761
        if (slot->key == entry.key && slot->color == entry.color) {
×
2762
            slot->value = entry.value;
×
2763
            return slot;
×
2764
        }
2765
        if (slot->distance < entry.distance) {
×
2766
            tmp = *slot;
×
2767
            *slot = entry;
×
2768
            entry = tmp;
×
2769
        }
2770
        index = (index + 1U) & mask;
×
2771
        entry.distance++;
×
2772
    }
2773
}
2774

2775
static SIXELSTATUS
2776
robinhood_table32_grow(struct robinhood_table32 *table)
×
2777
{
2778
    struct robinhood_slot32 *old_slots;
2779
    size_t old_capacity;
2780
    size_t new_capacity;
2781
    size_t i;
2782

2783
    if (table->allocator == NULL) {
×
2784
        return SIXEL_BAD_ALLOCATION;
×
2785
    }
2786
    if (table->capacity == 0U) {
×
2787
        new_capacity = 16U;
×
2788
    } else {
2789
        if (table->capacity > SIZE_MAX / 2U) {
×
2790
            return SIXEL_BAD_ALLOCATION;
×
2791
        }
2792
        new_capacity = table->capacity << 1U;
×
2793
    }
2794
    new_capacity = robinhood_round_capacity(new_capacity);
×
2795
    if (new_capacity <= table->capacity) {
×
2796
        return SIXEL_BAD_ALLOCATION;
×
2797
    }
2798

2799
    old_slots = table->slots;
×
2800
    old_capacity = table->capacity;
×
2801
    table->slots = (struct robinhood_slot32 *)
×
2802
        sixel_allocator_calloc(table->allocator,
×
2803
                               new_capacity,
2804
                               sizeof(struct robinhood_slot32));
2805
    if (table->slots == NULL) {
×
2806
        table->slots = old_slots;
×
2807
        table->capacity = old_capacity;
×
2808
        return SIXEL_BAD_ALLOCATION;
×
2809
    }
2810
    table->capacity = new_capacity;
×
2811
    table->count = 0U;
×
2812

2813
    for (i = 0U; i < old_capacity; ++i) {
×
2814
        struct robinhood_slot32 entry;
2815

2816
        if (old_slots[i].value == 0U) {
×
2817
            continue;
×
2818
        }
2819
        entry.key = old_slots[i].key;
×
2820
        entry.color = old_slots[i].color;
×
2821
        entry.value = old_slots[i].value;
×
2822
        entry.distance = 0U;
×
2823
        entry.pad = 0U;  /* ensure padding bytes are initialized */
×
2824
        (void)robinhood_table32_place(table, entry);
×
2825
    }
2826

2827
    sixel_allocator_free(table->allocator, old_slots);
×
2828

2829
    return SIXEL_OK;
×
2830
}
2831

2832
static SIXELSTATUS
2833
robinhood_table32_insert(struct robinhood_table32 *table,
×
2834
                         uint32_t key,
2835
                         uint32_t color,
2836
                         uint32_t value)
2837
{
2838
    SIXELSTATUS status;
2839
    struct robinhood_slot32 entry;
2840

2841
    if (table->slots == NULL || table->capacity == 0U) {
×
2842
        return SIXEL_BAD_ARGUMENT;
×
2843
    }
2844
    if (table->count * 2U >= table->capacity) {
×
2845
        status = robinhood_table32_grow(table);
×
2846
        if (SIXEL_FAILED(status)) {
×
2847
            return status;
×
2848
        }
2849
    }
2850

2851
    entry.key = key;
×
2852
    entry.color = color;
×
2853
    entry.value = value;
×
2854
    entry.distance = 0U;
×
2855
    entry.pad = 0U;  /* ensure padding bytes are initialized */
×
2856
    (void)robinhood_table32_place(table, entry);
×
2857

2858
    return SIXEL_OK;
×
2859
}
2860

2861
static SIXELSTATUS
2862
hopscotch_table32_init(struct hopscotch_table32 *table,
×
2863
                       size_t expected,
2864
                       sixel_allocator_t *allocator)
2865
{
2866
    size_t hint;
2867
    size_t capacity;
2868
    size_t i;
2869

2870
    if (table == NULL) {
×
2871
        return SIXEL_BAD_ARGUMENT;
×
2872
    }
2873

2874
    table->slots = NULL;
×
2875
    table->hopinfo = NULL;
×
2876
    table->capacity = 0U;
×
2877
    table->count = 0U;
×
2878
    table->neighborhood = HOPSCOTCH_DEFAULT_NEIGHBORHOOD;
×
2879
    table->allocator = allocator;
×
2880

2881
    if (expected < 16U) {
×
2882
        expected = 16U;
×
2883
    }
2884
    if (expected > SIZE_MAX / 2U) {
×
2885
        hint = SIZE_MAX / 2U;
×
2886
    } else {
2887
        hint = expected * 2U;
×
2888
    }
2889
    capacity = robinhood_round_capacity(hint);
×
2890
    if (capacity == SIZE_MAX && hint != SIZE_MAX) {
×
2891
        return SIXEL_BAD_ALLOCATION;
×
2892
    }
2893
    if (capacity < table->neighborhood) {
×
2894
        capacity = table->neighborhood;
×
2895
    }
2896

2897
    table->slots = (struct hopscotch_slot32 *)
×
2898
        sixel_allocator_malloc(allocator,
×
2899
                               capacity * sizeof(struct hopscotch_slot32));
2900
    if (table->slots == NULL) {
×
2901
        return SIXEL_BAD_ALLOCATION;
×
2902
    }
2903
    table->hopinfo = (uint32_t *)
×
2904
        sixel_allocator_calloc(allocator,
×
2905
                               capacity,
2906
                               sizeof(uint32_t));
2907
    if (table->hopinfo == NULL) {
×
2908
        sixel_allocator_free(allocator, table->slots);
×
2909
        table->slots = NULL;
×
2910
        return SIXEL_BAD_ALLOCATION;
×
2911
    }
2912

2913
    for (i = 0U; i < capacity; ++i) {
×
2914
        table->slots[i].key = HOPSCOTCH_EMPTY_KEY;
×
2915
        table->slots[i].color = 0U;
×
2916
        table->slots[i].value = 0U;
×
2917
    }
2918
    table->capacity = capacity;
×
2919
    table->count = 0U;
×
2920
    if (table->neighborhood > 32U) {
×
2921
        table->neighborhood = 32U;
×
2922
    }
2923
    if (table->neighborhood > table->capacity) {
×
2924
        table->neighborhood = table->capacity;
×
2925
    }
2926

2927
    return SIXEL_OK;
×
2928
}
2929

2930
static void
2931
hopscotch_table32_fini(struct hopscotch_table32 *table)
×
2932
{
2933
    sixel_allocator_t *allocator;
2934

2935
    if (table == NULL) {
×
2936
        return;
×
2937
    }
2938

2939
    allocator = table->allocator;
×
2940
    if (allocator != NULL) {
×
2941
        if (table->slots != NULL) {
×
2942
            sixel_allocator_free(allocator, table->slots);
×
2943
        }
2944
        if (table->hopinfo != NULL) {
×
2945
            sixel_allocator_free(allocator, table->hopinfo);
×
2946
        }
2947
    }
2948
    table->slots = NULL;
×
2949
    table->hopinfo = NULL;
×
2950
    table->capacity = 0U;
×
2951
    table->count = 0U;
×
2952
}
2953

2954
static struct hopscotch_slot32 *
2955
hopscotch_table32_lookup(struct hopscotch_table32 *table,
×
2956
                         uint32_t key,
2957
                         uint32_t color)
2958
{
2959
    size_t index;
2960
    size_t bit;
2961
    size_t candidate;
2962
    uint32_t hop;
2963
    size_t mask;
2964
    size_t neighborhood;
2965

2966
    if (table == NULL || table->slots == NULL || table->hopinfo == NULL) {
×
2967
        return NULL;
×
2968
    }
2969
    if (table->capacity == 0U) {
×
2970
        return NULL;
×
2971
    }
2972

2973
    mask = table->capacity - 1U;
×
2974
    index = (size_t)key & mask;
×
2975
    hop = table->hopinfo[index];
×
2976
    neighborhood = table->neighborhood;
×
2977
    for (bit = 0U; bit < neighborhood; ++bit) {
×
2978
        if ((hop & (1U << bit)) == 0U) {
×
2979
            continue;
×
2980
        }
2981
        candidate = (index + bit) & mask;
×
2982
        if (table->slots[candidate].key == key
×
2983
            && table->slots[candidate].color == color) {
×
2984
            return &table->slots[candidate];
×
2985
        }
2986
    }
2987

2988
    return NULL;
×
2989
}
2990

2991
static SIXELSTATUS
2992
hopscotch_table32_grow(struct hopscotch_table32 *table)
×
2993
{
2994
    SIXELSTATUS status;
2995
    struct hopscotch_table32 tmp;
2996
    size_t i;
2997
    struct hopscotch_slot32 *slot;
2998

2999
    tmp.slots = NULL;
×
3000
    tmp.hopinfo = NULL;
×
3001
    tmp.capacity = 0U;
×
3002
    tmp.count = 0U;
×
3003
    tmp.neighborhood = table->neighborhood;
×
3004
    tmp.allocator = table->allocator;
×
3005

3006
    status = hopscotch_table32_init(&tmp,
×
3007
                                    table->capacity * 2U,
×
3008
                                    table->allocator);
3009
    if (SIXEL_FAILED(status)) {
×
3010
        return status;
×
3011
    }
3012
    if (tmp.neighborhood > table->neighborhood) {
×
3013
        tmp.neighborhood = table->neighborhood;
×
3014
    }
3015

3016
    for (i = 0U; i < table->capacity; ++i) {
×
3017
        slot = &table->slots[i];
×
3018
        if (slot->key == HOPSCOTCH_EMPTY_KEY) {
×
3019
            continue;
×
3020
        }
3021
        status = hopscotch_table32_insert(&tmp,
×
3022
                                          slot->key,
3023
                                          slot->color,
3024
                                          slot->value);
3025
        if (SIXEL_FAILED(status)) {
×
3026
            hopscotch_table32_fini(&tmp);
×
3027
            return status;
×
3028
        }
3029
    }
3030

3031
    hopscotch_table32_fini(table);
×
3032

3033
    table->slots = tmp.slots;
×
3034
    table->hopinfo = tmp.hopinfo;
×
3035
    table->capacity = tmp.capacity;
×
3036
    table->count = tmp.count;
×
3037
    table->neighborhood = tmp.neighborhood;
×
3038
    table->allocator = tmp.allocator;
×
3039

3040
    return SIXEL_OK;
×
3041
}
3042

3043
static SIXELSTATUS
3044
hopscotch_table32_insert(struct hopscotch_table32 *table,
×
3045
                         uint32_t key,
3046
                         uint32_t color,
3047
                         uint32_t value)
3048
{
3049
    SIXELSTATUS status;
3050
    struct hopscotch_slot32 *slot;
3051
    size_t index;
3052
    size_t mask;
3053
    size_t distance;
3054
    size_t attempts;
3055
    size_t free_index;
3056
    size_t neighborhood;
3057
    int relocated;
3058
    size_t offset;
3059
    uint32_t hop;
3060
    size_t bit;
3061
    size_t move_index;
3062
    struct hopscotch_slot32 tmp_slot;
3063

3064
    if (table == NULL || table->slots == NULL || table->hopinfo == NULL) {
×
3065
        return SIXEL_BAD_ARGUMENT;
×
3066
    }
3067
    if (table->capacity == 0U) {
×
3068
        return SIXEL_BAD_ARGUMENT;
×
3069
    }
3070

3071
    slot = hopscotch_table32_lookup(table, key, color);
×
3072
    if (slot != NULL) {
×
3073
        slot->value = value;
×
3074
        return SIXEL_OK;
×
3075
    }
3076

3077
    if (table->count * 2U >= table->capacity) {
×
3078
        status = hopscotch_table32_grow(table);
×
3079
        if (SIXEL_FAILED(status)) {
×
3080
            return status;
×
3081
        }
3082
        return hopscotch_table32_insert(table, key, color, value);
×
3083
    }
3084

3085
    mask = table->capacity - 1U;
×
3086
    neighborhood = table->neighborhood;
×
3087
    index = (size_t)key & mask;
×
3088
    slot = NULL;
×
3089
    free_index = index;
×
3090
    distance = 0U;
×
3091
    for (attempts = 0U; attempts < HOPSCOTCH_INSERT_RANGE; ++attempts) {
×
3092
        free_index = (index + attempts) & mask;
×
3093
        slot = &table->slots[free_index];
×
3094
        if (slot->key == HOPSCOTCH_EMPTY_KEY) {
×
3095
            distance = attempts;
×
3096
            break;
×
3097
        }
3098
    }
3099
    if (slot == NULL || slot->key != HOPSCOTCH_EMPTY_KEY) {
×
3100
        status = hopscotch_table32_grow(table);
×
3101
        if (SIXEL_FAILED(status)) {
×
3102
            return status;
×
3103
        }
3104
        return hopscotch_table32_insert(table, key, color, value);
×
3105
    }
3106

3107
    /*
3108
     * Relocation diagram:
3109
     *
3110
     *   free slot <--- hop window <--- [base bucket]
3111
     *      ^ move resident outward until distance < neighborhood
3112
     */
3113
    while (distance >= neighborhood) {
×
3114
        relocated = 0;
×
3115
        for (offset = neighborhood - 1U; offset > 0U; --offset) {
×
3116
            size_t base;
3117

3118
            base = (free_index + table->capacity - offset) & mask;
×
3119
            hop = table->hopinfo[base];
×
3120
            if (hop == 0U) {
×
3121
                continue;
×
3122
            }
3123
            for (bit = 0U; bit < offset; ++bit) {
×
3124
                if ((hop & (1U << bit)) == 0U) {
×
3125
                    continue;
×
3126
                }
3127
                move_index = (base + bit) & mask;
×
3128
                tmp_slot = table->slots[move_index];
×
3129
                table->slots[free_index] = tmp_slot;
×
3130
                table->slots[move_index].key = HOPSCOTCH_EMPTY_KEY;
×
3131
                table->slots[move_index].color = 0U;
×
3132
                table->slots[move_index].value = 0U;
×
3133
                table->hopinfo[base] &= (uint32_t)~(1U << bit);
×
3134
                table->hopinfo[base] |= (uint32_t)(1U << offset);
×
3135
                free_index = move_index;
×
3136
                if (free_index >= index) {
×
3137
                    distance = free_index - index;
×
3138
                } else {
3139
                    distance = free_index + table->capacity - index;
×
3140
                }
3141
                relocated = 1;
×
3142
                break;
×
3143
            }
3144
            if (relocated) {
×
3145
                break;
×
3146
            }
3147
        }
3148
        if (!relocated) {
×
3149
            status = hopscotch_table32_grow(table);
×
3150
            if (SIXEL_FAILED(status)) {
×
3151
                return status;
×
3152
            }
3153
            return hopscotch_table32_insert(table, key, color, value);
×
3154
        }
3155
    }
3156

3157
    if (distance >= 32U) {
×
3158
        status = hopscotch_table32_grow(table);
×
3159
        if (SIXEL_FAILED(status)) {
×
3160
            return status;
×
3161
        }
3162
        return hopscotch_table32_insert(table, key, color, value);
×
3163
    }
3164

3165
    table->slots[free_index].key = key;
×
3166
    table->slots[free_index].color = color;
×
3167
    table->slots[free_index].value = value;
×
3168
    table->hopinfo[index] |= (uint32_t)(1U << distance);
×
3169
    table->count++;
×
3170

3171
    return SIXEL_OK;
×
3172
}
3173

3174
/*
3175
 * The cuckoo hash backend stores entries in fixed-width buckets.
3176
 *
3177
 *   [bucket 0] -> key0 key1 key2 key3
3178
 *                 val0 val1 val2 val3
3179
 *   [bucket 1] -> ...
3180
 *
3181
 * Each key is compared against the 128-bit lane using SIMD instructions.
3182
 * Two hash functions map a key to its primary and secondary buckets.  When
3183
 * both are full, the eviction loop "kicks" an entry toward its alternate
3184
 * bucket, as illustrated below:
3185
 *
3186
 *   bucket A --kick--> bucket B --kick--> bucket A ...
3187
 *
3188
 * A tiny stash buffers entries when the table momentarily fills up.  This
3189
 * keeps lookups fast while letting us grow the table lazily.
3190
 */
3191
static size_t
3192
cuckoo_round_buckets(size_t hint)
264✔
3193
{
3194
    size_t desired;
3195
    size_t buckets;
3196
    size_t prev;
3197

3198
    if (hint < CUCKOO_BUCKET_SIZE) {
264!
3199
        hint = CUCKOO_BUCKET_SIZE;
×
3200
    }
3201
    if (hint > SIZE_MAX / 2U) {
264!
3202
        hint = SIZE_MAX / 2U;
×
3203
    }
3204
    desired = (hint * 2U + CUCKOO_BUCKET_SIZE - 1U) / CUCKOO_BUCKET_SIZE;
264✔
3205
    if (desired == 0U) {
264!
3206
        desired = 1U;
×
3207
    }
3208

3209
    buckets = 1U;
264✔
3210
    while (buckets < desired) {
4,752✔
3211
        prev = buckets;
4,488✔
3212
        if (buckets > SIZE_MAX / 2U) {
4,488!
3213
            buckets = prev;
×
3214
            break;
×
3215
        }
3216
        buckets <<= 1U;
4,488✔
3217
        if (buckets < prev) {
4,488!
3218
            buckets = prev;
×
3219
            break;
×
3220
        }
3221
    }
3222
    if (buckets == 0U) {
264!
3223
        buckets = 1U;
×
3224
    }
3225

3226
    return buckets;
264✔
3227
}
3228

3229
static size_t
3230
cuckoo_hash_primary(uint32_t key, size_t mask)
43,908,960✔
3231
{
3232
    uint32_t mix;
3233

3234
    mix = key * 0x9e3779b1U;
43,908,960✔
3235
    return (size_t)(mix & (uint32_t)mask);
43,908,960✔
3236
}
3237

3238
static size_t
3239
cuckoo_hash_secondary(uint32_t key, size_t mask)
3,837,993✔
3240
{
3241
    uint32_t mix;
3242

3243
    mix = key ^ 0x85ebca6bU;
3,837,993✔
3244
    mix ^= mix >> 13;
3,837,993✔
3245
    mix *= 0xc2b2ae35U;
3,837,993✔
3246
    return (size_t)(mix & (uint32_t)mask);
3,837,993✔
3247
}
3248

3249
static size_t
3250
cuckoo_hash_alternate(uint32_t key, size_t bucket, size_t mask)
52✔
3251
{
3252
    size_t primary;
3253
    size_t secondary;
3254

3255
    primary = cuckoo_hash_primary(key, mask);
52✔
3256
    secondary = cuckoo_hash_secondary(key, mask);
52✔
3257
    if (primary == bucket) {
52!
3258
        if (secondary == primary) {
52!
3259
            secondary = (secondary + 1U) & mask;
×
3260
        }
3261
        return secondary;
52✔
3262
    }
3263

3264
    return primary;
×
3265
}
10✔
3266

3267
static uint32_t *
3268
cuckoo_bucket_find(struct cuckoo_bucket32 *bucket, uint32_t key)
45,831,374✔
3269
{
3270
    size_t i;
3271

3272
#if defined(SIXEL_USE_SSE2)
3273
    __m128i needle;
3274
    __m128i keys;
3275
    __m128i cmp;
3276
    int mask;
3277

3278
    needle = _mm_set1_epi32((int)key);
3279
    keys = _mm_loadu_si128((const __m128i *)bucket->key);
3280
    cmp = _mm_cmpeq_epi32(needle, keys);
3281
    mask = _mm_movemask_ps(_mm_castsi128_ps(cmp));
3282
    if ((mask & 1) != 0 && bucket->value[0] != 0U) {
3283
        return &bucket->value[0];
3284
    }
3285
    if ((mask & 2) != 0 && bucket->value[1] != 0U) {
3286
        return &bucket->value[1];
3287
    }
3288
    if ((mask & 4) != 0 && bucket->value[2] != 0U) {
3289
        return &bucket->value[2];
3290
    }
3291
    if ((mask & 8) != 0 && bucket->value[3] != 0U) {
3292
        return &bucket->value[3];
3293
    }
3294
#elif defined(SIXEL_USE_NEON)
3295
    uint32x4_t needle;
3296
    uint32x4_t keys;
3297
    uint32x4_t cmp;
3298

3299
    needle = vdupq_n_u32(key);
3300
    keys = vld1q_u32(bucket->key);
3301
    cmp = vceqq_u32(needle, keys);
3302
    if (vgetq_lane_u32(cmp, 0) != 0U && bucket->value[0] != 0U) {
3303
        return &bucket->value[0];
3304
    }
3305
    if (vgetq_lane_u32(cmp, 1) != 0U && bucket->value[1] != 0U) {
3306
        return &bucket->value[1];
3307
    }
3308
    if (vgetq_lane_u32(cmp, 2) != 0U && bucket->value[2] != 0U) {
3309
        return &bucket->value[2];
3310
    }
3311
    if (vgetq_lane_u32(cmp, 3) != 0U && bucket->value[3] != 0U) {
3312
        return &bucket->value[3];
3313
    }
3314
#else
3315
    for (i = 0U; i < CUCKOO_BUCKET_SIZE; ++i) {
77,866,608✔
3316
        if (bucket->value[i] != 0U && bucket->key[i] == key) {
70,197,717✔
3317
            return &bucket->value[i];
38,162,483✔
3318
        }
3319
    }
10,249,102✔
3320
#endif
3321

3322
    return NULL;
7,668,891✔
3323
}
20,846,518✔
3324

3325
static int
3326
cuckoo_bucket_insert_direct(struct cuckoo_bucket32 *bucket,
1,915,527✔
3327
                            uint32_t key,
3328
                            uint32_t value)
3329
{
3330
    size_t i;
3331

3332
    for (i = 0U; i < CUCKOO_BUCKET_SIZE; ++i) {
2,112,289✔
3333
        if (bucket->value[i] == 0U) {
2,112,237✔
3334
            bucket->key[i] = key;
1,915,475✔
3335
            bucket->value[i] = value;
1,915,475✔
3336
            return 1;
1,915,475✔
3337
        }
3338
    }
59,286✔
3339

3340
    return 0;
52✔
3341
}
613,553✔
3342

3343
static SIXELSTATUS
3344
cuckoo_table32_init(struct cuckoo_table32 *table,
261✔
3345
                    size_t expected,
3346
                    sixel_allocator_t *allocator)
3347
{
3348
    size_t buckets;
3349
    size_t i;
3350
    size_t j;
3351

3352
    if (table == NULL || allocator == NULL) {
261!
3353
        return SIXEL_BAD_ARGUMENT;
×
3354
    }
3355

3356
    buckets = cuckoo_round_buckets(expected);
261✔
3357
    if (buckets == 0U
261!
3358
        || buckets > SIZE_MAX / sizeof(struct cuckoo_bucket32)) {
261!
3359
        sixel_helper_set_additional_message(
×
3360
            "unable to size cuckoo bucket array.");
3361
        return SIXEL_BAD_ALLOCATION;
×
3362
    }
3363

3364
    table->buckets = (struct cuckoo_bucket32 *)sixel_allocator_malloc(
261✔
3365
        allocator, buckets * sizeof(struct cuckoo_bucket32));
107✔
3366
    if (table->buckets == NULL) {
261!
3367
        sixel_helper_set_additional_message(
×
3368
            "unable to allocate cuckoo buckets.");
3369
        return SIXEL_BAD_ALLOCATION;
×
3370
    }
3371

3372
    table->bucket_count = buckets;
261✔
3373
    table->bucket_mask = buckets - 1U;
261✔
3374
    table->stash_count = 0U;
261✔
3375
    table->entry_count = 0U;
261✔
3376
    table->allocator = allocator;
261✔
3377
    for (i = 0U; i < buckets; ++i) {
34,210,053✔
3378
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
171,048,960✔
3379
            table->buckets[i].key[j] = CUCKOO_EMPTY_KEY;
136,839,168✔
3380
            table->buckets[i].value[j] = 0U;
136,839,168✔
3381
        }
56,098,816✔
3382
    }
14,024,704✔
3383
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
8,613✔
3384
        table->stash_key[i] = CUCKOO_EMPTY_KEY;
8,352✔
3385
        table->stash_value[i] = 0U;
8,352✔
3386
    }
3,424✔
3387

3388
    return SIXEL_OK;
261✔
3389
}
107✔
3390

3391
static void
3392
cuckoo_table32_clear(struct cuckoo_table32 *table)
267✔
3393
{
3394
    size_t i;
3395
    size_t j;
3396

3397
    if (table == NULL || table->buckets == NULL) {
267!
3398
        return;
×
3399
    }
3400

3401
    table->stash_count = 0U;
267✔
3402
    table->entry_count = 0U;
267✔
3403
    for (i = 0U; i < table->bucket_count; ++i) {
34,996,491✔
3404
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
174,981,120✔
3405
            table->buckets[i].key[j] = CUCKOO_EMPTY_KEY;
139,984,896✔
3406
            table->buckets[i].value[j] = 0U;
139,984,896✔
3407
        }
57,147,392✔
3408
    }
14,286,848✔
3409
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
8,811✔
3410
        table->stash_key[i] = CUCKOO_EMPTY_KEY;
8,544✔
3411
        table->stash_value[i] = 0U;
8,544✔
3412
    }
3,488✔
3413
}
109✔
3414

3415
static void
3416
cuckoo_table32_fini(struct cuckoo_table32 *table)
261✔
3417
{
3418
    if (table == NULL || table->allocator == NULL) {
261!
3419
        return;
×
3420
    }
3421
    if (table->buckets != NULL) {
261!
3422
        sixel_allocator_free(table->allocator, table->buckets);
261✔
3423
        table->buckets = NULL;
261✔
3424
    }
107✔
3425
    table->bucket_count = 0U;
261✔
3426
    table->bucket_mask = 0U;
261✔
3427
    table->stash_count = 0U;
261✔
3428
    table->entry_count = 0U;
261✔
3429
}
107✔
3430

3431
static uint32_t *
3432
cuckoo_table32_lookup(struct cuckoo_table32 *table, uint32_t key)
41,993,433✔
3433
{
3434
    size_t index;
3435
    size_t i;
3436
    uint32_t *slot;
3437

3438
    if (table == NULL || table->buckets == NULL) {
41,993,433!
3439
        return NULL;
×
3440
    }
3441

3442
    index = cuckoo_hash_primary(key, table->bucket_mask);
41,993,433✔
3443
    slot = cuckoo_bucket_find(&table->buckets[index], key);
41,993,433✔
3444
    if (slot != NULL) {
41,993,433✔
3445
        return slot;
38,155,492✔
3446
    }
3447

3448
    index = cuckoo_hash_secondary(key, table->bucket_mask);
3,837,941✔
3449
    slot = cuckoo_bucket_find(&table->buckets[index], key);
3,837,941✔
3450
    if (slot != NULL) {
3,837,941✔
3451
        return slot;
6,991✔
3452
    }
3453

3454
    for (i = 0U; i < table->stash_count; ++i) {
3,830,950!
3455
        if (table->stash_value[i] != 0U && table->stash_key[i] == key) {
×
3456
            return &table->stash_value[i];
×
3457
        }
3458
    }
3459

3460
    return NULL;
3,830,950✔
3461
}
19,619,361✔
3462

3463
static SIXELSTATUS
3464
cuckoo_table32_grow(struct cuckoo_table32 *table)
×
3465
{
3466
    struct cuckoo_table32 tmp;
3467
    struct cuckoo_bucket32 *old_buckets;
3468
    size_t old_count;
3469
    size_t i;
3470
    size_t j;
3471
    SIXELSTATUS status;
3472

3473
    if (table == NULL || table->allocator == NULL) {
×
3474
        return SIXEL_BAD_ARGUMENT;
×
3475
    }
3476

3477
    tmp.buckets = NULL;
×
3478
    tmp.bucket_count = 0U;
×
3479
    tmp.bucket_mask = 0U;
×
3480
    tmp.stash_count = 0U;
×
3481
    tmp.entry_count = 0U;
×
3482
    tmp.allocator = table->allocator;
×
3483
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
×
3484
        tmp.stash_key[i] = CUCKOO_EMPTY_KEY;
×
3485
        tmp.stash_value[i] = 0U;
×
3486
    }
3487

3488
    status = cuckoo_table32_init(&tmp,
×
3489
                                 (table->entry_count + 1U) * 2U,
×
3490
                                 table->allocator);
3491
    if (SIXEL_FAILED(status)) {
×
3492
        return status;
×
3493
    }
3494

3495
    old_buckets = table->buckets;
×
3496
    old_count = table->bucket_count;
×
3497
    for (i = 0U; i < old_count; ++i) {
×
3498
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
×
3499
            if (old_buckets[i].value[j] != 0U) {
×
3500
                status = cuckoo_table32_insert(&tmp,
×
3501
                                               old_buckets[i].key[j],
×
3502
                                               old_buckets[i].value[j]);
×
3503
                if (SIXEL_FAILED(status)) {
×
3504
                    cuckoo_table32_fini(&tmp);
×
3505
                    return status;
×
3506
                }
3507
            }
3508
        }
3509
    }
3510
    for (i = 0U; i < table->stash_count; ++i) {
×
3511
        if (table->stash_value[i] != 0U) {
×
3512
            status = cuckoo_table32_insert(&tmp,
×
3513
                                           table->stash_key[i],
3514
                                           table->stash_value[i]);
3515
            if (SIXEL_FAILED(status)) {
×
3516
                cuckoo_table32_fini(&tmp);
×
3517
                return status;
×
3518
            }
3519
        }
3520
    }
3521

3522
    sixel_allocator_free(table->allocator, old_buckets);
×
3523
    *table = tmp;
×
3524

3525
    return SIXEL_OK;
×
3526
}
3527

3528
static SIXELSTATUS
3529
cuckoo_table32_insert(struct cuckoo_table32 *table,
1,915,475✔
3530
                      uint32_t key,
3531
                      uint32_t value)
3532
{
3533
    uint32_t *slot;
3534
    uint32_t cur_key;
3535
    uint32_t cur_value;
3536
    uint32_t victim_key;
3537
    uint32_t victim_value;
3538
    size_t bucket_index;
3539
    size_t kicks;
3540
    size_t victim_slot;
3541
    struct cuckoo_bucket32 *bucket;
3542
    SIXELSTATUS status;
3543

3544
    if (table == NULL || table->buckets == NULL) {
1,915,475!
3545
        return SIXEL_BAD_ARGUMENT;
×
3546
    }
3547

3548
    slot = cuckoo_table32_lookup(table, key);
1,915,475✔
3549
    if (slot != NULL) {
1,915,475!
3550
        *slot = value;
×
3551
        return SIXEL_OK;
×
3552
    }
3553

3554
    cur_key = key;
1,915,475✔
3555
    cur_value = value;
1,915,475✔
3556
    bucket_index = cuckoo_hash_primary(cur_key, table->bucket_mask);
1,915,475✔
3557
    for (kicks = 0U; kicks < CUCKOO_MAX_KICKS; ++kicks) {
1,915,527!
3558
        bucket = &table->buckets[bucket_index];
1,915,527✔
3559
        if (cuckoo_bucket_insert_direct(bucket, cur_key, cur_value)) {
1,915,527✔
3560
            table->entry_count++;
1,915,475✔
3561
            return SIXEL_OK;
1,915,475✔
3562
        }
3563
        victim_slot = (size_t)((cur_key + kicks) &
52✔
3564
                               (CUCKOO_BUCKET_SIZE - 1U));
3565
        victim_key = bucket->key[victim_slot];
52✔
3566
        victim_value = bucket->value[victim_slot];
52✔
3567
        bucket->key[victim_slot] = cur_key;
52✔
3568
        bucket->value[victim_slot] = cur_value;
52✔
3569
        cur_key = victim_key;
52✔
3570
        cur_value = victim_value;
52✔
3571
        bucket_index = cuckoo_hash_alternate(cur_key,
62✔
3572
                                             bucket_index,
10✔
3573
                                             table->bucket_mask);
10✔
3574
    }
10✔
3575

3576
    if (table->stash_count < CUCKOO_STASH_SIZE) {
×
3577
        table->stash_key[table->stash_count] = cur_key;
×
3578
        table->stash_value[table->stash_count] = cur_value;
×
3579
        table->stash_count++;
×
3580
        table->entry_count++;
×
3581
        return SIXEL_OK;
×
3582
    }
3583

3584
    status = cuckoo_table32_grow(table);
×
3585
    if (SIXEL_FAILED(status)) {
×
3586
        return status;
×
3587
    }
3588

3589
    return cuckoo_table32_insert(table, cur_key, cur_value);
×
3590
}
613,543✔
3591

3592
static SIXELSTATUS
3593
computeHistogram_robinhood(unsigned char const *data,
3594
                           unsigned int length,
3595
                           unsigned long depth,
3596
                           unsigned int step,
3597
                           unsigned int max_sample,
3598
                           tupletable2 * const colorfreqtableP,
3599
                           struct histogram_control const *control,
3600
                           int use_reversible,
3601
                           sixel_allocator_t *allocator);
3602

3603
static SIXELSTATUS
3604
computeHistogram_hopscotch(unsigned char const *data,
3605
                           unsigned int length,
3606
                           unsigned long depth,
3607
                           unsigned int step,
3608
                           unsigned int max_sample,
3609
                           tupletable2 * const colorfreqtableP,
3610
                           struct histogram_control const *control,
3611
                           int use_reversible,
3612
                           sixel_allocator_t *allocator);
3613

3614
static SIXELSTATUS
3615
computeHistogram(unsigned char const    /* in */  *data,
260✔
3616
                 unsigned int           /* in */  length,
3617
                 unsigned long const    /* in */  depth,
3618
                 tupletable2 * const    /* out */ colorfreqtableP,
3619
                 int const              /* in */  qualityMode,
3620
                 int const              /* in */  use_reversible,
3621
                 sixel_allocator_t      /* in */  *allocator)
3622
{
3623
    SIXELSTATUS status = SIXEL_FALSE;
260✔
3624
    typedef uint32_t unit_t;
3625
    unsigned int i, n;
3626
    unit_t *histogram = NULL;
260✔
3627
    unit_t *refmap = NULL;
260✔
3628
    unit_t *ref;
3629
    unsigned int bucket_index;
3630
    unsigned int step;
3631
    unsigned int max_sample;
3632
    size_t hist_size;
3633
    unit_t bucket_value;
3634
    unsigned int component;
3635
    unsigned int reconstructed;
3636
    struct histogram_control control;
3637
    unsigned int depth_u;
3638
    unsigned char reversible_pixel[4];
3639

3640
    switch (qualityMode) {
260!
3641
    case SIXEL_QUALITY_LOW:
120✔
3642
        max_sample = 18383;
227✔
3643
        break;
227✔
3644
    case SIXEL_QUALITY_HIGH:
20✔
3645
        max_sample = 1118383;
30✔
3646
        break;
30✔
3647
    case SIXEL_QUALITY_FULL:
3✔
3648
    default:
3649
        max_sample = 4003079;
3✔
3650
        break;
3✔
3651
    }
3652

3653
    step = length / depth / max_sample * depth;
260✔
3654
    if (step <= 0) {
260✔
3655
        step = depth;
156✔
3656
    }
52✔
3657

3658
    quant_trace(stderr, "making histogram...\n");
260✔
3659

3660
    depth_u = (unsigned int)depth;
260✔
3661
    control = histogram_control_make(depth_u);
260✔
3662
    if (use_reversible) {
260!
3663
        control.reversible_rounding = 1;
×
3664
    }
3665
    if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
260!
3666
        || histogram_lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
260!
3667
        if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD) {
×
3668
            status = computeHistogram_robinhood(data,
×
3669
                                                length,
3670
                                                depth,
3671
                                                step,
3672
                                                max_sample,
3673
                                                colorfreqtableP,
3674
                                                &control,
3675
                                                use_reversible,
3676
                                                allocator);
3677
        } else {
3678
            status = computeHistogram_hopscotch(data,
×
3679
                                                length,
3680
                                                depth,
3681
                                                step,
3682
                                                max_sample,
3683
                                                colorfreqtableP,
3684
                                                &control,
3685
                                                use_reversible,
3686
                                                allocator);
3687
        }
3688
        goto end;
×
3689
    }
3690

3691
    hist_size = histogram_dense_size((unsigned int)depth, &control);
260✔
3692
    histogram = (unit_t *)sixel_allocator_calloc(allocator,
378✔
3693
                                                 hist_size,
118✔
3694
                                                 sizeof(unit_t));
3695
    if (histogram == NULL) {
260!
3696
        sixel_helper_set_additional_message(
×
3697
            "unable to allocate memory for histogram.");
3698
        status = SIXEL_BAD_ALLOCATION;
×
3699
        goto end;
×
3700
    }
3701
    ref = refmap = (unit_t *)sixel_allocator_malloc(allocator,
378✔
3702
                                                    hist_size *
118✔
3703
                                                    sizeof(unit_t));
3704
    if (refmap == NULL) {
260!
3705
        sixel_helper_set_additional_message(
×
3706
            "unable to allocate memory for lookup table.");
3707
        status = SIXEL_BAD_ALLOCATION;
×
3708
        goto end;
×
3709
    }
3710

3711
    for (i = 0; i < length; i += step) {
3,572,726✔
3712
        if (use_reversible) {
3,572,466!
3713
            sixel_quant_reversible_pixel(data + i,
×
3714
                                         depth_u,
3715
                                         reversible_pixel);
3716
            bucket_index = histogram_pack_color(reversible_pixel,
×
3717
                                                depth_u,
3718
                                                &control);
3719
        } else {
3720
            bucket_index = histogram_pack_color(data + i,
5,349,672✔
3721
                                                depth_u,
1,777,206✔
3722
                                                &control);
3723
        }
3724
        if (histogram[bucket_index] == 0) {
3,572,466✔
3725
            *ref++ = bucket_index;
294,113✔
3726
        }
98,297✔
3727
        if (histogram[bucket_index] < UINT32_MAX) {
3,572,466!
3728
            histogram[bucket_index]++;
3,572,466✔
3729
        }
1,777,206✔
3730
    }
1,777,206✔
3731

3732
    colorfreqtableP->size = (unsigned int)(ref - refmap);
260✔
3733

3734
    status = alloctupletable(&colorfreqtableP->table,
378✔
3735
                             depth,
118✔
3736
                             (unsigned int)(ref - refmap),
260✔
3737
                             allocator);
118✔
3738
    if (SIXEL_FAILED(status)) {
260!
3739
        goto end;
×
3740
    }
3741
    for (i = 0; i < colorfreqtableP->size; ++i) {
294,373✔
3742
        bucket_value = refmap[i];
294,113✔
3743
        if (histogram[bucket_value] > 0) {
294,113!
3744
            colorfreqtableP->table[i]->value = histogram[bucket_value];
294,113✔
3745
            for (n = 0; n < depth; n++) {
1,176,452✔
3746
                component = (unsigned int)
882,339✔
3747
                    ((bucket_value >> (n * control.channel_bits)) &
1,177,230✔
3748
                     control.channel_mask);
882,339✔
3749
                reconstructed = histogram_reconstruct(component,
882,339✔
3750
                                                      &control);
3751
                if (use_reversible) {
882,339!
3752
                    reconstructed =
×
3753
                        (unsigned int)sixel_quant_reversible_value(
×
3754
                            reconstructed);
3755
                }
3756
                colorfreqtableP->table[i]->tuple[depth - 1 - n]
882,339✔
3757
                    = (sample)reconstructed;
1,177,230✔
3758
            }
294,891✔
3759
        }
98,297✔
3760
    }
98,297✔
3761

3762
    quant_trace(stderr, "%u colors found\n", colorfreqtableP->size);
260✔
3763

3764
    status = SIXEL_OK;
260✔
3765

3766
end:
142✔
3767
    sixel_allocator_free(allocator, refmap);
260✔
3768
    sixel_allocator_free(allocator, histogram);
260✔
3769

3770
    return status;
260✔
3771
}
3772

3773
static SIXELSTATUS
3774
computeHistogram_robinhood(unsigned char const *data,
×
3775
                           unsigned int length,
3776
                           unsigned long depth,
3777
                           unsigned int step,
3778
                           unsigned int max_sample,
3779
                           tupletable2 * const colorfreqtableP,
3780
                           struct histogram_control const *control,
3781
                           int use_reversible,
3782
                           sixel_allocator_t *allocator)
3783
{
3784
    SIXELSTATUS status = SIXEL_FALSE;
×
3785
    struct robinhood_table32 table;
3786
    size_t expected;
3787
    size_t cap_limit;
3788
    size_t index;
3789
    unsigned int depth_u;
3790
    unsigned int i;
3791
    unsigned int n;
3792
    uint32_t bucket_color;
3793
    uint32_t bucket_hash;
3794
    uint32_t entry_color;
3795
    struct robinhood_slot32 *slot;
3796
    unsigned int component;
3797
    unsigned int reconstructed;
3798
    unsigned char reversible_pixel[4];
3799

3800
    /*
3801
     * The ASCII sketch below shows how the sparse table stores samples:
3802
     *
3803
     *   [hash]->(key,value,distance)  Robin Hood probing keeps dense tails.
3804
     */
3805
    table.slots = NULL;
×
3806
    table.capacity = 0U;
×
3807
    table.count = 0U;
×
3808
    table.allocator = allocator;
×
3809
    cap_limit = (size_t)1U << 20;
×
3810
    expected = max_sample;
×
3811
    if (expected < 256U) {
×
3812
        expected = 256U;
×
3813
    }
3814
    if (expected > cap_limit) {
×
3815
        expected = cap_limit;
×
3816
    }
3817

3818
    status = robinhood_table32_init(&table, expected, allocator);
×
3819
    if (SIXEL_FAILED(status)) {
×
3820
        sixel_helper_set_additional_message(
×
3821
            "unable to allocate robinhood histogram.");
3822
        goto end;
×
3823
    }
3824

3825
    depth_u = (unsigned int)depth;
×
3826
    for (i = 0U; i < length; i += step) {
×
3827
        if (use_reversible) {
×
3828
            sixel_quant_reversible_pixel(data + i, depth_u,
×
3829
                                         reversible_pixel);
3830
            bucket_color = histogram_pack_color(reversible_pixel,
×
3831
                                                depth_u, control);
3832
        } else {
3833
            bucket_color = histogram_pack_color(data + i,
×
3834
                                                depth_u, control);
3835
        }
3836
        bucket_hash = histogram_hash_mix(bucket_color);
×
3837
        /*
3838
         * Hash probing uses the mixed key while the slot stores the
3839
         * original quantized RGB value:
3840
         *
3841
         *   hash --> [slot]
3842
         *             |color|count|
3843
         */
3844
        slot = robinhood_table32_lookup(&table,
×
3845
                                        bucket_hash,
3846
                                        bucket_color);
3847
        if (slot == NULL) {
×
3848
            status = robinhood_table32_insert(&table,
×
3849
                                              bucket_hash,
3850
                                              bucket_color,
3851
                                              1U);
3852
            if (SIXEL_FAILED(status)) {
×
3853
                sixel_helper_set_additional_message(
×
3854
                    "unable to grow robinhood histogram.");
3855
                goto end;
×
3856
            }
3857
        } else if (slot->value < UINT32_MAX) {
×
3858
            slot->value++;
×
3859
        }
3860
    }
3861

3862
    if (table.count > UINT_MAX) {
×
3863
        sixel_helper_set_additional_message(
×
3864
            "too many unique colors for histogram.");
3865
        status = SIXEL_BAD_ARGUMENT;
×
3866
        goto end;
×
3867
    }
3868

3869
    colorfreqtableP->size = (unsigned int)table.count;
×
3870
    status = alloctupletable(&colorfreqtableP->table,
×
3871
                             depth_u,
3872
                             (unsigned int)table.count,
×
3873
                             allocator);
3874
    if (SIXEL_FAILED(status)) {
×
3875
        goto end;
×
3876
    }
3877

3878
    index = 0U;
×
3879
    /*
3880
     * Stream slots in the hash traversal order to avoid qsort overhead.
3881
     * This favors throughput over identical palette ordering.
3882
     */
3883
    for (i = 0U; i < table.capacity; ++i) {
×
3884
        slot = &table.slots[i];
×
3885
        if (slot->value == 0U) {
×
3886
            continue;
×
3887
        }
3888
        if (index >= colorfreqtableP->size) {
×
3889
            break;
×
3890
        }
3891
        entry_color = slot->color;
×
3892
        colorfreqtableP->table[index]->value = slot->value;
×
3893
        for (n = 0U; n < depth_u; ++n) {
×
3894
            component = (unsigned int)
×
3895
                ((entry_color >> (n * control->channel_bits))
×
3896
                 & control->channel_mask);
×
3897
            reconstructed = histogram_reconstruct(component, control);
×
3898
            if (use_reversible) {
×
3899
                reconstructed =
×
3900
                    (unsigned int)sixel_quant_reversible_value(
×
3901
                        reconstructed);
3902
            }
3903
            colorfreqtableP->table[index]->tuple[depth_u - 1U - n]
×
3904
                = (sample)reconstructed;
×
3905
        }
3906
        index++;
×
3907
    }
3908

3909
    status = SIXEL_OK;
×
3910

3911
end:
3912
    robinhood_table32_fini(&table);
×
3913

3914
    return status;
×
3915
}
3916

3917
static SIXELSTATUS
3918
computeHistogram_hopscotch(unsigned char const *data,
×
3919
                           unsigned int length,
3920
                           unsigned long depth,
3921
                           unsigned int step,
3922
                           unsigned int max_sample,
3923
                           tupletable2 * const colorfreqtableP,
3924
                           struct histogram_control const *control,
3925
                           int use_reversible,
3926
                           sixel_allocator_t *allocator)
3927
{
3928
    SIXELSTATUS status = SIXEL_FALSE;
×
3929
    struct hopscotch_table32 table;
3930
    size_t expected;
3931
    size_t cap_limit;
3932
    size_t index;
3933
    unsigned int depth_u;
3934
    unsigned int i;
3935
    unsigned int n;
3936
    uint32_t bucket_color;
3937
    uint32_t bucket_hash;
3938
    uint32_t entry_color;
3939
    struct hopscotch_slot32 *slot;
3940
    unsigned int component;
3941
    unsigned int reconstructed;
3942
    unsigned char reversible_pixel[4];
3943

3944
    /*
3945
     * Hopscotch hashing stores the local neighbourhood using the map below:
3946
     *
3947
     *   [home] hopinfo bits ---> |slot+0|slot+1|slot+2| ...
3948
     *                              ^ entries hop within this window.
3949
     */
3950
    table.slots = NULL;
×
3951
    table.hopinfo = NULL;
×
3952
    table.capacity = 0U;
×
3953
    table.count = 0U;
×
3954
    table.neighborhood = HOPSCOTCH_DEFAULT_NEIGHBORHOOD;
×
3955
    table.allocator = allocator;
×
3956
    cap_limit = (size_t)1U << 20;
×
3957
    expected = max_sample;
×
3958
    if (expected < 256U) {
×
3959
        expected = 256U;
×
3960
    }
3961
    if (expected > cap_limit) {
×
3962
        expected = cap_limit;
×
3963
    }
3964

3965
    status = hopscotch_table32_init(&table, expected, allocator);
×
3966
    if (SIXEL_FAILED(status)) {
×
3967
        sixel_helper_set_additional_message(
×
3968
            "unable to allocate hopscotch histogram.");
3969
        goto end;
×
3970
    }
3971

3972
    depth_u = (unsigned int)depth;
×
3973
    for (i = 0U; i < length; i += step) {
×
3974
        if (use_reversible) {
×
3975
            sixel_quant_reversible_pixel(data + i, depth_u,
×
3976
                                         reversible_pixel);
3977
            bucket_color = histogram_pack_color(reversible_pixel,
×
3978
                                                depth_u, control);
3979
        } else {
3980
            bucket_color = histogram_pack_color(data + i,
×
3981
                                                depth_u, control);
3982
        }
3983
        bucket_hash = histogram_hash_mix(bucket_color);
×
3984
        /*
3985
         * Hopscotch buckets mirror the robinhood layout, keeping the
3986
         * quantized color next to the count so we never derive it from
3987
         * the scrambled hash key.
3988
         */
3989
        slot = hopscotch_table32_lookup(&table,
×
3990
                                        bucket_hash,
3991
                                        bucket_color);
3992
        if (slot == NULL) {
×
3993
            status = hopscotch_table32_insert(&table,
×
3994
                                              bucket_hash,
3995
                                              bucket_color,
3996
                                              1U);
3997
            if (SIXEL_FAILED(status)) {
×
3998
                sixel_helper_set_additional_message(
×
3999
                    "unable to grow hopscotch histogram.");
4000
                goto end;
×
4001
            }
4002
        } else if (slot->value < UINT32_MAX) {
×
4003
            slot->value++;
×
4004
        }
4005
    }
4006

4007
    if (table.count > UINT_MAX) {
×
4008
        sixel_helper_set_additional_message(
×
4009
            "too many unique colors for histogram.");
4010
        status = SIXEL_BAD_ARGUMENT;
×
4011
        goto end;
×
4012
    }
4013

4014
    colorfreqtableP->size = (unsigned int)table.count;
×
4015
    status = alloctupletable(&colorfreqtableP->table,
×
4016
                             depth_u,
4017
                             (unsigned int)table.count,
×
4018
                             allocator);
4019
    if (SIXEL_FAILED(status)) {
×
4020
        goto end;
×
4021
    }
4022

4023
    index = 0U;
×
4024
    /*
4025
     * Stream slots in the hash traversal order to avoid qsort overhead.
4026
     * This favors throughput over identical palette ordering.
4027
     */
4028
    for (i = 0U; i < table.capacity; ++i) {
×
4029
        slot = &table.slots[i];
×
4030
        if (slot->key == HOPSCOTCH_EMPTY_KEY || slot->value == 0U) {
×
4031
            continue;
×
4032
        }
4033
        if (index >= colorfreqtableP->size) {
×
4034
            break;
×
4035
        }
4036
        entry_color = slot->color;
×
4037
        colorfreqtableP->table[index]->value = slot->value;
×
4038
        for (n = 0U; n < depth_u; ++n) {
×
4039
            component = (unsigned int)
×
4040
                ((entry_color >> (n * control->channel_bits))
×
4041
                 & control->channel_mask);
×
4042
            reconstructed = histogram_reconstruct(component, control);
×
4043
            if (use_reversible) {
×
4044
                reconstructed =
×
4045
                    (unsigned int)sixel_quant_reversible_value(
×
4046
                        reconstructed);
4047
            }
4048
            colorfreqtableP->table[index]->tuple[depth_u - 1U - n]
×
4049
                = (sample)reconstructed;
×
4050
        }
4051
        index++;
×
4052
    }
4053

4054
    status = SIXEL_OK;
×
4055

4056
end:
4057
    hopscotch_table32_fini(&table);
×
4058

4059
    return status;
×
4060
}
4061

4062
SIXELSTATUS
4063
sixel_quant_cache_prepare(unsigned short **cachetable,
264✔
4064
                          size_t *cachetable_size,
4065
                          int lut_policy,
4066
                          int reqcolor,
4067
                          sixel_allocator_t *allocator)
4068
{
4069
    SIXELSTATUS status;
4070
    struct histogram_control control;
4071
    struct cuckoo_table32 *table;
4072
    size_t expected;
4073
    size_t buckets;
4074
    size_t cap_limit;
4075
    int normalized;
4076

4077
    if (cachetable == NULL || allocator == NULL) {
264!
4078
        return SIXEL_BAD_ARGUMENT;
×
4079
    }
4080

4081
    /*
4082
     * The cache pointer always references the same ladder:
4083
     *
4084
     *   cache -> cuckoo_table32 -> buckets
4085
     *                           -> stash
4086
     */
4087
    normalized = lut_policy;
264✔
4088
    if (normalized == SIXEL_LUT_POLICY_AUTO) {
264!
4089
        normalized = histogram_lut_policy;
264✔
4090
    }
108✔
4091
    if (normalized == SIXEL_LUT_POLICY_AUTO) {
264!
4092
        normalized = SIXEL_LUT_POLICY_6BIT;
264✔
4093
    }
108✔
4094

4095
    control = histogram_control_make_for_policy(3U, normalized);
264✔
4096
    if (control.channel_shift == 0U) {
264!
4097
        expected = (size_t)reqcolor * 64U;
×
4098
        cap_limit = (size_t)1U << 20;
×
4099
        if (expected < 512U) {
×
4100
            expected = 512U;
×
4101
        }
4102
        if (expected > cap_limit) {
×
4103
            expected = cap_limit;
×
4104
        }
4105
    } else {
4106
        expected = histogram_dense_size(3U, &control);
264✔
4107
        if (expected == 0U) {
264!
4108
            expected = CUCKOO_BUCKET_SIZE;
×
4109
        }
4110
    }
4111

4112
    table = (struct cuckoo_table32 *)*cachetable;
264✔
4113
    if (table != NULL) {
264✔
4114
        buckets = cuckoo_round_buckets(expected);
3✔
4115
        if (table->bucket_count < buckets) {
3!
4116
            cuckoo_table32_fini(table);
×
4117
            sixel_allocator_free(allocator, table);
×
4118
            table = NULL;
×
4119
            *cachetable = NULL;
×
4120
        } else {
4121
            cuckoo_table32_clear(table);
3✔
4122
        }
4123
    }
1✔
4124
    if (table == NULL) {
264✔
4125
        table = (struct cuckoo_table32 *)sixel_allocator_malloc(
261✔
4126
            allocator, sizeof(struct cuckoo_table32));
107✔
4127
        if (table == NULL) {
261!
4128
            sixel_helper_set_additional_message(
×
4129
                "unable to allocate cuckoo cache state.");
4130
            return SIXEL_BAD_ALLOCATION;
×
4131
        }
4132
        memset(table, 0, sizeof(struct cuckoo_table32));
261✔
4133
        status = cuckoo_table32_init(table, expected, allocator);
261✔
4134
        if (SIXEL_FAILED(status)) {
261!
4135
            sixel_allocator_free(allocator, table);
×
4136
            sixel_helper_set_additional_message(
×
4137
                "unable to initialize cuckoo cache.");
4138
            return status;
×
4139
        }
4140
        *cachetable = (unsigned short *)table;
261✔
4141
    }
107✔
4142

4143
    if (cachetable_size != NULL) {
264!
4144
        *cachetable_size = table->bucket_count * CUCKOO_BUCKET_SIZE;
264✔
4145
    }
108✔
4146

4147
    return SIXEL_OK;
264✔
4148
}
108✔
4149

4150
void
4151
sixel_quant_cache_clear(unsigned short *cachetable,
264✔
4152
                        int lut_policy)
4153
{
4154
    struct cuckoo_table32 *table;
4155

4156
    (void)lut_policy;
108✔
4157
    if (cachetable == NULL) {
264!
4158
        return;
×
4159
    }
4160

4161
    table = (struct cuckoo_table32 *)cachetable;
264✔
4162
    cuckoo_table32_clear(table);
264✔
4163
}
108✔
4164

4165
void
4166
sixel_quant_cache_release(unsigned short *cachetable,
261✔
4167
                          int lut_policy,
4168
                          sixel_allocator_t *allocator)
4169
{
4170
    struct cuckoo_table32 *table;
4171

4172
    (void)lut_policy;
107✔
4173
    if (cachetable == NULL || allocator == NULL) {
261!
UNCOV
4174
        return;
×
4175
    }
4176

4177
    table = (struct cuckoo_table32 *)cachetable;
261✔
4178
    cuckoo_table32_fini(table);
261✔
4179
    sixel_allocator_free(allocator, table);
261✔
4180
}
107✔
4181

4182

4183
int
4184
computeColorMapFromInput(unsigned char const *data,
260✔
4185
                         unsigned int const length,
4186
                         unsigned int const depth,
4187
                         unsigned int const reqColors,
4188
                         int const methodForLargest,
4189
                         int const methodForRep,
4190
                         int const qualityMode,
4191
                         int const force_palette,
4192
                         int const use_reversible,
4193
                         int const final_merge_mode,
4194
                         tupletable2 * const colormapP,
4195
                         unsigned int *origcolors,
4196
                         sixel_allocator_t *allocator)
4197
{
4198
/*----------------------------------------------------------------------------
4199
   Produce a colormap containing the best colors to represent the
4200
   image stream in file 'ifP'.  Figure it out using the median cut
4201
   technique.
4202

4203
   The colormap will have 'reqcolors' or fewer colors in it, unless
4204
   'allcolors' is true, in which case it will have all the colors that
4205
   are in the input.
4206

4207
   The colormap has the same maxval as the input.
4208

4209
   Put the colormap in newly allocated storage as a tupletable2
4210
   and return its address as *colormapP.  Return the number of colors in
4211
   it as *colorsP and its maxval as *colormapMaxvalP.
4212

4213
   Return the characteristics of the input file as
4214
   *formatP and *freqPamP.  (This information is not really
4215
   relevant to our colormap mission; just a fringe benefit).
4216
-----------------------------------------------------------------------------*/
4217
    SIXELSTATUS status = SIXEL_FALSE;
260✔
4218
    tupletable2 colorfreqtable = {0, NULL};
260✔
4219
    unsigned int i;
4220
    unsigned int n;
4221

4222
    status = computeHistogram(data, length, depth,
378✔
4223
                              &colorfreqtable, qualityMode,
118✔
4224
                              use_reversible, allocator);
118✔
4225
    if (SIXEL_FAILED(status)) {
260!
4226
        goto end;
×
4227
    }
4228
    if (origcolors) {
260!
4229
        *origcolors = colorfreqtable.size;
260✔
4230
    }
118✔
4231

4232
    if (colorfreqtable.size <= reqColors) {
260✔
4233
        quant_trace(stderr,
286✔
4234
                    "Image already has few enough colors (<=%d).  "
4235
                    "Keeping same colors.\n", reqColors);
95✔
4236
        /* *colormapP = colorfreqtable; */
4237
        colormapP->size = colorfreqtable.size;
191✔
4238
        status = alloctupletable(&colormapP->table, depth, colorfreqtable.size, allocator);
191✔
4239
        if (SIXEL_FAILED(status)) {
191!
4240
            goto end;
×
4241
        }
4242
        for (i = 0; i < colorfreqtable.size; ++i) {
3,205✔
4243
            colormapP->table[i]->value = colorfreqtable.table[i]->value;
3,014✔
4244
            for (n = 0; n < depth; ++n) {
12,056✔
4245
                colormapP->table[i]->tuple[n] = colorfreqtable.table[i]->tuple[n];
9,042✔
4246
            }
4,224✔
4247
            if (use_reversible) {
3,014!
4248
                sixel_quant_reversible_tuple(colormapP->table[i]->tuple,
×
4249
                                             depth);
4250
            }
4251
        }
1,408✔
4252
    } else {
95✔
4253
        quant_trace(stderr, "choosing %d colors...\n", reqColors);
69✔
4254
        status = mediancut(colorfreqtable, depth, reqColors,
92✔
4255
                           methodForLargest, methodForRep,
23✔
4256
                           use_reversible, final_merge_mode,
23✔
4257
                           colormapP, allocator);
23✔
4258
        if (SIXEL_FAILED(status)) {
69!
4259
            goto end;
×
4260
        }
4261
        quant_trace(stderr, "%d colors are choosed.\n", colorfreqtable.size);
69✔
4262
    }
4263

4264
    if (force_palette) {
260!
4265
        status = force_palette_completion(colormapP, depth, reqColors,
×
4266
                                          colorfreqtable, allocator);
4267
        if (SIXEL_FAILED(status)) {
×
4268
            goto end;
×
4269
        }
4270
    }
4271

4272
    status = SIXEL_OK;
260✔
4273

4274
end:
142✔
4275
    sixel_allocator_free(allocator, colorfreqtable.table);
260✔
4276
    return status;
260✔
4277
}
4278

4279

4280
/* diffuse error energy to surround pixels (normal strategy) */
4281
static void
4282
error_diffuse_normal(
281,667,771✔
4283
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4284
    int           /* in */    pos,        /* address of the destination pixel */
4285
    int           /* in */    depth,      /* color depth in bytes */
4286
    int           /* in */    error,      /* error energy */
4287
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4288
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4289
{
4290
    int c;
4291

4292
    data += pos * depth;
281,667,771✔
4293

4294
    c = *data + (error * numerator * 2 / denominator + 1) / 2;
281,667,771✔
4295
    if (c < 0) {
281,667,771✔
4296
        c = 0;
2,567,509✔
4297
    }
811,245✔
4298
    if (c >= 1 << 8) {
281,667,771✔
4299
        c = (1 << 8) - 1;
959,648✔
4300
    }
304,614✔
4301
    *data = (unsigned char)c;
281,667,771✔
4302
}
281,667,771✔
4303

4304
/* error diffusion with fast strategy */
4305
static void
4306
error_diffuse_fast(
169,469,067✔
4307
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4308
    int           /* in */    pos,        /* address of the destination pixel */
4309
    int           /* in */    depth,      /* color depth in bytes */
4310
    int           /* in */    error,      /* error energy */
4311
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4312
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4313
{
4314
    int c;
4315

4316
    data += pos * depth;
169,469,067✔
4317

4318
    c = *data + error * numerator / denominator;
169,469,067✔
4319
    if (c < 0) {
169,469,067✔
4320
        c = 0;
7,291,990✔
4321
    }
2,721,044✔
4322
    if (c >= 1 << 8) {
169,469,067✔
4323
        c = (1 << 8) - 1;
459,666✔
4324
    }
150,804✔
4325
    *data = (unsigned char)c;
169,469,067✔
4326
}
169,469,067✔
4327

4328

4329
/* error diffusion with precise strategy */
4330
static void
4331
error_diffuse_precise(
6,111,369✔
4332
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4333
    int           /* in */    pos,        /* address of the destination pixel */
4334
    int           /* in */    depth,      /* color depth in bytes */
4335
    int           /* in */    error,      /* error energy */
4336
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4337
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4338
{
4339
    int c;
4340

4341
    data += pos * depth;
6,111,369✔
4342

4343
    c = (int)(*data + error * numerator / (double)denominator + 0.5);
6,111,369✔
4344
    if (c < 0) {
6,111,369✔
4345
        c = 0;
70,455✔
4346
    }
25,487✔
4347
    if (c >= 1 << 8) {
6,111,369!
4348
        c = (1 << 8) - 1;
×
4349
    }
4350
    *data = (unsigned char)c;
6,111,369✔
4351
}
6,111,369✔
4352

4353

4354
typedef void (*diffuse_fixed_carry_mode)(int32_t *carry_curr,
4355
                                         int32_t *carry_next,
4356
                                         int32_t *carry_far,
4357
                                         int width,
4358
                                         int height,
4359
                                         int depth,
4360
                                         int x,
4361
                                         int y,
4362
                                         int32_t error,
4363
                                         int direction,
4364
                                         int channel);
4365

4366

4367
typedef void (*diffuse_varerr_mode)(unsigned char *data,
4368
                                    int width,
4369
                                    int height,
4370
                                    int x,
4371
                                    int y,
4372
                                    int depth,
4373
                                    int32_t error,
4374
                                    int index,
4375
                                    int direction);
4376

4377
typedef void (*diffuse_varerr_carry_mode)(int32_t *carry_curr,
4378
                                          int32_t *carry_next,
4379
                                          int32_t *carry_far,
4380
                                          int width,
4381
                                          int height,
4382
                                          int depth,
4383
                                          int x,
4384
                                          int y,
4385
                                          int32_t error,
4386
                                          int index,
4387
                                          int direction,
4388
                                          int channel);
4389

4390

4391
static int32_t
4392
diffuse_varerr_term(int32_t error, int weight, int denom)
×
4393
{
4394
    int64_t delta;
4395

4396
    delta = (int64_t)error * (int64_t)weight;
×
4397
    if (delta >= 0) {
×
4398
        delta = (delta + denom / 2) / denom;
×
4399
    } else {
4400
        delta = (delta - denom / 2) / denom;
×
4401
    }
4402

4403
    return (int32_t)delta;
×
4404
}
4405

4406

4407
static int32_t
4408
diffuse_fixed_term(int32_t error, int numerator, int denominator)
×
4409
{
4410
    int64_t delta;
4411

4412
    delta = (int64_t)error * (int64_t)numerator;
×
4413
    if (delta >= 0) {
×
4414
        delta = (delta + denominator / 2) / denominator;
×
4415
    } else {
4416
        delta = (delta - denominator / 2) / denominator;
×
4417
    }
4418

4419
    return (int32_t)delta;
×
4420
}
4421

4422

4423
static void
4424
diffuse_varerr_apply_direct(unsigned char *target, int depth, size_t offset,
×
4425
                            int32_t delta)
4426
{
4427
    int64_t value;
4428
    int result;
4429

4430
    value = (int64_t)target[offset * depth] << VARERR_SCALE_SHIFT;
×
4431
    value += delta;
×
4432
    if (value < 0) {
×
4433
        value = 0;
×
4434
    } else {
4435
        int64_t max_value;
4436

4437
        max_value = VARERR_MAX_VALUE;
×
4438
        if (value > max_value) {
×
4439
            value = max_value;
×
4440
        }
4441
    }
4442

4443
    result = (int)((value + VARERR_ROUND) >> VARERR_SCALE_SHIFT);
×
4444
    if (result < 0) {
×
4445
        result = 0;
×
4446
    }
4447
    if (result > 255) {
×
4448
        result = 255;
×
4449
    }
4450
    target[offset * depth] = (unsigned char)result;
×
4451
}
×
4452

4453

4454
static void
4455
diffuse_lso2(unsigned char *data, int width, int height,
×
4456
             int x, int y, int depth, int32_t error,
4457
             int index, int direction)
4458
{
4459
    const int (*table)[7];
4460
    const int *entry;
4461
    int denom;
4462
    int32_t term_r = 0;
×
4463
    int32_t term_r2 = 0;
×
4464
    int32_t term_dl = 0;
×
4465
    int32_t term_d = 0;
×
4466
    int32_t term_dr = 0;
×
4467
    int32_t term_d2 = 0;
×
4468
    size_t offset;
4469

4470
    if (error == 0) {
×
4471
        return;
×
4472
    }
4473
    if (index < 0) {
×
4474
        index = 0;
×
4475
    }
4476
    if (index > 255) {
×
4477
        index = 255;
×
4478
    }
4479

4480
    table = lso2_table();
×
4481
    entry = table[index];
×
4482
    denom = entry[6];
×
4483
    if (denom == 0) {
×
4484
        return;
×
4485
    }
4486

4487
    term_r = diffuse_varerr_term(error, entry[0], denom);
×
4488
    term_r2 = diffuse_varerr_term(error, entry[1], denom);
×
4489
    term_dl = diffuse_varerr_term(error, entry[2], denom);
×
4490
    term_d = diffuse_varerr_term(error, entry[3], denom);
×
4491
    term_dr = diffuse_varerr_term(error, entry[4], denom);
×
4492
    term_d2 = diffuse_varerr_term(error, entry[5], denom);
×
4493

4494

4495
    if (direction >= 0) {
×
4496
        if (x + 1 < width) {
×
4497
            offset = (size_t)y * (size_t)width + (size_t)(x + 1);
×
4498
            diffuse_varerr_apply_direct(data, depth, offset, term_r);
×
4499
        }
4500
        if (x + 2 < width) {
×
4501
            offset = (size_t)y * (size_t)width + (size_t)(x + 2);
×
4502
            diffuse_varerr_apply_direct(data, depth, offset, term_r2);
×
4503
        }
4504
        if (y + 1 < height && x - 1 >= 0) {
×
4505
            offset = (size_t)(y + 1) * (size_t)width;
×
4506
            offset += (size_t)(x - 1);
×
4507
            diffuse_varerr_apply_direct(data, depth, offset, term_dl);
×
4508
        }
4509
        if (y + 1 < height) {
×
4510
            offset = (size_t)(y + 1) * (size_t)width + (size_t)x;
×
4511
            diffuse_varerr_apply_direct(data, depth, offset, term_d);
×
4512
        }
4513
        if (y + 1 < height && x + 1 < width) {
×
4514
            offset = (size_t)(y + 1) * (size_t)width;
×
4515
            offset += (size_t)(x + 1);
×
4516
            diffuse_varerr_apply_direct(data, depth, offset, term_dr);
×
4517
        }
4518
        if (y + 2 < height) {
×
4519
            offset = (size_t)(y + 2) * (size_t)width + (size_t)x;
×
4520
            diffuse_varerr_apply_direct(data, depth, offset, term_d2);
×
4521
        }
4522
    } else {
4523
        if (x - 1 >= 0) {
×
4524
            offset = (size_t)y * (size_t)width + (size_t)(x - 1);
×
4525
            diffuse_varerr_apply_direct(data, depth, offset, term_r);
×
4526
        }
4527
        if (x - 2 >= 0) {
×
4528
            offset = (size_t)y * (size_t)width + (size_t)(x - 2);
×
4529
            diffuse_varerr_apply_direct(data, depth, offset, term_r2);
×
4530
        }
4531
        if (y + 1 < height && x + 1 < width) {
×
4532
            offset = (size_t)(y + 1) * (size_t)width;
×
4533
            offset += (size_t)(x + 1);
×
4534
            diffuse_varerr_apply_direct(data, depth, offset, term_dl);
×
4535
        }
4536
        if (y + 1 < height) {
×
4537
            offset = (size_t)(y + 1) * (size_t)width + (size_t)x;
×
4538
            diffuse_varerr_apply_direct(data, depth, offset, term_d);
×
4539
        }
4540
        if (y + 1 < height && x - 1 >= 0) {
×
4541
            offset = (size_t)(y + 1) * (size_t)width;
×
4542
            offset += (size_t)(x - 1);
×
4543
            diffuse_varerr_apply_direct(data, depth, offset, term_dr);
×
4544
        }
4545
        if (y + 2 < height) {
×
4546
            offset = (size_t)(y + 2) * (size_t)width + (size_t)x;
×
4547
            diffuse_varerr_apply_direct(data, depth, offset, term_d2);
×
4548
        }
4549
    }
4550
}
4551

4552

4553
static void
4554
diffuse_lso2_carry(int32_t *carry_curr, int32_t *carry_next, int32_t *carry_far,
×
4555
                   int width, int height, int depth,
4556
                   int x, int y, int32_t error,
4557
                   int index, int direction, int channel)
4558
{
4559
    const int (*table)[7];
4560
    const int *entry;
4561
    int denom;
4562
    int32_t term_r = 0;
×
4563
    int32_t term_r2 = 0;
×
4564
    int32_t term_dl = 0;
×
4565
    int32_t term_d = 0;
×
4566
    int32_t term_dr = 0;
×
4567
    int32_t term_d2 = 0;
×
4568
    size_t base;
4569

4570
    if (error == 0) {
×
4571
        return;
×
4572
    }
4573
    if (index < 0) {
×
4574
        index = 0;
×
4575
    }
4576
    if (index > 255) {
×
4577
        index = 255;
×
4578
    }
4579

4580
    table = lso2_table();
×
4581
    entry = table[index];
×
4582
    denom = entry[6];
×
4583
    if (denom == 0) {
×
4584
        return;
×
4585
    }
4586

4587
    term_r = diffuse_varerr_term(error, entry[0], denom);
×
4588
    term_r2 = diffuse_varerr_term(error, entry[1], denom);
×
4589
    term_dl = diffuse_varerr_term(error, entry[2], denom);
×
4590
    term_d = diffuse_varerr_term(error, entry[3], denom);
×
4591
    term_dr = diffuse_varerr_term(error, entry[4], denom);
×
4592
    term_d2 = error - term_r - term_r2 - term_dl - term_d - term_dr;
×
4593

4594
    if (direction >= 0) {
×
4595
        if (x + 1 < width) {
×
4596
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4597
            carry_curr[base] += term_r;
×
4598
        }
4599
        if (x + 2 < width) {
×
4600
            base = ((size_t)(x + 2) * (size_t)depth) + (size_t)channel;
×
4601
            carry_curr[base] += term_r2;
×
4602
        }
4603
        if (y + 1 < height && x - 1 >= 0) {
×
4604
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4605
            carry_next[base] += term_dl;
×
4606
        }
4607
        if (y + 1 < height) {
×
4608
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4609
            carry_next[base] += term_d;
×
4610
        }
4611
        if (y + 1 < height && x + 1 < width) {
×
4612
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4613
            carry_next[base] += term_dr;
×
4614
        }
4615
        if (y + 2 < height) {
×
4616
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4617
            carry_far[base] += term_d2;
×
4618
        }
4619
    } else {
4620
        if (x - 1 >= 0) {
×
4621
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4622
            carry_curr[base] += term_r;
×
4623
        }
4624
        if (x - 2 >= 0) {
×
4625
            base = ((size_t)(x - 2) * (size_t)depth) + (size_t)channel;
×
4626
            carry_curr[base] += term_r;
×
4627
        }
4628
        if (y + 1 < height && x + 1 < width) {
×
4629
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4630
            carry_next[base] += term_dl;
×
4631
        }
4632
        if (y + 1 < height) {
×
4633
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4634
            carry_next[base] += term_d;
×
4635
        }
4636
        if (y + 1 < height && x - 1 >= 0) {
×
4637
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4638
            carry_next[base] += term_dr;
×
4639
        }
4640
        if (y + 2 < height) {
×
4641
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4642
            carry_far[base] += term_d2;
×
4643
        }
4644
    }
4645
}
4646

4647

4648
static void
4649
scanline_params(int serpentine, int index, int limit,
59,650✔
4650
                int *start, int *end, int *step, int *direction)
4651
{
4652
    if (serpentine && (index & 1)) {
59,650✔
4653
        *start = limit - 1;
675✔
4654
        *end = -1;
675✔
4655
        *step = -1;
675✔
4656
        *direction = -1;
675✔
4657
    } else {
225✔
4658
        *start = 0;
58,975✔
4659
        *end = limit;
58,975✔
4660
        *step = 1;
58,975✔
4661
        *direction = 1;
58,975✔
4662
    }
4663
}
59,650✔
4664

4665

4666
static SIXELSTATUS
4667
apply_palette_positional(
×
4668
    sixel_index_t *result,
4669
    unsigned char *data,
4670
    int width,
4671
    int height,
4672
    int depth,
4673
    unsigned char *palette,
4674
    int reqcolor,
4675
    int methodForDiffuse,
4676
    int methodForScan,
4677
    int foptimize_palette,
4678
    int (*f_lookup)(const unsigned char *pixel,
4679
                    int depth,
4680
                    const unsigned char *palette,
4681
                    int reqcolor,
4682
                    unsigned short *cachetable,
4683
                    int complexion),
4684
    unsigned short *indextable,
4685
    int complexion,
4686
    unsigned char copy[],
4687
    unsigned char new_palette[],
4688
    unsigned short migration_map[],
4689
    int *ncolors)
4690
{
4691
    int serpentine;
4692
    int y;
4693
    float (*f_mask)(int x, int y, int c);
4694

4695
    switch (methodForDiffuse) {
×
4696
    case SIXEL_DIFFUSE_A_DITHER:
4697
        f_mask = mask_a;
×
4698
        break;
×
4699
    case SIXEL_DIFFUSE_X_DITHER:
×
4700
    default:
4701
        f_mask = mask_x;
×
4702
        break;
×
4703
    }
4704

4705
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
×
4706

4707
    if (foptimize_palette) {
×
4708
        int x;
4709

4710
        *ncolors = 0;
×
4711
        memset(new_palette, 0x00,
×
4712
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
4713
        memset(migration_map, 0x00,
×
4714
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
4715
        for (y = 0; y < height; ++y) {
×
4716
            int start;
4717
            int end;
4718
            int step;
4719
            int direction;
4720

4721
            scanline_params(serpentine, y, width,
×
4722
                            &start, &end, &step, &direction);
4723
            (void)direction;
4724
            for (x = start; x != end; x += step) {
×
4725
                int pos;
4726
                int d;
4727
                int color_index;
4728

4729
                pos = y * width + x;
×
4730
                for (d = 0; d < depth; ++d) {
×
4731
                    int val;
4732

4733
                    val = data[pos * depth + d]
×
4734
                        + f_mask(x, y, d) * 32;
×
4735
                    copy[d] = val < 0 ? 0
×
4736
                               : val > 255 ? 255 : val;
×
4737
                }
4738
                color_index = f_lookup(copy, depth, palette,
×
4739
                                       reqcolor, indextable,
4740
                                       complexion);
4741
                if (migration_map[color_index] == 0) {
×
4742
                    result[pos] = *ncolors;
×
4743
                    for (d = 0; d < depth; ++d) {
×
4744
                        new_palette[*ncolors * depth + d]
×
4745
                            = palette[color_index * depth + d];
×
4746
                    }
4747
                    ++*ncolors;
×
4748
                    migration_map[color_index] = *ncolors;
×
4749
                } else {
4750
                    result[pos] = migration_map[color_index] - 1;
×
4751
                }
4752
            }
4753
        }
4754
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
×
4755
    } else {
4756
        int x;
4757

4758
        for (y = 0; y < height; ++y) {
×
4759
            int start;
4760
            int end;
4761
            int step;
4762
            int direction;
4763

4764
            scanline_params(serpentine, y, width,
×
4765
                            &start, &end, &step, &direction);
4766
            (void)direction;
4767
            for (x = start; x != end; x += step) {
×
4768
                int pos;
4769
                int d;
4770

4771
                pos = y * width + x;
×
4772
                for (d = 0; d < depth; ++d) {
×
4773
                    int val;
4774

4775
                    val = data[pos * depth + d]
×
4776
                        + f_mask(x, y, d) * 32;
×
4777
                    copy[d] = val < 0 ? 0
×
4778
                               : val > 255 ? 255 : val;
×
4779
                }
4780
                result[pos] = f_lookup(copy, depth, palette,
×
4781
                                       reqcolor, indextable,
4782
                                       complexion);
4783
            }
4784
        }
4785
        *ncolors = reqcolor;
×
4786
    }
4787

4788
    return SIXEL_OK;
×
4789
}
4790

4791

4792
static SIXELSTATUS
4793
apply_palette_variable(
×
4794
    sixel_index_t *result,
4795
    unsigned char *data,
4796
    int width,
4797
    int height,
4798
    int depth,
4799
    unsigned char *palette,
4800
    int reqcolor,
4801
    int methodForScan,
4802
    int foptimize_palette,
4803
    int (*f_lookup)(const unsigned char *pixel,
4804
                    int depth,
4805
                    const unsigned char *palette,
4806
                    int reqcolor,
4807
                    unsigned short *cachetable,
4808
                    int complexion),
4809
    unsigned short *indextable,
4810
    int complexion,
4811
    unsigned char new_palette[],
4812
    unsigned short migration_map[],
4813
    int *ncolors,
4814
    int methodForDiffuse,
4815
    int methodForCarry)
4816
{
4817
    SIXELSTATUS status = SIXEL_FALSE;
×
4818
#if _MSC_VER
4819
    enum { max_channels = 4 };
4820
#else
4821
    const int max_channels = 4;
×
4822
#endif
4823
    int serpentine;
4824
    int y;
4825
    diffuse_varerr_mode varerr_diffuse;
4826
    diffuse_varerr_carry_mode varerr_diffuse_carry;
4827
    int use_carry;
4828
    size_t carry_len;
4829
    int32_t *carry_curr = NULL;
×
4830
    int32_t *carry_next = NULL;
×
4831
    int32_t *carry_far = NULL;
×
4832
    unsigned char corrected[max_channels];
4833
    int32_t sample_scaled[max_channels];
4834
    int32_t accum_scaled[max_channels];
4835
    int start;
4836
    int end;
4837
    int step;
4838
    int direction;
4839
    int x;
4840
    int pos;
4841
    size_t base;
4842
    size_t carry_base;
4843
    const unsigned char *source_pixel;
4844
    int color_index;
4845
    int output_index;
4846
    int n;
4847
    int palette_value;
4848
    int diff;
4849
    int table_index;
4850
    int64_t accum;
4851
    int64_t clamped;
4852
    int32_t target_scaled;
4853
    int32_t error_scaled;
4854
    int32_t *tmp;
4855

4856
    if (depth > max_channels) {
×
4857
        status = SIXEL_BAD_ARGUMENT;
×
4858
        goto end;
×
4859
    }
4860

4861
    use_carry = (methodForCarry == SIXEL_CARRY_ENABLE);
×
4862
    carry_len = 0;
×
4863

4864
    switch (methodForDiffuse) {
×
4865
    case SIXEL_DIFFUSE_LSO2:
4866
        varerr_diffuse = diffuse_lso2;
×
4867
        varerr_diffuse_carry = diffuse_lso2_carry;
×
4868
        break;
×
4869
    default:
4870
        varerr_diffuse = diffuse_lso2;
×
4871
        varerr_diffuse_carry = diffuse_lso2_carry;
×
4872
        break;
×
4873
    }
4874

4875
    if (use_carry) {
×
4876
        carry_len = (size_t)width * (size_t)depth;
×
4877
        carry_curr = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4878
        if (carry_curr == NULL) {
×
4879
            status = SIXEL_BAD_ALLOCATION;
×
4880
            goto end;
×
4881
        }
4882
        carry_next = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4883
        if (carry_next == NULL) {
×
4884
            status = SIXEL_BAD_ALLOCATION;
×
4885
            goto end;
×
4886
        }
4887
        carry_far = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4888
        if (carry_far == NULL) {
×
4889
            status = SIXEL_BAD_ALLOCATION;
×
4890
            goto end;
×
4891
        }
4892
    }
4893

4894
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
×
4895

4896
    if (foptimize_palette) {
×
4897
        *ncolors = 0;
×
4898
        memset(new_palette, 0x00,
×
4899
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
4900
        memset(migration_map, 0x00,
×
4901
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
4902
    }
4903

4904
    for (y = 0; y < height; ++y) {
×
4905
        scanline_params(serpentine, y, width,
×
4906
                        &start, &end, &step, &direction);
4907
        for (x = start; x != end; x += step) {
×
4908
            pos = y * width + x;
×
4909
            base = (size_t)pos * (size_t)depth;
×
4910
            carry_base = (size_t)x * (size_t)depth;
×
4911
            if (use_carry) {
×
4912
                for (n = 0; n < depth; ++n) {
×
4913
                    accum = ((int64_t)data[base + n]
×
4914
                             << VARERR_SCALE_SHIFT)
×
4915
                          + carry_curr[carry_base + (size_t)n];
×
4916
                    if (accum < INT32_MIN) {
×
4917
                        accum = INT32_MIN;
×
4918
                    } else if (accum > INT32_MAX) {
×
4919
                        accum = INT32_MAX;
×
4920
                    }
4921
                    carry_curr[carry_base + (size_t)n] = 0;
×
4922
                    clamped = accum;
×
4923
                    if (clamped < 0) {
×
4924
                        clamped = 0;
×
4925
                    } else if (clamped > VARERR_MAX_VALUE) {
×
4926
                        clamped = VARERR_MAX_VALUE;
×
4927
                    }
4928
                    accum_scaled[n] = (int32_t)clamped;
×
4929
                    corrected[n]
4930
                        = (unsigned char)((clamped + VARERR_ROUND)
×
4931
                                          >> VARERR_SCALE_SHIFT);
×
4932
                }
4933
                source_pixel = corrected;
×
4934
            } else {
4935
                for (n = 0; n < depth; ++n) {
×
4936
                    sample_scaled[n]
4937
                        = (int32_t)data[base + n]
×
4938
                        << VARERR_SCALE_SHIFT;
×
4939
                    corrected[n] = data[base + n];
×
4940
                }
4941
                source_pixel = data + base;
×
4942
            }
4943

4944
            color_index = f_lookup(source_pixel, depth, palette,
×
4945
                                   reqcolor, indextable,
4946
                                   complexion);
4947

4948
            if (foptimize_palette) {
×
4949
                if (migration_map[color_index] == 0) {
×
4950
                    output_index = *ncolors;
×
4951
                    for (n = 0; n < depth; ++n) {
×
4952
                        new_palette[output_index * depth + n]
×
4953
                            = palette[color_index * depth + n];
×
4954
                    }
4955
                    ++*ncolors;
×
4956
                    migration_map[color_index] = *ncolors;
×
4957
                } else {
4958
                    output_index = migration_map[color_index] - 1;
×
4959
                }
4960
                result[pos] = output_index;
×
4961
            } else {
4962
                output_index = color_index;
×
4963
                result[pos] = output_index;
×
4964
            }
4965

4966
            for (n = 0; n < depth; ++n) {
×
4967
                if (foptimize_palette) {
×
4968
                    palette_value = new_palette[output_index * depth + n];
×
4969
                } else {
4970
                    palette_value = palette[color_index * depth + n];
×
4971
                }
4972
                diff = (int)source_pixel[n] - palette_value;
×
4973
                if (diff < 0) {
×
4974
                    diff = -diff;
×
4975
                }
4976
                if (diff > 255) {
×
4977
                    diff = 255;
×
4978
                }
4979
                table_index = diff;
×
4980
                if (use_carry) {
×
4981
                    target_scaled = (int32_t)palette_value
×
4982
                                  << VARERR_SCALE_SHIFT;
4983
                    error_scaled = accum_scaled[n] - target_scaled;
×
4984
                    varerr_diffuse_carry(carry_curr, carry_next, carry_far,
×
4985
                                         width, height, depth,
4986
                                         x, y, error_scaled,
4987
                                         table_index,
4988
                                         direction, n);
4989
                } else {
4990
                    target_scaled = (int32_t)palette_value
×
4991
                                  << VARERR_SCALE_SHIFT;
4992
                    error_scaled = sample_scaled[n] - target_scaled;
×
4993
                    varerr_diffuse(data + n, width, height,
×
4994
                                   x, y, depth, error_scaled,
4995
                                   table_index,
4996
                                   direction);
4997
                }
4998
            }
4999
        }
5000
        if (use_carry) {
×
5001
            tmp = carry_curr;
×
5002
            carry_curr = carry_next;
×
5003
            carry_next = carry_far;
×
5004
            carry_far = tmp;
×
5005
            if (carry_len > 0) {
×
5006
                memset(carry_far, 0x00, carry_len * sizeof(int32_t));
×
5007
            }
5008
        }
5009
    }
5010

5011
    if (foptimize_palette) {
×
5012
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
×
5013
    } else {
5014
        *ncolors = reqcolor;
×
5015
    }
5016

5017
    status = SIXEL_OK;
×
5018

5019
end:
5020
    free(carry_next);
×
5021
    free(carry_curr);
×
5022
    free(carry_far);
×
5023
    return status;
×
5024
}
5025

5026

5027
static SIXELSTATUS
5028
apply_palette_fixed(
285✔
5029
    sixel_index_t *result,
5030
    unsigned char *data,
5031
    int width,
5032
    int height,
5033
    int depth,
5034
    unsigned char *palette,
5035
    int reqcolor,
5036
    int methodForScan,
5037
    int foptimize_palette,
5038
    int (*f_lookup)(const unsigned char *pixel,
5039
                    int depth,
5040
                    const unsigned char *palette,
5041
                    int reqcolor,
5042
                    unsigned short *cachetable,
5043
                    int complexion),
5044
    unsigned short *indextable,
5045
    int complexion,
5046
    unsigned char new_palette[],
5047
    unsigned short migration_map[],
5048
    int *ncolors,
5049
    int methodForDiffuse,
5050
    int methodForCarry)
5051
{
170✔
5052
#if _MSC_VER
5053
    enum { max_channels = 4 };
5054
#else
5055
    const int max_channels = 4;
285✔
5056
#endif
5057
    SIXELSTATUS status = SIXEL_FALSE;
285✔
5058
    int serpentine;
5059
    int y;
5060
    void (*f_diffuse)(unsigned char *data,
5061
                      int width,
5062
                      int height,
5063
                      int x,
5064
                      int y,
5065
                      int depth,
5066
                      int offset,
5067
                      int direction);
5068
    diffuse_fixed_carry_mode f_diffuse_carry;
5069
    int use_carry;
5070
    size_t carry_len;
5071
    int32_t *carry_curr = NULL;
285✔
5072
    int32_t *carry_next = NULL;
285✔
5073
    int32_t *carry_far = NULL;
285✔
5074
    unsigned char corrected[max_channels];
170✔
5075
    int32_t accum_scaled[max_channels];
170✔
5076
    int start;
5077
    int end;
5078
    int step;
5079
    int direction;
5080
    int x;
5081
    int pos;
5082
    size_t base;
5083
    size_t carry_base;
5084
    const unsigned char *source_pixel;
5085
    int color_index;
5086
    int output_index;
5087
    int n;
5088
    int palette_value;
5089
    int64_t accum;
5090
    int64_t clamped;
5091
    int32_t target_scaled;
5092
    int32_t error_scaled;
5093
    int offset;
5094
    int32_t *tmp;
5095

5096
    if (depth > max_channels) {
285!
5097
        status = SIXEL_BAD_ARGUMENT;
×
5098
        goto end;
×
5099
    }
5100

5101
    use_carry = (methodForCarry == SIXEL_CARRY_ENABLE);
285✔
5102
    carry_len = 0;
285✔
5103

5104
    if (depth != 3) {
285!
5105
        f_diffuse = diffuse_none;
×
5106
        f_diffuse_carry = diffuse_none_carry;
×
5107
        use_carry = 0;
×
5108
    } else {
5109
        switch (methodForDiffuse) {
285!
5110
        case SIXEL_DIFFUSE_NONE:
86✔
5111
            f_diffuse = diffuse_none;
158✔
5112
            f_diffuse_carry = diffuse_none_carry;
158✔
5113
            break;
158✔
5114
        case SIXEL_DIFFUSE_ATKINSON:
44✔
5115
            f_diffuse = diffuse_atkinson;
67✔
5116
            f_diffuse_carry = diffuse_atkinson_carry;
67✔
5117
            break;
67✔
5118
        case SIXEL_DIFFUSE_FS:
34✔
5119
            f_diffuse = diffuse_fs;
51✔
5120
            f_diffuse_carry = diffuse_fs_carry;
51✔
5121
            break;
51✔
5122
        case SIXEL_DIFFUSE_JAJUNI:
2✔
5123
            f_diffuse = diffuse_jajuni;
3✔
5124
            f_diffuse_carry = diffuse_jajuni_carry;
3✔
5125
            break;
3✔
5126
        case SIXEL_DIFFUSE_STUCKI:
2✔
5127
            f_diffuse = diffuse_stucki;
3✔
5128
            f_diffuse_carry = diffuse_stucki_carry;
3✔
5129
            break;
3✔
5130
        case SIXEL_DIFFUSE_BURKES:
2✔
5131
            f_diffuse = diffuse_burkes;
3✔
5132
            f_diffuse_carry = diffuse_burkes_carry;
3✔
5133
            break;
3✔
5134
        case SIXEL_DIFFUSE_SIERRA1:
5135
            f_diffuse = diffuse_sierra1;
×
5136
            f_diffuse_carry = diffuse_sierra1_carry;
×
5137
            break;
×
5138
        case SIXEL_DIFFUSE_SIERRA2:
5139
            f_diffuse = diffuse_sierra2;
×
5140
            f_diffuse_carry = diffuse_sierra2_carry;
×
5141
            break;
×
5142
        case SIXEL_DIFFUSE_SIERRA3:
5143
            f_diffuse = diffuse_sierra3;
×
5144
            f_diffuse_carry = diffuse_sierra3_carry;
×
5145
            break;
×
5146
        default:
5147
            quant_trace(stderr,
×
5148
                        "Internal error: invalid methodForDiffuse: %d\n",
5149
                        methodForDiffuse);
5150
            f_diffuse = diffuse_none;
×
5151
            f_diffuse_carry = diffuse_none_carry;
×
5152
            break;
×
5153
        }
5154
    }
5155

5156
    if (use_carry) {
285!
5157
        carry_len = (size_t)width * (size_t)depth;
×
5158
        if (carry_len > 0) {
×
5159
            carry_curr = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5160
            if (carry_curr == NULL) {
×
5161
                status = SIXEL_BAD_ALLOCATION;
×
5162
                goto end;
×
5163
            }
5164
            carry_next = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5165
            if (carry_next == NULL) {
×
5166
                status = SIXEL_BAD_ALLOCATION;
×
5167
                goto end;
×
5168
            }
5169
            carry_far = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5170
            if (carry_far == NULL) {
×
5171
                status = SIXEL_BAD_ALLOCATION;
×
5172
                goto end;
×
5173
            }
5174
        } else {
5175
            use_carry = 0;
×
5176
        }
5177
    }
5178

5179
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
285✔
5180

5181
    if (foptimize_palette) {
285✔
5182
        *ncolors = 0;
206✔
5183
        memset(new_palette, 0x00,
206✔
5184
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
130✔
5185
        memset(migration_map, 0x00,
206✔
5186
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
5187
    } else {
76✔
5188
        *ncolors = reqcolor;
79✔
5189
    }
5190

5191
    for (y = 0; y < height; ++y) {
59,935✔
5192
        scanline_params(serpentine, y, width,
59,650✔
5193
                        &start, &end, &step, &direction);
5194
        for (x = start; x != end; x += step) {
43,528,028✔
5195
            pos = y * width + x;
43,468,378✔
5196
            base = (size_t)pos * (size_t)depth;
43,468,378✔
5197
            carry_base = (size_t)x * (size_t)depth;
43,468,378✔
5198
            if (use_carry) {
43,468,378!
5199
                for (n = 0; n < depth; ++n) {
×
5200
                    accum = ((int64_t)data[base + n]
×
5201
                             << VARERR_SCALE_SHIFT)
×
5202
                           + carry_curr[carry_base + (size_t)n];
×
5203
                    if (accum < INT32_MIN) {
×
5204
                        accum = INT32_MIN;
×
5205
                    } else if (accum > INT32_MAX) {
×
5206
                        accum = INT32_MAX;
×
5207
                    }
5208
                    clamped = accum;
×
5209
                    if (clamped < 0) {
×
5210
                        clamped = 0;
×
5211
                    } else if (clamped > VARERR_MAX_VALUE) {
×
5212
                        clamped = VARERR_MAX_VALUE;
×
5213
                    }
5214
                    accum_scaled[n] = (int32_t)clamped;
×
5215
                    corrected[n]
5216
                        = (unsigned char)((clamped + VARERR_ROUND)
×
5217
                                          >> VARERR_SCALE_SHIFT);
×
5218
                    data[base + n] = corrected[n];
×
5219
                    carry_curr[carry_base + (size_t)n] = 0;
×
5220
                }
5221
                source_pixel = corrected;
×
5222
            } else {
5223
                source_pixel = data + base;
43,468,378✔
5224
            }
5225

5226
            color_index = f_lookup(source_pixel, depth, palette,
63,604,336✔
5227
                                   reqcolor, indextable,
20,135,958✔
5228
                                   complexion);
20,135,958✔
5229

5230
            if (foptimize_palette) {
43,468,378✔
5231
                if (migration_map[color_index] == 0) {
31,366,878✔
5232
                    output_index = *ncolors;
16,650✔
5233
                    for (n = 0; n < depth; ++n) {
66,600✔
5234
                        new_palette[output_index * depth + n]
49,950✔
5235
                            = palette[color_index * depth + n];
67,068✔
5236
                    }
17,118✔
5237
                    ++*ncolors;
16,650✔
5238
                    migration_map[color_index] = *ncolors;
16,650✔
5239
                } else {
5,706✔
5240
                    output_index = migration_map[color_index] - 1;
31,350,228✔
5241
                }
5242
                result[pos] = output_index;
31,366,878✔
5243
            } else {
14,296,138✔
5244
                output_index = color_index;
12,101,500✔
5245
                result[pos] = output_index;
12,101,500✔
5246
            }
5247

5248
            for (n = 0; n < depth; ++n) {
173,873,512✔
5249
                if (foptimize_palette) {
130,405,134✔
5250
                    palette_value = new_palette[output_index * depth + n];
94,100,634✔
5251
                } else {
42,888,414✔
5252
                    palette_value = palette[color_index * depth + n];
36,304,500✔
5253
                }
5254
                if (use_carry) {
130,405,134!
5255
                    target_scaled = (int32_t)palette_value
×
5256
                                  << VARERR_SCALE_SHIFT;
5257
                    error_scaled = accum_scaled[n] - target_scaled;
×
5258
                    f_diffuse_carry(carry_curr, carry_next, carry_far,
×
5259
                                    width, height, depth,
5260
                                    x, y, error_scaled, direction, n);
5261
                } else {
5262
                    offset = (int)source_pixel[n] - palette_value;
130,405,134✔
5263
                    f_diffuse(data + n, width, height, x, y,
190,813,008✔
5264
                              depth, offset, direction);
60,407,874✔
5265
                }
5266
            }
60,407,874✔
5267
        }
20,135,958✔
5268
        if (use_carry) {
59,650!
5269
            tmp = carry_curr;
×
5270
            carry_curr = carry_next;
×
5271
            carry_next = carry_far;
×
5272
            carry_far = tmp;
×
5273
            if (carry_len > 0) {
×
5274
                memset(carry_far, 0x00, carry_len * sizeof(int32_t));
×
5275
            }
5276
        }
5277
    }
26,750✔
5278

5279
    if (foptimize_palette) {
285✔
5280
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
206✔
5281
    }
76✔
5282

5283
    status = SIXEL_OK;
285✔
5284

5285
end:
170✔
5286
    free(carry_far);
285✔
5287
    free(carry_next);
285✔
5288
    free(carry_curr);
285✔
5289
    return status;
285✔
5290
}
5291

5292

5293
static void
5294
diffuse_none(unsigned char *data, int width, int height,
31,038,318✔
5295
             int x, int y, int depth, int error, int direction)
5296
{
5297
    /* unused */ (void) data;
27,259,002✔
5298
    /* unused */ (void) width;
27,259,002✔
5299
    /* unused */ (void) height;
27,259,002✔
5300
    /* unused */ (void) x;
27,259,002✔
5301
    /* unused */ (void) y;
27,259,002✔
5302
    /* unused */ (void) depth;
27,259,002✔
5303
    /* unused */ (void) error;
27,259,002✔
5304
    /* unused */ (void) direction;
27,259,002✔
5305
}
31,038,318✔
5306

5307

5308
static void
5309
diffuse_none_carry(int32_t *carry_curr, int32_t *carry_next,
×
5310
                   int32_t *carry_far, int width, int height,
5311
                   int depth, int x, int y, int32_t error,
5312
                   int direction, int channel)
5313
{
5314
    /* unused */ (void) carry_curr;
5315
    /* unused */ (void) carry_next;
5316
    /* unused */ (void) carry_far;
5317
    /* unused */ (void) width;
5318
    /* unused */ (void) height;
5319
    /* unused */ (void) depth;
5320
    /* unused */ (void) x;
5321
    /* unused */ (void) y;
5322
    /* unused */ (void) error;
5323
    /* unused */ (void) direction;
5324
    /* unused */ (void) channel;
5325
}
×
5326

5327

5328
static void
5329
diffuse_fs(unsigned char *data, int width, int height,
70,430,256✔
5330
           int x, int y, int depth, int error, int direction)
5331
{
5332
    /* Floyd Steinberg Method
5333
     *          curr    7/16
5334
     *  3/16    5/16    1/16
5335
     */
5336
    int pos;
5337
    int forward;
5338

5339
    pos = y * width + x;
70,430,256✔
5340
    forward = direction >= 0;
70,430,256✔
5341

5342
    if (forward) {
70,430,256✔
5343
        if (x < width - 1) {
69,215,256✔
5344
            error_diffuse_normal(data, pos + 1, depth, error, 7, 16);
69,144,759✔
5345
        }
23,048,253✔
5346
        if (y < height - 1) {
69,215,256✔
5347
            if (x > 0) {
69,121,332✔
5348
                error_diffuse_normal(data,
92,067,972✔
5349
                                     pos + width - 1,
69,050,979✔
5350
                                     depth, error, 3, 16);
23,016,993✔
5351
            }
23,016,993✔
5352
            error_diffuse_normal(data,
92,161,776✔
5353
                                 pos + width,
23,040,444✔
5354
                                 depth, error, 5, 16);
23,040,444✔
5355
            if (x < width - 1) {
69,121,332✔
5356
                error_diffuse_normal(data,
92,067,972✔
5357
                                     pos + width + 1,
69,050,979✔
5358
                                     depth, error, 1, 16);
23,016,993✔
5359
            }
23,016,993✔
5360
        }
23,040,444✔
5361
    } else {
23,071,752✔
5362
        if (x > 0) {
1,215,000✔
5363
            error_diffuse_normal(data, pos - 1, depth, error, 7, 16);
1,212,975✔
5364
        }
404,325✔
5365
        if (y < height - 1) {
1,215,000✔
5366
            if (x < width - 1) {
1,209,600✔
5367
                error_diffuse_normal(data,
1,610,112✔
5368
                                     pos + width + 1,
1,207,584✔
5369
                                     depth, error, 3, 16);
402,528✔
5370
            }
402,528✔
5371
            error_diffuse_normal(data,
1,612,800✔
5372
                                 pos + width,
403,200✔
5373
                                 depth, error, 5, 16);
403,200✔
5374
            if (x > 0) {
1,209,600✔
5375
                error_diffuse_normal(data,
1,610,112✔
5376
                                     pos + width - 1,
1,207,584✔
5377
                                     depth, error, 1, 16);
402,528✔
5378
            }
402,528✔
5379
        }
403,200✔
5380
    }
5381
}
70,430,256✔
5382

5383

5384
static void
5385
diffuse_fs_carry(int32_t *carry_curr, int32_t *carry_next,
×
5386
                 int32_t *carry_far, int width, int height,
5387
                 int depth, int x, int y, int32_t error,
5388
                 int direction, int channel)
5389
{
5390
    /* Floyd Steinberg Method
5391
     *          curr    7/16
5392
     *  3/16    5/48    1/16
5393
     */
5394
    int forward;
5395

5396
    /* unused */ (void) carry_far;
5397
    if (error == 0) {
×
5398
        return;
×
5399
    }
5400

5401
    forward = direction >= 0;
×
5402
    if (forward) {
×
5403
        if (x + 1 < width) {
×
5404
            size_t base;
5405
            int32_t term;
5406

5407
            base = ((size_t)(x + 1) * (size_t)depth)
×
5408
                 + (size_t)channel;
×
5409
            term = diffuse_fixed_term(error, 7, 16);
×
5410
            carry_curr[base] += term;
×
5411
        }
5412
        if (y + 1 < height) {
×
5413
            if (x > 0) {
×
5414
                size_t base;
5415
                int32_t term;
5416

5417
                base = ((size_t)(x - 1) * (size_t)depth)
×
5418
                     + (size_t)channel;
×
5419
                term = diffuse_fixed_term(error, 3, 16);
×
5420
                carry_next[base] += term;
×
5421
            }
5422
            {
5423
                size_t base;
5424
                int32_t term;
5425

5426
                base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5427
                term = diffuse_fixed_term(error, 5, 16);
×
5428
                carry_next[base] += term;
×
5429
            }
5430
            if (x + 1 < width) {
×
5431
                size_t base;
5432
                int32_t term;
5433

5434
                base = ((size_t)(x + 1) * (size_t)depth)
×
5435
                     + (size_t)channel;
×
5436
                term = diffuse_fixed_term(error, 1, 16);
×
5437
                carry_next[base] += term;
×
5438
            }
5439
        }
5440
    } else {
5441
        if (x - 1 >= 0) {
×
5442
            size_t base;
5443
            int32_t term;
5444

5445
            base = ((size_t)(x - 1) * (size_t)depth)
×
5446
                 + (size_t)channel;
×
5447
            term = diffuse_fixed_term(error, 7, 16);
×
5448
            carry_curr[base] += term;
×
5449
        }
5450
        if (y + 1 < height) {
×
5451
            if (x + 1 < width) {
×
5452
                size_t base;
5453
                int32_t term;
5454

5455
                base = ((size_t)(x + 1) * (size_t)depth)
×
5456
                     + (size_t)channel;
×
5457
                term = diffuse_fixed_term(error, 3, 16);
×
5458
                carry_next[base] += term;
×
5459
            }
5460
            {
5461
                size_t base;
5462
                int32_t term;
5463

5464
                base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5465
                term = diffuse_fixed_term(error, 5, 16);
×
5466
                carry_next[base] += term;
×
5467
            }
5468
            if (x - 1 >= 0) {
×
5469
                size_t base;
5470
                int32_t term;
5471

5472
                base = ((size_t)(x - 1) * (size_t)depth)
×
5473
                     + (size_t)channel;
×
5474
                term = diffuse_fixed_term(error, 1, 16);
×
5475
                carry_next[base] += term;
×
5476
            }
5477
        }
5478
    }
5479
}
5480

5481

5482
static void
5483
diffuse_atkinson(unsigned char *data, int width, int height,
28,352,460✔
5484
                 int x, int y, int depth, int error, int direction)
5485
{
5486
    /* Atkinson's Method
5487
     *          curr    1/8    1/8
5488
     *   1/8     1/8    1/8
5489
     *           1/8
5490
     */
5491
    int pos;
5492
    int sign;
5493

5494
    pos = y * width + x;
28,352,460✔
5495
    sign = direction >= 0 ? 1 : -1;
28,352,460!
5496

5497
    if (x + sign >= 0 && x + sign < width) {
28,352,460!
5498
        error_diffuse_fast(data, pos + sign, depth, error, 1, 8);
28,296,945✔
5499
    }
9,458,715✔
5500
    if (x + sign * 2 >= 0 && x + sign * 2 < width) {
28,352,460!
5501
        error_diffuse_fast(data, pos + sign * 2, depth, error, 1, 8);
28,241,430✔
5502
    }
9,440,010✔
5503
    if (y < height - 1) {
28,352,460✔
5504
        int row;
5505

5506
        row = pos + width;
28,278,756✔
5507
        if (x - sign >= 0 && x - sign < width) {
28,278,756!
5508
            error_diffuse_fast(data,
37,657,392✔
5509
                               row + (-sign),
9,433,950✔
5510
                               depth, error, 1, 8);
9,433,950✔
5511
        }
9,433,950✔
5512
        error_diffuse_fast(data, row, depth, error, 1, 8);
28,278,756✔
5513
        if (x + sign >= 0 && x + sign < width) {
28,278,756!
5514
            error_diffuse_fast(data,
37,657,392✔
5515
                               row + sign,
9,433,950✔
5516
                               depth, error, 1, 8);
9,433,950✔
5517
        }
9,433,950✔
5518
    }
9,452,586✔
5519
    if (y < height - 2) {
28,352,460✔
5520
        error_diffuse_fast(data, pos + width * 2, depth, error, 1, 8);
28,205,052✔
5521
    }
9,427,752✔
5522
}
28,352,460✔
5523

5524

5525
static void
5526
diffuse_atkinson_carry(int32_t *carry_curr, int32_t *carry_next,
×
5527
                       int32_t *carry_far, int width, int height,
5528
                       int depth, int x, int y, int32_t error,
5529
                       int direction, int channel)
5530
{
5531
    /* Atkinson's Method
5532
     *          curr    1/8    1/8
5533
     *   1/8     1/8    1/8
5534
     *           1/8
5535
     */
5536
    int sign;
5537
    int32_t term;
5538

5539
    if (error == 0) {
×
5540
        return;
×
5541
    }
5542

5543
    term = diffuse_fixed_term(error, 1, 8);
×
5544
    sign = direction >= 0 ? 1 : -1;
×
5545
    if (x + sign >= 0 && x + sign < width) {
×
5546
        size_t base;
5547

5548
        base = ((size_t)(x + sign) * (size_t)depth)
×
5549
             + (size_t)channel;
×
5550
        carry_curr[base] += term;
×
5551
    }
5552
    if (x + sign * 2 >= 0 && x + sign * 2 < width) {
×
5553
        size_t base;
5554

5555
        base = ((size_t)(x + sign * 2) * (size_t)depth)
×
5556
             + (size_t)channel;
×
5557
        carry_curr[base] += term;
×
5558
    }
5559
    if (y + 1 < height) {
×
5560
        if (x - sign >= 0 && x - sign < width) {
×
5561
            size_t base;
5562

5563
            base = ((size_t)(x - sign) * (size_t)depth)
×
5564
                 + (size_t)channel;
×
5565
            carry_next[base] += term;
×
5566
        }
5567
        {
5568
            size_t base;
5569

5570
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5571
            carry_next[base] += term;
×
5572
        }
5573
        if (x + sign >= 0 && x + sign < width) {
×
5574
            size_t base;
5575

5576
            base = ((size_t)(x + sign) * (size_t)depth)
×
5577
                 + (size_t)channel;
×
5578
            carry_next[base] += term;
×
5579
        }
5580
    }
5581
    if (y + 2 < height) {
×
5582
        size_t base;
5583

5584
        base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5585
        carry_far[base] += term;
×
5586
    }
5587
}
5588

5589

5590
static void
5591
diffuse_jajuni(unsigned char *data, int width, int height,
396,900✔
5592
               int x, int y, int depth, int error, int direction)
5593
{
5594
    /* Jarvis, Judice & Ninke Method
5595
     *                  curr    7/48    5/48
5596
     *  3/48    5/48    7/48    5/48    3/48
5597
     *  1/48    3/48    5/48    3/48    1/48
5598
     */
5599
    int pos;
5600
    int sign;
5601
    static const int row0_offsets[] = { 1, 2 };
5602
    static const int row0_weights[] = { 7, 5 };
5603
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5604
    static const int row1_weights[] = { 3, 5, 7, 5, 3 };
5605
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5606
    static const int row2_weights[] = { 1, 3, 5, 3, 1 };
5607
    int i;
5608

5609
    pos = y * width + x;
396,900✔
5610
    sign = direction >= 0 ? 1 : -1;
396,900!
5611

5612
    for (i = 0; i < 2; ++i) {
1,190,700✔
5613
        int neighbor;
5614

5615
        neighbor = x + sign * row0_offsets[i];
793,800✔
5616
        if (neighbor < 0 || neighbor >= width) {
793,800!
5617
            continue;
5,670✔
5618
        }
5619
        error_diffuse_precise(data,
1,050,840✔
5620
                              pos + (neighbor - x),
788,130✔
5621
                              depth, error,
262,710✔
5622
                              row0_weights[i], 48);
788,130✔
5623
    }
262,710✔
5624
    if (y < height - 1) {
396,900✔
5625
        int row;
5626

5627
        row = pos + width;
395,010✔
5628
        for (i = 0; i < 5; ++i) {
2,370,060✔
5629
            int neighbor;
5630

5631
            neighbor = x + sign * row1_offsets[i];
1,975,050✔
5632
            if (neighbor < 0 || neighbor >= width) {
1,975,050✔
5633
                continue;
11,286✔
5634
            }
5635
            error_diffuse_precise(data,
2,618,352✔
5636
                                  row + (neighbor - x),
1,963,764✔
5637
                                  depth, error,
654,588✔
5638
                                  row1_weights[i], 48);
1,963,764✔
5639
        }
654,588✔
5640
    }
131,670✔
5641
    if (y < height - 2) {
396,900✔
5642
        int row;
5643

5644
        row = pos + width * 2;
393,120✔
5645
        for (i = 0; i < 5; ++i) {
2,358,720✔
5646
            int neighbor;
5647

5648
            neighbor = x + sign * row2_offsets[i];
1,965,600✔
5649
            if (neighbor < 0 || neighbor >= width) {
1,965,600✔
5650
                continue;
11,232✔
5651
            }
5652
            error_diffuse_precise(data,
2,605,824✔
5653
                                  row + (neighbor - x),
1,954,368✔
5654
                                  depth, error,
651,456✔
5655
                                  row2_weights[i], 48);
1,954,368✔
5656
        }
651,456✔
5657
    }
131,040✔
5658
}
396,900✔
5659

5660

5661
static void
5662
diffuse_jajuni_carry(int32_t *carry_curr, int32_t *carry_next,
×
5663
                     int32_t *carry_far, int width, int height,
5664
                     int depth, int x, int y, int32_t error,
5665
                     int direction, int channel)
5666
{
5667
    /* Jarvis, Judice & Ninke Method
5668
     *                  curr    7/48    5/48
5669
     *  3/48    5/48    7/48    5/48    3/48
5670
     *  1/48    3/48    5/48    3/48    1/48
5671
     */
5672
    static const int row0_offsets[] = { 1, 2 };
5673
    static const int row0_weights[] = { 7, 5 };
5674
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5675
    static const int row1_weights[] = { 3, 5, 7, 5, 3 };
5676
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5677
    static const int row2_weights[] = { 1, 3, 5, 3, 1 };
5678
    int sign;
5679
    int i;
5680

5681
    if (error == 0) {
×
5682
        return;
×
5683
    }
5684

5685
    sign = direction >= 0 ? 1 : -1;
×
5686
    for (i = 0; i < 2; ++i) {
×
5687
        int neighbor;
5688
        int32_t term;
5689

5690
        neighbor = x + sign * row0_offsets[i];
×
5691
        if (neighbor < 0 || neighbor >= width) {
×
5692
            continue;
×
5693
        }
5694
        term = diffuse_fixed_term(error, row0_weights[i], 48);
×
5695
        carry_curr[((size_t)neighbor * (size_t)depth)
×
5696
                   + (size_t)channel] += term;
×
5697
    }
5698
    if (y + 1 < height) {
×
5699
        for (i = 0; i < 5; ++i) {
×
5700
            int neighbor;
5701
            int32_t term;
5702

5703
            neighbor = x + sign * row1_offsets[i];
×
5704
            if (neighbor < 0 || neighbor >= width) {
×
5705
                continue;
×
5706
            }
5707
            term = diffuse_fixed_term(error, row1_weights[i], 48);
×
5708
            carry_next[((size_t)neighbor * (size_t)depth)
×
5709
                       + (size_t)channel] += term;
×
5710
        }
5711
    }
5712
    if (y + 2 < height) {
×
5713
        for (i = 0; i < 5; ++i) {
×
5714
            int neighbor;
5715
            int32_t term;
5716

5717
            neighbor = x + sign * row2_offsets[i];
×
5718
            if (neighbor < 0 || neighbor >= width) {
×
5719
                continue;
×
5720
            }
5721
            term = diffuse_fixed_term(error, row2_weights[i], 48);
×
5722
            carry_far[((size_t)neighbor * (size_t)depth)
×
5723
                      + (size_t)channel] += term;
×
5724
        }
5725
    }
5726
}
5727

5728

5729
static void
5730
diffuse_stucki(unsigned char *data, int width, int height,
119,700✔
5731
               int x, int y, int depth, int error, int direction)
5732
{
5733
    /* Stucki's Method
5734
     *                  curr    8/48    4/48
5735
     *  2/48    4/48    8/48    4/48    2/48
5736
     *  1/48    2/48    4/48    2/48    1/48
5737
     */
5738
    int pos;
5739
    int sign;
5740
    static const int row0_offsets[] = { 1, 2 };
5741
    static const int row0_num[] = { 1, 1 };
5742
    static const int row0_den[] = { 6, 12 };
5743
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5744
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5745
    static const int row1_den[] = { 24, 12, 6, 12, 24 };
5746
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5747
    static const int row2_num[] = { 1, 1, 1, 1, 1 };
5748
    static const int row2_den[] = { 48, 24, 12, 24, 48 };
5749
    int i;
5750

5751
    pos = y * width + x;
119,700✔
5752
    sign = direction >= 0 ? 1 : -1;
119,700!
5753

5754
    for (i = 0; i < 2; ++i) {
359,100✔
5755
        int neighbor;
5756

5757
        neighbor = x + sign * row0_offsets[i];
239,400✔
5758
        if (neighbor < 0 || neighbor >= width) {
239,400!
5759
            continue;
2,700✔
5760
        }
5761
        error_diffuse_precise(data,
315,600✔
5762
                              pos + (neighbor - x),
236,700✔
5763
                              depth, error,
78,900✔
5764
                              row0_num[i], row0_den[i]);
236,700✔
5765
    }
78,900✔
5766
    if (y < height - 1) {
119,700✔
5767
        int row;
5768

5769
        row = pos + width;
118,503✔
5770
        for (i = 0; i < 5; ++i) {
711,018✔
5771
            int neighbor;
5772

5773
            neighbor = x + sign * row1_offsets[i];
592,515✔
5774
            if (neighbor < 0 || neighbor >= width) {
592,515✔
5775
                continue;
5,346✔
5776
            }
5777
            error_diffuse_precise(data,
782,892✔
5778
                                  row + (neighbor - x),
587,169✔
5779
                                  depth, error,
195,723✔
5780
                                  row1_num[i], row1_den[i]);
587,169✔
5781
        }
195,723✔
5782
    }
39,501✔
5783
    if (y < height - 2) {
119,700✔
5784
        int row;
5785

5786
        row = pos + width * 2;
117,306✔
5787
        for (i = 0; i < 5; ++i) {
703,836✔
5788
            int neighbor;
5789

5790
            neighbor = x + sign * row2_offsets[i];
586,530✔
5791
            if (neighbor < 0 || neighbor >= width) {
586,530✔
5792
                continue;
5,292✔
5793
            }
5794
            error_diffuse_precise(data,
774,984✔
5795
                                  row + (neighbor - x),
581,238✔
5796
                                  depth, error,
193,746✔
5797
                                  row2_num[i], row2_den[i]);
581,238✔
5798
        }
193,746✔
5799
    }
39,102✔
5800
}
119,700✔
5801

5802

5803
static void
5804
diffuse_stucki_carry(int32_t *carry_curr, int32_t *carry_next,
×
5805
                     int32_t *carry_far, int width, int height,
5806
                     int depth, int x, int y, int32_t error,
5807
                     int direction, int channel)
5808
{
5809
    /* Stucki's Method
5810
     *                  curr    8/48    4/48
5811
     *  2/48    4/48    8/48    4/48    2/48
5812
     *  1/48    2/48    4/48    2/48    1/48
5813
     */
5814
    static const int row0_offsets[] = { 1, 2 };
5815
    static const int row0_num[] = { 1, 1 };
5816
    static const int row0_den[] = { 6, 12 };
5817
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5818
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5819
    static const int row1_den[] = { 24, 12, 6, 12, 24 };
5820
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5821
    static const int row2_num[] = { 1, 1, 1, 1, 1 };
5822
    static const int row2_den[] = { 48, 24, 12, 24, 48 };
5823
    int sign;
5824
    int i;
5825

5826
    if (error == 0) {
×
5827
        return;
×
5828
    }
5829

5830
    sign = direction >= 0 ? 1 : -1;
×
5831
    for (i = 0; i < 2; ++i) {
×
5832
        int neighbor;
5833
        int32_t term;
5834

5835
        neighbor = x + sign * row0_offsets[i];
×
5836
        if (neighbor < 0 || neighbor >= width) {
×
5837
            continue;
×
5838
        }
5839
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
5840
        carry_curr[((size_t)neighbor * (size_t)depth)
×
5841
                   + (size_t)channel] += term;
×
5842
    }
5843
    if (y + 1 < height) {
×
5844
        for (i = 0; i < 5; ++i) {
×
5845
            int neighbor;
5846
            int32_t term;
5847

5848
            neighbor = x + sign * row1_offsets[i];
×
5849
            if (neighbor < 0 || neighbor >= width) {
×
5850
                continue;
×
5851
            }
5852
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
5853
            carry_next[((size_t)neighbor * (size_t)depth)
×
5854
                       + (size_t)channel] += term;
×
5855
        }
5856
    }
5857
    if (y + 2 < height) {
×
5858
        for (i = 0; i < 5; ++i) {
×
5859
            int neighbor;
5860
            int32_t term;
5861

5862
            neighbor = x + sign * row2_offsets[i];
×
5863
            if (neighbor < 0 || neighbor >= width) {
×
5864
                continue;
×
5865
            }
5866
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
5867
            carry_far[((size_t)neighbor * (size_t)depth)
×
5868
                      + (size_t)channel] += term;
×
5869
        }
5870
    }
5871
}
5872

5873

5874
static void
5875
diffuse_burkes(unsigned char *data, int width, int height,
67,500✔
5876
               int x, int y, int depth, int error, int direction)
5877
{
5878
    /* Burkes' Method
5879
     *                  curr    4/16    2/16
5880
     *  1/16    2/16    4/16    2/16    1/16
5881
     */
5882
    int pos;
5883
    int sign;
5884
    static const int row0_offsets[] = { 1, 2 };
5885
    static const int row0_num[] = { 1, 1 };
5886
    static const int row0_den[] = { 4, 8 };
5887
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5888
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5889
    static const int row1_den[] = { 16, 8, 4, 8, 16 };
5890
    int i;
5891

5892
    pos = y * width + x;
67,500✔
5893
    sign = direction >= 0 ? 1 : -1;
67,500!
5894

5895
    for (i = 0; i < 2; ++i) {
202,500✔
5896
        int neighbor;
5897

5898
        neighbor = x + sign * row0_offsets[i];
135,000✔
5899
        if (neighbor < 0 || neighbor >= width) {
135,000!
5900
            continue;
2,025✔
5901
        }
5902
        error_diffuse_normal(data,
177,300✔
5903
                             pos + (neighbor - x),
132,975✔
5904
                             depth, error,
44,325✔
5905
                             row0_num[i], row0_den[i]);
132,975✔
5906
    }
44,325✔
5907
    if (y < height - 1) {
67,500✔
5908
        int row;
5909

5910
        row = pos + width;
66,600✔
5911
        for (i = 0; i < 5; ++i) {
399,600✔
5912
            int neighbor;
5913

5914
            neighbor = x + sign * row1_offsets[i];
333,000✔
5915
            if (neighbor < 0 || neighbor >= width) {
333,000✔
5916
                continue;
3,996✔
5917
            }
5918
            error_diffuse_normal(data,
438,672✔
5919
                                 row + (neighbor - x),
329,004✔
5920
                                 depth, error,
109,668✔
5921
                                 row1_num[i], row1_den[i]);
329,004✔
5922
        }
109,668✔
5923
    }
22,200✔
5924
}
67,500✔
5925

5926
static void
5927
diffuse_burkes_carry(int32_t *carry_curr, int32_t *carry_next,
×
5928
                     int32_t *carry_far, int width, int height,
5929
                     int depth, int x, int y, int32_t error,
5930
                     int direction, int channel)
5931
{
5932
    /* Burkes' Method
5933
     *                  curr    4/16    2/16
5934
     *  1/16    2/16    4/16    2/16    1/16
5935
     */
5936
    static const int row0_offsets[] = { 1, 2 };
5937
    static const int row0_num[] = { 1, 1 };
5938
    static const int row0_den[] = { 4, 8 };
5939
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5940
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5941
    static const int row1_den[] = { 16, 8, 4, 8, 16 };
5942
    int sign;
5943
    int i;
5944

5945
    /* unused */ (void) carry_far;
5946

5947
    if (error == 0) {
×
5948
        return;
×
5949
    }
5950

5951
    sign = direction >= 0 ? 1 : -1;
×
5952
    for (i = 0; i < 2; ++i) {
×
5953
        int neighbor;
5954
        int32_t term;
5955

5956
        neighbor = x + sign * row0_offsets[i];
×
5957
        if (neighbor < 0 || neighbor >= width) {
×
5958
            continue;
×
5959
        }
5960
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
5961
        carry_curr[((size_t)neighbor * (size_t)depth)
×
5962
                   + (size_t)channel] += term;
×
5963
    }
5964
    if (y + 1 < height) {
×
5965
        for (i = 0; i < 5; ++i) {
×
5966
            int neighbor;
5967
            int32_t term;
5968

5969
            neighbor = x + sign * row1_offsets[i];
×
5970
            if (neighbor < 0 || neighbor >= width) {
×
5971
                continue;
×
5972
            }
5973
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
5974
            carry_next[((size_t)neighbor * (size_t)depth)
×
5975
                       + (size_t)channel] += term;
×
5976
        }
5977
    }
5978
}
5979

5980
static void
5981
diffuse_sierra1(unsigned char *data, int width, int height,
×
5982
                int x, int y, int depth, int error, int direction)
5983
{
5984
    /* Sierra Lite Method
5985
     *          curr    2/4
5986
     *  1/4     1/4
5987
     */
5988
    static const int row0_offsets[] = { 1 };
5989
    static const int row0_num[] = { 1 };
5990
    static const int row0_den[] = { 2 };
5991
    static const int row1_offsets[] = { -1, 0 };
5992
    static const int row1_num[] = { 1, 1 };
5993
    static const int row1_den[] = { 4, 4 };
5994
    int pos;
5995
    int sign;
5996
    int i;
5997
    int neighbor;
5998
    int row;
5999

6000
    pos = y * width + x;
×
6001
    sign = direction >= 0 ? 1 : -1;
×
6002

6003
    for (i = 0; i < 1; ++i) {
×
6004
        neighbor = x + sign * row0_offsets[i];
×
6005
        if (neighbor < 0 || neighbor >= width) {
×
6006
            continue;
×
6007
        }
6008
        error_diffuse_normal(data,
×
6009
                             pos + (neighbor - x),
×
6010
                             depth, error,
6011
                             row0_num[i], row0_den[i]);
×
6012
    }
6013
    if (y < height - 1) {
×
6014
        row = pos + width;
×
6015
        for (i = 0; i < 2; ++i) {
×
6016
            neighbor = x + sign * row1_offsets[i];
×
6017
            if (neighbor < 0 || neighbor >= width) {
×
6018
                continue;
×
6019
            }
6020
            error_diffuse_normal(data,
×
6021
                                 row + (neighbor - x),
×
6022
                                 depth, error,
6023
                                 row1_num[i], row1_den[i]);
×
6024
        }
6025
    }
6026
}
×
6027

6028

6029
static void
6030
diffuse_sierra1_carry(int32_t *carry_curr, int32_t *carry_next,
×
6031
                      int32_t *carry_far, int width, int height,
6032
                      int depth, int x, int y, int32_t error,
6033
                      int direction, int channel)
6034
{
6035
    /* Sierra Lite Method
6036
     *          curr    2/4
6037
     *  1/4     1/4
6038
     */
6039
    static const int row0_offsets[] = { 1 };
6040
    static const int row0_num[] = { 1 };
6041
    static const int row0_den[] = { 2 };
6042
    static const int row1_offsets[] = { -1, 0 };
6043
    static const int row1_num[] = { 1, 1 };
6044
    static const int row1_den[] = { 4, 4 };
6045
    int sign;
6046
    int i;
6047
    int neighbor;
6048
    int32_t term;
6049

6050
    /* unused */ (void) carry_far;
6051

6052
    if (error == 0) {
×
6053
        return;
×
6054
    }
6055

6056
    sign = direction >= 0 ? 1 : -1;
×
6057
    for (i = 0; i < 1; ++i) {
×
6058
        neighbor = x + sign * row0_offsets[i];
×
6059
        if (neighbor < 0 || neighbor >= width) {
×
6060
            continue;
×
6061
        }
6062
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6063
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6064
                   + (size_t)channel] += term;
×
6065
    }
6066
    if (y + 1 < height) {
×
6067
        for (i = 0; i < 2; ++i) {
×
6068
            neighbor = x + sign * row1_offsets[i];
×
6069
            if (neighbor < 0 || neighbor >= width) {
×
6070
                continue;
×
6071
            }
6072
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6073
            carry_next[((size_t)neighbor * (size_t)depth)
×
6074
                       + (size_t)channel] += term;
×
6075
        }
6076
    }
6077
}
6078

6079

6080
static void
6081
diffuse_sierra2(unsigned char *data, int width, int height,
×
6082
                int x, int y, int depth, int error, int direction)
6083
{
6084
    /* Sierra Two-row Method
6085
     *                  curr    4/32    3/32
6086
     *  1/32    2/32    3/32    2/32    1/32
6087
     *                  2/32    3/32    2/32
6088
     */
6089
    static const int row0_offsets[] = { 1, 2 };
6090
    static const int row0_num[] = { 4, 3 };
6091
    static const int row0_den[] = { 32, 32 };
6092
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6093
    static const int row1_num[] = { 1, 2, 3, 2, 1 };
6094
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6095
    static const int row2_offsets[] = { -1, 0, 1 };
6096
    static const int row2_num[] = { 2, 3, 2 };
6097
    static const int row2_den[] = { 32, 32, 32 };
6098
    int pos;
6099
    int sign;
6100
    int i;
6101
    int neighbor;
6102
    int row;
6103

6104
    pos = y * width + x;
×
6105
    sign = direction >= 0 ? 1 : -1;
×
6106

6107
    for (i = 0; i < 2; ++i) {
×
6108
        neighbor = x + sign * row0_offsets[i];
×
6109
        if (neighbor < 0 || neighbor >= width) {
×
6110
            continue;
×
6111
        }
6112
        error_diffuse_precise(data,
×
6113
                              pos + (neighbor - x),
×
6114
                              depth, error,
6115
                              row0_num[i], row0_den[i]);
×
6116
    }
6117
    if (y < height - 1) {
×
6118
        row = pos + width;
×
6119
        for (i = 0; i < 5; ++i) {
×
6120
            neighbor = x + sign * row1_offsets[i];
×
6121
            if (neighbor < 0 || neighbor >= width) {
×
6122
                continue;
×
6123
            }
6124
            error_diffuse_precise(data,
×
6125
                                  row + (neighbor - x),
×
6126
                                  depth, error,
6127
                                  row1_num[i], row1_den[i]);
×
6128
        }
6129
    }
6130
    if (y < height - 2) {
×
6131
        row = pos + width * 2;
×
6132
        for (i = 0; i < 3; ++i) {
×
6133
            neighbor = x + sign * row2_offsets[i];
×
6134
            if (neighbor < 0 || neighbor >= width) {
×
6135
                continue;
×
6136
            }
6137
            error_diffuse_precise(data,
×
6138
                                  row + (neighbor - x),
×
6139
                                  depth, error,
6140
                                  row2_num[i], row2_den[i]);
×
6141
        }
6142
    }
6143
}
×
6144

6145

6146
static void
6147
diffuse_sierra2_carry(int32_t *carry_curr, int32_t *carry_next,
×
6148
                      int32_t *carry_far, int width, int height,
6149
                      int depth, int x, int y, int32_t error,
6150
                      int direction, int channel)
6151
{
6152
    /* Sierra Two-row Method
6153
     *                  curr    4/32    3/32
6154
     *  1/32    2/32    3/32    2/32    1/32
6155
     *                  2/32    3/32    2/32
6156
     */
6157
    static const int row0_offsets[] = { 1, 2 };
6158
    static const int row0_num[] = { 4, 3 };
6159
    static const int row0_den[] = { 32, 32 };
6160
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6161
    static const int row1_num[] = { 1, 2, 3, 2, 1 };
6162
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6163
    static const int row2_offsets[] = { -1, 0, 1 };
6164
    static const int row2_num[] = { 2, 3, 2 };
6165
    static const int row2_den[] = { 32, 32, 32 };
6166
    int sign;
6167
    int i;
6168
    int neighbor;
6169
    int32_t term;
6170

6171
    if (error == 0) {
×
6172
        return;
×
6173
    }
6174

6175
    sign = direction >= 0 ? 1 : -1;
×
6176
    for (i = 0; i < 2; ++i) {
×
6177
        neighbor = x + sign * row0_offsets[i];
×
6178
        if (neighbor < 0 || neighbor >= width) {
×
6179
            continue;
×
6180
        }
6181
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6182
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6183
                   + (size_t)channel] += term;
×
6184
    }
6185
    if (y + 1 < height) {
×
6186
        for (i = 0; i < 5; ++i) {
×
6187
            neighbor = x + sign * row1_offsets[i];
×
6188
            if (neighbor < 0 || neighbor >= width) {
×
6189
                continue;
×
6190
            }
6191
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6192
            carry_next[((size_t)neighbor * (size_t)depth)
×
6193
                       + (size_t)channel] += term;
×
6194
        }
6195
    }
6196
    if (y + 2 < height) {
×
6197
        for (i = 0; i < 3; ++i) {
×
6198
            neighbor = x + sign * row2_offsets[i];
×
6199
            if (neighbor < 0 || neighbor >= width) {
×
6200
                continue;
×
6201
            }
6202
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
6203
            carry_far[((size_t)neighbor * (size_t)depth)
×
6204
                      + (size_t)channel] += term;
×
6205
        }
6206
    }
6207
}
6208

6209

6210
static void
6211
diffuse_sierra3(unsigned char *data, int width, int height,
×
6212
                int x, int y, int depth, int error, int direction)
6213
{
6214
    /* Sierra-3 Method
6215
     *                  curr    5/32    3/32
6216
     *  2/32    4/32    5/32    4/32    2/32
6217
     *                  2/32    3/32    2/32
6218
     */
6219
    static const int row0_offsets[] = { 1, 2 };
6220
    static const int row0_num[] = { 5, 3 };
6221
    static const int row0_den[] = { 32, 32 };
6222
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6223
    static const int row1_num[] = { 2, 4, 5, 4, 2 };
6224
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6225
    static const int row2_offsets[] = { -1, 0, 1 };
6226
    static const int row2_num[] = { 2, 3, 2 };
6227
    static const int row2_den[] = { 32, 32, 32 };
6228
    int pos;
6229
    int sign;
6230
    int i;
6231
    int neighbor;
6232
    int row;
6233

6234
    pos = y * width + x;
×
6235
    sign = direction >= 0 ? 1 : -1;
×
6236

6237
    for (i = 0; i < 2; ++i) {
×
6238
        neighbor = x + sign * row0_offsets[i];
×
6239
        if (neighbor < 0 || neighbor >= width) {
×
6240
            continue;
×
6241
        }
6242
        error_diffuse_precise(data,
×
6243
                              pos + (neighbor - x),
×
6244
                              depth, error,
6245
                              row0_num[i], row0_den[i]);
×
6246
    }
6247
    if (y < height - 1) {
×
6248
        row = pos + width;
×
6249
        for (i = 0; i < 5; ++i) {
×
6250
            neighbor = x + sign * row1_offsets[i];
×
6251
            if (neighbor < 0 || neighbor >= width) {
×
6252
                continue;
×
6253
            }
6254
            error_diffuse_precise(data,
×
6255
                                  row + (neighbor - x),
×
6256
                                  depth, error,
6257
                                  row1_num[i], row1_den[i]);
×
6258
        }
6259
    }
6260
    if (y < height - 2) {
×
6261
        row = pos + width * 2;
×
6262
        for (i = 0; i < 3; ++i) {
×
6263
            neighbor = x + sign * row2_offsets[i];
×
6264
            if (neighbor < 0 || neighbor >= width) {
×
6265
                continue;
×
6266
            }
6267
            error_diffuse_precise(data,
×
6268
                                  row + (neighbor - x),
×
6269
                                  depth, error,
6270
                                  row2_num[i], row2_den[i]);
×
6271
        }
6272
    }
6273
}
×
6274

6275

6276
static void
6277
diffuse_sierra3_carry(int32_t *carry_curr, int32_t *carry_next,
×
6278
                      int32_t *carry_far, int width, int height,
6279
                      int depth, int x, int y, int32_t error,
6280
                      int direction, int channel)
6281
{
6282
    /* Sierra-3 Method
6283
     *                  curr    5/32    3/32
6284
     *  2/32    4/32    5/32    4/32    2/32
6285
     *                  2/32    3/32    2/32
6286
     */
6287
    static const int row0_offsets[] = { 1, 2 };
6288
    static const int row0_num[] = { 5, 3 };
6289
    static const int row0_den[] = { 32, 32 };
6290
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6291
    static const int row1_num[] = { 2, 4, 5, 4, 2 };
6292
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6293
    static const int row2_offsets[] = { -1, 0, 1 };
6294
    static const int row2_num[] = { 2, 3, 2 };
6295
    static const int row2_den[] = { 32, 32, 32 };
6296
    int sign;
6297
    int i;
6298
    int neighbor;
6299
    int32_t term;
6300

6301
    if (error == 0) {
×
6302
        return;
×
6303
    }
6304

6305
    sign = direction >= 0 ? 1 : -1;
×
6306
    for (i = 0; i < 2; ++i) {
×
6307
        neighbor = x + sign * row0_offsets[i];
×
6308
        if (neighbor < 0 || neighbor >= width) {
×
6309
            continue;
×
6310
        }
6311
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6312
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6313
                   + (size_t)channel] += term;
×
6314
    }
6315
    if (y + 1 < height) {
×
6316
        for (i = 0; i < 5; ++i) {
×
6317
            neighbor = x + sign * row1_offsets[i];
×
6318
            if (neighbor < 0 || neighbor >= width) {
×
6319
                continue;
×
6320
            }
6321
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6322
            carry_next[((size_t)neighbor * (size_t)depth)
×
6323
                       + (size_t)channel] += term;
×
6324
        }
6325
    }
6326
    if (y + 2 < height) {
×
6327
        for (i = 0; i < 3; ++i) {
×
6328
            neighbor = x + sign * row2_offsets[i];
×
6329
            if (neighbor < 0 || neighbor >= width) {
×
6330
                continue;
×
6331
            }
6332
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
6333
            carry_far[((size_t)neighbor * (size_t)depth)
×
6334
                      + (size_t)channel] += term;
×
6335
        }
6336
    }
6337
}
6338

6339

6340
static float
6341
mask_a (int x, int y, int c)
×
6342
{
6343
    return ((((x + c * 67) + y * 236) * 119) & 255 ) / 128.0 - 1.0;
×
6344
}
6345

6346
static float
6347
mask_x (int x, int y, int c)
×
6348
{
6349
    return ((((x + c * 29) ^ y* 149) * 1234) & 511 ) / 256.0 - 1.0;
×
6350
}
6351

6352
/* lookup closest color from palette with "normal" strategy */
6353
static int
6354
lookup_normal(unsigned char const * const pixel,
849,900✔
6355
              int const depth,
6356
              unsigned char const * const palette,
6357
              int const reqcolor,
6358
              unsigned short * const cachetable,
6359
              int const complexion)
6360
{
6361
    int result;
6362
    int diff;
6363
    int r;
6364
    int i;
6365
    int n;
6366
    int distant;
6367

6368
    result = (-1);
849,900✔
6369
    diff = INT_MAX;
849,900✔
6370

6371
    /* don't use cachetable in 'normal' strategy */
6372
    (void) cachetable;
283,300✔
6373

6374
    for (i = 0; i < reqcolor; i++) {
15,309,900✔
6375
        distant = 0;
14,460,000✔
6376
        r = pixel[0] - palette[i * depth + 0];
14,460,000✔
6377
        distant += r * r * complexion;
14,460,000✔
6378
        for (n = 1; n < depth; ++n) {
43,380,000✔
6379
            r = pixel[n] - palette[i * depth + n];
28,920,000✔
6380
            distant += r * r;
28,920,000✔
6381
        }
9,640,000✔
6382
        if (distant < diff) {
14,460,000✔
6383
            diff = distant;
2,234,570✔
6384
            result = i;
2,234,570✔
6385
        }
744,404✔
6386
    }
4,820,000✔
6387

6388
    return result;
849,900✔
6389
}
6390

6391

6392
/*
6393
 * Shared fast lookup flow:
6394
 *
6395
 *   pixel --> quantize --> cuckoo cache --> palette index
6396
 */
6397
static int
6398
lookup_fast_common(unsigned char const *pixel,
40,077,958✔
6399
                   unsigned char const *palette,
6400
                   int reqcolor,
6401
                   unsigned short *cachetable,
6402
                   int complexion,
6403
                   struct histogram_control control)
6404
{
6405
    int result;
6406
    unsigned int hash;
6407
    int diff;
6408
    int i;
6409
    int distant;
6410
    struct cuckoo_table32 *table;
6411
    uint32_t *slot;
6412
    SIXELSTATUS status;
6413
    unsigned char const *entry;
6414
    unsigned char const *end;
6415
    int pixel0;
6416
    int pixel1;
6417
    int pixel2;
6418
    int delta;
6419

6420
    result = (-1);
40,077,958✔
6421
    diff = INT_MAX;
40,077,958✔
6422
    hash = computeHash(pixel, 3U, &control);
40,077,958✔
6423

6424
    table = (struct cuckoo_table32 *)cachetable;
40,077,958✔
6425
    if (table != NULL) {
40,077,958!
6426
        slot = cuckoo_table32_lookup(table, hash);
40,077,958✔
6427
        if (slot != NULL && *slot != 0U) {
40,077,958!
6428
            return (int)(*slot - 1U);
38,162,483✔
6429
        }
6430
    }
613,543✔
6431

6432
    entry = palette;
1,915,475✔
6433
    end = palette + (size_t)reqcolor * 3;
1,915,475✔
6434
    pixel0 = (int)pixel[0];
1,915,475✔
6435
    pixel1 = (int)pixel[1];
1,915,475✔
6436
    pixel2 = (int)pixel[2];
1,915,475✔
6437
    /*
6438
     * Palette traversal as RGB triplets keeps the stride linear:
6439
     *
6440
     *   i -> [R][G][B]
6441
     *        |  |  |
6442
     *        `--+--'
6443
     *           v
6444
     *         entry
6445
     */
6446
    for (i = 0; entry < end; ++i, entry += 3) {
360,360,099✔
6447
        delta = pixel0 - (int)entry[0];
358,444,624✔
6448
        distant = delta * delta * complexion;
358,444,624✔
6449
        delta = pixel1 - (int)entry[1];
358,444,624✔
6450
        distant += delta * delta;
358,444,624✔
6451
        delta = pixel2 - (int)entry[2];
358,444,624✔
6452
        distant += delta * delta;
358,444,624✔
6453
        if (distant < diff) {
358,444,624✔
6454
            diff = distant;
15,983,557✔
6455
            result = i;
15,983,557✔
6456
        }
5,029,727✔
6457
    }
114,439,102✔
6458

6459
    if (table != NULL && result >= 0) {
1,915,475!
6460
        status = cuckoo_table32_insert(table,
2,529,018✔
6461
                                       hash,
613,543✔
6462
                                       (uint32_t)(result + 1));
1,915,475✔
6463
        if (SIXEL_FAILED(status)) {
1,915,475!
6464
            /* ignore cache update failure */
6465
        }
6466
    }
613,543✔
6467

6468
    return result;
1,915,475✔
6469
}
19,005,818✔
6470

6471
/* lookup closest color from palette with "fast" strategy */
6472
static int
6473
lookup_fast(unsigned char const * const pixel,
40,077,958✔
6474
            int const depth,
6475
            unsigned char const * const palette,
6476
            int const reqcolor,
6477
            unsigned short * const cachetable,
6478
            int const complexion)
6479
{
6480
    struct histogram_control control;
6481

6482
    (void)depth;
19,005,818✔
6483

6484
    control = histogram_control_make(3U);
40,077,958✔
6485

6486
    return lookup_fast_common(pixel,
59,083,776✔
6487
                              palette,
19,005,818✔
6488
                              reqcolor,
19,005,818✔
6489
                              cachetable,
19,005,818✔
6490
                              complexion,
19,005,818✔
6491
                              control);
6492
}
6493

6494
static int
6495
lookup_fast_robinhood(unsigned char const * const pixel,
×
6496
                      int const depth,
6497
                      unsigned char const * const palette,
6498
                      int const reqcolor,
6499
                      unsigned short * const cachetable,
6500
                      int const complexion)
6501
{
6502
    struct histogram_control control;
6503

6504
    (void)depth;
6505

6506
    control = histogram_control_make_for_policy(3U,
×
6507
                                                SIXEL_LUT_POLICY_ROBINHOOD);
6508

6509
    return lookup_fast_common(pixel,
×
6510
                              palette,
6511
                              reqcolor,
6512
                              cachetable,
6513
                              complexion,
6514
                              control);
6515
}
6516

6517
static int
6518
lookup_fast_hopscotch(unsigned char const * const pixel,
×
6519
                      int const depth,
6520
                      unsigned char const * const palette,
6521
                      int const reqcolor,
6522
                      unsigned short * const cachetable,
6523
                      int const complexion)
6524
{
6525
    struct histogram_control control;
6526

6527
    (void)depth;
6528

6529
    control = histogram_control_make_for_policy(3U,
×
6530
                                                SIXEL_LUT_POLICY_HOPSCOTCH);
6531

6532
    return lookup_fast_common(pixel,
×
6533
                              palette,
6534
                              reqcolor,
6535
                              cachetable,
6536
                              complexion,
6537
                              control);
6538
}
6539

6540
static int
6541
lookup_fast_certlut(unsigned char const * const pixel,
×
6542
                    int const depth,
6543
                    unsigned char const * const palette,
6544
                    int const reqcolor,
6545
                    unsigned short * const cachetable,
6546
                    int const complexion)
6547
{
6548
    (void)depth;
6549
    (void)palette;
6550
    (void)reqcolor;
6551
    (void)cachetable;
6552
    (void)complexion;
6553

6554
    if (certlut_context == NULL) {
×
6555
        return 0;
×
6556
    }
6557

6558
    return (int)sixel_certlut_lookup(certlut_context,
×
6559
                                     pixel[0],
×
6560
                                     pixel[1],
×
6561
                                     pixel[2]);
×
6562
}
6563

6564

6565
static int
6566
lookup_mono_darkbg(unsigned char const * const pixel,
1,730,520✔
6567
                   int const depth,
6568
                   unsigned char const * const palette,
6569
                   int const reqcolor,
6570
                   unsigned short * const cachetable,
6571
                   int const complexion)
6572
{
6573
    int n;
6574
    int distant;
6575

6576
    /* unused */ (void) palette;
576,840✔
6577
    /* unused */ (void) cachetable;
576,840✔
6578
    /* unused */ (void) complexion;
576,840✔
6579

6580
    distant = 0;
1,730,520✔
6581
    for (n = 0; n < depth; ++n) {
6,922,080✔
6582
        distant += pixel[n];
5,191,560✔
6583
    }
1,730,520✔
6584
    return distant >= 128 * reqcolor ? 1: 0;
1,730,520✔
6585
}
6586

6587

6588
static int
6589
lookup_mono_lightbg(unsigned char const * const pixel,
810,000✔
6590
                    int const depth,
6591
                    unsigned char const * const palette,
6592
                    int const reqcolor,
6593
                    unsigned short * const cachetable,
6594
                    int const complexion)
6595
{
6596
    int n;
6597
    int distant;
6598

6599
    /* unused */ (void) palette;
270,000✔
6600
    /* unused */ (void) cachetable;
270,000✔
6601
    /* unused */ (void) complexion;
270,000✔
6602

6603
    distant = 0;
810,000✔
6604
    for (n = 0; n < depth; ++n) {
3,240,000✔
6605
        distant += pixel[n];
2,430,000✔
6606
    }
810,000✔
6607
    return distant < 128 * reqcolor ? 1: 0;
810,000✔
6608
}
6609

6610

6611
SIXELSTATUS
6612
build_palette_kmeans(unsigned char **result,
×
6613
                     unsigned char const *data,
6614
                     unsigned int length,
6615
                     unsigned int depth,
6616
                     unsigned int reqcolors,
6617
                     unsigned int *ncolors,
6618
                     unsigned int *origcolors,
6619
                     int quality_mode,
6620
                     int force_palette,
6621
                     int final_merge_mode,
6622
                     sixel_allocator_t *allocator)
6623
{
6624
    SIXELSTATUS status;
6625
    unsigned int channels;
6626
    unsigned int pixel_count;
6627
    unsigned int sample_limit;
6628
    unsigned int sample_cap;
6629
    unsigned int valid_seen;
6630
    unsigned int sample_count;
6631
    unsigned int k;
6632
    unsigned int index;
6633
    unsigned int channel;
6634
    unsigned int center_index;
6635
    unsigned int sample_index;
6636
    unsigned int replace;
6637
    unsigned int max_iterations;
6638
    unsigned int iteration;
6639
    unsigned int best_index;
6640
    unsigned int old_cluster;
6641
    unsigned int farthest_index;
6642
    unsigned int fill;
6643
    unsigned int source;
6644
    unsigned int swap_temp;
6645
    unsigned int base;
6646
    unsigned int extra_component;
6647
    unsigned int *membership;
6648
    unsigned int *order;
6649
    unsigned char *samples;
6650
    unsigned char *palette;
6651
    unsigned char *new_palette;
6652
    double *centers;
6653
    double *distance_cache;
6654
    double total_weight;
6655
    double random_point;
6656
    double best_distance;
6657
    double distance;
6658
    double diff;
6659
    double update;
6660
    double farthest_distance;
6661
    unsigned long *counts;
6662
    unsigned long *accum;
6663
    unsigned long *channel_sum;
6664
    unsigned long rand_value;
6665
    int changed;
6666
    int apply_merge;
6667
    unsigned int overshoot;
6668
    unsigned int refine_iterations;
6669
    int cluster_total;
6670

6671
    status = SIXEL_BAD_ARGUMENT;
×
6672
    channels = depth;
×
6673
    pixel_count = 0U;
×
6674
    sample_limit = 50000U;
×
6675
    sample_cap = sample_limit;
×
6676
    valid_seen = 0U;
×
6677
    sample_count = 0U;
×
6678
    k = 0U;
×
6679
    index = 0U;
×
6680
    channel = 0U;
×
6681
    center_index = 0U;
×
6682
    sample_index = 0U;
×
6683
    replace = 0U;
×
6684
    max_iterations = 0U;
×
6685
    iteration = 0U;
×
6686
    best_index = 0U;
×
6687
    old_cluster = 0U;
×
6688
    farthest_index = 0U;
×
6689
    fill = 0U;
×
6690
    source = 0U;
×
6691
    swap_temp = 0U;
×
6692
    base = 0U;
×
6693
    extra_component = 0U;
×
6694
    membership = NULL;
×
6695
    order = NULL;
×
6696
    samples = NULL;
×
6697
    palette = NULL;
×
6698
    new_palette = NULL;
×
6699
    centers = NULL;
×
6700
    distance_cache = NULL;
×
6701
    counts = NULL;
×
6702
    accum = NULL;
×
6703
    channel_sum = NULL;
×
6704
    rand_value = 0UL;
×
6705
    total_weight = 0.0;
×
6706
    random_point = 0.0;
×
6707
    best_distance = 0.0;
×
6708
    distance = 0.0;
×
6709
    diff = 0.0;
×
6710
    update = 0.0;
×
6711
    farthest_distance = 0.0;
×
6712
    changed = 0;
×
6713
    apply_merge = 0;
×
6714
    overshoot = 0U;
×
6715
    refine_iterations = 0U;
×
6716
    cluster_total = 0;
×
6717

6718
    if (result != NULL) {
×
6719
        *result = NULL;
×
6720
    }
6721
    if (ncolors != NULL) {
×
6722
        *ncolors = 0U;
×
6723
    }
6724
    if (origcolors != NULL) {
×
6725
        *origcolors = 0U;
×
6726
    }
6727

6728
    if (channels != 3U && channels != 4U) {
×
6729
        goto end;
×
6730
    }
6731
    if (channels == 0U) {
×
6732
        goto end;
×
6733
    }
6734

6735
    pixel_count = length / channels;
×
6736
    if (pixel_count == 0U) {
×
6737
        goto end;
×
6738
    }
6739
    if (pixel_count < sample_cap) {
×
6740
        sample_cap = pixel_count;
×
6741
    }
6742
    if (sample_cap == 0U) {
×
6743
        sample_cap = 1U;
×
6744
    }
6745

6746
    samples = (unsigned char *)sixel_allocator_malloc(
×
6747
        allocator, (size_t)sample_cap * 3U);
×
6748
    if (samples == NULL) {
×
6749
        status = SIXEL_BAD_ALLOCATION;
×
6750
        goto end;
×
6751
    }
6752

6753
    /*
6754
     * Reservoir sampling keeps the distribution fair when the image is
6755
     * larger than our budget. Transparent pixels are skipped so that the
6756
     * solver only sees visible colors.
6757
     */
6758
    for (index = 0U; index < pixel_count; ++index) {
×
6759
        base = index * channels;
×
6760
        if (channels == 4U && data[base + 3U] == 0U) {
×
6761
            continue;
×
6762
        }
6763
        ++valid_seen;
×
6764
        if (sample_count < sample_cap) {
×
6765
            for (channel = 0U; channel < 3U; ++channel) {
×
6766
                samples[sample_count * 3U + channel] =
×
6767
                    data[base + channel];
×
6768
            }
6769
            ++sample_count;
×
6770
        } else {
6771
            rand_value = (unsigned long)rand();
×
6772
            replace = (unsigned int)(rand_value % valid_seen);
×
6773
            if (replace < sample_cap) {
×
6774
                for (channel = 0U; channel < 3U; ++channel) {
×
6775
                    samples[replace * 3U + channel] =
×
6776
                        data[base + channel];
×
6777
                }
6778
            }
6779
        }
6780
    }
6781

6782
    if (origcolors != NULL) {
×
6783
        *origcolors = valid_seen;
×
6784
    }
6785
    if (sample_count == 0U) {
×
6786
        goto end;
×
6787
    }
6788

6789
    if (reqcolors == 0U) {
×
6790
        reqcolors = 1U;
×
6791
    }
6792
    apply_merge = (final_merge_mode == SIXEL_FINAL_MERGE_AUTO
×
6793
                   || final_merge_mode == SIXEL_FINAL_MERGE_WARD);
×
6794
    refine_iterations = 2U;
×
6795
    overshoot = reqcolors;
×
6796
    /* Oversplit so the subsequent Ward merge has room to consolidate. */
6797
    if (apply_merge) {
×
6798
        overshoot = sixel_final_merge_target(reqcolors,
×
6799
                                             final_merge_mode);
6800
        quant_trace(stderr, "overshoot: %d\n", overshoot);
×
6801
    }
6802
    if (overshoot > sample_count) {
×
6803
        overshoot = sample_count;
×
6804
    }
6805
    k = overshoot;
×
6806
    if (k == 0U) {
×
6807
        goto end;
×
6808
    }
6809

6810
    centers = (double *)sixel_allocator_malloc(
×
6811
        allocator, (size_t)k * 3U * sizeof(double));
×
6812
    distance_cache = (double *)sixel_allocator_malloc(
×
6813
        allocator, (size_t)sample_count * sizeof(double));
×
6814
    counts = (unsigned long *)sixel_allocator_malloc(
×
6815
        allocator, (size_t)k * sizeof(unsigned long));
×
6816
    accum = (unsigned long *)sixel_allocator_malloc(
×
6817
        allocator, (size_t)k * 3U * sizeof(unsigned long));
×
6818
    membership = (unsigned int *)sixel_allocator_malloc(
×
6819
        allocator, (size_t)sample_count * sizeof(unsigned int));
×
6820
    if (centers == NULL || distance_cache == NULL || counts == NULL ||
×
6821
            accum == NULL || membership == NULL) {
×
6822
        status = SIXEL_BAD_ALLOCATION;
×
6823
        goto end;
×
6824
    }
6825

6826
    /*
6827
     * Seed the first center uniformly from the sampled set. Subsequent
6828
     * centers use k-means++ to favour distant samples.
6829
     */
6830
    rand_value = (unsigned long)rand();
×
6831
    replace = (unsigned int)(rand_value % sample_count);
×
6832
    for (channel = 0U; channel < 3U; ++channel) {
×
6833
        centers[channel] =
×
6834
            (double)samples[replace * 3U + channel];
×
6835
    }
6836
    for (sample_index = 0U; sample_index < sample_count; ++sample_index) {
×
6837
        distance = 0.0;
×
6838
        for (channel = 0U; channel < 3U; ++channel) {
×
6839
            diff = (double)samples[sample_index * 3U + channel]
×
6840
                - centers[channel];
×
6841
            distance += diff * diff;
×
6842
        }
6843
        distance_cache[sample_index] = distance;
×
6844
    }
6845

6846
    for (center_index = 1U; center_index < k; ++center_index) {
×
6847
        total_weight = 0.0;
×
6848
        for (sample_index = 0U; sample_index < sample_count;
×
6849
                ++sample_index) {
×
6850
            total_weight += distance_cache[sample_index];
×
6851
        }
6852
        random_point = 0.0;
×
6853
        if (total_weight > 0.0) {
×
6854
            random_point =
×
6855
                ((double)rand() / ((double)RAND_MAX + 1.0)) *
×
6856
                total_weight;
6857
        }
6858
        sample_index = 0U;
×
6859
        while (sample_index + 1U < sample_count &&
×
6860
               random_point > distance_cache[sample_index]) {
×
6861
            random_point -= distance_cache[sample_index];
×
6862
            ++sample_index;
×
6863
        }
6864
        for (channel = 0U; channel < 3U; ++channel) {
×
6865
            centers[center_index * 3U + channel] =
×
6866
                (double)samples[sample_index * 3U + channel];
×
6867
        }
6868
        for (index = 0U; index < sample_count; ++index) {
×
6869
            distance = 0.0;
×
6870
            for (channel = 0U; channel < 3U; ++channel) {
×
6871
                diff = (double)samples[index * 3U + channel]
×
6872
                    - centers[center_index * 3U + channel];
×
6873
                distance += diff * diff;
×
6874
            }
6875
            if (distance < distance_cache[index]) {
×
6876
                distance_cache[index] = distance;
×
6877
            }
6878
        }
6879
    }
6880

6881
    switch (quality_mode) {
×
6882
    case SIXEL_QUALITY_LOW:
6883
        max_iterations = 6U;
×
6884
        break;
×
6885
    case SIXEL_QUALITY_HIGH:
6886
        max_iterations = 24U;
×
6887
        break;
×
6888
    case SIXEL_QUALITY_FULL:
6889
        max_iterations = 48U;
×
6890
        break;
×
6891
    case SIXEL_QUALITY_HIGHCOLOR:
6892
        max_iterations = 24U;
×
6893
        break;
×
6894
    case SIXEL_QUALITY_AUTO:
×
6895
    default:
6896
        max_iterations = 12U;
×
6897
        break;
×
6898
    }
6899
    if (max_iterations == 0U) {
×
6900
        max_iterations = 1U;
×
6901
    }
6902
    if (max_iterations > 20U) {
×
6903
        /*
6904
         * The requirements cap the Lloyd refinement at twenty passes to
6905
         * keep runtime bounded even for demanding quality presets.
6906
         */
6907
        max_iterations = 20U;
×
6908
    }
6909

6910
    /*
6911
     * Lloyd refinement assigns samples to their nearest center and moves
6912
     * each center to the mean of its cluster. Empty clusters are reseeded
6913
     * using the farthest sample to improve stability.
6914
     */
6915
    for (iteration = 0U; iteration < max_iterations; ++iteration) {
×
6916
        for (index = 0U; index < k; ++index) {
×
6917
            counts[index] = 0UL;
×
6918
        }
6919
        for (index = 0U; index < k * 3U; ++index) {
×
6920
            accum[index] = 0UL;
×
6921
        }
6922
        for (sample_index = 0U; sample_index < sample_count;
×
6923
                ++sample_index) {
×
6924
            best_index = 0U;
×
6925
            distance = 0.0;
×
6926
            for (channel = 0U; channel < 3U; ++channel) {
×
6927
                diff = (double)samples[sample_index * 3U + channel]
×
6928
                    - centers[channel];
×
6929
                distance += diff * diff;
×
6930
            }
6931
            best_distance = distance;
×
6932
            for (center_index = 1U; center_index < k;
×
6933
                    ++center_index) {
×
6934
                distance = 0.0;
×
6935
                for (channel = 0U; channel < 3U; ++channel) {
×
6936
                    diff = (double)samples[sample_index * 3U + channel]
×
6937
                        - centers[center_index * 3U + channel];
×
6938
                    distance += diff * diff;
×
6939
                }
6940
                if (distance < best_distance) {
×
6941
                    best_distance = distance;
×
6942
                    best_index = center_index;
×
6943
                }
6944
            }
6945
            membership[sample_index] = best_index;
×
6946
            distance_cache[sample_index] = best_distance;
×
6947
            counts[best_index] += 1UL;
×
6948
            channel_sum = accum + (size_t)best_index * 3U;
×
6949
            for (channel = 0U; channel < 3U; ++channel) {
×
6950
                channel_sum[channel] +=
×
6951
                    (unsigned long)samples[sample_index * 3U + channel];
×
6952
            }
6953
        }
6954
        for (center_index = 0U; center_index < k; ++center_index) {
×
6955
            if (counts[center_index] != 0UL) {
×
6956
                continue;
×
6957
            }
6958
            farthest_distance = -1.0;
×
6959
            farthest_index = 0U;
×
6960
            for (sample_index = 0U; sample_index < sample_count;
×
6961
                    ++sample_index) {
×
6962
                if (distance_cache[sample_index] > farthest_distance) {
×
6963
                    farthest_distance = distance_cache[sample_index];
×
6964
                    farthest_index = sample_index;
×
6965
                }
6966
            }
6967
            old_cluster = membership[farthest_index];
×
6968
            if (counts[old_cluster] > 0UL) {
×
6969
                counts[old_cluster] -= 1UL;
×
6970
                channel_sum = accum + (size_t)old_cluster * 3U;
×
6971
                for (channel = 0U; channel < 3U; ++channel) {
×
6972
                    extra_component =
×
6973
                        (unsigned int)samples[farthest_index * 3U + channel];
×
6974
                    if (channel_sum[channel] >=
×
6975
                            (unsigned long)extra_component) {
×
6976
                        channel_sum[channel] -=
×
6977
                            (unsigned long)extra_component;
×
6978
                    } else {
6979
                        channel_sum[channel] = 0UL;
×
6980
                    }
6981
                }
6982
            }
6983
            membership[farthest_index] = center_index;
×
6984
            counts[center_index] = 1UL;
×
6985
            channel_sum = accum + (size_t)center_index * 3U;
×
6986
            for (channel = 0U; channel < 3U; ++channel) {
×
6987
                channel_sum[channel] =
×
6988
                    (unsigned long)samples[farthest_index * 3U + channel];
×
6989
            }
6990
            distance_cache[farthest_index] = 0.0;
×
6991
        }
6992
        changed = 0;
×
6993
        for (center_index = 0U; center_index < k; ++center_index) {
×
6994
            if (counts[center_index] == 0UL) {
×
6995
                continue;
×
6996
            }
6997
            channel_sum = accum + (size_t)center_index * 3U;
×
6998
            for (channel = 0U; channel < 3U; ++channel) {
×
6999
                update = (double)channel_sum[channel]
×
7000
                    / (double)counts[center_index];
×
7001
                diff = centers[center_index * 3U + channel] - update;
×
7002
                if (diff < 0.0) {
×
7003
                    diff = -diff;
×
7004
                }
7005
                if (diff > 0.5) {
×
7006
                    changed = 1;
×
7007
                }
7008
                centers[center_index * 3U + channel] = update;
×
7009
            }
7010
        }
7011
        if (!changed) {
×
7012
            break;
×
7013
        }
7014
    }
7015

7016
    if (apply_merge && k > reqcolors) {
×
7017
        /* Merge the provisional clusters and polish with a few Lloyd steps. */
7018
        cluster_total = (int)k;
×
7019
        sixel_merge_clusters_ward(counts, accum, 3U,
×
7020
                                  &cluster_total, (int)reqcolors);
7021
        if (cluster_total < 1) {
×
7022
            cluster_total = 1;
×
7023
        }
7024
        if ((unsigned int)cluster_total > reqcolors) {
×
7025
            cluster_total = (int)reqcolors;
×
7026
        }
7027
        k = (unsigned int)cluster_total;
×
7028
        if (k == 0U) {
×
7029
            k = 1U;
×
7030
        }
7031
        for (center_index = 0U; center_index < k; ++center_index) {
×
7032
            if (counts[center_index] == 0UL) {
×
7033
                counts[center_index] = 1UL;
×
7034
            }
7035
            channel_sum = accum + (size_t)center_index * 3U;
×
7036
            for (channel = 0U; channel < 3U; ++channel) {
×
7037
                centers[center_index * 3U + channel] =
×
7038
                    (double)channel_sum[channel]
×
7039
                    / (double)counts[center_index];
×
7040
            }
7041
        }
7042
        for (iteration = 0U; iteration < refine_iterations; ++iteration) {
×
7043
            for (index = 0U; index < k; ++index) {
×
7044
                counts[index] = 0UL;
×
7045
            }
7046
            for (index = 0U; index < k * 3U; ++index) {
×
7047
                accum[index] = 0UL;
×
7048
            }
7049
            for (sample_index = 0U; sample_index < sample_count;
×
7050
                    ++sample_index) {
×
7051
                best_index = 0U;
×
7052
                best_distance = 0.0;
×
7053
                for (channel = 0U; channel < 3U; ++channel) {
×
7054
                    diff = (double)samples[sample_index * 3U + channel]
×
7055
                        - centers[channel];
×
7056
                    best_distance += diff * diff;
×
7057
                }
7058
                for (center_index = 1U; center_index < k;
×
7059
                        ++center_index) {
×
7060
                    distance = 0.0;
×
7061
                    for (channel = 0U; channel < 3U; ++channel) {
×
7062
                        diff = (double)samples[sample_index * 3U + channel]
×
7063
                            - centers[center_index * 3U + channel];
×
7064
                        distance += diff * diff;
×
7065
                    }
7066
                    if (distance < best_distance) {
×
7067
                        best_distance = distance;
×
7068
                        best_index = center_index;
×
7069
                    }
7070
                }
7071
                membership[sample_index] = best_index;
×
7072
                distance_cache[sample_index] = best_distance;
×
7073
                counts[best_index] += 1UL;
×
7074
                channel_sum = accum + (size_t)best_index * 3U;
×
7075
                for (channel = 0U; channel < 3U; ++channel) {
×
7076
                    channel_sum[channel] +=
×
7077
                        (unsigned long)samples[sample_index * 3U + channel];
×
7078
                }
7079
            }
7080
            for (center_index = 0U; center_index < k; ++center_index) {
×
7081
                if (counts[center_index] != 0UL) {
×
7082
                    continue;
×
7083
                }
7084
                farthest_distance = -1.0;
×
7085
                farthest_index = 0U;
×
7086
                for (sample_index = 0U; sample_index < sample_count;
×
7087
                        ++sample_index) {
×
7088
                    if (distance_cache[sample_index] > farthest_distance) {
×
7089
                        farthest_distance = distance_cache[sample_index];
×
7090
                        farthest_index = sample_index;
×
7091
                    }
7092
                }
7093
                old_cluster = membership[farthest_index];
×
7094
                if (counts[old_cluster] > 0UL) {
×
7095
                    counts[old_cluster] -= 1UL;
×
7096
                    channel_sum = accum + (size_t)old_cluster * 3U;
×
7097
                    for (channel = 0U; channel < 3U; ++channel) {
×
7098
                        extra_component =
×
7099
                            (unsigned int)samples[farthest_index * 3U + channel];
×
7100
                        if (channel_sum[channel] >=
×
7101
                                (unsigned long)extra_component) {
×
7102
                            channel_sum[channel] -=
×
7103
                                (unsigned long)extra_component;
×
7104
                        } else {
7105
                            channel_sum[channel] = 0UL;
×
7106
                        }
7107
                    }
7108
                }
7109
                membership[farthest_index] = center_index;
×
7110
                counts[center_index] = 1UL;
×
7111
                channel_sum = accum + (size_t)center_index * 3U;
×
7112
                for (channel = 0U; channel < 3U; ++channel) {
×
7113
                    channel_sum[channel] =
×
7114
                        (unsigned long)samples[farthest_index * 3U + channel];
×
7115
                }
7116
                distance_cache[farthest_index] = 0.0;
×
7117
            }
7118
            changed = 0;
×
7119
            for (center_index = 0U; center_index < k; ++center_index) {
×
7120
                if (counts[center_index] == 0UL) {
×
7121
                    continue;
×
7122
                }
7123
                channel_sum = accum + (size_t)center_index * 3U;
×
7124
                for (channel = 0U; channel < 3U; ++channel) {
×
7125
                    update = (double)channel_sum[channel]
×
7126
                        / (double)counts[center_index];
×
7127
                    diff = centers[center_index * 3U + channel] - update;
×
7128
                    if (diff < 0.0) {
×
7129
                        diff = -diff;
×
7130
                    }
7131
                    if (diff > 0.5) {
×
7132
                        changed = 1;
×
7133
                    }
7134
                    centers[center_index * 3U + channel] = update;
×
7135
                }
7136
            }
7137
            if (!changed) {
×
7138
                break;
×
7139
            }
7140
        }
7141
    }
7142

7143
    /*
7144
     * Convert the floating point centers back into the byte palette that
7145
     * callers expect.
7146
     */
7147
    palette = (unsigned char *)sixel_allocator_malloc(
×
7148
        allocator, (size_t)k * 3U);
×
7149
    if (palette == NULL) {
×
7150
        status = SIXEL_BAD_ALLOCATION;
×
7151
        goto end;
×
7152
    }
7153
    for (center_index = 0U; center_index < k; ++center_index) {
×
7154
        for (channel = 0U; channel < 3U; ++channel) {
×
7155
            update = centers[center_index * 3U + channel];
×
7156
            if (update < 0.0) {
×
7157
                update = 0.0;
×
7158
            }
7159
            if (update > 255.0) {
×
7160
                update = 255.0;
×
7161
            }
7162
            palette[center_index * 3U + channel] =
×
7163
                (unsigned char)(update + 0.5);
×
7164
        }
7165
    }
7166

7167
    if (force_palette && k < reqcolors) {
×
7168
        /*
7169
         * Populate the tail of the palette by repeating the most frequent
7170
         * clusters so the caller still receives the requested palette size.
7171
         */
7172
        new_palette = (unsigned char *)sixel_allocator_malloc(
×
7173
            allocator, (size_t)reqcolors * 3U);
×
7174
        if (new_palette == NULL) {
×
7175
            status = SIXEL_BAD_ALLOCATION;
×
7176
            goto end;
×
7177
        }
7178
        for (index = 0U; index < k * 3U; ++index) {
×
7179
            new_palette[index] = palette[index];
×
7180
        }
7181
        order = (unsigned int *)sixel_allocator_malloc(
×
7182
            allocator, (size_t)k * sizeof(unsigned int));
×
7183
        if (order == NULL) {
×
7184
            status = SIXEL_BAD_ALLOCATION;
×
7185
            goto end;
×
7186
        }
7187
        for (index = 0U; index < k; ++index) {
×
7188
            order[index] = index;
×
7189
        }
7190
        for (index = 0U; index < k; ++index) {
×
7191
            for (center_index = index + 1U; center_index < k;
×
7192
                    ++center_index) {
×
7193
                if (counts[order[center_index]] >
×
7194
                        counts[order[index]]) {
×
7195
                    swap_temp = order[index];
×
7196
                    order[index] = order[center_index];
×
7197
                    order[center_index] = swap_temp;
×
7198
                }
7199
            }
7200
        }
7201
        fill = k;
×
7202
        source = 0U;
×
7203
        while (fill < reqcolors && k > 0U) {
×
7204
            center_index = order[source];
×
7205
            for (channel = 0U; channel < 3U; ++channel) {
×
7206
                new_palette[fill * 3U + channel] =
×
7207
                    palette[center_index * 3U + channel];
×
7208
            }
7209
            ++fill;
×
7210
            ++source;
×
7211
            if (source >= k) {
×
7212
                source = 0U;
×
7213
            }
7214
        }
7215
        sixel_allocator_free(allocator, palette);
×
7216
        palette = new_palette;
×
7217
        new_palette = NULL;
×
7218
        k = reqcolors;
×
7219
    }
7220

7221
    status = SIXEL_OK;
×
7222
    if (result != NULL) {
×
7223
        *result = palette;
×
7224
    } else {
7225
        palette = NULL;
×
7226
    }
7227
    if (ncolors != NULL) {
×
7228
        *ncolors = k;
×
7229
    }
7230

7231
end:
7232
    if (status != SIXEL_OK && palette != NULL) {
×
7233
        sixel_allocator_free(allocator, palette);
×
7234
    }
7235
    if (new_palette != NULL) {
×
7236
        sixel_allocator_free(allocator, new_palette);
×
7237
    }
7238
    if (order != NULL) {
×
7239
        sixel_allocator_free(allocator, order);
×
7240
    }
7241
    if (membership != NULL) {
×
7242
        sixel_allocator_free(allocator, membership);
×
7243
    }
7244
    if (accum != NULL) {
×
7245
        sixel_allocator_free(allocator, accum);
×
7246
    }
7247
    if (counts != NULL) {
×
7248
        sixel_allocator_free(allocator, counts);
×
7249
    }
7250
    if (distance_cache != NULL) {
×
7251
        sixel_allocator_free(allocator, distance_cache);
×
7252
    }
7253
    if (centers != NULL) {
×
7254
        sixel_allocator_free(allocator, centers);
×
7255
    }
7256
    if (samples != NULL) {
×
7257
        sixel_allocator_free(allocator, samples);
×
7258
    }
7259
    return status;
×
7260
}
7261

7262

7263
/* choose colors using median-cut method */
7264
SIXELSTATUS
7265
sixel_quant_make_palette(
260✔
7266
    unsigned char          /* out */ **result,
7267
    unsigned char const    /* in */  *data,
7268
    unsigned int           /* in */  length,
7269
    int                    /* in */  pixelformat,
7270
    unsigned int           /* in */  reqcolors,
7271
    unsigned int           /* in */  *ncolors,
7272
    unsigned int           /* in */  *origcolors,
7273
    int                    /* in */  methodForLargest,
7274
    int                    /* in */  methodForRep,
7275
    int                    /* in */  qualityMode,
7276
    int                    /* in */  force_palette,
7277
    int                    /* in */  use_reversible,
7278
    int                    /* in */  quantize_model,
7279
    int                    /* in */  final_merge_mode,
7280
    sixel_allocator_t      /* in */  *allocator)
7281
{
7282
    SIXELSTATUS status = SIXEL_FALSE;
260✔
7283
    sixel_palette_t *palette = NULL;
260✔
7284
    sixel_allocator_t *work_allocator;
7285
    size_t payload_size;
7286
    unsigned int depth;
7287

7288
    if (result == NULL) {
260!
NEW
7289
        return SIXEL_BAD_ARGUMENT;
×
7290
    }
7291
    *result = NULL;
260✔
7292

7293
    status = sixel_palette_new(&palette, allocator);
260✔
7294
    if (SIXEL_FAILED(status)) {
260!
NEW
7295
        goto end;
×
7296
    }
7297

7298
    palette->requested_colors = reqcolors;
260✔
7299
    palette->method_for_largest = methodForLargest;
260✔
7300
    palette->method_for_rep = methodForRep;
260✔
7301
    palette->quality_mode = qualityMode;
260✔
7302
    palette->force_palette = force_palette;
260✔
7303
    palette->use_reversible = use_reversible;
260✔
7304
    palette->quantize_model = quantize_model;
260✔
7305
    palette->final_merge_mode = final_merge_mode;
260✔
7306
    palette->lut_policy = histogram_lut_policy;
260✔
7307

7308
    status = sixel_palette_generate(palette,
378✔
7309
                                    data,
118✔
7310
                                    length,
118✔
7311
                                    pixelformat,
118✔
7312
                                    allocator);
118✔
7313
    if (SIXEL_FAILED(status)) {
260!
UNCOV
7314
        goto end;
×
7315
    }
7316

7317
    if (ncolors != NULL) {
260!
7318
        *ncolors = palette->entry_count;
260✔
7319
    }
118✔
7320
    if (origcolors != NULL) {
260!
7321
        *origcolors = palette->original_colors;
260✔
7322
    }
118✔
7323

7324
    if (palette->depth <= 0 || palette->entry_count == 0U) {
260!
NEW
7325
        status = SIXEL_OK;
×
NEW
7326
        goto end;
×
7327
    }
7328

7329
    depth = (unsigned int)palette->depth;
260✔
7330
    payload_size = (size_t)palette->entry_count * (size_t)depth;
260✔
7331
    work_allocator = (allocator != NULL) ? allocator : palette->allocator;
260!
7332
    if (work_allocator == NULL) {
260!
NEW
7333
        status = SIXEL_BAD_ARGUMENT;
×
NEW
7334
        goto end;
×
7335
    }
7336

7337
    *result = (unsigned char *)sixel_allocator_malloc(work_allocator,
378✔
7338
                                                      payload_size);
118✔
7339
    if (*result == NULL) {
260!
NEW
7340
        sixel_helper_set_additional_message(
×
7341
            "sixel_quant_make_palette: allocation failed.");
NEW
7342
        status = SIXEL_BAD_ALLOCATION;
×
NEW
7343
        goto end;
×
7344
    }
7345
    memcpy(*result, palette->entries, payload_size);
260✔
7346

7347
    status = SIXEL_OK;
260✔
7348

7349
end:
142✔
7350
    if (palette != NULL) {
260!
7351
        sixel_palette_unref(palette);
260✔
7352
    }
118✔
7353
    return status;
260✔
7354
}
118✔
7355

7356

7357
/* apply color palette into specified pixel buffers */
7358
SIXELSTATUS
7359
sixel_quant_apply_palette(
285✔
7360
    sixel_index_t     /* out */ *result,
7361
    unsigned char     /* in */  *data,
7362
    int               /* in */  width,
7363
    int               /* in */  height,
7364
    int               /* in */  depth,
7365
    unsigned char     /* in */  *palette,
7366
    int               /* in */  reqcolor,
7367
    int               /* in */  methodForDiffuse,
7368
    int               /* in */  methodForScan,
7369
    int               /* in */  methodForCarry,
7370
    int               /* in */  foptimize,
7371
    int               /* in */  foptimize_palette,
7372
    int               /* in */  complexion,
7373
    unsigned short    /* in */  *cachetable,
7374
    int               /* in */  *ncolors,
7375
    sixel_allocator_t /* in */  *allocator)
7376
{
170✔
7377
#if _MSC_VER
7378
    enum { max_depth = 4 };
7379
#else
7380
    const size_t max_depth = 4;
285✔
7381
#endif
7382
    unsigned char copy[max_depth];
170✔
7383
    SIXELSTATUS status = SIXEL_FALSE;
285✔
7384
    int sum1, sum2;
7385
    int n;
7386
    unsigned short *indextable;
7387
    size_t cache_size;
7388
    int allocated_cache;
7389
    int use_cache;
7390
    int cache_policy;
7391
    unsigned char new_palette[SIXEL_PALETTE_MAX * 4];
7392
    unsigned short migration_map[SIXEL_PALETTE_MAX];
7393
    int (*f_lookup)(unsigned char const * const pixel,
7394
                    int const depth,
7395
                    unsigned char const * const palette,
7396
                    int const reqcolor,
7397
                    unsigned short * const cachetable,
7398
                    int const complexion);
7399
    int use_varerr;
7400
    int use_positional;
7401
    int carry_mode;
7402
    sixel_certlut_t certlut;
7403
    int certlut_ready;
7404
    int wR;
7405
    int wG;
7406
    int wB;
7407

7408
    certlut_ready = 0;
285✔
7409
    memset(&certlut, 0, sizeof(certlut));
285✔
7410

7411
    /* check bad reqcolor */
7412
    if (reqcolor < 1) {
285!
7413
        status = SIXEL_BAD_ARGUMENT;
×
7414
        sixel_helper_set_additional_message(
×
7415
            "sixel_quant_apply_palette: "
7416
            "a bad argument is detected, reqcolor < 0.");
7417
        goto end;
×
7418
    }
7419

7420
    /* NOTE: diffuse_jajuni, diffuse_stucki, and diffuse_burkes reference at
7421
     * minimum the position pos + width * 1 - 2, so width must be at least 2
7422
     * to avoid underflow.
7423
     * On the other hand, diffuse_fs and diffuse_atkinson
7424
     * reference pos + width * 1 - 1, but since these functions are only called
7425
     * when width >= 1, they do not cause underflow.
7426
     */
7427
    use_varerr = (depth == 3
400✔
7428
                  && methodForDiffuse == SIXEL_DIFFUSE_LSO2);
285!
7429
    use_positional = (methodForDiffuse == SIXEL_DIFFUSE_A_DITHER
400✔
7430
                      || methodForDiffuse == SIXEL_DIFFUSE_X_DITHER);
285!
7431
    carry_mode = (methodForCarry == SIXEL_CARRY_ENABLE)
285✔
7432
               ? SIXEL_CARRY_ENABLE
7433
               : SIXEL_CARRY_DISABLE;
170!
7434

7435
    f_lookup = NULL;
285✔
7436
    if (reqcolor == 2) {
285✔
7437
        sum1 = 0;
19✔
7438
        sum2 = 0;
19✔
7439
        for (n = 0; n < depth; ++n) {
76✔
7440
            sum1 += palette[n];
57✔
7441
        }
21✔
7442
        for (n = depth; n < depth + depth; ++n) {
76✔
7443
            sum2 += palette[n];
57✔
7444
        }
21✔
7445
        if (sum1 == 0 && sum2 == 255 * 3) {
19!
7446
            f_lookup = lookup_mono_darkbg;
12✔
7447
        } else if (sum1 == 255 * 3 && sum2 == 0) {
11!
7448
            f_lookup = lookup_mono_lightbg;
3✔
7449
        }
1✔
7450
    }
7✔
7451
    if (f_lookup == NULL) {
285✔
7452
        if (foptimize && depth == 3) {
270!
7453
            f_lookup = lookup_fast;
264✔
7454
        } else {
108✔
7455
            f_lookup = lookup_normal;
6✔
7456
        }
7457
    }
110✔
7458

7459
    if (f_lookup == lookup_fast) {
285✔
7460
        if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD) {
264!
7461
            f_lookup = lookup_fast_robinhood;
×
7462
        } else if (histogram_lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
264!
7463
            f_lookup = lookup_fast_hopscotch;
×
7464
        } else if (histogram_lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
264!
7465
            f_lookup = lookup_fast_certlut;
×
7466
        }
7467
    }
108✔
7468

7469
    indextable = cachetable;
285✔
7470
    allocated_cache = 0;
285✔
7471
    cache_policy = SIXEL_LUT_POLICY_AUTO;
285✔
7472
    use_cache = 0;
285✔
7473
    if (f_lookup == lookup_fast) {
285✔
7474
        cache_policy = histogram_lut_policy;
264✔
7475
        use_cache = 1;
264✔
7476
    } else if (f_lookup == lookup_fast_robinhood) {
129!
7477
        cache_policy = SIXEL_LUT_POLICY_ROBINHOOD;
×
7478
        use_cache = 1;
×
7479
    } else if (f_lookup == lookup_fast_hopscotch) {
21!
7480
        cache_policy = SIXEL_LUT_POLICY_HOPSCOTCH;
×
7481
        use_cache = 1;
×
7482
    }
7483
    if (cache_policy == SIXEL_LUT_POLICY_AUTO) {
285!
7484
        cache_policy = SIXEL_LUT_POLICY_6BIT;
285✔
7485
    }
115✔
7486
    if (use_cache) {
285✔
7487
        if (cachetable == NULL) {
264!
7488
            status = sixel_quant_cache_prepare(&indextable,
×
7489
                                               &cache_size,
7490
                                               cache_policy,
7491
                                               reqcolor,
7492
                                               allocator);
7493
            if (SIXEL_FAILED(status)) {
×
7494
                quant_trace(stderr,
×
7495
                            "Unable to allocate lookup cache.\n");
7496
                goto end;
×
7497
            }
7498
            allocated_cache = 1;
×
7499
        } else {
7500
            sixel_quant_cache_clear(indextable, cache_policy);
264✔
7501
        }
7502
    }
108✔
7503

7504
    if (f_lookup == lookup_fast_certlut) {
285!
7505
        if (depth != 3) {
×
7506
            status = SIXEL_BAD_ARGUMENT;
×
7507
            sixel_helper_set_additional_message(
×
7508
                "sixel_quant_apply_palette: "
7509
                "certlut requires RGB pixels.");
7510
            goto end;
×
7511
        }
7512
        if (quant_method_for_largest == SIXEL_LARGE_LUM) {
×
7513
            wR = complexion * 299;
×
7514
            wG = 587;
×
7515
            wB = 114;
×
7516
        } else {
7517
            wR = complexion;
×
7518
            wG = 1;
×
7519
            wB = 1;
×
7520
        }
7521
        status = sixel_certlut_build(&certlut,
×
7522
                                     (sixel_color_t const *)palette,
7523
                                     reqcolor,
7524
                                     wR,
7525
                                     wG,
7526
                                     wB);
7527
        if (SIXEL_FAILED(status)) {
×
7528
            goto end;
×
7529
        }
7530
        certlut_context = &certlut;
×
7531
        certlut_ready = 1;
×
7532
    }
7533

7534
    if (use_positional) {
285!
7535
        status = apply_palette_positional(result, data, width, height,
×
7536
                                          depth, palette, reqcolor,
7537
                                          methodForDiffuse, methodForScan,
7538
                                          foptimize_palette, f_lookup,
7539
                                          indextable, complexion, copy,
7540
                                          new_palette, migration_map,
7541
                                          ncolors);
7542
    } else if (use_varerr) {
285!
7543
        status = apply_palette_variable(result, data, width, height,
×
7544
                                        depth, palette, reqcolor,
7545
                                        methodForScan, foptimize_palette,
7546
                                        f_lookup, indextable, complexion,
7547
                                        new_palette, migration_map,
7548
                                        ncolors,
7549
                                        methodForDiffuse,
7550
                                        carry_mode);
7551
    } else {
7552
        status = apply_palette_fixed(result, data, width, height,
400✔
7553
                                     depth, palette, reqcolor,
115✔
7554
                                     methodForScan, foptimize_palette,
115✔
7555
                                     f_lookup, indextable, complexion,
115✔
7556
                                     new_palette, migration_map,
115✔
7557
                                     ncolors, methodForDiffuse,
115✔
7558
                                     carry_mode);
115✔
7559
    }
7560
    if (SIXEL_FAILED(status)) {
285!
7561
        goto end;
×
7562
    }
7563

7564
    if (certlut_ready) {
285!
7565
        certlut_context = NULL;
×
7566
        sixel_certlut_free(&certlut);
×
7567
        certlut_ready = 0;
×
7568
    }
7569

7570
    if (allocated_cache) {
285!
7571
        sixel_quant_cache_release(indextable,
×
7572
                                  cache_policy,
7573
                                  allocator);
7574
    }
7575

7576
    status = SIXEL_OK;
285✔
7577

7578
end:
170✔
7579
    if (certlut_ready) {
285!
7580
        certlut_context = NULL;
×
7581
        sixel_certlut_free(&certlut);
×
7582
    }
7583
    return status;
285✔
7584
}
7585

7586

7587
void
7588
sixel_quant_free_palette(
260✔
7589
    unsigned char       /* in */ *data,
7590
    sixel_allocator_t   /* in */ *allocator)
7591
{
7592
    sixel_allocator_free(allocator, data);
260✔
7593
}
260✔
7594

7595

7596
#if HAVE_TESTS
7597
static int
7598
test1(void)
×
7599
{
7600
    int nret = EXIT_FAILURE;
×
7601
    sample minval[1] = { 1 };
×
7602
    sample maxval[1] = { 2 };
×
7603
    unsigned int retval;
7604

7605
    retval = largestByLuminosity(minval, maxval, 1);
×
7606
    if (retval != 0) {
×
7607
        goto error;
×
7608
    }
7609
    nret = EXIT_SUCCESS;
×
7610

7611
error:
7612
    return nret;
×
7613
}
7614

7615

7616
int
7617
sixel_quant_tests_main(void)
×
7618
{
7619
    int nret = EXIT_FAILURE;
×
7620
    size_t i;
7621
    typedef int (* testcase)(void);
7622

7623
    static testcase const testcases[] = {
7624
        test1,
7625
    };
7626

7627
    for (i = 0; i < sizeof(testcases) / sizeof(testcase); ++i) {
×
7628
        nret = testcases[i]();
×
7629
        if (nret != EXIT_SUCCESS) {
×
7630
            goto error;
×
7631
        }
7632
    }
7633

7634
    nret = EXIT_SUCCESS;
×
7635

7636
error:
7637
    return nret;
×
7638
}
7639
#endif  /* HAVE_TESTS */
7640

7641
/* emacs Local Variables:      */
7642
/* emacs mode: c               */
7643
/* emacs tab-width: 4          */
7644
/* emacs indent-tabs-mode: nil */
7645
/* emacs c-basic-offset: 4     */
7646
/* emacs End:                  */
7647
/* vim: set expandtab ts=4 sts=4 sw=4 : */
7648
/* EOF */
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc