• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

saitoha / libsixel / 19204137627

09 Nov 2025 05:44AM UTC coverage: 45.897% (-0.8%) from 46.739%
19204137627

push

github

saitoha
feat: add new lut policy -L certlut (certified hierarchical LUT)

8080 of 26001 branches covered (31.08%)

19 of 489 new or added lines in 2 files covered. (3.89%)

2 existing lines in 2 files now uncovered.

11470 of 24991 relevant lines covered (45.9%)

2338550.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.04
/src/quant.c
1
/*
2
 *
3
 * mediancut algorithm implementation is imported from pnmcolormap.c
4
 * in netpbm library.
5
 * http://netpbm.sourceforge.net/
6
 *
7
 * *******************************************************************************
8
 *                  original license block of pnmcolormap.c
9
 * *******************************************************************************
10
 *
11
 *   Derived from ppmquant, originally by Jef Poskanzer.
12
 *
13
 *   Copyright (C) 1989, 1991 by Jef Poskanzer.
14
 *   Copyright (C) 2001 by Bryan Henderson.
15
 *
16
 *   Permission to use, copy, modify, and distribute this software and its
17
 *   documentation for any purpose and without fee is hereby granted, provided
18
 *   that the above copyright notice appear in all copies and that both that
19
 *   copyright notice and this permission notice appear in supporting
20
 *   documentation.  This software is provided "as is" without express or
21
 *   implied warranty.
22
 *
23
 * ******************************************************************************
24
 *
25
 * Copyright (c) 2014-2018 Hayaki Saito
26
 *
27
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
28
 * this software and associated documentation files (the "Software"), to deal in
29
 * the Software without restriction, including without limitation the rights to
30
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
31
 * the Software, and to permit persons to whom the Software is furnished to do so,
32
 * subject to the following conditions:
33
 *
34
 * The above copyright notice and this permission notice shall be included in all
35
 * copies or substantial portions of the Software.
36
 *
37
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
38
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
39
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
40
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
41
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
42
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
43
 *
44
 *
45
 */
46

47
#include "config.h"
48

49
/* STDC_HEADERS */
50
#include <stdlib.h>
51
#include <stdio.h>
52
#include <time.h>
53

54
#if HAVE_STRING_H
55
# include <string.h>
56
#endif  /* HAVE_STRING_H */
57
#if HAVE_MATH_H
58
#include <math.h>
59
#endif  /* HAVE_MATH_H */
60
#if HAVE_LIMITS_H
61
# include <limits.h>
62
#endif  /* HAVE_MATH_H */
63
#if HAVE_STDINT_H
64
# include <stdint.h>
65
#else
66
# if HAVE_INTTYPES_H
67
#  include <inttypes.h>
68
# endif
69
#endif  /* HAVE_STDINT_H */
70

71
#if defined(SIXEL_USE_SSE2)
72
# include <emmintrin.h>
73
# include <xmmintrin.h>
74
#elif defined(SIXEL_USE_NEON)
75
# include <arm_neon.h>
76
#endif
77

78
#include "quant.h"
79
#include "compat_stub.h"
80

81
typedef struct {
82
    uint8_t r;
83
    uint8_t g;
84
    uint8_t b;
85
} sixel_color_t;
86

87
static float env_final_merge_target_factor = 1.81;
88

89
static unsigned char const sixel_safe_tones[256] = {
90
    0,   0,   3,   3,   5,   5,   5,   8,   8,   10,  10,  10,  13,  13,  13,
91
    15,  15,  18,  18,  18,  20,  20,  23,  23,  23,  26,  26,  28,  28,  28,
92
    31,  31,  33,  33,  33,  36,  36,  38,  38,  38,  41,  41,  41,  43,  43,
93
    46,  46,  46,  48,  48,  51,  51,  51,  54,  54,  56,  56,  56,  59,  59,
94
    61,  61,  61,  64,  64,  64,  66,  66,  69,  69,  69,  71,  71,  74,  74,
95
    74,  77,  77,  79,  79,  79,  82,  82,  84,  84,  84,  87,  87,  89,  89,
96
    89,  92,  92,  92,  94,  94,  97,  97,  97,  99,  99,  102, 102, 102, 105,
97
    105, 107, 107, 107, 110, 110, 112, 112, 112, 115, 115, 115, 117, 117, 120,
98
    120, 120, 122, 122, 125, 125, 125, 128, 128, 130, 130, 130, 133, 133, 135,
99
    135, 135, 138, 138, 140, 140, 140, 143, 143, 143, 145, 145, 148, 148, 148,
100
    150, 150, 153, 153, 153, 156, 156, 158, 158, 158, 161, 161, 163, 163, 163,
101
    166, 166, 166, 168, 168, 171, 171, 171, 173, 173, 176, 176, 176, 179, 179,
102
    181, 181, 181, 184, 184, 186, 186, 186, 189, 189, 191, 191, 191, 194, 194,
103
    194, 196, 196, 199, 199, 199, 201, 201, 204, 204, 204, 207, 207, 209, 209,
104
    209, 212, 212, 214, 214, 214, 217, 217, 217, 219, 219, 222, 222, 222, 224,
105
    224, 227, 227, 227, 230, 230, 232, 232, 232, 235, 235, 237, 237, 237, 240,
106
    240, 242, 242, 242, 245, 245, 245, 247, 247, 250, 250, 250, 252, 252, 255,
107
    255
108
};
109

110

111
static float mask_a(int x, int y, int c);
112
static float mask_x(int x, int y, int c);
113
static void diffuse_none(unsigned char *data, int width, int height,
114
                         int x, int y, int depth, int error, int direction);
115
static void diffuse_fs(unsigned char *data, int width, int height,
116
                       int x, int y, int depth, int error, int direction);
117
static void diffuse_atkinson(unsigned char *data, int width, int height,
118
                             int x, int y, int depth, int error,
119
                             int direction);
120
static void diffuse_jajuni(unsigned char *data, int width, int height,
121
                           int x, int y, int depth, int error,
122
                           int direction);
123
static void diffuse_stucki(unsigned char *data, int width, int height,
124
                           int x, int y, int depth, int error,
125
                           int direction);
126
static void diffuse_burkes(unsigned char *data, int width, int height,
127
                           int x, int y, int depth, int error,
128
                           int direction);
129
static void diffuse_sierra1(unsigned char *data, int width, int height,
130
                            int x, int y, int depth, int error,
131
                            int direction);
132
static void diffuse_sierra2(unsigned char *data, int width, int height,
133
                            int x, int y, int depth, int error,
134
                            int direction);
135
static void diffuse_sierra3(unsigned char *data, int width, int height,
136
                            int x, int y, int depth, int error,
137
                            int direction);
138
static void diffuse_none_carry(int32_t *carry_curr, int32_t *carry_next,
139
                               int32_t *carry_far, int width, int height,
140
                               int depth, int x, int y, int32_t error,
141
                               int direction, int channel);
142
static void diffuse_fs_carry(int32_t *carry_curr, int32_t *carry_next,
143
                             int32_t *carry_far, int width, int height,
144
                             int depth, int x, int y, int32_t error,
145
                             int direction, int channel);
146
static void diffuse_atkinson_carry(int32_t *carry_curr, int32_t *carry_next,
147
                                   int32_t *carry_far, int width,
148
                                   int height, int depth, int x, int y,
149
                                   int32_t error, int direction,
150
                                   int channel);
151
static void diffuse_jajuni_carry(int32_t *carry_curr, int32_t *carry_next,
152
                                 int32_t *carry_far, int width, int height,
153
                                 int depth, int x, int y, int32_t error,
154
                                 int direction, int channel);
155
static void diffuse_stucki_carry(int32_t *carry_curr, int32_t *carry_next,
156
                                 int32_t *carry_far, int width, int height,
157
                                 int depth, int x, int y, int32_t error,
158
                                 int direction, int channel);
159
static void diffuse_burkes_carry(int32_t *carry_curr, int32_t *carry_next,
160
                                 int32_t *carry_far, int width, int height,
161
                                 int depth, int x, int y, int32_t error,
162
                                 int direction, int channel);
163
static void diffuse_sierra1_carry(int32_t *carry_curr, int32_t *carry_next,
164
                                  int32_t *carry_far, int width, int height,
165
                                  int depth, int x, int y, int32_t error,
166
                                  int direction, int channel);
167
static void diffuse_sierra2_carry(int32_t *carry_curr, int32_t *carry_next,
168
                                  int32_t *carry_far, int width, int height,
169
                                  int depth, int x, int y, int32_t error,
170
                                  int direction, int channel);
171
static void diffuse_sierra3_carry(int32_t *carry_curr, int32_t *carry_next,
172
                                  int32_t *carry_far, int width, int height,
173
                                  int depth, int x, int y, int32_t error,
174
                                  int direction, int channel);
175

176
typedef struct {
177
    int index;
178
    int left;
179
    int right;
180
    unsigned char axis;
181
} sixel_certlut_node_t;
182

183
typedef struct {
184
    uint32_t *level0;
185
    uint8_t *pool;
186
    uint32_t pool_size;
187
    uint32_t pool_capacity;
188
    int wR;
189
    int wG;
190
    int wB;
191
    uint64_t wR2;
192
    uint64_t wG2;
193
    uint64_t wB2;
194
    int32_t wr_scale[256];
195
    int32_t wg_scale[256];
196
    int32_t wb_scale[256];
197
    int32_t *wr_palette;
198
    int32_t *wg_palette;
199
    int32_t *wb_palette;
200
    sixel_color_t const *palette;
201
    int ncolors;
202
    sixel_certlut_node_t *kdnodes;
203
    int kdnodes_count;
204
    int kdtree_root;
205
} sixel_certlut_t;
206

207
#define SIXEL_CERTLUT_BRANCH_FLAG 0x40000000U
208
/* #define DEBUG_CERTLUT_TRACE 1 */
209

210
static void sixel_quant_cell_center(int rmin, int gmin, int bmin, int size,
211
                                    int *cr, int *cg, int *cb);
212
static void sixel_quant_weight_init(sixel_certlut_t *lut, int wR, int wG,
213
                                    int wB);
214
static uint64_t sixel_certlut_distance_precomputed(
215
    sixel_certlut_t const *lut,
216
    int index,
217
    int32_t wr_r,
218
    int32_t wg_g,
219
    int32_t wb_b);
220
static int sixel_certlut_palette_component(sixel_certlut_t const *lut,
221
                                           int index, int axis);
222
static void sixel_certlut_sort_indices(sixel_certlut_t const *lut,
223
                                       int *indices, int count, int axis);
224
static SIXELSTATUS sixel_certlut_kdtree_build(sixel_certlut_t *lut);
225
static int sixel_certlut_kdtree_build_recursive(sixel_certlut_t *lut,
226
                                                int *indices,
227
                                                int count,
228
                                                int depth);
229
static uint64_t sixel_certlut_axis_distance(sixel_certlut_t const *lut,
230
                                            int diff, int axis);
231
static void sixel_certlut_consider_candidate(sixel_certlut_t const *lut,
232
                                             int candidate,
233
                                             int32_t wr_r,
234
                                             int32_t wg_g,
235
                                             int32_t wb_b,
236
                                             int *best_idx,
237
                                             uint64_t *best_dist,
238
                                             int *second_idx,
239
                                             uint64_t *second_dist);
240
static void sixel_certlut_kdtree_search(sixel_certlut_t const *lut,
241
                                        int node_index,
242
                                        int r,
243
                                        int g,
244
                                        int b,
245
                                        int32_t wr_r,
246
                                        int32_t wg_g,
247
                                        int32_t wb_b,
248
                                        int *best_idx,
249
                                        uint64_t *best_dist,
250
                                        int *second_idx,
251
                                        uint64_t *second_dist);
252
static void sixel_quant_distance_pair(sixel_certlut_t const *lut, int r,
253
                                      int g, int b, int *best_idx,
254
                                      int *second_idx, uint64_t *best_dist,
255
                                      uint64_t *second_dist);
256
static int sixel_quant_is_cell_safe(sixel_certlut_t const *lut, int best_idx,
257
                                    int second_idx, int size,
258
                                    uint64_t best_dist,
259
                                    uint64_t second_dist);
260
static uint32_t sixel_quant_pool_alloc(sixel_certlut_t *lut, int *status);
261
static void sixel_quant_assign_leaf(uint32_t *cell, int palette_index);
262
static void sixel_quant_assign_branch(uint32_t *cell, uint32_t offset);
263
static uint8_t sixel_certlut_fallback(sixel_certlut_t const *lut,
264
                                      int r, int g, int b);
265
static int sixel_certlut_build_cell(sixel_certlut_t *lut, uint32_t *cell,
266
                                    int rmin, int gmin, int bmin, int size);
267
static int sixel_certlut_init(sixel_certlut_t *lut);
268
static void sixel_certlut_release(sixel_certlut_t *lut);
269
static int sixel_certlut_prepare_palette_terms(sixel_certlut_t *lut);
270

271
static sixel_certlut_t *certlut_context = NULL;
272

273
static const int (*
274
lso2_table(void))[7]
×
275
{
276
#include "lso2.h"
277
    return var_coefs;
278
}
279

280
#define VARERR_SCALE_SHIFT 12
281
#define VARERR_SCALE       (1 << VARERR_SCALE_SHIFT)
282
#define VARERR_ROUND       (1 << (VARERR_SCALE_SHIFT - 1))
283
#define VARERR_MAX_VALUE   (255 * VARERR_SCALE)
284

285
#if HAVE_DEBUG
286
#define quant_trace fprintf
287
#else
288
static inline void quant_trace(FILE *f, ...) { (void) f; }
289
#endif
290

291
/*****************************************************************************
292
 *
293
 * quantization
294
 *
295
 *****************************************************************************/
296

297
typedef struct box* boxVector;
298
struct box {
299
    unsigned int ind;
300
    unsigned int colors;
301
    unsigned int sum;
302
};
303

304
typedef unsigned long sample;
305
typedef sample * tuple;
306

307
static unsigned char
308
sixel_quant_reversible_value(unsigned int sample)
×
309
{
310
    if (sample > 255U) {
×
311
        sample = 255U;
×
312
    }
313

314
    return sixel_safe_tones[sample];
×
315
}
316

317
static void
318
sixel_quant_reversible_pixel(unsigned char const *src,
×
319
                             unsigned int depth,
320
                             unsigned char *dst)
321
{
322
    unsigned int plane;
323

324
    for (plane = 0U; plane < depth; ++plane) {
×
325
        dst[plane] = sixel_quant_reversible_value(src[plane]);
×
326
    }
327
}
×
328

329
static void
330
sixel_quant_reversible_tuple(sample *tuple,
×
331
                             unsigned int depth)
332
{
333
    unsigned int plane;
334
    unsigned int sample_value;
335

336
    for (plane = 0U; plane < depth; ++plane) {
×
337
        sample_value = (unsigned int)tuple[plane];
×
338
        tuple[plane] =
×
339
            (sample)sixel_quant_reversible_value(sample_value);
×
340
    }
341
}
×
342

343
static void
344
sixel_quant_reversible_palette(unsigned char *palette,
×
345
                               unsigned int colors,
346
                               unsigned int depth)
347
{
348
    unsigned int index;
349
    unsigned int plane;
350
    unsigned int sample_value;
351
    size_t offset;
352

353
    for (index = 0U; index < colors; ++index) {
×
354
        for (plane = 0U; plane < depth; ++plane) {
×
355
            offset = (size_t)index * (size_t)depth + (size_t)plane;
×
356
            sample_value = (unsigned int)palette[offset];
×
357
            palette[offset] =
×
358
                sixel_quant_reversible_value(sample_value);
×
359
        }
360
    }
361
}
×
362

363
struct tupleint {
364
    /* An ordered pair of a tuple value and an integer, such as you
365
       would find in a tuple table or tuple hash.
366
       Note that this is a variable length structure.
367
    */
368
    unsigned int value;
369
    sample tuple[1];
370
    /* This is actually a variable size array -- its size is the
371
       depth of the tuple in question.  Some compilers do not let us
372
       declare a variable length array.
373
    */
374
};
375
typedef struct tupleint ** tupletable;
376

377
typedef struct {
378
    unsigned int size;
379
    tupletable table;
380
} tupletable2;
381

382
static unsigned int compareplanePlane;
383
static tupletable2 const *force_palette_source;
384
    /* This is a parameter to compareplane().  We use this global variable
385
       so that compareplane() can be called by qsort(), to compare two
386
       tuples.  qsort() doesn't pass any arguments except the two tuples.
387
    */
388
static int
389
compareplane(const void * const arg1,
12,089,356✔
390
             const void * const arg2)
391
{
392
    int lhs, rhs;
393

394
    typedef const struct tupleint * const * const sortarg;
395
    sortarg comparandPP  = (sortarg) arg1;
12,089,356✔
396
    sortarg comparatorPP = (sortarg) arg2;
12,089,356✔
397
    lhs = (int)(*comparandPP)->tuple[compareplanePlane];
12,089,356✔
398
    rhs = (int)(*comparatorPP)->tuple[compareplanePlane];
12,089,356✔
399

400
    return lhs - rhs;
12,089,356✔
401
}
402

403

404
static int
405
sumcompare(const void * const b1, const void * const b2)
27,883,574✔
406
{
407
    return (int)((boxVector)b2)->sum - (int)((boxVector)b1)->sum;
27,883,574✔
408
}
409

410

411
static SIXELSTATUS
412
alloctupletable(
520✔
413
    tupletable          /* out */ *result,
414
    unsigned int const  /* in */  depth,
415
    unsigned int const  /* in */  size,
416
    sixel_allocator_t   /* in */  *allocator)
417
{
418
    SIXELSTATUS status = SIXEL_FALSE;
520✔
419
    enum { message_buffer_size = 256 };
420
    char message[message_buffer_size];
421
    int nwrite;
422
    unsigned int mainTableSize;
423
    unsigned int tupleIntSize;
424
    unsigned int allocSize;
425
    void * pool;
426
    tupletable tbl;
427
    unsigned int i;
428

429
    if (UINT_MAX / sizeof(struct tupleint) < size) {
520!
430
        nwrite = sixel_compat_snprintf(
×
431
            message,
432
            sizeof(message),
433
            "size %u is too big for arithmetic",
434
            size);
435
        if (nwrite > 0) {
×
436
            sixel_helper_set_additional_message(message);
×
437
        }
438
        status = SIXEL_RUNTIME_ERROR;
×
439
        goto end;
×
440
    }
441

442
    mainTableSize = size * sizeof(struct tupleint *);
520✔
443
    tupleIntSize = sizeof(struct tupleint) - sizeof(sample)
520✔
444
        + depth * sizeof(sample);
236✔
445

446
    /* To save the enormous amount of time it could take to allocate
447
       each individual tuple, we do a trick here and allocate everything
448
       as a single malloc block and suballocate internally.
449
    */
450
    if ((UINT_MAX - mainTableSize) / tupleIntSize < size) {
520!
451
        nwrite = sixel_compat_snprintf(
×
452
            message,
453
            sizeof(message),
454
            "size %u is too big for arithmetic",
455
            size);
456
        if (nwrite > 0) {
×
457
            sixel_helper_set_additional_message(message);
×
458
        }
459
        status = SIXEL_RUNTIME_ERROR;
×
460
        goto end;
×
461
    }
462

463
    allocSize = mainTableSize + size * tupleIntSize;
520✔
464

465
    pool = sixel_allocator_malloc(allocator, allocSize);
520✔
466
    if (pool == NULL) {
520!
467
        sixel_compat_snprintf(
×
468
            message,
469
            sizeof(message),
470
            "unable to allocate %u bytes for a %u-entry tuple table",
471
            allocSize,
472
            size);
473
        sixel_helper_set_additional_message(message);
×
474
        status = SIXEL_BAD_ALLOCATION;
×
475
        goto end;
×
476
    }
477
    tbl = (tupletable) pool;
520✔
478

479
    for (i = 0; i < size; ++i)
311,411✔
480
        tbl[i] = (struct tupleint *)
310,891✔
481
            ((char*)pool + mainTableSize + i * tupleIntSize);
310,891✔
482

483
    *result = tbl;
520✔
484

485
    status = SIXEL_OK;
520✔
486

487
end:
284✔
488
    return status;
520✔
489
}
490

491

492
/*
493
** Here is the fun part, the median-cut colormap generator.  This is based
494
** on Paul Heckbert's paper "Color Image Quantization for Frame Buffer
495
** Display", SIGGRAPH '82 Proceedings, page 297.
496
*/
497

498
static tupletable2
499
newColorMap(unsigned int const newcolors, unsigned int const depth, sixel_allocator_t *allocator)
69✔
500
{
501
    SIXELSTATUS status = SIXEL_FALSE;
69✔
502
    tupletable2 colormap;
503
    unsigned int i;
504

505
    colormap.size = 0;
69✔
506
    status = alloctupletable(&colormap.table, depth, newcolors, allocator);
69✔
507
    if (SIXEL_FAILED(status)) {
69!
508
        goto end;
×
509
    }
510
    if (colormap.table) {
92!
511
        for (i = 0; i < newcolors; ++i) {
13,833✔
512
            unsigned int plane;
513
            for (plane = 0; plane < depth; ++plane)
55,056✔
514
                colormap.table[i]->tuple[plane] = 0;
41,292✔
515
        }
4,588✔
516
        colormap.size = newcolors;
69✔
517
    }
23✔
518

519
end:
520
    return colormap;
69✔
521
}
522

523

524
static boxVector
525
newBoxVector(
69✔
526
    unsigned int const  /* in */ colors,
527
    unsigned int const  /* in */ sum,
528
    unsigned int const  /* in */ newcolors,
529
    sixel_allocator_t   /* in */ *allocator)
530
{
531
    boxVector bv;
532

533
    bv = (boxVector)sixel_allocator_malloc(allocator,
92✔
534
                                           sizeof(struct box) * (size_t)newcolors);
69✔
535
    if (bv == NULL) {
69!
536
        quant_trace(stderr, "out of memory allocating box vector table\n");
×
537
        return NULL;
×
538
    }
539

540
    /* Set up the initial box. */
541
    bv[0].ind = 0;
69✔
542
    bv[0].colors = colors;
69✔
543
    bv[0].sum = sum;
69✔
544

545
    return bv;
69✔
546
}
23✔
547

548

549
static void
550
findBoxBoundaries(tupletable2  const colorfreqtable,
24,813✔
551
                  unsigned int const depth,
552
                  unsigned int const boxStart,
553
                  unsigned int const boxSize,
554
                  sample             minval[],
555
                  sample             maxval[])
556
{
557
/*----------------------------------------------------------------------------
558
  Go through the box finding the minimum and maximum of each
559
  component - the boundaries of the box.
560
-----------------------------------------------------------------------------*/
561
    unsigned int plane;
562
    unsigned int i;
563

564
    for (plane = 0; plane < depth; ++plane) {
99,252✔
565
        minval[plane] = colorfreqtable.table[boxStart]->tuple[plane];
74,439✔
566
        maxval[plane] = minval[plane];
74,439✔
567
    }
24,813✔
568

569
    for (i = 1; i < boxSize; ++i) {
2,226,612✔
570
        for (plane = 0; plane < depth; ++plane) {
8,807,196✔
571
            sample const v = colorfreqtable.table[boxStart + i]->tuple[plane];
6,605,397✔
572
            if (v < minval[plane]) minval[plane] = v;
6,605,397✔
573
            if (v > maxval[plane]) maxval[plane] = v;
6,605,397✔
574
        }
2,202,753✔
575
    }
734,251✔
576
}
24,813✔
577

578

579

580
static unsigned int
581
largestByNorm(sample minval[], sample maxval[], unsigned int const depth)
23,427✔
582
{
583

584
    unsigned int largestDimension;
585
    unsigned int plane;
586
    sample largestSpreadSoFar;
587

588
    largestSpreadSoFar = 0;
23,427✔
589
    largestDimension = 0;
23,427✔
590
    for (plane = 0; plane < depth; ++plane) {
93,708✔
591
        sample const spread = maxval[plane]-minval[plane];
70,281✔
592
        if (spread > largestSpreadSoFar) {
70,281✔
593
            largestDimension = plane;
39,144✔
594
            largestSpreadSoFar = spread;
39,144✔
595
        }
12,946✔
596
    }
23,427✔
597
    return largestDimension;
23,427✔
598
}
599

600

601

602
static unsigned int
603
largestByLuminosity(sample minval[], sample maxval[], unsigned int const depth)
1,386✔
604
{
605
/*----------------------------------------------------------------------------
606
   This subroutine presumes that the tuple type is either
607
   BLACKANDWHITE, GRAYSCALE, or RGB (which implies pamP->depth is 1 or 3).
608
   To save time, we don't actually check it.
609
-----------------------------------------------------------------------------*/
610
    unsigned int retval;
611

612
    double lumin_factor[3] = {0.2989, 0.5866, 0.1145};
1,386✔
613

614
    if (depth == 1) {
1,386!
615
        retval = 0;
×
616
    } else {
617
        /* An RGB tuple */
618
        unsigned int largestDimension;
619
        unsigned int plane;
620
        double largestSpreadSoFar;
621

622
        largestSpreadSoFar = 0.0;
1,386✔
623
        largestDimension = 0;
1,386✔
624

625
        for (plane = 0; plane < 3; ++plane) {
5,544✔
626
            double const spread =
4,158✔
627
                lumin_factor[plane] * (maxval[plane]-minval[plane]);
4,158✔
628
            if (spread > largestSpreadSoFar) {
4,158✔
629
                largestDimension = plane;
2,369✔
630
                largestSpreadSoFar = spread;
2,369✔
631
            }
809✔
632
        }
1,386✔
633
        retval = largestDimension;
1,386✔
634
    }
635
    return retval;
1,386✔
636
}
637

638

639

640
static void
641
centerBox(unsigned int const boxStart,
×
642
          unsigned int const boxSize,
643
          tupletable2  const colorfreqtable,
644
          unsigned int const depth,
645
          tuple        const newTuple)
646
{
647

648
    unsigned int plane;
649
    sample minval, maxval;
650
    unsigned int i;
651

652
    for (plane = 0; plane < depth; ++plane) {
×
653
        minval = maxval = colorfreqtable.table[boxStart]->tuple[plane];
×
654

655
        for (i = 1; i < boxSize; ++i) {
×
656
            sample v = colorfreqtable.table[boxStart + i]->tuple[plane];
×
657
            minval = minval < v ? minval: v;
×
658
            maxval = maxval > v ? maxval: v;
×
659
        }
660
        newTuple[plane] = (minval + maxval) / 2;
×
661
    }
662
}
×
663

664

665

666
static void
667
averageColors(unsigned int const boxStart,
×
668
              unsigned int const boxSize,
669
              tupletable2  const colorfreqtable,
670
              unsigned int const depth,
671
              tuple        const newTuple)
672
{
673
    unsigned int plane;
674
    sample sum;
675
    unsigned int i;
676

677
    for (plane = 0; plane < depth; ++plane) {
×
678
        sum = 0;
×
679

680
        for (i = 0; i < boxSize; ++i) {
×
681
            sum += colorfreqtable.table[boxStart + i]->tuple[plane];
×
682
        }
683

684
        newTuple[plane] = sum / boxSize;
×
685
    }
686
}
×
687

688

689

690
static void
691
averagePixels(unsigned int const boxStart,
×
692
              unsigned int const boxSize,
693
              tupletable2 const colorfreqtable,
694
              unsigned int const depth,
695
              tuple const newTuple)
696
{
697

698
    unsigned int n;
699
        /* Number of tuples represented by the box */
700
    unsigned int plane;
701
    unsigned int i;
702

703
    /* Count the tuples in question */
704
    n = 0;  /* initial value */
×
705
    for (i = 0; i < boxSize; ++i) {
×
706
        n += (unsigned int)colorfreqtable.table[boxStart + i]->value;
×
707
    }
708

709
    for (plane = 0; plane < depth; ++plane) {
×
710
        sample sum;
711

712
        sum = 0;
×
713

714
        for (i = 0; i < boxSize; ++i) {
×
715
            sum += colorfreqtable.table[boxStart + i]->tuple[plane]
×
716
                * (unsigned int)colorfreqtable.table[boxStart + i]->value;
×
717
        }
718

719
        newTuple[plane] = sum / n;
×
720
    }
721
}
×
722

723

724

725
static tupletable2
726
colormapFromBv(unsigned int const newcolors,
×
727
               boxVector const bv,
728
               unsigned int const boxes,
729
               tupletable2 const colorfreqtable,
730
               unsigned int const depth,
731
               int const methodForRep,
732
               int const use_reversible,
733
               sixel_allocator_t *allocator)
734
{
735
    /*
736
    ** Ok, we've got enough boxes.  Now choose a representative color for
737
    ** each box.  There are a number of possible ways to make this choice.
738
    ** One would be to choose the center of the box; this ignores any structure
739
    ** within the boxes.  Another method would be to average all the colors in
740
    ** the box - this is the method specified in Heckbert's paper.  A third
741
    ** method is to average all the pixels in the box.
742
    */
743
    tupletable2 colormap;
744
    unsigned int bi;
745

746
    colormap = newColorMap(newcolors, depth, allocator);
×
747
    if (!colormap.size) {
×
748
        return colormap;
×
749
    }
750

751
    for (bi = 0; bi < boxes; ++bi) {
×
752
        switch (methodForRep) {
×
753
        case SIXEL_REP_CENTER_BOX:
754
            centerBox(bv[bi].ind, bv[bi].colors,
×
755
                      colorfreqtable, depth,
756
                      colormap.table[bi]->tuple);
×
757
            break;
×
758
        case SIXEL_REP_AVERAGE_COLORS:
759
            averageColors(bv[bi].ind, bv[bi].colors,
×
760
                          colorfreqtable, depth,
761
                          colormap.table[bi]->tuple);
×
762
            break;
×
763
        case SIXEL_REP_AVERAGE_PIXELS:
764
            averagePixels(bv[bi].ind, bv[bi].colors,
×
765
                          colorfreqtable, depth,
766
                          colormap.table[bi]->tuple);
×
767
            break;
×
768
        default:
769
            quant_trace(stderr, "Internal error: "
×
770
                                "invalid value of methodForRep: %d\n",
771
                        methodForRep);
772
        }
773
        if (use_reversible) {
×
774
            sixel_quant_reversible_tuple(colormap.table[bi]->tuple,
×
775
                                         depth);
776
        }
777
    }
778
    return colormap;
×
779
}
780

781

782
static int
783
force_palette_compare(const void *lhs, const void *rhs)
×
784
{
785
    unsigned int left;
786
    unsigned int right;
787
    unsigned int left_value;
788
    unsigned int right_value;
789

790
    left = *(const unsigned int *)lhs;
×
791
    right = *(const unsigned int *)rhs;
×
792
    left_value = force_palette_source->table[left]->value;
×
793
    right_value = force_palette_source->table[right]->value;
×
794
    if (left_value > right_value) {
×
795
        return -1;
×
796
    }
797
    if (left_value < right_value) {
×
798
        return 1;
×
799
    }
800
    if (left < right) {
×
801
        return -1;
×
802
    }
803
    if (left > right) {
×
804
        return 1;
×
805
    }
806
    return 0;
×
807
}
808

809

810
static SIXELSTATUS
811
force_palette_completion(tupletable2 *colormapP,
×
812
                         unsigned int depth,
813
                         unsigned int reqColors,
814
                         tupletable2 const colorfreqtable,
815
                         sixel_allocator_t *allocator)
816
{
817
    /*
818
     * We enqueue "losers" from the histogram so that we can revive them:
819
     *
820
     *   histogram --> sort by hit count --> append to palette tail
821
     *        ^                             |
822
     *        +-----------------------------+
823
     *
824
     * The ASCII loop shows how discarded colors walk back into the
825
     * palette when the user demands an exact size.
826
     */
827
    SIXELSTATUS status = SIXEL_FALSE;
×
828
    tupletable new_table = NULL;
×
829
    unsigned int *order = NULL;
×
830
    unsigned int current;
831
    unsigned int fill;
832
    unsigned int candidate;
833
    unsigned int plane;
834
    unsigned int source;
835

836
    current = colormapP->size;
×
837
    if (current >= reqColors) {
×
838
        return SIXEL_OK;
×
839
    }
840

841
    status = alloctupletable(&new_table, depth, reqColors, allocator);
×
842
    if (SIXEL_FAILED(status)) {
×
843
        goto end;
×
844
    }
845

846
    if (colorfreqtable.size > 0U) {
×
847
        order = (unsigned int *)sixel_allocator_malloc(
×
848
            allocator, colorfreqtable.size * sizeof(unsigned int));
×
849
        if (order == NULL) {
×
850
            status = SIXEL_BAD_ALLOCATION;
×
851
            goto end;
×
852
        }
853
        for (candidate = 0; candidate < colorfreqtable.size; ++candidate) {
×
854
            order[candidate] = candidate;
×
855
        }
856
        force_palette_source = &colorfreqtable;
×
857
        qsort(order, colorfreqtable.size, sizeof(unsigned int),
×
858
              force_palette_compare);
859
        force_palette_source = NULL;
×
860
    }
861

862
    for (fill = 0; fill < current; ++fill) {
×
863
        new_table[fill]->value = colormapP->table[fill]->value;
×
864
        for (plane = 0; plane < depth; ++plane) {
×
865
            new_table[fill]->tuple[plane] =
×
866
                colormapP->table[fill]->tuple[plane];
×
867
        }
868
    }
869

870
    candidate = 0U;
×
871
    fill = current;
×
872
    if (order != NULL) {
×
873
        while (fill < reqColors && candidate < colorfreqtable.size) {
×
874
            unsigned int index;
875

876
            index = order[candidate];
×
877
            new_table[fill]->value = colorfreqtable.table[index]->value;
×
878
            for (plane = 0; plane < depth; ++plane) {
×
879
                new_table[fill]->tuple[plane] =
×
880
                    colorfreqtable.table[index]->tuple[plane];
×
881
            }
882
            ++fill;
×
883
            ++candidate;
×
884
        }
885
    }
886

887
    if (fill < reqColors && fill == 0U) {
×
888
        new_table[fill]->value = 0U;
×
889
        for (plane = 0; plane < depth; ++plane) {
×
890
            new_table[fill]->tuple[plane] = 0U;
×
891
        }
892
        ++fill;
×
893
    }
894

895
    source = 0U;
×
896
    while (fill < reqColors) {
×
897
        new_table[fill]->value = new_table[source]->value;
×
898
        for (plane = 0; plane < depth; ++plane) {
×
899
            new_table[fill]->tuple[plane] = new_table[source]->tuple[plane];
×
900
        }
901
        ++fill;
×
902
        ++source;
×
903
        if (source >= fill) {
×
904
            source = 0U;
×
905
        }
906
    }
907

908
    sixel_allocator_free(allocator, colormapP->table);
×
909
    colormapP->table = new_table;
×
910
    colormapP->size = reqColors;
×
911
    status = SIXEL_OK;
×
912

913
end:
914
    if (status != SIXEL_OK && new_table != NULL) {
×
915
        sixel_allocator_free(allocator, new_table);
×
916
    }
917
    if (order != NULL) {
×
918
        sixel_allocator_free(allocator, order);
×
919
    }
920
    return status;
×
921
}
922

923

924
static SIXELSTATUS
925
splitBox(boxVector const bv,
24,813✔
926
         unsigned int *const boxesP,
927
         unsigned int const bi,
928
         tupletable2 const colorfreqtable,
929
         unsigned int const depth,
930
         int const methodForLargest)
931
{
932
/*----------------------------------------------------------------------------
933
   Split Box 'bi' in the box vector bv (so that bv contains one more box
934
   than it did as input).  Split it so that each new box represents about
935
   half of the pixels in the distribution given by 'colorfreqtable' for
936
   the colors in the original box, but with distinct colors in each of the
937
   two new boxes.
938

939
   Assume the box contains at least two colors.
940
-----------------------------------------------------------------------------*/
941
    SIXELSTATUS status = SIXEL_FALSE;
24,813✔
942
    unsigned int const boxStart = bv[bi].ind;
24,813✔
943
    unsigned int const boxSize  = bv[bi].colors;
24,813✔
944
    unsigned int const sm       = bv[bi].sum;
24,813✔
945

946
    enum { max_depth= 16 };
947
    sample minval[max_depth];
948
    sample maxval[max_depth];
949

950
    /* assert(max_depth >= depth); */
951

952
    unsigned int largestDimension;
953
    /* number of the plane with the largest spread */
954
    unsigned int medianIndex;
955
    unsigned int lowersum;
956

957
    /* Number of pixels whose value is "less than" the median */
958
    findBoxBoundaries(colorfreqtable, depth, boxStart, boxSize,
33,084✔
959
                      minval, maxval);
8,271✔
960

961
    /* Find the largest dimension, and sort by that component.  I have
962
       included two methods for determining the "largest" dimension;
963
       first by simply comparing the range in RGB space, and second by
964
       transforming into luminosities before the comparison.
965
    */
966
    switch (methodForLargest) {
24,813!
967
    case SIXEL_LARGE_NORM:
15,618✔
968
        largestDimension = largestByNorm(minval, maxval, depth);
23,427✔
969
        break;
23,427✔
970
    case SIXEL_LARGE_LUM:
924✔
971
        largestDimension = largestByLuminosity(minval, maxval, depth);
1,386✔
972
        break;
1,386✔
973
    default:
974
        sixel_helper_set_additional_message(
×
975
            "Internal error: invalid value of methodForLargest.");
976
        status = SIXEL_LOGIC_ERROR;
×
977
        goto end;
×
978
    }
979

980
    /* TODO: I think this sort should go after creating a box,
981
       not before splitting.  Because you need the sort to use
982
       the SIXEL_REP_CENTER_BOX method of choosing a color to
983
       represent the final boxes
984
    */
985

986
    /* Set the gross global variable 'compareplanePlane' as a
987
       parameter to compareplane(), which is called by qsort().
988
    */
989
    compareplanePlane = largestDimension;
24,813✔
990
    qsort((char*) &colorfreqtable.table[boxStart], boxSize,
24,813✔
991
          sizeof(colorfreqtable.table[boxStart]),
992
          compareplane);
993

994
    {
995
        /* Now find the median based on the counts, so that about half
996
           the pixels (not colors, pixels) are in each subdivision.  */
997

998
        unsigned int i;
999

1000
        lowersum = colorfreqtable.table[boxStart]->value; /* initial value */
24,813✔
1001
        for (i = 1; i < boxSize - 1 && lowersum < sm / 2; ++i) {
1,038,483✔
1002
            lowersum += colorfreqtable.table[boxStart + i]->value;
1,013,670✔
1003
        }
336,688✔
1004
        medianIndex = i;
24,813✔
1005
    }
1006
    /* Split the box, and sort to bring the biggest boxes to the top.  */
1007

1008
    bv[bi].colors = medianIndex;
24,813✔
1009
    bv[bi].sum = lowersum;
24,813✔
1010
    bv[*boxesP].ind = boxStart + medianIndex;
24,813✔
1011
    bv[*boxesP].colors = boxSize - medianIndex;
24,813✔
1012
    bv[*boxesP].sum = sm - lowersum;
24,813✔
1013
    ++(*boxesP);
24,813✔
1014
    qsort((char*) bv, *boxesP, sizeof(struct box), sumcompare);
24,813✔
1015

1016
    status = SIXEL_OK;
24,813✔
1017

1018
end:
16,542✔
1019
    return status;
24,813✔
1020
}
1021

1022

1023

1024
static unsigned int sixel_final_merge_target(unsigned int reqcolors,
1025
                                             int final_merge_mode);
1026
static void sixel_merge_clusters_ward(unsigned long *weights,
1027
                                      unsigned long *sums,
1028
                                      unsigned int depth,
1029
                                      int *cluster_count,
1030
                                      int target);
1031
static SIXELSTATUS sixel_quant_clusters_to_colormap(unsigned long *weights,
1032
                                                    unsigned long *sums,
1033
                                                    unsigned int depth,
1034
                                                    unsigned int cluster_count,
1035
                                                    int use_reversible,
1036
                                                    tupletable2 *colormapP,
1037
                                                    sixel_allocator_t *allocator);
1038

1039
static SIXELSTATUS
1040
mediancut(tupletable2 const colorfreqtable,
69✔
1041
          unsigned int const depth,
1042
          unsigned int const newcolors,
1043
          int const methodForLargest,
1044
          int const methodForRep,
1045
          int const use_reversible,
1046
          int const final_merge_mode,
1047
          tupletable2 *const colormapP,
1048
          sixel_allocator_t *allocator)
1049
{
1050
/*----------------------------------------------------------------------------
1051
   Compute a set of only 'newcolors' colors that best represent an
1052
   image whose pixels are summarized by the histogram
1053
   'colorfreqtable'.  Each tuple in that table has depth 'depth'.
1054
   colorfreqtable.table[i] tells the number of pixels in the subject image
1055
   have a particular color.
1056

1057
   As a side effect, sort 'colorfreqtable'.
1058
-----------------------------------------------------------------------------*/
1059
    boxVector bv;
1060
    unsigned int bi;
1061
    unsigned int boxes;
1062
    int multicolorBoxesExist;
1063
    unsigned int i;
1064
    unsigned int sum;
1065
    unsigned int working_colors;
1066
    int apply_merge;
1067
    unsigned long *cluster_weight;
1068
    unsigned long *cluster_sums;
1069
    int cluster_total;
1070
    unsigned int plane;
1071
    unsigned int offset;
1072
    unsigned int size;
1073
    unsigned long value;
1074
    struct tupleint *entry;
1075
    SIXELSTATUS merge_status;
1076
    SIXELSTATUS status = SIXEL_FALSE;
69✔
1077

1078
    sum = 0;
69✔
1079
    working_colors = newcolors;
69✔
1080
    apply_merge = (final_merge_mode == SIXEL_FINAL_MERGE_AUTO
69✔
1081
                   || final_merge_mode == SIXEL_FINAL_MERGE_WARD);
69!
1082
    bv = NULL;
69✔
1083
    cluster_weight = NULL;
69✔
1084
    cluster_sums = NULL;
69✔
1085
    cluster_total = 0;
69✔
1086
    plane = 0U;
69✔
1087
    offset = 0U;
69✔
1088
    size = 0U;
69✔
1089
    value = 0UL;
69✔
1090
    entry = NULL;
69✔
1091
    merge_status = SIXEL_OK;
69✔
1092

1093
    for (i = 0; i < colorfreqtable.size; ++i) {
291,168✔
1094
        sum += colorfreqtable.table[i]->value;
291,099✔
1095
    }
96,889✔
1096

1097
    if (apply_merge) {
69!
1098
        /* Choose an oversplit target so that the merge stage has slack. */
1099
        working_colors = sixel_final_merge_target(newcolors,
92✔
1100
                                                  final_merge_mode);
23✔
1101
        if (working_colors > colorfreqtable.size) {
69!
1102
            working_colors = colorfreqtable.size;
×
1103
        }
1104
        quant_trace(stderr, "overshoot: %d\n", working_colors);
69✔
1105
    }
23✔
1106
    if (working_colors == 0U) {
69!
1107
        working_colors = 1U;
×
1108
    }
1109

1110
    /* There is at least one box that contains at least 2 colors; ergo,
1111
       there is more splitting we can do.  */
1112
    bv = newBoxVector(colorfreqtable.size, sum, working_colors, allocator);
69✔
1113
    if (bv == NULL) {
69!
1114
        goto end;
×
1115
    }
1116
    boxes = 1;
69✔
1117
    multicolorBoxesExist = (colorfreqtable.size > 1);
69✔
1118

1119
    /* Main loop: split boxes until we have enough. */
1120
    while (boxes < working_colors && multicolorBoxesExist) {
24,882!
1121
        /* Find the first splittable box. */
1122
        for (bi = 0; bi < boxes && bv[bi].colors < 2; ++bi)
442,226!
1123
            ;
1124
        if (bi >= boxes) {
24,813!
1125
            multicolorBoxesExist = 0;
×
1126
        } else {
1127
            status = splitBox(bv, &boxes, bi,
33,084✔
1128
                              colorfreqtable, depth,
8,271✔
1129
                              methodForLargest);
8,271✔
1130
            if (SIXEL_FAILED(status)) {
24,813!
1131
                goto end;
×
1132
            }
1133
        }
1134
    }
1135
    if (apply_merge && boxes > newcolors) {
69!
1136
        /* Capture weight and component sums for each temporary box. */
1137
        cluster_weight = (unsigned long *)sixel_allocator_malloc(
69✔
1138
            allocator, (size_t)boxes * sizeof(unsigned long));
69✔
1139
        cluster_sums = (unsigned long *)sixel_allocator_malloc(
69✔
1140
            allocator, (size_t)boxes * (size_t)depth * sizeof(unsigned long));
69✔
1141
        if (cluster_weight == NULL || cluster_sums == NULL) {
69!
1142
            status = SIXEL_BAD_ALLOCATION;
×
1143
            goto end;
×
1144
        }
1145
        for (bi = 0U; bi < boxes; ++bi) {
24,951✔
1146
            offset = bv[bi].ind;
24,882✔
1147
            size = bv[bi].colors;
24,882✔
1148
            cluster_weight[bi] = 0UL;
24,882✔
1149
            for (plane = 0U; plane < depth; ++plane) {
99,528✔
1150
                cluster_sums[(size_t)bi * (size_t)depth + plane] = 0UL;
74,646✔
1151
            }
24,882✔
1152
            for (i = 0U; i < size; ++i) {
315,981✔
1153
                entry = colorfreqtable.table[offset + i];
291,099✔
1154
                value = (unsigned long)entry->value;
291,099✔
1155
                cluster_weight[bi] += value;
291,099✔
1156
                for (plane = 0U; plane < depth; ++plane) {
1,164,396✔
1157
                    cluster_sums[(size_t)bi * (size_t)depth + plane] +=
873,297✔
1158
                        (unsigned long)entry->tuple[plane] * value;
873,297✔
1159
                }
290,667✔
1160
            }
96,889✔
1161
        }
8,294✔
1162
        cluster_total = (int)boxes;
69✔
1163
        /* Merge clusters greedily using Ward's minimum variance rule. */
1164
        sixel_merge_clusters_ward(cluster_weight, cluster_sums, depth,
92✔
1165
                                  &cluster_total, (int)newcolors);
23✔
1166
        if (cluster_total < 1) {
69!
1167
            cluster_total = 1;
×
1168
        }
1169
        if ((unsigned int)cluster_total > newcolors) {
69!
1170
            cluster_total = (int)newcolors;
×
1171
        }
1172
        /* Rebuild the palette using the merged cluster statistics. */
1173
        merge_status = sixel_quant_clusters_to_colormap(cluster_weight,
92✔
1174
                                                        cluster_sums,
23✔
1175
                                                        depth,
23✔
1176
                                                        (unsigned int)cluster_total,
23✔
1177
                                                        use_reversible,
23✔
1178
                                                        colormapP,
23✔
1179
                                                        allocator);
23✔
1180
        if (SIXEL_FAILED(merge_status)) {
69!
1181
            status = merge_status;
×
1182
            goto end;
×
1183
        }
1184
    } else {
23✔
1185
        *colormapP = colormapFromBv(newcolors, bv, boxes,
×
1186
                                    colorfreqtable, depth,
1187
                                    methodForRep, use_reversible,
1188
                                    allocator);
1189
    }
1190

1191
    status = SIXEL_OK;
69✔
1192

1193
end:
46✔
1194
    if (bv != NULL) {
69!
1195
        sixel_allocator_free(allocator, bv);
69✔
1196
    }
23✔
1197
    if (cluster_sums != NULL) {
69!
1198
        sixel_allocator_free(allocator, cluster_sums);
69✔
1199
    }
23✔
1200
    if (cluster_weight != NULL) {
69!
1201
        sixel_allocator_free(allocator, cluster_weight);
69✔
1202
    }
23✔
1203
    return status;
69✔
1204
}
1205

1206

1207
/* Determine how many clusters to create before the final merge step. */
1208
static unsigned int
1209
sixel_final_merge_target(unsigned int reqcolors,
69✔
1210
                         int final_merge_mode)
1211
{
1212
    double factor;
1213
    unsigned int scaled;
1214

1215
    if (final_merge_mode != SIXEL_FINAL_MERGE_AUTO
69!
1216
        && final_merge_mode != SIXEL_FINAL_MERGE_WARD) {
23!
1217
        return reqcolors;
×
1218
    }
1219
    factor = env_final_merge_target_factor;
69✔
1220
    scaled = (unsigned int)((double)reqcolors * factor);
69✔
1221
    if (scaled <= reqcolors) {
69!
1222
        scaled = reqcolors;
×
1223
    }
1224
    if (scaled < 1U) {
69!
1225
        scaled = 1U;
×
1226
    }
1227

1228
    return scaled;
69✔
1229
}
23✔
1230

1231

1232
/* Merge clusters to the requested size using Ward linkage. */
1233
static void
1234
sixel_merge_clusters_ward(unsigned long *weights,
69✔
1235
                          unsigned long *sums,
1236
                          unsigned int depth,
1237
                          int *cluster_count,
1238
                          int target)
1239
{
1240
    int n;
1241
    int desired;
1242
    int best_i;
1243
    int best_j;
1244
    int i;
1245
    int j;
1246
    int k;
1247
    int channel;
1248
    double best_cost;
1249
    double wi;
1250
    double wj;
1251
    double distance_sq;
1252
    double delta;
1253
    double mean_i;
1254
    double mean_j;
1255
    double diff;
1256
    size_t offset_i;
1257
    size_t offset_j;
1258
    size_t dst;
1259
    size_t src;
1260

1261
    if (cluster_count == NULL) {
69!
1262
        return;
×
1263
    }
1264
    n = *cluster_count;
69✔
1265
    desired = target;
69✔
1266
    if (desired < 1) {
69!
1267
        desired = 1;
×
1268
    }
1269
    while (n > desired) {
11,187✔
1270
        best_i = -1;
11,118✔
1271
        best_j = -1;
11,118✔
1272
        best_cost = 1.0e300;
11,118✔
1273
        for (i = 0; i < n; ++i) {
3,948,603✔
1274
            if (weights[i] == 0UL) {
3,937,485!
1275
                continue;
×
1276
            }
1277
            wi = (double)weights[i];
3,937,485✔
1278
            offset_i = (size_t)i * (size_t)depth;
3,937,485✔
1279
            for (j = i + 1; j < n; ++j) {
724,512,615✔
1280
                if (weights[j] == 0UL) {
720,575,130!
1281
                    continue;
×
1282
                }
1283
                wj = (double)weights[j];
720,575,130✔
1284
                offset_j = (size_t)j * (size_t)depth;
720,575,130✔
1285
                distance_sq = 0.0;
720,575,130✔
1286
                for (channel = 0; channel < (int)depth; ++channel) {
2,147,483,647✔
1287
                    mean_i = (double)sums[offset_i + (size_t)channel] / wi;
2,147,483,647✔
1288
                    mean_j = (double)sums[offset_j + (size_t)channel] / wj;
2,147,483,647✔
1289
                    diff = mean_i - mean_j;
2,147,483,647✔
1290
                    distance_sq += diff * diff;
2,147,483,647✔
1291
                }
720,575,130✔
1292
                delta = (wi * wj) / (wi + wj) * distance_sq;
720,575,130✔
1293
                if (delta < best_cost) {
720,575,130✔
1294
                    best_cost = delta;
124,929✔
1295
                    best_i = i;
124,929✔
1296
                    best_j = j;
124,929✔
1297
                }
41,091✔
1298
            }
240,191,710✔
1299
        }
1,312,495✔
1300
        if (best_i < 0 || best_j < 0) {
11,118!
1301
            break;
1302
        }
1303
        weights[best_i] += weights[best_j];
11,118✔
1304
        offset_i = (size_t)best_i * (size_t)depth;
11,118✔
1305
        offset_j = (size_t)best_j * (size_t)depth;
11,118✔
1306
        for (channel = 0; channel < (int)depth; ++channel) {
44,472✔
1307
            sums[offset_i + (size_t)channel] +=
33,354✔
1308
                sums[offset_j + (size_t)channel];
33,354✔
1309
        }
11,118✔
1310
        for (k = best_j; k < n - 1; ++k) {
941,582✔
1311
            weights[k] = weights[k + 1];
930,464✔
1312
            dst = (size_t)k * (size_t)depth;
930,464✔
1313
            src = (size_t)(k + 1) * (size_t)depth;
930,464✔
1314
            for (channel = 0; channel < (int)depth; ++channel) {
3,721,856✔
1315
                sums[dst + (size_t)channel] = sums[src + (size_t)channel];
2,791,392✔
1316
            }
948,216✔
1317
        }
316,072✔
1318
        --n;
11,118✔
1319
    }
1320
    *cluster_count = n;
69✔
1321
}
23✔
1322

1323

1324
/* Translate merged cluster statistics into a tupletable palette. */
1325
static SIXELSTATUS
1326
sixel_quant_clusters_to_colormap(unsigned long *weights,
69✔
1327
                               unsigned long *sums,
1328
                               unsigned int depth,
1329
                               unsigned int cluster_count,
1330
                               int use_reversible,
1331
                               tupletable2 *colormapP,
1332
                               sixel_allocator_t *allocator)
1333
{
1334
    SIXELSTATUS status;
1335
    tupletable2 colormap;
1336
    unsigned int index;
1337
    unsigned int plane;
1338
    double component;
1339
    unsigned long weight;
1340

1341
    status = SIXEL_BAD_ARGUMENT;
69✔
1342
    if (colormapP == NULL) {
69!
1343
        return status;
×
1344
    }
1345
    colormap = newColorMap(cluster_count, depth, allocator);
69✔
1346
    if (colormap.size == 0U) {
69!
1347
        return SIXEL_BAD_ALLOCATION;
×
1348
    }
1349
    for (index = 0U; index < cluster_count; ++index) {
13,833✔
1350
        weight = weights[index];
13,764✔
1351
        if (weight == 0UL) {
13,764!
1352
            weight = 1UL;
×
1353
        }
1354
        colormap.table[index]->value =
13,764✔
1355
            (unsigned int)((weight > (unsigned long)UINT_MAX)
13,764!
1356
                ? UINT_MAX
1357
                : weight);
4,588✔
1358
        for (plane = 0U; plane < depth; ++plane) {
55,056✔
1359
            component = (double)sums[(size_t)index * (size_t)depth + plane];
41,292✔
1360
            component /= (double)weight;
41,292✔
1361
            if (component < 0.0) {
41,292!
1362
                component = 0.0;
×
1363
            }
1364
            if (component > 255.0) {
41,292!
1365
                component = 255.0;
×
1366
            }
1367
            colormap.table[index]->tuple[plane] =
41,292✔
1368
                (sample)(component + 0.5);
41,292✔
1369
        }
13,764✔
1370
        if (use_reversible) {
13,764!
1371
            sixel_quant_reversible_tuple(colormap.table[index]->tuple,
×
1372
                                         depth);
1373
        }
1374
    }
4,588✔
1375
    *colormapP = colormap;
69✔
1376
    status = SIXEL_OK;
69✔
1377

1378
    return status;
69✔
1379
}
23✔
1380

1381

1382
static int histogram_lut_policy = SIXEL_LUT_POLICY_AUTO;
1383
static int quant_method_for_largest = SIXEL_LARGE_NORM;
1384

1385
void
1386
sixel_quant_set_lut_policy(int lut_policy)
545✔
1387
{
1388
    int normalized;
1389

1390
    normalized = SIXEL_LUT_POLICY_AUTO;
545✔
1391
    if (lut_policy == SIXEL_LUT_POLICY_5BIT
778!
1392
        || lut_policy == SIXEL_LUT_POLICY_6BIT
545!
1393
        || lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
545!
1394
        || lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH
545!
1395
        || lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
545!
UNCOV
1396
        normalized = lut_policy;
×
1397
    }
1398

1399
    histogram_lut_policy = normalized;
545✔
1400
}
545✔
1401

1402
void
1403
sixel_quant_set_method_for_largest(int method)
285✔
1404
{
1405
    int normalized;
1406

1407
    normalized = SIXEL_LARGE_NORM;
285✔
1408
    if (method == SIXEL_LARGE_NORM || method == SIXEL_LARGE_LUM) {
285!
1409
        normalized = method;
285✔
1410
    } else if (method == SIXEL_LARGE_AUTO) {
115!
NEW
1411
        normalized = SIXEL_LARGE_NORM;
×
1412
    }
1413

1414
    quant_method_for_largest = normalized;
285✔
1415
}
285✔
1416

1417
struct histogram_control {
1418
    unsigned int channel_shift;
1419
    unsigned int channel_bits;
1420
    unsigned int channel_mask;
1421
    int reversible_rounding;         /* nonzero biases quantization upward */
1422
};
1423

1424
static struct histogram_control
1425
histogram_control_make(unsigned int depth);
1426
static struct histogram_control
1427
histogram_control_make_for_policy(unsigned int depth, int lut_policy);
1428
static size_t histogram_dense_size(unsigned int depth,
1429
                                   struct histogram_control const
1430
                                       *control);
1431
static unsigned int histogram_reconstruct(unsigned int quantized,
1432
                                          struct histogram_control const
1433
                                              *control);
1434

1435
static size_t
1436
histogram_dense_size(unsigned int depth,
527✔
1437
                     struct histogram_control const *control)
1438
{
1439
    size_t size;
1440
    unsigned int exponent;
1441
    unsigned int i;
1442

1443
    size = 1U;
527✔
1444
    exponent = depth * control->channel_bits;
527✔
1445
    for (i = 0U; i < exponent; ++i) {
10,013✔
1446
        if (size > SIZE_MAX / 2U) {
9,486!
1447
            size = SIZE_MAX;
×
1448
            break;
×
1449
        }
1450
        size <<= 1U;
9,486✔
1451
    }
4,086✔
1452

1453
    return size;
527✔
1454
}
1455

1456
static struct histogram_control
1457
histogram_control_make_for_policy(unsigned int depth, int lut_policy)
40,078,485✔
1458
{
1459
    struct histogram_control control;
1460

1461
    control.reversible_rounding = 0;
40,078,485✔
1462
    /*
1463
     * The ASCII ladder below shows how each policy selects bucket width.
1464
     *
1465
     *   auto / 6bit RGB : |--6--|
1466
     *   forced 5bit     : |---5---|
1467
     *   robinhood       : |------8------|
1468
     *   alpha fallback  : |---5---|  (avoids 2^(6*4) buckets)
1469
     */
1470
    control.channel_shift = 2U;
40,078,485✔
1471
    if (depth > 3U) {
40,078,485!
1472
        control.channel_shift = 3U;
×
1473
    }
1474
    if (lut_policy == SIXEL_LUT_POLICY_5BIT) {
40,078,485!
1475
        control.channel_shift = 3U;
×
1476
    } else if (lut_policy == SIXEL_LUT_POLICY_6BIT) {
40,078,485✔
1477
        control.channel_shift = 2U;
267✔
1478
        if (depth > 3U) {
267!
1479
            control.channel_shift = 3U;
×
1480
        }
1481
    } else if (lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
40,078,327!
1482
               || lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
40,078,218!
1483
        control.channel_shift = 0U;
×
1484
    } else if (lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
40,078,218!
NEW
1485
        control.channel_shift = 2U;
×
1486
    }
1487
    control.channel_bits = 8U - control.channel_shift;
40,078,485✔
1488
    control.channel_mask = (1U << control.channel_bits) - 1U;
40,078,485✔
1489

1490
    return control;
40,078,485✔
1491
}
1492

1493
static unsigned int
1494
histogram_reconstruct(unsigned int quantized,
882,339✔
1495
                      struct histogram_control const *control)
1496
{
1497
    unsigned int value;
1498

1499
    value = quantized << control->channel_shift;
882,339✔
1500
    if (quantized == control->channel_mask) {
882,339✔
1501
        value = 255U;
1,203✔
1502
    } else {
517✔
1503
        if (control->channel_shift > 0U) {
881,136!
1504
            value |= (1U << (control->channel_shift - 1U));
881,136✔
1505
        }
294,374✔
1506
    }
1507
    if (value > 255U) {
882,339!
1508
        value = 255U;
×
1509
    }
1510

1511
    return value;
882,339✔
1512
}
1513

1514
static unsigned int
1515
histogram_quantize(unsigned int sample8,
130,951,272✔
1516
                   struct histogram_control const *control)
1517
{
1518
    unsigned int quantized;
1519
    unsigned int shift;
1520
    unsigned int mask;
1521
    unsigned int rounding;
1522

1523
    /*
1524
     * In reversible mode we already rounded once when snapping to
1525
     * sixel_safe_tones[].  If we rounded to the nearest midpoint
1526
     * again, the second pass would fall back to the lower bucket and break
1527
     * the round-trip.  Biasing towards the upper edge keeps the bucket
1528
     * stable after decoding and encoding again.  Non-reversible runs keep
1529
     * symmetric midpoints to avoid nudging colors upwards.
1530
     *
1531
     *        midpoint bias        upper-edge bias
1532
     *   |----*----|----*----|    |----|----*----|
1533
     *   0         8        16    0    8        16
1534
     *
1535
     * The asterisk marks the midpoint captured by a bucket.  Moving that
1536
     * midpoint to the upper edge keeps reversible tones from drifting.
1537
     */
1538
    shift = control->channel_shift;
130,951,272✔
1539
    mask = control->channel_mask;
130,951,272✔
1540
    if (shift == 0U) {
130,951,272!
1541
        quantized = sample8;
×
1542
    } else {
1543
        if (control->reversible_rounding) {
130,951,272!
1544
            rounding = 1U << shift;
×
1545
        } else {
1546
            rounding = 1U << (shift - 1U);
130,951,272✔
1547
        }
1548
        quantized = (sample8 + rounding) >> shift;
130,951,272✔
1549
        if (quantized > mask) {
130,951,272✔
1550
            quantized = mask;
3,133,447✔
1551
        }
1,500,381✔
1552
    }
1553

1554
    return quantized;
130,951,272✔
1555
}
1556

1557
static uint32_t
1558
histogram_pack_color(unsigned char const *data,
43,650,424✔
1559
                     unsigned int const depth,
1560
                     struct histogram_control const *control)
1561
{
1562
    uint32_t packed;
1563
    unsigned int n;
1564
    unsigned int sample8;
1565
    unsigned int bits;
1566

1567
    packed = 0U;
43,650,424✔
1568
    bits = control->channel_bits;
43,650,424✔
1569
    if (control->channel_shift == 0U) {
43,650,424!
1570
        /*
1571
         * The channel shift being zero means each component keeps eight
1572
         * bits.  We therefore pack pixels in RGB order, as illustrated
1573
         * below:
1574
         *
1575
         *      R   G   B
1576
         *     [ ]-[ ]-[ ]
1577
         *      |   |   |
1578
         *      v   v   v
1579
         *     0xRRGGBB
1580
         */
1581
        for (n = 0U; n < depth; ++n) {
×
1582
            packed |= (uint32_t)data[depth - 1U - n] << (n * bits);
×
1583
        }
1584
        return packed;
×
1585
    }
1586

1587
    for (n = 0U; n < depth; ++n) {
174,601,696✔
1588
        sample8 = (unsigned int)data[depth - 1U - n];
130,951,272✔
1589
        packed |= histogram_quantize(sample8, control) << (n * bits);
130,951,272✔
1590
    }
62,349,072✔
1591

1592
    return packed;
43,650,424✔
1593
}
20,783,024✔
1594

1595
static uint32_t
1596
histogram_hash_mix(uint32_t key)
40,077,958✔
1597
{
1598
    uint32_t hash;
1599

1600
    /*
1601
     * Multiplicative mixing with two avalanching rounds keeps nearby
1602
     * colors far apart in the hash domain.  The final tweak avoids the
1603
     * 0xffffffff sentinel used by hopscotch slots.
1604
     */
1605
    hash = key * 0x9e3779b9U;
40,077,958✔
1606
    hash ^= hash >> 16;
40,077,958✔
1607
    hash *= 0x7feb352dU;
40,077,958✔
1608
    hash ^= hash >> 15;
40,077,958✔
1609
    hash *= 0x846ca68bU;
40,077,958✔
1610
    hash ^= hash >> 16;
40,077,958✔
1611
    if (hash == 0xffffffffU) {
40,077,958!
1612
        hash ^= 0x632be59bU;
×
1613
    }
1614

1615
    return hash;
40,077,958✔
1616
}
1617

1618
static unsigned int
1619
computeHash(unsigned char const *data,
40,077,958✔
1620
            unsigned int const depth,
1621
            struct histogram_control const *control)
1622
{
1623
    uint32_t packed;
1624

1625
    packed = histogram_pack_color(data, depth, control);
40,077,958✔
1626

1627
    return histogram_hash_mix(packed);
40,077,958✔
1628
}
1629

1630
#define CUCKOO_BUCKET_SIZE 4U
1631
#define CUCKOO_MAX_KICKS 128U
1632
#define CUCKOO_STASH_SIZE 32U
1633
#define CUCKOO_EMPTY_KEY 0xffffffffU
1634

1635
struct cuckoo_bucket32 {
1636
    uint32_t key[CUCKOO_BUCKET_SIZE];
1637
    uint32_t value[CUCKOO_BUCKET_SIZE];
1638
};
1639

1640
struct cuckoo_table32 {
1641
    struct cuckoo_bucket32 *buckets;
1642
    uint32_t stash_key[CUCKOO_STASH_SIZE];
1643
    uint32_t stash_value[CUCKOO_STASH_SIZE];
1644
    size_t bucket_count;
1645
    size_t bucket_mask;
1646
    size_t stash_count;
1647
    size_t entry_count;
1648
    sixel_allocator_t *allocator;
1649
};
1650

1651
static size_t cuckoo_round_buckets(size_t hint);
1652
static size_t cuckoo_hash_primary(uint32_t key, size_t mask);
1653
static size_t cuckoo_hash_secondary(uint32_t key, size_t mask);
1654
static size_t cuckoo_hash_alternate(uint32_t key,
1655
                                    size_t bucket,
1656
                                    size_t mask);
1657
static uint32_t *cuckoo_bucket_find(struct cuckoo_bucket32 *bucket,
1658
                                    uint32_t key);
1659
static int cuckoo_bucket_insert_direct(struct cuckoo_bucket32 *bucket,
1660
                                       uint32_t key,
1661
                                       uint32_t value);
1662
static SIXELSTATUS cuckoo_table32_init(struct cuckoo_table32 *table,
1663
                                       size_t expected,
1664
                                       sixel_allocator_t *allocator);
1665
static void cuckoo_table32_clear(struct cuckoo_table32 *table);
1666
static void cuckoo_table32_fini(struct cuckoo_table32 *table);
1667
static uint32_t *cuckoo_table32_lookup(struct cuckoo_table32 *table,
1668
                                       uint32_t key);
1669
static SIXELSTATUS cuckoo_table32_insert(struct cuckoo_table32 *table,
1670
                                         uint32_t key,
1671
                                         uint32_t value);
1672
static SIXELSTATUS cuckoo_table32_grow(struct cuckoo_table32 *table);
1673

1674
struct robinhood_slot32 {
1675
    uint32_t key;
1676
    uint32_t color;
1677
    uint32_t value;
1678
    uint16_t distance;
1679
    uint16_t pad;
1680
};
1681

1682
struct robinhood_table32 {
1683
    struct robinhood_slot32 *slots;
1684
    size_t capacity;
1685
    size_t count;
1686
    sixel_allocator_t *allocator;
1687
};
1688

1689
static size_t robinhood_round_capacity(size_t hint);
1690
static SIXELSTATUS robinhood_table32_init(struct robinhood_table32 *table,
1691
                                         size_t expected,
1692
                                         sixel_allocator_t *allocator);
1693
static void robinhood_table32_fini(struct robinhood_table32 *table);
1694
static struct robinhood_slot32 *
1695
robinhood_table32_lookup(struct robinhood_table32 *table,
1696
                         uint32_t key,
1697
                         uint32_t color);
1698
static SIXELSTATUS robinhood_table32_insert(struct robinhood_table32 *table,
1699
                                           uint32_t key,
1700
                                           uint32_t color,
1701
                                           uint32_t value);
1702
static SIXELSTATUS robinhood_table32_grow(struct robinhood_table32 *table);
1703
static struct robinhood_slot32 *
1704
robinhood_table32_place(struct robinhood_table32 *table,
1705
                        struct robinhood_slot32 entry);
1706

1707
#define HOPSCOTCH_EMPTY_KEY 0xffffffffU
1708
#define HOPSCOTCH_DEFAULT_NEIGHBORHOOD 32U
1709
#define HOPSCOTCH_INSERT_RANGE 256U
1710

1711
struct hopscotch_slot32 {
1712
    uint32_t key;
1713
    uint32_t color;
1714
    uint32_t value;
1715
};
1716

1717
struct hopscotch_table32 {
1718
    struct hopscotch_slot32 *slots;
1719
    uint32_t *hopinfo;
1720
    size_t capacity;
1721
    size_t count;
1722
    size_t neighborhood;
1723
    sixel_allocator_t *allocator;
1724
};
1725

1726
static SIXELSTATUS hopscotch_table32_init(struct hopscotch_table32 *table,
1727
                                          size_t expected,
1728
                                          sixel_allocator_t *allocator);
1729
static void hopscotch_table32_fini(struct hopscotch_table32 *table);
1730
static struct hopscotch_slot32 *
1731
hopscotch_table32_lookup(struct hopscotch_table32 *table,
1732
                         uint32_t key,
1733
                         uint32_t color);
1734
static SIXELSTATUS hopscotch_table32_insert(struct hopscotch_table32 *table,
1735
                                            uint32_t key,
1736
                                            uint32_t color,
1737
                                            uint32_t value);
1738
static SIXELSTATUS hopscotch_table32_grow(struct hopscotch_table32 *table);
1739

1740
static struct histogram_control
1741
histogram_control_make(unsigned int depth)
40,078,218✔
1742
{
1743
    struct histogram_control control;
1744

1745
    control = histogram_control_make_for_policy(depth,
59,084,154✔
1746
                                                histogram_lut_policy);
19,005,936✔
1747

1748
    return control;
40,078,218✔
1749
}
1750

1751
static int
NEW
1752
sixel_certlut_init(sixel_certlut_t *lut)
×
1753
{
1754
    int status;
1755

NEW
1756
    status = SIXEL_FALSE;
×
NEW
1757
    if (lut == NULL) {
×
NEW
1758
        goto end;
×
1759
    }
1760

NEW
1761
    lut->level0 = NULL;
×
NEW
1762
    lut->pool = NULL;
×
NEW
1763
    lut->pool_size = 0U;
×
NEW
1764
    lut->pool_capacity = 0U;
×
NEW
1765
    lut->wR = 1;
×
NEW
1766
    lut->wG = 1;
×
NEW
1767
    lut->wB = 1;
×
NEW
1768
    lut->wR2 = 1U;
×
NEW
1769
    lut->wG2 = 1U;
×
NEW
1770
    lut->wB2 = 1U;
×
NEW
1771
    memset(lut->wr_scale, 0, sizeof(lut->wr_scale));
×
NEW
1772
    memset(lut->wg_scale, 0, sizeof(lut->wg_scale));
×
NEW
1773
    memset(lut->wb_scale, 0, sizeof(lut->wb_scale));
×
NEW
1774
    lut->wr_palette = NULL;
×
NEW
1775
    lut->wg_palette = NULL;
×
NEW
1776
    lut->wb_palette = NULL;
×
NEW
1777
    lut->palette = NULL;
×
NEW
1778
    lut->ncolors = 0;
×
NEW
1779
    lut->kdnodes = NULL;
×
NEW
1780
    lut->kdnodes_count = 0;
×
NEW
1781
    lut->kdtree_root = -1;
×
NEW
1782
    status = SIXEL_OK;
×
1783

1784
end:
NEW
1785
    return status;
×
1786
}
1787

1788
static void
NEW
1789
sixel_certlut_release(sixel_certlut_t *lut)
×
1790
{
NEW
1791
    if (lut == NULL) {
×
NEW
1792
        return;
×
1793
    }
NEW
1794
    free(lut->level0);
×
NEW
1795
    free(lut->pool);
×
NEW
1796
    free(lut->wr_palette);
×
NEW
1797
    free(lut->wg_palette);
×
NEW
1798
    free(lut->wb_palette);
×
NEW
1799
    free(lut->kdnodes);
×
NEW
1800
    lut->level0 = NULL;
×
NEW
1801
    lut->pool = NULL;
×
NEW
1802
    lut->pool_size = 0U;
×
NEW
1803
    lut->pool_capacity = 0U;
×
NEW
1804
    lut->wr_palette = NULL;
×
NEW
1805
    lut->wg_palette = NULL;
×
NEW
1806
    lut->wb_palette = NULL;
×
NEW
1807
    lut->kdnodes = NULL;
×
NEW
1808
    lut->kdnodes_count = 0;
×
NEW
1809
    lut->kdtree_root = -1;
×
1810
}
1811

1812
static int
NEW
1813
sixel_certlut_prepare_palette_terms(sixel_certlut_t *lut)
×
1814
{
1815
    int status;
1816
    size_t count;
1817
    int index;
1818
    int32_t *wr_terms;
1819
    int32_t *wg_terms;
1820
    int32_t *wb_terms;
1821

NEW
1822
    status = SIXEL_FALSE;
×
NEW
1823
    wr_terms = NULL;
×
NEW
1824
    wg_terms = NULL;
×
NEW
1825
    wb_terms = NULL;
×
NEW
1826
    if (lut == NULL) {
×
NEW
1827
        goto end;
×
1828
    }
NEW
1829
    if (lut->ncolors <= 0) {
×
NEW
1830
        status = SIXEL_OK;
×
NEW
1831
        goto end;
×
1832
    }
NEW
1833
    count = (size_t)lut->ncolors;
×
NEW
1834
    wr_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
NEW
1835
    if (wr_terms == NULL) {
×
NEW
1836
        goto end;
×
1837
    }
NEW
1838
    wg_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
NEW
1839
    if (wg_terms == NULL) {
×
NEW
1840
        goto end;
×
1841
    }
NEW
1842
    wb_terms = (int32_t *)malloc(count * sizeof(int32_t));
×
NEW
1843
    if (wb_terms == NULL) {
×
NEW
1844
        goto end;
×
1845
    }
NEW
1846
    for (index = 0; index < lut->ncolors; ++index) {
×
NEW
1847
        wr_terms[index] = lut->wR * (int)lut->palette[index].r;
×
NEW
1848
        wg_terms[index] = lut->wG * (int)lut->palette[index].g;
×
NEW
1849
        wb_terms[index] = lut->wB * (int)lut->palette[index].b;
×
1850
    }
NEW
1851
    free(lut->wr_palette);
×
NEW
1852
    free(lut->wg_palette);
×
NEW
1853
    free(lut->wb_palette);
×
NEW
1854
    lut->wr_palette = wr_terms;
×
NEW
1855
    lut->wg_palette = wg_terms;
×
NEW
1856
    lut->wb_palette = wb_terms;
×
NEW
1857
    wr_terms = NULL;
×
NEW
1858
    wg_terms = NULL;
×
NEW
1859
    wb_terms = NULL;
×
NEW
1860
    status = SIXEL_OK;
×
1861

1862
end:
NEW
1863
    free(wr_terms);
×
NEW
1864
    free(wg_terms);
×
NEW
1865
    free(wb_terms);
×
NEW
1866
    return status;
×
1867
}
1868

1869
static void
NEW
1870
sixel_quant_cell_center(int rmin, int gmin, int bmin, int size,
×
1871
                        int *cr, int *cg, int *cb)
1872
{
1873
    int half;
1874

NEW
1875
    half = size / 2;
×
NEW
1876
    *cr = rmin + half;
×
NEW
1877
    *cg = gmin + half;
×
NEW
1878
    *cb = bmin + half;
×
NEW
1879
    if (size == 1) {
×
NEW
1880
        *cr = rmin;
×
NEW
1881
        *cg = gmin;
×
NEW
1882
        *cb = bmin;
×
1883
    }
NEW
1884
}
×
1885

1886
static void
NEW
1887
sixel_quant_weight_init(sixel_certlut_t *lut, int wR, int wG, int wB)
×
1888
{
1889
    int i;
1890

NEW
1891
    lut->wR = wR;
×
NEW
1892
    lut->wG = wG;
×
NEW
1893
    lut->wB = wB;
×
NEW
1894
    lut->wR2 = (uint64_t)wR * (uint64_t)wR;
×
NEW
1895
    lut->wG2 = (uint64_t)wG * (uint64_t)wG;
×
NEW
1896
    lut->wB2 = (uint64_t)wB * (uint64_t)wB;
×
NEW
1897
    for (i = 0; i < 256; ++i) {
×
NEW
1898
        lut->wr_scale[i] = wR * i;
×
NEW
1899
        lut->wg_scale[i] = wG * i;
×
NEW
1900
        lut->wb_scale[i] = wB * i;
×
1901
    }
NEW
1902
}
×
1903

1904
static uint64_t
NEW
1905
sixel_certlut_distance_precomputed(sixel_certlut_t const *lut,
×
1906
                                   int index,
1907
                                   int32_t wr_r,
1908
                                   int32_t wg_g,
1909
                                   int32_t wb_b)
1910
{
1911
    uint64_t distance;
1912
    int64_t diff;
1913

NEW
1914
    diff = (int64_t)wr_r - (int64_t)lut->wr_palette[index];
×
NEW
1915
    distance = (uint64_t)(diff * diff);
×
NEW
1916
    diff = (int64_t)wg_g - (int64_t)lut->wg_palette[index];
×
NEW
1917
    distance += (uint64_t)(diff * diff);
×
NEW
1918
    diff = (int64_t)wb_b - (int64_t)lut->wb_palette[index];
×
NEW
1919
    distance += (uint64_t)(diff * diff);
×
1920

NEW
1921
    return distance;
×
1922
}
1923

1924
static void
NEW
1925
sixel_quant_distance_pair(sixel_certlut_t const *lut, int r, int g, int b,
×
1926
                          int *best_idx, int *second_idx,
1927
                          uint64_t *best_dist, uint64_t *second_dist)
1928
{
1929
    int i;
1930
    int best_candidate;
1931
    int second_candidate;
1932
    uint64_t best_value;
1933
    uint64_t second_value;
1934
    uint64_t distance;
1935
    int rr;
1936
    int gg;
1937
    int bb;
1938
    int32_t wr_r;
1939
    int32_t wg_g;
1940
    int32_t wb_b;
1941

NEW
1942
    best_candidate = (-1);
×
NEW
1943
    second_candidate = (-1);
×
NEW
1944
    best_value = UINT64_MAX;
×
NEW
1945
    second_value = UINT64_MAX;
×
NEW
1946
    rr = r;
×
NEW
1947
    gg = g;
×
NEW
1948
    bb = b;
×
NEW
1949
    if (rr < 0) {
×
NEW
1950
        rr = 0;
×
NEW
1951
    } else if (rr > 255) {
×
NEW
1952
        rr = 255;
×
1953
    }
NEW
1954
    if (gg < 0) {
×
NEW
1955
        gg = 0;
×
NEW
1956
    } else if (gg > 255) {
×
NEW
1957
        gg = 255;
×
1958
    }
NEW
1959
    if (bb < 0) {
×
NEW
1960
        bb = 0;
×
NEW
1961
    } else if (bb > 255) {
×
NEW
1962
        bb = 255;
×
1963
    }
NEW
1964
    wr_r = lut->wr_scale[rr];
×
NEW
1965
    wg_g = lut->wg_scale[gg];
×
NEW
1966
    wb_b = lut->wb_scale[bb];
×
NEW
1967
    if (lut->kdnodes != NULL && lut->kdtree_root >= 0) {
×
NEW
1968
        sixel_certlut_kdtree_search(lut,
×
NEW
1969
                                    lut->kdtree_root,
×
1970
                                    r,
1971
                                    g,
1972
                                    b,
1973
                                    wr_r,
1974
                                    wg_g,
1975
                                    wb_b,
1976
                                    &best_candidate,
1977
                                    &best_value,
1978
                                    &second_candidate,
1979
                                    &second_value);
1980
    } else {
NEW
1981
        for (i = 0; i < lut->ncolors; ++i) {
×
NEW
1982
            distance = sixel_certlut_distance_precomputed(lut,
×
1983
                                                          i,
1984
                                                          wr_r,
1985
                                                          wg_g,
1986
                                                          wb_b);
NEW
1987
            if (distance < best_value) {
×
NEW
1988
                second_value = best_value;
×
NEW
1989
                second_candidate = best_candidate;
×
NEW
1990
                best_value = distance;
×
NEW
1991
                best_candidate = i;
×
NEW
1992
            } else if (distance < second_value) {
×
NEW
1993
                second_value = distance;
×
NEW
1994
                second_candidate = i;
×
1995
            }
1996
        }
1997
    }
NEW
1998
    if (second_candidate < 0) {
×
NEW
1999
        second_candidate = best_candidate;
×
NEW
2000
        second_value = best_value;
×
2001
    }
NEW
2002
    *best_idx = best_candidate;
×
NEW
2003
    *second_idx = second_candidate;
×
NEW
2004
    *best_dist = best_value;
×
NEW
2005
    *second_dist = second_value;
×
NEW
2006
}
×
2007

2008
static int
NEW
2009
sixel_quant_is_cell_safe(sixel_certlut_t const *lut, int best_idx,
×
2010
                         int second_idx, int size, uint64_t best_dist,
2011
                         uint64_t second_dist)
2012
{
2013
    uint64_t delta_sq;
2014
    uint64_t rhs;
2015
    uint64_t weight_term;
2016
    int64_t wr_delta;
2017
    int64_t wg_delta;
2018
    int64_t wb_delta;
2019

NEW
2020
    if (best_idx < 0 || second_idx < 0) {
×
NEW
2021
        return 1;
×
2022
    }
2023

2024
    /*
2025
     * The certification bound compares the squared distance gap against the
2026
     * palette separation scaled by the cube diameter.  If the gap wins the
2027
     * entire cube maps to the current best palette entry.
2028
     */
NEW
2029
    delta_sq = second_dist - best_dist;
×
NEW
2030
    wr_delta = (int64_t)lut->wr_palette[second_idx]
×
NEW
2031
        - (int64_t)lut->wr_palette[best_idx];
×
NEW
2032
    wg_delta = (int64_t)lut->wg_palette[second_idx]
×
NEW
2033
        - (int64_t)lut->wg_palette[best_idx];
×
NEW
2034
    wb_delta = (int64_t)lut->wb_palette[second_idx]
×
NEW
2035
        - (int64_t)lut->wb_palette[best_idx];
×
NEW
2036
    weight_term = (uint64_t)(wr_delta * wr_delta);
×
NEW
2037
    weight_term += (uint64_t)(wg_delta * wg_delta);
×
NEW
2038
    weight_term += (uint64_t)(wb_delta * wb_delta);
×
NEW
2039
    rhs = (uint64_t)3 * (uint64_t)size * (uint64_t)size * weight_term;
×
2040

NEW
2041
    return delta_sq * delta_sq > rhs;
×
2042
}
2043

2044
static uint32_t
NEW
2045
sixel_quant_pool_alloc(sixel_certlut_t *lut, int *status)
×
2046
{
2047
    uint32_t required;
2048
    uint32_t next_capacity;
2049
    uint32_t offset;
2050
    uint8_t *resized;
2051

NEW
2052
    offset = 0U;
×
NEW
2053
    if (status != NULL) {
×
NEW
2054
        *status = SIXEL_FALSE;
×
2055
    }
NEW
2056
    required = lut->pool_size + (uint32_t)(8 * sizeof(uint32_t));
×
NEW
2057
    if (required > lut->pool_capacity) {
×
NEW
2058
        next_capacity = lut->pool_capacity;
×
NEW
2059
        if (next_capacity == 0U) {
×
NEW
2060
            next_capacity = (uint32_t)(8 * sizeof(uint32_t));
×
2061
        }
NEW
2062
        while (next_capacity < required) {
×
NEW
2063
            if (next_capacity > UINT32_MAX / 2U) {
×
NEW
2064
                return 0U;
×
2065
            }
NEW
2066
            next_capacity *= 2U;
×
2067
        }
NEW
2068
        resized = (uint8_t *)realloc(lut->pool, next_capacity);
×
NEW
2069
        if (resized == NULL) {
×
NEW
2070
            return 0U;
×
2071
        }
NEW
2072
        lut->pool = resized;
×
NEW
2073
        lut->pool_capacity = next_capacity;
×
2074
    }
NEW
2075
    offset = lut->pool_size;
×
NEW
2076
    memset(lut->pool + offset, 0, 8 * sizeof(uint32_t));
×
NEW
2077
    lut->pool_size = required;
×
NEW
2078
    if (status != NULL) {
×
NEW
2079
        *status = SIXEL_OK;
×
2080
    }
2081

NEW
2082
    return offset;
×
2083
}
2084

2085
static void
NEW
2086
sixel_quant_assign_leaf(uint32_t *cell, int palette_index)
×
2087
{
NEW
2088
    *cell = 0x80000000U | (uint32_t)(palette_index & 0xff);
×
NEW
2089
}
×
2090

2091
static void
NEW
2092
sixel_quant_assign_branch(uint32_t *cell, uint32_t offset)
×
2093
{
NEW
2094
    *cell = SIXEL_CERTLUT_BRANCH_FLAG | (offset & 0x3fffffffU);
×
NEW
2095
}
×
2096

2097
static int
NEW
2098
sixel_certlut_palette_component(sixel_certlut_t const *lut,
×
2099
                                int index, int axis)
2100
{
2101
    sixel_color_t const *color;
2102

NEW
2103
    color = &lut->palette[index];
×
NEW
2104
    if (axis == 0) {
×
NEW
2105
        return (int)color->r;
×
2106
    }
NEW
2107
    if (axis == 1) {
×
NEW
2108
        return (int)color->g;
×
2109
    }
NEW
2110
    return (int)color->b;
×
2111
}
2112

2113
static void
NEW
2114
sixel_certlut_sort_indices(sixel_certlut_t const *lut,
×
2115
                           int *indices, int count, int axis)
2116
{
2117
    int i;
2118
    int j;
2119
    int key;
2120
    int key_value;
2121
    int current_value;
2122

NEW
2123
    for (i = 1; i < count; ++i) {
×
NEW
2124
        key = indices[i];
×
NEW
2125
        key_value = sixel_certlut_palette_component(lut, key, axis);
×
NEW
2126
        j = i - 1;
×
NEW
2127
        while (j >= 0) {
×
NEW
2128
            current_value = sixel_certlut_palette_component(lut,
×
NEW
2129
                                                            indices[j],
×
2130
                                                            axis);
NEW
2131
            if (current_value <= key_value) {
×
NEW
2132
                break;
×
2133
            }
NEW
2134
            indices[j + 1] = indices[j];
×
NEW
2135
            --j;
×
2136
        }
NEW
2137
        indices[j + 1] = key;
×
2138
    }
NEW
2139
}
×
2140

2141
static int
NEW
2142
sixel_certlut_kdtree_build_recursive(sixel_certlut_t *lut,
×
2143
                                     int *indices,
2144
                                     int count,
2145
                                     int depth)
2146
{
2147
    int axis;
2148
    int median;
2149
    int node_index;
2150

NEW
2151
    if (count <= 0) {
×
NEW
2152
        return -1;
×
2153
    }
2154

NEW
2155
    axis = depth % 3;
×
NEW
2156
    sixel_certlut_sort_indices(lut, indices, count, axis);
×
NEW
2157
    median = count / 2;
×
NEW
2158
    node_index = lut->kdnodes_count;
×
NEW
2159
    if (node_index >= lut->ncolors) {
×
NEW
2160
        return -1;
×
2161
    }
NEW
2162
    lut->kdnodes_count++;
×
NEW
2163
    lut->kdnodes[node_index].index = indices[median];
×
NEW
2164
    lut->kdnodes[node_index].axis = (unsigned char)axis;
×
NEW
2165
    lut->kdnodes[node_index].left =
×
NEW
2166
        sixel_certlut_kdtree_build_recursive(lut,
×
2167
                                             indices,
2168
                                             median,
2169
                                             depth + 1);
NEW
2170
    lut->kdnodes[node_index].right =
×
NEW
2171
        sixel_certlut_kdtree_build_recursive(lut,
×
NEW
2172
                                             indices + median + 1,
×
NEW
2173
                                             count - median - 1,
×
2174
                                             depth + 1);
2175

NEW
2176
    return node_index;
×
2177
}
2178

2179
static SIXELSTATUS
NEW
2180
sixel_certlut_kdtree_build(sixel_certlut_t *lut)
×
2181
{
2182
    SIXELSTATUS status;
2183
    int *indices;
2184
    int i;
2185

NEW
2186
    status = SIXEL_FALSE;
×
NEW
2187
    indices = NULL;
×
NEW
2188
    lut->kdnodes = NULL;
×
NEW
2189
    lut->kdnodes_count = 0;
×
NEW
2190
    lut->kdtree_root = -1;
×
NEW
2191
    if (lut->ncolors <= 0) {
×
NEW
2192
        status = SIXEL_OK;
×
NEW
2193
        goto end;
×
2194
    }
NEW
2195
    lut->kdnodes = (sixel_certlut_node_t *)
×
NEW
2196
        calloc((size_t)lut->ncolors, sizeof(sixel_certlut_node_t));
×
NEW
2197
    if (lut->kdnodes == NULL) {
×
NEW
2198
        goto end;
×
2199
    }
NEW
2200
    indices = (int *)malloc((size_t)lut->ncolors * sizeof(int));
×
NEW
2201
    if (indices == NULL) {
×
NEW
2202
        goto end;
×
2203
    }
NEW
2204
    for (i = 0; i < lut->ncolors; ++i) {
×
NEW
2205
        indices[i] = i;
×
2206
    }
NEW
2207
    lut->kdnodes_count = 0;
×
NEW
2208
    lut->kdtree_root = sixel_certlut_kdtree_build_recursive(lut,
×
2209
                                                            indices,
2210
                                                            lut->ncolors,
2211
                                                            0);
NEW
2212
    if (lut->kdtree_root < 0) {
×
NEW
2213
        goto end;
×
2214
    }
NEW
2215
    status = SIXEL_OK;
×
2216

2217
end:
NEW
2218
    free(indices);
×
NEW
2219
    if (SIXEL_FAILED(status)) {
×
NEW
2220
        free(lut->kdnodes);
×
NEW
2221
        lut->kdnodes = NULL;
×
NEW
2222
        lut->kdnodes_count = 0;
×
NEW
2223
        lut->kdtree_root = -1;
×
2224
    }
2225

NEW
2226
    return status;
×
2227
}
2228

2229
static uint64_t
NEW
2230
sixel_certlut_axis_distance(sixel_certlut_t const *lut, int diff, int axis)
×
2231
{
2232
    uint64_t weight;
2233
    uint64_t abs_diff;
2234

NEW
2235
    abs_diff = (uint64_t)(diff < 0 ? -diff : diff);
×
NEW
2236
    if (axis == 0) {
×
NEW
2237
        weight = lut->wR2;
×
NEW
2238
    } else if (axis == 1) {
×
NEW
2239
        weight = lut->wG2;
×
2240
    } else {
NEW
2241
        weight = lut->wB2;
×
2242
    }
2243

NEW
2244
    return weight * abs_diff * abs_diff;
×
2245
}
2246

2247
static void
NEW
2248
sixel_certlut_consider_candidate(sixel_certlut_t const *lut,
×
2249
                                 int candidate,
2250
                                 int32_t wr_r,
2251
                                 int32_t wg_g,
2252
                                 int32_t wb_b,
2253
                                 int *best_idx,
2254
                                 uint64_t *best_dist,
2255
                                 int *second_idx,
2256
                                 uint64_t *second_dist)
2257
{
2258
    uint64_t distance;
2259

NEW
2260
    distance = sixel_certlut_distance_precomputed(lut,
×
2261
                                                  candidate,
2262
                                                  wr_r,
2263
                                                  wg_g,
2264
                                                  wb_b);
NEW
2265
    if (distance < *best_dist) {
×
NEW
2266
        *second_dist = *best_dist;
×
NEW
2267
        *second_idx = *best_idx;
×
NEW
2268
        *best_dist = distance;
×
NEW
2269
        *best_idx = candidate;
×
NEW
2270
    } else if (distance < *second_dist) {
×
NEW
2271
        *second_dist = distance;
×
NEW
2272
        *second_idx = candidate;
×
2273
    }
NEW
2274
}
×
2275

2276
static void
NEW
2277
sixel_certlut_kdtree_search(sixel_certlut_t const *lut,
×
2278
                            int node_index,
2279
                            int r,
2280
                            int g,
2281
                            int b,
2282
                            int32_t wr_r,
2283
                            int32_t wg_g,
2284
                            int32_t wb_b,
2285
                            int *best_idx,
2286
                            uint64_t *best_dist,
2287
                            int *second_idx,
2288
                            uint64_t *second_dist)
2289
{
2290
    sixel_certlut_node_t const *node;
2291
    int axis;
2292
    int value;
2293
    int diff;
2294
    int near_child;
2295
    int far_child;
2296
    uint64_t axis_bound;
2297
    int component;
2298

NEW
2299
    if (node_index < 0) {
×
NEW
2300
        return;
×
2301
    }
NEW
2302
    node = &lut->kdnodes[node_index];
×
NEW
2303
    sixel_certlut_consider_candidate(lut,
×
NEW
2304
                                     node->index,
×
2305
                                     wr_r,
2306
                                     wg_g,
2307
                                     wb_b,
2308
                                     best_idx,
2309
                                     best_dist,
2310
                                     second_idx,
2311
                                     second_dist);
2312

NEW
2313
    axis = (int)node->axis;
×
NEW
2314
    value = sixel_certlut_palette_component(lut, node->index, axis);
×
NEW
2315
    if (axis == 0) {
×
NEW
2316
        component = r;
×
NEW
2317
    } else if (axis == 1) {
×
NEW
2318
        component = g;
×
2319
    } else {
NEW
2320
        component = b;
×
2321
    }
NEW
2322
    diff = component - value;
×
NEW
2323
    if (diff <= 0) {
×
NEW
2324
        near_child = node->left;
×
NEW
2325
        far_child = node->right;
×
2326
    } else {
NEW
2327
        near_child = node->right;
×
NEW
2328
        far_child = node->left;
×
2329
    }
NEW
2330
    if (near_child >= 0) {
×
NEW
2331
        sixel_certlut_kdtree_search(lut,
×
2332
                                    near_child,
2333
                                    r,
2334
                                    g,
2335
                                    b,
2336
                                    wr_r,
2337
                                    wg_g,
2338
                                    wb_b,
2339
                                    best_idx,
2340
                                    best_dist,
2341
                                    second_idx,
2342
                                    second_dist);
2343
    }
NEW
2344
    axis_bound = sixel_certlut_axis_distance(lut, diff, axis);
×
NEW
2345
    if (far_child >= 0 && axis_bound <= *second_dist) {
×
NEW
2346
        sixel_certlut_kdtree_search(lut,
×
2347
                                    far_child,
2348
                                    r,
2349
                                    g,
2350
                                    b,
2351
                                    wr_r,
2352
                                    wg_g,
2353
                                    wb_b,
2354
                                    best_idx,
2355
                                    best_dist,
2356
                                    second_idx,
2357
                                    second_dist);
2358
    }
2359
}
2360

2361
static uint8_t
NEW
2362
sixel_certlut_fallback(sixel_certlut_t const *lut, int r, int g, int b)
×
2363
{
2364
    int best_idx;
2365
    int second_idx;
2366
    uint64_t best_dist;
2367
    uint64_t second_dist;
2368

NEW
2369
    best_idx = -1;
×
NEW
2370
    second_idx = -1;
×
NEW
2371
    best_dist = 0U;
×
NEW
2372
    second_dist = 0U;
×
NEW
2373
    if (lut == NULL) {
×
NEW
2374
        return 0U;
×
2375
    }
2376
    /*
2377
     * The lazy builder may fail when allocations run out.  Fall back to a
2378
     * direct brute-force palette search so lookups still succeed even in low
2379
     * memory conditions.
2380
     */
NEW
2381
    sixel_quant_distance_pair(lut,
×
2382
                              r,
2383
                              g,
2384
                              b,
2385
                              &best_idx,
2386
                              &second_idx,
2387
                              &best_dist,
2388
                              &second_dist);
NEW
2389
    if (best_idx < 0) {
×
NEW
2390
        return 0U;
×
2391
    }
2392

NEW
2393
    return (uint8_t)best_idx;
×
2394
}
2395

2396
static int
NEW
2397
sixel_certlut_build_cell(sixel_certlut_t *lut, uint32_t *cell,
×
2398
                         int rmin, int gmin, int bmin, int size)
2399
{
2400
    SIXELSTATUS status;
2401
    int cr;
2402
    int cg;
2403
    int cb;
2404
    int best_idx;
2405
    int second_idx;
2406
    uint64_t best_dist;
2407
    uint64_t second_dist;
2408
    uint32_t offset;
2409
    int branch_status;
2410
    uint8_t *pool_before;
2411
    size_t pool_size_before;
2412
    uint32_t cell_offset;
2413
    int cell_in_pool;
2414

NEW
2415
    if (cell == NULL || lut == NULL) {
×
NEW
2416
        status = SIXEL_BAD_ARGUMENT;
×
NEW
2417
        goto end;
×
2418
    }
NEW
2419
    if (*cell == 0U) {
×
2420
#ifdef DEBUG_CERTLUT_TRACE
2421
        fprintf(stderr,
2422
                "build_cell rmin=%d gmin=%d bmin=%d size=%d\n",
2423
                rmin,
2424
                gmin,
2425
                bmin,
2426
                size);
2427
#endif
2428
    }
NEW
2429
    if (*cell != 0U) {
×
NEW
2430
        status = SIXEL_OK;
×
NEW
2431
        goto end;
×
2432
    }
2433

2434
    /*
2435
     * Each node represents an axis-aligned cube in RGB space.  The builder
2436
     * certifies the dominant palette index by checking the distance gap at
2437
     * the cell center.  When certification fails the cube is split into eight
2438
     * octants backed by a pool block.  Children remain unbuilt until lookups
2439
     * descend into them, keeping the workload proportional to actual queries.
2440
     */
NEW
2441
    status = SIXEL_FALSE;
×
NEW
2442
    sixel_quant_cell_center(rmin, gmin, bmin, size, &cr, &cg, &cb);
×
NEW
2443
    sixel_quant_distance_pair(lut, cr, cg, cb, &best_idx, &second_idx,
×
2444
                              &best_dist, &second_dist);
NEW
2445
    if (best_idx < 0) {
×
NEW
2446
        best_idx = 0;
×
2447
    }
NEW
2448
    if (size == 1) {
×
NEW
2449
        sixel_quant_assign_leaf(cell, best_idx);
×
2450
#ifdef DEBUG_CERTLUT_TRACE
2451
        fprintf(stderr,
2452
                "  leaf idx=%d\n",
2453
                best_idx);
2454
#endif
NEW
2455
        status = SIXEL_OK;
×
NEW
2456
        goto end;
×
2457
    }
NEW
2458
    if (sixel_quant_is_cell_safe(lut, best_idx, second_idx, size,
×
2459
                                 best_dist, second_dist)) {
NEW
2460
        sixel_quant_assign_leaf(cell, best_idx);
×
2461
#ifdef DEBUG_CERTLUT_TRACE
2462
        fprintf(stderr,
2463
                "  safe leaf idx=%d\n",
2464
                best_idx);
2465
#endif
NEW
2466
        status = SIXEL_OK;
×
NEW
2467
        goto end;
×
2468
    }
NEW
2469
    pool_before = lut->pool;
×
NEW
2470
    pool_size_before = lut->pool_size;
×
NEW
2471
    cell_in_pool = 0;
×
NEW
2472
    cell_offset = 0U;
×
2473
    /*
2474
     * The pool may grow while building descendants.  Remember the caller's
2475
     * offset so the cell pointer can be refreshed after realloc moves the
2476
     * backing storage.
2477
     */
NEW
2478
    if (pool_before != NULL) {
×
NEW
2479
        if ((uint8_t *)(void *)cell >= pool_before
×
NEW
2480
                && (size_t)((uint8_t *)(void *)cell - pool_before)
×
2481
                        < pool_size_before) {
NEW
2482
            cell_in_pool = 1;
×
NEW
2483
            cell_offset = (uint32_t)((uint8_t *)(void *)cell - pool_before);
×
2484
        }
2485
    }
NEW
2486
    offset = sixel_quant_pool_alloc(lut, &branch_status);
×
NEW
2487
    if (branch_status != SIXEL_OK) {
×
NEW
2488
        goto end;
×
2489
    }
NEW
2490
    if (cell_in_pool != 0) {
×
NEW
2491
        cell = (uint32_t *)(void *)(lut->pool + cell_offset);
×
2492
    }
NEW
2493
    sixel_quant_assign_branch(cell, offset);
×
2494
#ifdef DEBUG_CERTLUT_TRACE
2495
    fprintf(stderr,
2496
            "  branch offset=%u\n",
2497
            offset);
2498
#endif
NEW
2499
    status = SIXEL_OK;
×
2500

2501
end:
NEW
2502
    return status;
×
2503
}
2504

2505
static int
NEW
2506
sixel_certlut_build(sixel_certlut_t *lut, sixel_color_t const *palette,
×
2507
                    int ncolors, int wR, int wG, int wB)
2508
{
2509
    SIXELSTATUS status;
2510
    int initialized;
2511
    size_t level0_count;
NEW
2512
    status = SIXEL_FALSE;
×
NEW
2513
    initialized = sixel_certlut_init(lut);
×
NEW
2514
    if (SIXEL_FAILED(initialized)) {
×
NEW
2515
        goto end;
×
2516
    }
NEW
2517
    lut->palette = palette;
×
NEW
2518
    lut->ncolors = ncolors;
×
NEW
2519
    sixel_quant_weight_init(lut, wR, wG, wB);
×
NEW
2520
    status = sixel_certlut_prepare_palette_terms(lut);
×
NEW
2521
    if (SIXEL_FAILED(status)) {
×
NEW
2522
        goto end;
×
2523
    }
NEW
2524
    status = sixel_certlut_kdtree_build(lut);
×
NEW
2525
    if (SIXEL_FAILED(status)) {
×
NEW
2526
        goto end;
×
2527
    }
NEW
2528
    level0_count = (size_t)64 * (size_t)64 * (size_t)64;
×
NEW
2529
    lut->level0 = (uint32_t *)calloc(level0_count, sizeof(uint32_t));
×
NEW
2530
    if (lut->level0 == NULL) {
×
NEW
2531
        goto end;
×
2532
    }
2533
    /*
2534
     * Level 0 cells start uninitialized.  The lookup routine materializes
2535
     * individual subtrees on demand so we avoid evaluating the entire
2536
     * 64x64x64 grid upfront.
2537
     */
NEW
2538
    status = SIXEL_OK;
×
2539

2540
end:
NEW
2541
    if (SIXEL_FAILED(status)) {
×
NEW
2542
        sixel_certlut_release(lut);
×
2543
    }
NEW
2544
    return status;
×
2545
}
2546

2547
static uint8_t
NEW
2548
sixel_certlut_lookup(sixel_certlut_t *lut, uint8_t r, uint8_t g, uint8_t b)
×
2549
{
2550
    uint32_t entry;
2551
    uint32_t offset;
2552
    uint32_t index;
2553
    uint32_t *children;
2554
    uint32_t *cell;
2555
    int shift;
2556
    int child;
2557
    int status;
2558
    int size;
2559
    int rmin;
2560
    int gmin;
2561
    int bmin;
2562
    int step;
NEW
2563
    if (lut == NULL || lut->level0 == NULL) {
×
NEW
2564
        return 0U;
×
2565
    }
2566
    /*
2567
     * Cells are created lazily.  A zero entry indicates an uninitialized
2568
     * subtree, so the builder is invoked with the cube bounds of the current
2569
     * traversal.  Should allocation fail we fall back to a direct brute-force
2570
     * palette search for the queried pixel.
2571
     */
NEW
2572
    index = ((uint32_t)(r >> 2) << 12)
×
NEW
2573
          | ((uint32_t)(g >> 2) << 6)
×
NEW
2574
          | (uint32_t)(b >> 2);
×
NEW
2575
    cell = lut->level0 + index;
×
NEW
2576
    size = 4;
×
NEW
2577
    rmin = (int)(r & 0xfc);
×
NEW
2578
    gmin = (int)(g & 0xfc);
×
NEW
2579
    bmin = (int)(b & 0xfc);
×
NEW
2580
    entry = *cell;
×
NEW
2581
    if (entry == 0U) {
×
2582
#ifdef DEBUG_CERTLUT_TRACE
2583
        fprintf(stderr,
2584
                "lookup build level0 r=%u g=%u b=%u\n",
2585
                (unsigned int)r,
2586
                (unsigned int)g,
2587
                (unsigned int)b);
2588
#endif
NEW
2589
        status = sixel_certlut_build_cell(lut, cell, rmin, gmin, bmin, size);
×
NEW
2590
        if (SIXEL_FAILED(status)) {
×
NEW
2591
            return sixel_certlut_fallback(lut,
×
2592
                                          (int)r,
2593
                                          (int)g,
2594
                                          (int)b);
2595
        }
NEW
2596
        entry = *cell;
×
2597
    }
NEW
2598
    shift = 1;
×
NEW
2599
    while ((entry & 0x80000000U) == 0U) {
×
NEW
2600
        offset = entry & 0x3fffffffU;
×
NEW
2601
        children = (uint32_t *)(void *)(lut->pool + offset);
×
NEW
2602
        child = (((int)(r >> shift) & 1) << 2)
×
NEW
2603
              | (((int)(g >> shift) & 1) << 1)
×
NEW
2604
              | ((int)(b >> shift) & 1);
×
2605
#ifdef DEBUG_CERTLUT_TRACE
2606
        fprintf(stderr,
2607
                "descend child=%d size=%d offset=%u\n",
2608
                child,
2609
                size,
2610
                offset);
2611
#endif
NEW
2612
        step = size / 2;
×
NEW
2613
        if (step <= 0) {
×
NEW
2614
            step = 1;
×
2615
        }
NEW
2616
        rmin += step * ((child >> 2) & 1);
×
NEW
2617
        gmin += step * ((child >> 1) & 1);
×
NEW
2618
        bmin += step * (child & 1);
×
NEW
2619
        size = step;
×
NEW
2620
        cell = children + (size_t)child;
×
NEW
2621
        entry = *cell;
×
NEW
2622
        if (entry == 0U) {
×
2623
#ifdef DEBUG_CERTLUT_TRACE
2624
            fprintf(stderr,
2625
                    "lookup build child size=%d rmin=%d gmin=%d bmin=%d\n",
2626
                    size,
2627
                    rmin,
2628
                    gmin,
2629
                    bmin);
2630
#endif
NEW
2631
            status = sixel_certlut_build_cell(lut,
×
2632
                                              cell,
2633
                                              rmin,
2634
                                              gmin,
2635
                                              bmin,
2636
                                              size);
NEW
2637
            if (SIXEL_FAILED(status)) {
×
NEW
2638
                return sixel_certlut_fallback(lut,
×
2639
                                              (int)r,
2640
                                              (int)g,
2641
                                              (int)b);
2642
            }
NEW
2643
            children = (uint32_t *)(void *)(lut->pool + offset);
×
NEW
2644
            cell = children + (size_t)child;
×
NEW
2645
            entry = *cell;
×
2646
        }
NEW
2647
        if (size == 1) {
×
NEW
2648
            break;
×
2649
        }
NEW
2650
        if (shift == 0) {
×
NEW
2651
            break;
×
2652
        }
NEW
2653
        --shift;
×
2654
    }
2655

NEW
2656
    return (uint8_t)(entry & 0xffU);
×
2657
}
2658

2659
static void
NEW
2660
sixel_certlut_free(sixel_certlut_t *lut)
×
2661
{
NEW
2662
    sixel_certlut_release(lut);
×
NEW
2663
    if (lut != NULL) {
×
NEW
2664
        lut->palette = NULL;
×
NEW
2665
        lut->ncolors = 0;
×
2666
    }
NEW
2667
}
×
2668

2669
static size_t
2670
robinhood_round_capacity(size_t hint)
×
2671
{
2672
    size_t capacity;
2673

2674
    capacity = 16U;
×
2675
    if (hint < capacity) {
×
2676
        return capacity;
×
2677
    }
2678

2679
    capacity = hint - 1U;
×
2680
    capacity |= capacity >> 1;
×
2681
    capacity |= capacity >> 2;
×
2682
    capacity |= capacity >> 4;
×
2683
    capacity |= capacity >> 8;
×
2684
    capacity |= capacity >> 16;
×
2685
#if SIZE_MAX > UINT32_MAX
2686
    capacity |= capacity >> 32;
×
2687
#endif
2688
    if (capacity == SIZE_MAX) {
×
2689
        return SIZE_MAX;
×
2690
    }
2691
    capacity++;
×
2692
    if (capacity < 16U) {
×
2693
        capacity = 16U;
×
2694
    }
2695

2696
    return capacity;
×
2697
}
2698

2699
static SIXELSTATUS
2700
robinhood_table32_init(struct robinhood_table32 *table,
×
2701
                       size_t expected,
2702
                       sixel_allocator_t *allocator)
2703
{
2704
    size_t hint;
2705
    size_t capacity;
2706

2707
    table->slots = NULL;
×
2708
    table->capacity = 0U;
×
2709
    table->count = 0U;
×
2710
    table->allocator = allocator;
×
2711

2712
    if (expected < 16U) {
×
2713
        expected = 16U;
×
2714
    }
2715
    if (expected > SIZE_MAX / 2U) {
×
2716
        hint = SIZE_MAX / 2U;
×
2717
    } else {
2718
        hint = expected * 2U;
×
2719
    }
2720
    capacity = robinhood_round_capacity(hint);
×
2721
    if (capacity == SIZE_MAX && hint != SIZE_MAX) {
×
2722
        return SIXEL_BAD_ALLOCATION;
×
2723
    }
2724

2725
    table->slots = (struct robinhood_slot32 *)
×
2726
        sixel_allocator_calloc(allocator,
×
2727
                               capacity,
2728
                               sizeof(struct robinhood_slot32));
2729
    if (table->slots == NULL) {
×
2730
        table->capacity = 0U;
×
2731
        table->count = 0U;
×
2732
        return SIXEL_BAD_ALLOCATION;
×
2733
    }
2734
    table->capacity = capacity;
×
2735
    table->count = 0U;
×
2736

2737
    return SIXEL_OK;
×
2738
}
2739

2740
static void
2741
robinhood_table32_fini(struct robinhood_table32 *table)
×
2742
{
2743
    if (table->slots != NULL) {
×
2744
        sixel_allocator_free(table->allocator, table->slots);
×
2745
        table->slots = NULL;
×
2746
    }
2747
    table->capacity = 0U;
×
2748
    table->count = 0U;
×
2749
    table->allocator = NULL;
×
2750
}
×
2751

2752
static struct robinhood_slot32 *
2753
robinhood_table32_lookup(struct robinhood_table32 *table,
×
2754
                         uint32_t key,
2755
                         uint32_t color)
2756
{
2757
    size_t mask;
2758
    size_t index;
2759
    uint16_t distance;
2760
    struct robinhood_slot32 *slot;
2761

2762
    if (table->capacity == 0U || table->slots == NULL) {
×
2763
        return NULL;
×
2764
    }
2765

2766
    mask = table->capacity - 1U;
×
2767
    index = (size_t)(key & mask);
×
2768
    distance = 0U;
×
2769

2770
    for (;;) {
2771
        slot = &table->slots[index];
×
2772
        if (slot->value == 0U) {
×
2773
            return NULL;
×
2774
        }
2775
        if (slot->key == key && slot->color == color) {
×
2776
            return slot;
×
2777
        }
2778
        if (slot->distance < distance) {
×
2779
            return NULL;
×
2780
        }
2781
        index = (index + 1U) & mask;
×
2782
        distance++;
×
2783
    }
2784
}
2785

2786
static struct robinhood_slot32 *
2787
robinhood_table32_place(struct robinhood_table32 *table,
×
2788
                        struct robinhood_slot32 entry)
2789
{
2790
    size_t mask;
2791
    size_t index;
2792
    struct robinhood_slot32 *slot;
2793
    struct robinhood_slot32 tmp;
2794

2795
    mask = table->capacity - 1U;
×
2796
    index = (size_t)(entry.key & mask);
×
2797

2798
    for (;;) {
2799
        slot = &table->slots[index];
×
2800
        if (slot->value == 0U) {
×
2801
            *slot = entry;
×
2802
            table->count++;
×
2803
            return slot;
×
2804
        }
2805
        if (slot->key == entry.key && slot->color == entry.color) {
×
2806
            slot->value = entry.value;
×
2807
            return slot;
×
2808
        }
2809
        if (slot->distance < entry.distance) {
×
2810
            tmp = *slot;
×
2811
            *slot = entry;
×
2812
            entry = tmp;
×
2813
        }
2814
        index = (index + 1U) & mask;
×
2815
        entry.distance++;
×
2816
    }
2817
}
2818

2819
static SIXELSTATUS
2820
robinhood_table32_grow(struct robinhood_table32 *table)
×
2821
{
2822
    struct robinhood_slot32 *old_slots;
2823
    size_t old_capacity;
2824
    size_t new_capacity;
2825
    size_t i;
2826

2827
    if (table->allocator == NULL) {
×
2828
        return SIXEL_BAD_ALLOCATION;
×
2829
    }
2830
    if (table->capacity == 0U) {
×
2831
        new_capacity = 16U;
×
2832
    } else {
2833
        if (table->capacity > SIZE_MAX / 2U) {
×
2834
            return SIXEL_BAD_ALLOCATION;
×
2835
        }
2836
        new_capacity = table->capacity << 1U;
×
2837
    }
2838
    new_capacity = robinhood_round_capacity(new_capacity);
×
2839
    if (new_capacity <= table->capacity) {
×
2840
        return SIXEL_BAD_ALLOCATION;
×
2841
    }
2842

2843
    old_slots = table->slots;
×
2844
    old_capacity = table->capacity;
×
2845
    table->slots = (struct robinhood_slot32 *)
×
2846
        sixel_allocator_calloc(table->allocator,
×
2847
                               new_capacity,
2848
                               sizeof(struct robinhood_slot32));
2849
    if (table->slots == NULL) {
×
2850
        table->slots = old_slots;
×
2851
        table->capacity = old_capacity;
×
2852
        return SIXEL_BAD_ALLOCATION;
×
2853
    }
2854
    table->capacity = new_capacity;
×
2855
    table->count = 0U;
×
2856

2857
    for (i = 0U; i < old_capacity; ++i) {
×
2858
        struct robinhood_slot32 entry;
2859

2860
        if (old_slots[i].value == 0U) {
×
2861
            continue;
×
2862
        }
2863
        entry.key = old_slots[i].key;
×
2864
        entry.color = old_slots[i].color;
×
2865
        entry.value = old_slots[i].value;
×
2866
        entry.distance = 0U;
×
2867
        entry.pad = 0U;  /* ensure padding bytes are initialized */
×
2868
        (void)robinhood_table32_place(table, entry);
×
2869
    }
2870

2871
    sixel_allocator_free(table->allocator, old_slots);
×
2872

2873
    return SIXEL_OK;
×
2874
}
2875

2876
static SIXELSTATUS
2877
robinhood_table32_insert(struct robinhood_table32 *table,
×
2878
                         uint32_t key,
2879
                         uint32_t color,
2880
                         uint32_t value)
2881
{
2882
    SIXELSTATUS status;
2883
    struct robinhood_slot32 entry;
2884

2885
    if (table->slots == NULL || table->capacity == 0U) {
×
2886
        return SIXEL_BAD_ARGUMENT;
×
2887
    }
2888
    if (table->count * 2U >= table->capacity) {
×
2889
        status = robinhood_table32_grow(table);
×
2890
        if (SIXEL_FAILED(status)) {
×
2891
            return status;
×
2892
        }
2893
    }
2894

2895
    entry.key = key;
×
2896
    entry.color = color;
×
2897
    entry.value = value;
×
2898
    entry.distance = 0U;
×
2899
    entry.pad = 0U;  /* ensure padding bytes are initialized */
×
2900
    (void)robinhood_table32_place(table, entry);
×
2901

2902
    return SIXEL_OK;
×
2903
}
2904

2905
static SIXELSTATUS
2906
hopscotch_table32_init(struct hopscotch_table32 *table,
×
2907
                       size_t expected,
2908
                       sixel_allocator_t *allocator)
2909
{
2910
    size_t hint;
2911
    size_t capacity;
2912
    size_t i;
2913

2914
    if (table == NULL) {
×
2915
        return SIXEL_BAD_ARGUMENT;
×
2916
    }
2917

2918
    table->slots = NULL;
×
2919
    table->hopinfo = NULL;
×
2920
    table->capacity = 0U;
×
2921
    table->count = 0U;
×
2922
    table->neighborhood = HOPSCOTCH_DEFAULT_NEIGHBORHOOD;
×
2923
    table->allocator = allocator;
×
2924

2925
    if (expected < 16U) {
×
2926
        expected = 16U;
×
2927
    }
2928
    if (expected > SIZE_MAX / 2U) {
×
2929
        hint = SIZE_MAX / 2U;
×
2930
    } else {
2931
        hint = expected * 2U;
×
2932
    }
2933
    capacity = robinhood_round_capacity(hint);
×
2934
    if (capacity == SIZE_MAX && hint != SIZE_MAX) {
×
2935
        return SIXEL_BAD_ALLOCATION;
×
2936
    }
2937
    if (capacity < table->neighborhood) {
×
2938
        capacity = table->neighborhood;
×
2939
    }
2940

2941
    table->slots = (struct hopscotch_slot32 *)
×
2942
        sixel_allocator_malloc(allocator,
×
2943
                               capacity * sizeof(struct hopscotch_slot32));
2944
    if (table->slots == NULL) {
×
2945
        return SIXEL_BAD_ALLOCATION;
×
2946
    }
2947
    table->hopinfo = (uint32_t *)
×
2948
        sixel_allocator_calloc(allocator,
×
2949
                               capacity,
2950
                               sizeof(uint32_t));
2951
    if (table->hopinfo == NULL) {
×
2952
        sixel_allocator_free(allocator, table->slots);
×
2953
        table->slots = NULL;
×
2954
        return SIXEL_BAD_ALLOCATION;
×
2955
    }
2956

2957
    for (i = 0U; i < capacity; ++i) {
×
2958
        table->slots[i].key = HOPSCOTCH_EMPTY_KEY;
×
2959
        table->slots[i].color = 0U;
×
2960
        table->slots[i].value = 0U;
×
2961
    }
2962
    table->capacity = capacity;
×
2963
    table->count = 0U;
×
2964
    if (table->neighborhood > 32U) {
×
2965
        table->neighborhood = 32U;
×
2966
    }
2967
    if (table->neighborhood > table->capacity) {
×
2968
        table->neighborhood = table->capacity;
×
2969
    }
2970

2971
    return SIXEL_OK;
×
2972
}
2973

2974
static void
2975
hopscotch_table32_fini(struct hopscotch_table32 *table)
×
2976
{
2977
    sixel_allocator_t *allocator;
2978

2979
    if (table == NULL) {
×
2980
        return;
×
2981
    }
2982

2983
    allocator = table->allocator;
×
2984
    if (allocator != NULL) {
×
2985
        if (table->slots != NULL) {
×
2986
            sixel_allocator_free(allocator, table->slots);
×
2987
        }
2988
        if (table->hopinfo != NULL) {
×
2989
            sixel_allocator_free(allocator, table->hopinfo);
×
2990
        }
2991
    }
2992
    table->slots = NULL;
×
2993
    table->hopinfo = NULL;
×
2994
    table->capacity = 0U;
×
2995
    table->count = 0U;
×
2996
}
2997

2998
static struct hopscotch_slot32 *
2999
hopscotch_table32_lookup(struct hopscotch_table32 *table,
×
3000
                         uint32_t key,
3001
                         uint32_t color)
3002
{
3003
    size_t index;
3004
    size_t bit;
3005
    size_t candidate;
3006
    uint32_t hop;
3007
    size_t mask;
3008
    size_t neighborhood;
3009

3010
    if (table == NULL || table->slots == NULL || table->hopinfo == NULL) {
×
3011
        return NULL;
×
3012
    }
3013
    if (table->capacity == 0U) {
×
3014
        return NULL;
×
3015
    }
3016

3017
    mask = table->capacity - 1U;
×
3018
    index = (size_t)key & mask;
×
3019
    hop = table->hopinfo[index];
×
3020
    neighborhood = table->neighborhood;
×
3021
    for (bit = 0U; bit < neighborhood; ++bit) {
×
3022
        if ((hop & (1U << bit)) == 0U) {
×
3023
            continue;
×
3024
        }
3025
        candidate = (index + bit) & mask;
×
3026
        if (table->slots[candidate].key == key
×
3027
            && table->slots[candidate].color == color) {
×
3028
            return &table->slots[candidate];
×
3029
        }
3030
    }
3031

3032
    return NULL;
×
3033
}
3034

3035
static SIXELSTATUS
3036
hopscotch_table32_grow(struct hopscotch_table32 *table)
×
3037
{
3038
    SIXELSTATUS status;
3039
    struct hopscotch_table32 tmp;
3040
    size_t i;
3041
    struct hopscotch_slot32 *slot;
3042

3043
    tmp.slots = NULL;
×
3044
    tmp.hopinfo = NULL;
×
3045
    tmp.capacity = 0U;
×
3046
    tmp.count = 0U;
×
3047
    tmp.neighborhood = table->neighborhood;
×
3048
    tmp.allocator = table->allocator;
×
3049

3050
    status = hopscotch_table32_init(&tmp,
×
3051
                                    table->capacity * 2U,
×
3052
                                    table->allocator);
3053
    if (SIXEL_FAILED(status)) {
×
3054
        return status;
×
3055
    }
3056
    if (tmp.neighborhood > table->neighborhood) {
×
3057
        tmp.neighborhood = table->neighborhood;
×
3058
    }
3059

3060
    for (i = 0U; i < table->capacity; ++i) {
×
3061
        slot = &table->slots[i];
×
3062
        if (slot->key == HOPSCOTCH_EMPTY_KEY) {
×
3063
            continue;
×
3064
        }
3065
        status = hopscotch_table32_insert(&tmp,
×
3066
                                          slot->key,
3067
                                          slot->color,
3068
                                          slot->value);
3069
        if (SIXEL_FAILED(status)) {
×
3070
            hopscotch_table32_fini(&tmp);
×
3071
            return status;
×
3072
        }
3073
    }
3074

3075
    hopscotch_table32_fini(table);
×
3076

3077
    table->slots = tmp.slots;
×
3078
    table->hopinfo = tmp.hopinfo;
×
3079
    table->capacity = tmp.capacity;
×
3080
    table->count = tmp.count;
×
3081
    table->neighborhood = tmp.neighborhood;
×
3082
    table->allocator = tmp.allocator;
×
3083

3084
    return SIXEL_OK;
×
3085
}
3086

3087
static SIXELSTATUS
3088
hopscotch_table32_insert(struct hopscotch_table32 *table,
×
3089
                         uint32_t key,
3090
                         uint32_t color,
3091
                         uint32_t value)
3092
{
3093
    SIXELSTATUS status;
3094
    struct hopscotch_slot32 *slot;
3095
    size_t index;
3096
    size_t mask;
3097
    size_t distance;
3098
    size_t attempts;
3099
    size_t free_index;
3100
    size_t neighborhood;
3101
    int relocated;
3102
    size_t offset;
3103
    uint32_t hop;
3104
    size_t bit;
3105
    size_t move_index;
3106
    struct hopscotch_slot32 tmp_slot;
3107

3108
    if (table == NULL || table->slots == NULL || table->hopinfo == NULL) {
×
3109
        return SIXEL_BAD_ARGUMENT;
×
3110
    }
3111
    if (table->capacity == 0U) {
×
3112
        return SIXEL_BAD_ARGUMENT;
×
3113
    }
3114

3115
    slot = hopscotch_table32_lookup(table, key, color);
×
3116
    if (slot != NULL) {
×
3117
        slot->value = value;
×
3118
        return SIXEL_OK;
×
3119
    }
3120

3121
    if (table->count * 2U >= table->capacity) {
×
3122
        status = hopscotch_table32_grow(table);
×
3123
        if (SIXEL_FAILED(status)) {
×
3124
            return status;
×
3125
        }
3126
        return hopscotch_table32_insert(table, key, color, value);
×
3127
    }
3128

3129
    mask = table->capacity - 1U;
×
3130
    neighborhood = table->neighborhood;
×
3131
    index = (size_t)key & mask;
×
3132
    slot = NULL;
×
3133
    free_index = index;
×
3134
    distance = 0U;
×
3135
    for (attempts = 0U; attempts < HOPSCOTCH_INSERT_RANGE; ++attempts) {
×
3136
        free_index = (index + attempts) & mask;
×
3137
        slot = &table->slots[free_index];
×
3138
        if (slot->key == HOPSCOTCH_EMPTY_KEY) {
×
3139
            distance = attempts;
×
3140
            break;
×
3141
        }
3142
    }
3143
    if (slot == NULL || slot->key != HOPSCOTCH_EMPTY_KEY) {
×
3144
        status = hopscotch_table32_grow(table);
×
3145
        if (SIXEL_FAILED(status)) {
×
3146
            return status;
×
3147
        }
3148
        return hopscotch_table32_insert(table, key, color, value);
×
3149
    }
3150

3151
    /*
3152
     * Relocation diagram:
3153
     *
3154
     *   free slot <--- hop window <--- [base bucket]
3155
     *      ^ move resident outward until distance < neighborhood
3156
     */
3157
    while (distance >= neighborhood) {
×
3158
        relocated = 0;
×
3159
        for (offset = neighborhood - 1U; offset > 0U; --offset) {
×
3160
            size_t base;
3161

3162
            base = (free_index + table->capacity - offset) & mask;
×
3163
            hop = table->hopinfo[base];
×
3164
            if (hop == 0U) {
×
3165
                continue;
×
3166
            }
3167
            for (bit = 0U; bit < offset; ++bit) {
×
3168
                if ((hop & (1U << bit)) == 0U) {
×
3169
                    continue;
×
3170
                }
3171
                move_index = (base + bit) & mask;
×
3172
                tmp_slot = table->slots[move_index];
×
3173
                table->slots[free_index] = tmp_slot;
×
3174
                table->slots[move_index].key = HOPSCOTCH_EMPTY_KEY;
×
3175
                table->slots[move_index].color = 0U;
×
3176
                table->slots[move_index].value = 0U;
×
3177
                table->hopinfo[base] &= (uint32_t)~(1U << bit);
×
3178
                table->hopinfo[base] |= (uint32_t)(1U << offset);
×
3179
                free_index = move_index;
×
3180
                if (free_index >= index) {
×
3181
                    distance = free_index - index;
×
3182
                } else {
3183
                    distance = free_index + table->capacity - index;
×
3184
                }
3185
                relocated = 1;
×
3186
                break;
×
3187
            }
3188
            if (relocated) {
×
3189
                break;
×
3190
            }
3191
        }
3192
        if (!relocated) {
×
3193
            status = hopscotch_table32_grow(table);
×
3194
            if (SIXEL_FAILED(status)) {
×
3195
                return status;
×
3196
            }
3197
            return hopscotch_table32_insert(table, key, color, value);
×
3198
        }
3199
    }
3200

3201
    if (distance >= 32U) {
×
3202
        status = hopscotch_table32_grow(table);
×
3203
        if (SIXEL_FAILED(status)) {
×
3204
            return status;
×
3205
        }
3206
        return hopscotch_table32_insert(table, key, color, value);
×
3207
    }
3208

3209
    table->slots[free_index].key = key;
×
3210
    table->slots[free_index].color = color;
×
3211
    table->slots[free_index].value = value;
×
3212
    table->hopinfo[index] |= (uint32_t)(1U << distance);
×
3213
    table->count++;
×
3214

3215
    return SIXEL_OK;
×
3216
}
3217

3218
/*
3219
 * The cuckoo hash backend stores entries in fixed-width buckets.
3220
 *
3221
 *   [bucket 0] -> key0 key1 key2 key3
3222
 *                 val0 val1 val2 val3
3223
 *   [bucket 1] -> ...
3224
 *
3225
 * Each key is compared against the 128-bit lane using SIMD instructions.
3226
 * Two hash functions map a key to its primary and secondary buckets.  When
3227
 * both are full, the eviction loop "kicks" an entry toward its alternate
3228
 * bucket, as illustrated below:
3229
 *
3230
 *   bucket A --kick--> bucket B --kick--> bucket A ...
3231
 *
3232
 * A tiny stash buffers entries when the table momentarily fills up.  This
3233
 * keeps lookups fast while letting us grow the table lazily.
3234
 */
3235
static size_t
3236
cuckoo_round_buckets(size_t hint)
267✔
3237
{
3238
    size_t desired;
3239
    size_t buckets;
3240
    size_t prev;
3241

3242
    if (hint < CUCKOO_BUCKET_SIZE) {
267!
3243
        hint = CUCKOO_BUCKET_SIZE;
×
3244
    }
3245
    if (hint > SIZE_MAX / 2U) {
267!
3246
        hint = SIZE_MAX / 2U;
×
3247
    }
3248
    desired = (hint * 2U + CUCKOO_BUCKET_SIZE - 1U) / CUCKOO_BUCKET_SIZE;
267✔
3249
    if (desired == 0U) {
267!
3250
        desired = 1U;
×
3251
    }
3252

3253
    buckets = 1U;
267✔
3254
    while (buckets < desired) {
4,806✔
3255
        prev = buckets;
4,539✔
3256
        if (buckets > SIZE_MAX / 2U) {
4,539!
3257
            buckets = prev;
×
3258
            break;
×
3259
        }
3260
        buckets <<= 1U;
4,539✔
3261
        if (buckets < prev) {
4,539!
3262
            buckets = prev;
×
3263
            break;
×
3264
        }
3265
    }
3266
    if (buckets == 0U) {
267!
3267
        buckets = 1U;
×
3268
    }
3269

3270
    return buckets;
267✔
3271
}
3272

3273
static size_t
3274
cuckoo_hash_primary(uint32_t key, size_t mask)
43,908,960✔
3275
{
3276
    uint32_t mix;
3277

3278
    mix = key * 0x9e3779b1U;
43,908,960✔
3279
    return (size_t)(mix & (uint32_t)mask);
43,908,960✔
3280
}
3281

3282
static size_t
3283
cuckoo_hash_secondary(uint32_t key, size_t mask)
3,837,993✔
3284
{
3285
    uint32_t mix;
3286

3287
    mix = key ^ 0x85ebca6bU;
3,837,993✔
3288
    mix ^= mix >> 13;
3,837,993✔
3289
    mix *= 0xc2b2ae35U;
3,837,993✔
3290
    return (size_t)(mix & (uint32_t)mask);
3,837,993✔
3291
}
3292

3293
static size_t
3294
cuckoo_hash_alternate(uint32_t key, size_t bucket, size_t mask)
52✔
3295
{
3296
    size_t primary;
3297
    size_t secondary;
3298

3299
    primary = cuckoo_hash_primary(key, mask);
52✔
3300
    secondary = cuckoo_hash_secondary(key, mask);
52✔
3301
    if (primary == bucket) {
52!
3302
        if (secondary == primary) {
52!
3303
            secondary = (secondary + 1U) & mask;
×
3304
        }
3305
        return secondary;
52✔
3306
    }
3307

3308
    return primary;
×
3309
}
10✔
3310

3311
static uint32_t *
3312
cuckoo_bucket_find(struct cuckoo_bucket32 *bucket, uint32_t key)
45,831,374✔
3313
{
3314
    size_t i;
3315

3316
#if defined(SIXEL_USE_SSE2)
3317
    __m128i needle;
3318
    __m128i keys;
3319
    __m128i cmp;
3320
    int mask;
3321

3322
    needle = _mm_set1_epi32((int)key);
3323
    keys = _mm_loadu_si128((const __m128i *)bucket->key);
3324
    cmp = _mm_cmpeq_epi32(needle, keys);
3325
    mask = _mm_movemask_ps(_mm_castsi128_ps(cmp));
3326
    if ((mask & 1) != 0 && bucket->value[0] != 0U) {
3327
        return &bucket->value[0];
3328
    }
3329
    if ((mask & 2) != 0 && bucket->value[1] != 0U) {
3330
        return &bucket->value[1];
3331
    }
3332
    if ((mask & 4) != 0 && bucket->value[2] != 0U) {
3333
        return &bucket->value[2];
3334
    }
3335
    if ((mask & 8) != 0 && bucket->value[3] != 0U) {
3336
        return &bucket->value[3];
3337
    }
3338
#elif defined(SIXEL_USE_NEON)
3339
    uint32x4_t needle;
3340
    uint32x4_t keys;
3341
    uint32x4_t cmp;
3342

3343
    needle = vdupq_n_u32(key);
3344
    keys = vld1q_u32(bucket->key);
3345
    cmp = vceqq_u32(needle, keys);
3346
    if (vgetq_lane_u32(cmp, 0) != 0U && bucket->value[0] != 0U) {
3347
        return &bucket->value[0];
3348
    }
3349
    if (vgetq_lane_u32(cmp, 1) != 0U && bucket->value[1] != 0U) {
3350
        return &bucket->value[1];
3351
    }
3352
    if (vgetq_lane_u32(cmp, 2) != 0U && bucket->value[2] != 0U) {
3353
        return &bucket->value[2];
3354
    }
3355
    if (vgetq_lane_u32(cmp, 3) != 0U && bucket->value[3] != 0U) {
3356
        return &bucket->value[3];
3357
    }
3358
#else
3359
    for (i = 0U; i < CUCKOO_BUCKET_SIZE; ++i) {
77,866,608✔
3360
        if (bucket->value[i] != 0U && bucket->key[i] == key) {
70,197,717✔
3361
            return &bucket->value[i];
38,162,483✔
3362
        }
3363
    }
10,249,102✔
3364
#endif
3365

3366
    return NULL;
7,668,891✔
3367
}
20,846,518✔
3368

3369
static int
3370
cuckoo_bucket_insert_direct(struct cuckoo_bucket32 *bucket,
1,915,527✔
3371
                            uint32_t key,
3372
                            uint32_t value)
3373
{
3374
    size_t i;
3375

3376
    for (i = 0U; i < CUCKOO_BUCKET_SIZE; ++i) {
2,112,289✔
3377
        if (bucket->value[i] == 0U) {
2,112,237✔
3378
            bucket->key[i] = key;
1,915,475✔
3379
            bucket->value[i] = value;
1,915,475✔
3380
            return 1;
1,915,475✔
3381
        }
3382
    }
59,286✔
3383

3384
    return 0;
52✔
3385
}
613,553✔
3386

3387
static SIXELSTATUS
3388
cuckoo_table32_init(struct cuckoo_table32 *table,
264✔
3389
                    size_t expected,
3390
                    sixel_allocator_t *allocator)
3391
{
3392
    size_t buckets;
3393
    size_t i;
3394
    size_t j;
3395

3396
    if (table == NULL || allocator == NULL) {
264!
3397
        return SIXEL_BAD_ARGUMENT;
×
3398
    }
3399

3400
    buckets = cuckoo_round_buckets(expected);
264✔
3401
    if (buckets == 0U
264!
3402
        || buckets > SIZE_MAX / sizeof(struct cuckoo_bucket32)) {
264!
3403
        sixel_helper_set_additional_message(
×
3404
            "unable to size cuckoo bucket array.");
3405
        return SIXEL_BAD_ALLOCATION;
×
3406
    }
3407

3408
    table->buckets = (struct cuckoo_bucket32 *)sixel_allocator_malloc(
264✔
3409
        allocator, buckets * sizeof(struct cuckoo_bucket32));
108✔
3410
    if (table->buckets == NULL) {
264!
3411
        sixel_helper_set_additional_message(
×
3412
            "unable to allocate cuckoo buckets.");
3413
        return SIXEL_BAD_ALLOCATION;
×
3414
    }
3415

3416
    table->bucket_count = buckets;
264✔
3417
    table->bucket_mask = buckets - 1U;
264✔
3418
    table->stash_count = 0U;
264✔
3419
    table->entry_count = 0U;
264✔
3420
    table->allocator = allocator;
264✔
3421
    for (i = 0U; i < buckets; ++i) {
34,603,272✔
3422
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
173,015,040✔
3423
            table->buckets[i].key[j] = CUCKOO_EMPTY_KEY;
138,412,032✔
3424
            table->buckets[i].value[j] = 0U;
138,412,032✔
3425
        }
56,623,104✔
3426
    }
14,155,776✔
3427
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
8,712✔
3428
        table->stash_key[i] = CUCKOO_EMPTY_KEY;
8,448✔
3429
        table->stash_value[i] = 0U;
8,448✔
3430
    }
3,456✔
3431

3432
    return SIXEL_OK;
264✔
3433
}
108✔
3434

3435
static void
3436
cuckoo_table32_clear(struct cuckoo_table32 *table)
267✔
3437
{
3438
    size_t i;
3439
    size_t j;
3440

3441
    if (table == NULL || table->buckets == NULL) {
267!
3442
        return;
×
3443
    }
3444

3445
    table->stash_count = 0U;
267✔
3446
    table->entry_count = 0U;
267✔
3447
    for (i = 0U; i < table->bucket_count; ++i) {
34,996,491✔
3448
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
174,981,120✔
3449
            table->buckets[i].key[j] = CUCKOO_EMPTY_KEY;
139,984,896✔
3450
            table->buckets[i].value[j] = 0U;
139,984,896✔
3451
        }
57,147,392✔
3452
    }
14,286,848✔
3453
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
8,811✔
3454
        table->stash_key[i] = CUCKOO_EMPTY_KEY;
8,544✔
3455
        table->stash_value[i] = 0U;
8,544✔
3456
    }
3,488✔
3457
}
109✔
3458

3459
static void
3460
cuckoo_table32_fini(struct cuckoo_table32 *table)
264✔
3461
{
3462
    if (table == NULL || table->allocator == NULL) {
264!
3463
        return;
×
3464
    }
3465
    if (table->buckets != NULL) {
264!
3466
        sixel_allocator_free(table->allocator, table->buckets);
264✔
3467
        table->buckets = NULL;
264✔
3468
    }
108✔
3469
    table->bucket_count = 0U;
264✔
3470
    table->bucket_mask = 0U;
264✔
3471
    table->stash_count = 0U;
264✔
3472
    table->entry_count = 0U;
264✔
3473
}
108✔
3474

3475
static uint32_t *
3476
cuckoo_table32_lookup(struct cuckoo_table32 *table, uint32_t key)
41,993,433✔
3477
{
3478
    size_t index;
3479
    size_t i;
3480
    uint32_t *slot;
3481

3482
    if (table == NULL || table->buckets == NULL) {
41,993,433!
3483
        return NULL;
×
3484
    }
3485

3486
    index = cuckoo_hash_primary(key, table->bucket_mask);
41,993,433✔
3487
    slot = cuckoo_bucket_find(&table->buckets[index], key);
41,993,433✔
3488
    if (slot != NULL) {
41,993,433✔
3489
        return slot;
38,155,492✔
3490
    }
3491

3492
    index = cuckoo_hash_secondary(key, table->bucket_mask);
3,837,941✔
3493
    slot = cuckoo_bucket_find(&table->buckets[index], key);
3,837,941✔
3494
    if (slot != NULL) {
3,837,941✔
3495
        return slot;
6,991✔
3496
    }
3497

3498
    for (i = 0U; i < table->stash_count; ++i) {
3,830,950!
3499
        if (table->stash_value[i] != 0U && table->stash_key[i] == key) {
×
3500
            return &table->stash_value[i];
×
3501
        }
3502
    }
3503

3504
    return NULL;
3,830,950✔
3505
}
19,619,361✔
3506

3507
static SIXELSTATUS
3508
cuckoo_table32_grow(struct cuckoo_table32 *table)
×
3509
{
3510
    struct cuckoo_table32 tmp;
3511
    struct cuckoo_bucket32 *old_buckets;
3512
    size_t old_count;
3513
    size_t i;
3514
    size_t j;
3515
    SIXELSTATUS status;
3516

3517
    if (table == NULL || table->allocator == NULL) {
×
3518
        return SIXEL_BAD_ARGUMENT;
×
3519
    }
3520

3521
    tmp.buckets = NULL;
×
3522
    tmp.bucket_count = 0U;
×
3523
    tmp.bucket_mask = 0U;
×
3524
    tmp.stash_count = 0U;
×
3525
    tmp.entry_count = 0U;
×
3526
    tmp.allocator = table->allocator;
×
3527
    for (i = 0U; i < CUCKOO_STASH_SIZE; ++i) {
×
3528
        tmp.stash_key[i] = CUCKOO_EMPTY_KEY;
×
3529
        tmp.stash_value[i] = 0U;
×
3530
    }
3531

3532
    status = cuckoo_table32_init(&tmp,
×
3533
                                 (table->entry_count + 1U) * 2U,
×
3534
                                 table->allocator);
3535
    if (SIXEL_FAILED(status)) {
×
3536
        return status;
×
3537
    }
3538

3539
    old_buckets = table->buckets;
×
3540
    old_count = table->bucket_count;
×
3541
    for (i = 0U; i < old_count; ++i) {
×
3542
        for (j = 0U; j < CUCKOO_BUCKET_SIZE; ++j) {
×
3543
            if (old_buckets[i].value[j] != 0U) {
×
3544
                status = cuckoo_table32_insert(&tmp,
×
3545
                                               old_buckets[i].key[j],
×
3546
                                               old_buckets[i].value[j]);
×
3547
                if (SIXEL_FAILED(status)) {
×
3548
                    cuckoo_table32_fini(&tmp);
×
3549
                    return status;
×
3550
                }
3551
            }
3552
        }
3553
    }
3554
    for (i = 0U; i < table->stash_count; ++i) {
×
3555
        if (table->stash_value[i] != 0U) {
×
3556
            status = cuckoo_table32_insert(&tmp,
×
3557
                                           table->stash_key[i],
3558
                                           table->stash_value[i]);
3559
            if (SIXEL_FAILED(status)) {
×
3560
                cuckoo_table32_fini(&tmp);
×
3561
                return status;
×
3562
            }
3563
        }
3564
    }
3565

3566
    sixel_allocator_free(table->allocator, old_buckets);
×
3567
    *table = tmp;
×
3568

3569
    return SIXEL_OK;
×
3570
}
3571

3572
static SIXELSTATUS
3573
cuckoo_table32_insert(struct cuckoo_table32 *table,
1,915,475✔
3574
                      uint32_t key,
3575
                      uint32_t value)
3576
{
3577
    uint32_t *slot;
3578
    uint32_t cur_key;
3579
    uint32_t cur_value;
3580
    uint32_t victim_key;
3581
    uint32_t victim_value;
3582
    size_t bucket_index;
3583
    size_t kicks;
3584
    size_t victim_slot;
3585
    struct cuckoo_bucket32 *bucket;
3586
    SIXELSTATUS status;
3587

3588
    if (table == NULL || table->buckets == NULL) {
1,915,475!
3589
        return SIXEL_BAD_ARGUMENT;
×
3590
    }
3591

3592
    slot = cuckoo_table32_lookup(table, key);
1,915,475✔
3593
    if (slot != NULL) {
1,915,475!
3594
        *slot = value;
×
3595
        return SIXEL_OK;
×
3596
    }
3597

3598
    cur_key = key;
1,915,475✔
3599
    cur_value = value;
1,915,475✔
3600
    bucket_index = cuckoo_hash_primary(cur_key, table->bucket_mask);
1,915,475✔
3601
    for (kicks = 0U; kicks < CUCKOO_MAX_KICKS; ++kicks) {
1,915,527!
3602
        bucket = &table->buckets[bucket_index];
1,915,527✔
3603
        if (cuckoo_bucket_insert_direct(bucket, cur_key, cur_value)) {
1,915,527✔
3604
            table->entry_count++;
1,915,475✔
3605
            return SIXEL_OK;
1,915,475✔
3606
        }
3607
        victim_slot = (size_t)((cur_key + kicks) &
52✔
3608
                               (CUCKOO_BUCKET_SIZE - 1U));
3609
        victim_key = bucket->key[victim_slot];
52✔
3610
        victim_value = bucket->value[victim_slot];
52✔
3611
        bucket->key[victim_slot] = cur_key;
52✔
3612
        bucket->value[victim_slot] = cur_value;
52✔
3613
        cur_key = victim_key;
52✔
3614
        cur_value = victim_value;
52✔
3615
        bucket_index = cuckoo_hash_alternate(cur_key,
62✔
3616
                                             bucket_index,
10✔
3617
                                             table->bucket_mask);
10✔
3618
    }
10✔
3619

3620
    if (table->stash_count < CUCKOO_STASH_SIZE) {
×
3621
        table->stash_key[table->stash_count] = cur_key;
×
3622
        table->stash_value[table->stash_count] = cur_value;
×
3623
        table->stash_count++;
×
3624
        table->entry_count++;
×
3625
        return SIXEL_OK;
×
3626
    }
3627

3628
    status = cuckoo_table32_grow(table);
×
3629
    if (SIXEL_FAILED(status)) {
×
3630
        return status;
×
3631
    }
3632

3633
    return cuckoo_table32_insert(table, cur_key, cur_value);
×
3634
}
613,543✔
3635

3636
static SIXELSTATUS
3637
computeHistogram_robinhood(unsigned char const *data,
3638
                           unsigned int length,
3639
                           unsigned long depth,
3640
                           unsigned int step,
3641
                           unsigned int max_sample,
3642
                           tupletable2 * const colorfreqtableP,
3643
                           struct histogram_control const *control,
3644
                           int use_reversible,
3645
                           sixel_allocator_t *allocator);
3646

3647
static SIXELSTATUS
3648
computeHistogram_hopscotch(unsigned char const *data,
3649
                           unsigned int length,
3650
                           unsigned long depth,
3651
                           unsigned int step,
3652
                           unsigned int max_sample,
3653
                           tupletable2 * const colorfreqtableP,
3654
                           struct histogram_control const *control,
3655
                           int use_reversible,
3656
                           sixel_allocator_t *allocator);
3657

3658
static SIXELSTATUS
3659
computeHistogram(unsigned char const    /* in */  *data,
260✔
3660
                 unsigned int           /* in */  length,
3661
                 unsigned long const    /* in */  depth,
3662
                 tupletable2 * const    /* out */ colorfreqtableP,
3663
                 int const              /* in */  qualityMode,
3664
                 int const              /* in */  use_reversible,
3665
                 sixel_allocator_t      /* in */  *allocator)
3666
{
3667
    SIXELSTATUS status = SIXEL_FALSE;
260✔
3668
    typedef uint32_t unit_t;
3669
    unsigned int i, n;
3670
    unit_t *histogram = NULL;
260✔
3671
    unit_t *refmap = NULL;
260✔
3672
    unit_t *ref;
3673
    unsigned int bucket_index;
3674
    unsigned int step;
3675
    unsigned int max_sample;
3676
    size_t hist_size;
3677
    unit_t bucket_value;
3678
    unsigned int component;
3679
    unsigned int reconstructed;
3680
    struct histogram_control control;
3681
    unsigned int depth_u;
3682
    unsigned char reversible_pixel[4];
3683

3684
    switch (qualityMode) {
260!
3685
    case SIXEL_QUALITY_LOW:
120✔
3686
        max_sample = 18383;
227✔
3687
        break;
227✔
3688
    case SIXEL_QUALITY_HIGH:
20✔
3689
        max_sample = 1118383;
30✔
3690
        break;
30✔
3691
    case SIXEL_QUALITY_FULL:
3✔
3692
    default:
3693
        max_sample = 4003079;
3✔
3694
        break;
3✔
3695
    }
3696

3697
    step = length / depth / max_sample * depth;
260✔
3698
    if (step <= 0) {
260✔
3699
        step = depth;
156✔
3700
    }
52✔
3701

3702
    quant_trace(stderr, "making histogram...\n");
260✔
3703

3704
    depth_u = (unsigned int)depth;
260✔
3705
    control = histogram_control_make(depth_u);
260✔
3706
    if (use_reversible) {
260!
3707
        control.reversible_rounding = 1;
×
3708
    }
3709
    if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD
260!
3710
        || histogram_lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
260!
3711
        if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD) {
×
3712
            status = computeHistogram_robinhood(data,
×
3713
                                                length,
3714
                                                depth,
3715
                                                step,
3716
                                                max_sample,
3717
                                                colorfreqtableP,
3718
                                                &control,
3719
                                                use_reversible,
3720
                                                allocator);
3721
        } else {
3722
            status = computeHistogram_hopscotch(data,
×
3723
                                                length,
3724
                                                depth,
3725
                                                step,
3726
                                                max_sample,
3727
                                                colorfreqtableP,
3728
                                                &control,
3729
                                                use_reversible,
3730
                                                allocator);
3731
        }
3732
        goto end;
×
3733
    }
3734

3735
    hist_size = histogram_dense_size((unsigned int)depth, &control);
260✔
3736
    histogram = (unit_t *)sixel_allocator_calloc(allocator,
378✔
3737
                                                 hist_size,
118✔
3738
                                                 sizeof(unit_t));
3739
    if (histogram == NULL) {
260!
3740
        sixel_helper_set_additional_message(
×
3741
            "unable to allocate memory for histogram.");
3742
        status = SIXEL_BAD_ALLOCATION;
×
3743
        goto end;
×
3744
    }
3745
    ref = refmap = (unit_t *)sixel_allocator_malloc(allocator,
378✔
3746
                                                    hist_size *
118✔
3747
                                                    sizeof(unit_t));
3748
    if (refmap == NULL) {
260!
3749
        sixel_helper_set_additional_message(
×
3750
            "unable to allocate memory for lookup table.");
3751
        status = SIXEL_BAD_ALLOCATION;
×
3752
        goto end;
×
3753
    }
3754

3755
    for (i = 0; i < length; i += step) {
3,572,726✔
3756
        if (use_reversible) {
3,572,466!
3757
            sixel_quant_reversible_pixel(data + i,
×
3758
                                         depth_u,
3759
                                         reversible_pixel);
3760
            bucket_index = histogram_pack_color(reversible_pixel,
×
3761
                                                depth_u,
3762
                                                &control);
3763
        } else {
3764
            bucket_index = histogram_pack_color(data + i,
5,349,672✔
3765
                                                depth_u,
1,777,206✔
3766
                                                &control);
3767
        }
3768
        if (histogram[bucket_index] == 0) {
3,572,466✔
3769
            *ref++ = bucket_index;
294,113✔
3770
        }
98,297✔
3771
        if (histogram[bucket_index] < UINT32_MAX) {
3,572,466!
3772
            histogram[bucket_index]++;
3,572,466✔
3773
        }
1,777,206✔
3774
    }
1,777,206✔
3775

3776
    colorfreqtableP->size = (unsigned int)(ref - refmap);
260✔
3777

3778
    status = alloctupletable(&colorfreqtableP->table,
378✔
3779
                             depth,
118✔
3780
                             (unsigned int)(ref - refmap),
260✔
3781
                             allocator);
118✔
3782
    if (SIXEL_FAILED(status)) {
260!
3783
        goto end;
×
3784
    }
3785
    for (i = 0; i < colorfreqtableP->size; ++i) {
294,373✔
3786
        bucket_value = refmap[i];
294,113✔
3787
        if (histogram[bucket_value] > 0) {
294,113!
3788
            colorfreqtableP->table[i]->value = histogram[bucket_value];
294,113✔
3789
            for (n = 0; n < depth; n++) {
1,176,452✔
3790
                component = (unsigned int)
882,339✔
3791
                    ((bucket_value >> (n * control.channel_bits)) &
1,177,230✔
3792
                     control.channel_mask);
882,339✔
3793
                reconstructed = histogram_reconstruct(component,
882,339✔
3794
                                                      &control);
3795
                if (use_reversible) {
882,339!
3796
                    reconstructed =
×
3797
                        (unsigned int)sixel_quant_reversible_value(
×
3798
                            reconstructed);
3799
                }
3800
                colorfreqtableP->table[i]->tuple[depth - 1 - n]
882,339✔
3801
                    = (sample)reconstructed;
1,177,230✔
3802
            }
294,891✔
3803
        }
98,297✔
3804
    }
98,297✔
3805

3806
    quant_trace(stderr, "%u colors found\n", colorfreqtableP->size);
260✔
3807

3808
    status = SIXEL_OK;
260✔
3809

3810
end:
142✔
3811
    sixel_allocator_free(allocator, refmap);
260✔
3812
    sixel_allocator_free(allocator, histogram);
260✔
3813

3814
    return status;
260✔
3815
}
3816

3817
static SIXELSTATUS
3818
computeHistogram_robinhood(unsigned char const *data,
×
3819
                           unsigned int length,
3820
                           unsigned long depth,
3821
                           unsigned int step,
3822
                           unsigned int max_sample,
3823
                           tupletable2 * const colorfreqtableP,
3824
                           struct histogram_control const *control,
3825
                           int use_reversible,
3826
                           sixel_allocator_t *allocator)
3827
{
3828
    SIXELSTATUS status = SIXEL_FALSE;
×
3829
    struct robinhood_table32 table;
3830
    size_t expected;
3831
    size_t cap_limit;
3832
    size_t index;
3833
    unsigned int depth_u;
3834
    unsigned int i;
3835
    unsigned int n;
3836
    uint32_t bucket_color;
3837
    uint32_t bucket_hash;
3838
    uint32_t entry_color;
3839
    struct robinhood_slot32 *slot;
3840
    unsigned int component;
3841
    unsigned int reconstructed;
3842
    unsigned char reversible_pixel[4];
3843

3844
    /*
3845
     * The ASCII sketch below shows how the sparse table stores samples:
3846
     *
3847
     *   [hash]->(key,value,distance)  Robin Hood probing keeps dense tails.
3848
     */
3849
    table.slots = NULL;
×
3850
    table.capacity = 0U;
×
3851
    table.count = 0U;
×
3852
    table.allocator = allocator;
×
3853
    cap_limit = (size_t)1U << 20;
×
3854
    expected = max_sample;
×
3855
    if (expected < 256U) {
×
3856
        expected = 256U;
×
3857
    }
3858
    if (expected > cap_limit) {
×
3859
        expected = cap_limit;
×
3860
    }
3861

3862
    status = robinhood_table32_init(&table, expected, allocator);
×
3863
    if (SIXEL_FAILED(status)) {
×
3864
        sixel_helper_set_additional_message(
×
3865
            "unable to allocate robinhood histogram.");
3866
        goto end;
×
3867
    }
3868

3869
    depth_u = (unsigned int)depth;
×
3870
    for (i = 0U; i < length; i += step) {
×
3871
        if (use_reversible) {
×
3872
            sixel_quant_reversible_pixel(data + i, depth_u,
×
3873
                                         reversible_pixel);
3874
            bucket_color = histogram_pack_color(reversible_pixel,
×
3875
                                                depth_u, control);
3876
        } else {
3877
            bucket_color = histogram_pack_color(data + i,
×
3878
                                                depth_u, control);
3879
        }
3880
        bucket_hash = histogram_hash_mix(bucket_color);
×
3881
        /*
3882
         * Hash probing uses the mixed key while the slot stores the
3883
         * original quantized RGB value:
3884
         *
3885
         *   hash --> [slot]
3886
         *             |color|count|
3887
         */
3888
        slot = robinhood_table32_lookup(&table,
×
3889
                                        bucket_hash,
3890
                                        bucket_color);
3891
        if (slot == NULL) {
×
3892
            status = robinhood_table32_insert(&table,
×
3893
                                              bucket_hash,
3894
                                              bucket_color,
3895
                                              1U);
3896
            if (SIXEL_FAILED(status)) {
×
3897
                sixel_helper_set_additional_message(
×
3898
                    "unable to grow robinhood histogram.");
3899
                goto end;
×
3900
            }
3901
        } else if (slot->value < UINT32_MAX) {
×
3902
            slot->value++;
×
3903
        }
3904
    }
3905

3906
    if (table.count > UINT_MAX) {
×
3907
        sixel_helper_set_additional_message(
×
3908
            "too many unique colors for histogram.");
3909
        status = SIXEL_BAD_ARGUMENT;
×
3910
        goto end;
×
3911
    }
3912

3913
    colorfreqtableP->size = (unsigned int)table.count;
×
3914
    status = alloctupletable(&colorfreqtableP->table,
×
3915
                             depth_u,
3916
                             (unsigned int)table.count,
×
3917
                             allocator);
3918
    if (SIXEL_FAILED(status)) {
×
3919
        goto end;
×
3920
    }
3921

3922
    index = 0U;
×
3923
    /*
3924
     * Stream slots in the hash traversal order to avoid qsort overhead.
3925
     * This favors throughput over identical palette ordering.
3926
     */
3927
    for (i = 0U; i < table.capacity; ++i) {
×
3928
        slot = &table.slots[i];
×
3929
        if (slot->value == 0U) {
×
3930
            continue;
×
3931
        }
3932
        if (index >= colorfreqtableP->size) {
×
3933
            break;
×
3934
        }
3935
        entry_color = slot->color;
×
3936
        colorfreqtableP->table[index]->value = slot->value;
×
3937
        for (n = 0U; n < depth_u; ++n) {
×
3938
            component = (unsigned int)
×
3939
                ((entry_color >> (n * control->channel_bits))
×
3940
                 & control->channel_mask);
×
3941
            reconstructed = histogram_reconstruct(component, control);
×
3942
            if (use_reversible) {
×
3943
                reconstructed =
×
3944
                    (unsigned int)sixel_quant_reversible_value(
×
3945
                        reconstructed);
3946
            }
3947
            colorfreqtableP->table[index]->tuple[depth_u - 1U - n]
×
3948
                = (sample)reconstructed;
×
3949
        }
3950
        index++;
×
3951
    }
3952

3953
    status = SIXEL_OK;
×
3954

3955
end:
3956
    robinhood_table32_fini(&table);
×
3957

3958
    return status;
×
3959
}
3960

3961
static SIXELSTATUS
3962
computeHistogram_hopscotch(unsigned char const *data,
×
3963
                           unsigned int length,
3964
                           unsigned long depth,
3965
                           unsigned int step,
3966
                           unsigned int max_sample,
3967
                           tupletable2 * const colorfreqtableP,
3968
                           struct histogram_control const *control,
3969
                           int use_reversible,
3970
                           sixel_allocator_t *allocator)
3971
{
3972
    SIXELSTATUS status = SIXEL_FALSE;
×
3973
    struct hopscotch_table32 table;
3974
    size_t expected;
3975
    size_t cap_limit;
3976
    size_t index;
3977
    unsigned int depth_u;
3978
    unsigned int i;
3979
    unsigned int n;
3980
    uint32_t bucket_color;
3981
    uint32_t bucket_hash;
3982
    uint32_t entry_color;
3983
    struct hopscotch_slot32 *slot;
3984
    unsigned int component;
3985
    unsigned int reconstructed;
3986
    unsigned char reversible_pixel[4];
3987

3988
    /*
3989
     * Hopscotch hashing stores the local neighbourhood using the map below:
3990
     *
3991
     *   [home] hopinfo bits ---> |slot+0|slot+1|slot+2| ...
3992
     *                              ^ entries hop within this window.
3993
     */
3994
    table.slots = NULL;
×
3995
    table.hopinfo = NULL;
×
3996
    table.capacity = 0U;
×
3997
    table.count = 0U;
×
3998
    table.neighborhood = HOPSCOTCH_DEFAULT_NEIGHBORHOOD;
×
3999
    table.allocator = allocator;
×
4000
    cap_limit = (size_t)1U << 20;
×
4001
    expected = max_sample;
×
4002
    if (expected < 256U) {
×
4003
        expected = 256U;
×
4004
    }
4005
    if (expected > cap_limit) {
×
4006
        expected = cap_limit;
×
4007
    }
4008

4009
    status = hopscotch_table32_init(&table, expected, allocator);
×
4010
    if (SIXEL_FAILED(status)) {
×
4011
        sixel_helper_set_additional_message(
×
4012
            "unable to allocate hopscotch histogram.");
4013
        goto end;
×
4014
    }
4015

4016
    depth_u = (unsigned int)depth;
×
4017
    for (i = 0U; i < length; i += step) {
×
4018
        if (use_reversible) {
×
4019
            sixel_quant_reversible_pixel(data + i, depth_u,
×
4020
                                         reversible_pixel);
4021
            bucket_color = histogram_pack_color(reversible_pixel,
×
4022
                                                depth_u, control);
4023
        } else {
4024
            bucket_color = histogram_pack_color(data + i,
×
4025
                                                depth_u, control);
4026
        }
4027
        bucket_hash = histogram_hash_mix(bucket_color);
×
4028
        /*
4029
         * Hopscotch buckets mirror the robinhood layout, keeping the
4030
         * quantized color next to the count so we never derive it from
4031
         * the scrambled hash key.
4032
         */
4033
        slot = hopscotch_table32_lookup(&table,
×
4034
                                        bucket_hash,
4035
                                        bucket_color);
4036
        if (slot == NULL) {
×
4037
            status = hopscotch_table32_insert(&table,
×
4038
                                              bucket_hash,
4039
                                              bucket_color,
4040
                                              1U);
4041
            if (SIXEL_FAILED(status)) {
×
4042
                sixel_helper_set_additional_message(
×
4043
                    "unable to grow hopscotch histogram.");
4044
                goto end;
×
4045
            }
4046
        } else if (slot->value < UINT32_MAX) {
×
4047
            slot->value++;
×
4048
        }
4049
    }
4050

4051
    if (table.count > UINT_MAX) {
×
4052
        sixel_helper_set_additional_message(
×
4053
            "too many unique colors for histogram.");
4054
        status = SIXEL_BAD_ARGUMENT;
×
4055
        goto end;
×
4056
    }
4057

4058
    colorfreqtableP->size = (unsigned int)table.count;
×
4059
    status = alloctupletable(&colorfreqtableP->table,
×
4060
                             depth_u,
4061
                             (unsigned int)table.count,
×
4062
                             allocator);
4063
    if (SIXEL_FAILED(status)) {
×
4064
        goto end;
×
4065
    }
4066

4067
    index = 0U;
×
4068
    /*
4069
     * Stream slots in the hash traversal order to avoid qsort overhead.
4070
     * This favors throughput over identical palette ordering.
4071
     */
4072
    for (i = 0U; i < table.capacity; ++i) {
×
4073
        slot = &table.slots[i];
×
4074
        if (slot->key == HOPSCOTCH_EMPTY_KEY || slot->value == 0U) {
×
4075
            continue;
×
4076
        }
4077
        if (index >= colorfreqtableP->size) {
×
4078
            break;
×
4079
        }
4080
        entry_color = slot->color;
×
4081
        colorfreqtableP->table[index]->value = slot->value;
×
4082
        for (n = 0U; n < depth_u; ++n) {
×
4083
            component = (unsigned int)
×
4084
                ((entry_color >> (n * control->channel_bits))
×
4085
                 & control->channel_mask);
×
4086
            reconstructed = histogram_reconstruct(component, control);
×
4087
            if (use_reversible) {
×
4088
                reconstructed =
×
4089
                    (unsigned int)sixel_quant_reversible_value(
×
4090
                        reconstructed);
4091
            }
4092
            colorfreqtableP->table[index]->tuple[depth_u - 1U - n]
×
4093
                = (sample)reconstructed;
×
4094
        }
4095
        index++;
×
4096
    }
4097

4098
    status = SIXEL_OK;
×
4099

4100
end:
4101
    hopscotch_table32_fini(&table);
×
4102

4103
    return status;
×
4104
}
4105

4106
SIXELSTATUS
4107
sixel_quant_cache_prepare(unsigned short **cachetable,
267✔
4108
                          size_t *cachetable_size,
4109
                          int lut_policy,
4110
                          int reqcolor,
4111
                          sixel_allocator_t *allocator)
4112
{
4113
    SIXELSTATUS status;
4114
    struct histogram_control control;
4115
    struct cuckoo_table32 *table;
4116
    size_t expected;
4117
    size_t buckets;
4118
    size_t cap_limit;
4119
    int normalized;
4120

4121
    if (cachetable == NULL || allocator == NULL) {
267!
4122
        return SIXEL_BAD_ARGUMENT;
×
4123
    }
4124

4125
    /*
4126
     * The cache pointer always references the same ladder:
4127
     *
4128
     *   cache -> cuckoo_table32 -> buckets
4129
     *                           -> stash
4130
     */
4131
    normalized = lut_policy;
267✔
4132
    if (normalized == SIXEL_LUT_POLICY_AUTO) {
267!
4133
        normalized = histogram_lut_policy;
267✔
4134
    }
109✔
4135
    if (normalized == SIXEL_LUT_POLICY_AUTO) {
267!
4136
        normalized = SIXEL_LUT_POLICY_6BIT;
267✔
4137
    }
109✔
4138

4139
    control = histogram_control_make_for_policy(3U, normalized);
267✔
4140
    if (control.channel_shift == 0U) {
267!
4141
        expected = (size_t)reqcolor * 64U;
×
4142
        cap_limit = (size_t)1U << 20;
×
4143
        if (expected < 512U) {
×
4144
            expected = 512U;
×
4145
        }
4146
        if (expected > cap_limit) {
×
4147
            expected = cap_limit;
×
4148
        }
4149
    } else {
4150
        expected = histogram_dense_size(3U, &control);
267✔
4151
        if (expected == 0U) {
267!
4152
            expected = CUCKOO_BUCKET_SIZE;
×
4153
        }
4154
    }
4155

4156
    table = (struct cuckoo_table32 *)*cachetable;
267✔
4157
    if (table != NULL) {
267✔
4158
        buckets = cuckoo_round_buckets(expected);
3✔
4159
        if (table->bucket_count < buckets) {
3!
4160
            cuckoo_table32_fini(table);
×
4161
            sixel_allocator_free(allocator, table);
×
4162
            table = NULL;
×
4163
            *cachetable = NULL;
×
4164
        } else {
4165
            cuckoo_table32_clear(table);
3✔
4166
        }
4167
    }
1✔
4168
    if (table == NULL) {
267✔
4169
        table = (struct cuckoo_table32 *)sixel_allocator_malloc(
264✔
4170
            allocator, sizeof(struct cuckoo_table32));
108✔
4171
        if (table == NULL) {
264!
4172
            sixel_helper_set_additional_message(
×
4173
                "unable to allocate cuckoo cache state.");
4174
            return SIXEL_BAD_ALLOCATION;
×
4175
        }
4176
        memset(table, 0, sizeof(struct cuckoo_table32));
264✔
4177
        status = cuckoo_table32_init(table, expected, allocator);
264✔
4178
        if (SIXEL_FAILED(status)) {
264!
4179
            sixel_allocator_free(allocator, table);
×
4180
            sixel_helper_set_additional_message(
×
4181
                "unable to initialize cuckoo cache.");
4182
            return status;
×
4183
        }
4184
        *cachetable = (unsigned short *)table;
264✔
4185
    }
108✔
4186

4187
    if (cachetable_size != NULL) {
267!
4188
        *cachetable_size = table->bucket_count * CUCKOO_BUCKET_SIZE;
267✔
4189
    }
109✔
4190

4191
    return SIXEL_OK;
267✔
4192
}
109✔
4193

4194
void
4195
sixel_quant_cache_clear(unsigned short *cachetable,
264✔
4196
                        int lut_policy)
4197
{
4198
    struct cuckoo_table32 *table;
4199

4200
    (void)lut_policy;
108✔
4201
    if (cachetable == NULL) {
264!
4202
        return;
×
4203
    }
4204

4205
    table = (struct cuckoo_table32 *)cachetable;
264✔
4206
    cuckoo_table32_clear(table);
264✔
4207
}
108✔
4208

4209
void
4210
sixel_quant_cache_release(unsigned short *cachetable,
529✔
4211
                          int lut_policy,
4212
                          sixel_allocator_t *allocator)
4213
{
4214
    struct cuckoo_table32 *table;
4215

4216
    (void)lut_policy;
183✔
4217
    if (cachetable == NULL || allocator == NULL) {
529!
4218
        return;
265✔
4219
    }
4220

4221
    table = (struct cuckoo_table32 *)cachetable;
264✔
4222
    cuckoo_table32_fini(table);
264✔
4223
    sixel_allocator_free(allocator, table);
264✔
4224
}
183✔
4225

4226

4227
static int
4228
computeColorMapFromInput(unsigned char const *data,
260✔
4229
                         unsigned int const length,
4230
                         unsigned int const depth,
4231
                         unsigned int const reqColors,
4232
                         int const methodForLargest,
4233
                         int const methodForRep,
4234
                         int const qualityMode,
4235
                         int const force_palette,
4236
                         int const use_reversible,
4237
                         int const final_merge_mode,
4238
                         tupletable2 * const colormapP,
4239
                         unsigned int *origcolors,
4240
                         sixel_allocator_t *allocator)
4241
{
4242
/*----------------------------------------------------------------------------
4243
   Produce a colormap containing the best colors to represent the
4244
   image stream in file 'ifP'.  Figure it out using the median cut
4245
   technique.
4246

4247
   The colormap will have 'reqcolors' or fewer colors in it, unless
4248
   'allcolors' is true, in which case it will have all the colors that
4249
   are in the input.
4250

4251
   The colormap has the same maxval as the input.
4252

4253
   Put the colormap in newly allocated storage as a tupletable2
4254
   and return its address as *colormapP.  Return the number of colors in
4255
   it as *colorsP and its maxval as *colormapMaxvalP.
4256

4257
   Return the characteristics of the input file as
4258
   *formatP and *freqPamP.  (This information is not really
4259
   relevant to our colormap mission; just a fringe benefit).
4260
-----------------------------------------------------------------------------*/
4261
    SIXELSTATUS status = SIXEL_FALSE;
260✔
4262
    tupletable2 colorfreqtable = {0, NULL};
260✔
4263
    unsigned int i;
4264
    unsigned int n;
4265

4266
    status = computeHistogram(data, length, depth,
378✔
4267
                              &colorfreqtable, qualityMode,
118✔
4268
                              use_reversible, allocator);
118✔
4269
    if (SIXEL_FAILED(status)) {
260!
4270
        goto end;
×
4271
    }
4272
    if (origcolors) {
260!
4273
        *origcolors = colorfreqtable.size;
260✔
4274
    }
118✔
4275

4276
    if (colorfreqtable.size <= reqColors) {
260✔
4277
        quant_trace(stderr,
286✔
4278
                    "Image already has few enough colors (<=%d).  "
4279
                    "Keeping same colors.\n", reqColors);
95✔
4280
        /* *colormapP = colorfreqtable; */
4281
        colormapP->size = colorfreqtable.size;
191✔
4282
        status = alloctupletable(&colormapP->table, depth, colorfreqtable.size, allocator);
191✔
4283
        if (SIXEL_FAILED(status)) {
191!
4284
            goto end;
×
4285
        }
4286
        for (i = 0; i < colorfreqtable.size; ++i) {
3,205✔
4287
            colormapP->table[i]->value = colorfreqtable.table[i]->value;
3,014✔
4288
            for (n = 0; n < depth; ++n) {
12,056✔
4289
                colormapP->table[i]->tuple[n] = colorfreqtable.table[i]->tuple[n];
9,042✔
4290
            }
4,224✔
4291
            if (use_reversible) {
3,014!
4292
                sixel_quant_reversible_tuple(colormapP->table[i]->tuple,
×
4293
                                             depth);
4294
            }
4295
        }
1,408✔
4296
    } else {
95✔
4297
        quant_trace(stderr, "choosing %d colors...\n", reqColors);
69✔
4298
        status = mediancut(colorfreqtable, depth, reqColors,
92✔
4299
                           methodForLargest, methodForRep,
23✔
4300
                           use_reversible, final_merge_mode,
23✔
4301
                           colormapP, allocator);
23✔
4302
        if (SIXEL_FAILED(status)) {
69!
4303
            goto end;
×
4304
        }
4305
        quant_trace(stderr, "%d colors are choosed.\n", colorfreqtable.size);
69✔
4306
    }
4307

4308
    if (force_palette) {
260!
4309
        status = force_palette_completion(colormapP, depth, reqColors,
×
4310
                                          colorfreqtable, allocator);
4311
        if (SIXEL_FAILED(status)) {
×
4312
            goto end;
×
4313
        }
4314
    }
4315

4316
    status = SIXEL_OK;
260✔
4317

4318
end:
142✔
4319
    sixel_allocator_free(allocator, colorfreqtable.table);
260✔
4320
    return status;
260✔
4321
}
4322

4323

4324
/* diffuse error energy to surround pixels (normal strategy) */
4325
static void
4326
error_diffuse_normal(
281,667,771✔
4327
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4328
    int           /* in */    pos,        /* address of the destination pixel */
4329
    int           /* in */    depth,      /* color depth in bytes */
4330
    int           /* in */    error,      /* error energy */
4331
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4332
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4333
{
4334
    int c;
4335

4336
    data += pos * depth;
281,667,771✔
4337

4338
    c = *data + (error * numerator * 2 / denominator + 1) / 2;
281,667,771✔
4339
    if (c < 0) {
281,667,771✔
4340
        c = 0;
2,567,509✔
4341
    }
811,245✔
4342
    if (c >= 1 << 8) {
281,667,771✔
4343
        c = (1 << 8) - 1;
959,648✔
4344
    }
304,614✔
4345
    *data = (unsigned char)c;
281,667,771✔
4346
}
281,667,771✔
4347

4348
/* error diffusion with fast strategy */
4349
static void
4350
error_diffuse_fast(
169,469,067✔
4351
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4352
    int           /* in */    pos,        /* address of the destination pixel */
4353
    int           /* in */    depth,      /* color depth in bytes */
4354
    int           /* in */    error,      /* error energy */
4355
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4356
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4357
{
4358
    int c;
4359

4360
    data += pos * depth;
169,469,067✔
4361

4362
    c = *data + error * numerator / denominator;
169,469,067✔
4363
    if (c < 0) {
169,469,067✔
4364
        c = 0;
7,291,990✔
4365
    }
2,721,044✔
4366
    if (c >= 1 << 8) {
169,469,067✔
4367
        c = (1 << 8) - 1;
459,666✔
4368
    }
150,804✔
4369
    *data = (unsigned char)c;
169,469,067✔
4370
}
169,469,067✔
4371

4372

4373
/* error diffusion with precise strategy */
4374
static void
4375
error_diffuse_precise(
6,111,369✔
4376
    unsigned char /* in */    *data,      /* base address of pixel buffer */
4377
    int           /* in */    pos,        /* address of the destination pixel */
4378
    int           /* in */    depth,      /* color depth in bytes */
4379
    int           /* in */    error,      /* error energy */
4380
    int           /* in */    numerator,  /* numerator of diffusion coefficient */
4381
    int           /* in */    denominator /* denominator of diffusion coefficient */)
4382
{
4383
    int c;
4384

4385
    data += pos * depth;
6,111,369✔
4386

4387
    c = (int)(*data + error * numerator / (double)denominator + 0.5);
6,111,369✔
4388
    if (c < 0) {
6,111,369✔
4389
        c = 0;
70,455✔
4390
    }
25,487✔
4391
    if (c >= 1 << 8) {
6,111,369!
4392
        c = (1 << 8) - 1;
×
4393
    }
4394
    *data = (unsigned char)c;
6,111,369✔
4395
}
6,111,369✔
4396

4397

4398
typedef void (*diffuse_fixed_carry_mode)(int32_t *carry_curr,
4399
                                         int32_t *carry_next,
4400
                                         int32_t *carry_far,
4401
                                         int width,
4402
                                         int height,
4403
                                         int depth,
4404
                                         int x,
4405
                                         int y,
4406
                                         int32_t error,
4407
                                         int direction,
4408
                                         int channel);
4409

4410

4411
typedef void (*diffuse_varerr_mode)(unsigned char *data,
4412
                                    int width,
4413
                                    int height,
4414
                                    int x,
4415
                                    int y,
4416
                                    int depth,
4417
                                    int32_t error,
4418
                                    int index,
4419
                                    int direction);
4420

4421
typedef void (*diffuse_varerr_carry_mode)(int32_t *carry_curr,
4422
                                          int32_t *carry_next,
4423
                                          int32_t *carry_far,
4424
                                          int width,
4425
                                          int height,
4426
                                          int depth,
4427
                                          int x,
4428
                                          int y,
4429
                                          int32_t error,
4430
                                          int index,
4431
                                          int direction,
4432
                                          int channel);
4433

4434

4435
static int32_t
4436
diffuse_varerr_term(int32_t error, int weight, int denom)
×
4437
{
4438
    int64_t delta;
4439

4440
    delta = (int64_t)error * (int64_t)weight;
×
4441
    if (delta >= 0) {
×
4442
        delta = (delta + denom / 2) / denom;
×
4443
    } else {
4444
        delta = (delta - denom / 2) / denom;
×
4445
    }
4446

4447
    return (int32_t)delta;
×
4448
}
4449

4450

4451
static int32_t
4452
diffuse_fixed_term(int32_t error, int numerator, int denominator)
×
4453
{
4454
    int64_t delta;
4455

4456
    delta = (int64_t)error * (int64_t)numerator;
×
4457
    if (delta >= 0) {
×
4458
        delta = (delta + denominator / 2) / denominator;
×
4459
    } else {
4460
        delta = (delta - denominator / 2) / denominator;
×
4461
    }
4462

4463
    return (int32_t)delta;
×
4464
}
4465

4466

4467
static void
4468
diffuse_varerr_apply_direct(unsigned char *target, int depth, size_t offset,
×
4469
                            int32_t delta)
4470
{
4471
    int64_t value;
4472
    int result;
4473

4474
    value = (int64_t)target[offset * depth] << VARERR_SCALE_SHIFT;
×
4475
    value += delta;
×
4476
    if (value < 0) {
×
4477
        value = 0;
×
4478
    } else {
4479
        int64_t max_value;
4480

4481
        max_value = VARERR_MAX_VALUE;
×
4482
        if (value > max_value) {
×
4483
            value = max_value;
×
4484
        }
4485
    }
4486

4487
    result = (int)((value + VARERR_ROUND) >> VARERR_SCALE_SHIFT);
×
4488
    if (result < 0) {
×
4489
        result = 0;
×
4490
    }
4491
    if (result > 255) {
×
4492
        result = 255;
×
4493
    }
4494
    target[offset * depth] = (unsigned char)result;
×
4495
}
×
4496

4497

4498
static void
4499
diffuse_lso2(unsigned char *data, int width, int height,
×
4500
             int x, int y, int depth, int32_t error,
4501
             int index, int direction)
4502
{
4503
    const int (*table)[7];
4504
    const int *entry;
4505
    int denom;
4506
    int32_t term_r = 0;
×
4507
    int32_t term_r2 = 0;
×
4508
    int32_t term_dl = 0;
×
4509
    int32_t term_d = 0;
×
4510
    int32_t term_dr = 0;
×
4511
    int32_t term_d2 = 0;
×
4512
    size_t offset;
4513

4514
    if (error == 0) {
×
4515
        return;
×
4516
    }
4517
    if (index < 0) {
×
4518
        index = 0;
×
4519
    }
4520
    if (index > 255) {
×
4521
        index = 255;
×
4522
    }
4523

4524
    table = lso2_table();
×
4525
    entry = table[index];
×
4526
    denom = entry[6];
×
4527
    if (denom == 0) {
×
4528
        return;
×
4529
    }
4530

4531
    term_r = diffuse_varerr_term(error, entry[0], denom);
×
4532
    term_r2 = diffuse_varerr_term(error, entry[1], denom);
×
4533
    term_dl = diffuse_varerr_term(error, entry[2], denom);
×
4534
    term_d = diffuse_varerr_term(error, entry[3], denom);
×
4535
    term_dr = diffuse_varerr_term(error, entry[4], denom);
×
4536
    term_d2 = diffuse_varerr_term(error, entry[5], denom);
×
4537

4538

4539
    if (direction >= 0) {
×
4540
        if (x + 1 < width) {
×
4541
            offset = (size_t)y * (size_t)width + (size_t)(x + 1);
×
4542
            diffuse_varerr_apply_direct(data, depth, offset, term_r);
×
4543
        }
4544
        if (x + 2 < width) {
×
4545
            offset = (size_t)y * (size_t)width + (size_t)(x + 2);
×
4546
            diffuse_varerr_apply_direct(data, depth, offset, term_r2);
×
4547
        }
4548
        if (y + 1 < height && x - 1 >= 0) {
×
4549
            offset = (size_t)(y + 1) * (size_t)width;
×
4550
            offset += (size_t)(x - 1);
×
4551
            diffuse_varerr_apply_direct(data, depth, offset, term_dl);
×
4552
        }
4553
        if (y + 1 < height) {
×
4554
            offset = (size_t)(y + 1) * (size_t)width + (size_t)x;
×
4555
            diffuse_varerr_apply_direct(data, depth, offset, term_d);
×
4556
        }
4557
        if (y + 1 < height && x + 1 < width) {
×
4558
            offset = (size_t)(y + 1) * (size_t)width;
×
4559
            offset += (size_t)(x + 1);
×
4560
            diffuse_varerr_apply_direct(data, depth, offset, term_dr);
×
4561
        }
4562
        if (y + 2 < height) {
×
4563
            offset = (size_t)(y + 2) * (size_t)width + (size_t)x;
×
4564
            diffuse_varerr_apply_direct(data, depth, offset, term_d2);
×
4565
        }
4566
    } else {
4567
        if (x - 1 >= 0) {
×
4568
            offset = (size_t)y * (size_t)width + (size_t)(x - 1);
×
4569
            diffuse_varerr_apply_direct(data, depth, offset, term_r);
×
4570
        }
4571
        if (x - 2 >= 0) {
×
4572
            offset = (size_t)y * (size_t)width + (size_t)(x - 2);
×
4573
            diffuse_varerr_apply_direct(data, depth, offset, term_r2);
×
4574
        }
4575
        if (y + 1 < height && x + 1 < width) {
×
4576
            offset = (size_t)(y + 1) * (size_t)width;
×
4577
            offset += (size_t)(x + 1);
×
4578
            diffuse_varerr_apply_direct(data, depth, offset, term_dl);
×
4579
        }
4580
        if (y + 1 < height) {
×
4581
            offset = (size_t)(y + 1) * (size_t)width + (size_t)x;
×
4582
            diffuse_varerr_apply_direct(data, depth, offset, term_d);
×
4583
        }
4584
        if (y + 1 < height && x - 1 >= 0) {
×
4585
            offset = (size_t)(y + 1) * (size_t)width;
×
4586
            offset += (size_t)(x - 1);
×
4587
            diffuse_varerr_apply_direct(data, depth, offset, term_dr);
×
4588
        }
4589
        if (y + 2 < height) {
×
4590
            offset = (size_t)(y + 2) * (size_t)width + (size_t)x;
×
4591
            diffuse_varerr_apply_direct(data, depth, offset, term_d2);
×
4592
        }
4593
    }
4594
}
4595

4596

4597
static void
4598
diffuse_lso2_carry(int32_t *carry_curr, int32_t *carry_next, int32_t *carry_far,
×
4599
                   int width, int height, int depth,
4600
                   int x, int y, int32_t error,
4601
                   int index, int direction, int channel)
4602
{
4603
    const int (*table)[7];
4604
    const int *entry;
4605
    int denom;
4606
    int32_t term_r = 0;
×
4607
    int32_t term_r2 = 0;
×
4608
    int32_t term_dl = 0;
×
4609
    int32_t term_d = 0;
×
4610
    int32_t term_dr = 0;
×
4611
    int32_t term_d2 = 0;
×
4612
    size_t base;
4613

4614
    if (error == 0) {
×
4615
        return;
×
4616
    }
4617
    if (index < 0) {
×
4618
        index = 0;
×
4619
    }
4620
    if (index > 255) {
×
4621
        index = 255;
×
4622
    }
4623

4624
    table = lso2_table();
×
4625
    entry = table[index];
×
4626
    denom = entry[6];
×
4627
    if (denom == 0) {
×
4628
        return;
×
4629
    }
4630

4631
    term_r = diffuse_varerr_term(error, entry[0], denom);
×
4632
    term_r2 = diffuse_varerr_term(error, entry[1], denom);
×
4633
    term_dl = diffuse_varerr_term(error, entry[2], denom);
×
4634
    term_d = diffuse_varerr_term(error, entry[3], denom);
×
4635
    term_dr = diffuse_varerr_term(error, entry[4], denom);
×
4636
    term_d2 = error - term_r - term_r2 - term_dl - term_d - term_dr;
×
4637

4638
    if (direction >= 0) {
×
4639
        if (x + 1 < width) {
×
4640
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4641
            carry_curr[base] += term_r;
×
4642
        }
4643
        if (x + 2 < width) {
×
4644
            base = ((size_t)(x + 2) * (size_t)depth) + (size_t)channel;
×
4645
            carry_curr[base] += term_r2;
×
4646
        }
4647
        if (y + 1 < height && x - 1 >= 0) {
×
4648
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4649
            carry_next[base] += term_dl;
×
4650
        }
4651
        if (y + 1 < height) {
×
4652
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4653
            carry_next[base] += term_d;
×
4654
        }
4655
        if (y + 1 < height && x + 1 < width) {
×
4656
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4657
            carry_next[base] += term_dr;
×
4658
        }
4659
        if (y + 2 < height) {
×
4660
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4661
            carry_far[base] += term_d2;
×
4662
        }
4663
    } else {
4664
        if (x - 1 >= 0) {
×
4665
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4666
            carry_curr[base] += term_r;
×
4667
        }
4668
        if (x - 2 >= 0) {
×
4669
            base = ((size_t)(x - 2) * (size_t)depth) + (size_t)channel;
×
4670
            carry_curr[base] += term_r;
×
4671
        }
4672
        if (y + 1 < height && x + 1 < width) {
×
4673
            base = ((size_t)(x + 1) * (size_t)depth) + (size_t)channel;
×
4674
            carry_next[base] += term_dl;
×
4675
        }
4676
        if (y + 1 < height) {
×
4677
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4678
            carry_next[base] += term_d;
×
4679
        }
4680
        if (y + 1 < height && x - 1 >= 0) {
×
4681
            base = ((size_t)(x - 1) * (size_t)depth) + (size_t)channel;
×
4682
            carry_next[base] += term_dr;
×
4683
        }
4684
        if (y + 2 < height) {
×
4685
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
4686
            carry_far[base] += term_d2;
×
4687
        }
4688
    }
4689
}
4690

4691

4692
static void
4693
scanline_params(int serpentine, int index, int limit,
59,650✔
4694
                int *start, int *end, int *step, int *direction)
4695
{
4696
    if (serpentine && (index & 1)) {
59,650✔
4697
        *start = limit - 1;
675✔
4698
        *end = -1;
675✔
4699
        *step = -1;
675✔
4700
        *direction = -1;
675✔
4701
    } else {
225✔
4702
        *start = 0;
58,975✔
4703
        *end = limit;
58,975✔
4704
        *step = 1;
58,975✔
4705
        *direction = 1;
58,975✔
4706
    }
4707
}
59,650✔
4708

4709

4710
static SIXELSTATUS
4711
apply_palette_positional(
×
4712
    sixel_index_t *result,
4713
    unsigned char *data,
4714
    int width,
4715
    int height,
4716
    int depth,
4717
    unsigned char *palette,
4718
    int reqcolor,
4719
    int methodForDiffuse,
4720
    int methodForScan,
4721
    int foptimize_palette,
4722
    int (*f_lookup)(const unsigned char *pixel,
4723
                    int depth,
4724
                    const unsigned char *palette,
4725
                    int reqcolor,
4726
                    unsigned short *cachetable,
4727
                    int complexion),
4728
    unsigned short *indextable,
4729
    int complexion,
4730
    unsigned char copy[],
4731
    unsigned char new_palette[],
4732
    unsigned short migration_map[],
4733
    int *ncolors)
4734
{
4735
    int serpentine;
4736
    int y;
4737
    float (*f_mask)(int x, int y, int c);
4738

4739
    switch (methodForDiffuse) {
×
4740
    case SIXEL_DIFFUSE_A_DITHER:
4741
        f_mask = mask_a;
×
4742
        break;
×
4743
    case SIXEL_DIFFUSE_X_DITHER:
×
4744
    default:
4745
        f_mask = mask_x;
×
4746
        break;
×
4747
    }
4748

4749
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
×
4750

4751
    if (foptimize_palette) {
×
4752
        int x;
4753

4754
        *ncolors = 0;
×
4755
        memset(new_palette, 0x00,
×
4756
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
4757
        memset(migration_map, 0x00,
×
4758
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
4759
        for (y = 0; y < height; ++y) {
×
4760
            int start;
4761
            int end;
4762
            int step;
4763
            int direction;
4764

4765
            scanline_params(serpentine, y, width,
×
4766
                            &start, &end, &step, &direction);
4767
            (void)direction;
4768
            for (x = start; x != end; x += step) {
×
4769
                int pos;
4770
                int d;
4771
                int color_index;
4772

4773
                pos = y * width + x;
×
4774
                for (d = 0; d < depth; ++d) {
×
4775
                    int val;
4776

4777
                    val = data[pos * depth + d]
×
4778
                        + f_mask(x, y, d) * 32;
×
4779
                    copy[d] = val < 0 ? 0
×
4780
                               : val > 255 ? 255 : val;
×
4781
                }
4782
                color_index = f_lookup(copy, depth, palette,
×
4783
                                       reqcolor, indextable,
4784
                                       complexion);
4785
                if (migration_map[color_index] == 0) {
×
4786
                    result[pos] = *ncolors;
×
4787
                    for (d = 0; d < depth; ++d) {
×
4788
                        new_palette[*ncolors * depth + d]
×
4789
                            = palette[color_index * depth + d];
×
4790
                    }
4791
                    ++*ncolors;
×
4792
                    migration_map[color_index] = *ncolors;
×
4793
                } else {
4794
                    result[pos] = migration_map[color_index] - 1;
×
4795
                }
4796
            }
4797
        }
4798
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
×
4799
    } else {
4800
        int x;
4801

4802
        for (y = 0; y < height; ++y) {
×
4803
            int start;
4804
            int end;
4805
            int step;
4806
            int direction;
4807

4808
            scanline_params(serpentine, y, width,
×
4809
                            &start, &end, &step, &direction);
4810
            (void)direction;
4811
            for (x = start; x != end; x += step) {
×
4812
                int pos;
4813
                int d;
4814

4815
                pos = y * width + x;
×
4816
                for (d = 0; d < depth; ++d) {
×
4817
                    int val;
4818

4819
                    val = data[pos * depth + d]
×
4820
                        + f_mask(x, y, d) * 32;
×
4821
                    copy[d] = val < 0 ? 0
×
4822
                               : val > 255 ? 255 : val;
×
4823
                }
4824
                result[pos] = f_lookup(copy, depth, palette,
×
4825
                                       reqcolor, indextable,
4826
                                       complexion);
4827
            }
4828
        }
4829
        *ncolors = reqcolor;
×
4830
    }
4831

4832
    return SIXEL_OK;
×
4833
}
4834

4835

4836
static SIXELSTATUS
4837
apply_palette_variable(
×
4838
    sixel_index_t *result,
4839
    unsigned char *data,
4840
    int width,
4841
    int height,
4842
    int depth,
4843
    unsigned char *palette,
4844
    int reqcolor,
4845
    int methodForScan,
4846
    int foptimize_palette,
4847
    int (*f_lookup)(const unsigned char *pixel,
4848
                    int depth,
4849
                    const unsigned char *palette,
4850
                    int reqcolor,
4851
                    unsigned short *cachetable,
4852
                    int complexion),
4853
    unsigned short *indextable,
4854
    int complexion,
4855
    unsigned char new_palette[],
4856
    unsigned short migration_map[],
4857
    int *ncolors,
4858
    int methodForDiffuse,
4859
    int methodForCarry)
4860
{
4861
    SIXELSTATUS status = SIXEL_FALSE;
×
4862
#if _MSC_VER
4863
    enum { max_channels = 4 };
4864
#else
4865
    const int max_channels = 4;
×
4866
#endif
4867
    int serpentine;
4868
    int y;
4869
    diffuse_varerr_mode varerr_diffuse;
4870
    diffuse_varerr_carry_mode varerr_diffuse_carry;
4871
    int use_carry;
4872
    size_t carry_len;
4873
    int32_t *carry_curr = NULL;
×
4874
    int32_t *carry_next = NULL;
×
4875
    int32_t *carry_far = NULL;
×
4876
    unsigned char corrected[max_channels];
4877
    int32_t sample_scaled[max_channels];
4878
    int32_t accum_scaled[max_channels];
4879
    int start;
4880
    int end;
4881
    int step;
4882
    int direction;
4883
    int x;
4884
    int pos;
4885
    size_t base;
4886
    size_t carry_base;
4887
    const unsigned char *source_pixel;
4888
    int color_index;
4889
    int output_index;
4890
    int n;
4891
    int palette_value;
4892
    int diff;
4893
    int table_index;
4894
    int64_t accum;
4895
    int64_t clamped;
4896
    int32_t target_scaled;
4897
    int32_t error_scaled;
4898
    int32_t *tmp;
4899

4900
    if (depth > max_channels) {
×
4901
        status = SIXEL_BAD_ARGUMENT;
×
4902
        goto end;
×
4903
    }
4904

4905
    use_carry = (methodForCarry == SIXEL_CARRY_ENABLE);
×
4906
    carry_len = 0;
×
4907

4908
    switch (methodForDiffuse) {
×
4909
    case SIXEL_DIFFUSE_LSO2:
4910
        varerr_diffuse = diffuse_lso2;
×
4911
        varerr_diffuse_carry = diffuse_lso2_carry;
×
4912
        break;
×
4913
    default:
4914
        varerr_diffuse = diffuse_lso2;
×
4915
        varerr_diffuse_carry = diffuse_lso2_carry;
×
4916
        break;
×
4917
    }
4918

4919
    if (use_carry) {
×
4920
        carry_len = (size_t)width * (size_t)depth;
×
4921
        carry_curr = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4922
        if (carry_curr == NULL) {
×
4923
            status = SIXEL_BAD_ALLOCATION;
×
4924
            goto end;
×
4925
        }
4926
        carry_next = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4927
        if (carry_next == NULL) {
×
4928
            status = SIXEL_BAD_ALLOCATION;
×
4929
            goto end;
×
4930
        }
4931
        carry_far = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
4932
        if (carry_far == NULL) {
×
4933
            status = SIXEL_BAD_ALLOCATION;
×
4934
            goto end;
×
4935
        }
4936
    }
4937

4938
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
×
4939

4940
    if (foptimize_palette) {
×
4941
        *ncolors = 0;
×
4942
        memset(new_palette, 0x00,
×
4943
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
4944
        memset(migration_map, 0x00,
×
4945
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
4946
    }
4947

4948
    for (y = 0; y < height; ++y) {
×
4949
        scanline_params(serpentine, y, width,
×
4950
                        &start, &end, &step, &direction);
4951
        for (x = start; x != end; x += step) {
×
4952
            pos = y * width + x;
×
4953
            base = (size_t)pos * (size_t)depth;
×
4954
            carry_base = (size_t)x * (size_t)depth;
×
4955
            if (use_carry) {
×
4956
                for (n = 0; n < depth; ++n) {
×
4957
                    accum = ((int64_t)data[base + n]
×
4958
                             << VARERR_SCALE_SHIFT)
×
4959
                          + carry_curr[carry_base + (size_t)n];
×
4960
                    if (accum < INT32_MIN) {
×
4961
                        accum = INT32_MIN;
×
4962
                    } else if (accum > INT32_MAX) {
×
4963
                        accum = INT32_MAX;
×
4964
                    }
4965
                    carry_curr[carry_base + (size_t)n] = 0;
×
4966
                    clamped = accum;
×
4967
                    if (clamped < 0) {
×
4968
                        clamped = 0;
×
4969
                    } else if (clamped > VARERR_MAX_VALUE) {
×
4970
                        clamped = VARERR_MAX_VALUE;
×
4971
                    }
4972
                    accum_scaled[n] = (int32_t)clamped;
×
4973
                    corrected[n]
4974
                        = (unsigned char)((clamped + VARERR_ROUND)
×
4975
                                          >> VARERR_SCALE_SHIFT);
×
4976
                }
4977
                source_pixel = corrected;
×
4978
            } else {
4979
                for (n = 0; n < depth; ++n) {
×
4980
                    sample_scaled[n]
4981
                        = (int32_t)data[base + n]
×
4982
                        << VARERR_SCALE_SHIFT;
×
4983
                    corrected[n] = data[base + n];
×
4984
                }
4985
                source_pixel = data + base;
×
4986
            }
4987

4988
            color_index = f_lookup(source_pixel, depth, palette,
×
4989
                                   reqcolor, indextable,
4990
                                   complexion);
4991

4992
            if (foptimize_palette) {
×
4993
                if (migration_map[color_index] == 0) {
×
4994
                    output_index = *ncolors;
×
4995
                    for (n = 0; n < depth; ++n) {
×
4996
                        new_palette[output_index * depth + n]
×
4997
                            = palette[color_index * depth + n];
×
4998
                    }
4999
                    ++*ncolors;
×
5000
                    migration_map[color_index] = *ncolors;
×
5001
                } else {
5002
                    output_index = migration_map[color_index] - 1;
×
5003
                }
5004
                result[pos] = output_index;
×
5005
            } else {
5006
                output_index = color_index;
×
5007
                result[pos] = output_index;
×
5008
            }
5009

5010
            for (n = 0; n < depth; ++n) {
×
5011
                if (foptimize_palette) {
×
5012
                    palette_value = new_palette[output_index * depth + n];
×
5013
                } else {
5014
                    palette_value = palette[color_index * depth + n];
×
5015
                }
5016
                diff = (int)source_pixel[n] - palette_value;
×
5017
                if (diff < 0) {
×
5018
                    diff = -diff;
×
5019
                }
5020
                if (diff > 255) {
×
5021
                    diff = 255;
×
5022
                }
5023
                table_index = diff;
×
5024
                if (use_carry) {
×
5025
                    target_scaled = (int32_t)palette_value
×
5026
                                  << VARERR_SCALE_SHIFT;
5027
                    error_scaled = accum_scaled[n] - target_scaled;
×
5028
                    varerr_diffuse_carry(carry_curr, carry_next, carry_far,
×
5029
                                         width, height, depth,
5030
                                         x, y, error_scaled,
5031
                                         table_index,
5032
                                         direction, n);
5033
                } else {
5034
                    target_scaled = (int32_t)palette_value
×
5035
                                  << VARERR_SCALE_SHIFT;
5036
                    error_scaled = sample_scaled[n] - target_scaled;
×
5037
                    varerr_diffuse(data + n, width, height,
×
5038
                                   x, y, depth, error_scaled,
5039
                                   table_index,
5040
                                   direction);
5041
                }
5042
            }
5043
        }
5044
        if (use_carry) {
×
5045
            tmp = carry_curr;
×
5046
            carry_curr = carry_next;
×
5047
            carry_next = carry_far;
×
5048
            carry_far = tmp;
×
5049
            if (carry_len > 0) {
×
5050
                memset(carry_far, 0x00, carry_len * sizeof(int32_t));
×
5051
            }
5052
        }
5053
    }
5054

5055
    if (foptimize_palette) {
×
5056
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
×
5057
    } else {
5058
        *ncolors = reqcolor;
×
5059
    }
5060

5061
    status = SIXEL_OK;
×
5062

5063
end:
5064
    free(carry_next);
×
5065
    free(carry_curr);
×
5066
    free(carry_far);
×
5067
    return status;
×
5068
}
5069

5070

5071
static SIXELSTATUS
5072
apply_palette_fixed(
285✔
5073
    sixel_index_t *result,
5074
    unsigned char *data,
5075
    int width,
5076
    int height,
5077
    int depth,
5078
    unsigned char *palette,
5079
    int reqcolor,
5080
    int methodForScan,
5081
    int foptimize_palette,
5082
    int (*f_lookup)(const unsigned char *pixel,
5083
                    int depth,
5084
                    const unsigned char *palette,
5085
                    int reqcolor,
5086
                    unsigned short *cachetable,
5087
                    int complexion),
5088
    unsigned short *indextable,
5089
    int complexion,
5090
    unsigned char new_palette[],
5091
    unsigned short migration_map[],
5092
    int *ncolors,
5093
    int methodForDiffuse,
5094
    int methodForCarry)
5095
{
170✔
5096
#if _MSC_VER
5097
    enum { max_channels = 4 };
5098
#else
5099
    const int max_channels = 4;
285✔
5100
#endif
5101
    SIXELSTATUS status = SIXEL_FALSE;
285✔
5102
    int serpentine;
5103
    int y;
5104
    void (*f_diffuse)(unsigned char *data,
5105
                      int width,
5106
                      int height,
5107
                      int x,
5108
                      int y,
5109
                      int depth,
5110
                      int offset,
5111
                      int direction);
5112
    diffuse_fixed_carry_mode f_diffuse_carry;
5113
    int use_carry;
5114
    size_t carry_len;
5115
    int32_t *carry_curr = NULL;
285✔
5116
    int32_t *carry_next = NULL;
285✔
5117
    int32_t *carry_far = NULL;
285✔
5118
    unsigned char corrected[max_channels];
170✔
5119
    int32_t accum_scaled[max_channels];
170✔
5120
    int start;
5121
    int end;
5122
    int step;
5123
    int direction;
5124
    int x;
5125
    int pos;
5126
    size_t base;
5127
    size_t carry_base;
5128
    const unsigned char *source_pixel;
5129
    int color_index;
5130
    int output_index;
5131
    int n;
5132
    int palette_value;
5133
    int64_t accum;
5134
    int64_t clamped;
5135
    int32_t target_scaled;
5136
    int32_t error_scaled;
5137
    int offset;
5138
    int32_t *tmp;
5139

5140
    if (depth > max_channels) {
285!
5141
        status = SIXEL_BAD_ARGUMENT;
×
5142
        goto end;
×
5143
    }
5144

5145
    use_carry = (methodForCarry == SIXEL_CARRY_ENABLE);
285✔
5146
    carry_len = 0;
285✔
5147

5148
    if (depth != 3) {
285!
5149
        f_diffuse = diffuse_none;
×
5150
        f_diffuse_carry = diffuse_none_carry;
×
5151
        use_carry = 0;
×
5152
    } else {
5153
        switch (methodForDiffuse) {
285!
5154
        case SIXEL_DIFFUSE_NONE:
86✔
5155
            f_diffuse = diffuse_none;
158✔
5156
            f_diffuse_carry = diffuse_none_carry;
158✔
5157
            break;
158✔
5158
        case SIXEL_DIFFUSE_ATKINSON:
44✔
5159
            f_diffuse = diffuse_atkinson;
67✔
5160
            f_diffuse_carry = diffuse_atkinson_carry;
67✔
5161
            break;
67✔
5162
        case SIXEL_DIFFUSE_FS:
34✔
5163
            f_diffuse = diffuse_fs;
51✔
5164
            f_diffuse_carry = diffuse_fs_carry;
51✔
5165
            break;
51✔
5166
        case SIXEL_DIFFUSE_JAJUNI:
2✔
5167
            f_diffuse = diffuse_jajuni;
3✔
5168
            f_diffuse_carry = diffuse_jajuni_carry;
3✔
5169
            break;
3✔
5170
        case SIXEL_DIFFUSE_STUCKI:
2✔
5171
            f_diffuse = diffuse_stucki;
3✔
5172
            f_diffuse_carry = diffuse_stucki_carry;
3✔
5173
            break;
3✔
5174
        case SIXEL_DIFFUSE_BURKES:
2✔
5175
            f_diffuse = diffuse_burkes;
3✔
5176
            f_diffuse_carry = diffuse_burkes_carry;
3✔
5177
            break;
3✔
5178
        case SIXEL_DIFFUSE_SIERRA1:
5179
            f_diffuse = diffuse_sierra1;
×
5180
            f_diffuse_carry = diffuse_sierra1_carry;
×
5181
            break;
×
5182
        case SIXEL_DIFFUSE_SIERRA2:
5183
            f_diffuse = diffuse_sierra2;
×
5184
            f_diffuse_carry = diffuse_sierra2_carry;
×
5185
            break;
×
5186
        case SIXEL_DIFFUSE_SIERRA3:
5187
            f_diffuse = diffuse_sierra3;
×
5188
            f_diffuse_carry = diffuse_sierra3_carry;
×
5189
            break;
×
5190
        default:
5191
            quant_trace(stderr,
×
5192
                        "Internal error: invalid methodForDiffuse: %d\n",
5193
                        methodForDiffuse);
5194
            f_diffuse = diffuse_none;
×
5195
            f_diffuse_carry = diffuse_none_carry;
×
5196
            break;
×
5197
        }
5198
    }
5199

5200
    if (use_carry) {
285!
5201
        carry_len = (size_t)width * (size_t)depth;
×
5202
        if (carry_len > 0) {
×
5203
            carry_curr = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5204
            if (carry_curr == NULL) {
×
5205
                status = SIXEL_BAD_ALLOCATION;
×
5206
                goto end;
×
5207
            }
5208
            carry_next = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5209
            if (carry_next == NULL) {
×
5210
                status = SIXEL_BAD_ALLOCATION;
×
5211
                goto end;
×
5212
            }
5213
            carry_far = (int32_t *)calloc(carry_len, sizeof(int32_t));
×
5214
            if (carry_far == NULL) {
×
5215
                status = SIXEL_BAD_ALLOCATION;
×
5216
                goto end;
×
5217
            }
5218
        } else {
5219
            use_carry = 0;
×
5220
        }
5221
    }
5222

5223
    serpentine = (methodForScan == SIXEL_SCAN_SERPENTINE);
285✔
5224

5225
    if (foptimize_palette) {
285✔
5226
        *ncolors = 0;
206✔
5227
        memset(new_palette, 0x00,
206✔
5228
               (size_t)SIXEL_PALETTE_MAX * (size_t)depth);
130✔
5229
        memset(migration_map, 0x00,
206✔
5230
               sizeof(unsigned short) * (size_t)SIXEL_PALETTE_MAX);
5231
    } else {
76✔
5232
        *ncolors = reqcolor;
79✔
5233
    }
5234

5235
    for (y = 0; y < height; ++y) {
59,935✔
5236
        scanline_params(serpentine, y, width,
59,650✔
5237
                        &start, &end, &step, &direction);
5238
        for (x = start; x != end; x += step) {
43,528,028✔
5239
            pos = y * width + x;
43,468,378✔
5240
            base = (size_t)pos * (size_t)depth;
43,468,378✔
5241
            carry_base = (size_t)x * (size_t)depth;
43,468,378✔
5242
            if (use_carry) {
43,468,378!
5243
                for (n = 0; n < depth; ++n) {
×
5244
                    accum = ((int64_t)data[base + n]
×
5245
                             << VARERR_SCALE_SHIFT)
×
5246
                           + carry_curr[carry_base + (size_t)n];
×
5247
                    if (accum < INT32_MIN) {
×
5248
                        accum = INT32_MIN;
×
5249
                    } else if (accum > INT32_MAX) {
×
5250
                        accum = INT32_MAX;
×
5251
                    }
5252
                    clamped = accum;
×
5253
                    if (clamped < 0) {
×
5254
                        clamped = 0;
×
5255
                    } else if (clamped > VARERR_MAX_VALUE) {
×
5256
                        clamped = VARERR_MAX_VALUE;
×
5257
                    }
5258
                    accum_scaled[n] = (int32_t)clamped;
×
5259
                    corrected[n]
5260
                        = (unsigned char)((clamped + VARERR_ROUND)
×
5261
                                          >> VARERR_SCALE_SHIFT);
×
5262
                    data[base + n] = corrected[n];
×
5263
                    carry_curr[carry_base + (size_t)n] = 0;
×
5264
                }
5265
                source_pixel = corrected;
×
5266
            } else {
5267
                source_pixel = data + base;
43,468,378✔
5268
            }
5269

5270
            color_index = f_lookup(source_pixel, depth, palette,
63,604,336✔
5271
                                   reqcolor, indextable,
20,135,958✔
5272
                                   complexion);
20,135,958✔
5273

5274
            if (foptimize_palette) {
43,468,378✔
5275
                if (migration_map[color_index] == 0) {
31,366,878✔
5276
                    output_index = *ncolors;
16,650✔
5277
                    for (n = 0; n < depth; ++n) {
66,600✔
5278
                        new_palette[output_index * depth + n]
49,950✔
5279
                            = palette[color_index * depth + n];
67,068✔
5280
                    }
17,118✔
5281
                    ++*ncolors;
16,650✔
5282
                    migration_map[color_index] = *ncolors;
16,650✔
5283
                } else {
5,706✔
5284
                    output_index = migration_map[color_index] - 1;
31,350,228✔
5285
                }
5286
                result[pos] = output_index;
31,366,878✔
5287
            } else {
14,296,138✔
5288
                output_index = color_index;
12,101,500✔
5289
                result[pos] = output_index;
12,101,500✔
5290
            }
5291

5292
            for (n = 0; n < depth; ++n) {
173,873,512✔
5293
                if (foptimize_palette) {
130,405,134✔
5294
                    palette_value = new_palette[output_index * depth + n];
94,100,634✔
5295
                } else {
42,888,414✔
5296
                    palette_value = palette[color_index * depth + n];
36,304,500✔
5297
                }
5298
                if (use_carry) {
130,405,134!
5299
                    target_scaled = (int32_t)palette_value
×
5300
                                  << VARERR_SCALE_SHIFT;
5301
                    error_scaled = accum_scaled[n] - target_scaled;
×
5302
                    f_diffuse_carry(carry_curr, carry_next, carry_far,
×
5303
                                    width, height, depth,
5304
                                    x, y, error_scaled, direction, n);
5305
                } else {
5306
                    offset = (int)source_pixel[n] - palette_value;
130,405,134✔
5307
                    f_diffuse(data + n, width, height, x, y,
190,813,008✔
5308
                              depth, offset, direction);
60,407,874✔
5309
                }
5310
            }
60,407,874✔
5311
        }
20,135,958✔
5312
        if (use_carry) {
59,650!
5313
            tmp = carry_curr;
×
5314
            carry_curr = carry_next;
×
5315
            carry_next = carry_far;
×
5316
            carry_far = tmp;
×
5317
            if (carry_len > 0) {
×
5318
                memset(carry_far, 0x00, carry_len * sizeof(int32_t));
×
5319
            }
5320
        }
5321
    }
26,750✔
5322

5323
    if (foptimize_palette) {
285✔
5324
        memcpy(palette, new_palette, (size_t)(*ncolors * depth));
206✔
5325
    }
76✔
5326

5327
    status = SIXEL_OK;
285✔
5328

5329
end:
170✔
5330
    free(carry_far);
285✔
5331
    free(carry_next);
285✔
5332
    free(carry_curr);
285✔
5333
    return status;
285✔
5334
}
5335

5336

5337
static void
5338
diffuse_none(unsigned char *data, int width, int height,
31,038,318✔
5339
             int x, int y, int depth, int error, int direction)
5340
{
5341
    /* unused */ (void) data;
27,259,002✔
5342
    /* unused */ (void) width;
27,259,002✔
5343
    /* unused */ (void) height;
27,259,002✔
5344
    /* unused */ (void) x;
27,259,002✔
5345
    /* unused */ (void) y;
27,259,002✔
5346
    /* unused */ (void) depth;
27,259,002✔
5347
    /* unused */ (void) error;
27,259,002✔
5348
    /* unused */ (void) direction;
27,259,002✔
5349
}
31,038,318✔
5350

5351

5352
static void
5353
diffuse_none_carry(int32_t *carry_curr, int32_t *carry_next,
×
5354
                   int32_t *carry_far, int width, int height,
5355
                   int depth, int x, int y, int32_t error,
5356
                   int direction, int channel)
5357
{
5358
    /* unused */ (void) carry_curr;
5359
    /* unused */ (void) carry_next;
5360
    /* unused */ (void) carry_far;
5361
    /* unused */ (void) width;
5362
    /* unused */ (void) height;
5363
    /* unused */ (void) depth;
5364
    /* unused */ (void) x;
5365
    /* unused */ (void) y;
5366
    /* unused */ (void) error;
5367
    /* unused */ (void) direction;
5368
    /* unused */ (void) channel;
5369
}
×
5370

5371

5372
static void
5373
diffuse_fs(unsigned char *data, int width, int height,
70,430,256✔
5374
           int x, int y, int depth, int error, int direction)
5375
{
5376
    /* Floyd Steinberg Method
5377
     *          curr    7/16
5378
     *  3/16    5/16    1/16
5379
     */
5380
    int pos;
5381
    int forward;
5382

5383
    pos = y * width + x;
70,430,256✔
5384
    forward = direction >= 0;
70,430,256✔
5385

5386
    if (forward) {
70,430,256✔
5387
        if (x < width - 1) {
69,215,256✔
5388
            error_diffuse_normal(data, pos + 1, depth, error, 7, 16);
69,144,759✔
5389
        }
23,048,253✔
5390
        if (y < height - 1) {
69,215,256✔
5391
            if (x > 0) {
69,121,332✔
5392
                error_diffuse_normal(data,
92,067,972✔
5393
                                     pos + width - 1,
69,050,979✔
5394
                                     depth, error, 3, 16);
23,016,993✔
5395
            }
23,016,993✔
5396
            error_diffuse_normal(data,
92,161,776✔
5397
                                 pos + width,
23,040,444✔
5398
                                 depth, error, 5, 16);
23,040,444✔
5399
            if (x < width - 1) {
69,121,332✔
5400
                error_diffuse_normal(data,
92,067,972✔
5401
                                     pos + width + 1,
69,050,979✔
5402
                                     depth, error, 1, 16);
23,016,993✔
5403
            }
23,016,993✔
5404
        }
23,040,444✔
5405
    } else {
23,071,752✔
5406
        if (x > 0) {
1,215,000✔
5407
            error_diffuse_normal(data, pos - 1, depth, error, 7, 16);
1,212,975✔
5408
        }
404,325✔
5409
        if (y < height - 1) {
1,215,000✔
5410
            if (x < width - 1) {
1,209,600✔
5411
                error_diffuse_normal(data,
1,610,112✔
5412
                                     pos + width + 1,
1,207,584✔
5413
                                     depth, error, 3, 16);
402,528✔
5414
            }
402,528✔
5415
            error_diffuse_normal(data,
1,612,800✔
5416
                                 pos + width,
403,200✔
5417
                                 depth, error, 5, 16);
403,200✔
5418
            if (x > 0) {
1,209,600✔
5419
                error_diffuse_normal(data,
1,610,112✔
5420
                                     pos + width - 1,
1,207,584✔
5421
                                     depth, error, 1, 16);
402,528✔
5422
            }
402,528✔
5423
        }
403,200✔
5424
    }
5425
}
70,430,256✔
5426

5427

5428
static void
5429
diffuse_fs_carry(int32_t *carry_curr, int32_t *carry_next,
×
5430
                 int32_t *carry_far, int width, int height,
5431
                 int depth, int x, int y, int32_t error,
5432
                 int direction, int channel)
5433
{
5434
    /* Floyd Steinberg Method
5435
     *          curr    7/16
5436
     *  3/16    5/48    1/16
5437
     */
5438
    int forward;
5439

5440
    /* unused */ (void) carry_far;
5441
    if (error == 0) {
×
5442
        return;
×
5443
    }
5444

5445
    forward = direction >= 0;
×
5446
    if (forward) {
×
5447
        if (x + 1 < width) {
×
5448
            size_t base;
5449
            int32_t term;
5450

5451
            base = ((size_t)(x + 1) * (size_t)depth)
×
5452
                 + (size_t)channel;
×
5453
            term = diffuse_fixed_term(error, 7, 16);
×
5454
            carry_curr[base] += term;
×
5455
        }
5456
        if (y + 1 < height) {
×
5457
            if (x > 0) {
×
5458
                size_t base;
5459
                int32_t term;
5460

5461
                base = ((size_t)(x - 1) * (size_t)depth)
×
5462
                     + (size_t)channel;
×
5463
                term = diffuse_fixed_term(error, 3, 16);
×
5464
                carry_next[base] += term;
×
5465
            }
5466
            {
5467
                size_t base;
5468
                int32_t term;
5469

5470
                base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5471
                term = diffuse_fixed_term(error, 5, 16);
×
5472
                carry_next[base] += term;
×
5473
            }
5474
            if (x + 1 < width) {
×
5475
                size_t base;
5476
                int32_t term;
5477

5478
                base = ((size_t)(x + 1) * (size_t)depth)
×
5479
                     + (size_t)channel;
×
5480
                term = diffuse_fixed_term(error, 1, 16);
×
5481
                carry_next[base] += term;
×
5482
            }
5483
        }
5484
    } else {
5485
        if (x - 1 >= 0) {
×
5486
            size_t base;
5487
            int32_t term;
5488

5489
            base = ((size_t)(x - 1) * (size_t)depth)
×
5490
                 + (size_t)channel;
×
5491
            term = diffuse_fixed_term(error, 7, 16);
×
5492
            carry_curr[base] += term;
×
5493
        }
5494
        if (y + 1 < height) {
×
5495
            if (x + 1 < width) {
×
5496
                size_t base;
5497
                int32_t term;
5498

5499
                base = ((size_t)(x + 1) * (size_t)depth)
×
5500
                     + (size_t)channel;
×
5501
                term = diffuse_fixed_term(error, 3, 16);
×
5502
                carry_next[base] += term;
×
5503
            }
5504
            {
5505
                size_t base;
5506
                int32_t term;
5507

5508
                base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5509
                term = diffuse_fixed_term(error, 5, 16);
×
5510
                carry_next[base] += term;
×
5511
            }
5512
            if (x - 1 >= 0) {
×
5513
                size_t base;
5514
                int32_t term;
5515

5516
                base = ((size_t)(x - 1) * (size_t)depth)
×
5517
                     + (size_t)channel;
×
5518
                term = diffuse_fixed_term(error, 1, 16);
×
5519
                carry_next[base] += term;
×
5520
            }
5521
        }
5522
    }
5523
}
5524

5525

5526
static void
5527
diffuse_atkinson(unsigned char *data, int width, int height,
28,352,460✔
5528
                 int x, int y, int depth, int error, int direction)
5529
{
5530
    /* Atkinson's Method
5531
     *          curr    1/8    1/8
5532
     *   1/8     1/8    1/8
5533
     *           1/8
5534
     */
5535
    int pos;
5536
    int sign;
5537

5538
    pos = y * width + x;
28,352,460✔
5539
    sign = direction >= 0 ? 1 : -1;
28,352,460!
5540

5541
    if (x + sign >= 0 && x + sign < width) {
28,352,460!
5542
        error_diffuse_fast(data, pos + sign, depth, error, 1, 8);
28,296,945✔
5543
    }
9,458,715✔
5544
    if (x + sign * 2 >= 0 && x + sign * 2 < width) {
28,352,460!
5545
        error_diffuse_fast(data, pos + sign * 2, depth, error, 1, 8);
28,241,430✔
5546
    }
9,440,010✔
5547
    if (y < height - 1) {
28,352,460✔
5548
        int row;
5549

5550
        row = pos + width;
28,278,756✔
5551
        if (x - sign >= 0 && x - sign < width) {
28,278,756!
5552
            error_diffuse_fast(data,
37,657,392✔
5553
                               row + (-sign),
9,433,950✔
5554
                               depth, error, 1, 8);
9,433,950✔
5555
        }
9,433,950✔
5556
        error_diffuse_fast(data, row, depth, error, 1, 8);
28,278,756✔
5557
        if (x + sign >= 0 && x + sign < width) {
28,278,756!
5558
            error_diffuse_fast(data,
37,657,392✔
5559
                               row + sign,
9,433,950✔
5560
                               depth, error, 1, 8);
9,433,950✔
5561
        }
9,433,950✔
5562
    }
9,452,586✔
5563
    if (y < height - 2) {
28,352,460✔
5564
        error_diffuse_fast(data, pos + width * 2, depth, error, 1, 8);
28,205,052✔
5565
    }
9,427,752✔
5566
}
28,352,460✔
5567

5568

5569
static void
5570
diffuse_atkinson_carry(int32_t *carry_curr, int32_t *carry_next,
×
5571
                       int32_t *carry_far, int width, int height,
5572
                       int depth, int x, int y, int32_t error,
5573
                       int direction, int channel)
5574
{
5575
    /* Atkinson's Method
5576
     *          curr    1/8    1/8
5577
     *   1/8     1/8    1/8
5578
     *           1/8
5579
     */
5580
    int sign;
5581
    int32_t term;
5582

5583
    if (error == 0) {
×
5584
        return;
×
5585
    }
5586

5587
    term = diffuse_fixed_term(error, 1, 8);
×
5588
    sign = direction >= 0 ? 1 : -1;
×
5589
    if (x + sign >= 0 && x + sign < width) {
×
5590
        size_t base;
5591

5592
        base = ((size_t)(x + sign) * (size_t)depth)
×
5593
             + (size_t)channel;
×
5594
        carry_curr[base] += term;
×
5595
    }
5596
    if (x + sign * 2 >= 0 && x + sign * 2 < width) {
×
5597
        size_t base;
5598

5599
        base = ((size_t)(x + sign * 2) * (size_t)depth)
×
5600
             + (size_t)channel;
×
5601
        carry_curr[base] += term;
×
5602
    }
5603
    if (y + 1 < height) {
×
5604
        if (x - sign >= 0 && x - sign < width) {
×
5605
            size_t base;
5606

5607
            base = ((size_t)(x - sign) * (size_t)depth)
×
5608
                 + (size_t)channel;
×
5609
            carry_next[base] += term;
×
5610
        }
5611
        {
5612
            size_t base;
5613

5614
            base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5615
            carry_next[base] += term;
×
5616
        }
5617
        if (x + sign >= 0 && x + sign < width) {
×
5618
            size_t base;
5619

5620
            base = ((size_t)(x + sign) * (size_t)depth)
×
5621
                 + (size_t)channel;
×
5622
            carry_next[base] += term;
×
5623
        }
5624
    }
5625
    if (y + 2 < height) {
×
5626
        size_t base;
5627

5628
        base = ((size_t)x * (size_t)depth) + (size_t)channel;
×
5629
        carry_far[base] += term;
×
5630
    }
5631
}
5632

5633

5634
static void
5635
diffuse_jajuni(unsigned char *data, int width, int height,
396,900✔
5636
               int x, int y, int depth, int error, int direction)
5637
{
5638
    /* Jarvis, Judice & Ninke Method
5639
     *                  curr    7/48    5/48
5640
     *  3/48    5/48    7/48    5/48    3/48
5641
     *  1/48    3/48    5/48    3/48    1/48
5642
     */
5643
    int pos;
5644
    int sign;
5645
    static const int row0_offsets[] = { 1, 2 };
5646
    static const int row0_weights[] = { 7, 5 };
5647
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5648
    static const int row1_weights[] = { 3, 5, 7, 5, 3 };
5649
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5650
    static const int row2_weights[] = { 1, 3, 5, 3, 1 };
5651
    int i;
5652

5653
    pos = y * width + x;
396,900✔
5654
    sign = direction >= 0 ? 1 : -1;
396,900!
5655

5656
    for (i = 0; i < 2; ++i) {
1,190,700✔
5657
        int neighbor;
5658

5659
        neighbor = x + sign * row0_offsets[i];
793,800✔
5660
        if (neighbor < 0 || neighbor >= width) {
793,800!
5661
            continue;
5,670✔
5662
        }
5663
        error_diffuse_precise(data,
1,050,840✔
5664
                              pos + (neighbor - x),
788,130✔
5665
                              depth, error,
262,710✔
5666
                              row0_weights[i], 48);
788,130✔
5667
    }
262,710✔
5668
    if (y < height - 1) {
396,900✔
5669
        int row;
5670

5671
        row = pos + width;
395,010✔
5672
        for (i = 0; i < 5; ++i) {
2,370,060✔
5673
            int neighbor;
5674

5675
            neighbor = x + sign * row1_offsets[i];
1,975,050✔
5676
            if (neighbor < 0 || neighbor >= width) {
1,975,050✔
5677
                continue;
11,286✔
5678
            }
5679
            error_diffuse_precise(data,
2,618,352✔
5680
                                  row + (neighbor - x),
1,963,764✔
5681
                                  depth, error,
654,588✔
5682
                                  row1_weights[i], 48);
1,963,764✔
5683
        }
654,588✔
5684
    }
131,670✔
5685
    if (y < height - 2) {
396,900✔
5686
        int row;
5687

5688
        row = pos + width * 2;
393,120✔
5689
        for (i = 0; i < 5; ++i) {
2,358,720✔
5690
            int neighbor;
5691

5692
            neighbor = x + sign * row2_offsets[i];
1,965,600✔
5693
            if (neighbor < 0 || neighbor >= width) {
1,965,600✔
5694
                continue;
11,232✔
5695
            }
5696
            error_diffuse_precise(data,
2,605,824✔
5697
                                  row + (neighbor - x),
1,954,368✔
5698
                                  depth, error,
651,456✔
5699
                                  row2_weights[i], 48);
1,954,368✔
5700
        }
651,456✔
5701
    }
131,040✔
5702
}
396,900✔
5703

5704

5705
static void
5706
diffuse_jajuni_carry(int32_t *carry_curr, int32_t *carry_next,
×
5707
                     int32_t *carry_far, int width, int height,
5708
                     int depth, int x, int y, int32_t error,
5709
                     int direction, int channel)
5710
{
5711
    /* Jarvis, Judice & Ninke Method
5712
     *                  curr    7/48    5/48
5713
     *  3/48    5/48    7/48    5/48    3/48
5714
     *  1/48    3/48    5/48    3/48    1/48
5715
     */
5716
    static const int row0_offsets[] = { 1, 2 };
5717
    static const int row0_weights[] = { 7, 5 };
5718
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5719
    static const int row1_weights[] = { 3, 5, 7, 5, 3 };
5720
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5721
    static const int row2_weights[] = { 1, 3, 5, 3, 1 };
5722
    int sign;
5723
    int i;
5724

5725
    if (error == 0) {
×
5726
        return;
×
5727
    }
5728

5729
    sign = direction >= 0 ? 1 : -1;
×
5730
    for (i = 0; i < 2; ++i) {
×
5731
        int neighbor;
5732
        int32_t term;
5733

5734
        neighbor = x + sign * row0_offsets[i];
×
5735
        if (neighbor < 0 || neighbor >= width) {
×
5736
            continue;
×
5737
        }
5738
        term = diffuse_fixed_term(error, row0_weights[i], 48);
×
5739
        carry_curr[((size_t)neighbor * (size_t)depth)
×
5740
                   + (size_t)channel] += term;
×
5741
    }
5742
    if (y + 1 < height) {
×
5743
        for (i = 0; i < 5; ++i) {
×
5744
            int neighbor;
5745
            int32_t term;
5746

5747
            neighbor = x + sign * row1_offsets[i];
×
5748
            if (neighbor < 0 || neighbor >= width) {
×
5749
                continue;
×
5750
            }
5751
            term = diffuse_fixed_term(error, row1_weights[i], 48);
×
5752
            carry_next[((size_t)neighbor * (size_t)depth)
×
5753
                       + (size_t)channel] += term;
×
5754
        }
5755
    }
5756
    if (y + 2 < height) {
×
5757
        for (i = 0; i < 5; ++i) {
×
5758
            int neighbor;
5759
            int32_t term;
5760

5761
            neighbor = x + sign * row2_offsets[i];
×
5762
            if (neighbor < 0 || neighbor >= width) {
×
5763
                continue;
×
5764
            }
5765
            term = diffuse_fixed_term(error, row2_weights[i], 48);
×
5766
            carry_far[((size_t)neighbor * (size_t)depth)
×
5767
                      + (size_t)channel] += term;
×
5768
        }
5769
    }
5770
}
5771

5772

5773
static void
5774
diffuse_stucki(unsigned char *data, int width, int height,
119,700✔
5775
               int x, int y, int depth, int error, int direction)
5776
{
5777
    /* Stucki's Method
5778
     *                  curr    8/48    4/48
5779
     *  2/48    4/48    8/48    4/48    2/48
5780
     *  1/48    2/48    4/48    2/48    1/48
5781
     */
5782
    int pos;
5783
    int sign;
5784
    static const int row0_offsets[] = { 1, 2 };
5785
    static const int row0_num[] = { 1, 1 };
5786
    static const int row0_den[] = { 6, 12 };
5787
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5788
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5789
    static const int row1_den[] = { 24, 12, 6, 12, 24 };
5790
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5791
    static const int row2_num[] = { 1, 1, 1, 1, 1 };
5792
    static const int row2_den[] = { 48, 24, 12, 24, 48 };
5793
    int i;
5794

5795
    pos = y * width + x;
119,700✔
5796
    sign = direction >= 0 ? 1 : -1;
119,700!
5797

5798
    for (i = 0; i < 2; ++i) {
359,100✔
5799
        int neighbor;
5800

5801
        neighbor = x + sign * row0_offsets[i];
239,400✔
5802
        if (neighbor < 0 || neighbor >= width) {
239,400!
5803
            continue;
2,700✔
5804
        }
5805
        error_diffuse_precise(data,
315,600✔
5806
                              pos + (neighbor - x),
236,700✔
5807
                              depth, error,
78,900✔
5808
                              row0_num[i], row0_den[i]);
236,700✔
5809
    }
78,900✔
5810
    if (y < height - 1) {
119,700✔
5811
        int row;
5812

5813
        row = pos + width;
118,503✔
5814
        for (i = 0; i < 5; ++i) {
711,018✔
5815
            int neighbor;
5816

5817
            neighbor = x + sign * row1_offsets[i];
592,515✔
5818
            if (neighbor < 0 || neighbor >= width) {
592,515✔
5819
                continue;
5,346✔
5820
            }
5821
            error_diffuse_precise(data,
782,892✔
5822
                                  row + (neighbor - x),
587,169✔
5823
                                  depth, error,
195,723✔
5824
                                  row1_num[i], row1_den[i]);
587,169✔
5825
        }
195,723✔
5826
    }
39,501✔
5827
    if (y < height - 2) {
119,700✔
5828
        int row;
5829

5830
        row = pos + width * 2;
117,306✔
5831
        for (i = 0; i < 5; ++i) {
703,836✔
5832
            int neighbor;
5833

5834
            neighbor = x + sign * row2_offsets[i];
586,530✔
5835
            if (neighbor < 0 || neighbor >= width) {
586,530✔
5836
                continue;
5,292✔
5837
            }
5838
            error_diffuse_precise(data,
774,984✔
5839
                                  row + (neighbor - x),
581,238✔
5840
                                  depth, error,
193,746✔
5841
                                  row2_num[i], row2_den[i]);
581,238✔
5842
        }
193,746✔
5843
    }
39,102✔
5844
}
119,700✔
5845

5846

5847
static void
5848
diffuse_stucki_carry(int32_t *carry_curr, int32_t *carry_next,
×
5849
                     int32_t *carry_far, int width, int height,
5850
                     int depth, int x, int y, int32_t error,
5851
                     int direction, int channel)
5852
{
5853
    /* Stucki's Method
5854
     *                  curr    8/48    4/48
5855
     *  2/48    4/48    8/48    4/48    2/48
5856
     *  1/48    2/48    4/48    2/48    1/48
5857
     */
5858
    static const int row0_offsets[] = { 1, 2 };
5859
    static const int row0_num[] = { 1, 1 };
5860
    static const int row0_den[] = { 6, 12 };
5861
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5862
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5863
    static const int row1_den[] = { 24, 12, 6, 12, 24 };
5864
    static const int row2_offsets[] = { -2, -1, 0, 1, 2 };
5865
    static const int row2_num[] = { 1, 1, 1, 1, 1 };
5866
    static const int row2_den[] = { 48, 24, 12, 24, 48 };
5867
    int sign;
5868
    int i;
5869

5870
    if (error == 0) {
×
5871
        return;
×
5872
    }
5873

5874
    sign = direction >= 0 ? 1 : -1;
×
5875
    for (i = 0; i < 2; ++i) {
×
5876
        int neighbor;
5877
        int32_t term;
5878

5879
        neighbor = x + sign * row0_offsets[i];
×
5880
        if (neighbor < 0 || neighbor >= width) {
×
5881
            continue;
×
5882
        }
5883
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
5884
        carry_curr[((size_t)neighbor * (size_t)depth)
×
5885
                   + (size_t)channel] += term;
×
5886
    }
5887
    if (y + 1 < height) {
×
5888
        for (i = 0; i < 5; ++i) {
×
5889
            int neighbor;
5890
            int32_t term;
5891

5892
            neighbor = x + sign * row1_offsets[i];
×
5893
            if (neighbor < 0 || neighbor >= width) {
×
5894
                continue;
×
5895
            }
5896
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
5897
            carry_next[((size_t)neighbor * (size_t)depth)
×
5898
                       + (size_t)channel] += term;
×
5899
        }
5900
    }
5901
    if (y + 2 < height) {
×
5902
        for (i = 0; i < 5; ++i) {
×
5903
            int neighbor;
5904
            int32_t term;
5905

5906
            neighbor = x + sign * row2_offsets[i];
×
5907
            if (neighbor < 0 || neighbor >= width) {
×
5908
                continue;
×
5909
            }
5910
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
5911
            carry_far[((size_t)neighbor * (size_t)depth)
×
5912
                      + (size_t)channel] += term;
×
5913
        }
5914
    }
5915
}
5916

5917

5918
static void
5919
diffuse_burkes(unsigned char *data, int width, int height,
67,500✔
5920
               int x, int y, int depth, int error, int direction)
5921
{
5922
    /* Burkes' Method
5923
     *                  curr    4/16    2/16
5924
     *  1/16    2/16    4/16    2/16    1/16
5925
     */
5926
    int pos;
5927
    int sign;
5928
    static const int row0_offsets[] = { 1, 2 };
5929
    static const int row0_num[] = { 1, 1 };
5930
    static const int row0_den[] = { 4, 8 };
5931
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5932
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5933
    static const int row1_den[] = { 16, 8, 4, 8, 16 };
5934
    int i;
5935

5936
    pos = y * width + x;
67,500✔
5937
    sign = direction >= 0 ? 1 : -1;
67,500!
5938

5939
    for (i = 0; i < 2; ++i) {
202,500✔
5940
        int neighbor;
5941

5942
        neighbor = x + sign * row0_offsets[i];
135,000✔
5943
        if (neighbor < 0 || neighbor >= width) {
135,000!
5944
            continue;
2,025✔
5945
        }
5946
        error_diffuse_normal(data,
177,300✔
5947
                             pos + (neighbor - x),
132,975✔
5948
                             depth, error,
44,325✔
5949
                             row0_num[i], row0_den[i]);
132,975✔
5950
    }
44,325✔
5951
    if (y < height - 1) {
67,500✔
5952
        int row;
5953

5954
        row = pos + width;
66,600✔
5955
        for (i = 0; i < 5; ++i) {
399,600✔
5956
            int neighbor;
5957

5958
            neighbor = x + sign * row1_offsets[i];
333,000✔
5959
            if (neighbor < 0 || neighbor >= width) {
333,000✔
5960
                continue;
3,996✔
5961
            }
5962
            error_diffuse_normal(data,
438,672✔
5963
                                 row + (neighbor - x),
329,004✔
5964
                                 depth, error,
109,668✔
5965
                                 row1_num[i], row1_den[i]);
329,004✔
5966
        }
109,668✔
5967
    }
22,200✔
5968
}
67,500✔
5969

5970
static void
5971
diffuse_burkes_carry(int32_t *carry_curr, int32_t *carry_next,
×
5972
                     int32_t *carry_far, int width, int height,
5973
                     int depth, int x, int y, int32_t error,
5974
                     int direction, int channel)
5975
{
5976
    /* Burkes' Method
5977
     *                  curr    4/16    2/16
5978
     *  1/16    2/16    4/16    2/16    1/16
5979
     */
5980
    static const int row0_offsets[] = { 1, 2 };
5981
    static const int row0_num[] = { 1, 1 };
5982
    static const int row0_den[] = { 4, 8 };
5983
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
5984
    static const int row1_num[] = { 1, 1, 1, 1, 1 };
5985
    static const int row1_den[] = { 16, 8, 4, 8, 16 };
5986
    int sign;
5987
    int i;
5988

5989
    /* unused */ (void) carry_far;
5990

5991
    if (error == 0) {
×
5992
        return;
×
5993
    }
5994

5995
    sign = direction >= 0 ? 1 : -1;
×
5996
    for (i = 0; i < 2; ++i) {
×
5997
        int neighbor;
5998
        int32_t term;
5999

6000
        neighbor = x + sign * row0_offsets[i];
×
6001
        if (neighbor < 0 || neighbor >= width) {
×
6002
            continue;
×
6003
        }
6004
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6005
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6006
                   + (size_t)channel] += term;
×
6007
    }
6008
    if (y + 1 < height) {
×
6009
        for (i = 0; i < 5; ++i) {
×
6010
            int neighbor;
6011
            int32_t term;
6012

6013
            neighbor = x + sign * row1_offsets[i];
×
6014
            if (neighbor < 0 || neighbor >= width) {
×
6015
                continue;
×
6016
            }
6017
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6018
            carry_next[((size_t)neighbor * (size_t)depth)
×
6019
                       + (size_t)channel] += term;
×
6020
        }
6021
    }
6022
}
6023

6024
static void
6025
diffuse_sierra1(unsigned char *data, int width, int height,
×
6026
                int x, int y, int depth, int error, int direction)
6027
{
6028
    /* Sierra Lite Method
6029
     *          curr    2/4
6030
     *  1/4     1/4
6031
     */
6032
    static const int row0_offsets[] = { 1 };
6033
    static const int row0_num[] = { 1 };
6034
    static const int row0_den[] = { 2 };
6035
    static const int row1_offsets[] = { -1, 0 };
6036
    static const int row1_num[] = { 1, 1 };
6037
    static const int row1_den[] = { 4, 4 };
6038
    int pos;
6039
    int sign;
6040
    int i;
6041
    int neighbor;
6042
    int row;
6043

6044
    pos = y * width + x;
×
6045
    sign = direction >= 0 ? 1 : -1;
×
6046

6047
    for (i = 0; i < 1; ++i) {
×
6048
        neighbor = x + sign * row0_offsets[i];
×
6049
        if (neighbor < 0 || neighbor >= width) {
×
6050
            continue;
×
6051
        }
6052
        error_diffuse_normal(data,
×
6053
                             pos + (neighbor - x),
×
6054
                             depth, error,
6055
                             row0_num[i], row0_den[i]);
×
6056
    }
6057
    if (y < height - 1) {
×
6058
        row = pos + width;
×
6059
        for (i = 0; i < 2; ++i) {
×
6060
            neighbor = x + sign * row1_offsets[i];
×
6061
            if (neighbor < 0 || neighbor >= width) {
×
6062
                continue;
×
6063
            }
6064
            error_diffuse_normal(data,
×
6065
                                 row + (neighbor - x),
×
6066
                                 depth, error,
6067
                                 row1_num[i], row1_den[i]);
×
6068
        }
6069
    }
6070
}
×
6071

6072

6073
static void
6074
diffuse_sierra1_carry(int32_t *carry_curr, int32_t *carry_next,
×
6075
                      int32_t *carry_far, int width, int height,
6076
                      int depth, int x, int y, int32_t error,
6077
                      int direction, int channel)
6078
{
6079
    /* Sierra Lite Method
6080
     *          curr    2/4
6081
     *  1/4     1/4
6082
     */
6083
    static const int row0_offsets[] = { 1 };
6084
    static const int row0_num[] = { 1 };
6085
    static const int row0_den[] = { 2 };
6086
    static const int row1_offsets[] = { -1, 0 };
6087
    static const int row1_num[] = { 1, 1 };
6088
    static const int row1_den[] = { 4, 4 };
6089
    int sign;
6090
    int i;
6091
    int neighbor;
6092
    int32_t term;
6093

6094
    /* unused */ (void) carry_far;
6095

6096
    if (error == 0) {
×
6097
        return;
×
6098
    }
6099

6100
    sign = direction >= 0 ? 1 : -1;
×
6101
    for (i = 0; i < 1; ++i) {
×
6102
        neighbor = x + sign * row0_offsets[i];
×
6103
        if (neighbor < 0 || neighbor >= width) {
×
6104
            continue;
×
6105
        }
6106
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6107
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6108
                   + (size_t)channel] += term;
×
6109
    }
6110
    if (y + 1 < height) {
×
6111
        for (i = 0; i < 2; ++i) {
×
6112
            neighbor = x + sign * row1_offsets[i];
×
6113
            if (neighbor < 0 || neighbor >= width) {
×
6114
                continue;
×
6115
            }
6116
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6117
            carry_next[((size_t)neighbor * (size_t)depth)
×
6118
                       + (size_t)channel] += term;
×
6119
        }
6120
    }
6121
}
6122

6123

6124
static void
6125
diffuse_sierra2(unsigned char *data, int width, int height,
×
6126
                int x, int y, int depth, int error, int direction)
6127
{
6128
    /* Sierra Two-row Method
6129
     *                  curr    4/32    3/32
6130
     *  1/32    2/32    3/32    2/32    1/32
6131
     *                  2/32    3/32    2/32
6132
     */
6133
    static const int row0_offsets[] = { 1, 2 };
6134
    static const int row0_num[] = { 4, 3 };
6135
    static const int row0_den[] = { 32, 32 };
6136
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6137
    static const int row1_num[] = { 1, 2, 3, 2, 1 };
6138
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6139
    static const int row2_offsets[] = { -1, 0, 1 };
6140
    static const int row2_num[] = { 2, 3, 2 };
6141
    static const int row2_den[] = { 32, 32, 32 };
6142
    int pos;
6143
    int sign;
6144
    int i;
6145
    int neighbor;
6146
    int row;
6147

6148
    pos = y * width + x;
×
6149
    sign = direction >= 0 ? 1 : -1;
×
6150

6151
    for (i = 0; i < 2; ++i) {
×
6152
        neighbor = x + sign * row0_offsets[i];
×
6153
        if (neighbor < 0 || neighbor >= width) {
×
6154
            continue;
×
6155
        }
6156
        error_diffuse_precise(data,
×
6157
                              pos + (neighbor - x),
×
6158
                              depth, error,
6159
                              row0_num[i], row0_den[i]);
×
6160
    }
6161
    if (y < height - 1) {
×
6162
        row = pos + width;
×
6163
        for (i = 0; i < 5; ++i) {
×
6164
            neighbor = x + sign * row1_offsets[i];
×
6165
            if (neighbor < 0 || neighbor >= width) {
×
6166
                continue;
×
6167
            }
6168
            error_diffuse_precise(data,
×
6169
                                  row + (neighbor - x),
×
6170
                                  depth, error,
6171
                                  row1_num[i], row1_den[i]);
×
6172
        }
6173
    }
6174
    if (y < height - 2) {
×
6175
        row = pos + width * 2;
×
6176
        for (i = 0; i < 3; ++i) {
×
6177
            neighbor = x + sign * row2_offsets[i];
×
6178
            if (neighbor < 0 || neighbor >= width) {
×
6179
                continue;
×
6180
            }
6181
            error_diffuse_precise(data,
×
6182
                                  row + (neighbor - x),
×
6183
                                  depth, error,
6184
                                  row2_num[i], row2_den[i]);
×
6185
        }
6186
    }
6187
}
×
6188

6189

6190
static void
6191
diffuse_sierra2_carry(int32_t *carry_curr, int32_t *carry_next,
×
6192
                      int32_t *carry_far, int width, int height,
6193
                      int depth, int x, int y, int32_t error,
6194
                      int direction, int channel)
6195
{
6196
    /* Sierra Two-row Method
6197
     *                  curr    4/32    3/32
6198
     *  1/32    2/32    3/32    2/32    1/32
6199
     *                  2/32    3/32    2/32
6200
     */
6201
    static const int row0_offsets[] = { 1, 2 };
6202
    static const int row0_num[] = { 4, 3 };
6203
    static const int row0_den[] = { 32, 32 };
6204
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6205
    static const int row1_num[] = { 1, 2, 3, 2, 1 };
6206
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6207
    static const int row2_offsets[] = { -1, 0, 1 };
6208
    static const int row2_num[] = { 2, 3, 2 };
6209
    static const int row2_den[] = { 32, 32, 32 };
6210
    int sign;
6211
    int i;
6212
    int neighbor;
6213
    int32_t term;
6214

6215
    if (error == 0) {
×
6216
        return;
×
6217
    }
6218

6219
    sign = direction >= 0 ? 1 : -1;
×
6220
    for (i = 0; i < 2; ++i) {
×
6221
        neighbor = x + sign * row0_offsets[i];
×
6222
        if (neighbor < 0 || neighbor >= width) {
×
6223
            continue;
×
6224
        }
6225
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6226
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6227
                   + (size_t)channel] += term;
×
6228
    }
6229
    if (y + 1 < height) {
×
6230
        for (i = 0; i < 5; ++i) {
×
6231
            neighbor = x + sign * row1_offsets[i];
×
6232
            if (neighbor < 0 || neighbor >= width) {
×
6233
                continue;
×
6234
            }
6235
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6236
            carry_next[((size_t)neighbor * (size_t)depth)
×
6237
                       + (size_t)channel] += term;
×
6238
        }
6239
    }
6240
    if (y + 2 < height) {
×
6241
        for (i = 0; i < 3; ++i) {
×
6242
            neighbor = x + sign * row2_offsets[i];
×
6243
            if (neighbor < 0 || neighbor >= width) {
×
6244
                continue;
×
6245
            }
6246
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
6247
            carry_far[((size_t)neighbor * (size_t)depth)
×
6248
                      + (size_t)channel] += term;
×
6249
        }
6250
    }
6251
}
6252

6253

6254
static void
6255
diffuse_sierra3(unsigned char *data, int width, int height,
×
6256
                int x, int y, int depth, int error, int direction)
6257
{
6258
    /* Sierra-3 Method
6259
     *                  curr    5/32    3/32
6260
     *  2/32    4/32    5/32    4/32    2/32
6261
     *                  2/32    3/32    2/32
6262
     */
6263
    static const int row0_offsets[] = { 1, 2 };
6264
    static const int row0_num[] = { 5, 3 };
6265
    static const int row0_den[] = { 32, 32 };
6266
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6267
    static const int row1_num[] = { 2, 4, 5, 4, 2 };
6268
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6269
    static const int row2_offsets[] = { -1, 0, 1 };
6270
    static const int row2_num[] = { 2, 3, 2 };
6271
    static const int row2_den[] = { 32, 32, 32 };
6272
    int pos;
6273
    int sign;
6274
    int i;
6275
    int neighbor;
6276
    int row;
6277

6278
    pos = y * width + x;
×
6279
    sign = direction >= 0 ? 1 : -1;
×
6280

6281
    for (i = 0; i < 2; ++i) {
×
6282
        neighbor = x + sign * row0_offsets[i];
×
6283
        if (neighbor < 0 || neighbor >= width) {
×
6284
            continue;
×
6285
        }
6286
        error_diffuse_precise(data,
×
6287
                              pos + (neighbor - x),
×
6288
                              depth, error,
6289
                              row0_num[i], row0_den[i]);
×
6290
    }
6291
    if (y < height - 1) {
×
6292
        row = pos + width;
×
6293
        for (i = 0; i < 5; ++i) {
×
6294
            neighbor = x + sign * row1_offsets[i];
×
6295
            if (neighbor < 0 || neighbor >= width) {
×
6296
                continue;
×
6297
            }
6298
            error_diffuse_precise(data,
×
6299
                                  row + (neighbor - x),
×
6300
                                  depth, error,
6301
                                  row1_num[i], row1_den[i]);
×
6302
        }
6303
    }
6304
    if (y < height - 2) {
×
6305
        row = pos + width * 2;
×
6306
        for (i = 0; i < 3; ++i) {
×
6307
            neighbor = x + sign * row2_offsets[i];
×
6308
            if (neighbor < 0 || neighbor >= width) {
×
6309
                continue;
×
6310
            }
6311
            error_diffuse_precise(data,
×
6312
                                  row + (neighbor - x),
×
6313
                                  depth, error,
6314
                                  row2_num[i], row2_den[i]);
×
6315
        }
6316
    }
6317
}
×
6318

6319

6320
static void
6321
diffuse_sierra3_carry(int32_t *carry_curr, int32_t *carry_next,
×
6322
                      int32_t *carry_far, int width, int height,
6323
                      int depth, int x, int y, int32_t error,
6324
                      int direction, int channel)
6325
{
6326
    /* Sierra-3 Method
6327
     *                  curr    5/32    3/32
6328
     *  2/32    4/32    5/32    4/32    2/32
6329
     *                  2/32    3/32    2/32
6330
     */
6331
    static const int row0_offsets[] = { 1, 2 };
6332
    static const int row0_num[] = { 5, 3 };
6333
    static const int row0_den[] = { 32, 32 };
6334
    static const int row1_offsets[] = { -2, -1, 0, 1, 2 };
6335
    static const int row1_num[] = { 2, 4, 5, 4, 2 };
6336
    static const int row1_den[] = { 32, 32, 32, 32, 32 };
6337
    static const int row2_offsets[] = { -1, 0, 1 };
6338
    static const int row2_num[] = { 2, 3, 2 };
6339
    static const int row2_den[] = { 32, 32, 32 };
6340
    int sign;
6341
    int i;
6342
    int neighbor;
6343
    int32_t term;
6344

6345
    if (error == 0) {
×
6346
        return;
×
6347
    }
6348

6349
    sign = direction >= 0 ? 1 : -1;
×
6350
    for (i = 0; i < 2; ++i) {
×
6351
        neighbor = x + sign * row0_offsets[i];
×
6352
        if (neighbor < 0 || neighbor >= width) {
×
6353
            continue;
×
6354
        }
6355
        term = diffuse_fixed_term(error, row0_num[i], row0_den[i]);
×
6356
        carry_curr[((size_t)neighbor * (size_t)depth)
×
6357
                   + (size_t)channel] += term;
×
6358
    }
6359
    if (y + 1 < height) {
×
6360
        for (i = 0; i < 5; ++i) {
×
6361
            neighbor = x + sign * row1_offsets[i];
×
6362
            if (neighbor < 0 || neighbor >= width) {
×
6363
                continue;
×
6364
            }
6365
            term = diffuse_fixed_term(error, row1_num[i], row1_den[i]);
×
6366
            carry_next[((size_t)neighbor * (size_t)depth)
×
6367
                       + (size_t)channel] += term;
×
6368
        }
6369
    }
6370
    if (y + 2 < height) {
×
6371
        for (i = 0; i < 3; ++i) {
×
6372
            neighbor = x + sign * row2_offsets[i];
×
6373
            if (neighbor < 0 || neighbor >= width) {
×
6374
                continue;
×
6375
            }
6376
            term = diffuse_fixed_term(error, row2_num[i], row2_den[i]);
×
6377
            carry_far[((size_t)neighbor * (size_t)depth)
×
6378
                      + (size_t)channel] += term;
×
6379
        }
6380
    }
6381
}
6382

6383

6384
static float
6385
mask_a (int x, int y, int c)
×
6386
{
6387
    return ((((x + c * 67) + y * 236) * 119) & 255 ) / 128.0 - 1.0;
×
6388
}
6389

6390
static float
6391
mask_x (int x, int y, int c)
×
6392
{
6393
    return ((((x + c * 29) ^ y* 149) * 1234) & 511 ) / 256.0 - 1.0;
×
6394
}
6395

6396
/* lookup closest color from palette with "normal" strategy */
6397
static int
6398
lookup_normal(unsigned char const * const pixel,
849,900✔
6399
              int const depth,
6400
              unsigned char const * const palette,
6401
              int const reqcolor,
6402
              unsigned short * const cachetable,
6403
              int const complexion)
6404
{
6405
    int result;
6406
    int diff;
6407
    int r;
6408
    int i;
6409
    int n;
6410
    int distant;
6411

6412
    result = (-1);
849,900✔
6413
    diff = INT_MAX;
849,900✔
6414

6415
    /* don't use cachetable in 'normal' strategy */
6416
    (void) cachetable;
283,300✔
6417

6418
    for (i = 0; i < reqcolor; i++) {
15,309,900✔
6419
        distant = 0;
14,460,000✔
6420
        r = pixel[0] - palette[i * depth + 0];
14,460,000✔
6421
        distant += r * r * complexion;
14,460,000✔
6422
        for (n = 1; n < depth; ++n) {
43,380,000✔
6423
            r = pixel[n] - palette[i * depth + n];
28,920,000✔
6424
            distant += r * r;
28,920,000✔
6425
        }
9,640,000✔
6426
        if (distant < diff) {
14,460,000✔
6427
            diff = distant;
2,234,570✔
6428
            result = i;
2,234,570✔
6429
        }
744,404✔
6430
    }
4,820,000✔
6431

6432
    return result;
849,900✔
6433
}
6434

6435

6436
/*
6437
 * Shared fast lookup flow:
6438
 *
6439
 *   pixel --> quantize --> cuckoo cache --> palette index
6440
 */
6441
static int
6442
lookup_fast_common(unsigned char const *pixel,
40,077,958✔
6443
                   unsigned char const *palette,
6444
                   int reqcolor,
6445
                   unsigned short *cachetable,
6446
                   int complexion,
6447
                   struct histogram_control control)
6448
{
6449
    int result;
6450
    unsigned int hash;
6451
    int diff;
6452
    int i;
6453
    int distant;
6454
    struct cuckoo_table32 *table;
6455
    uint32_t *slot;
6456
    SIXELSTATUS status;
6457
    unsigned char const *entry;
6458
    unsigned char const *end;
6459
    int pixel0;
6460
    int pixel1;
6461
    int pixel2;
6462
    int delta;
6463

6464
    result = (-1);
40,077,958✔
6465
    diff = INT_MAX;
40,077,958✔
6466
    hash = computeHash(pixel, 3U, &control);
40,077,958✔
6467

6468
    table = (struct cuckoo_table32 *)cachetable;
40,077,958✔
6469
    if (table != NULL) {
40,077,958!
6470
        slot = cuckoo_table32_lookup(table, hash);
40,077,958✔
6471
        if (slot != NULL && *slot != 0U) {
40,077,958!
6472
            return (int)(*slot - 1U);
38,162,483✔
6473
        }
6474
    }
613,543✔
6475

6476
    entry = palette;
1,915,475✔
6477
    end = palette + (size_t)reqcolor * 3;
1,915,475✔
6478
    pixel0 = (int)pixel[0];
1,915,475✔
6479
    pixel1 = (int)pixel[1];
1,915,475✔
6480
    pixel2 = (int)pixel[2];
1,915,475✔
6481
    /*
6482
     * Palette traversal as RGB triplets keeps the stride linear:
6483
     *
6484
     *   i -> [R][G][B]
6485
     *        |  |  |
6486
     *        `--+--'
6487
     *           v
6488
     *         entry
6489
     */
6490
    for (i = 0; entry < end; ++i, entry += 3) {
360,360,099✔
6491
        delta = pixel0 - (int)entry[0];
358,444,624✔
6492
        distant = delta * delta * complexion;
358,444,624✔
6493
        delta = pixel1 - (int)entry[1];
358,444,624✔
6494
        distant += delta * delta;
358,444,624✔
6495
        delta = pixel2 - (int)entry[2];
358,444,624✔
6496
        distant += delta * delta;
358,444,624✔
6497
        if (distant < diff) {
358,444,624✔
6498
            diff = distant;
15,983,557✔
6499
            result = i;
15,983,557✔
6500
        }
5,029,727✔
6501
    }
114,439,102✔
6502

6503
    if (table != NULL && result >= 0) {
1,915,475!
6504
        status = cuckoo_table32_insert(table,
2,529,018✔
6505
                                       hash,
613,543✔
6506
                                       (uint32_t)(result + 1));
1,915,475✔
6507
        if (SIXEL_FAILED(status)) {
1,915,475!
6508
            /* ignore cache update failure */
6509
        }
6510
    }
613,543✔
6511

6512
    return result;
1,915,475✔
6513
}
19,005,818✔
6514

6515
/* lookup closest color from palette with "fast" strategy */
6516
static int
6517
lookup_fast(unsigned char const * const pixel,
40,077,958✔
6518
            int const depth,
6519
            unsigned char const * const palette,
6520
            int const reqcolor,
6521
            unsigned short * const cachetable,
6522
            int const complexion)
6523
{
6524
    struct histogram_control control;
6525

6526
    (void)depth;
19,005,818✔
6527

6528
    control = histogram_control_make(3U);
40,077,958✔
6529

6530
    return lookup_fast_common(pixel,
59,083,776✔
6531
                              palette,
19,005,818✔
6532
                              reqcolor,
19,005,818✔
6533
                              cachetable,
19,005,818✔
6534
                              complexion,
19,005,818✔
6535
                              control);
6536
}
6537

6538
static int
6539
lookup_fast_robinhood(unsigned char const * const pixel,
×
6540
                      int const depth,
6541
                      unsigned char const * const palette,
6542
                      int const reqcolor,
6543
                      unsigned short * const cachetable,
6544
                      int const complexion)
6545
{
6546
    struct histogram_control control;
6547

6548
    (void)depth;
6549

6550
    control = histogram_control_make_for_policy(3U,
×
6551
                                                SIXEL_LUT_POLICY_ROBINHOOD);
6552

6553
    return lookup_fast_common(pixel,
×
6554
                              palette,
6555
                              reqcolor,
6556
                              cachetable,
6557
                              complexion,
6558
                              control);
6559
}
6560

6561
static int
6562
lookup_fast_hopscotch(unsigned char const * const pixel,
×
6563
                      int const depth,
6564
                      unsigned char const * const palette,
6565
                      int const reqcolor,
6566
                      unsigned short * const cachetable,
6567
                      int const complexion)
6568
{
6569
    struct histogram_control control;
6570

6571
    (void)depth;
6572

6573
    control = histogram_control_make_for_policy(3U,
×
6574
                                                SIXEL_LUT_POLICY_HOPSCOTCH);
6575

6576
    return lookup_fast_common(pixel,
×
6577
                              palette,
6578
                              reqcolor,
6579
                              cachetable,
6580
                              complexion,
6581
                              control);
6582
}
6583

6584
static int
NEW
6585
lookup_fast_certlut(unsigned char const * const pixel,
×
6586
                    int const depth,
6587
                    unsigned char const * const palette,
6588
                    int const reqcolor,
6589
                    unsigned short * const cachetable,
6590
                    int const complexion)
6591
{
6592
    (void)depth;
6593
    (void)palette;
6594
    (void)reqcolor;
6595
    (void)cachetable;
6596
    (void)complexion;
6597

NEW
6598
    if (certlut_context == NULL) {
×
NEW
6599
        return 0;
×
6600
    }
6601

NEW
6602
    return (int)sixel_certlut_lookup(certlut_context,
×
NEW
6603
                                     pixel[0],
×
NEW
6604
                                     pixel[1],
×
NEW
6605
                                     pixel[2]);
×
6606
}
6607

6608

6609
static int
6610
lookup_mono_darkbg(unsigned char const * const pixel,
1,730,520✔
6611
                   int const depth,
6612
                   unsigned char const * const palette,
6613
                   int const reqcolor,
6614
                   unsigned short * const cachetable,
6615
                   int const complexion)
6616
{
6617
    int n;
6618
    int distant;
6619

6620
    /* unused */ (void) palette;
576,840✔
6621
    /* unused */ (void) cachetable;
576,840✔
6622
    /* unused */ (void) complexion;
576,840✔
6623

6624
    distant = 0;
1,730,520✔
6625
    for (n = 0; n < depth; ++n) {
6,922,080✔
6626
        distant += pixel[n];
5,191,560✔
6627
    }
1,730,520✔
6628
    return distant >= 128 * reqcolor ? 1: 0;
1,730,520✔
6629
}
6630

6631

6632
static int
6633
lookup_mono_lightbg(unsigned char const * const pixel,
810,000✔
6634
                    int const depth,
6635
                    unsigned char const * const palette,
6636
                    int const reqcolor,
6637
                    unsigned short * const cachetable,
6638
                    int const complexion)
6639
{
6640
    int n;
6641
    int distant;
6642

6643
    /* unused */ (void) palette;
270,000✔
6644
    /* unused */ (void) cachetable;
270,000✔
6645
    /* unused */ (void) complexion;
270,000✔
6646

6647
    distant = 0;
810,000✔
6648
    for (n = 0; n < depth; ++n) {
3,240,000✔
6649
        distant += pixel[n];
2,430,000✔
6650
    }
810,000✔
6651
    return distant < 128 * reqcolor ? 1: 0;
810,000✔
6652
}
6653

6654

6655
static SIXELSTATUS
6656
build_palette_kmeans(unsigned char **result,
×
6657
                     unsigned char const *data,
6658
                     unsigned int length,
6659
                     unsigned int depth,
6660
                     unsigned int reqcolors,
6661
                     unsigned int *ncolors,
6662
                     unsigned int *origcolors,
6663
                     int quality_mode,
6664
                     int force_palette,
6665
                     int final_merge_mode,
6666
                     sixel_allocator_t *allocator)
6667
{
6668
    SIXELSTATUS status;
6669
    unsigned int channels;
6670
    unsigned int pixel_count;
6671
    unsigned int sample_limit;
6672
    unsigned int sample_cap;
6673
    unsigned int valid_seen;
6674
    unsigned int sample_count;
6675
    unsigned int k;
6676
    unsigned int index;
6677
    unsigned int channel;
6678
    unsigned int center_index;
6679
    unsigned int sample_index;
6680
    unsigned int replace;
6681
    unsigned int max_iterations;
6682
    unsigned int iteration;
6683
    unsigned int best_index;
6684
    unsigned int old_cluster;
6685
    unsigned int farthest_index;
6686
    unsigned int fill;
6687
    unsigned int source;
6688
    unsigned int swap_temp;
6689
    unsigned int base;
6690
    unsigned int extra_component;
6691
    unsigned int *membership;
6692
    unsigned int *order;
6693
    unsigned char *samples;
6694
    unsigned char *palette;
6695
    unsigned char *new_palette;
6696
    double *centers;
6697
    double *distance_cache;
6698
    double total_weight;
6699
    double random_point;
6700
    double best_distance;
6701
    double distance;
6702
    double diff;
6703
    double update;
6704
    double farthest_distance;
6705
    unsigned long *counts;
6706
    unsigned long *accum;
6707
    unsigned long *channel_sum;
6708
    unsigned long rand_value;
6709
    int changed;
6710
    int apply_merge;
6711
    unsigned int overshoot;
6712
    unsigned int refine_iterations;
6713
    int cluster_total;
6714

6715
    status = SIXEL_BAD_ARGUMENT;
×
6716
    channels = depth;
×
6717
    pixel_count = 0U;
×
6718
    sample_limit = 50000U;
×
6719
    sample_cap = sample_limit;
×
6720
    valid_seen = 0U;
×
6721
    sample_count = 0U;
×
6722
    k = 0U;
×
6723
    index = 0U;
×
6724
    channel = 0U;
×
6725
    center_index = 0U;
×
6726
    sample_index = 0U;
×
6727
    replace = 0U;
×
6728
    max_iterations = 0U;
×
6729
    iteration = 0U;
×
6730
    best_index = 0U;
×
6731
    old_cluster = 0U;
×
6732
    farthest_index = 0U;
×
6733
    fill = 0U;
×
6734
    source = 0U;
×
6735
    swap_temp = 0U;
×
6736
    base = 0U;
×
6737
    extra_component = 0U;
×
6738
    membership = NULL;
×
6739
    order = NULL;
×
6740
    samples = NULL;
×
6741
    palette = NULL;
×
6742
    new_palette = NULL;
×
6743
    centers = NULL;
×
6744
    distance_cache = NULL;
×
6745
    counts = NULL;
×
6746
    accum = NULL;
×
6747
    channel_sum = NULL;
×
6748
    rand_value = 0UL;
×
6749
    total_weight = 0.0;
×
6750
    random_point = 0.0;
×
6751
    best_distance = 0.0;
×
6752
    distance = 0.0;
×
6753
    diff = 0.0;
×
6754
    update = 0.0;
×
6755
    farthest_distance = 0.0;
×
6756
    changed = 0;
×
6757
    apply_merge = 0;
×
6758
    overshoot = 0U;
×
6759
    refine_iterations = 0U;
×
6760
    cluster_total = 0;
×
6761

6762
    if (result != NULL) {
×
6763
        *result = NULL;
×
6764
    }
6765
    if (ncolors != NULL) {
×
6766
        *ncolors = 0U;
×
6767
    }
6768
    if (origcolors != NULL) {
×
6769
        *origcolors = 0U;
×
6770
    }
6771

6772
    if (channels != 3U && channels != 4U) {
×
6773
        goto end;
×
6774
    }
6775
    if (channels == 0U) {
×
6776
        goto end;
×
6777
    }
6778

6779
    pixel_count = length / channels;
×
6780
    if (pixel_count == 0U) {
×
6781
        goto end;
×
6782
    }
6783
    if (pixel_count < sample_cap) {
×
6784
        sample_cap = pixel_count;
×
6785
    }
6786
    if (sample_cap == 0U) {
×
6787
        sample_cap = 1U;
×
6788
    }
6789

6790
    samples = (unsigned char *)sixel_allocator_malloc(
×
6791
        allocator, (size_t)sample_cap * 3U);
×
6792
    if (samples == NULL) {
×
6793
        status = SIXEL_BAD_ALLOCATION;
×
6794
        goto end;
×
6795
    }
6796

6797
    /*
6798
     * Reservoir sampling keeps the distribution fair when the image is
6799
     * larger than our budget. Transparent pixels are skipped so that the
6800
     * solver only sees visible colors.
6801
     */
6802
    for (index = 0U; index < pixel_count; ++index) {
×
6803
        base = index * channels;
×
6804
        if (channels == 4U && data[base + 3U] == 0U) {
×
6805
            continue;
×
6806
        }
6807
        ++valid_seen;
×
6808
        if (sample_count < sample_cap) {
×
6809
            for (channel = 0U; channel < 3U; ++channel) {
×
6810
                samples[sample_count * 3U + channel] =
×
6811
                    data[base + channel];
×
6812
            }
6813
            ++sample_count;
×
6814
        } else {
6815
            rand_value = (unsigned long)rand();
×
6816
            replace = (unsigned int)(rand_value % valid_seen);
×
6817
            if (replace < sample_cap) {
×
6818
                for (channel = 0U; channel < 3U; ++channel) {
×
6819
                    samples[replace * 3U + channel] =
×
6820
                        data[base + channel];
×
6821
                }
6822
            }
6823
        }
6824
    }
6825

6826
    if (origcolors != NULL) {
×
6827
        *origcolors = valid_seen;
×
6828
    }
6829
    if (sample_count == 0U) {
×
6830
        goto end;
×
6831
    }
6832

6833
    if (reqcolors == 0U) {
×
6834
        reqcolors = 1U;
×
6835
    }
6836
    apply_merge = (final_merge_mode == SIXEL_FINAL_MERGE_AUTO
×
6837
                   || final_merge_mode == SIXEL_FINAL_MERGE_WARD);
×
6838
    refine_iterations = 2U;
×
6839
    overshoot = reqcolors;
×
6840
    /* Oversplit so the subsequent Ward merge has room to consolidate. */
6841
    if (apply_merge) {
×
6842
        overshoot = sixel_final_merge_target(reqcolors,
×
6843
                                             final_merge_mode);
6844
        quant_trace(stderr, "overshoot: %d\n", overshoot);
×
6845
    }
6846
    if (overshoot > sample_count) {
×
6847
        overshoot = sample_count;
×
6848
    }
6849
    k = overshoot;
×
6850
    if (k == 0U) {
×
6851
        goto end;
×
6852
    }
6853

6854
    centers = (double *)sixel_allocator_malloc(
×
6855
        allocator, (size_t)k * 3U * sizeof(double));
×
6856
    distance_cache = (double *)sixel_allocator_malloc(
×
6857
        allocator, (size_t)sample_count * sizeof(double));
×
6858
    counts = (unsigned long *)sixel_allocator_malloc(
×
6859
        allocator, (size_t)k * sizeof(unsigned long));
×
6860
    accum = (unsigned long *)sixel_allocator_malloc(
×
6861
        allocator, (size_t)k * 3U * sizeof(unsigned long));
×
6862
    membership = (unsigned int *)sixel_allocator_malloc(
×
6863
        allocator, (size_t)sample_count * sizeof(unsigned int));
×
6864
    if (centers == NULL || distance_cache == NULL || counts == NULL ||
×
6865
            accum == NULL || membership == NULL) {
×
6866
        status = SIXEL_BAD_ALLOCATION;
×
6867
        goto end;
×
6868
    }
6869

6870
    /*
6871
     * Seed the first center uniformly from the sampled set. Subsequent
6872
     * centers use k-means++ to favour distant samples.
6873
     */
6874
    rand_value = (unsigned long)rand();
×
6875
    replace = (unsigned int)(rand_value % sample_count);
×
6876
    for (channel = 0U; channel < 3U; ++channel) {
×
6877
        centers[channel] =
×
6878
            (double)samples[replace * 3U + channel];
×
6879
    }
6880
    for (sample_index = 0U; sample_index < sample_count; ++sample_index) {
×
6881
        distance = 0.0;
×
6882
        for (channel = 0U; channel < 3U; ++channel) {
×
6883
            diff = (double)samples[sample_index * 3U + channel]
×
6884
                - centers[channel];
×
6885
            distance += diff * diff;
×
6886
        }
6887
        distance_cache[sample_index] = distance;
×
6888
    }
6889

6890
    for (center_index = 1U; center_index < k; ++center_index) {
×
6891
        total_weight = 0.0;
×
6892
        for (sample_index = 0U; sample_index < sample_count;
×
6893
                ++sample_index) {
×
6894
            total_weight += distance_cache[sample_index];
×
6895
        }
6896
        random_point = 0.0;
×
6897
        if (total_weight > 0.0) {
×
6898
            random_point =
×
6899
                ((double)rand() / ((double)RAND_MAX + 1.0)) *
×
6900
                total_weight;
6901
        }
6902
        sample_index = 0U;
×
6903
        while (sample_index + 1U < sample_count &&
×
6904
               random_point > distance_cache[sample_index]) {
×
6905
            random_point -= distance_cache[sample_index];
×
6906
            ++sample_index;
×
6907
        }
6908
        for (channel = 0U; channel < 3U; ++channel) {
×
6909
            centers[center_index * 3U + channel] =
×
6910
                (double)samples[sample_index * 3U + channel];
×
6911
        }
6912
        for (index = 0U; index < sample_count; ++index) {
×
6913
            distance = 0.0;
×
6914
            for (channel = 0U; channel < 3U; ++channel) {
×
6915
                diff = (double)samples[index * 3U + channel]
×
6916
                    - centers[center_index * 3U + channel];
×
6917
                distance += diff * diff;
×
6918
            }
6919
            if (distance < distance_cache[index]) {
×
6920
                distance_cache[index] = distance;
×
6921
            }
6922
        }
6923
    }
6924

6925
    switch (quality_mode) {
×
6926
    case SIXEL_QUALITY_LOW:
6927
        max_iterations = 6U;
×
6928
        break;
×
6929
    case SIXEL_QUALITY_HIGH:
6930
        max_iterations = 24U;
×
6931
        break;
×
6932
    case SIXEL_QUALITY_FULL:
6933
        max_iterations = 48U;
×
6934
        break;
×
6935
    case SIXEL_QUALITY_HIGHCOLOR:
6936
        max_iterations = 24U;
×
6937
        break;
×
6938
    case SIXEL_QUALITY_AUTO:
×
6939
    default:
6940
        max_iterations = 12U;
×
6941
        break;
×
6942
    }
6943
    if (max_iterations == 0U) {
×
6944
        max_iterations = 1U;
×
6945
    }
6946
    if (max_iterations > 20U) {
×
6947
        /*
6948
         * The requirements cap the Lloyd refinement at twenty passes to
6949
         * keep runtime bounded even for demanding quality presets.
6950
         */
6951
        max_iterations = 20U;
×
6952
    }
6953

6954
    /*
6955
     * Lloyd refinement assigns samples to their nearest center and moves
6956
     * each center to the mean of its cluster. Empty clusters are reseeded
6957
     * using the farthest sample to improve stability.
6958
     */
6959
    for (iteration = 0U; iteration < max_iterations; ++iteration) {
×
6960
        for (index = 0U; index < k; ++index) {
×
6961
            counts[index] = 0UL;
×
6962
        }
6963
        for (index = 0U; index < k * 3U; ++index) {
×
6964
            accum[index] = 0UL;
×
6965
        }
6966
        for (sample_index = 0U; sample_index < sample_count;
×
6967
                ++sample_index) {
×
6968
            best_index = 0U;
×
6969
            distance = 0.0;
×
6970
            for (channel = 0U; channel < 3U; ++channel) {
×
6971
                diff = (double)samples[sample_index * 3U + channel]
×
6972
                    - centers[channel];
×
6973
                distance += diff * diff;
×
6974
            }
6975
            best_distance = distance;
×
6976
            for (center_index = 1U; center_index < k;
×
6977
                    ++center_index) {
×
6978
                distance = 0.0;
×
6979
                for (channel = 0U; channel < 3U; ++channel) {
×
6980
                    diff = (double)samples[sample_index * 3U + channel]
×
6981
                        - centers[center_index * 3U + channel];
×
6982
                    distance += diff * diff;
×
6983
                }
6984
                if (distance < best_distance) {
×
6985
                    best_distance = distance;
×
6986
                    best_index = center_index;
×
6987
                }
6988
            }
6989
            membership[sample_index] = best_index;
×
6990
            distance_cache[sample_index] = best_distance;
×
6991
            counts[best_index] += 1UL;
×
6992
            channel_sum = accum + (size_t)best_index * 3U;
×
6993
            for (channel = 0U; channel < 3U; ++channel) {
×
6994
                channel_sum[channel] +=
×
6995
                    (unsigned long)samples[sample_index * 3U + channel];
×
6996
            }
6997
        }
6998
        for (center_index = 0U; center_index < k; ++center_index) {
×
6999
            if (counts[center_index] != 0UL) {
×
7000
                continue;
×
7001
            }
7002
            farthest_distance = -1.0;
×
7003
            farthest_index = 0U;
×
7004
            for (sample_index = 0U; sample_index < sample_count;
×
7005
                    ++sample_index) {
×
7006
                if (distance_cache[sample_index] > farthest_distance) {
×
7007
                    farthest_distance = distance_cache[sample_index];
×
7008
                    farthest_index = sample_index;
×
7009
                }
7010
            }
7011
            old_cluster = membership[farthest_index];
×
7012
            if (counts[old_cluster] > 0UL) {
×
7013
                counts[old_cluster] -= 1UL;
×
7014
                channel_sum = accum + (size_t)old_cluster * 3U;
×
7015
                for (channel = 0U; channel < 3U; ++channel) {
×
7016
                    extra_component =
×
7017
                        (unsigned int)samples[farthest_index * 3U + channel];
×
7018
                    if (channel_sum[channel] >=
×
7019
                            (unsigned long)extra_component) {
×
7020
                        channel_sum[channel] -=
×
7021
                            (unsigned long)extra_component;
×
7022
                    } else {
7023
                        channel_sum[channel] = 0UL;
×
7024
                    }
7025
                }
7026
            }
7027
            membership[farthest_index] = center_index;
×
7028
            counts[center_index] = 1UL;
×
7029
            channel_sum = accum + (size_t)center_index * 3U;
×
7030
            for (channel = 0U; channel < 3U; ++channel) {
×
7031
                channel_sum[channel] =
×
7032
                    (unsigned long)samples[farthest_index * 3U + channel];
×
7033
            }
7034
            distance_cache[farthest_index] = 0.0;
×
7035
        }
7036
        changed = 0;
×
7037
        for (center_index = 0U; center_index < k; ++center_index) {
×
7038
            if (counts[center_index] == 0UL) {
×
7039
                continue;
×
7040
            }
7041
            channel_sum = accum + (size_t)center_index * 3U;
×
7042
            for (channel = 0U; channel < 3U; ++channel) {
×
7043
                update = (double)channel_sum[channel]
×
7044
                    / (double)counts[center_index];
×
7045
                diff = centers[center_index * 3U + channel] - update;
×
7046
                if (diff < 0.0) {
×
7047
                    diff = -diff;
×
7048
                }
7049
                if (diff > 0.5) {
×
7050
                    changed = 1;
×
7051
                }
7052
                centers[center_index * 3U + channel] = update;
×
7053
            }
7054
        }
7055
        if (!changed) {
×
7056
            break;
×
7057
        }
7058
    }
7059

7060
    if (apply_merge && k > reqcolors) {
×
7061
        /* Merge the provisional clusters and polish with a few Lloyd steps. */
7062
        cluster_total = (int)k;
×
7063
        sixel_merge_clusters_ward(counts, accum, 3U,
×
7064
                                  &cluster_total, (int)reqcolors);
7065
        if (cluster_total < 1) {
×
7066
            cluster_total = 1;
×
7067
        }
7068
        if ((unsigned int)cluster_total > reqcolors) {
×
7069
            cluster_total = (int)reqcolors;
×
7070
        }
7071
        k = (unsigned int)cluster_total;
×
7072
        if (k == 0U) {
×
7073
            k = 1U;
×
7074
        }
7075
        for (center_index = 0U; center_index < k; ++center_index) {
×
7076
            if (counts[center_index] == 0UL) {
×
7077
                counts[center_index] = 1UL;
×
7078
            }
7079
            channel_sum = accum + (size_t)center_index * 3U;
×
7080
            for (channel = 0U; channel < 3U; ++channel) {
×
7081
                centers[center_index * 3U + channel] =
×
7082
                    (double)channel_sum[channel]
×
7083
                    / (double)counts[center_index];
×
7084
            }
7085
        }
7086
        for (iteration = 0U; iteration < refine_iterations; ++iteration) {
×
7087
            for (index = 0U; index < k; ++index) {
×
7088
                counts[index] = 0UL;
×
7089
            }
7090
            for (index = 0U; index < k * 3U; ++index) {
×
7091
                accum[index] = 0UL;
×
7092
            }
7093
            for (sample_index = 0U; sample_index < sample_count;
×
7094
                    ++sample_index) {
×
7095
                best_index = 0U;
×
7096
                best_distance = 0.0;
×
7097
                for (channel = 0U; channel < 3U; ++channel) {
×
7098
                    diff = (double)samples[sample_index * 3U + channel]
×
7099
                        - centers[channel];
×
7100
                    best_distance += diff * diff;
×
7101
                }
7102
                for (center_index = 1U; center_index < k;
×
7103
                        ++center_index) {
×
7104
                    distance = 0.0;
×
7105
                    for (channel = 0U; channel < 3U; ++channel) {
×
7106
                        diff = (double)samples[sample_index * 3U + channel]
×
7107
                            - centers[center_index * 3U + channel];
×
7108
                        distance += diff * diff;
×
7109
                    }
7110
                    if (distance < best_distance) {
×
7111
                        best_distance = distance;
×
7112
                        best_index = center_index;
×
7113
                    }
7114
                }
7115
                membership[sample_index] = best_index;
×
7116
                distance_cache[sample_index] = best_distance;
×
7117
                counts[best_index] += 1UL;
×
7118
                channel_sum = accum + (size_t)best_index * 3U;
×
7119
                for (channel = 0U; channel < 3U; ++channel) {
×
7120
                    channel_sum[channel] +=
×
7121
                        (unsigned long)samples[sample_index * 3U + channel];
×
7122
                }
7123
            }
7124
            for (center_index = 0U; center_index < k; ++center_index) {
×
7125
                if (counts[center_index] != 0UL) {
×
7126
                    continue;
×
7127
                }
7128
                farthest_distance = -1.0;
×
7129
                farthest_index = 0U;
×
7130
                for (sample_index = 0U; sample_index < sample_count;
×
7131
                        ++sample_index) {
×
7132
                    if (distance_cache[sample_index] > farthest_distance) {
×
7133
                        farthest_distance = distance_cache[sample_index];
×
7134
                        farthest_index = sample_index;
×
7135
                    }
7136
                }
7137
                old_cluster = membership[farthest_index];
×
7138
                if (counts[old_cluster] > 0UL) {
×
7139
                    counts[old_cluster] -= 1UL;
×
7140
                    channel_sum = accum + (size_t)old_cluster * 3U;
×
7141
                    for (channel = 0U; channel < 3U; ++channel) {
×
7142
                        extra_component =
×
7143
                            (unsigned int)samples[farthest_index * 3U + channel];
×
7144
                        if (channel_sum[channel] >=
×
7145
                                (unsigned long)extra_component) {
×
7146
                            channel_sum[channel] -=
×
7147
                                (unsigned long)extra_component;
×
7148
                        } else {
7149
                            channel_sum[channel] = 0UL;
×
7150
                        }
7151
                    }
7152
                }
7153
                membership[farthest_index] = center_index;
×
7154
                counts[center_index] = 1UL;
×
7155
                channel_sum = accum + (size_t)center_index * 3U;
×
7156
                for (channel = 0U; channel < 3U; ++channel) {
×
7157
                    channel_sum[channel] =
×
7158
                        (unsigned long)samples[farthest_index * 3U + channel];
×
7159
                }
7160
                distance_cache[farthest_index] = 0.0;
×
7161
            }
7162
            changed = 0;
×
7163
            for (center_index = 0U; center_index < k; ++center_index) {
×
7164
                if (counts[center_index] == 0UL) {
×
7165
                    continue;
×
7166
                }
7167
                channel_sum = accum + (size_t)center_index * 3U;
×
7168
                for (channel = 0U; channel < 3U; ++channel) {
×
7169
                    update = (double)channel_sum[channel]
×
7170
                        / (double)counts[center_index];
×
7171
                    diff = centers[center_index * 3U + channel] - update;
×
7172
                    if (diff < 0.0) {
×
7173
                        diff = -diff;
×
7174
                    }
7175
                    if (diff > 0.5) {
×
7176
                        changed = 1;
×
7177
                    }
7178
                    centers[center_index * 3U + channel] = update;
×
7179
                }
7180
            }
7181
            if (!changed) {
×
7182
                break;
×
7183
            }
7184
        }
7185
    }
7186

7187
    /*
7188
     * Convert the floating point centers back into the byte palette that
7189
     * callers expect.
7190
     */
7191
    palette = (unsigned char *)sixel_allocator_malloc(
×
7192
        allocator, (size_t)k * 3U);
×
7193
    if (palette == NULL) {
×
7194
        status = SIXEL_BAD_ALLOCATION;
×
7195
        goto end;
×
7196
    }
7197
    for (center_index = 0U; center_index < k; ++center_index) {
×
7198
        for (channel = 0U; channel < 3U; ++channel) {
×
7199
            update = centers[center_index * 3U + channel];
×
7200
            if (update < 0.0) {
×
7201
                update = 0.0;
×
7202
            }
7203
            if (update > 255.0) {
×
7204
                update = 255.0;
×
7205
            }
7206
            palette[center_index * 3U + channel] =
×
7207
                (unsigned char)(update + 0.5);
×
7208
        }
7209
    }
7210

7211
    if (force_palette && k < reqcolors) {
×
7212
        /*
7213
         * Populate the tail of the palette by repeating the most frequent
7214
         * clusters so the caller still receives the requested palette size.
7215
         */
7216
        new_palette = (unsigned char *)sixel_allocator_malloc(
×
7217
            allocator, (size_t)reqcolors * 3U);
×
7218
        if (new_palette == NULL) {
×
7219
            status = SIXEL_BAD_ALLOCATION;
×
7220
            goto end;
×
7221
        }
7222
        for (index = 0U; index < k * 3U; ++index) {
×
7223
            new_palette[index] = palette[index];
×
7224
        }
7225
        order = (unsigned int *)sixel_allocator_malloc(
×
7226
            allocator, (size_t)k * sizeof(unsigned int));
×
7227
        if (order == NULL) {
×
7228
            status = SIXEL_BAD_ALLOCATION;
×
7229
            goto end;
×
7230
        }
7231
        for (index = 0U; index < k; ++index) {
×
7232
            order[index] = index;
×
7233
        }
7234
        for (index = 0U; index < k; ++index) {
×
7235
            for (center_index = index + 1U; center_index < k;
×
7236
                    ++center_index) {
×
7237
                if (counts[order[center_index]] >
×
7238
                        counts[order[index]]) {
×
7239
                    swap_temp = order[index];
×
7240
                    order[index] = order[center_index];
×
7241
                    order[center_index] = swap_temp;
×
7242
                }
7243
            }
7244
        }
7245
        fill = k;
×
7246
        source = 0U;
×
7247
        while (fill < reqcolors && k > 0U) {
×
7248
            center_index = order[source];
×
7249
            for (channel = 0U; channel < 3U; ++channel) {
×
7250
                new_palette[fill * 3U + channel] =
×
7251
                    palette[center_index * 3U + channel];
×
7252
            }
7253
            ++fill;
×
7254
            ++source;
×
7255
            if (source >= k) {
×
7256
                source = 0U;
×
7257
            }
7258
        }
7259
        sixel_allocator_free(allocator, palette);
×
7260
        palette = new_palette;
×
7261
        new_palette = NULL;
×
7262
        k = reqcolors;
×
7263
    }
7264

7265
    status = SIXEL_OK;
×
7266
    if (result != NULL) {
×
7267
        *result = palette;
×
7268
    } else {
7269
        palette = NULL;
×
7270
    }
7271
    if (ncolors != NULL) {
×
7272
        *ncolors = k;
×
7273
    }
7274

7275
end:
7276
    if (status != SIXEL_OK && palette != NULL) {
×
7277
        sixel_allocator_free(allocator, palette);
×
7278
    }
7279
    if (new_palette != NULL) {
×
7280
        sixel_allocator_free(allocator, new_palette);
×
7281
    }
7282
    if (order != NULL) {
×
7283
        sixel_allocator_free(allocator, order);
×
7284
    }
7285
    if (membership != NULL) {
×
7286
        sixel_allocator_free(allocator, membership);
×
7287
    }
7288
    if (accum != NULL) {
×
7289
        sixel_allocator_free(allocator, accum);
×
7290
    }
7291
    if (counts != NULL) {
×
7292
        sixel_allocator_free(allocator, counts);
×
7293
    }
7294
    if (distance_cache != NULL) {
×
7295
        sixel_allocator_free(allocator, distance_cache);
×
7296
    }
7297
    if (centers != NULL) {
×
7298
        sixel_allocator_free(allocator, centers);
×
7299
    }
7300
    if (samples != NULL) {
×
7301
        sixel_allocator_free(allocator, samples);
×
7302
    }
7303
    return status;
×
7304
}
7305

7306

7307
/* choose colors using median-cut method */
7308
SIXELSTATUS
7309
sixel_quant_make_palette(
260✔
7310
    unsigned char          /* out */ **result,
7311
    unsigned char const    /* in */  *data,
7312
    unsigned int           /* in */  length,
7313
    int                    /* in */  pixelformat,
7314
    unsigned int           /* in */  reqcolors,
7315
    unsigned int           /* in */  *ncolors,
7316
    unsigned int           /* in */  *origcolors,
7317
    int                    /* in */  methodForLargest,
7318
    int                    /* in */  methodForRep,
7319
    int                    /* in */  qualityMode,
7320
    int                    /* in */  force_palette,
7321
    int                    /* in */  use_reversible,
7322
    int                    /* in */  quantize_model,
7323
    int                    /* in */  final_merge_mode,
7324
    sixel_allocator_t      /* in */  *allocator)
7325
{
7326
    SIXELSTATUS status = SIXEL_FALSE;
260✔
7327
    unsigned int i;
7328
    unsigned int n;
7329
    int ret;
7330
    tupletable2 colormap;
7331
    unsigned int depth;
7332
    int result_depth;
7333

7334
    result_depth = sixel_helper_compute_depth(pixelformat);
260✔
7335
    if (result_depth <= 0) {
260!
7336
        *result = NULL;
×
7337
        goto end;
×
7338
    }
7339

7340
    depth = (unsigned int)result_depth;
260✔
7341

7342
    if (quantize_model == SIXEL_QUANTIZE_MODEL_KMEANS) {
260!
7343
        status = build_palette_kmeans(result,
×
7344
                                      data,
7345
                                      length,
7346
                                      depth,
7347
                                      reqcolors,
7348
                                      ncolors,
7349
                                      origcolors,
7350
                                      qualityMode,
7351
                                      force_palette,
7352
                                      final_merge_mode,
7353
                                      allocator);
7354
        if (SIXEL_SUCCEEDED(status)) {
×
7355
            if (use_reversible) {
×
7356
                sixel_quant_reversible_palette(*result,
×
7357
                                               *ncolors,
7358
                                               depth);
7359
            }
7360
            return status;
×
7361
        }
7362
    }
7363

7364
    ret = computeColorMapFromInput(data, length, depth,
378✔
7365
                                   reqcolors, methodForLargest,
118✔
7366
                                   methodForRep, qualityMode,
118✔
7367
                                   force_palette, use_reversible,
118✔
7368
                                   final_merge_mode,
118✔
7369
                                   &colormap, origcolors, allocator);
118✔
7370
    if (ret != 0) {
260!
7371
        *result = NULL;
×
7372
        goto end;
×
7373
    }
7374
    *ncolors = colormap.size;
260✔
7375
    quant_trace(stderr, "tupletable size: %d\n", *ncolors);
260✔
7376
    *result = (unsigned char *)sixel_allocator_malloc(allocator, *ncolors * depth);
260✔
7377
    for (i = 0; i < *ncolors; i++) {
17,038✔
7378
        for (n = 0; n < depth; ++n) {
67,112✔
7379
            (*result)[i * depth + n] = colormap.table[i]->tuple[n];
50,334✔
7380
        }
17,988✔
7381
    }
5,996✔
7382

7383
    if (use_reversible) {
260!
7384
        sixel_quant_reversible_palette(*result, *ncolors, depth);
×
7385
    }
7386

7387
    sixel_allocator_free(allocator, colormap.table);
260✔
7388

7389
    status = SIXEL_OK;
260✔
7390

7391
end:
142✔
7392
    return status;
260✔
7393
}
118✔
7394

7395

7396
/* apply color palette into specified pixel buffers */
7397
SIXELSTATUS
7398
sixel_quant_apply_palette(
285✔
7399
    sixel_index_t     /* out */ *result,
7400
    unsigned char     /* in */  *data,
7401
    int               /* in */  width,
7402
    int               /* in */  height,
7403
    int               /* in */  depth,
7404
    unsigned char     /* in */  *palette,
7405
    int               /* in */  reqcolor,
7406
    int               /* in */  methodForDiffuse,
7407
    int               /* in */  methodForScan,
7408
    int               /* in */  methodForCarry,
7409
    int               /* in */  foptimize,
7410
    int               /* in */  foptimize_palette,
7411
    int               /* in */  complexion,
7412
    unsigned short    /* in */  *cachetable,
7413
    int               /* in */  *ncolors,
7414
    sixel_allocator_t /* in */  *allocator)
7415
{
170✔
7416
#if _MSC_VER
7417
    enum { max_depth = 4 };
7418
#else
7419
    const size_t max_depth = 4;
285✔
7420
#endif
7421
    unsigned char copy[max_depth];
170✔
7422
    SIXELSTATUS status = SIXEL_FALSE;
285✔
7423
    int sum1, sum2;
7424
    int n;
7425
    unsigned short *indextable;
7426
    size_t cache_size;
7427
    int allocated_cache;
7428
    int use_cache;
7429
    int cache_policy;
7430
    unsigned char new_palette[SIXEL_PALETTE_MAX * 4];
7431
    unsigned short migration_map[SIXEL_PALETTE_MAX];
7432
    int (*f_lookup)(unsigned char const * const pixel,
7433
                    int const depth,
7434
                    unsigned char const * const palette,
7435
                    int const reqcolor,
7436
                    unsigned short * const cachetable,
7437
                    int const complexion);
7438
    int use_varerr;
7439
    int use_positional;
7440
    int carry_mode;
7441
    sixel_certlut_t certlut;
7442
    int certlut_ready;
7443
    int wR;
7444
    int wG;
7445
    int wB;
7446

7447
    certlut_ready = 0;
285✔
7448
    memset(&certlut, 0, sizeof(certlut));
285✔
7449

7450
    /* check bad reqcolor */
7451
    if (reqcolor < 1) {
285!
7452
        status = SIXEL_BAD_ARGUMENT;
×
7453
        sixel_helper_set_additional_message(
×
7454
            "sixel_quant_apply_palette: "
7455
            "a bad argument is detected, reqcolor < 0.");
7456
        goto end;
×
7457
    }
7458

7459
    /* NOTE: diffuse_jajuni, diffuse_stucki, and diffuse_burkes reference at
7460
     * minimum the position pos + width * 1 - 2, so width must be at least 2
7461
     * to avoid underflow.
7462
     * On the other hand, diffuse_fs and diffuse_atkinson
7463
     * reference pos + width * 1 - 1, but since these functions are only called
7464
     * when width >= 1, they do not cause underflow.
7465
     */
7466
    use_varerr = (depth == 3
400✔
7467
                  && methodForDiffuse == SIXEL_DIFFUSE_LSO2);
285!
7468
    use_positional = (methodForDiffuse == SIXEL_DIFFUSE_A_DITHER
400✔
7469
                      || methodForDiffuse == SIXEL_DIFFUSE_X_DITHER);
285!
7470
    carry_mode = (methodForCarry == SIXEL_CARRY_ENABLE)
285✔
7471
               ? SIXEL_CARRY_ENABLE
7472
               : SIXEL_CARRY_DISABLE;
170!
7473

7474
    f_lookup = NULL;
285✔
7475
    if (reqcolor == 2) {
285✔
7476
        sum1 = 0;
19✔
7477
        sum2 = 0;
19✔
7478
        for (n = 0; n < depth; ++n) {
76✔
7479
            sum1 += palette[n];
57✔
7480
        }
21✔
7481
        for (n = depth; n < depth + depth; ++n) {
76✔
7482
            sum2 += palette[n];
57✔
7483
        }
21✔
7484
        if (sum1 == 0 && sum2 == 255 * 3) {
19!
7485
            f_lookup = lookup_mono_darkbg;
12✔
7486
        } else if (sum1 == 255 * 3 && sum2 == 0) {
11!
7487
            f_lookup = lookup_mono_lightbg;
3✔
7488
        }
1✔
7489
    }
7✔
7490
    if (f_lookup == NULL) {
285✔
7491
        if (foptimize && depth == 3) {
270!
7492
            f_lookup = lookup_fast;
264✔
7493
        } else {
108✔
7494
            f_lookup = lookup_normal;
6✔
7495
        }
7496
    }
110✔
7497

7498
    if (f_lookup == lookup_fast) {
285✔
7499
        if (histogram_lut_policy == SIXEL_LUT_POLICY_ROBINHOOD) {
264!
7500
            f_lookup = lookup_fast_robinhood;
×
7501
        } else if (histogram_lut_policy == SIXEL_LUT_POLICY_HOPSCOTCH) {
264!
7502
            f_lookup = lookup_fast_hopscotch;
×
7503
        } else if (histogram_lut_policy == SIXEL_LUT_POLICY_CERTLUT) {
264!
NEW
7504
            f_lookup = lookup_fast_certlut;
×
7505
        }
7506
    }
108✔
7507

7508
    indextable = cachetable;
285✔
7509
    allocated_cache = 0;
285✔
7510
    cache_policy = SIXEL_LUT_POLICY_AUTO;
285✔
7511
    use_cache = 0;
285✔
7512
    if (f_lookup == lookup_fast) {
285✔
7513
        cache_policy = histogram_lut_policy;
264✔
7514
        use_cache = 1;
264✔
7515
    } else if (f_lookup == lookup_fast_robinhood) {
129!
7516
        cache_policy = SIXEL_LUT_POLICY_ROBINHOOD;
×
7517
        use_cache = 1;
×
7518
    } else if (f_lookup == lookup_fast_hopscotch) {
21!
7519
        cache_policy = SIXEL_LUT_POLICY_HOPSCOTCH;
×
7520
        use_cache = 1;
×
7521
    }
7522
    if (cache_policy == SIXEL_LUT_POLICY_AUTO) {
285!
7523
        cache_policy = SIXEL_LUT_POLICY_6BIT;
285✔
7524
    }
115✔
7525
    if (use_cache) {
285✔
7526
        if (cachetable == NULL) {
264!
7527
            status = sixel_quant_cache_prepare(&indextable,
×
7528
                                               &cache_size,
7529
                                               cache_policy,
7530
                                               reqcolor,
7531
                                               allocator);
7532
            if (SIXEL_FAILED(status)) {
×
7533
                quant_trace(stderr,
×
7534
                            "Unable to allocate lookup cache.\n");
7535
                goto end;
×
7536
            }
7537
            allocated_cache = 1;
×
7538
        } else {
7539
            sixel_quant_cache_clear(indextable, cache_policy);
264✔
7540
        }
7541
    }
108✔
7542

7543
    if (f_lookup == lookup_fast_certlut) {
285!
NEW
7544
        if (depth != 3) {
×
NEW
7545
            status = SIXEL_BAD_ARGUMENT;
×
NEW
7546
            sixel_helper_set_additional_message(
×
7547
                "sixel_quant_apply_palette: "
7548
                "certlut requires RGB pixels.");
NEW
7549
            goto end;
×
7550
        }
NEW
7551
        if (quant_method_for_largest == SIXEL_LARGE_LUM) {
×
NEW
7552
            wR = complexion * 299;
×
NEW
7553
            wG = 587;
×
NEW
7554
            wB = 114;
×
7555
        } else {
NEW
7556
            wR = complexion;
×
NEW
7557
            wG = 1;
×
NEW
7558
            wB = 1;
×
7559
        }
NEW
7560
        status = sixel_certlut_build(&certlut,
×
7561
                                     (sixel_color_t const *)palette,
7562
                                     reqcolor,
7563
                                     wR,
7564
                                     wG,
7565
                                     wB);
NEW
7566
        if (SIXEL_FAILED(status)) {
×
NEW
7567
            goto end;
×
7568
        }
NEW
7569
        certlut_context = &certlut;
×
NEW
7570
        certlut_ready = 1;
×
7571
    }
7572

7573
    if (use_positional) {
285!
7574
        status = apply_palette_positional(result, data, width, height,
×
7575
                                          depth, palette, reqcolor,
7576
                                          methodForDiffuse, methodForScan,
7577
                                          foptimize_palette, f_lookup,
7578
                                          indextable, complexion, copy,
7579
                                          new_palette, migration_map,
7580
                                          ncolors);
7581
    } else if (use_varerr) {
285!
7582
        status = apply_palette_variable(result, data, width, height,
×
7583
                                        depth, palette, reqcolor,
7584
                                        methodForScan, foptimize_palette,
7585
                                        f_lookup, indextable, complexion,
7586
                                        new_palette, migration_map,
7587
                                        ncolors,
7588
                                        methodForDiffuse,
7589
                                        carry_mode);
7590
    } else {
7591
        status = apply_palette_fixed(result, data, width, height,
400✔
7592
                                     depth, palette, reqcolor,
115✔
7593
                                     methodForScan, foptimize_palette,
115✔
7594
                                     f_lookup, indextable, complexion,
115✔
7595
                                     new_palette, migration_map,
115✔
7596
                                     ncolors, methodForDiffuse,
115✔
7597
                                     carry_mode);
115✔
7598
    }
7599
    if (SIXEL_FAILED(status)) {
285!
7600
        goto end;
×
7601
    }
7602

7603
    if (certlut_ready) {
285!
NEW
7604
        certlut_context = NULL;
×
NEW
7605
        sixel_certlut_free(&certlut);
×
NEW
7606
        certlut_ready = 0;
×
7607
    }
7608

7609
    if (allocated_cache) {
285!
7610
        sixel_quant_cache_release(indextable,
×
7611
                                  cache_policy,
7612
                                  allocator);
7613
    }
7614

7615
    status = SIXEL_OK;
285✔
7616

7617
end:
170✔
7618
    if (certlut_ready) {
285!
NEW
7619
        certlut_context = NULL;
×
NEW
7620
        sixel_certlut_free(&certlut);
×
7621
    }
7622
    return status;
285✔
7623
}
7624

7625

7626
void
7627
sixel_quant_free_palette(
260✔
7628
    unsigned char       /* in */ *data,
7629
    sixel_allocator_t   /* in */ *allocator)
7630
{
7631
    sixel_allocator_free(allocator, data);
260✔
7632
}
260✔
7633

7634

7635
#if HAVE_TESTS
7636
static int
7637
test1(void)
×
7638
{
7639
    int nret = EXIT_FAILURE;
×
7640
    sample minval[1] = { 1 };
×
7641
    sample maxval[1] = { 2 };
×
7642
    unsigned int retval;
7643

7644
    retval = largestByLuminosity(minval, maxval, 1);
×
7645
    if (retval != 0) {
×
7646
        goto error;
×
7647
    }
7648
    nret = EXIT_SUCCESS;
×
7649

7650
error:
7651
    return nret;
×
7652
}
7653

7654

7655
int
7656
sixel_quant_tests_main(void)
×
7657
{
7658
    int nret = EXIT_FAILURE;
×
7659
    size_t i;
7660
    typedef int (* testcase)(void);
7661

7662
    static testcase const testcases[] = {
7663
        test1,
7664
    };
7665

7666
    for (i = 0; i < sizeof(testcases) / sizeof(testcase); ++i) {
×
7667
        nret = testcases[i]();
×
7668
        if (nret != EXIT_SUCCESS) {
×
7669
            goto error;
×
7670
        }
7671
    }
7672

7673
    nret = EXIT_SUCCESS;
×
7674

7675
error:
7676
    return nret;
×
7677
}
7678
#endif  /* HAVE_TESTS */
7679

7680
/* emacs Local Variables:      */
7681
/* emacs mode: c               */
7682
/* emacs tab-width: 4          */
7683
/* emacs indent-tabs-mode: nil */
7684
/* emacs c-basic-offset: 4     */
7685
/* emacs End:                  */
7686
/* vim: set expandtab ts=4 sts=4 sw=4 : */
7687
/* EOF */
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc