• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

geo-engine / geoengine / 7006568925

27 Nov 2023 02:07PM UTC coverage: 89.651% (+0.2%) from 89.498%
7006568925

push

github

web-flow
Merge pull request #888 from geo-engine/raster_stacks

raster stacking

4032 of 4274 new or added lines in 107 files covered. (94.34%)

12 existing lines in 8 files now uncovered.

113020 of 126066 relevant lines covered (89.65%)

59901.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.87
/operators/src/pro/cache/cache_tiles.rs
1
use super::cache_stream::CacheStream;
2
use super::error::CacheError;
3
use super::shared_cache::{
4
    CacheBackendElement, CacheBackendElementExt, CacheElement, CacheElementsContainer,
5
    CacheElementsContainerInfos, LandingZoneElementsContainer, RasterCacheQueryEntry,
6
    RasterLandingQueryEntry,
7
};
8
use crate::util::Result;
9
use geoengine_datatypes::primitives::SpatialPartitioned;
10
use geoengine_datatypes::raster::{
11
    BaseTile, EmptyGrid, Grid, GridOrEmpty, GridShape2D, GridSize, GridSpaceToLinearSpace,
12
    MaskedGrid, RasterTile,
13
};
14
use geoengine_datatypes::{
15
    primitives::RasterQueryRectangle,
16
    raster::{Pixel, RasterTile2D},
17
    util::ByteSize,
18
};
19
use std::marker::PhantomData;
20
use std::sync::Arc;
21

22
#[derive(Debug)]
×
23
pub enum CachedTiles {
24
    U8(Arc<Vec<CompressedRasterTile2D<u8>>>),
25
    U16(Arc<Vec<CompressedRasterTile2D<u16>>>),
26
    U32(Arc<Vec<CompressedRasterTile2D<u32>>>),
27
    U64(Arc<Vec<CompressedRasterTile2D<u64>>>),
28
    I8(Arc<Vec<CompressedRasterTile2D<i8>>>),
29
    I16(Arc<Vec<CompressedRasterTile2D<i16>>>),
30
    I32(Arc<Vec<CompressedRasterTile2D<i32>>>),
31
    I64(Arc<Vec<CompressedRasterTile2D<i64>>>),
32
    F32(Arc<Vec<CompressedRasterTile2D<f32>>>),
33
    F64(Arc<Vec<CompressedRasterTile2D<f64>>>),
34
}
35

36
impl ByteSize for CachedTiles {
37
    fn heap_byte_size(&self) -> usize {
10✔
38
        // we need to use `byte_size` instead of `heap_byte_size` here, because `Arc` stores its data on the heap
10✔
39
        match self {
10✔
40
            CachedTiles::U8(tiles) => tiles.byte_size(),
10✔
41
            CachedTiles::U16(tiles) => tiles.byte_size(),
×
42
            CachedTiles::U32(tiles) => tiles.byte_size(),
×
43
            CachedTiles::U64(tiles) => tiles.byte_size(),
×
44
            CachedTiles::I8(tiles) => tiles.byte_size(),
×
45
            CachedTiles::I16(tiles) => tiles.byte_size(),
×
46
            CachedTiles::I32(tiles) => tiles.byte_size(),
×
47
            CachedTiles::I64(tiles) => tiles.byte_size(),
×
48
            CachedTiles::F32(tiles) => tiles.byte_size(),
×
49
            CachedTiles::F64(tiles) => tiles.byte_size(),
×
50
        }
51
    }
10✔
52
}
53

54
impl CachedTiles {
55
    fn is_expired(&self) -> bool {
7✔
56
        match self {
7✔
57
            CachedTiles::U8(v) => v.iter().any(|t| t.cache_hint.is_expired()),
7✔
58
            CachedTiles::U16(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
59
            CachedTiles::U32(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
60
            CachedTiles::U64(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
61
            CachedTiles::I8(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
62
            CachedTiles::I16(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
63
            CachedTiles::I32(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
64
            CachedTiles::I64(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
65
            CachedTiles::F32(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
66
            CachedTiles::F64(v) => v.iter().any(|t| t.cache_hint.is_expired()),
×
67
        }
68
    }
7✔
69
}
70

71
#[derive(Debug)]
×
72
pub enum LandingZoneQueryTiles {
73
    U8(Vec<CompressedRasterTile2D<u8>>),
74
    U16(Vec<CompressedRasterTile2D<u16>>),
75
    U32(Vec<CompressedRasterTile2D<u32>>),
76
    U64(Vec<CompressedRasterTile2D<u64>>),
77
    I8(Vec<CompressedRasterTile2D<i8>>),
78
    I16(Vec<CompressedRasterTile2D<i16>>),
79
    I32(Vec<CompressedRasterTile2D<i32>>),
80
    I64(Vec<CompressedRasterTile2D<i64>>),
81
    F32(Vec<CompressedRasterTile2D<f32>>),
82
    F64(Vec<CompressedRasterTile2D<f64>>),
83
}
84

85
impl LandingZoneQueryTiles {
86
    pub fn len(&self) -> usize {
1✔
87
        match self {
1✔
88
            LandingZoneQueryTiles::U8(v) => v.len(),
1✔
89
            LandingZoneQueryTiles::U16(v) => v.len(),
×
90
            LandingZoneQueryTiles::U32(v) => v.len(),
×
91
            LandingZoneQueryTiles::U64(v) => v.len(),
×
92
            LandingZoneQueryTiles::I8(v) => v.len(),
×
93
            LandingZoneQueryTiles::I16(v) => v.len(),
×
94
            LandingZoneQueryTiles::I32(v) => v.len(),
×
95
            LandingZoneQueryTiles::I64(v) => v.len(),
×
96
            LandingZoneQueryTiles::F32(v) => v.len(),
×
97
            LandingZoneQueryTiles::F64(v) => v.len(),
×
98
        }
99
    }
1✔
100

101
    pub fn is_empty(&self) -> bool {
1✔
102
        self.len() == 0
1✔
103
    }
1✔
104
}
105

106
impl ByteSize for LandingZoneQueryTiles {
107
    fn heap_byte_size(&self) -> usize {
14✔
108
        // we need to use `byte_size` instead of `heap_byte_size` here, because `Vec` stores its data on the heap
14✔
109
        match self {
14✔
110
            LandingZoneQueryTiles::U8(v) => v.byte_size(),
14✔
111
            LandingZoneQueryTiles::U16(v) => v.byte_size(),
×
112
            LandingZoneQueryTiles::U32(v) => v.byte_size(),
×
113
            LandingZoneQueryTiles::U64(v) => v.byte_size(),
×
114
            LandingZoneQueryTiles::I8(v) => v.byte_size(),
×
115
            LandingZoneQueryTiles::I16(v) => v.byte_size(),
×
116
            LandingZoneQueryTiles::I32(v) => v.byte_size(),
×
117
            LandingZoneQueryTiles::I64(v) => v.byte_size(),
×
118
            LandingZoneQueryTiles::F32(v) => v.byte_size(),
×
119
            LandingZoneQueryTiles::F64(v) => v.byte_size(),
×
120
        }
121
    }
14✔
122
}
123

124
impl From<LandingZoneQueryTiles> for CachedTiles {
125
    fn from(value: LandingZoneQueryTiles) -> Self {
7✔
126
        match value {
7✔
127
            LandingZoneQueryTiles::U8(t) => CachedTiles::U8(Arc::new(t)),
7✔
128
            LandingZoneQueryTiles::U16(t) => CachedTiles::U16(Arc::new(t)),
×
129
            LandingZoneQueryTiles::U32(t) => CachedTiles::U32(Arc::new(t)),
×
130
            LandingZoneQueryTiles::U64(t) => CachedTiles::U64(Arc::new(t)),
×
131
            LandingZoneQueryTiles::I8(t) => CachedTiles::I8(Arc::new(t)),
×
132
            LandingZoneQueryTiles::I16(t) => CachedTiles::I16(Arc::new(t)),
×
133
            LandingZoneQueryTiles::I32(t) => CachedTiles::I32(Arc::new(t)),
×
134
            LandingZoneQueryTiles::I64(t) => CachedTiles::I64(Arc::new(t)),
×
135
            LandingZoneQueryTiles::F32(t) => CachedTiles::F32(Arc::new(t)),
×
136
            LandingZoneQueryTiles::F64(t) => CachedTiles::F64(Arc::new(t)),
×
137
        }
138
    }
7✔
139
}
140

141
impl CacheElementsContainerInfos<RasterQueryRectangle> for CachedTiles {
142
    fn is_expired(&self) -> bool {
6✔
143
        self.is_expired()
6✔
144
    }
6✔
145
}
146

147
impl<T> CacheElementsContainer<RasterQueryRectangle, CompressedRasterTile2D<T>> for CachedTiles
148
where
149
    T: Pixel,
150
    CompressedRasterTile2D<T>: CacheBackendElementExt<CacheContainer = CachedTiles>,
151
{
152
    fn results_arc(&self) -> Option<Arc<Vec<CompressedRasterTile2D<T>>>> {
5✔
153
        CompressedRasterTile2D::<T>::results_arc(self)
5✔
154
    }
5✔
155
}
156

157
impl<T> LandingZoneElementsContainer<CompressedRasterTile2D<T>> for LandingZoneQueryTiles
158
where
159
    T: Pixel,
160
    CompressedRasterTile2D<T>: CacheBackendElementExt<LandingZoneContainer = LandingZoneQueryTiles>,
161
{
162
    fn insert_element(
5✔
163
        &mut self,
5✔
164
        element: CompressedRasterTile2D<T>,
5✔
165
    ) -> Result<(), super::error::CacheError> {
5✔
166
        CompressedRasterTile2D::<T>::move_element_into_landing_zone(element, self)
5✔
167
    }
5✔
168

169
    fn create_empty() -> Self {
8✔
170
        CompressedRasterTile2D::<T>::create_empty_landing_zone()
8✔
171
    }
8✔
172
}
173

174
impl From<RasterLandingQueryEntry> for RasterCacheQueryEntry {
175
    fn from(value: RasterLandingQueryEntry) -> Self {
7✔
176
        Self {
7✔
177
            query: value.query,
7✔
178
            elements: value.elements.into(),
7✔
179
        }
7✔
180
    }
7✔
181
}
182

183
impl<T> CacheBackendElement for CompressedRasterTile2D<T>
184
where
185
    T: Pixel,
186
{
187
    type Query = RasterQueryRectangle;
188

189
    fn cache_hint(&self) -> geoengine_datatypes::primitives::CacheHint {
6✔
190
        self.cache_hint
6✔
191
    }
6✔
192

193
    fn typed_canonical_operator_name(
5✔
194
        key: crate::engine::CanonicOperatorName,
5✔
195
    ) -> super::shared_cache::TypedCanonicOperatorName {
5✔
196
        super::shared_cache::TypedCanonicOperatorName::Raster(key)
5✔
197
    }
5✔
198

199
    fn update_stored_query(&self, query: &mut Self::Query) -> Result<(), CacheError> {
5✔
200
        query.spatial_bounds.extend(&self.spatial_partition());
5✔
201
        query.time_interval = query
5✔
202
            .time_interval
5✔
203
            .union(&self.time)
5✔
204
            .map_err(|_| CacheError::ElementAndQueryDoNotIntersect)?;
5✔
205
        Ok(())
5✔
206
    }
5✔
207

208
    fn intersects_query(&self, query: &Self::Query) -> bool {
13✔
209
        self.spatial_partition().intersects(&query.spatial_bounds)
13✔
210
            && self.time.intersects(&query.time_interval)
8✔
211
    }
13✔
212
}
213

214
macro_rules! impl_cache_element_subtype {
215
    ($t:ty, $variant:ident) => {
216
        impl CacheBackendElementExt for CompressedRasterTile2D<$t> {
217
            type LandingZoneContainer = LandingZoneQueryTiles;
218
            type CacheContainer = CachedTiles;
219

220
            fn move_element_into_landing_zone(
7✔
221
                self,
7✔
222
                landing_zone: &mut LandingZoneQueryTiles,
7✔
223
            ) -> Result<(), super::error::CacheError> {
7✔
224
                match landing_zone {
7✔
225
                    LandingZoneQueryTiles::$variant(v) => {
7✔
226
                        v.push(self);
7✔
227
                        Ok(())
7✔
228
                    }
229
                    _ => Err(super::error::CacheError::InvalidTypeForInsertion),
×
230
                }
231
            }
7✔
232

233
            fn create_empty_landing_zone() -> LandingZoneQueryTiles {
10✔
234
                LandingZoneQueryTiles::$variant(Vec::new())
10✔
235
            }
10✔
236

237
            fn results_arc(cache_elements_container: &CachedTiles) -> Option<Arc<Vec<Self>>> {
238
                if let CachedTiles::$variant(v) = cache_elements_container {
5✔
239
                    Some(v.clone())
5✔
240
                } else {
241
                    None
×
242
                }
243
            }
5✔
244

245
            fn landing_zone_to_cache_entry(
6✔
246
                landing_zone_entry: RasterLandingQueryEntry,
6✔
247
            ) -> RasterCacheQueryEntry {
6✔
248
                landing_zone_entry.into()
6✔
249
            }
6✔
250
        }
251
    };
252
}
253
impl_cache_element_subtype!(i8, I8);
254
impl_cache_element_subtype!(u8, U8);
255
impl_cache_element_subtype!(i16, I16);
256
impl_cache_element_subtype!(u16, U16);
257
impl_cache_element_subtype!(i32, I32);
258
impl_cache_element_subtype!(u32, U32);
259
impl_cache_element_subtype!(i64, I64);
260
impl_cache_element_subtype!(u64, U64);
261
impl_cache_element_subtype!(f32, F32);
262
impl_cache_element_subtype!(f64, F64);
263

264
#[derive(Clone, Debug)]
1✔
265
pub struct CompressedMaskedGrid<D, T, C> {
266
    shape: D,
267
    type_marker: PhantomData<T>,
268
    data: Vec<u8>,
269
    mask: Vec<u8>,
270
    compression_marker: PhantomData<C>,
271
}
272

273
#[derive(Clone, Debug)]
1✔
274
pub enum CompressedGridOrEmpty<D, T, F> {
275
    Empty(EmptyGrid<D, T>),
276
    Compressed(CompressedMaskedGrid<D, T, F>),
277
}
278

279
pub type CompressedRasterTile<D, T> = BaseTile<CompressedGridOrEmpty<D, T, Lz4FlexCompression>>;
280
pub type CompressedRasterTile2D<T> = CompressedRasterTile<GridShape2D, T>;
281

282
impl<D, T, C: TileCompression> CompressedMaskedGrid<D, T, C> {
283
    #[cfg(test)]
284
    pub(crate) fn new(shape: D, data: Vec<u8>, mask: Vec<u8>) -> Self {
9✔
285
        Self {
9✔
286
            shape,
9✔
287
            type_marker: PhantomData,
9✔
288
            data,
9✔
289
            mask,
9✔
290
            compression_marker: PhantomData,
9✔
291
        }
9✔
292
    }
9✔
293

294
    pub fn compressed_data_len(&self) -> usize {
×
295
        self.data.len()
×
296
    }
×
297

298
    pub fn compressed_mask_len(&self) -> usize {
×
299
        self.mask.len()
×
300
    }
×
301

302
    pub fn compressed_len(&self) -> usize {
×
303
        self.compressed_data_len() + self.compressed_mask_len()
×
304
    }
×
305

306
    pub fn compressed_data_slice(&self) -> &[u8] {
×
307
        &self.data
×
308
    }
×
309

310
    pub fn compressed_mask_slice(&self) -> &[u8] {
×
311
        &self.mask
×
312
    }
×
313

314
    pub fn shape(&self) -> &D {
36✔
315
        &self.shape
36✔
316
    }
36✔
317

318
    pub fn compress_masked_grid(grid: &MaskedGrid<D, T>) -> Self
11✔
319
    where
11✔
320
        D: Clone + GridSize + PartialEq,
11✔
321
        T: Copy,
11✔
322
    {
11✔
323
        let grid_data_compressed = C::compress(&grid.inner_grid.data);
11✔
324
        let grid_mask_compressed = C::compress(&grid.validity_mask.data);
11✔
325

11✔
326
        Self {
11✔
327
            shape: grid.shape().clone(),
11✔
328
            type_marker: PhantomData,
11✔
329
            data: grid_data_compressed,
11✔
330
            mask: grid_mask_compressed,
11✔
331
            compression_marker: PhantomData,
11✔
332
        }
11✔
333
    }
11✔
334

335
    pub fn decompress_masked_grid(&self) -> Result<MaskedGrid<D, T>, CacheError>
×
336
    where
×
337
        D: Clone + GridSize + PartialEq,
×
338
        T: Copy,
×
339
    {
×
340
        let elements = self.shape().number_of_elements();
×
341

342
        let grid_data = C::decompress(&self.data, elements)?;
×
343
        let grid_mask = C::decompress(&self.mask, elements)?;
×
344

345
        let masked_grid = MaskedGrid {
×
346
            inner_grid: Grid {
×
347
                shape: self.shape.clone(),
×
348
                data: grid_data,
×
349
            },
×
350
            validity_mask: Grid {
×
351
                shape: self.shape.clone(),
×
352
                data: grid_mask,
×
353
            },
×
354
        };
×
355

×
356
        Ok(masked_grid)
×
357
    }
×
358
}
359

360
impl<D, T, C: TileCompression> CompressedGridOrEmpty<D, T, C> {
361
    pub fn compressed_data_len(&self) -> usize {
×
362
        match self {
×
363
            Self::Empty(_empty_grid) => 0,
×
364
            Self::Compressed(compressed_grid) => compressed_grid.compressed_data_len(),
×
365
        }
366
    }
×
367

368
    pub fn compressed_mask_len(&self) -> usize {
×
369
        match self {
×
370
            Self::Empty(_empty_grid) => 0,
×
371
            Self::Compressed(compressed_grid) => compressed_grid.compressed_mask_len(),
×
372
        }
373
    }
×
374

375
    pub fn compressed_len(&self) -> usize {
×
376
        match self {
×
377
            Self::Empty(_empty_grid) => 0,
×
378
            Self::Compressed(compressed_grid) => compressed_grid.compressed_len(),
×
379
        }
380
    }
×
381

382
    pub fn shape(&self) -> &D {
36✔
383
        match self {
36✔
384
            Self::Empty(empty_grid) => &empty_grid.shape,
×
385
            Self::Compressed(compressed_grid) => compressed_grid.shape(),
36✔
386
        }
387
    }
36✔
388

389
    pub fn is_empty(&self) -> bool {
×
390
        match self {
×
391
            Self::Empty(_) => true,
×
392
            Self::Compressed(_) => false,
×
393
        }
394
    }
×
395

396
    pub fn compress_grid(grid: &GridOrEmpty<D, T>) -> Self
19✔
397
    where
19✔
398
        D: Clone + GridSize + PartialEq,
19✔
399
        T: Copy,
19✔
400
    {
19✔
401
        match grid {
19✔
402
            GridOrEmpty::Empty(empty_grid) => Self::Empty(empty_grid.clone()),
8✔
403
            GridOrEmpty::Grid(grid) => {
11✔
404
                let compressed_grid = CompressedMaskedGrid::compress_masked_grid(grid);
11✔
405
                Self::Compressed(compressed_grid)
11✔
406
            }
407
        }
408
    }
19✔
409

410
    pub fn decompress_grid(&self) -> Result<GridOrEmpty<D, T>, CacheError>
×
411
    where
×
412
        D: Clone + GridSize + PartialEq,
×
413
        T: Copy,
×
414
    {
×
415
        match self {
×
416
            Self::Empty(empty_grid) => Ok(GridOrEmpty::Empty(empty_grid.clone())),
×
417
            Self::Compressed(compressed_grid) => {
×
418
                let decompressed_grid = compressed_grid.decompress_masked_grid()?;
×
419
                Ok(GridOrEmpty::Grid(decompressed_grid))
×
420
            }
421
        }
422
    }
×
423
}
424

425
impl<D, T, C> GridSize for CompressedGridOrEmpty<D, T, C>
426
where
427
    D: GridSize + GridSpaceToLinearSpace + PartialEq + Clone,
428
    T: Clone + Default,
429
    C: TileCompression,
430
{
431
    type ShapeArray = D::ShapeArray;
432

433
    const NDIM: usize = D::NDIM;
434

435
    fn axis_size(&self) -> Self::ShapeArray {
36✔
436
        self.shape().axis_size()
36✔
437
    }
36✔
438

439
    fn number_of_elements(&self) -> usize {
×
440
        self.shape().number_of_elements()
×
441
    }
×
442
}
443

444
impl<D, T, C> ByteSize for CompressedGridOrEmpty<D, T, C> {
445
    fn heap_byte_size(&self) -> usize {
23✔
446
        match self {
23✔
447
            Self::Empty(empty_grid) => empty_grid.heap_byte_size(),
×
448
            Self::Compressed(compressed_grid) => compressed_grid.heap_byte_size(),
23✔
449
        }
450
    }
23✔
451
}
452

453
impl<D, T, C> ByteSize for CompressedMaskedGrid<D, T, C> {
454
    fn heap_byte_size(&self) -> usize {
23✔
455
        self.data.heap_byte_size() + self.mask.heap_byte_size()
23✔
456
    }
23✔
457
}
458

459
pub trait CompressedRasterTileExt<D, T>
460
where
461
    Self: Sized,
462
{
463
    fn compress_tile(tile: RasterTile<D, T>) -> Self;
464

465
    fn decompress_tile(&self) -> Result<RasterTile<D, T>, CacheError>;
466

467
    fn compressed_data_len(&self) -> usize;
468
}
469

470
impl<T, C> CompressedRasterTileExt<GridShape2D, T>
471
    for BaseTile<CompressedGridOrEmpty<GridShape2D, T, C>>
472
where
473
    T: Copy,
474
    C: TileCompression,
475
{
476
    fn compress_tile(tile: RasterTile<GridShape2D, T>) -> Self {
19✔
477
        let compressed_grid = CompressedGridOrEmpty::compress_grid(&tile.grid_array);
19✔
478
        Self {
19✔
479
            grid_array: compressed_grid,
19✔
480
            time: tile.time,
19✔
481
            cache_hint: tile.cache_hint,
19✔
482
            global_geo_transform: tile.global_geo_transform,
19✔
483
            properties: tile.properties,
19✔
484
            tile_position: tile.tile_position,
19✔
485
            band: tile.band,
19✔
486
        }
19✔
487
    }
19✔
488

489
    fn decompress_tile(&self) -> Result<RasterTile<GridShape2D, T>, CacheError> {
×
490
        let grid_array = match &self.grid_array {
×
491
            CompressedGridOrEmpty::Empty(empty_grid) => GridOrEmpty::Empty(*empty_grid),
×
492
            CompressedGridOrEmpty::Compressed(compressed_grid) => {
×
493
                let decompressed_grid = compressed_grid.decompress_masked_grid()?;
×
494
                GridOrEmpty::Grid(decompressed_grid)
×
495
            }
496
        };
497

498
        Ok(RasterTile {
×
499
            grid_array,
×
500
            time: self.time,
×
501
            cache_hint: self.cache_hint,
×
502
            global_geo_transform: self.global_geo_transform,
×
503
            properties: self.properties.clone(),
×
504
            tile_position: self.tile_position,
×
NEW
505
            band: self.band,
×
506
        })
×
507
    }
×
508

509
    fn compressed_data_len(&self) -> usize {
×
510
        self.grid_array.compressed_data_len()
×
511
    }
×
512
}
513

514
impl<T> CacheElement for RasterTile2D<T>
515
where
516
    T: Pixel,
517
    CompressedRasterTile2D<T>: CompressedRasterTileExt<GridShape2D, T>
518
        + CacheBackendElement<Query = RasterQueryRectangle>
519
        + CacheBackendElementExt,
520
{
521
    type StoredCacheElement = CompressedRasterTile2D<T>;
522
    type Query = RasterQueryRectangle;
523
    type ResultStream =
524
        CacheStream<CompressedRasterTile2D<T>, RasterTile2D<T>, RasterQueryRectangle>;
525

526
    fn into_stored_element(self) -> Self::StoredCacheElement {
9✔
527
        CompressedRasterTile2D::compress_tile(self)
9✔
528
    }
9✔
529

530
    fn from_stored_element_ref(stored: &Self::StoredCacheElement) -> Result<Self, CacheError> {
×
531
        stored.decompress_tile()
×
532
    }
×
533

534
    fn result_stream(
1✔
535
        stored_data: Arc<Vec<Self::StoredCacheElement>>,
1✔
536
        query: Self::Query,
1✔
537
    ) -> Self::ResultStream {
1✔
538
        CacheStream::new(stored_data, query)
1✔
539
    }
1✔
540
}
541

542
pub trait TileCompression {
543
    fn compress<T>(data: &[T]) -> Vec<u8>
544
    where
545
        T: Copy;
546

547
    fn decompress<T>(data: &[u8], elements: usize) -> Result<Vec<T>, CacheError>
548
    where
549
        T: Copy;
550
}
551

552
#[derive(Debug, Copy, Clone)]
×
553
pub struct Lz4FlexCompression;
554

555
impl Lz4FlexCompression {
556
    fn cast_data_slice_to_u8_slice<T>(data: &[T]) -> &[u8]
22✔
557
    where
22✔
558
        T: Copy,
22✔
559
    {
22✔
560
        unsafe {
22✔
561
            std::slice::from_raw_parts(data.as_ptr().cast::<u8>(), std::mem::size_of_val(data))
22✔
562
        }
22✔
563
    }
22✔
564

565
    fn cast_u8_slice_to_data_slice<T>(data: &[u8]) -> &[T]
×
566
    where
×
567
        T: Copy,
×
568
    {
×
569
        unsafe {
×
570
            std::slice::from_raw_parts(
×
571
                data.as_ptr().cast::<T>(),
×
572
                data.len() / std::mem::size_of::<T>(),
×
573
            )
×
574
        }
×
575
    }
×
576
}
577

578
impl TileCompression for Lz4FlexCompression {
579
    fn compress<T>(data: &[T]) -> Vec<u8>
22✔
580
    where
22✔
581
        T: Copy,
22✔
582
    {
22✔
583
        let data_as_u8_slice = Self::cast_data_slice_to_u8_slice(data);
22✔
584
        lz4_flex::compress(data_as_u8_slice)
22✔
585
    }
22✔
586

587
    fn decompress<T>(data: &[u8], elements: usize) -> Result<Vec<T>, CacheError>
×
588
    where
×
589
        T: Copy,
×
590
    {
×
591
        let stored_bytes = elements * std::mem::size_of::<T>();
×
592
        let decompressed_data = lz4_flex::decompress(data, stored_bytes)
×
593
            .map_err(|source| CacheError::CouldNotDecompressElement { source })?;
×
594
        let decompressed_data_as_t_slice =
×
595
            Self::cast_u8_slice_to_data_slice(decompressed_data.as_slice());
×
596
        Ok(decompressed_data_as_t_slice.to_vec())
×
597
    }
×
598
}
599

600
#[cfg(test)]
601
mod tests {
602
    use std::sync::Arc;
603

604
    use geoengine_datatypes::{
605
        primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution},
606
        raster::GeoTransform,
607
        util::test::TestDefault,
608
    };
609

610
    use crate::pro::cache::{
611
        cache_tiles::CachedTiles,
612
        shared_cache::{
613
            CacheBackendElement, CacheBackendElementExt, CacheQueryMatch, RasterCacheQueryEntry,
614
            RasterLandingQueryEntry,
615
        },
616
    };
617

618
    use super::{
619
        CompressedGridOrEmpty, CompressedMaskedGrid, CompressedRasterTile2D, LandingZoneQueryTiles,
620
    };
621

622
    fn create_test_tile() -> CompressedRasterTile2D<u8> {
3✔
623
        CompressedRasterTile2D {
3✔
624
            grid_array: CompressedGridOrEmpty::Compressed(CompressedMaskedGrid::<_, u8, _> {
3✔
625
                shape: [2, 2].into(),
3✔
626
                type_marker: Default::default(),
3✔
627
                data: vec![16; 4],
3✔
628
                mask: vec![1; 4],
3✔
629
                compression_marker: Default::default(),
3✔
630
            }),
3✔
631
            time: Default::default(),
3✔
632
            cache_hint: Default::default(),
3✔
633
            global_geo_transform: GeoTransform::test_default(),
3✔
634
            properties: Default::default(),
3✔
635
            tile_position: [0, 0].into(),
3✔
636
            band: 0,
3✔
637
        }
3✔
638
    }
3✔
639

640
    #[test]
1✔
641
    fn create_empty_landing_zone() {
1✔
642
        let landing_zone = super::CompressedRasterTile2D::<u8>::create_empty_landing_zone();
1✔
643
        assert!(landing_zone.is_empty());
1✔
644
        if let super::LandingZoneQueryTiles::U8(v) = landing_zone {
1✔
645
            assert!(v.is_empty());
1✔
646
        } else {
647
            panic!("wrong type");
×
648
        }
649
    }
1✔
650

651
    #[test]
1✔
652
    fn move_element_to_landing_zone() {
1✔
653
        let mut landing_zone = CompressedRasterTile2D::<u8>::create_empty_landing_zone();
1✔
654
        let tile = create_test_tile();
1✔
655
        tile.move_element_into_landing_zone(&mut landing_zone)
1✔
656
            .unwrap();
1✔
657
        if let LandingZoneQueryTiles::U8(v) = landing_zone {
1✔
658
            assert_eq!(v.len(), 1);
1✔
659
        } else {
660
            panic!("wrong type");
×
661
        }
662
    }
1✔
663

664
    #[test]
1✔
665
    fn landing_zone_to_cache_entry() {
1✔
666
        let tile = create_test_tile();
1✔
667
        let query = RasterQueryRectangle {
1✔
668
            spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., 1.).into()),
1✔
669
            time_interval: Default::default(),
1✔
670
            spatial_resolution: SpatialResolution::zero_point_one(),
1✔
671
            attributes: BandSelection::first(),
1✔
672
        };
1✔
673
        let mut lq =
1✔
674
            RasterLandingQueryEntry::create_empty::<CompressedRasterTile2D<u8>>(query.clone());
1✔
675
        tile.move_element_into_landing_zone(lq.elements_mut())
1✔
676
            .unwrap();
1✔
677
        let mut cache_entry = CompressedRasterTile2D::<u8>::landing_zone_to_cache_entry(lq);
1✔
678
        assert_eq!(cache_entry.query(), &query);
1✔
679
        assert!(cache_entry.elements_mut().is_expired());
1✔
680
    }
1✔
681

682
    #[test]
1✔
683
    fn cache_element_hit() {
1✔
684
        let tile = create_test_tile();
1✔
685

1✔
686
        // tile is fully contained
1✔
687
        let query = RasterQueryRectangle {
1✔
688
            spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()),
1✔
689
            time_interval: Default::default(),
1✔
690
            spatial_resolution: SpatialResolution::one(),
1✔
691
            attributes: BandSelection::first(),
1✔
692
        };
1✔
693
        assert!(tile.intersects_query(&query));
1✔
694

695
        // tile is partially contained
696
        let query = RasterQueryRectangle {
1✔
697
            spatial_bounds: SpatialPartition2D::new_unchecked(
1✔
698
                (0.5, -0.5).into(),
1✔
699
                (1.5, -1.5).into(),
1✔
700
            ),
1✔
701
            time_interval: Default::default(),
1✔
702
            spatial_resolution: SpatialResolution::one(),
1✔
703
            attributes: BandSelection::first(),
1✔
704
        };
1✔
705
        assert!(tile.intersects_query(&query));
1✔
706

707
        // tile is not contained
708
        let query = RasterQueryRectangle {
1✔
709
            spatial_bounds: SpatialPartition2D::new_unchecked(
1✔
710
                (10., -10.).into(),
1✔
711
                (11., -11.).into(),
1✔
712
            ),
1✔
713
            time_interval: Default::default(),
1✔
714
            spatial_resolution: SpatialResolution::one(),
1✔
715
            attributes: BandSelection::first(),
1✔
716
        };
1✔
717
        assert!(!tile.intersects_query(&query));
1✔
718
    }
1✔
719

720
    #[test]
1✔
721
    fn cache_entry_matches() {
1✔
722
        let cache_entry_bounds = RasterQueryRectangle {
1✔
723
            spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()),
1✔
724
            time_interval: Default::default(),
1✔
725
            spatial_resolution: SpatialResolution::one(),
1✔
726
            attributes: BandSelection::first(),
1✔
727
        };
1✔
728
        let cache_query_entry = RasterCacheQueryEntry {
1✔
729
            query: cache_entry_bounds.clone(),
1✔
730
            elements: CachedTiles::U8(Arc::new(Vec::new())),
1✔
731
        };
1✔
732

1✔
733
        // query is equal
1✔
734
        let query = cache_entry_bounds;
1✔
735
        assert!(cache_query_entry.query().is_match(&query));
1✔
736

737
        // query is fully contained
738
        let query2 = RasterQueryRectangle {
1✔
739
            spatial_bounds: SpatialPartition2D::new_unchecked(
1✔
740
                (0.1, -0.1).into(),
1✔
741
                (0.9, -0.9).into(),
1✔
742
            ),
1✔
743
            time_interval: Default::default(),
1✔
744
            spatial_resolution: SpatialResolution::one(),
1✔
745
            attributes: BandSelection::first(),
1✔
746
        };
1✔
747
        assert!(cache_query_entry.query().is_match(&query2));
1✔
748

749
        // query is exceeds cached bounds
750
        let query3 = RasterQueryRectangle {
1✔
751
            spatial_bounds: SpatialPartition2D::new_unchecked(
1✔
752
                (-0.1, 0.1).into(),
1✔
753
                (1.1, -1.1).into(),
1✔
754
            ),
1✔
755
            time_interval: Default::default(),
1✔
756
            spatial_resolution: SpatialResolution::one(),
1✔
757
            attributes: BandSelection::first(),
1✔
758
        };
1✔
759
        assert!(!cache_query_entry.query().is_match(&query3));
1✔
760
    }
1✔
761
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc