23965535588

Committed 03 Apr 2026 11:07PM UTC coverage: 59.873% (-1.8%) from 61.699%

Build # 23965535588

Build Type

push

github

Committed by

web-flow

Commit Message

chore: deck.gl 9.2 upgrade & loaders.gl, luma.gl upgrades (#3271)

* chore: upgrade to deckgl 9.2.11

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix lint follow up

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix blending

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix geojson layer

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* more fixes for aggregation layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix h3 layer

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* potential fix for issues with geoarrow in point and line layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix postprocessing effects

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for light and shadow effect

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* shadow and light effect - restore uniform shadow during the nighttime

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* don't hide line and arc layers when layer type changed

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* restore filters for aggregation layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix aggregation layers - hightlight outlines

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for raster tile layer - raster pmtiles related

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for raster tiles shader modules updates

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* more lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix CI lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix CI tests

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* install webgpu

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix Ci test env setup

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* cont fix setup browser env

Signed-o... (continued)

Coverage Stats

6517 of 12977 branches covered (50.22%)

Branch coverage included in aggregate %.

324 of 1031 new or added lines in 58 files covered. (31.43%)

123 existing lines in 18 files now uncovered.

13313 of 20143 relevant lines covered (66.09%)

78.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.79

/src/layers/src/raster-tile/gpu-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

/**
 * Functions and constants for handling webgl/luma.gl/deck.gl entities
 */

import {parse, fetchFile, load, FetchError, Loader} from '@loaders.gl/core';
import {ImageLoader} from '@loaders.gl/images';
import {NPYLoader} from '@loaders.gl/textures';
// @ts-ignore GL resolution depends on moduleResolution setting
import {GL} from '@luma.gl/constants';

/**
 * Loose texture data descriptor passed around before actual luma.gl Texture creation.
 * Not the same as luma.gl's strict TextureProps (which requires width/height).
 */
type Texture2DProps = Record<string, any>;

import {sleep} from '@kepler.gl/common-utils';
import {getLoaderOptions} from '@kepler.gl/constants';
import {RasterWebGL} from '@kepler.gl/deckgl-layers';
import {getApplicationConfig} from '@kepler.gl/utils';

type ShaderModule = RasterWebGL.ShaderModule;
const {
  combineBandsFloat,
  combineBandsInt,
  combineBandsUint,
  maskFloat,
  maskInt,
  maskUint,
  linearRescale,
  gammaContrast,
  sigmoidalContrast,
  normalizedDifference,
  enhancedVegetationIndex,
  soilAdjustedVegetationIndex,
  modifiedSoilAdjustedVegetationIndex,
  colormap: colormapModule,
  filter,
  saturation,
  reorderBands,
  rgbaImage
} = RasterWebGL;

import {
  CATEGORICAL_TEXTURE_WIDTH,
  dtypeMaxValue,
  generateCategoricalBitmapArray,
  isColormapAllowed,
  isFilterAllowed,
  isRescalingAllowed
} from './raster-tile-utils';
import {
  CategoricalColormapOptions,
  ImageData,
  NPYLoaderDataTypes,
  NPYLoaderResponse,
  RenderSubLayersProps
} from './types';
import {getRequestThrottle} from './request-throttle';

/**
 * Describe WebGL2 Texture parameters to use for given input data type
 */
interface WebGLTextureFormat {
  format: number;
  dataFormat: number;
  type: number;
}

/**
 * Convert TypedArray to WebGL2 Texture Parameters
 */
function getWebGL2TextureParameters(data: NPYLoaderDataTypes): WebGLTextureFormat | never {
  if (data instanceof Uint8Array || data instanceof Uint8ClampedArray) {
    return {
      // Note: texture data has no auto-rescaling; pixel values stay as 0-255
      format: GL.R8UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_BYTE
    };
  }

  if (data instanceof Uint16Array) {
    return {
      format: GL.R16UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_SHORT
    };
  }

  if (data instanceof Uint32Array) {
    return {
      format: GL.R32UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_INT
    };
  }

  if (data instanceof Int8Array) {
    return {
      format: GL.R8I,
      dataFormat: GL.RED_INTEGER,
      type: GL.BYTE
    };
  }

  if (data instanceof Int16Array) {
    return {
      format: GL.R16I,
      dataFormat: GL.RED_INTEGER,
      type: GL.SHORT
    };
  }
  if (data instanceof Int32Array) {
    return {
      format: GL.R32I,
      dataFormat: GL.RED_INTEGER,
      type: GL.INT
    };
  }
  if (data instanceof Float32Array) {
    return {
      format: GL.R32F,
      dataFormat: GL.RED,
      type: GL.FLOAT
    };
  }

  if (data instanceof Float64Array) {
    return {
      format: GL.R32F,
      dataFormat: GL.RED,
      type: GL.FLOAT
    };
  }

  // For exhaustive check above; following should never occur
  // https://stackoverflow.com/a/58009992
  const unexpectedInput: never = data;
  throw new Error(unexpectedInput);
}

/**
 * Discrete-valued colormaps (e.g. from the output of
 * classification algorithms) in the raster layer. Previously, the values passed to
 * `TEXTURE_MIN_FILTER` and `TEXTURE_MAG_FILTER` were `GL.LINEAR`, which meant that the GPU would
 * linearly interpolate values between two neighboring colormap pixel values. Setting these values
 * to NEAREST means that the GPU will choose the nearest value on the texture2D lookup operation,
 * which fixes precision issues for discrete-valued colormaps. This should be ok for continuous
 * colormaps as long as the color difference between each pixel on the colormap is small.
 */
export const COLORMAP_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
  [GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

const DEFAULT_8BIT_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.LINEAR_MIPMAP_LINEAR,
  [GL.TEXTURE_MAG_FILTER]: GL.LINEAR,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

const DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
  [GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

/**
 * Select correct module type for "combineBands"
 *
 * combineBands joins up to four 2D arrays (contained in imageBands) into a single "rgba" image
 * texture on the GPU. That shader code needs to have the same data type as the actual image data.
 * E.g. for float data the texture needs to be `sampler2D`, for uint data the texture needs to be
 * `usampler2D` and for int data the texture needs to be `isampler2D`.
 */
export function getCombineBandsModule(imageBands: Texture2DProps[]): ShaderModule {
  // Each image array is expected/required to be of the same data type
  switch (imageBands[0].format) {
    case GL.R8UI:
      return combineBandsUint;
    case GL.R16UI:
      return combineBandsUint;
    case GL.R32UI:
      return combineBandsUint;
    case GL.R8I:
      return combineBandsInt;
    case GL.R16I:
      return combineBandsInt;
    case GL.R32I:
      return combineBandsInt;
    case GL.R32F:
      return combineBandsFloat;
    default:
      throw new Error('bad format');
  }
}

/** Select correct image masking shader module for mask data type
 * The imageMask could (at least in the future, theoretically) be of a different data format than
 * the imageBands data itself.
 */
export function getImageMaskModule(imageMask: Texture2DProps): ShaderModule {
  switch (imageMask.format) {
    case GL.R8UI:
      return maskUint;
    case GL.R16UI:
      return maskUint;
    case GL.R32UI:
      return maskUint;
    case GL.R8I:
      return maskInt;
    case GL.R16I:
      return maskInt;
    case GL.R32I:
      return maskInt;
    case GL.R32F:
      return maskFloat;
    default:
      throw new Error('bad format');
  }
}

/**
 * Load image and wrap with default WebGL texture parameters
 *
 * @param url URL to load image
 * @param textureParams parameters to pass to Texture2D
 *
 * @return image object to pass to Texture2D constructor
 */
export async function loadImage(
  url: string,
  textureParams: Texture2DProps = {},
  requestOptions: RequestInit = {}
): Promise<Texture2DProps> {
  const response = await fetchFile(url, requestOptions);
  const image = await parse(response, ImageLoader);

  return {
    data: image,
    parameters: DEFAULT_8BIT_TEXTURE_PARAMETERS,
    format: GL.RGB,
    ...textureParams
  };
}

type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
type LoadingOptions = {
  fetch?: typeof fetch | FetchLike;
};

type NpyRequest = {
  url: string;
  rasterServerUrl: string;
  options: RequestInit;
  rasterServerMaxRetries?: number;
  rasterServerRetryDelay?: number;
  rasterServerServerErrorsToRetry?: number[];
  rasterServerMaxPerServerRequests?: number;
};

/**
 * Load NPY Array
 *
 * The NPY format is described here: https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html.
 * It's designed to be a very simple file format to hold an N-dimensional block of data. The header describes the data type, shape, and order (either C or Fortran) of the array.
 *
 * @param url URL to load NPY Array
 * @param split Whether to split single typed array representing an N-dimensional array into an Array with each dimension as its own typed array
 *
 * @return image object to pass to Texture2D constructor
 */
export async function loadNpyArray(
  request: NpyRequest,
  split: true,
  options?: LoadingOptions
): Promise<Texture2DProps[] | null>;
export async function loadNpyArray(
  request: NpyRequest,
  split: false,
  options?: LoadingOptions
): Promise<Texture2DProps | null>;
export async function loadNpyArray(
  request: NpyRequest,
  split: boolean,
  options?: LoadingOptions
): Promise<Texture2DProps | Texture2DProps[] | null> {
  const numAttempts =
    1 + (request.rasterServerMaxRetries ?? getApplicationConfig().rasterServerMaxRetries);

  const asset = await getRequestThrottle().throttleRequest(
    request.rasterServerUrl,
    async () => {
      for (let attempt = 0; attempt < numAttempts; attempt++) {
        try {
          const {npy: npyOptions} = getLoaderOptions();

          const response = (await load(request.url, NPYLoader as Loader<NPYLoaderResponse>, {
            npy: npyOptions,
            fetch: options?.fetch
          })) as NPYLoaderResponse;

          if (!response || !response.data || request.options.signal?.aborted) {
            return null;
          }

          // Float64 data needs to be coerced to Float32 for the GPU
          if (response.data instanceof Float64Array) {
            response.data = Float32Array.from(response.data);
          }

          const {data, header} = response;
          const {shape} = header;
          const {format, dataFormat, type} = getWebGL2TextureParameters(data);

          // TODO: check height-width or width-height
          // Regardless, images usually square
          // TODO: handle cases of 256x256x1 instead of 1x256x256
          const [z, height, width] = shape;

          // Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
          const mipmaps = false;
          const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;

          if (!split) {
            return {
              data,
              width,
              height,
              format,
              dataFormat,
              type,
              parameters,
              mipmaps
            };
          }

          // Split into individual arrays
          const channels: Texture2DProps[] = [];
          const channelSize = height * width;
          for (let i = 0; i < z; i++) {
            channels.push({
              data: data.subarray(i * channelSize, (i + 1) * channelSize),
              width,
              height,
              format,
              dataFormat,
              type,
              parameters,
              mipmaps
            });
          }
          return channels;
        } catch (error) {
          // Retry if Service Temporarily Unavailable 503 error etc.
          if (
            attempt < numAttempts &&
            error instanceof FetchError &&
            (
              request.rasterServerServerErrorsToRetry ??
              getApplicationConfig().rasterServerServerErrorsToRetry
            )?.includes(error.response?.status as number)
          ) {
            await sleep(
              request.rasterServerRetryDelay ?? getApplicationConfig().rasterServerRetryDelay
            );
            continue;
          }
        }
      }
      return null;
    },
    request.rasterServerMaxPerServerRequests
  );

  return asset;
}

/**
 * Create texture data for categorical colormap scale
 * @param categoricalOptions - color map configuration and min-max values of categorical band
 * @returns texture data
 */
export function generateCategoricalColormapTexture(
  categoricalOptions: CategoricalColormapOptions
): Texture2DProps {
  const data = generateCategoricalBitmapArray(categoricalOptions);
  return {
    data,
    width: CATEGORICAL_TEXTURE_WIDTH,
    height: 1,
    format: GL.RGBA,
    dataFormat: GL.RGBA,
    type: GL.UNSIGNED_BYTE,
    parameters: COLORMAP_TEXTURE_PARAMETERS,
    mipmaps: false
  };
}

// TODO: would probably be simpler to only pass in the props actually used by this function. That
// would mean a smaller object than RenderSubLayersProps
// eslint-disable-next-line max-statements, complexity
export function getModules({
  images,
  props
}: {
  images: Partial<ImageData>;
  props?: RenderSubLayersProps;
}): {
  modules: ShaderModule[];
  moduleProps: Record<string, any>;
} {
  const moduleProps: Record<string, any> = {};
  // Array of luma.gl WebGL modules to pass to the RasterLayer
  const modules: ShaderModule[] = [];

  // use rgba image directly. Used for raster .pmtiles rendering
  if (images.imageRgba) {
    modules.push(rgbaImage);

    // no support for other modules atm for direct rgba mode
    return {modules, moduleProps};
  }

  if (!props) {
    return {modules, moduleProps};
  }

  const {
    renderBandIndexes,
    nonLinearRescaling,
    linearRescalingFactor,
    minPixelValue,
    maxPixelValue,
    gammaContrastFactor,
    sigmoidalContrastFactor,
    sigmoidalBiasFactor,
    saturationValue,
    bandCombination,
    filterEnabled,
    filterRange,
    dataType,
    minCategoricalBandValue,
    maxCategoricalBandValue,
    hasCategoricalColorMap
  } = props;

  if (Array.isArray(images.imageBands) && images.imageBands.length > 0) {
    modules.push(getCombineBandsModule(images.imageBands));
  }

  if (images.imageMask) {
    modules.push(getImageMaskModule(images.imageMask));
    // In general, data masks are 0 for nodata and the maximum value for valid data, e.g. 255 or
    // 65535 for uint8 or uint16 data, respectively
    moduleProps.maskKeepMin = 1;
  }

  if (Array.isArray(renderBandIndexes)) {
    modules.push(reorderBands);
    moduleProps.ordering = renderBandIndexes;
  }

  const globalRange = maxPixelValue - minPixelValue;
  // Fix rescaling if we are sure that dataset is categorical
  if (hasCategoricalColorMap) {
    modules.push(linearRescale);
    moduleProps.linearRescaleScaler = 1 / maxPixelValue;
    moduleProps.linearRescaleOffset = 0;
  } else if (isRescalingAllowed(bandCombination)) {
    if (!nonLinearRescaling) {
      const [min, max] = linearRescalingFactor;
      const localRange = max - min;

      // Add linear rescaling module
      modules.push(linearRescale);

      // Divide by local range * global range
      moduleProps.linearRescaleScaler = 1 / (localRange * globalRange);

      // Subtract off the local min
      moduleProps.linearRescaleOffset = -min;

      // Clamp to [0, 1] done automatically?
    } else {
      modules.push(linearRescale);
      moduleProps.linearRescaleScaler = 1 / maxPixelValue;
      moduleProps.linearRescaleOffset = 0;

      modules.push(gammaContrast);
      moduleProps.gammaContrastValue = gammaContrastFactor;

      modules.push(sigmoidalContrast);
      moduleProps.sigmoidalContrast = sigmoidalContrastFactor;
      moduleProps.sigmoidalBias = sigmoidalBiasFactor;
    }

    if (Number.isFinite(saturationValue) && saturationValue !== 1) {
      modules.push(saturation);
      moduleProps.saturationValue = saturationValue;
    }
  }

  switch (bandCombination) {
    case 'normalizedDifference':
      modules.push(normalizedDifference);
      break;
    case 'enhancedVegetationIndex':
      modules.push(enhancedVegetationIndex);
      break;
    case 'soilAdjustedVegetationIndex':
      modules.push(soilAdjustedVegetationIndex);
      break;
    case 'modifiedSoilAdjustedVegetationIndex':
      modules.push(modifiedSoilAdjustedVegetationIndex);
      break;
    default:
      break;
  }

  if (isFilterAllowed(bandCombination) && filterEnabled) {
    modules.push(filter);
    moduleProps.filterMin1 = filterRange[0];
    moduleProps.filterMax1 = filterRange[1];
  }

  // Apply colormap
  if (isColormapAllowed(bandCombination) && images.imageColormap) {
    modules.push(colormapModule);
    moduleProps.minCategoricalBandValue = minCategoricalBandValue;
    moduleProps.maxCategoricalBandValue = maxCategoricalBandValue;
    moduleProps.dataTypeMaxValue = dtypeMaxValue[dataType];
    moduleProps.maxPixelValue = maxPixelValue;
  }

  return {modules, moduleProps};
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	/**
5	* Functions and constants for handling webgl/luma.gl/deck.gl entities
6	*/
7
8	import {parse, fetchFile, load, FetchError, Loader} from '@loaders.gl/core';
9	import {ImageLoader} from '@loaders.gl/images';
10	import {NPYLoader} from '@loaders.gl/textures';
11	// @ts-ignore GL resolution depends on moduleResolution setting
12	import {GL} from '@luma.gl/constants';
13
14	/**
15	* Loose texture data descriptor passed around before actual luma.gl Texture creation.
16	* Not the same as luma.gl's strict TextureProps (which requires width/height).
17	*/
18	type Texture2DProps = Record<string, any>;
19
20	import {sleep} from '@kepler.gl/common-utils';
21	import {getLoaderOptions} from '@kepler.gl/constants';
22	import {RasterWebGL} from '@kepler.gl/deckgl-layers';
23	import {getApplicationConfig} from '@kepler.gl/utils';
24
25	type ShaderModule = RasterWebGL.ShaderModule;
26	const {
27	combineBandsFloat,
28	combineBandsInt,
29	combineBandsUint,
30	maskFloat,
31	maskInt,
32	maskUint,
33	linearRescale,
34	gammaContrast,
35	sigmoidalContrast,
36	normalizedDifference,
37	enhancedVegetationIndex,
38	soilAdjustedVegetationIndex,
39	modifiedSoilAdjustedVegetationIndex,
40	colormap: colormapModule,
41	filter,
42	saturation,
43	reorderBands,
44	rgbaImage
45	} = RasterWebGL;	13✔
46
47	import {
48	CATEGORICAL_TEXTURE_WIDTH,
49	dtypeMaxValue,
50	generateCategoricalBitmapArray,
51	isColormapAllowed,
52	isFilterAllowed,
53	isRescalingAllowed
54	} from './raster-tile-utils';
55	import {
56	CategoricalColormapOptions,
57	ImageData,
58	NPYLoaderDataTypes,
59	NPYLoaderResponse,
60	RenderSubLayersProps
61	} from './types';
62	import {getRequestThrottle} from './request-throttle';
63
64	/**
65	* Describe WebGL2 Texture parameters to use for given input data type
66	*/
67	interface WebGLTextureFormat {
68	format: number;
69	dataFormat: number;
70	type: number;
71	}
72
73	/**
74	* Convert TypedArray to WebGL2 Texture Parameters
75	*/
76	function getWebGL2TextureParameters(data: NPYLoaderDataTypes): WebGLTextureFormat \| never {
77	if (data instanceof Uint8Array \|\| data instanceof Uint8ClampedArray) {	×
78	return {	×
79	// Note: texture data has no auto-rescaling; pixel values stay as 0-255
80	format: GL.R8UI,
81	dataFormat: GL.RED_INTEGER,
82	type: GL.UNSIGNED_BYTE
83	};
84	}
85
86	if (data instanceof Uint16Array) {	×
87	return {	×
88	format: GL.R16UI,
89	dataFormat: GL.RED_INTEGER,
90	type: GL.UNSIGNED_SHORT
91	};
92	}
93
94	if (data instanceof Uint32Array) {	×
95	return {	×
96	format: GL.R32UI,
97	dataFormat: GL.RED_INTEGER,
98	type: GL.UNSIGNED_INT
99	};
100	}
101
102	if (data instanceof Int8Array) {	×
103	return {	×
104	format: GL.R8I,
105	dataFormat: GL.RED_INTEGER,
106	type: GL.BYTE
107	};
108	}
109
110	if (data instanceof Int16Array) {	×
111	return {	×
112	format: GL.R16I,
113	dataFormat: GL.RED_INTEGER,
114	type: GL.SHORT
115	};
116	}
117	if (data instanceof Int32Array) {	×
118	return {	×
119	format: GL.R32I,
120	dataFormat: GL.RED_INTEGER,
121	type: GL.INT
122	};
123	}
124	if (data instanceof Float32Array) {	×
125	return {	×
126	format: GL.R32F,
127	dataFormat: GL.RED,
128	type: GL.FLOAT
129	};
130	}
131
132	if (data instanceof Float64Array) {	×
133	return {	×
134	format: GL.R32F,
135	dataFormat: GL.RED,
136	type: GL.FLOAT
137	};
138	}
139
140	// For exhaustive check above; following should never occur
141	// https://stackoverflow.com/a/58009992
142	const unexpectedInput: never = data;	×
143	throw new Error(unexpectedInput);	×
144	}
145
146	/**
147	* Discrete-valued colormaps (e.g. from the output of
148	* classification algorithms) in the raster layer. Previously, the values passed to
149	* `TEXTURE_MIN_FILTER` and `TEXTURE_MAG_FILTER` were `GL.LINEAR`, which meant that the GPU would
150	* linearly interpolate values between two neighboring colormap pixel values. Setting these values
151	* to NEAREST means that the GPU will choose the nearest value on the texture2D lookup operation,
152	* which fixes precision issues for discrete-valued colormaps. This should be ok for continuous
153	* colormaps as long as the color difference between each pixel on the colormap is small.
154	*/
155	export const COLORMAP_TEXTURE_PARAMETERS = {	13✔
156	[GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
157	[GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
158	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
159	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
160	};
161
162	const DEFAULT_8BIT_TEXTURE_PARAMETERS = {	13✔
163	[GL.TEXTURE_MIN_FILTER]: GL.LINEAR_MIPMAP_LINEAR,
164	[GL.TEXTURE_MAG_FILTER]: GL.LINEAR,
165	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
166	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
167	};
168
169	const DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS = {	13✔
170	[GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
171	[GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
172	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
173	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
174	};
175
176	/**
177	* Select correct module type for "combineBands"
178	*
179	* combineBands joins up to four 2D arrays (contained in imageBands) into a single "rgba" image
180	* texture on the GPU. That shader code needs to have the same data type as the actual image data.
181	* E.g. for float data the texture needs to be `sampler2D`, for uint data the texture needs to be
182	* `usampler2D` and for int data the texture needs to be `isampler2D`.
183	*/
184	export function getCombineBandsModule(imageBands: Texture2DProps[]): ShaderModule {
185	// Each image array is expected/required to be of the same data type
186	switch (imageBands[0].format) {	×
187	case GL.R8UI:
188	return combineBandsUint;	×
189	case GL.R16UI:
190	return combineBandsUint;	×
191	case GL.R32UI:
192	return combineBandsUint;	×
193	case GL.R8I:
194	return combineBandsInt;	×
195	case GL.R16I:
196	return combineBandsInt;	×
197	case GL.R32I:
198	return combineBandsInt;	×
199	case GL.R32F:
200	return combineBandsFloat;	×
201	default:
202	throw new Error('bad format');	×
203	}
204	}
205
206	/** Select correct image masking shader module for mask data type
207	* The imageMask could (at least in the future, theoretically) be of a different data format than
208	* the imageBands data itself.
209	*/
210	export function getImageMaskModule(imageMask: Texture2DProps): ShaderModule {
211	switch (imageMask.format) {	×
212	case GL.R8UI:
213	return maskUint;	×
214	case GL.R16UI:
215	return maskUint;	×
216	case GL.R32UI:
217	return maskUint;	×
218	case GL.R8I:
219	return maskInt;	×
220	case GL.R16I:
221	return maskInt;	×
222	case GL.R32I:
223	return maskInt;	×
224	case GL.R32F:
225	return maskFloat;	×
226	default:
227	throw new Error('bad format');	×
228	}
229	}
230
231	/**
232	* Load image and wrap with default WebGL texture parameters
233	*
234	* @param url URL to load image
235	* @param textureParams parameters to pass to Texture2D
236	*
237	* @return image object to pass to Texture2D constructor
238	*/
239	export async function loadImage(
240	url: string,
241	textureParams: Texture2DProps = {},	×
242	requestOptions: RequestInit = {}	×
243	): Promise<Texture2DProps> {
244	const response = await fetchFile(url, requestOptions);	×
245	const image = await parse(response, ImageLoader);	×
246
247	return {	×
248	data: image,
249	parameters: DEFAULT_8BIT_TEXTURE_PARAMETERS,
250	format: GL.RGB,
251	...textureParams
252	};
253	}
254
255	type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
256	type LoadingOptions = {
257	fetch?: typeof fetch \| FetchLike;
258	};
259
260	type NpyRequest = {
261	url: string;
262	rasterServerUrl: string;
263	options: RequestInit;
264	rasterServerMaxRetries?: number;
265	rasterServerRetryDelay?: number;
266	rasterServerServerErrorsToRetry?: number[];
267	rasterServerMaxPerServerRequests?: number;
268	};
269
270	/**
271	* Load NPY Array
272	*
273	* The NPY format is described here: https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html.
274	* It's designed to be a very simple file format to hold an N-dimensional block of data. The header describes the data type, shape, and order (either C or Fortran) of the array.
275	*
276	* @param url URL to load NPY Array
277	* @param split Whether to split single typed array representing an N-dimensional array into an Array with each dimension as its own typed array
278	*
279	* @return image object to pass to Texture2D constructor
280	*/
281	export async function loadNpyArray(
282	request: NpyRequest,
283	split: true,
284	options?: LoadingOptions
285	): Promise<Texture2DProps[] \| null>;
286	export async function loadNpyArray(
287	request: NpyRequest,
288	split: false,
289	options?: LoadingOptions
290	): Promise<Texture2DProps \| null>;
291	export async function loadNpyArray(
292	request: NpyRequest,
293	split: boolean,
294	options?: LoadingOptions
295	): Promise<Texture2DProps \| Texture2DProps[] \| null> {
296	const numAttempts =
297	1 + (request.rasterServerMaxRetries ?? getApplicationConfig().rasterServerMaxRetries);	×
298
299	const asset = await getRequestThrottle().throttleRequest(	×
300	request.rasterServerUrl,
301	async () => {
302	for (let attempt = 0; attempt < numAttempts; attempt++) {	×
303	try {	×
304	const {npy: npyOptions} = getLoaderOptions();	×
305
NEW 306	const response = (await load(request.url, NPYLoader as Loader<NPYLoaderResponse>, {	×
307	npy: npyOptions,
308	fetch: options?.fetch
309	})) as NPYLoaderResponse;
310
311	if (!response \|\| !response.data \|\| request.options.signal?.aborted) {	×
312	return null;	×
313	}
314
315	// Float64 data needs to be coerced to Float32 for the GPU
316	if (response.data instanceof Float64Array) {	×
317	response.data = Float32Array.from(response.data);	×
318	}
319
320	const {data, header} = response;	×
321	const {shape} = header;	×
322	const {format, dataFormat, type} = getWebGL2TextureParameters(data);	×
323
324	// TODO: check height-width or width-height
325	// Regardless, images usually square
326	// TODO: handle cases of 256x256x1 instead of 1x256x256
327	const [z, height, width] = shape;	×
328
329	// Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
330	const mipmaps = false;	×
331	const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;	×
332
333	if (!split) {	×
334	return {	×
335	data,
336	width,
337	height,
338	format,
339	dataFormat,
340	type,
341	parameters,
342	mipmaps
343	};
344	}
345
346	// Split into individual arrays
347	const channels: Texture2DProps[] = [];	×
348	const channelSize = height * width;	×
349	for (let i = 0; i < z; i++) {	×
350	channels.push({	×
351	data: data.subarray(i * channelSize, (i + 1) * channelSize),
352	width,
353	height,
354	format,
355	dataFormat,
356	type,
357	parameters,
358	mipmaps
359	});
360	}
361	return channels;	×
362	} catch (error) {
363	// Retry if Service Temporarily Unavailable 503 error etc.
364	if (	×
365	attempt < numAttempts &&	×
366	error instanceof FetchError &&
367	(
368	request.rasterServerServerErrorsToRetry ??	×
369	getApplicationConfig().rasterServerServerErrorsToRetry
370	)?.includes(error.response?.status as number)
371	) {
372	await sleep(	×
373	request.rasterServerRetryDelay ?? getApplicationConfig().rasterServerRetryDelay	×
374	);
375	continue;	×
376	}
377	}
378	}
379	return null;	×
380	},
381	request.rasterServerMaxPerServerRequests
382	);
383
384	return asset;	×
385	}
386
387	/**
388	* Create texture data for categorical colormap scale
389	* @param categoricalOptions - color map configuration and min-max values of categorical band
390	* @returns texture data
391	*/
392	export function generateCategoricalColormapTexture(
393	categoricalOptions: CategoricalColormapOptions
394	): Texture2DProps {
395	const data = generateCategoricalBitmapArray(categoricalOptions);	×
396	return {	×
397	data,
398	width: CATEGORICAL_TEXTURE_WIDTH,
399	height: 1,
400	format: GL.RGBA,
401	dataFormat: GL.RGBA,
402	type: GL.UNSIGNED_BYTE,
403	parameters: COLORMAP_TEXTURE_PARAMETERS,
404	mipmaps: false
405	};
406	}
407
408	// TODO: would probably be simpler to only pass in the props actually used by this function. That
409	// would mean a smaller object than RenderSubLayersProps
410	// eslint-disable-next-line max-statements, complexity
411	export function getModules({
412	images,
413	props
414	}: {
415	images: Partial<ImageData>;
416	props?: RenderSubLayersProps;
417	}): {
418	modules: ShaderModule[];
419	moduleProps: Record<string, any>;
420	} {
421	const moduleProps: Record<string, any> = {};	×
422	// Array of luma.gl WebGL modules to pass to the RasterLayer
423	const modules: ShaderModule[] = [];	×
424
425	// use rgba image directly. Used for raster .pmtiles rendering
426	if (images.imageRgba) {	×
427	modules.push(rgbaImage);	×
428
429	// no support for other modules atm for direct rgba mode
430	return {modules, moduleProps};	×
431	}
432
433	if (!props) {	×
434	return {modules, moduleProps};	×
435	}
436
437	const {
438	renderBandIndexes,
439	nonLinearRescaling,
440	linearRescalingFactor,
441	minPixelValue,
442	maxPixelValue,
443	gammaContrastFactor,
444	sigmoidalContrastFactor,
445	sigmoidalBiasFactor,
446	saturationValue,
447	bandCombination,
448	filterEnabled,
449	filterRange,
450	dataType,
451	minCategoricalBandValue,
452	maxCategoricalBandValue,
453	hasCategoricalColorMap
454	} = props;	×
455
456	if (Array.isArray(images.imageBands) && images.imageBands.length > 0) {	×
457	modules.push(getCombineBandsModule(images.imageBands));	×
458	}
459
460	if (images.imageMask) {	×
461	modules.push(getImageMaskModule(images.imageMask));	×
462	// In general, data masks are 0 for nodata and the maximum value for valid data, e.g. 255 or
463	// 65535 for uint8 or uint16 data, respectively
464	moduleProps.maskKeepMin = 1;	×
465	}
466
467	if (Array.isArray(renderBandIndexes)) {	×
468	modules.push(reorderBands);	×
469	moduleProps.ordering = renderBandIndexes;	×
470	}
471
472	const globalRange = maxPixelValue - minPixelValue;	×
473	// Fix rescaling if we are sure that dataset is categorical
474	if (hasCategoricalColorMap) {	×
475	modules.push(linearRescale);	×
476	moduleProps.linearRescaleScaler = 1 / maxPixelValue;	×
477	moduleProps.linearRescaleOffset = 0;	×
478	} else if (isRescalingAllowed(bandCombination)) {	×
479	if (!nonLinearRescaling) {	×
480	const [min, max] = linearRescalingFactor;	×
481	const localRange = max - min;	×
482
483	// Add linear rescaling module
484	modules.push(linearRescale);	×
485
486	// Divide by local range * global range
487	moduleProps.linearRescaleScaler = 1 / (localRange * globalRange);	×
488
489	// Subtract off the local min
490	moduleProps.linearRescaleOffset = -min;	×
491
492	// Clamp to [0, 1] done automatically?
493	} else {
494	modules.push(linearRescale);	×
495	moduleProps.linearRescaleScaler = 1 / maxPixelValue;	×
496	moduleProps.linearRescaleOffset = 0;	×
497
498	modules.push(gammaContrast);	×
499	moduleProps.gammaContrastValue = gammaContrastFactor;	×
500
501	modules.push(sigmoidalContrast);	×
502	moduleProps.sigmoidalContrast = sigmoidalContrastFactor;	×
503	moduleProps.sigmoidalBias = sigmoidalBiasFactor;	×
504	}
505
506	if (Number.isFinite(saturationValue) && saturationValue !== 1) {	×
507	modules.push(saturation);	×
508	moduleProps.saturationValue = saturationValue;	×
509	}
510	}
511
512	switch (bandCombination) {	×
513	case 'normalizedDifference':
514	modules.push(normalizedDifference);	×
515	break;	×
516	case 'enhancedVegetationIndex':
517	modules.push(enhancedVegetationIndex);	×
518	break;	×
519	case 'soilAdjustedVegetationIndex':
520	modules.push(soilAdjustedVegetationIndex);	×
521	break;	×
522	case 'modifiedSoilAdjustedVegetationIndex':
523	modules.push(modifiedSoilAdjustedVegetationIndex);	×
524	break;	×
525	default:
526	break;	×
527	}
528
529	if (isFilterAllowed(bandCombination) && filterEnabled) {	×
530	modules.push(filter);	×
531	moduleProps.filterMin1 = filterRange[0];	×
532	moduleProps.filterMax1 = filterRange[1];	×
533	}
534
535	// Apply colormap
536	if (isColormapAllowed(bandCombination) && images.imageColormap) {	×
537	modules.push(colormapModule);	×
538	moduleProps.minCategoricalBandValue = minCategoricalBandValue;	×
539	moduleProps.maxCategoricalBandValue = maxCategoricalBandValue;	×
540	moduleProps.dataTypeMaxValue = dtypeMaxValue[dataType];	×
541	moduleProps.maxPixelValue = maxPixelValue;	×
542	}
543
544	return {modules, moduleProps};	×
545	}

keplergl / kepler.gl / 23965535588

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous