17114267830

Committed 21 Aug 2025 01:07AM UTC coverage: 61.984% (-0.04%) from 62.027%

Build # 17114267830

Build Type

push

github

Committed by

web-flow

Commit Message

[fix] save raster layer config with layer, don't rely on app config (#3184)

* fix: save raster layer config with layer, dont rely on app config

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* nit

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

Coverage Stats

6315 of 12072 branches covered (52.31%)

Branch coverage included in aggregate %.

0 of 38 new or added lines in 5 files covered. (0.0%)

4 existing lines in 3 files now uncovered.

13022 of 19125 relevant lines covered (68.09%)

81.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.79

/src/layers/src/raster-tile/gpu-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

/**
 * Functions and constants for handling webgl/luma.gl/deck.gl entities
 */

import {parse, fetchFile, load, FetchError} from '@loaders.gl/core';
import {ImageLoader} from '@loaders.gl/images';
import {NPYLoader} from '@loaders.gl/textures';
import GL from '@luma.gl/constants';
import {Texture2DProps} from '@luma.gl/webgl';

import {sleep} from '@kepler.gl/common-utils';
import {getLoaderOptions} from '@kepler.gl/constants';
import {RasterWebGL} from '@kepler.gl/deckgl-layers';
import {getApplicationConfig} from '@kepler.gl/utils';

type ShaderModule = RasterWebGL.ShaderModule;
const {
  combineBandsFloat,
  combineBandsInt,
  combineBandsUint,
  maskFloat,
  maskInt,
  maskUint,
  linearRescale,
  gammaContrast,
  sigmoidalContrast,
  normalizedDifference,
  enhancedVegetationIndex,
  soilAdjustedVegetationIndex,
  modifiedSoilAdjustedVegetationIndex,
  colormap: colormapModule,
  filter,
  saturation,
  reorderBands,
  rgbaImage
} = RasterWebGL;

import {
  CATEGORICAL_TEXTURE_WIDTH,
  dtypeMaxValue,
  generateCategoricalBitmapArray,
  isColormapAllowed,
  isFilterAllowed,
  isRescalingAllowed
} from './raster-tile-utils';
import {
  CategoricalColormapOptions,
  ImageData,
  NPYLoaderDataTypes,
  NPYLoaderResponse,
  RenderSubLayersProps
} from './types';
import {getRequestThrottle} from './request-throttle';

/**
 * Describe WebGL2 Texture parameters to use for given input data type
 */
interface WebGLTextureFormat {
  format: number;
  dataFormat: number;
  type: number;
}

/**
 * Convert TypedArray to WebGL2 Texture Parameters
 */
function getWebGL2TextureParameters(data: NPYLoaderDataTypes): WebGLTextureFormat | never {
  if (data instanceof Uint8Array || data instanceof Uint8ClampedArray) {
    return {
      // Note: texture data has no auto-rescaling; pixel values stay as 0-255
      format: GL.R8UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_BYTE
    };
  }

  if (data instanceof Uint16Array) {
    return {
      format: GL.R16UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_SHORT
    };
  }

  if (data instanceof Uint32Array) {
    return {
      format: GL.R32UI,
      dataFormat: GL.RED_INTEGER,
      type: GL.UNSIGNED_INT
    };
  }

  if (data instanceof Int8Array) {
    return {
      format: GL.R8I,
      dataFormat: GL.RED_INTEGER,
      type: GL.BYTE
    };
  }

  if (data instanceof Int16Array) {
    return {
      format: GL.R16I,
      dataFormat: GL.RED_INTEGER,
      type: GL.SHORT
    };
  }
  if (data instanceof Int32Array) {
    return {
      format: GL.R32I,
      dataFormat: GL.RED_INTEGER,
      type: GL.INT
    };
  }
  if (data instanceof Float32Array) {
    return {
      format: GL.R32F,
      dataFormat: GL.RED,
      type: GL.FLOAT
    };
  }

  if (data instanceof Float64Array) {
    return {
      format: GL.R32F,
      dataFormat: GL.RED,
      type: GL.FLOAT
    };
  }

  // For exhaustive check above; following should never occur
  // https://stackoverflow.com/a/58009992
  const unexpectedInput: never = data;
  throw new Error(unexpectedInput);
}

/**
 * Discrete-valued colormaps (e.g. from the output of
 * classification algorithms) in the raster layer. Previously, the values passed to
 * `TEXTURE_MIN_FILTER` and `TEXTURE_MAG_FILTER` were `GL.LINEAR`, which meant that the GPU would
 * linearly interpolate values between two neighboring colormap pixel values. Setting these values
 * to NEAREST means that the GPU will choose the nearest value on the texture2D lookup operation,
 * which fixes precision issues for discrete-valued colormaps. This should be ok for continuous
 * colormaps as long as the color difference between each pixel on the colormap is small.
 */
export const COLORMAP_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
  [GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

const DEFAULT_8BIT_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.LINEAR_MIPMAP_LINEAR,
  [GL.TEXTURE_MAG_FILTER]: GL.LINEAR,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

const DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS = {
  [GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
  [GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
  [GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
  [GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
};

/**
 * Select correct module type for "combineBands"
 *
 * combineBands joins up to four 2D arrays (contained in imageBands) into a single "rgba" image
 * texture on the GPU. That shader code needs to have the same data type as the actual image data.
 * E.g. for float data the texture needs to be `sampler2D`, for uint data the texture needs to be
 * `usampler2D` and for int data the texture needs to be `isampler2D`.
 */
export function getCombineBandsModule(imageBands: Texture2DProps[]): ShaderModule {
  // Each image array is expected/required to be of the same data type
  switch (imageBands[0].format) {
    case GL.R8UI:
      return combineBandsUint;
    case GL.R16UI:
      return combineBandsUint;
    case GL.R32UI:
      return combineBandsUint;
    case GL.R8I:
      return combineBandsInt;
    case GL.R16I:
      return combineBandsInt;
    case GL.R32I:
      return combineBandsInt;
    case GL.R32F:
      return combineBandsFloat;
    default:
      throw new Error('bad format');
  }
}

/** Select correct image masking shader module for mask data type
 * The imageMask could (at least in the future, theoretically) be of a different data format than
 * the imageBands data itself.
 */
export function getImageMaskModule(imageMask: Texture2DProps): ShaderModule {
  switch (imageMask.format) {
    case GL.R8UI:
      return maskUint;
    case GL.R16UI:
      return maskUint;
    case GL.R32UI:
      return maskUint;
    case GL.R8I:
      return maskInt;
    case GL.R16I:
      return maskInt;
    case GL.R32I:
      return maskInt;
    case GL.R32F:
      return maskFloat;
    default:
      throw new Error('bad format');
  }
}

/**
 * Load image and wrap with default WebGL texture parameters
 *
 * @param url URL to load image
 * @param textureParams parameters to pass to Texture2D
 *
 * @return image object to pass to Texture2D constructor
 */
export async function loadImage(
  url: string,
  textureParams: Texture2DProps = {},
  requestOptions: RequestInit = {}
): Promise<Texture2DProps> {
  const response = await fetchFile(url, requestOptions);
  const image = await parse(response, ImageLoader);

  return {
    data: image,
    parameters: DEFAULT_8BIT_TEXTURE_PARAMETERS,
    format: GL.RGB,
    ...textureParams
  };
}

type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
type LoadingOptions = {
  fetch?: typeof fetch | FetchLike;
};

type NpyRequest = {
  url: string;
  rasterServerUrl: string;
  options: RequestInit;
  rasterServerMaxRetries?: number;
  rasterServerRetryDelay?: number;
  rasterServerServerErrorsToRetry?: number[];
  rasterServerMaxPerServerRequests?: number;
};

/**
 * Load NPY Array
 *
 * The NPY format is described here: https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html.
 * It's designed to be a very simple file format to hold an N-dimensional block of data. The header describes the data type, shape, and order (either C or Fortran) of the array.
 *
 * @param url URL to load NPY Array
 * @param split Whether to split single typed array representing an N-dimensional array into an Array with each dimension as its own typed array
 *
 * @return image object to pass to Texture2D constructor
 */
export async function loadNpyArray(
  request: NpyRequest,
  split: true,
  options?: LoadingOptions
): Promise<Texture2DProps[] | null>;
export async function loadNpyArray(
  request: NpyRequest,
  split: false,
  options?: LoadingOptions
): Promise<Texture2DProps | null>;
export async function loadNpyArray(
  request: NpyRequest,
  split: boolean,
  options?: LoadingOptions
): Promise<Texture2DProps | Texture2DProps[] | null> {
  const numAttempts =
    1 + (request.rasterServerMaxRetries ?? getApplicationConfig().rasterServerMaxRetries);

  const asset = await getRequestThrottle().throttleRequest(
    request.rasterServerUrl,
    async () => {
      for (let attempt = 0; attempt < numAttempts; attempt++) {
        try {
          const {npy: npyOptions} = getLoaderOptions();
          const response: NPYLoaderResponse = await load(request.url, NPYLoader, {
            npy: npyOptions,
            fetch: options?.fetch
          });

          if (!response || !response.data || request.options.signal?.aborted) {
            return null;
          }

          // Float64 data needs to be coerced to Float32 for the GPU
          if (response.data instanceof Float64Array) {
            response.data = Float32Array.from(response.data);
          }

          const {data, header} = response;
          const {shape} = header;
          const {format, dataFormat, type} = getWebGL2TextureParameters(data);

          // TODO: check height-width or width-height
          // Regardless, images usually square
          // TODO: handle cases of 256x256x1 instead of 1x256x256
          const [z, height, width] = shape;

          // Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
          const mipmaps = false;
          const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;

          if (!split) {
            return {
              data,
              width,
              height,
              format,
              dataFormat,
              type,
              parameters,
              mipmaps
            };
          }

          // Split into individual arrays
          const channels: Texture2DProps[] = [];
          const channelSize = height * width;
          for (let i = 0; i < z; i++) {
            channels.push({
              data: data.subarray(i * channelSize, (i + 1) * channelSize),
              width,
              height,
              format,
              dataFormat,
              type,
              parameters,
              mipmaps
            });
          }
          return channels;
        } catch (error) {
          // Retry if Service Temporarily Unavailable 503 error etc.
          if (
            attempt < numAttempts &&
            error instanceof FetchError &&
            (
              request.rasterServerServerErrorsToRetry ??
              getApplicationConfig().rasterServerServerErrorsToRetry
            )?.includes(error.response?.status as number)
          ) {
            await sleep(
              request.rasterServerRetryDelay ?? getApplicationConfig().rasterServerRetryDelay
            );
            continue;
          }
        }
      }
      return null;
    },
    request.rasterServerMaxPerServerRequests
  );

  return asset;
}

/**
 * Create texture data for categorical colormap scale
 * @param categoricalOptions - color map configuration and min-max values of categorical band
 * @returns texture data
 */
export function generateCategoricalColormapTexture(
  categoricalOptions: CategoricalColormapOptions
): Texture2DProps {
  const data = generateCategoricalBitmapArray(categoricalOptions);
  return {
    data,
    width: CATEGORICAL_TEXTURE_WIDTH,
    height: 1,
    format: GL.RGBA,
    dataFormat: GL.RGBA,
    type: GL.UNSIGNED_BYTE,
    parameters: COLORMAP_TEXTURE_PARAMETERS,
    mipmaps: false
  };
}

// TODO: would probably be simpler to only pass in the props actually used by this function. That
// would mean a smaller object than RenderSubLayersProps
// eslint-disable-next-line max-statements, complexity
export function getModules({
  images,
  props
}: {
  images: Partial<ImageData>;
  props?: RenderSubLayersProps;
}): {
  modules: ShaderModule[];
  moduleProps: Record<string, any>;
} {
  const moduleProps: Record<string, any> = {};
  // Array of luma.gl WebGL modules to pass to the RasterLayer
  const modules: ShaderModule[] = [];

  // use rgba image directly. Used for raster .pmtiles rendering
  if (images.imageRgba) {
    modules.push(rgbaImage);

    // no support for other modules atm for direct rgba mode
    return {modules, moduleProps};
  }

  if (!props) {
    return {modules, moduleProps};
  }

  const {
    renderBandIndexes,
    nonLinearRescaling,
    linearRescalingFactor,
    minPixelValue,
    maxPixelValue,
    gammaContrastFactor,
    sigmoidalContrastFactor,
    sigmoidalBiasFactor,
    saturationValue,
    bandCombination,
    filterEnabled,
    filterRange,
    dataType,
    minCategoricalBandValue,
    maxCategoricalBandValue,
    hasCategoricalColorMap
  } = props;

  if (Array.isArray(images.imageBands) && images.imageBands.length > 0) {
    modules.push(getCombineBandsModule(images.imageBands));
  }

  if (images.imageMask) {
    modules.push(getImageMaskModule(images.imageMask));
    // In general, data masks are 0 for nodata and the maximum value for valid data, e.g. 255 or
    // 65535 for uint8 or uint16 data, respectively
    moduleProps.maskKeepMin = 1;
  }

  if (Array.isArray(renderBandIndexes)) {
    modules.push(reorderBands);
    moduleProps.ordering = renderBandIndexes;
  }

  const globalRange = maxPixelValue - minPixelValue;
  // Fix rescaling if we are sure that dataset is categorical
  if (hasCategoricalColorMap) {
    modules.push(linearRescale);
    moduleProps.linearRescaleScaler = 1 / maxPixelValue;
    moduleProps.linearRescaleOffset = 0;
  } else if (isRescalingAllowed(bandCombination)) {
    if (!nonLinearRescaling) {
      const [min, max] = linearRescalingFactor;
      const localRange = max - min;

      // Add linear rescaling module
      modules.push(linearRescale);

      // Divide by local range * global range
      moduleProps.linearRescaleScaler = 1 / (localRange * globalRange);

      // Subtract off the local min
      moduleProps.linearRescaleOffset = -min;

      // Clamp to [0, 1] done automatically?
    } else {
      modules.push(linearRescale);
      moduleProps.linearRescaleScaler = 1 / maxPixelValue;
      moduleProps.linearRescaleOffset = 0;

      modules.push(gammaContrast);
      moduleProps.gammaContrastValue = gammaContrastFactor;

      modules.push(sigmoidalContrast);
      moduleProps.sigmoidalContrast = sigmoidalContrastFactor;
      moduleProps.sigmoidalBias = sigmoidalBiasFactor;
    }

    if (Number.isFinite(saturationValue) && saturationValue !== 1) {
      modules.push(saturation);
      moduleProps.saturationValue = saturationValue;
    }
  }

  switch (bandCombination) {
    case 'normalizedDifference':
      modules.push(normalizedDifference);
      break;
    case 'enhancedVegetationIndex':
      modules.push(enhancedVegetationIndex);
      break;
    case 'soilAdjustedVegetationIndex':
      modules.push(soilAdjustedVegetationIndex);
      break;
    case 'modifiedSoilAdjustedVegetationIndex':
      modules.push(modifiedSoilAdjustedVegetationIndex);
      break;
    default:
      break;
  }

  if (isFilterAllowed(bandCombination) && filterEnabled) {
    modules.push(filter);
    moduleProps.filterMin1 = filterRange[0];
    moduleProps.filterMax1 = filterRange[1];
  }

  // Apply colormap
  if (isColormapAllowed(bandCombination) && images.imageColormap) {
    modules.push(colormapModule);
    moduleProps.minCategoricalBandValue = minCategoricalBandValue;
    moduleProps.maxCategoricalBandValue = maxCategoricalBandValue;
    moduleProps.dataTypeMaxValue = dtypeMaxValue[dataType];
    moduleProps.maxPixelValue = maxPixelValue;
  }

  return {modules, moduleProps};
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	/**
5	* Functions and constants for handling webgl/luma.gl/deck.gl entities
6	*/
7
8	import {parse, fetchFile, load, FetchError} from '@loaders.gl/core';
9	import {ImageLoader} from '@loaders.gl/images';
10	import {NPYLoader} from '@loaders.gl/textures';
11	import GL from '@luma.gl/constants';
12	import {Texture2DProps} from '@luma.gl/webgl';
13
14	import {sleep} from '@kepler.gl/common-utils';
15	import {getLoaderOptions} from '@kepler.gl/constants';
16	import {RasterWebGL} from '@kepler.gl/deckgl-layers';
17	import {getApplicationConfig} from '@kepler.gl/utils';
18
19	type ShaderModule = RasterWebGL.ShaderModule;
20	const {
21	combineBandsFloat,
22	combineBandsInt,
23	combineBandsUint,
24	maskFloat,
25	maskInt,
26	maskUint,
27	linearRescale,
28	gammaContrast,
29	sigmoidalContrast,
30	normalizedDifference,
31	enhancedVegetationIndex,
32	soilAdjustedVegetationIndex,
33	modifiedSoilAdjustedVegetationIndex,
34	colormap: colormapModule,
35	filter,
36	saturation,
37	reorderBands,
38	rgbaImage
39	} = RasterWebGL;	13✔
40
41	import {
42	CATEGORICAL_TEXTURE_WIDTH,
43	dtypeMaxValue,
44	generateCategoricalBitmapArray,
45	isColormapAllowed,
46	isFilterAllowed,
47	isRescalingAllowed
48	} from './raster-tile-utils';
49	import {
50	CategoricalColormapOptions,
51	ImageData,
52	NPYLoaderDataTypes,
53	NPYLoaderResponse,
54	RenderSubLayersProps
55	} from './types';
56	import {getRequestThrottle} from './request-throttle';
57
58	/**
59	* Describe WebGL2 Texture parameters to use for given input data type
60	*/
61	interface WebGLTextureFormat {
62	format: number;
63	dataFormat: number;
64	type: number;
65	}
66
67	/**
68	* Convert TypedArray to WebGL2 Texture Parameters
69	*/
70	function getWebGL2TextureParameters(data: NPYLoaderDataTypes): WebGLTextureFormat \| never {
71	if (data instanceof Uint8Array \|\| data instanceof Uint8ClampedArray) {	×
72	return {	×
73	// Note: texture data has no auto-rescaling; pixel values stay as 0-255
74	format: GL.R8UI,
75	dataFormat: GL.RED_INTEGER,
76	type: GL.UNSIGNED_BYTE
77	};
78	}
79
80	if (data instanceof Uint16Array) {	×
81	return {	×
82	format: GL.R16UI,
83	dataFormat: GL.RED_INTEGER,
84	type: GL.UNSIGNED_SHORT
85	};
86	}
87
88	if (data instanceof Uint32Array) {	×
89	return {	×
90	format: GL.R32UI,
91	dataFormat: GL.RED_INTEGER,
92	type: GL.UNSIGNED_INT
93	};
94	}
95
96	if (data instanceof Int8Array) {	×
97	return {	×
98	format: GL.R8I,
99	dataFormat: GL.RED_INTEGER,
100	type: GL.BYTE
101	};
102	}
103
104	if (data instanceof Int16Array) {	×
105	return {	×
106	format: GL.R16I,
107	dataFormat: GL.RED_INTEGER,
108	type: GL.SHORT
109	};
110	}
111	if (data instanceof Int32Array) {	×
112	return {	×
113	format: GL.R32I,
114	dataFormat: GL.RED_INTEGER,
115	type: GL.INT
116	};
117	}
118	if (data instanceof Float32Array) {	×
119	return {	×
120	format: GL.R32F,
121	dataFormat: GL.RED,
122	type: GL.FLOAT
123	};
124	}
125
126	if (data instanceof Float64Array) {	×
127	return {	×
128	format: GL.R32F,
129	dataFormat: GL.RED,
130	type: GL.FLOAT
131	};
132	}
133
134	// For exhaustive check above; following should never occur
135	// https://stackoverflow.com/a/58009992
136	const unexpectedInput: never = data;	×
137	throw new Error(unexpectedInput);	×
138	}
139
140	/**
141	* Discrete-valued colormaps (e.g. from the output of
142	* classification algorithms) in the raster layer. Previously, the values passed to
143	* `TEXTURE_MIN_FILTER` and `TEXTURE_MAG_FILTER` were `GL.LINEAR`, which meant that the GPU would
144	* linearly interpolate values between two neighboring colormap pixel values. Setting these values
145	* to NEAREST means that the GPU will choose the nearest value on the texture2D lookup operation,
146	* which fixes precision issues for discrete-valued colormaps. This should be ok for continuous
147	* colormaps as long as the color difference between each pixel on the colormap is small.
148	*/
149	export const COLORMAP_TEXTURE_PARAMETERS = {	13✔
150	[GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
151	[GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
152	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
153	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
154	};
155
156	const DEFAULT_8BIT_TEXTURE_PARAMETERS = {	13✔
157	[GL.TEXTURE_MIN_FILTER]: GL.LINEAR_MIPMAP_LINEAR,
158	[GL.TEXTURE_MAG_FILTER]: GL.LINEAR,
159	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
160	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
161	};
162
163	const DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS = {	13✔
164	[GL.TEXTURE_MIN_FILTER]: GL.NEAREST,
165	[GL.TEXTURE_MAG_FILTER]: GL.NEAREST,
166	[GL.TEXTURE_WRAP_S]: GL.CLAMP_TO_EDGE,
167	[GL.TEXTURE_WRAP_T]: GL.CLAMP_TO_EDGE
168	};
169
170	/**
171	* Select correct module type for "combineBands"
172	*
173	* combineBands joins up to four 2D arrays (contained in imageBands) into a single "rgba" image
174	* texture on the GPU. That shader code needs to have the same data type as the actual image data.
175	* E.g. for float data the texture needs to be `sampler2D`, for uint data the texture needs to be
176	* `usampler2D` and for int data the texture needs to be `isampler2D`.
177	*/
178	export function getCombineBandsModule(imageBands: Texture2DProps[]): ShaderModule {
179	// Each image array is expected/required to be of the same data type
180	switch (imageBands[0].format) {	×
181	case GL.R8UI:
182	return combineBandsUint;	×
183	case GL.R16UI:
184	return combineBandsUint;	×
185	case GL.R32UI:
186	return combineBandsUint;	×
187	case GL.R8I:
188	return combineBandsInt;	×
189	case GL.R16I:
190	return combineBandsInt;	×
191	case GL.R32I:
192	return combineBandsInt;	×
193	case GL.R32F:
194	return combineBandsFloat;	×
195	default:
196	throw new Error('bad format');	×
197	}
198	}
199
200	/** Select correct image masking shader module for mask data type
201	* The imageMask could (at least in the future, theoretically) be of a different data format than
202	* the imageBands data itself.
203	*/
204	export function getImageMaskModule(imageMask: Texture2DProps): ShaderModule {
205	switch (imageMask.format) {	×
206	case GL.R8UI:
207	return maskUint;	×
208	case GL.R16UI:
209	return maskUint;	×
210	case GL.R32UI:
211	return maskUint;	×
212	case GL.R8I:
213	return maskInt;	×
214	case GL.R16I:
215	return maskInt;	×
216	case GL.R32I:
217	return maskInt;	×
218	case GL.R32F:
219	return maskFloat;	×
220	default:
221	throw new Error('bad format');	×
222	}
223	}
224
225	/**
226	* Load image and wrap with default WebGL texture parameters
227	*
228	* @param url URL to load image
229	* @param textureParams parameters to pass to Texture2D
230	*
231	* @return image object to pass to Texture2D constructor
232	*/
233	export async function loadImage(
234	url: string,
235	textureParams: Texture2DProps = {},	×
236	requestOptions: RequestInit = {}	×
237	): Promise<Texture2DProps> {
238	const response = await fetchFile(url, requestOptions);	×
239	const image = await parse(response, ImageLoader);	×
240
241	return {	×
242	data: image,
243	parameters: DEFAULT_8BIT_TEXTURE_PARAMETERS,
244	format: GL.RGB,
245	...textureParams
246	};
247	}
248
249	type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
250	type LoadingOptions = {
251	fetch?: typeof fetch \| FetchLike;
252	};
253
254	type NpyRequest = {
255	url: string;
256	rasterServerUrl: string;
257	options: RequestInit;
258	rasterServerMaxRetries?: number;
259	rasterServerRetryDelay?: number;
260	rasterServerServerErrorsToRetry?: number[];
261	rasterServerMaxPerServerRequests?: number;
262	};
263
264	/**
265	* Load NPY Array
266	*
267	* The NPY format is described here: https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html.
268	* It's designed to be a very simple file format to hold an N-dimensional block of data. The header describes the data type, shape, and order (either C or Fortran) of the array.
269	*
270	* @param url URL to load NPY Array
271	* @param split Whether to split single typed array representing an N-dimensional array into an Array with each dimension as its own typed array
272	*
273	* @return image object to pass to Texture2D constructor
274	*/
275	export async function loadNpyArray(
276	request: NpyRequest,
277	split: true,
278	options?: LoadingOptions
279	): Promise<Texture2DProps[] \| null>;
280	export async function loadNpyArray(
281	request: NpyRequest,
282	split: false,
283	options?: LoadingOptions
284	): Promise<Texture2DProps \| null>;
285	export async function loadNpyArray(
286	request: NpyRequest,
287	split: boolean,
288	options?: LoadingOptions
289	): Promise<Texture2DProps \| Texture2DProps[] \| null> {
290	const numAttempts =
NEW 291	1 + (request.rasterServerMaxRetries ?? getApplicationConfig().rasterServerMaxRetries);	×
292
NEW 293	const asset = await getRequestThrottle().throttleRequest(	×
294	request.rasterServerUrl,
295	async () => {
NEW 296	for (let attempt = 0; attempt < numAttempts; attempt++) {	×
NEW 297	try {	×
NEW 298	const {npy: npyOptions} = getLoaderOptions();	×
NEW 299	const response: NPYLoaderResponse = await load(request.url, NPYLoader, {	×
300	npy: npyOptions,
301	fetch: options?.fetch
302	});
303
NEW 304	if (!response \|\| !response.data \|\| request.options.signal?.aborted) {	×
NEW 305	return null;	×
306	}
307
308	// Float64 data needs to be coerced to Float32 for the GPU
NEW 309	if (response.data instanceof Float64Array) {	×
NEW 310	response.data = Float32Array.from(response.data);	×
311	}
312
NEW 313	const {data, header} = response;	×
NEW 314	const {shape} = header;	×
NEW 315	const {format, dataFormat, type} = getWebGL2TextureParameters(data);	×
316
317	// TODO: check height-width or width-height
318	// Regardless, images usually square
319	// TODO: handle cases of 256x256x1 instead of 1x256x256
NEW 320	const [z, height, width] = shape;	×
321
322	// Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
NEW 323	const mipmaps = false;	×
NEW 324	const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;	×
325
NEW 326	if (!split) {	×
NEW 327	return {	×
328	data,
329	width,
330	height,
331	format,
332	dataFormat,
333	type,
334	parameters,
335	mipmaps
336	};
337	}
338
339	// Split into individual arrays
NEW 340	const channels: Texture2DProps[] = [];	×
NEW 341	const channelSize = height * width;	×
NEW 342	for (let i = 0; i < z; i++) {	×
NEW 343	channels.push({	×
344	data: data.subarray(i * channelSize, (i + 1) * channelSize),
345	width,
346	height,
347	format,
348	dataFormat,
349	type,
350	parameters,
351	mipmaps
352	});
353	}
NEW 354	return channels;	×
355	} catch (error) {
356	// Retry if Service Temporarily Unavailable 503 error etc.
NEW 357	if (	×
358	attempt < numAttempts &&	×
359	error instanceof FetchError &&
360	(
361	request.rasterServerServerErrorsToRetry ??	×
362	getApplicationConfig().rasterServerServerErrorsToRetry
363	)?.includes(error.response?.status as number)
364	) {
NEW 365	await sleep(	×
366	request.rasterServerRetryDelay ?? getApplicationConfig().rasterServerRetryDelay	×
367	);
NEW 368	continue;	×
369	}
370	}
371	}
NEW 372	return null;	×
373	},
374	request.rasterServerMaxPerServerRequests
375	);
376
377	return asset;	×
378	}
379
380	/**
381	* Create texture data for categorical colormap scale
382	* @param categoricalOptions - color map configuration and min-max values of categorical band
383	* @returns texture data
384	*/
385	export function generateCategoricalColormapTexture(
386	categoricalOptions: CategoricalColormapOptions
387	): Texture2DProps {
388	const data = generateCategoricalBitmapArray(categoricalOptions);	×
389	return {	×
390	data,
391	width: CATEGORICAL_TEXTURE_WIDTH,
392	height: 1,
393	format: GL.RGBA,
394	dataFormat: GL.RGBA,
395	type: GL.UNSIGNED_BYTE,
396	parameters: COLORMAP_TEXTURE_PARAMETERS,
397	mipmaps: false
398	};
399	}
400
401	// TODO: would probably be simpler to only pass in the props actually used by this function. That
402	// would mean a smaller object than RenderSubLayersProps
403	// eslint-disable-next-line max-statements, complexity
404	export function getModules({
405	images,
406	props
407	}: {
408	images: Partial<ImageData>;
409	props?: RenderSubLayersProps;
410	}): {
411	modules: ShaderModule[];
412	moduleProps: Record<string, any>;
413	} {
414	const moduleProps: Record<string, any> = {};	×
415	// Array of luma.gl WebGL modules to pass to the RasterLayer
416	const modules: ShaderModule[] = [];	×
417
418	// use rgba image directly. Used for raster .pmtiles rendering
419	if (images.imageRgba) {	×
420	modules.push(rgbaImage);	×
421
422	// no support for other modules atm for direct rgba mode
423	return {modules, moduleProps};	×
424	}
425
426	if (!props) {	×
427	return {modules, moduleProps};	×
428	}
429
430	const {
431	renderBandIndexes,
432	nonLinearRescaling,
433	linearRescalingFactor,
434	minPixelValue,
435	maxPixelValue,
436	gammaContrastFactor,
437	sigmoidalContrastFactor,
438	sigmoidalBiasFactor,
439	saturationValue,
440	bandCombination,
441	filterEnabled,
442	filterRange,
443	dataType,
444	minCategoricalBandValue,
445	maxCategoricalBandValue,
446	hasCategoricalColorMap
447	} = props;	×
448
449	if (Array.isArray(images.imageBands) && images.imageBands.length > 0) {	×
450	modules.push(getCombineBandsModule(images.imageBands));	×
451	}
452
453	if (images.imageMask) {	×
454	modules.push(getImageMaskModule(images.imageMask));	×
455	// In general, data masks are 0 for nodata and the maximum value for valid data, e.g. 255 or
456	// 65535 for uint8 or uint16 data, respectively
457	moduleProps.maskKeepMin = 1;	×
458	}
459
460	if (Array.isArray(renderBandIndexes)) {	×
461	modules.push(reorderBands);	×
462	moduleProps.ordering = renderBandIndexes;	×
463	}
464
465	const globalRange = maxPixelValue - minPixelValue;	×
466	// Fix rescaling if we are sure that dataset is categorical
467	if (hasCategoricalColorMap) {	×
468	modules.push(linearRescale);	×
469	moduleProps.linearRescaleScaler = 1 / maxPixelValue;	×
470	moduleProps.linearRescaleOffset = 0;	×
471	} else if (isRescalingAllowed(bandCombination)) {	×
472	if (!nonLinearRescaling) {	×
473	const [min, max] = linearRescalingFactor;	×
474	const localRange = max - min;	×
475
476	// Add linear rescaling module
477	modules.push(linearRescale);	×
478
479	// Divide by local range * global range
480	moduleProps.linearRescaleScaler = 1 / (localRange * globalRange);	×
481
482	// Subtract off the local min
483	moduleProps.linearRescaleOffset = -min;	×
484
485	// Clamp to [0, 1] done automatically?
486	} else {
487	modules.push(linearRescale);	×
488	moduleProps.linearRescaleScaler = 1 / maxPixelValue;	×
489	moduleProps.linearRescaleOffset = 0;	×
490
491	modules.push(gammaContrast);	×
492	moduleProps.gammaContrastValue = gammaContrastFactor;	×
493
494	modules.push(sigmoidalContrast);	×
495	moduleProps.sigmoidalContrast = sigmoidalContrastFactor;	×
496	moduleProps.sigmoidalBias = sigmoidalBiasFactor;	×
497	}
498
499	if (Number.isFinite(saturationValue) && saturationValue !== 1) {	×
500	modules.push(saturation);	×
501	moduleProps.saturationValue = saturationValue;	×
502	}
503	}
504
505	switch (bandCombination) {	×
506	case 'normalizedDifference':
507	modules.push(normalizedDifference);	×
508	break;	×
509	case 'enhancedVegetationIndex':
510	modules.push(enhancedVegetationIndex);	×
511	break;	×
512	case 'soilAdjustedVegetationIndex':
513	modules.push(soilAdjustedVegetationIndex);	×
514	break;	×
515	case 'modifiedSoilAdjustedVegetationIndex':
516	modules.push(modifiedSoilAdjustedVegetationIndex);	×
517	break;	×
518	default:
519	break;	×
520	}
521
522	if (isFilterAllowed(bandCombination) && filterEnabled) {	×
523	modules.push(filter);	×
524	moduleProps.filterMin1 = filterRange[0];	×
525	moduleProps.filterMax1 = filterRange[1];	×
526	}
527
528	// Apply colormap
529	if (isColormapAllowed(bandCombination) && images.imageColormap) {	×
530	modules.push(colormapModule);	×
531	moduleProps.minCategoricalBandValue = minCategoricalBandValue;	×
532	moduleProps.maxCategoricalBandValue = maxCategoricalBandValue;	×
533	moduleProps.dataTypeMaxValue = dtypeMaxValue[dataType];	×
534	moduleProps.maxPixelValue = maxPixelValue;	×
535	}
536
537	return {modules, moduleProps};	×
538	}

keplergl / kepler.gl / 17114267830

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous