• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

vanvalenlab / deepcell-label / 5720827273

pending completion
5720827273

Pull #519

github

web-flow
Merge 11c4d796a into 556f9652b
Pull Request #519: More reqs cleanup

470 of 1239 branches covered (37.93%)

Branch coverage included in aggregate %.

2559 of 4707 relevant lines covered (54.37%)

288.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.71
/frontend/src/Project/service/loadMachine.ts
1
/** Fetches data from the project storage API, including raw image data, label image data, and labels. */
2

3
import { loadOmeTiff } from '@hms-dbmi/viv';
4
import * as zip from '@zip.js/zip.js';
5
import { assign, createMachine, sendParent } from 'xstate';
6

7
type PropType<TObj, TProp extends keyof TObj> = TObj[TProp];
8
type UnboxPromise<T extends Promise<any>> = T extends Promise<infer U> ? U : never;
9

10
type OmeTiff = UnboxPromise<ReturnType<typeof loadOmeTiff>>;
11
type TiffPixelSource = PropType<OmeTiff, 'data'>[number];
12
type Spots = [number, number][];
13
type Divisions = { parent: number; daughters: number[]; t: number }[];
14
type Cells = { value: number; cell: number; t: number }[];
15
type CellTypes = { id: number; name: string; color: string; cells: number[] };
16
type Embeddings = number[][];
17
type Files = {
18
  [filename: string]: OmeTiff | Spots | Cells | CellTypes | Embeddings | Divisions;
19
};
20

21
async function parseZip(response: Response) {
22
  const blob = await response.blob();
14✔
23
  const reader = new zip.ZipReader(new zip.BlobReader(blob));
14✔
24
  const entries = await reader.getEntries();
14✔
25
  await reader.close();
14✔
26
  const files: Files = {};
14✔
27
  for (const entry of entries) {
14✔
28
    if (entry.filename.endsWith('.ome.tiff')) {
64✔
29
      // @ts-ignore
30
      const data = await entry.getData(new zip.BlobWriter());
24✔
31
      const omeTiff: OmeTiff = await loadOmeTiff(data);
24✔
32
      files[entry.filename] = omeTiff;
24✔
33
    }
34
    if (entry.filename === 'spots.csv') {
64✔
35
      // @ts-ignore
36
      const csv = await entry.getData(new zip.TextWriter());
2✔
37
      let spots: Spots = csv
2✔
38
        .split('\n')
39
        .map((row: string) => row.split(',').map(Number))
56✔
40
        .map(([x, y]: number[]) => [x, y]); // Use only x and y columns
56✔
41
      spots.shift(); // Remove header row
2✔
42
      spots.pop(); // Remove last empty row from final newline
2✔
43
      files[entry.filename] = spots;
2✔
44
    }
45
    if (entry.filename === 'divisions.json') {
64✔
46
      // @ts-ignore
47
      const json = await entry.getData(new zip.TextWriter());
12✔
48
      const divisions: Divisions = JSON.parse(json);
12✔
49
      files[entry.filename] = divisions;
12✔
50
    }
51
    if (entry.filename === 'cells.json') {
64✔
52
      // @ts-ignore
53
      const json = await entry.getData(new zip.TextWriter());
12✔
54
      const cells = JSON.parse(json);
12✔
55
      files[entry.filename] = cells;
12✔
56
    }
57
    if (entry.filename === 'cellTypes.json') {
64✔
58
      // @ts-ignore
59
      const json = await entry.getData(new zip.TextWriter());
10✔
60
      const cellTypes: CellTypes = JSON.parse(json);
8✔
61
      files[entry.filename] = cellTypes;
8✔
62
    }
63
    if (entry.filename === 'embeddings.json') {
62!
64
      // @ts-ignore
65
      const json = await entry.getData(new zip.TextWriter());
×
66
      const embeddings: Embeddings = JSON.parse(json);
×
67
      files[entry.filename] = embeddings;
×
68
    }
69
  }
70
  return { files };
12✔
71
}
72

73
function fetchZip(context: Context) {
74
  const { projectId } = context;
14✔
75
  const forceLoadOutput =
76
    new URLSearchParams(window.location.search).get('forceLoadOutput') === 'true';
14✔
77
  if (forceLoadOutput) {
14!
78
    const params = new URLSearchParams({ bucket: 'deepcell-label-output' });
×
79
    return fetch(`/api/project/${projectId}?` + params).then(parseZip);
×
80
  }
81
  return fetch(`/api/project/${projectId}`).then(parseZip);
14✔
82
}
83

84
async function splitArrays(files: Files) {
85
  const rawFile = files['X.ome.tiff'] as OmeTiff;
12✔
86
  const labeledFile = files['y.ome.tiff'] as OmeTiff;
12✔
87
  const raw = await getRawRasters(rawFile.data[0]);
12✔
88
  const labeled = await getLabelRasters(labeledFile.data[0]);
12✔
89
  const rawOriginal = await getRawOriginalRasters(rawFile.data[0]);
12✔
90
  return { raw, labeled, rawOriginal };
12✔
91
}
92

93
async function getRawOriginalRasters(source: TiffPixelSource) {
94
  const { labels, shape } = source;
12✔
95
  const c = shape[labels.indexOf('c')];
12✔
96
  const z = shape[labels.indexOf('z')];
12✔
97
  const channels = [];
12✔
98
  for (let i = 0; i < c; i++) {
12✔
99
    const frames = [];
32✔
100
    for (let j = 0; j < z; j++) {
32✔
101
      const selection = { t: 0, c: i, z: j };
34✔
102
      const raster = (await source.getRaster({ selection })) as Raster;
34✔
103
      const frame = splitRows(raster);
34✔
104
      frames.push(frame);
34✔
105
    }
106
    channels.push(frames);
32✔
107
  }
108
  return channels;
12✔
109
}
110

111
async function getRawRasters(source: TiffPixelSource) {
112
  const { labels, shape } = source;
12✔
113
  const c = shape[labels.indexOf('c')];
12✔
114
  const z = shape[labels.indexOf('z')];
12✔
115
  const channels = [];
12✔
116
  var max = Array(c).fill(0);
12✔
117
  var min = Array(c).fill(Infinity);
12✔
118
  for (let i = 0; i < c; i++) {
12✔
119
    const frames = [];
32✔
120
    for (let j = 0; j < z; j++) {
32✔
121
      const selection = { t: 0, c: i, z: j };
34✔
122
      const raster = (await source.getRaster({ selection })) as Raster;
34✔
123
      const frame = splitRows(raster);
34✔
124
      // Record max and min across frames
125
      for (let k = 0; k < frame.length; k++) {
34✔
126
        for (let l = 0; l < frame[k].length; l++) {
1,048✔
127
          if (frame[k][l] > max[i]) {
68,432✔
128
            max[i] = frame[k][l];
228✔
129
          }
130
          if (frame[k][l] < min[i]) {
68,432✔
131
            min[i] = frame[k][l];
124✔
132
          }
133
        }
134
      }
135
      frames.push(frame);
34✔
136
    }
137
    channels.push(frames);
32✔
138
  }
139
  const reshaped = reshapeRaw(channels, min, max);
12✔
140
  return reshaped;
12✔
141
}
142

143
async function getLabelRasters(source: TiffPixelSource) {
144
  const { labels, shape } = source;
12✔
145
  const c = shape[labels.indexOf('c')];
12✔
146
  const z = shape[labels.indexOf('z')];
12✔
147
  const channels = [];
12✔
148
  for (let i = 0; i < c; i++) {
12✔
149
    const frames = [];
14✔
150
    for (let j = 0; j < z; j++) {
14✔
151
      const selection = { t: 0, c: i, z: j };
16✔
152
      const raster = (await source.getRaster({ selection })) as Raster;
16✔
153
      const frame = splitRows(raster) as Int32Array[];
16✔
154
      frames.push(frame);
16✔
155
    }
156
    channels.push(frames);
14✔
157
  }
158
  return channels;
12✔
159
}
160

161
function reshapeRaw(channels: TypedArray[][][], min: number[], max: number[]) {
162
  const size_c = channels.length;
12✔
163
  const size_z = channels[0].length;
12✔
164
  const size_y = channels[0][0].length;
12✔
165
  const size_x = channels[0][0][0].length;
12✔
166
  const reshaped = [];
12✔
167
  // Normalize each pixel to 0-255 for rendering
168
  for (let c = 0; c < size_c; c++) {
12✔
169
    const channelMin = min[c];
32✔
170
    const channelMax = max[c];
32✔
171
    const frames = [];
32✔
172
    for (let z = 0; z < size_z; z++) {
32✔
173
      const frame = [];
34✔
174
      for (let y = 0; y < size_y; y++) {
34✔
175
        const row = new Uint8Array(size_x);
1,048✔
176
        for (let x = 0; x < size_x; x++) {
1,048✔
177
          row[x] = Math.round(
68,432✔
178
            ((channels[c][z][y][x] - channelMin) / (channelMax - channelMin)) * 255
179
          );
180
        }
181
        frame.push(row);
1,048✔
182
      }
183
      frames.push(frame);
34✔
184
    }
185
    reshaped.push(frames);
32✔
186
  }
187
  return reshaped;
12✔
188
}
189

190
type TypedArray =
191
  | Uint8Array
192
  | Int8Array
193
  | Uint16Array
194
  | Int16Array
195
  | Uint32Array
196
  | Int32Array
197
  | Float32Array
198
  | Float64Array;
199

200
type Raster = { data: TypedArray; width: number; height: number };
201

202
function splitRows(raster: Raster) {
203
  const { data, width, height } = raster;
84✔
204
  const frame = [];
84✔
205
  for (let i = 0; i < height; i++) {
84✔
206
    const row =
207
      data instanceof Uint8Array || data instanceof Int8Array
2,784✔
208
        ? new Uint8Array(data.buffer, width * i, width)
209
        : data instanceof Uint16Array || data instanceof Int16Array
2,064!
210
        ? new Uint16Array(data.buffer, width * i * 2, width)
211
        : data instanceof Uint32Array
688!
212
        ? new Uint32Array(data.buffer, width * i * 4, width)
213
        : data instanceof Float32Array
688!
214
        ? new Float32Array(data.buffer, width * i * 4, width)
215
        : data instanceof Float64Array
688!
216
        ? new Float64Array(data.buffer, width * i * 8, width)
217
        : new Int32Array(data.buffer, width * i * 4, width);
218
    frame.push(row);
2,784✔
219
  }
220
  return frame;
84✔
221
}
222

223
interface Context {
224
  projectId: string;
225
  // dimensions: string | null;
226
  // shape: [number, number, number, number] | null;
227
  width: number | null;
228
  height: number | null;
229
  t: number | null;
230
  channels: string[] | null;
231
  numChannels: number | null;
232
  numFeatures: number | null;
233
  raw: Uint8Array[][][] | null;
234
  labeled: Int32Array[][][] | null;
235
  rawOriginal: TypedArray[][][] | null;
236
  labels: Cells | null;
237
  spots: Spots | null;
238
  divisions: Divisions | null;
239
  cells: Cells | null;
240
  cellTypes: CellTypes | null;
241
  embeddings: Embeddings | null;
242
}
243

244
const createLoadMachine = (projectId: string) =>
17✔
245
  createMachine(
14✔
246
    {
247
      context: {
248
        projectId,
249
        // dimensions: null,
250
        // shape: null,
251
        width: null,
252
        height: null,
253
        t: null,
254
        channels: null,
255
        numChannels: null,
256
        numFeatures: null,
257
        raw: null,
258
        labeled: null,
259
        rawOriginal: null,
260
        labels: null,
261
        spots: null,
262
        divisions: null,
263
        cells: null,
264
        cellTypes: null,
265
        embeddings: null,
266
      },
267
      tsTypes: {} as import('./loadMachine.typegen').Typegen0,
268
      schema: {
269
        context: {} as Context,
270
        services: {} as {
271
          'fetch project zip': {
272
            data: {
273
              files: Files;
274
            };
275
          };
276
          'split arrays': {
277
            data: {
278
              raw: Uint8Array[][][];
279
              labeled: Int32Array[][][];
280
              rawOriginal: TypedArray[][][];
281
            };
282
          };
283
        },
284
      },
285
      id: 'load',
286
      initial: 'loading',
287
      states: {
288
        loading: {
289
          invoke: {
290
            src: 'fetch project zip',
291
            onDone: {
292
              target: 'splitArrays',
293
              actions: [
294
                'set spots',
295
                'set divisions',
296
                'set cells',
297
                'set cellTypes',
298
                'set embeddings',
299
                'set metadata',
300
              ],
301
            },
302
            onError: {
303
              actions: 'send project not in output bucket',
304
            },
305
          },
306
        },
307
        splitArrays: {
308
          invoke: {
309
            src: 'split arrays',
310
            onDone: { target: 'loaded', actions: 'set arrays' },
311
          },
312
        },
313
        loaded: {
314
          type: 'final',
315
          entry: 'send loaded',
316
        },
317
      },
318
    },
319
    {
320
      services: {
321
        'fetch project zip': fetchZip,
322
        'split arrays': (ctx, evt) => splitArrays(evt.data.files),
12✔
323
      },
324
      actions: {
325
        'send project not in output bucket': sendParent('PROJECT_NOT_IN_OUTPUT_BUCKET'),
326
        'set spots': assign({
327
          // @ts-ignore
328
          spots: (context, event) => event.data.files['spots.csv'] as Spots,
12✔
329
        }),
330
        'set divisions': assign({
331
          // @ts-ignore
332
          divisions: (context, event) => event.data.files['divisions.json'] as Divisions,
12✔
333
        }),
334
        'set cells': assign({
335
          // @ts-ignore
336
          cells: (context, event) => event.data.files['cells.json'] as Cells,
12✔
337
        }),
338
        'set cellTypes': assign({
339
          // @ts-ignore
340
          cellTypes: (context, event) => {
341
            const cellTypes = event.data.files['cellTypes.json'] as CellTypes;
12✔
342
            if (cellTypes) {
12✔
343
              return cellTypes;
8✔
344
            }
345
            return [];
4✔
346
          },
347
        }),
348
        'set embeddings': assign({
349
          // @ts-ignore
350
          embeddings: (context, event) => event.data.files['embeddings.json'] as Embeddings,
12✔
351
        }),
352
        'set metadata': assign((ctx, evt) => {
353
          // @ts-ignore
354
          const { metadata } = evt.data.files['X.ome.tiff'];
12✔
355
          const { SizeX, SizeY, SizeZ, SizeC, Channels } = metadata.Pixels;
12✔
356
          // @ts-ignore
357
          const { metadata: labelMetadata } = evt.data.files['y.ome.tiff'];
12✔
358
          const { SizeC: labelSizeC } = labelMetadata;
12✔
359
          const channelNames = Channels.map((i: any) => i.Name);
32✔
360
          return {
12✔
361
            width: SizeX,
362
            height: SizeY,
363
            t: SizeZ,
364
            // SizeT,
365
            numChannels: SizeC,
366
            numFeatures: labelSizeC,
367
            channels: channelNames,
368
          };
369
        }),
370
        'set arrays': assign({
371
          raw: (_, event) => event.data.raw,
12✔
372
          labeled: (_, event) => event.data.labeled,
12✔
373
          rawOriginal: (_, event) => event.data.rawOriginal,
12✔
374
        }),
375
        'send loaded': sendParent((ctx) => ({
12✔
376
          type: 'LOADED',
377
          raw: ctx.raw,
378
          labeled: ctx.labeled,
379
          rawOriginal: ctx.rawOriginal,
380
          spots: ctx.spots,
381
          divisions: ctx.divisions,
382
          cells: ctx.cells,
383
          cellTypes: ctx.cellTypes,
384
          channels: ctx.channels,
385
          embeddings: ctx.embeddings,
386
        })),
387
      },
388
    }
389
  );
390

391
export default createLoadMachine;
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc