18841170348

Committed 27 Oct 2025 12:36PM UTC coverage: 65.363% (+5.6%) from 59.735%

Build # 18841170348

Build Type

Pull #14

github

Committed by

web-flow

Commit Message

Merge 0a7e15414 into 921a76574

Pull Request Pull Request #14: Add multiple filter values support

Run Details

84 of 93 branches covered (90.32%)

Branch coverage included in aggregate %.

63 of 73 new or added lines in 3 files covered. (86.3%)

267 of 444 relevant lines covered (60.14%)

11.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.6

/lib/csv.js

import process from 'node:process'
import {pipeline} from 'node:stream/promises'

import contentDisposition from 'content-disposition'
import intoStream from 'into-stream'
import stringify from 'csv-write-stream'
import iconv from 'iconv-lite'
import createError from 'http-errors'
import {createGeocodeStream} from 'addok-geocode-stream'
import {previewCsvFromStream, validateCsvFromStream, createCsvReadStream} from '@livingdata/tabular-data-helpers'

const ADDOK_FILTERS = process.env.ADDOK_FILTERS ? process.env.ADDOK_FILTERS.split(',') : []

function ensureArray(value) {
  if (value) {
    return Array.isArray(value) ? value : [value]
  }

  return []
}

async function previewCsvBuffer(buffer) {
  const firstPassResult = await previewCsvFromStream(intoStream(buffer))

  if (firstPassResult.parseErrors && firstPassResult.parseErrors.includes('UndetectableDelimiter')) {
    return previewCsvFromStream(intoStream(buffer), {formatOptions: {delimiter: ','}})
  }

  return firstPassResult
}

export function csv({cluster, reverse}) {
  return async (req, res) => {
    if (!req.file) {
      throw createError(400, 'A CSV file must be provided in data field')
    }

    const {parseErrors, columns: columnsInFile, formatOptions} = await previewCsvBuffer(req.file.buffer)

    if (parseErrors) {
      throw createError(400, 'Errors in CSV file: ' + parseErrors.join(', '))
    }

    const logEntry = {
      type: 'csv',
      startedAt: new Date(),
      operation: reverse ? 'reverse' : 'search',
      columnsInFile,
      formatOptions,
      originalFileSize: req.file.buffer.length,
      originalFileName: req.file.originalName
    }

    await new Promise((resolve, reject) => {
      const fileStream = intoStream(req.file.buffer)

      validateCsvFromStream(fileStream, {formatOptions})
        .on('error', error => reject(createError(400, error.message)))
        .on('complete', ({readRows}) => {
          logEntry.rows = readRows
          resolve()
        })
    })

    const {originalName} = req.file

    const geocodeOptions = {}

    if (req.body.columns) {
      geocodeOptions.columns = ensureArray(req.body.columns)

      if (geocodeOptions.columns.some(c => !columnsInFile.includes(c))) {
        throw createError(400, 'At least one given column name is unknown')
      }
    }

    // Build filters object from ADDOK_FILTERS and req.body
    const filters = {}
    for (const filterName of ADDOK_FILTERS) {
      if (req.body[filterName]) {
        filters[filterName] = req.body[filterName]
      }
    }

    if (Object.keys(filters).length > 0) {
      geocodeOptions.filters = filters
    }

    if (req.body.lon) {
      geocodeOptions.lon = req.body.lon
    }

    if (req.body.lat) {
      geocodeOptions.lat = req.body.lat
    }

    if (req.body.result_columns) {
      geocodeOptions.resultColumns = ensureArray(req.body.result_columns)
    }

    logEntry.geocodeOptions = geocodeOptions

    const resultFileName = originalName ? 'geocoded-' + originalName : 'geocoded.csv'

    res
      .type('csv')
      .set('Content-Disposition', contentDisposition(resultFileName))

    try {
      await pipeline(
        intoStream(req.file.buffer),
        createCsvReadStream({formatOptions}),
        createGeocodeStream({
          cluster,
          strategy: 'cluster',
          reverse,
          ...geocodeOptions
        }),
        stringify({separator: formatOptions.delimiter, newline: formatOptions.linebreak}),
        iconv.encodeStream('utf8'),
        res
      )
    } catch (error) {
      res.destroy()
      console.log(error)
    } finally {
      logEntry.finishedAt = new Date()

      if (process.env.ENABLE_CSV_LOG === '1') {
        console.log(JSON.stringify(logEntry))
      }
    }
  }
}

1	import process from 'node:process'	4✔
2	import {pipeline} from 'node:stream/promises'	4✔
3		4✔
4	import contentDisposition from 'content-disposition'	4✔
5	import intoStream from 'into-stream'	4✔
6	import stringify from 'csv-write-stream'	4✔
7	import iconv from 'iconv-lite'	4✔
8	import createError from 'http-errors'	4✔
9	import {createGeocodeStream} from 'addok-geocode-stream'	4✔
10	import {previewCsvFromStream, validateCsvFromStream, createCsvReadStream} from '@livingdata/tabular-data-helpers'	4✔
11		4✔
12	const ADDOK_FILTERS = process.env.ADDOK_FILTERS ? process.env.ADDOK_FILTERS.split(',') : []	4!
13		4✔
14	function ensureArray(value) {	×
15	if (value) {	×
16	return Array.isArray(value) ? value : [value]	×
17	}	×
18		×
19	return []	×
20	}	×
21		4✔
22	async function previewCsvBuffer(buffer) {	×
23	const firstPassResult = await previewCsvFromStream(intoStream(buffer))	×
24		×
25	if (firstPassResult.parseErrors && firstPassResult.parseErrors.includes('UndetectableDelimiter')) {	×
26	return previewCsvFromStream(intoStream(buffer), {formatOptions: {delimiter: ','}})	×
27	}	×
28		×
29	return firstPassResult	×
30	}	×
31		4✔
32	export function csv({cluster, reverse}) {	4✔
33	return async (req, res) => {	64✔
34	if (!req.file) {	×
35	throw createError(400, 'A CSV file must be provided in data field')	×
36	}	×
37		×
38	const {parseErrors, columns: columnsInFile, formatOptions} = await previewCsvBuffer(req.file.buffer)	×
39		×
40	if (parseErrors) {	×
41	throw createError(400, 'Errors in CSV file: ' + parseErrors.join(', '))	×
42	}	×
43		×
44	const logEntry = {	×
45	type: 'csv',	×
46	startedAt: new Date(),	×
47	operation: reverse ? 'reverse' : 'search',	×
48	columnsInFile,	×
49	formatOptions,	×
50	originalFileSize: req.file.buffer.length,	×
51	originalFileName: req.file.originalName	×
52	}	×
53		×
54	await new Promise((resolve, reject) => {	×
55	const fileStream = intoStream(req.file.buffer)	×
56		×
57	validateCsvFromStream(fileStream, {formatOptions})	×
58	.on('error', error => reject(createError(400, error.message)))	×
59	.on('complete', ({readRows}) => {	×
60	logEntry.rows = readRows	×
61	resolve()	×
62	})	×
63	})	×
64		×
65	const {originalName} = req.file	×
66		×
67	const geocodeOptions = {}	×
68		×
69	if (req.body.columns) {	×
70	geocodeOptions.columns = ensureArray(req.body.columns)	×
71		×
72	if (geocodeOptions.columns.some(c => !columnsInFile.includes(c))) {	×
73	throw createError(400, 'At least one given column name is unknown')	×
74	}	×
75	}	×
76		×
NEW 77	// Build filters object from ADDOK_FILTERS and req.body	×
NEW 78	const filters = {}	×
NEW 79	for (const filterName of ADDOK_FILTERS) {	×
NEW 80	if (req.body[filterName]) {	×
NEW 81	filters[filterName] = req.body[filterName]	×
NEW 82	}	×
83	}	×
84		×
NEW 85	if (Object.keys(filters).length > 0) {	×
NEW 86	geocodeOptions.filters = filters	×
87	}	×
88		×
89	if (req.body.lon) {	×
90	geocodeOptions.lon = req.body.lon	×
91	}	×
92		×
93	if (req.body.lat) {	×
94	geocodeOptions.lat = req.body.lat	×
95	}	×
96		×
97	if (req.body.result_columns) {	×
98	geocodeOptions.resultColumns = ensureArray(req.body.result_columns)	×
99	}	×
100		×
101	logEntry.geocodeOptions = geocodeOptions	×
102		×
103	const resultFileName = originalName ? 'geocoded-' + originalName : 'geocoded.csv'	×
104		×
105	res	×
106	.type('csv')	×
107	.set('Content-Disposition', contentDisposition(resultFileName))	×
108		×
109	try {	×
110	await pipeline(	×
111	intoStream(req.file.buffer),	×
112	createCsvReadStream({formatOptions}),	×
113	createGeocodeStream({	×
114	cluster,	×
115	strategy: 'cluster',	×
116	reverse,	×
117	...geocodeOptions	×
118	}),	×
119	stringify({separator: formatOptions.delimiter, newline: formatOptions.linebreak}),	×
120	iconv.encodeStream('utf8'),	×
121	res	×
122	)	×
123	} catch (error) {	×
124	res.destroy()	×
125	console.log(error)	×
126	} finally {	×
127	logEntry.finishedAt = new Date()	×
128		×
129	if (process.env.ENABLE_CSV_LOG === '1') {	×
130	console.log(JSON.stringify(logEntry))	×
131	}	×
132	}	×
133	}	×
134	}	64✔

livingdata-co / addok-server / 18841170348

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous