27540052150

Committed 15 Jun 2026 10:27AM UTC coverage: 86.562% (-0.4%) from 86.922%

Build # 27540052150

Build Type

Pull #1644

github

Committed by

dawoldo

Commit Message

refactor(1598): migrate cave update integration tests to async/await syntax

Pull Request Pull Request #1644: Feat/1598 organization responsible geo entity

Coverage Stats

3603 of 4335 branches covered (83.11%)

Branch coverage included in aggregate %.

147 of 194 new or added lines in 14 files covered. (75.77%)

3 existing lines in 2 files now uncovered.

7232 of 8182 relevant lines covered (88.39%)

55.12 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.83

/api/utils/csvHelper.js

const http = require('http');
const https = require('https');
const mime = require('mime-types');
const moment = require('moment');
const CaverService = require('../services/CaverService');

// Usage example: https://ontology.uis-speleo.org/example/#Gouffre_Jean_Bernard
// Extract: Gouffre_Jean_Bernard
function extractUrlFragment(urlRaw) {
  const url = urlRaw.trim();
  if (url.startsWith('http')) return url.split('#')[1];
  return url;
}

// https://developer.mozilla.org/en-US/docs/Glossary/Truthy
function valIfTruthyOrNull(val) {
  return val || null;
}

function parsePersonName(fullName) {
  const nameParts = fullName.split(' ');
  if (nameParts.length > 3) return [fullName, undefined, undefined];

  const firstName = nameParts[0];
  const lastName = nameParts.slice(1).join(' ');
  return [fullName, firstName, lastName];
}

async function getOrCreateAuthor(data) {
  const creatorKey = 'karstlink:hasDescriptionDocument/dct:creator';
  const attributionKey = 'dct:rights/cc:attributionName';

  const author =
    valIfTruthyOrNull(data[creatorKey]) ??
    valIfTruthyOrNull(data[attributionKey]);

  const [nickname, name, surname] = parsePersonName(author);
  const cavers = await TCaver.find({ nickname });
  if (cavers.length > 0) return cavers[0].id;

  const caver = await CaverService.createNonUserCaver({
    name,
    surname,
    nickname,
  });
  return caver.id;
}

async function getCreator(creator) {
  const creatorName = extractUrlFragment(creator).replace('_', ' ');
  const cavers = await TCaver.find({ nickname: creatorName });
  if (cavers.length > 0) return cavers[0];

  const [nickname, name, surname] = parsePersonName(creatorName);
  const caver = await CaverService.createNonUserCaver({
    name,
    surname,
    nickname,
  });

  return caver;
}

function checkColumns(data, additionalRequiredColumns = []) {
  let requiredColumns = [
    'id',
    'rdf:type',
    'dct:rights/cc:attributionName',
    'dct:rights/karstlink:licenseType',
    'gn:countryCode',
  ];
  const requiredDescriptionColumns = [
    'karstlink:hasDescriptionDocument/dct:title',
    'karstlink:hasDescriptionDocument/dct:creator',
    'karstlink:hasDescriptionDocument/dc:language',
  ];
  const requiredLocationColumns = [
    'karstlink:hasAccessDocument/dct:description',
    'karstlink:hasAccessDocument/dc:language',
    'karstlink:hasAccessDocument/dct:creator',
  ];

  if (additionalRequiredColumns)
    requiredColumns = requiredColumns.concat(additionalRequiredColumns);

  const missingColumns = [];

  for (const requiredColumn of requiredColumns) {
    if (!valIfTruthyOrNull(data[requiredColumn]))
      missingColumns.push(requiredColumn);
  }

  if (valIfTruthyOrNull(data[requiredDescriptionColumns[0]])) {
    for (const requiredDescColumn of requiredDescriptionColumns) {
      if (!valIfTruthyOrNull(data[requiredDescColumn]))
        missingColumns.push(requiredDescColumn);
    }
  }

  if (valIfTruthyOrNull(data[requiredLocationColumns[0]])) {
    for (const requiredLocColumn of requiredLocationColumns) {
      if (!valIfTruthyOrNull(data[requiredLocColumn]))
        missingColumns.push(requiredLocColumn);
    }
  }

  return missingColumns;
}

// See https://ontology.uis-speleo.org/howto/ for more informations
const KARSTLINK_DATE_FORMAT = 'YYYY-MM-DD';
function getDateFromKarstlink(value) {
  return moment(value, KARSTLINK_DATE_FORMAT);
}

const MAX_DOWNLOADED_FILE_SIZE_MO = 200;
async function distantFileDownload({ url, allowedExtentions = [] } = {}) {
  return new Promise((resolve, reject) => {
    let fileUrl;
    try {
      fileUrl = new URL(url);
    } catch (_) {
      reject(new Error('Invalid URL'));
      return;
    }

    if (!['http:', 'https:'].includes(fileUrl.protocol)) {
      reject(new Error('Invalid protocol'));
      return;
    }

    const client = fileUrl.protocol === 'https:' ? https : http;
    // eslint-disable-next-line consistent-return
    client
      .get(fileUrl, (res) => {
        // Warn: redirect are not followed !
        if (res.statusCode < 200 || res.statusCode >= 300) {
          res.socket.end();
          reject(new Error('Invalid response status'));
          return;
        }

        const contentLenth = res.headers['content-length'];
        if (contentLenth > MAX_DOWNLOADED_FILE_SIZE_MO * 1024) {
          res.socket.end();
          reject(new Error('Invalid file size'));
          return;
        }

        const contentType = res.headers['content-type'];
        const extension = mime.extension(contentType);
        if (!allowedExtentions.includes(extension)) {
          res.socket.end();
          reject(new Error('Invalid file extention'));
          return;
        }

        const data = [];
        res
          .on('data', (chunk) => data.push(chunk))
          .on('end', () => {
            const fileBuffer = Buffer.concat(data);
            return resolve({
              buffer: fileBuffer,
              mimetype: contentType,
              originalname: decodeURI(fileUrl.href.split('/').pop()),
              size: Buffer.byteLength(fileBuffer),
            });
          });
      })
      .on('error', () => {
        reject(new Error('Download error'));
      });
  });
}

module.exports = {
  extractUrlFragment,
  valIfTruthyOrNull,
  parsePersonName,
  getOrCreateAuthor,
  getCreator,
  checkColumns,

  KARSTLINK_DATE_FORMAT,
  getDateFromKarstlink,

  MAX_DOWNLOADED_FILE_SIZE_MO,
  distantFileDownload,
};

1	const http = require('http');	7✔
2	const https = require('https');	7✔
3	const mime = require('mime-types');	7✔
4	const moment = require('moment');	7✔
5	const CaverService = require('../services/CaverService');	7✔
6
7	// Usage example: https://ontology.uis-speleo.org/example/#Gouffre_Jean_Bernard
8	// Extract: Gouffre_Jean_Bernard
9	function extractUrlFragment(urlRaw) {
10	const url = urlRaw.trim();	44✔
11	if (url.startsWith('http')) return url.split('#')[1];	44!
12	return url;	44✔
13	}
14
15	// https://developer.mozilla.org/en-US/docs/Glossary/Truthy
16	function valIfTruthyOrNull(val) {
17	return val \|\| null;	376✔
18	}
19
20	function parsePersonName(fullName) {
21	const nameParts = fullName.split(' ');	7✔
22	if (nameParts.length > 3) return [fullName, undefined, undefined];	7!
23
24	const firstName = nameParts[0];	7✔
25	const lastName = nameParts.slice(1).join(' ');	7✔
26	return [fullName, firstName, lastName];	7✔
27	}
28
29	async function getOrCreateAuthor(data) {
30	const creatorKey = 'karstlink:hasDescriptionDocument/dct:creator';	4✔
31	const attributionKey = 'dct:rights/cc:attributionName';	4✔
32
33	const author =
34	valIfTruthyOrNull(data[creatorKey]) ??	4✔
35	valIfTruthyOrNull(data[attributionKey]);
36
37	const [nickname, name, surname] = parsePersonName(author);	4✔
38	const cavers = await TCaver.find({ nickname });	4✔
39	if (cavers.length > 0) return cavers[0].id;	4!
40
41	const caver = await CaverService.createNonUserCaver({	4✔
42	name,
43	surname,
44	nickname,
45	});
46	return caver.id;	4✔
47	}
48
49	async function getCreator(creator) {
50	const creatorName = extractUrlFragment(creator).replace('_', ' ');	14✔
51	const cavers = await TCaver.find({ nickname: creatorName });	14✔
52	if (cavers.length > 0) return cavers[0];	14✔
53
54	const [nickname, name, surname] = parsePersonName(creatorName);	3✔
55	const caver = await CaverService.createNonUserCaver({	3✔
56	name,
57	surname,
58	nickname,
59	});
60
61	return caver;	3✔
62	}
63
64	function checkColumns(data, additionalRequiredColumns = []) {	3✔
65	let requiredColumns = [	7✔
66	'id',
67	'rdf:type',
68	'dct:rights/cc:attributionName',
69	'dct:rights/karstlink:licenseType',
70	'gn:countryCode',
71	];
72	const requiredDescriptionColumns = [	7✔
73	'karstlink:hasDescriptionDocument/dct:title',
74	'karstlink:hasDescriptionDocument/dct:creator',
75	'karstlink:hasDescriptionDocument/dc:language',
76	];
77	const requiredLocationColumns = [	7✔
78	'karstlink:hasAccessDocument/dct:description',
79	'karstlink:hasAccessDocument/dc:language',
80	'karstlink:hasAccessDocument/dct:creator',
81	];
82
83	if (additionalRequiredColumns)	7!
84	requiredColumns = requiredColumns.concat(additionalRequiredColumns);	7✔
85
86	const missingColumns = [];	7✔
87
88	for (const requiredColumn of requiredColumns) {	7✔
89	if (!valIfTruthyOrNull(data[requiredColumn]))	47✔
90	missingColumns.push(requiredColumn);	12✔
91	}
92
93	if (valIfTruthyOrNull(data[requiredDescriptionColumns[0]])) {	7!
94	for (const requiredDescColumn of requiredDescriptionColumns) {	×
95	if (!valIfTruthyOrNull(data[requiredDescColumn]))	×
96	missingColumns.push(requiredDescColumn);	×
97	}
98	}
99
100	if (valIfTruthyOrNull(data[requiredLocationColumns[0]])) {	7!
101	for (const requiredLocColumn of requiredLocationColumns) {	×
102	if (!valIfTruthyOrNull(data[requiredLocColumn]))	×
103	missingColumns.push(requiredLocColumn);	×
104	}
105	}
106
107	return missingColumns;	7✔
108	}
109
110	// See https://ontology.uis-speleo.org/howto/ for more informations
111	const KARSTLINK_DATE_FORMAT = 'YYYY-MM-DD';	7✔
112	function getDateFromKarstlink(value) {
113	return moment(value, KARSTLINK_DATE_FORMAT);	36✔
114	}
115
116	const MAX_DOWNLOADED_FILE_SIZE_MO = 200;	7✔
117	async function distantFileDownload({ url, allowedExtentions = [] } = {}) {	×
118	return new Promise((resolve, reject) => {	6✔
119	let fileUrl;
120	try {	6✔
121	fileUrl = new URL(url);	6✔
122	} catch (_) {
123	reject(new Error('Invalid URL'));	1✔
124	return;	1✔
125	}
126
127	if (!['http:', 'https:'].includes(fileUrl.protocol)) {	5✔
128	reject(new Error('Invalid protocol'));	1✔
129	return;	1✔
130	}
131
132	const client = fileUrl.protocol === 'https:' ? https : http;	4✔
133	// eslint-disable-next-line consistent-return
134	client	4✔
135	.get(fileUrl, (res) => {
136	// Warn: redirect are not followed !
137	if (res.statusCode < 200 \|\| res.statusCode >= 300) {	4✔
138	res.socket.end();	1✔
139	reject(new Error('Invalid response status'));	1✔
140	return;	1✔
141	}
142
143	const contentLenth = res.headers['content-length'];	3✔
144	if (contentLenth > MAX_DOWNLOADED_FILE_SIZE_MO * 1024) {	3!
UNCOV 145	res.socket.end();	×
146	reject(new Error('Invalid file size'));	×
147	return;	×
148	}
149
150	const contentType = res.headers['content-type'];	3✔
151	const extension = mime.extension(contentType);	3✔
152	if (!allowedExtentions.includes(extension)) {	3✔
153	res.socket.end();	2✔
154	reject(new Error('Invalid file extention'));	2✔
155	return;	2✔
156	}
157
158	const data = [];	1✔
159	res	1✔
160	.on('data', (chunk) => data.push(chunk))	1✔
161	.on('end', () => {
162	const fileBuffer = Buffer.concat(data);	1✔
163	return resolve({	1✔
164	buffer: fileBuffer,
165	mimetype: contentType,
166	originalname: decodeURI(fileUrl.href.split('/').pop()),
167	size: Buffer.byteLength(fileBuffer),
168	});
169	});
170	})
171	.on('error', () => {
UNCOV 172	reject(new Error('Download error'));	×
173	});
174	});
175	}
176
177	module.exports = {	7✔
178	extractUrlFragment,
179	valIfTruthyOrNull,
180	parsePersonName,
181	getOrCreateAuthor,
182	getCreator,
183	checkColumns,
184
185	KARSTLINK_DATE_FORMAT,
186	getDateFromKarstlink,
187
188	MAX_DOWNLOADED_FILE_SIZE_MO,
189	distantFileDownload,
190	};

GrottoCenter / grottocenter-api / 27540052150

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous