4088643824

Build Type

github

Committed by GitHub

Commit Message

Merge 3588a38ba into c1acf7dc8

Pull Request Pull Request #951: build(deps): bump http-cache-semantics from 4.1.0 to 4.1.1 in /packages/cli

Run Details

436 of 530 branches covered (82.26%)

Branch coverage included in aggregate %.

692 of 736 relevant lines covered (94.02%)

4.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.95

/packages/core/src/dataset-pool/format/csv.ts

import * as DataCook from '@pipcook/datacook';
import * as Papaparse from 'papaparse';
import { ArrayDatasetPoolImpl, Types } from '..';
import Csv = DataCook.Dataset.Types.Csv;

export interface Options {
  trainData?: string;
  testData?: string;
  validData?: string;
  predictedData?: string;
  hasHeader: boolean;
  delimiter?: string;
  labels?: string[];
}

function toSamples(
  parsedData: Papaparse.ParseResult<Record<string, string>>,
  labelFields?: Array<string>
): Array<Csv.Sample> {
  return parsedData.data.map((data) => {
    const label: Record<string, string> = {};
    const newData = { ...data };
    labelFields?.forEach((field) => {
      label[field] = newData[field];
      delete newData[field];
    });
    return {
      data: newData,
      label
    };
  });
}

export const makeDatasetPoolFromCsv = (options: Options): Types.DatasetPool<Csv.Sample, Types.Csv.DatasetMeta> => {
  const config = {
    header: options.hasHeader, delimiter: options.delimiter
  };
  const parsedTrainData = options.trainData ? Papaparse.parse<Record<string, string>>(options.trainData, config) : undefined;
  const parsedTestData = options.testData ? Papaparse.parse<Record<string, string>>(options.testData, config) : undefined;
  const parsedValidData = options.validData ? Papaparse.parse<Record<string, string>>(options.validData, config) : undefined;
  const parsedPredictedData = options.predictedData ? Papaparse.parse<Record<string, string>>(options.predictedData, config) : undefined;
  const data = {
    trainData: parsedTrainData ? toSamples(parsedTrainData, options.labels) : undefined,
    testData: parsedTestData ? toSamples(parsedTestData, options.labels) : undefined,
    validData: parsedValidData ? toSamples(parsedValidData, options.labels) : undefined,
    predictedData: parsedPredictedData ? toSamples(parsedPredictedData, options.labels) : undefined
  };
  const meta: Types.Csv.DatasetMeta = {
    type: DataCook.Dataset.Types.DatasetType.Table,
    size: {
      train: data.trainData?.length || 0,
      test: data.testData?.length || 0,
      valid: data.validData?.length || 0,
      predicted: data.predictedData?.length || 0
    }
  };
  return ArrayDatasetPoolImpl.from(data, meta);
};

1	import * as DataCook from '@pipcook/datacook';	6✔
2	import * as Papaparse from 'papaparse';	6✔
3	import { ArrayDatasetPoolImpl, Types } from '..';	6✔
4	import Csv = DataCook.Dataset.Types.Csv;
5
6	export interface Options {
7	trainData?: string;
8	testData?: string;
9	validData?: string;
10	predictedData?: string;
11	hasHeader: boolean;
12	delimiter?: string;
13	labels?: string[];
14	}
15
16	function toSamples(
17	parsedData: Papaparse.ParseResult<Record<string, string>>,
18	labelFields?: Array<string>
19	): Array<Csv.Sample> {
20	return parsedData.data.map((data) => {	8✔
21	const label: Record<string, string> = {};	24✔
22	const newData = { ...data };	24✔
23	labelFields?.forEach((field) => {	24!
24	label[field] = newData[field];	24✔
25	delete newData[field];	24✔
26	});
27	return {	24✔
28	data: newData,
29	label
30	};
31	});
32	}
33
34	export const makeDatasetPoolFromCsv = (options: Options): Types.DatasetPool<Csv.Sample, Types.Csv.DatasetMeta> => {	6✔
35	const config = {	3✔
36	header: options.hasHeader, delimiter: options.delimiter
37	};
38	const parsedTrainData = options.trainData ? Papaparse.parse<Record<string, string>>(options.trainData, config) : undefined;	3!
39	const parsedTestData = options.testData ? Papaparse.parse<Record<string, string>>(options.testData, config) : undefined;	3!
40	const parsedValidData = options.validData ? Papaparse.parse<Record<string, string>>(options.validData, config) : undefined;	3✔
41	const parsedPredictedData = options.predictedData ? Papaparse.parse<Record<string, string>>(options.predictedData, config) : undefined;	3!
42	const data = {	3✔
43	trainData: parsedTrainData ? toSamples(parsedTrainData, options.labels) : undefined,	3!
44	testData: parsedTestData ? toSamples(parsedTestData, options.labels) : undefined,	3!
45	validData: parsedValidData ? toSamples(parsedValidData, options.labels) : undefined,	3✔
46	predictedData: parsedPredictedData ? toSamples(parsedPredictedData, options.labels) : undefined	3!
47	};
48	const meta: Types.Csv.DatasetMeta = {	3✔
49	type: DataCook.Dataset.Types.DatasetType.Table,
50	size: {
51	train: data.trainData?.length \|\| 0,	12!
52	test: data.testData?.length \|\| 0,	12!
53	valid: data.validData?.length \|\| 0,	13✔
54	predicted: data.predictedData?.length \|\| 0	15!
55	}
56	};
57	return ArrayDatasetPoolImpl.from(data, meta);	3✔
58	};

alibaba / pipcook / 4088643824

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous