#677

Committed 07 May 2025 08:08PM UTC coverage: 87.245% (+0.01%) from 87.231%

Build # #677

Build Type

push

Committed by

Commit Message

Implement SonarCloud quality suggestions #159: fix blockers

Run Details

17 of 24 new or added lines in 9 files covered. (70.83%)

2 existing lines in 1 file now uncovered.

5520 of 6327 relevant lines covered (87.25%)

0.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.24

/src/main/java/de/gwdg/metadataqa/api/cli/App.java

package de.gwdg.metadataqa.api.cli;

import com.jayway.jsonpath.InvalidJsonException;
import com.opencsv.exceptions.CsvValidationException;
import de.gwdg.metadataqa.api.calculator.CalculatorFacade;
import de.gwdg.metadataqa.api.io.reader.XMLRecordReader;
import de.gwdg.metadataqa.api.io.writer.ResultWriter;
import de.gwdg.metadataqa.api.configuration.ConfigurationReader;
import de.gwdg.metadataqa.api.configuration.MeasurementConfiguration;
import de.gwdg.metadataqa.api.interfaces.MetricResult;
import de.gwdg.metadataqa.api.io.reader.RecordReader;
import de.gwdg.metadataqa.api.schema.Schema;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.MissingOptionException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;

public class App {

  private static final Logger logger = Logger.getLogger(App.class.getCanonicalName());

  private static final String APP_NAME = "mqa";
  private static final String APP_HEADER = "Command-line application for the Metadata Quality API (https://github.com/pkiraly/metadata-qa-api). Read line-based metadata records and output quality assessment results using various metrics.";

  // constants
  public static final String NDJSON = "ndjson";
  public static final String CSVJSON = "csvjson";
  public static final String CSV = "csv";
  public static final String JSON = "json";
  public static final String YAML = "yaml";

  // Arguments
  private static final String INPUT_FILE = "input";
  private static final String INPUT_FORMAT = "inputFormat";
  private static final String OUTPUT_FILE = "output";
  private static final String OUTPUT_FORMAT = "outputFormat";
  private static final String SCHEMA_CONFIG = "schema";
  private static final String SCHEMA_FORMAT = "schemaFormat";
  private static final String MEASUREMENTS_CONFIG = "measurements";
  private static final String HEADERS_CONFIG = "headers";
  private static final String MEASUREMENTS_FORMAT = "measurementsFormat";
  private static final String GZIP_FLAG = "gzip";
  private static final String RECORD_ADDRESS = "recordAddress";

  private final Schema schema;
  private final CalculatorFacade calculator;
  private final ResultWriter outputWriter;
  private final RecordReader inputReader;
  private final String recordAddress;

  public App(CommandLine cmd) throws IOException, CsvValidationException {
    // initialize schema
    String schemaFile = cmd.getOptionValue(SCHEMA_CONFIG);
    String schemaFormat = cmd.getOptionValue(SCHEMA_FORMAT, FilenameUtils.getExtension(schemaFile));
    switch (schemaFormat) {
      case YAML:
        this.schema = ConfigurationReader.readSchemaYaml(schemaFile).asSchema();
        break;
      case JSON:
      default:
        this.schema = ConfigurationReader.readSchemaJson(schemaFile).asSchema();
    }

    // Set the fields supplied by the command line to extractable fields
    if (cmd.hasOption(HEADERS_CONFIG)) {
      String[] headers = cmd.getOptionValues(HEADERS_CONFIG);
      for (String h : headers) {
        this.schema.addExtractableField(h, this.schema.getPathByLabel(h).getPath());
      }
    }

    // initialize config
    MeasurementConfiguration measurementConfig = new MeasurementConfiguration();
    if (cmd.hasOption(MEASUREMENTS_CONFIG)) {
      String measurementFile = cmd.getOptionValue(MEASUREMENTS_CONFIG);
      String measurementFormat = cmd.getOptionValue(MEASUREMENTS_FORMAT, FilenameUtils.getExtension(measurementFile));
      switch (measurementFormat) {
        case YAML:
          measurementConfig = ConfigurationReader.readMeasurementYaml(measurementFile);
          break;
        case JSON:
        default:
          measurementConfig = ConfigurationReader.readMeasurementJson(measurementFile);
      }
    }

    // initialize calculator
    this.calculator = new CalculatorFacade(measurementConfig);
    // set the schema which describes the source
    calculator.setSchema(schema);

    // initialize input
    String inputFile = cmd.getOptionValue(INPUT_FILE);
    InputFormat inputFormat = InputFormat.byCode(cmd.getOptionValue(INPUT_FORMAT));
    this.inputReader = RecordFactory.getRecordReader(inputFile, calculator, cmd.hasOption(GZIP_FLAG), inputFormat);

    // initialize output
    String outFormat = cmd.getOptionValue(OUTPUT_FORMAT, NDJSON);
    // write to std out if no file was given
    this.outputWriter = cmd.hasOption(OUTPUT_FILE)
      ? RecordFactory.getResultWriter(outFormat, cmd.getOptionValue(OUTPUT_FILE))
      : RecordFactory.getResultWriter(outFormat);

    this.recordAddress = (cmd.hasOption(RECORD_ADDRESS) && StringUtils.isNotBlank(cmd.getOptionValue(RECORD_ADDRESS)))
      ? cmd.getOptionValue(RECORD_ADDRESS)
      : null;
    if (inputReader instanceof XMLRecordReader && recordAddress != null)
      ((XMLRecordReader)inputReader).setRecordAddress(this.recordAddress);
  }

  public static void main(String[] args) {

    // Take input file
    final Options options = buildOptions();

    // create the parser
    CommandLineParser parser = new DefaultParser();

    // create the formatter
    HelpFormatter formatter = new HelpFormatter();

    try {
      // parse the command line arguments
      CommandLine cmd = parser.parse(options, args);
      new App(cmd).run();
    } catch (MissingOptionException ex) {
      List<String> missingOptions = new ArrayList<>();
      for (String option : (List<String>) ex.getMissingOptions()) {
        missingOptions.add(String.format("--%s (-%s)", options.getOption(option).getLongOpt(), option));
      }
      formatter.printHelp(APP_NAME, APP_HEADER, options, "ERROR\nMissing options: \n  " + StringUtils.join(missingOptions, "\n  "), true);
      System.exit(1);
    } catch (MissingArgumentException ex) {
      Option missingOption = ex.getOption();
      String err = String.format("%s for option --%s (-%s)", missingOption.getArgName(), missingOption.getLongOpt(), missingOption.getOpt());
      formatter.printHelp(APP_NAME, APP_HEADER, options, "ERROR\nMissing arguments: " + err, true);
      System.exit(1);
    } catch (Exception ex) {
      formatter.printHelp(APP_NAME, APP_HEADER, options, "Error: " + ex.getMessage(), true);
      ex.printStackTrace();
      System.exit(1);
    }
  }

  private static Options buildOptions() {
    final Options options = new Options();

    Option inputOption = Option.builder("i")
      .numberOfArgs(1)
      .argName("file")
      .required(true)
      .longOpt(INPUT_FILE)
      .desc("Input file.")
      .build();

    Option inputFormatOption = Option.builder("n")
      .numberOfArgs(1)
      .argName("inputFormat")
      .required(false)
      .longOpt(INPUT_FORMAT)
      .desc("Format of the input: json, ndjson (new line delimited JSON), json-array (JSON file that contains an array of objects). Default: ndjson.")
      .build();

    Option outputOption = Option.builder("o")
      .numberOfArgs(1)
      .argName("file")
      .required(false)
      .longOpt(OUTPUT_FILE)
      .desc("Output file.")
      .build();

    Option outputFormatOption = Option.builder("f")
      .numberOfArgs(1)
      .argName("format")
      .required(false)
      .longOpt(OUTPUT_FORMAT)
      .desc("Format of the output: json, ndjson (new line delimited JSON), csv, csvjson (json encoded in csv; useful for RDB bulk loading). Default: ndjson.")
      .build();

    Option schemaConfigOption = Option.builder("s")
      .numberOfArgs(1)
      .argName("file")
      .required(true)
      .longOpt(SCHEMA_CONFIG)
      .desc("Schema file to run assessment against.")
      .build();

    Option schemaFormatOption = Option.builder("v")
      .numberOfArgs(1)
      .argName("format")
      .required(false)
      .longOpt(SCHEMA_FORMAT)
      .desc("Format of schema file: json, yaml. Default: based on file extension, else json.")
      .build();

    Option measurementsConfigOption = Option.builder("m")
      .numberOfArgs(1)
      .argName("file")
      .required(true)
      .longOpt(MEASUREMENTS_CONFIG)
      .desc("Configuration file for measurements.")
      .build();

    Option measurementsFormatOption = Option.builder("w")
      .numberOfArgs(1)
      .argName("format")
      .required(false)
      .longOpt(MEASUREMENTS_FORMAT)
      .desc("Format of measurements config file: json, yaml. Default: based on file extension, else json.")
      .build();

    Option headersOption = Option.builder("h")
      .hasArgs()
      .required(false)
      .longOpt(HEADERS_CONFIG)
      .desc("Headers to copy from source")
      .build();

    Option gzipOption = Option.builder("z")
      .numberOfArgs(0)
      .required(false)
      .longOpt(GZIP_FLAG)
      .desc("Flag to indicate that input is gzipped.")
      .build();

    Option recordAddressOption = Option.builder("r")
      .numberOfArgs(1)
      .argName("path")
      .required(false)
      .longOpt(RECORD_ADDRESS)
      .desc("An XPath or JSONPath expression to separate individual records in an XML or JSON files.")
      .build();

    options.addOption(inputOption);
    options.addOption(inputFormatOption);
    options.addOption(outputOption);
    options.addOption(outputFormatOption);
    options.addOption(schemaConfigOption);
    options.addOption(schemaFormatOption);
    options.addOption(measurementsConfigOption);
    options.addOption(measurementsFormatOption);
    options.addOption(headersOption);
    options.addOption(gzipOption);
    options.addOption(recordAddressOption);
    return options;
  }

  private void run() {
    long counter = 0;
    try {
      // print header
      List<String> header = calculator.getHeader();
      outputWriter.writeHeader(header);

      while (inputReader.hasNext()) {
        Map<String, List<MetricResult>> measurement = inputReader.next();
        outputWriter.writeResult(measurement);

        // update process
        counter++;
        if (counter % 50 == 0) {
          logger.info(String.format("Processed %s records. ", counter));
        }
      }
      logger.info(String.format("Assessment completed successfully with %s records. ", counter));
      outputWriter.close();
    } catch (InvalidJsonException | IOException e) {
      logger.severe(String.format("Assessment failed with %s records. ", counter));
      logger.severe(e.getMessage());
      e.printStackTrace();
    }
  }
}

1	package de.gwdg.metadataqa.api.cli;
2
3	import com.jayway.jsonpath.InvalidJsonException;
4	import com.opencsv.exceptions.CsvValidationException;
5	import de.gwdg.metadataqa.api.calculator.CalculatorFacade;
6	import de.gwdg.metadataqa.api.io.reader.XMLRecordReader;
7	import de.gwdg.metadataqa.api.io.writer.ResultWriter;
8	import de.gwdg.metadataqa.api.configuration.ConfigurationReader;
9	import de.gwdg.metadataqa.api.configuration.MeasurementConfiguration;
10	import de.gwdg.metadataqa.api.interfaces.MetricResult;
11	import de.gwdg.metadataqa.api.io.reader.RecordReader;
12	import de.gwdg.metadataqa.api.schema.Schema;
13	import org.apache.commons.cli.CommandLine;
14	import org.apache.commons.cli.CommandLineParser;
15	import org.apache.commons.cli.DefaultParser;
16	import org.apache.commons.cli.HelpFormatter;
17	import org.apache.commons.cli.MissingArgumentException;
18	import org.apache.commons.cli.MissingOptionException;
19	import org.apache.commons.cli.Option;
20	import org.apache.commons.cli.Options;
21	import org.apache.commons.io.FilenameUtils;
22	import org.apache.commons.lang3.StringUtils;
23
24	import java.io.IOException;
25	import java.util.ArrayList;
26	import java.util.List;
27	import java.util.Map;
28	import java.util.logging.Logger;
29
30	public class App {
31
32	private static final Logger logger = Logger.getLogger(App.class.getCanonicalName());	1✔
33
34	private static final String APP_NAME = "mqa";
35	private static final String APP_HEADER = "Command-line application for the Metadata Quality API (https://github.com/pkiraly/metadata-qa-api). Read line-based metadata records and output quality assessment results using various metrics.";
36
37	// constants
38	public static final String NDJSON = "ndjson";
39	public static final String CSVJSON = "csvjson";
40	public static final String CSV = "csv";
41	public static final String JSON = "json";
42	public static final String YAML = "yaml";
43
44	// Arguments
45	private static final String INPUT_FILE = "input";
46	private static final String INPUT_FORMAT = "inputFormat";
47	private static final String OUTPUT_FILE = "output";
48	private static final String OUTPUT_FORMAT = "outputFormat";
49	private static final String SCHEMA_CONFIG = "schema";
50	private static final String SCHEMA_FORMAT = "schemaFormat";
51	private static final String MEASUREMENTS_CONFIG = "measurements";
52	private static final String HEADERS_CONFIG = "headers";
53	private static final String MEASUREMENTS_FORMAT = "measurementsFormat";
54	private static final String GZIP_FLAG = "gzip";
55	private static final String RECORD_ADDRESS = "recordAddress";
56
57	private final Schema schema;
58	private final CalculatorFacade calculator;
59	private final ResultWriter outputWriter;
60	private final RecordReader inputReader;
61	private final String recordAddress;
62
63	public App(CommandLine cmd) throws IOException, CsvValidationException {	1✔
64	// initialize schema
65	String schemaFile = cmd.getOptionValue(SCHEMA_CONFIG);	1✔
66	String schemaFormat = cmd.getOptionValue(SCHEMA_FORMAT, FilenameUtils.getExtension(schemaFile));	1✔
67	switch (schemaFormat) {	1✔
68	case YAML:
69	this.schema = ConfigurationReader.readSchemaYaml(schemaFile).asSchema();	1✔
70	break;	1✔
71	case JSON:
72	default:
73	this.schema = ConfigurationReader.readSchemaJson(schemaFile).asSchema();	1✔
74	}
75
76	// Set the fields supplied by the command line to extractable fields
77	if (cmd.hasOption(HEADERS_CONFIG)) {	1✔
78	String[] headers = cmd.getOptionValues(HEADERS_CONFIG);	×
79	for (String h : headers) {	×
80	this.schema.addExtractableField(h, this.schema.getPathByLabel(h).getPath());	×
81	}
82	}
83
84	// initialize config
85	MeasurementConfiguration measurementConfig = new MeasurementConfiguration();	1✔
86	if (cmd.hasOption(MEASUREMENTS_CONFIG)) {	1✔
87	String measurementFile = cmd.getOptionValue(MEASUREMENTS_CONFIG);	1✔
88	String measurementFormat = cmd.getOptionValue(MEASUREMENTS_FORMAT, FilenameUtils.getExtension(measurementFile));	1✔
89	switch (measurementFormat) {	1✔
90	case YAML:
91	measurementConfig = ConfigurationReader.readMeasurementYaml(measurementFile);	1✔
92	break;	1✔
93	case JSON:
94	default:
95	measurementConfig = ConfigurationReader.readMeasurementJson(measurementFile);	1✔
96	}
97	}
98
99	// initialize calculator
100	this.calculator = new CalculatorFacade(measurementConfig);	1✔
101	// set the schema which describes the source
102	calculator.setSchema(schema);	1✔
103
104	// initialize input
105	String inputFile = cmd.getOptionValue(INPUT_FILE);	1✔
106	InputFormat inputFormat = InputFormat.byCode(cmd.getOptionValue(INPUT_FORMAT));	1✔
107	this.inputReader = RecordFactory.getRecordReader(inputFile, calculator, cmd.hasOption(GZIP_FLAG), inputFormat);	1✔
108
109	// initialize output
110	String outFormat = cmd.getOptionValue(OUTPUT_FORMAT, NDJSON);	1✔
111	// write to std out if no file was given
112	this.outputWriter = cmd.hasOption(OUTPUT_FILE)	1✔
113	? RecordFactory.getResultWriter(outFormat, cmd.getOptionValue(OUTPUT_FILE))	1✔
114	: RecordFactory.getResultWriter(outFormat);	1✔
115
116	this.recordAddress = (cmd.hasOption(RECORD_ADDRESS) && StringUtils.isNotBlank(cmd.getOptionValue(RECORD_ADDRESS)))	1✔
117	? cmd.getOptionValue(RECORD_ADDRESS)	×
118	: null;	1✔
119	if (inputReader instanceof XMLRecordReader && recordAddress != null)	1✔
120	((XMLRecordReader)inputReader).setRecordAddress(this.recordAddress);	×
121	}	1✔
122
123	public static void main(String[] args) {
124
125	// Take input file
126	final Options options = buildOptions();	1✔
127
128	// create the parser
129	CommandLineParser parser = new DefaultParser();	1✔
130
131	// create the formatter
132	HelpFormatter formatter = new HelpFormatter();	1✔
133
134	try {
135	// parse the command line arguments
136	CommandLine cmd = parser.parse(options, args);	1✔
137	new App(cmd).run();	1✔
138	} catch (MissingOptionException ex) {	1✔
139	List<String> missingOptions = new ArrayList<>();	1✔
140	for (String option : (List<String>) ex.getMissingOptions()) {	1✔
141	missingOptions.add(String.format("--%s (-%s)", options.getOption(option).getLongOpt(), option));	1✔
142	}	1✔
143	formatter.printHelp(APP_NAME, APP_HEADER, options, "ERROR\nMissing options: \n " + StringUtils.join(missingOptions, "\n "), true);	1✔
144	System.exit(1);	×
145	} catch (MissingArgumentException ex) {	1✔
146	Option missingOption = ex.getOption();	1✔
147	String err = String.format("%s for option --%s (-%s)", missingOption.getArgName(), missingOption.getLongOpt(), missingOption.getOpt());	1✔
148	formatter.printHelp(APP_NAME, APP_HEADER, options, "ERROR\nMissing arguments: " + err, true);	1✔
149	System.exit(1);	×
150	} catch (Exception ex) {	×
NEW 151	formatter.printHelp(APP_NAME, APP_HEADER, options, "Error: " + ex.getMessage(), true);	×
152	ex.printStackTrace();	×
153	System.exit(1);	×
154	}	1✔
155	}	1✔
156
157	private static Options buildOptions() {
158	final Options options = new Options();	1✔
159
160	Option inputOption = Option.builder("i")	1✔
161	.numberOfArgs(1)	1✔
162	.argName("file")	1✔
163	.required(true)	1✔
164	.longOpt(INPUT_FILE)	1✔
165	.desc("Input file.")	1✔
166	.build();	1✔
167
168	Option inputFormatOption = Option.builder("n")	1✔
169	.numberOfArgs(1)	1✔
170	.argName("inputFormat")	1✔
171	.required(false)	1✔
172	.longOpt(INPUT_FORMAT)	1✔
173	.desc("Format of the input: json, ndjson (new line delimited JSON), json-array (JSON file that contains an array of objects). Default: ndjson.")	1✔
174	.build();	1✔
175
176	Option outputOption = Option.builder("o")	1✔
177	.numberOfArgs(1)	1✔
178	.argName("file")	1✔
179	.required(false)	1✔
180	.longOpt(OUTPUT_FILE)	1✔
181	.desc("Output file.")	1✔
182	.build();	1✔
183
184	Option outputFormatOption = Option.builder("f")	1✔
185	.numberOfArgs(1)	1✔
186	.argName("format")	1✔
187	.required(false)	1✔
188	.longOpt(OUTPUT_FORMAT)	1✔
189	.desc("Format of the output: json, ndjson (new line delimited JSON), csv, csvjson (json encoded in csv; useful for RDB bulk loading). Default: ndjson.")	1✔
190	.build();	1✔
191
192	Option schemaConfigOption = Option.builder("s")	1✔
193	.numberOfArgs(1)	1✔
194	.argName("file")	1✔
195	.required(true)	1✔
196	.longOpt(SCHEMA_CONFIG)	1✔
197	.desc("Schema file to run assessment against.")	1✔
198	.build();	1✔
199
200	Option schemaFormatOption = Option.builder("v")	1✔
201	.numberOfArgs(1)	1✔
202	.argName("format")	1✔
203	.required(false)	1✔
204	.longOpt(SCHEMA_FORMAT)	1✔
205	.desc("Format of schema file: json, yaml. Default: based on file extension, else json.")	1✔
206	.build();	1✔
207
208	Option measurementsConfigOption = Option.builder("m")	1✔
209	.numberOfArgs(1)	1✔
210	.argName("file")	1✔
211	.required(true)	1✔
212	.longOpt(MEASUREMENTS_CONFIG)	1✔
213	.desc("Configuration file for measurements.")	1✔
214	.build();	1✔
215
216	Option measurementsFormatOption = Option.builder("w")	1✔
217	.numberOfArgs(1)	1✔
218	.argName("format")	1✔
219	.required(false)	1✔
220	.longOpt(MEASUREMENTS_FORMAT)	1✔
221	.desc("Format of measurements config file: json, yaml. Default: based on file extension, else json.")	1✔
222	.build();	1✔
223
224	Option headersOption = Option.builder("h")	1✔
225	.hasArgs()	1✔
226	.required(false)	1✔
227	.longOpt(HEADERS_CONFIG)	1✔
228	.desc("Headers to copy from source")	1✔
229	.build();	1✔
230
231	Option gzipOption = Option.builder("z")	1✔
232	.numberOfArgs(0)	1✔
233	.required(false)	1✔
234	.longOpt(GZIP_FLAG)	1✔
235	.desc("Flag to indicate that input is gzipped.")	1✔
236	.build();	1✔
237
238	Option recordAddressOption = Option.builder("r")	1✔
239	.numberOfArgs(1)	1✔
240	.argName("path")	1✔
241	.required(false)	1✔
242	.longOpt(RECORD_ADDRESS)	1✔
243	.desc("An XPath or JSONPath expression to separate individual records in an XML or JSON files.")	1✔
244	.build();	1✔
245
246	options.addOption(inputOption);	1✔
247	options.addOption(inputFormatOption);	1✔
248	options.addOption(outputOption);	1✔
249	options.addOption(outputFormatOption);	1✔
250	options.addOption(schemaConfigOption);	1✔
251	options.addOption(schemaFormatOption);	1✔
252	options.addOption(measurementsConfigOption);	1✔
253	options.addOption(measurementsFormatOption);	1✔
254	options.addOption(headersOption);	1✔
255	options.addOption(gzipOption);	1✔
256	options.addOption(recordAddressOption);	1✔
257	return options;	1✔
258	}
259
260	private void run() {
261	long counter = 0;	1✔
262	try {
263	// print header
264	List<String> header = calculator.getHeader();	1✔
265	outputWriter.writeHeader(header);	1✔
266
267	while (inputReader.hasNext()) {	1✔
268	Map<String, List<MetricResult>> measurement = inputReader.next();	1✔
269	outputWriter.writeResult(measurement);	1✔
270
271	// update process
272	counter++;	1✔
273	if (counter % 50 == 0) {	1✔
274	logger.info(String.format("Processed %s records. ", counter));	×
275	}
276	}	1✔
277	logger.info(String.format("Assessment completed successfully with %s records. ", counter));	1✔
278	outputWriter.close();	1✔
279	} catch (InvalidJsonException \| IOException e) {	×
280	logger.severe(String.format("Assessment failed with %s records. ", counter));	×
281	logger.severe(e.getMessage());	×
282	e.printStackTrace();	×
283	}	1✔
284	}	1✔
285	}

pkiraly / metadata-qa-api / #677

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous