#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.75

/src/main/java/de/gwdg/metadataqa/marc/analysis/validator/DataFieldValidator.java

package de.gwdg.metadataqa.marc.analysis.validator;

import de.gwdg.metadataqa.marc.MarcSubfield;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.definition.Cardinality;
import de.gwdg.metadataqa.marc.definition.MarcVersion;
import de.gwdg.metadataqa.marc.definition.TagDefinitionLoader;
import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
import de.gwdg.metadataqa.marc.definition.structure.Indicator;
import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
import de.gwdg.metadataqa.marc.model.validation.ErrorsCollector;
import de.gwdg.metadataqa.marc.model.validation.ValidationErrorType;
import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_INVALID_VALUE;
import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_NON_EMPTY;
import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_OBSOLETE;
import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.SUBFIELD_NONREPEATABLE;
import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.SUBFIELD_UNDEFINED;

public class DataFieldValidator extends AbstractValidator {

  private ErrorsCollector errors;
  private DataField field;
  private DataFieldDefinition definition;

  public DataFieldValidator() {
    super(new ValidatorConfiguration());
  }

  public DataFieldValidator(ValidatorConfiguration configuration) {
    super(configuration);
  }

  public boolean validate(DataField field) {
    this.field = field;
    definition = field.getDefinition();
    validationErrors = new ArrayList<>();
    errors = new ErrorsCollector();

    // Check if the tag is defined only in case it is a MARC21 tag
    // I'm not sure how this is different to validating unhandled tags. However, it will be kept until further notice.
    boolean isMarc21TagDefined = isMarc21TagDefined();
    if (!isMarc21TagDefined) {
      addError(ValidationErrorType.FIELD_UNDEFINED, "");
      return false;
    }

    // From here on, we know the tag is either MARC21 and defined, or not MARC21
    // This LinkageHandler can later be further abstracted into an abstract LinkageHandler for UNIMARC in case that's needed
    Marc21LinkageHandler linkageHandler = new Marc21LinkageHandler(configuration, errors);
    DataField associatedField = linkageHandler.handleLinkage(field, definition);
    errors.addAll(linkageHandler.getErrors());

    // In case the associated field is not null, and due to compatibility with the rest of this class,
    // assign the referencer definition and the subfields of the associated field to the local variables
    // This assignment makes sure that the methods which use the attribute definition, use the definition
    // of the referencer field if needed
    DataFieldDefinition referencerDefinition = null;
    List<MarcSubfield> subfields = field.getSubfields();
    if (associatedField != null) {
      referencerDefinition = definition;
      definition = associatedField.getDefinition();
      subfields = associatedField.getSubfields();
    }

    validateIndicators(referencerDefinition);

    // This seems to never be executed because the addUnhandledSubfield method is never called
    if (field.getUnhandledSubfields() != null) {
      addError(SUBFIELD_UNDEFINED, StringUtils.join(field.getUnhandledSubfields(), ", "));
    }

    boolean isAmbiguousLinkage = linkageHandler.isAmbiguousLinkage();

    if (!isAmbiguousLinkage) {
      validateSubfields(subfields);
    }

    if (associatedField != null) {
      definition = referencerDefinition;
    }

    validationErrors.addAll(errors.getErrors());
    return errors.isEmpty();
  }

  private void validateIndicators(DataFieldDefinition referencerDefinition) {
    if (configuration.getSchemaType().equals(SchemaType.PICA)) {
      return;
    }

    if (field.getInd1() != null) {
      validateIndicator(definition.getInd1(), field.getInd1(), configuration.getMarcVersion(), referencerDefinition);
    }
    if (field.getInd2() != null) {
      validateIndicator(definition.getInd2(), field.getInd2(), configuration.getMarcVersion(), referencerDefinition);
    }
  }

  private boolean validateIndicator(Indicator indicatorDefinition,
                                    String value,
                                    MarcVersion marcVersion,
                                    DataFieldDefinition referencerDefinition) {
    String path = indicatorDefinition.getPath();

    if (referencerDefinition != null) {
      path = String.format("%s->%s", referencerDefinition.getTag(), path);
    }

    boolean isVersionSpecific = indicatorDefinition.isVersionSpecificCode(marcVersion, value);
    boolean indicatorExists = indicatorDefinition.exists();

    // The first invalid case is:
    // Indicator undefined but the value is not empty (and also it doesn't depend on the marc version used)
    if (!indicatorExists && !value.equals(" ") && !isVersionSpecific) {
        addError(path, INDICATOR_NON_EMPTY, value);
        return false;
    }

    // The second invalid case is when the indicator is defined, but the value isn't a valid code (and also it
    // doesn't depend on the marc version used).
    // In case the code used is some old (historical) code, then that's an obsolete code error, and if it is
    // just plainly a wrong code, then it is an invalid value error.
    if (indicatorExists && !indicatorDefinition.hasCode(value) && !isVersionSpecific) {
      // Maybe if the value is a space, it should be clarified in the error message a little differently
      if (indicatorDefinition.isHistoricalCode(value)) {
        addError(path, INDICATOR_OBSOLETE, value);
      } else {
        addError(path, INDICATOR_INVALID_VALUE, value);
      }
      return false;
    }

    // All other cases are valid: either the indicator is defined and the value is a valid code, or the indicator
    // is undefined and the value is empty (or it is a version specific code).
    return true;
  }

  private void validateSubfields(List<MarcSubfield> subfields) {
    SubfieldValidator subfieldValidator = new SubfieldValidator(configuration);

    for (MarcSubfield subfield : subfields) {
      addVersionSpecificDefinition(subfield);
      boolean isSubfieldValid = subfieldValidator.validate(subfield);

      if (!isSubfieldValid) {
        errors.addAll(subfieldValidator.getValidationErrors());
      }
    }

    // Count the number of subfield definitions
    Map<SubfieldDefinition, Long> counter = subfields.stream()
        .filter(subfield -> subfield.getDefinition() != null)
        .collect(Collectors.groupingBy(MarcSubfield::getDefinition, Collectors.counting()));

    // Add errors for all non-repeatable subfields which were repeated
    for (Map.Entry<SubfieldDefinition, Long> entry : counter.entrySet()) {
      SubfieldDefinition subfieldDefinition = entry.getKey();
      long count = entry.getValue();
      if (count > 1 && subfieldDefinition.getCardinality().equals(Cardinality.Nonrepeatable)) {
        // addError(subfieldDefinition, SUBFIELD_NONREPEATABLE, String.format("there are %d instances", count));
        addError(subfieldDefinition, SUBFIELD_NONREPEATABLE, String.format("there are multiple instances", count));
      }
    }

  }

  /**
   * Adds the definition of the currently specified marc version to the subfield if it exists.
   * @param subfield The subfield to which the definition should be added
   */
  private void addVersionSpecificDefinition(MarcSubfield subfield) {
    if (subfield.getDefinition() != null) {
      return;
    }

    if (definition.isVersionSpecificSubfields(configuration.getMarcVersion(), subfield.getCode())) {
      subfield.setDefinition(
          definition.getVersionSpecificDefinition(
              configuration.getMarcVersion(), subfield.getCode()));
    }
  }

  private void addError(ValidationErrorType type, String message) {
    addError(definition.getExtendedTag(), type, message);
  }

  private void addError(SubfieldDefinition subfieldDefinition, ValidationErrorType type, String message) {
    addError(subfieldDefinition.getPath(), type, message);
  }

  private void addError(String path, ValidationErrorType type, String message) {
    if (!isIgnorableType(type)) {
      String id = field.getBibliographicRecord() == null ? null : field.getBibliographicRecord().getId();
      String url = definition.getDescriptionUrl();
      errors.add(id, path, type, message, url);
    }
  }

  /**
   * @return True if the tag isn't MARC21 or if it's MARC21 and is defined. Otherwise, false.
   */
  private boolean isMarc21TagDefined() {
    boolean isMarc21 = configuration.getSchemaType().equals(SchemaType.MARC21);

    // If it is not MARC21, we don't need to check if the tag is defined
    if (!isMarc21) {
      return true;
    }

    // If tag definition could have been loaded, it is defined
    return TagDefinitionLoader.load(definition.getTag(), configuration.getMarcVersion()) != null;
  }
}

1	package de.gwdg.metadataqa.marc.analysis.validator;
2
3	import de.gwdg.metadataqa.marc.MarcSubfield;
4	import de.gwdg.metadataqa.marc.dao.DataField;
5	import de.gwdg.metadataqa.marc.definition.Cardinality;
6	import de.gwdg.metadataqa.marc.definition.MarcVersion;
7	import de.gwdg.metadataqa.marc.definition.TagDefinitionLoader;
8	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
9	import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
10	import de.gwdg.metadataqa.marc.definition.structure.Indicator;
11	import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
12	import de.gwdg.metadataqa.marc.model.validation.ErrorsCollector;
13	import de.gwdg.metadataqa.marc.model.validation.ValidationErrorType;
14	import org.apache.commons.lang3.StringUtils;
15
16	import java.util.ArrayList;
17	import java.util.List;
18	import java.util.Map;
19	import java.util.stream.Collectors;
20
21	import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_INVALID_VALUE;
22	import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_NON_EMPTY;
23	import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.INDICATOR_OBSOLETE;
24	import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.SUBFIELD_NONREPEATABLE;
25	import static de.gwdg.metadataqa.marc.model.validation.ValidationErrorType.SUBFIELD_UNDEFINED;
26
27	public class DataFieldValidator extends AbstractValidator {
28
29	private ErrorsCollector errors;
30	private DataField field;
31	private DataFieldDefinition definition;
32
33	public DataFieldValidator() {
34	super(new ValidatorConfiguration());	1✔
35	}	1✔
36
37	public DataFieldValidator(ValidatorConfiguration configuration) {
38	super(configuration);	1✔
39	}	1✔
40
41	public boolean validate(DataField field) {
42	this.field = field;	1✔
43	definition = field.getDefinition();	1✔
44	validationErrors = new ArrayList<>();	1✔
45	errors = new ErrorsCollector();	1✔
46
47	// Check if the tag is defined only in case it is a MARC21 tag
48	// I'm not sure how this is different to validating unhandled tags. However, it will be kept until further notice.
49	boolean isMarc21TagDefined = isMarc21TagDefined();	1✔
50	if (!isMarc21TagDefined) {	1✔
51	addError(ValidationErrorType.FIELD_UNDEFINED, "");	1✔
52	return false;	1✔
53	}
54
55	// From here on, we know the tag is either MARC21 and defined, or not MARC21
56	// This LinkageHandler can later be further abstracted into an abstract LinkageHandler for UNIMARC in case that's needed
57	Marc21LinkageHandler linkageHandler = new Marc21LinkageHandler(configuration, errors);	1✔
58	DataField associatedField = linkageHandler.handleLinkage(field, definition);	1✔
59	errors.addAll(linkageHandler.getErrors());	1✔
60
61	// In case the associated field is not null, and due to compatibility with the rest of this class,
62	// assign the referencer definition and the subfields of the associated field to the local variables
63	// This assignment makes sure that the methods which use the attribute definition, use the definition
64	// of the referencer field if needed
65	DataFieldDefinition referencerDefinition = null;	1✔
66	List<MarcSubfield> subfields = field.getSubfields();	1✔
67	if (associatedField != null) {	1✔
68	referencerDefinition = definition;	1✔
69	definition = associatedField.getDefinition();	1✔
70	subfields = associatedField.getSubfields();	1✔
71	}
72
73	validateIndicators(referencerDefinition);	1✔
74
75	// This seems to never be executed because the addUnhandledSubfield method is never called
76	if (field.getUnhandledSubfields() != null) {	1✔
UNCOV 77	addError(SUBFIELD_UNDEFINED, StringUtils.join(field.getUnhandledSubfields(), ", "));	×
78	}
79
80	boolean isAmbiguousLinkage = linkageHandler.isAmbiguousLinkage();	1✔
81
82	if (!isAmbiguousLinkage) {	1✔
83	validateSubfields(subfields);	1✔
84	}
85
86	if (associatedField != null) {	1✔
87	definition = referencerDefinition;	1✔
88	}
89
90	validationErrors.addAll(errors.getErrors());	1✔
91	return errors.isEmpty();	1✔
92	}
93
94	private void validateIndicators(DataFieldDefinition referencerDefinition) {
95	if (configuration.getSchemaType().equals(SchemaType.PICA)) {	1✔
96	return;	1✔
97	}
98
99	if (field.getInd1() != null) {	1✔
100	validateIndicator(definition.getInd1(), field.getInd1(), configuration.getMarcVersion(), referencerDefinition);	1✔
101	}
102	if (field.getInd2() != null) {	1✔
103	validateIndicator(definition.getInd2(), field.getInd2(), configuration.getMarcVersion(), referencerDefinition);	1✔
104	}
105	}	1✔
106
107	private boolean validateIndicator(Indicator indicatorDefinition,
108	String value,
109	MarcVersion marcVersion,
110	DataFieldDefinition referencerDefinition) {
111	String path = indicatorDefinition.getPath();	1✔
112
113	if (referencerDefinition != null) {	1✔
114	path = String.format("%s->%s", referencerDefinition.getTag(), path);	1✔
115	}
116
117	boolean isVersionSpecific = indicatorDefinition.isVersionSpecificCode(marcVersion, value);	1✔
118	boolean indicatorExists = indicatorDefinition.exists();	1✔
119
120	// The first invalid case is:
121	// Indicator undefined but the value is not empty (and also it doesn't depend on the marc version used)
122	if (!indicatorExists && !value.equals(" ") && !isVersionSpecific) {	1✔
123	addError(path, INDICATOR_NON_EMPTY, value);	1✔
124	return false;	1✔
125	}
126
127	// The second invalid case is when the indicator is defined, but the value isn't a valid code (and also it
128	// doesn't depend on the marc version used).
129	// In case the code used is some old (historical) code, then that's an obsolete code error, and if it is
130	// just plainly a wrong code, then it is an invalid value error.
131	if (indicatorExists && !indicatorDefinition.hasCode(value) && !isVersionSpecific) {	1✔
132	// Maybe if the value is a space, it should be clarified in the error message a little differently
133	if (indicatorDefinition.isHistoricalCode(value)) {	1✔
NEW 134	addError(path, INDICATOR_OBSOLETE, value);	×
135	} else {
136	addError(path, INDICATOR_INVALID_VALUE, value);	1✔
137	}
138	return false;	1✔
139	}
140
141	// All other cases are valid: either the indicator is defined and the value is a valid code, or the indicator
142	// is undefined and the value is empty (or it is a version specific code).
143	return true;	1✔
144	}
145
146	private void validateSubfields(List<MarcSubfield> subfields) {
147	SubfieldValidator subfieldValidator = new SubfieldValidator(configuration);	1✔
148
149	for (MarcSubfield subfield : subfields) {	1✔
150	addVersionSpecificDefinition(subfield);	1✔
151	boolean isSubfieldValid = subfieldValidator.validate(subfield);	1✔
152
153	if (!isSubfieldValid) {	1✔
154	errors.addAll(subfieldValidator.getValidationErrors());	1✔
155	}
156	}	1✔
157
158	// Count the number of subfield definitions
159	Map<SubfieldDefinition, Long> counter = subfields.stream()	1✔
160	.filter(subfield -> subfield.getDefinition() != null)	1✔
161	.collect(Collectors.groupingBy(MarcSubfield::getDefinition, Collectors.counting()));	1✔
162
163	// Add errors for all non-repeatable subfields which were repeated
164	for (Map.Entry<SubfieldDefinition, Long> entry : counter.entrySet()) {	1✔
165	SubfieldDefinition subfieldDefinition = entry.getKey();	1✔
166	long count = entry.getValue();	1✔
167	if (count > 1 && subfieldDefinition.getCardinality().equals(Cardinality.Nonrepeatable)) {	1✔
168	// addError(subfieldDefinition, SUBFIELD_NONREPEATABLE, String.format("there are %d instances", count));
169	addError(subfieldDefinition, SUBFIELD_NONREPEATABLE, String.format("there are multiple instances", count));	1✔
170	}
171	}	1✔
172
173	}	1✔
174
175	/**
176	* Adds the definition of the currently specified marc version to the subfield if it exists.
177	* @param subfield The subfield to which the definition should be added
178	*/
179	private void addVersionSpecificDefinition(MarcSubfield subfield) {
180	if (subfield.getDefinition() != null) {	1✔
181	return;	1✔
182	}
183
184	if (definition.isVersionSpecificSubfields(configuration.getMarcVersion(), subfield.getCode())) {	1✔
185	subfield.setDefinition(	1✔
186	definition.getVersionSpecificDefinition(	1✔
187	configuration.getMarcVersion(), subfield.getCode()));	1✔
188	}
189	}	1✔
190
191	private void addError(ValidationErrorType type, String message) {
192	addError(definition.getExtendedTag(), type, message);	1✔
193	}	1✔
194
195	private void addError(SubfieldDefinition subfieldDefinition, ValidationErrorType type, String message) {
196	addError(subfieldDefinition.getPath(), type, message);	1✔
197	}	1✔
198
199	private void addError(String path, ValidationErrorType type, String message) {
200	if (!isIgnorableType(type)) {	1✔
201	String id = field.getBibliographicRecord() == null ? null : field.getBibliographicRecord().getId();	1✔
202	String url = definition.getDescriptionUrl();	1✔
203	errors.add(id, path, type, message, url);	1✔
204	}
205	}	1✔
206
207	/**
208	* @return True if the tag isn't MARC21 or if it's MARC21 and is defined. Otherwise, false.
209	*/
210	private boolean isMarc21TagDefined() {
211	boolean isMarc21 = configuration.getSchemaType().equals(SchemaType.MARC21);	1✔
212
213	// If it is not MARC21, we don't need to check if the tag is defined
214	if (!isMarc21) {	1✔
215	return true;	1✔
216	}
217
218	// If tag definition could have been loaded, it is defined
219	return TagDefinitionLoader.load(definition.getTag(), configuration.getMarcVersion()) != null;	1✔
220	}
221	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous