#1632

Committed 02 Mar 2026 04:53PM UTC coverage: 90.198% (-0.08%) from 90.275%

Build # #1632

Build Type

push

Committed by

Commit Message

compound index in Solr #740

Coverage Stats

36734 of 40726 relevant lines covered (90.2%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.04

/src/main/java/de/gwdg/metadataqa/marc/cli/parameters/MarcToSolrParameters.java

package de.gwdg.metadataqa.marc.cli.parameters;

import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.model.SolrFieldType;
import de.gwdg.metadataqa.marc.utils.SchemaSpec;
import de.gwdg.metadataqa.marc.utils.marcspec.MarcSpecParser;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
import org.apache.commons.cli.ParseException;
import org.apache.solr.client.solrj.SolrClient;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;

public class MarcToSolrParameters extends CommonParameters {

  private static final Logger logger = Logger.getLogger(TranslationParameters.class.getCanonicalName());

  private int DEFAULT_COMMIT_AT = 10000;
  private boolean useEmbedded = false;
  private String solrUrl = null;
  private boolean doCommit = false;
  private SolrFieldType solrFieldType = SolrFieldType.MIXED;
  private SolrClient mainClient = null;
  private SolrClient validationClient = null;
  private boolean indexWithTokenizedField = false;

  private boolean isOptionSet = false;
  private int commitAt = DEFAULT_COMMIT_AT;
  private boolean indexFieldCounts = false;
  private boolean indexSubfieldCounts = false;
  private String fieldPrefix = null;
  private String compoundFieldsInput = null;
  private Map<String, List<SchemaSpec>> compoundFields = null;

  @Override
  protected void setOptions() {
    if (!isOptionSet) {
      super.setOptions();
      options.addOption("S", "solrUrl", true, "the URL of Solr server including the core (e.g. http://localhost:8983/solr/loc)");
      options.addOption("A", "doCommit", false, "commits Solr index regularly");
      options.addOption("T", "solrFieldType", true,
        "type of Solr fields, could be one of 'marc-tags', 'human-readable', or 'mixed'");
      options.addOption("B", "useEmbedded", false, "use embedded Solr server (used in tests only)");
      options.addOption("C", "indexWithTokenizedField", false, "index data elements as tokenized field as well");
      options.addOption("D", "commitAt", true, "commit index after this number of records");
      options.addOption("E", "indexFieldCounts", false, "index the count of field instances");
      options.addOption("G", "indexSubfieldCounts", false, "index the count of subfield instances");
      options.addOption("F", "fieldPrefix", true, "field prefix");
      options.addOption("H", "compoundFields", true, "compound fields");
      isOptionSet = true;
    }
  }

  public MarcToSolrParameters() {
    super();
  }

  public MarcToSolrParameters(String[] arguments) throws ParseException {
    super(arguments);

    if (cmd.hasOption("solrUrl"))
      solrUrl = cmd.getOptionValue("solrUrl");

    if (cmd.hasOption("doCommit"))
      doCommit = true;

    if (cmd.hasOption("solrFieldType"))
      solrFieldType = SolrFieldType.byCode(cmd.getOptionValue("solrFieldType"));

    if (cmd.hasOption("useEmbedded"))
      useEmbedded = true;

    if (cmd.hasOption("indexWithTokenizedField"))
      indexWithTokenizedField = true;

    if (cmd.hasOption("commitAt"))
      commitAt = Integer.valueOf(cmd.getOptionValue("commitAt"));

    if (cmd.hasOption("indexFieldCounts"))
      indexFieldCounts = true;

    if (cmd.hasOption("indexSubfieldCounts"))
      indexSubfieldCounts = true;

    if (cmd.hasOption("fieldPrefix"))
      fieldPrefix = cmd.getOptionValue("fieldPrefix");

    if (cmd.hasOption("compoundFields")) {
      compoundFieldsInput = cmd.getOptionValue("compoundFields");
      logger.info("compoundFieldsInput: " + compoundFieldsInput);
      try {
        compoundFields = parseCompoundFields(compoundFieldsInput);
      } catch (Exception e) {
        e.printStackTrace();
      }
      logger.info("compoundFields: " + compoundFields);
    }
  }

  /**
   * Parse the compound field input and transforms it to a maps of solr field - bibliographic fields pairs.
   * @param compoundFieldsInput The compound field configuration. It should fit the follwoing pattern:
   *                            solrfield1=biblfield1,biblfield2|solrfield2=biblfield3,biblfield4
   * @return
   */
  private Map<String, List<SchemaSpec>> parseCompoundFields(String compoundFieldsInput) {
    logger.info("parseCompoundFields: " + compoundFieldsInput);
    Map<String, List<SchemaSpec>> compoundFieldsMap = new HashMap<>();
    String[] fields = compoundFieldsInput.split("\\|");
    for (String field : fields) {
      logger.info("field: " + field);
      String[] fieldParts = field.split("=", 2);
      String solrField = fieldParts[0];
      String[] bibFields = fieldParts[1].split(",");
      List<SchemaSpec> bibliographicFields = new ArrayList<>();
      for (String bibField : bibFields) {
        if (getSchemaType().equals(SchemaType.PICA))
          bibliographicFields.add(new PicaSpec(bibField));
        else
          bibliographicFields.add(MarcSpecParser.parse(bibField));
      }
      compoundFieldsMap.put(solrField, bibliographicFields);
    }
    return compoundFieldsMap;
  }

  public String getSolrUrl() {
    return solrUrl;
  }

  public boolean isDoCommit() {
    return doCommit;
  }

  public SolrFieldType getSolrFieldType() {
    return solrFieldType;
  }

  public SolrClient getMainClient() {
    return mainClient;
  }

  public void setMainClient(SolrClient mainClient) {
    this.mainClient = mainClient;
  }

  public SolrClient getValidationClient() {
    return validationClient;
  }

  public void setValidationClient(SolrClient validationClient) {
    this.validationClient = validationClient;
  }

  public boolean isUseEmbedded() {
    return useEmbedded;
  }

  public boolean isIndexWithTokenizedField() {
    return indexWithTokenizedField;
  }

  public int getCommitAt() {
    return commitAt;
  }

  public boolean isIndexFieldCounts() {
    return indexFieldCounts;
  }

  public boolean isIndexSubfieldCounts() {
    return indexSubfieldCounts;
  }

  public String getFieldPrefix() {
    return fieldPrefix != null ? fieldPrefix : "";
  }

  public void setFieldPrefix(String fieldPrefix) {
    this.fieldPrefix = fieldPrefix;
  }

  public Map<String, List<SchemaSpec>> getCompoundFields() {
    return compoundFields;
  }

  @Override
  public String formatParameters() {
    String text = super.formatParameters();
    text += String.format("solrUrl: %s%n", solrUrl);
    text += String.format("doCommit: %s%n", doCommit);
    text += String.format("solrFieldType: %s%n", solrFieldType);
    text += String.format("indexWithTokenizedField: %s%n", indexWithTokenizedField);
    text += String.format("commitAt: %s%n", commitAt);
    text += String.format("indexFieldCounts: %s%n", indexFieldCounts);
    text += String.format("indexSubfieldCounts: %s%n", indexSubfieldCounts);
    text += String.format("fieldPrefix: %s%n", fieldPrefix);
    text += String.format("compoundFields: %s%n", compoundFieldsInput);
    return text;
  }

}

1	package de.gwdg.metadataqa.marc.cli.parameters;
2
3	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
4	import de.gwdg.metadataqa.marc.model.SolrFieldType;
5	import de.gwdg.metadataqa.marc.utils.SchemaSpec;
6	import de.gwdg.metadataqa.marc.utils.marcspec.MarcSpecParser;
7	import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
8	import org.apache.commons.cli.ParseException;
9	import org.apache.solr.client.solrj.SolrClient;
10
11	import java.util.ArrayList;
12	import java.util.HashMap;
13	import java.util.List;
14	import java.util.Map;
15	import java.util.logging.Logger;
16
17	public class MarcToSolrParameters extends CommonParameters {
18
19	private static final Logger logger = Logger.getLogger(TranslationParameters.class.getCanonicalName());	1✔
20
21	private int DEFAULT_COMMIT_AT = 10000;	1✔
22	private boolean useEmbedded = false;	1✔
23	private String solrUrl = null;	1✔
24	private boolean doCommit = false;	1✔
25	private SolrFieldType solrFieldType = SolrFieldType.MIXED;	1✔
26	private SolrClient mainClient = null;	1✔
27	private SolrClient validationClient = null;	1✔
28	private boolean indexWithTokenizedField = false;	1✔
29
30	private boolean isOptionSet = false;	1✔
31	private int commitAt = DEFAULT_COMMIT_AT;	1✔
32	private boolean indexFieldCounts = false;	1✔
33	private boolean indexSubfieldCounts = false;	1✔
34	private String fieldPrefix = null;	1✔
35	private String compoundFieldsInput = null;	1✔
36	private Map<String, List<SchemaSpec>> compoundFields = null;	1✔
37
38	@Override
39	protected void setOptions() {
40	if (!isOptionSet) {	1✔
41	super.setOptions();	1✔
42	options.addOption("S", "solrUrl", true, "the URL of Solr server including the core (e.g. http://localhost:8983/solr/loc)");	1✔
43	options.addOption("A", "doCommit", false, "commits Solr index regularly");	1✔
44	options.addOption("T", "solrFieldType", true,	1✔
45	"type of Solr fields, could be one of 'marc-tags', 'human-readable', or 'mixed'");
46	options.addOption("B", "useEmbedded", false, "use embedded Solr server (used in tests only)");	1✔
47	options.addOption("C", "indexWithTokenizedField", false, "index data elements as tokenized field as well");	1✔
48	options.addOption("D", "commitAt", true, "commit index after this number of records");	1✔
49	options.addOption("E", "indexFieldCounts", false, "index the count of field instances");	1✔
50	options.addOption("G", "indexSubfieldCounts", false, "index the count of subfield instances");	1✔
51	options.addOption("F", "fieldPrefix", true, "field prefix");	1✔
52	options.addOption("H", "compoundFields", true, "compound fields");	1✔
53	isOptionSet = true;	1✔
54	}
55	}	1✔
56
57	public MarcToSolrParameters() {
58	super();	1✔
59	}	1✔
60
61	public MarcToSolrParameters(String[] arguments) throws ParseException {
62	super(arguments);	1✔
63
64	if (cmd.hasOption("solrUrl"))	1✔
65	solrUrl = cmd.getOptionValue("solrUrl");	1✔
66
67	if (cmd.hasOption("doCommit"))	1✔
68	doCommit = true;	1✔
69
70	if (cmd.hasOption("solrFieldType"))	1✔
71	solrFieldType = SolrFieldType.byCode(cmd.getOptionValue("solrFieldType"));	1✔
72
73	if (cmd.hasOption("useEmbedded"))	1✔
74	useEmbedded = true;	1✔
75
76	if (cmd.hasOption("indexWithTokenizedField"))	1✔
77	indexWithTokenizedField = true;	×
78
79	if (cmd.hasOption("commitAt"))	1✔
80	commitAt = Integer.valueOf(cmd.getOptionValue("commitAt"));	×
81
82	if (cmd.hasOption("indexFieldCounts"))	1✔
83	indexFieldCounts = true;	1✔
84
85	if (cmd.hasOption("indexSubfieldCounts"))	1✔
86	indexSubfieldCounts = true;	×
87
88	if (cmd.hasOption("fieldPrefix"))	1✔
89	fieldPrefix = cmd.getOptionValue("fieldPrefix");	1✔
90
91	if (cmd.hasOption("compoundFields")) {	1✔
92	compoundFieldsInput = cmd.getOptionValue("compoundFields");	×
93	logger.info("compoundFieldsInput: " + compoundFieldsInput);	×
94	try {
95	compoundFields = parseCompoundFields(compoundFieldsInput);	×
96	} catch (Exception e) {	×
97	e.printStackTrace();	×
98	}	×
99	logger.info("compoundFields: " + compoundFields);	×
100	}
101	}	1✔
102
103	/**
104	* Parse the compound field input and transforms it to a maps of solr field - bibliographic fields pairs.
105	* @param compoundFieldsInput The compound field configuration. It should fit the follwoing pattern:
106	* solrfield1=biblfield1,biblfield2\|solrfield2=biblfield3,biblfield4
107	* @return
108	*/
109	private Map<String, List<SchemaSpec>> parseCompoundFields(String compoundFieldsInput) {
110	logger.info("parseCompoundFields: " + compoundFieldsInput);	×
111	Map<String, List<SchemaSpec>> compoundFieldsMap = new HashMap<>();	×
112	String[] fields = compoundFieldsInput.split("\\\|");	×
113	for (String field : fields) {	×
114	logger.info("field: " + field);	×
115	String[] fieldParts = field.split("=", 2);	×
116	String solrField = fieldParts[0];	×
117	String[] bibFields = fieldParts[1].split(",");	×
118	List<SchemaSpec> bibliographicFields = new ArrayList<>();	×
119	for (String bibField : bibFields) {	×
120	if (getSchemaType().equals(SchemaType.PICA))	×
121	bibliographicFields.add(new PicaSpec(bibField));	×
122	else
123	bibliographicFields.add(MarcSpecParser.parse(bibField));	×
124	}
125	compoundFieldsMap.put(solrField, bibliographicFields);	×
126	}
127	return compoundFieldsMap;	×
128	}
129
130	public String getSolrUrl() {
131	return solrUrl;	1✔
132	}
133
134	public boolean isDoCommit() {
135	return doCommit;	1✔
136	}
137
138	public SolrFieldType getSolrFieldType() {
139	return solrFieldType;	1✔
140	}
141
142	public SolrClient getMainClient() {
143	return mainClient;	1✔
144	}
145
146	public void setMainClient(SolrClient mainClient) {
147	this.mainClient = mainClient;	1✔
148	}	1✔
149
150	public SolrClient getValidationClient() {
151	return validationClient;	1✔
152	}
153
154	public void setValidationClient(SolrClient validationClient) {
155	this.validationClient = validationClient;	1✔
156	}	1✔
157
158	public boolean isUseEmbedded() {
159	return useEmbedded;	1✔
160	}
161
162	public boolean isIndexWithTokenizedField() {
163	return indexWithTokenizedField;	1✔
164	}
165
166	public int getCommitAt() {
167	return commitAt;	1✔
168	}
169
170	public boolean isIndexFieldCounts() {
171	return indexFieldCounts;	1✔
172	}
173
174	public boolean isIndexSubfieldCounts() {
175	return indexSubfieldCounts;	1✔
176	}
177
178	public String getFieldPrefix() {
179	return fieldPrefix != null ? fieldPrefix : "";	1✔
180	}
181
182	public void setFieldPrefix(String fieldPrefix) {
183	this.fieldPrefix = fieldPrefix;	×
184	}	×
185
186	public Map<String, List<SchemaSpec>> getCompoundFields() {
187	return compoundFields;	1✔
188	}
189
190	@Override
191	public String formatParameters() {
192	String text = super.formatParameters();	1✔
193	text += String.format("solrUrl: %s%n", solrUrl);	1✔
194	text += String.format("doCommit: %s%n", doCommit);	1✔
195	text += String.format("solrFieldType: %s%n", solrFieldType);	1✔
196	text += String.format("indexWithTokenizedField: %s%n", indexWithTokenizedField);	1✔
197	text += String.format("commitAt: %s%n", commitAt);	1✔
198	text += String.format("indexFieldCounts: %s%n", indexFieldCounts);	1✔
199	text += String.format("indexSubfieldCounts: %s%n", indexSubfieldCounts);	1✔
200	text += String.format("fieldPrefix: %s%n", fieldPrefix);	1✔
201	text += String.format("compoundFields: %s%n", compoundFieldsInput);	1✔
202	return text;	1✔
203	}
204
205	}

pkiraly / metadata-qa-marc / #1632

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous