#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.98

/src/main/java/de/gwdg/metadataqa/marc/datastore/MarcSolrClient.java

package de.gwdg.metadataqa.marc.datastore;

import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.util.NamedList;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

public class MarcSolrClient {

  private static final Logger logger = Logger.getLogger(MarcSolrClient.class.getCanonicalName());
  public static final String ID_QUERY = "id:\"%s\"";

  private final String defaultUrl = "http://localhost:8983/solr";
  private SolrClient solrClient;
  private boolean trimId = false;
  private boolean indexWithTokenizedField = false;
  private final String termFieldSuffix = "_tt";
  private String fieldPrefix = "";
  private final Map<String, String> termFieldNameCache = new HashMap<>();

  public MarcSolrClient() {
    initialize(defaultUrl);
  }

  public MarcSolrClient(String url) {
    initialize(url);
  }

  public MarcSolrClient(SolrClient client) {
    solrClient = client;
  }

  private void initialize(String url) {
    solrClient = new Http2SolrClient.Builder(url).build();
  }

  public void indexMap(String id, Map<String, List<String>> objectMap) {
    index(createSolrDoc(id, objectMap));
  }

  public void index(SolrInputDocument document) {
    try {
      solrClient.add(document);
      // logger.log(Level.INFO, document.getField("id").toString() + " indexed");
    } catch (BaseHttpSolrClient.RemoteSolrException | SolrServerException | IOException ex) {
      logger.log(Level.WARNING, "document", document);
      logger.log(Level.WARNING, "Commit exception", ex);
      throw new RuntimeException(ex);
    } catch (Exception e) {
      logger.log(Level.WARNING, "Other kind of commit exception", e);
      throw new RuntimeException(e);
    }
  }

  public SolrInputDocument createSolrDoc(String id, Map<String, List<String>> objectMap) {
    SolrInputDocument document = new SolrInputDocument();
    document.addField("id", (trimId ? id.trim() : id));
    for (Map.Entry<String, List<String>> entry : objectMap.entrySet()) {
      String fieldName = entry.getKey();
      Object value = entry.getValue();
      if (value == null) {
        continue;
      }
      if (!fieldName.endsWith("_sni") && !fieldName.endsWith("_ss")) {
        fieldName = fieldPrefix + fieldName;
        fieldName += "_ss";
      }
      document.addField(fieldName, value);

      if (indexWithTokenizedField && fieldName.endsWith("_ss"))
        document.addField(getTermFieldName(fieldName), value);
    }
    return document;
  }

  private String getTermFieldName(String phraseField) {
    termFieldNameCache.putIfAbsent(phraseField, phraseField.replaceAll("_ss$", termFieldSuffix));
    return termFieldNameCache.get(phraseField);
  }

  public void indexDuplumKey(String id, Map<String, Object> objectMap)
      throws IOException, SolrServerException {
    SolrInputDocument document = new SolrInputDocument();
    document.addField("id", id);
    for (Map.Entry<String, Object> entry : objectMap.entrySet()) {
      String key = entry.getKey();
      Object value = entry.getValue();
      if (value == null) {
        continue;
      }
      if (!key.endsWith("_sni") && !key.endsWith("_ss")) {
        key = fieldPrefix + key;
        key += "_ss";
      }
      document.addField(key, value);
    }

    try {
      solrClient.add(document);
    } catch (BaseHttpSolrClient.RemoteSolrException ex) {
      logger.log(Level.WARNING, "document", document);
      logger.log(Level.WARNING, "Commit exception", ex);
    }
  }

  public void commit() {
    try {
      solrClient.commit();
    } catch (IOException | SolrServerException e) {
      logger.log(Level.WARNING, "commit", e);
    }
  }

  public void optimize() {
    try {
      solrClient.optimize();
    } catch (IOException | SolrServerException e) {
      logger.log(Level.WARNING, "optimize", e);
    }
  }

  /**
   * Given an id, return the corresponding SolrDocument from the index, and also remove the id field.
   */
  public SolrDocument get(String id) {
    try {
      final QueryResponse response = solrClient.query(new MapSolrParams(Map.of("q", String.format(ID_QUERY, id))));
      final SolrDocumentList documents = response.getResults();
      if (documents.getNumFound() <= 0) {
        return null;
      }
      SolrDocument doc = documents.get(0);
      doc.removeFields("id");
      doc.removeFields("_version_");
      return doc;
    } catch (SolrServerException | IOException e) {
      throw new RuntimeException(e);
    }
  }

  public long getCount() {
    try {
      final QueryResponse response = solrClient.query(new MapSolrParams(Map.of(
        "q", "*:*",
        "rows", "0")));
      return response.getResults().getNumFound();
    } catch (SolrServerException | IOException e) {
      logger.severe(e.getMessage());
    }
    return 0;
  }

  public boolean getTrimId() {
    return trimId;
  }

  public void setTrimId(boolean trimId) {
    this.trimId = trimId;
  }

  public void indexWithTokenizedField(boolean indexWithTokenizedField) {
    this.indexWithTokenizedField = indexWithTokenizedField;
  }

  public String getFieldPrefix() {
    return fieldPrefix;
  }

  public void setFieldPrefix(String fieldPrefix) {
    this.fieldPrefix = fieldPrefix;
  }
}

1	package de.gwdg.metadataqa.marc.datastore;
2
3	import org.apache.solr.client.solrj.SolrClient;
4	import org.apache.solr.client.solrj.SolrServerException;
5	import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
6	import org.apache.solr.client.solrj.impl.Http2SolrClient;
7	import org.apache.solr.client.solrj.response.QueryResponse;
8	import org.apache.solr.common.SolrDocument;
9	import org.apache.solr.common.SolrDocumentList;
10	import org.apache.solr.common.SolrInputDocument;
11	import org.apache.solr.common.params.MapSolrParams;
12	import org.apache.solr.common.util.NamedList;
13
14	import java.io.IOException;
15	import java.util.HashMap;
16	import java.util.List;
17	import java.util.Map;
18	import java.util.logging.Level;
19	import java.util.logging.Logger;
20
21	public class MarcSolrClient {
22
23	private static final Logger logger = Logger.getLogger(MarcSolrClient.class.getCanonicalName());	1✔
24	public static final String ID_QUERY = "id:\"%s\"";
25
26	private final String defaultUrl = "http://localhost:8983/solr";	1✔
27	private SolrClient solrClient;
28	private boolean trimId = false;	1✔
29	private boolean indexWithTokenizedField = false;	1✔
30	private final String termFieldSuffix = "_tt";	1✔
31	private String fieldPrefix = "";	1✔
32	private final Map<String, String> termFieldNameCache = new HashMap<>();	1✔
33
UNCOV 34	public MarcSolrClient() {	×
UNCOV 35	initialize(defaultUrl);	×
36	}	×
37
38	public MarcSolrClient(String url) {	×
UNCOV 39	initialize(url);	×
40	}	×
41
42	public MarcSolrClient(SolrClient client) {	1✔
43	solrClient = client;	1✔
44	}	1✔
45
46	private void initialize(String url) {
NEW 47	solrClient = new Http2SolrClient.Builder(url).build();	×
UNCOV 48	}	×
49
50	public void indexMap(String id, Map<String, List<String>> objectMap) {
51	index(createSolrDoc(id, objectMap));	1✔
52	}	1✔
53
54	public void index(SolrInputDocument document) {
55	try {
56	solrClient.add(document);	1✔
57	// logger.log(Level.INFO, document.getField("id").toString() + " indexed");
NEW 58	} catch (BaseHttpSolrClient.RemoteSolrException \| SolrServerException \| IOException ex) {	×
UNCOV 59	logger.log(Level.WARNING, "document", document);	×
UNCOV 60	logger.log(Level.WARNING, "Commit exception", ex);	×
UNCOV 61	throw new RuntimeException(ex);	×
NEW 62	} catch (Exception e) {	×
NEW 63	logger.log(Level.WARNING, "Other kind of commit exception", e);	×
NEW 64	throw new RuntimeException(e);	×
65	}	1✔
66	}	1✔
67
68	public SolrInputDocument createSolrDoc(String id, Map<String, List<String>> objectMap) {
69	SolrInputDocument document = new SolrInputDocument();	1✔
70	document.addField("id", (trimId ? id.trim() : id));	1✔
71	for (Map.Entry<String, List<String>> entry : objectMap.entrySet()) {	1✔
72	String fieldName = entry.getKey();	1✔
73	Object value = entry.getValue();	1✔
74	if (value == null) {	1✔
NEW 75	continue;	×
76	}
77	if (!fieldName.endsWith("_sni") && !fieldName.endsWith("_ss")) {	1✔
78	fieldName = fieldPrefix + fieldName;	1✔
79	fieldName += "_ss";	1✔
80	}
81	document.addField(fieldName, value);	1✔
82
83	if (indexWithTokenizedField && fieldName.endsWith("_ss"))	1✔
84	document.addField(getTermFieldName(fieldName), value);	1✔
85	}	1✔
86	return document;	1✔
87	}
88
89	private String getTermFieldName(String phraseField) {
90	termFieldNameCache.putIfAbsent(phraseField, phraseField.replaceAll("_ss$", termFieldSuffix));	1✔
91	return termFieldNameCache.get(phraseField);	1✔
92	}
93
94	public void indexDuplumKey(String id, Map<String, Object> objectMap)
95	throws IOException, SolrServerException {
96	SolrInputDocument document = new SolrInputDocument();	×
97	document.addField("id", id);	×
98	for (Map.Entry<String, Object> entry : objectMap.entrySet()) {	×
99	String key = entry.getKey();	×
100	Object value = entry.getValue();	×
NEW 101	if (value == null) {	×
NEW 102	continue;	×
103	}
NEW 104	if (!key.endsWith("_sni") && !key.endsWith("_ss")) {	×
NEW 105	key = fieldPrefix + key;	×
NEW 106	key += "_ss";	×
107	}
NEW 108	document.addField(key, value);	×
109	}	×
110
111	try {
UNCOV 112	solrClient.add(document);	×
NEW 113	} catch (BaseHttpSolrClient.RemoteSolrException ex) {	×
114	logger.log(Level.WARNING, "document", document);	×
115	logger.log(Level.WARNING, "Commit exception", ex);	×
116	}	×
117	}	×
118
119	public void commit() {
120	try {
121	solrClient.commit();	1✔
UNCOV 122	} catch (IOException \| SolrServerException e) {	×
UNCOV 123	logger.log(Level.WARNING, "commit", e);	×
124	}	1✔
125	}	1✔
126
127	public void optimize() {
128	try {
UNCOV 129	solrClient.optimize();	×
UNCOV 130	} catch (IOException \| SolrServerException e) {	×
131	logger.log(Level.WARNING, "optimize", e);	×
132	}	×
133	}	×
134
135	/**
136	* Given an id, return the corresponding SolrDocument from the index, and also remove the id field.
137	*/
138	public SolrDocument get(String id) {
139	try {
140	final QueryResponse response = solrClient.query(new MapSolrParams(Map.of("q", String.format(ID_QUERY, id))));	1✔
141	final SolrDocumentList documents = response.getResults();	1✔
142	if (documents.getNumFound() <= 0) {	1✔
143	return null;	1✔
144	}
145	SolrDocument doc = documents.get(0);	1✔
146	doc.removeFields("id");	1✔
147	doc.removeFields("_version_");	1✔
148	return doc;	1✔
NEW 149	} catch (SolrServerException \| IOException e) {	×
UNCOV 150	throw new RuntimeException(e);	×
151	}
152	}
153
154	public long getCount() {
155	try {
NEW 156	final QueryResponse response = solrClient.query(new MapSolrParams(Map.of(	×
157	"q", ":",
158	"rows", "0")));
NEW 159	return response.getResults().getNumFound();	×
NEW 160	} catch (SolrServerException \| IOException e) {	×
NEW 161	logger.severe(e.getMessage());	×
162	}
NEW 163	return 0;	×
164	}
165
166	public boolean getTrimId() {
UNCOV 167	return trimId;	×
168	}
169
170	public void setTrimId(boolean trimId) {
171	this.trimId = trimId;	1✔
172	}	1✔
173
174	public void indexWithTokenizedField(boolean indexWithTokenizedField) {
175	this.indexWithTokenizedField = indexWithTokenizedField;	1✔
176	}	1✔
177
178	public String getFieldPrefix() {
NEW 179	return fieldPrefix;	×
180	}
181
182	public void setFieldPrefix(String fieldPrefix) {
183	this.fieldPrefix = fieldPrefix;	1✔
184	}	1✔
185	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous