#3650

Committed 24 Oct 2023 03:07PM UTC coverage: 66.012% (-8.4%) from 74.444%

Build # #3650

Build Type

push

Committed by

Commit Message

refactory repository history check

Run Details

9 of 9 new or added lines in 1 file covered. (100.0%)

38668 of 58577 relevant lines covered (66.01%)

0.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.4

/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileAnnotationCache.java

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * See LICENSE.txt included in this distribution for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at LICENSE.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
 */
package org.opengrok.indexer.history;

import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.fasterxml.jackson.dataformat.smile.SmileParser;
import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
import org.opengrok.indexer.Metrics;
import org.opengrok.indexer.logger.LoggerFactory;
import org.opengrok.indexer.util.Statistics;

import java.io.File;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;

public class FileAnnotationCache extends AbstractCache implements AnnotationCache {

    private static final Logger LOGGER = LoggerFactory.getLogger(FileAnnotationCache.class);

    private Counter fileAnnotationCacheHits;
    private Counter fileAnnotationCacheMisses;

    private static final String ANNOTATION_CACHE_DIR_NAME = "annotationcache";

    public void initialize() {
        MeterRegistry meterRegistry = Metrics.getRegistry();
        if (meterRegistry != null) {
            fileAnnotationCacheHits = Counter.builder("cache.annotation.file.get").
                    description("file annotation cache hits").
                    tag("what", "hits").
                    register(meterRegistry);
            fileAnnotationCacheMisses = Counter.builder("cache.annotation.file.get").
                    description("file annotation cache misses").
                    tag("what", "miss").
                    register(meterRegistry);
        }
    }

    /**
     * Read serialized {@link AnnotationData} from a file and create {@link Annotation} instance out of it.
     */
    static Annotation readCache(File file) throws IOException {
        ObjectMapper mapper = new SmileMapper();
        return new Annotation(mapper.readValue(file, AnnotationData.class));
    }

    /**
     * Retrieve revision from the cache for given file. This is done in a fashion that keeps I/O low.
     * Assumes that {@link AnnotationData#revision} is serialized in the cache file as the first member.
     * @param file source root file
     * @return revision from the cache file or {@code null}
     * @throws CacheException on error
     */
    @Nullable
    String getRevision(File file) throws CacheException {
        File cacheFile;
        try {
            cacheFile = getCachedFile(file);
        } catch (CacheException e) {
            throw new CacheException("failed to get annotation cache file", e);
        }

        SmileFactory factory = new SmileFactory();
        try (SmileParser parser = factory.createParser(cacheFile)) {
            parser.nextToken();
            while (parser.getCurrentToken() != null) {
                if (parser.getCurrentToken().equals(JsonToken.FIELD_NAME)) {
                    break;
                }
                parser.nextToken();
            }

            if (parser.getCurrentName().equals("revision")) {
                parser.nextToken();
                if (!parser.getCurrentToken().equals(JsonToken.VALUE_STRING)) {
                    LOGGER.log(Level.WARNING, "the value of the ''revision'' field in ''{0}'' is not string",
                            cacheFile);
                    return null;
                }
                return parser.getValueAsString();
            } else {
                LOGGER.log(Level.WARNING, "the first serialized field is not ''revision'' in ''{0}''", cacheFile);
                return null;
            }
        } catch (IOException e) {
            throw new CacheException(e);
        }
    }

    @Override
    public String getCacheFileSuffix() {
        return "";
    }

    @VisibleForTesting
    Annotation readAnnotation(File file) throws CacheException {
        File cacheFile;
        try {
            cacheFile = getCachedFile(file);
        } catch (CacheException e) {
            LOGGER.log(Level.WARNING, "failed to get annotation cache file", e);
            return null;
        }

        try {
            Statistics statistics = new Statistics();
            Annotation annotation = readCache(cacheFile);
            statistics.report(LOGGER, Level.FINEST, String.format("deserialized annotation from cache for '%s'", file));
            return annotation;
        } catch (IOException e) {
            throw new CacheException(String.format("failed to read annotation cache for '%s'", file), e);
        }
    }

    /**
     * This is potentially expensive operation as the cache entry has to be retrieved from disk
     * in order to tell whether it is stale or not.
     * @param file source file
     * @return indication whether the cache entry is fresh
     */
    public boolean isUpToDate(File file) {
        try {
            return get(file, null) != null;
        } catch (CacheException e) {
            return false;
        }
    }

    public Annotation get(File file, @Nullable String rev) throws CacheException {
        Annotation annotation = null;
        String latestRevision = LatestRevisionUtil.getLatestRevision(file);
        if (rev == null || (latestRevision != null && latestRevision.equals(rev))) {
            /*
             * Double check that the cached annotation is not stale by comparing the stored revision
             * with revision to be fetched.
             * This should be more robust than the file time stamp based check performed by history cache,
             * at the expense of having to read some content from the annotation cache.
             */
            final String storedRevision = getRevision(file);
            /*
             * Even though store() does not allow to store annotation with null revision, the check
             * should be present to catch weird cases of someone not using the store() or general badness.
             */
            if (storedRevision == null) {
                LOGGER.log(Level.FINER, "no stored revision in annotation cache for ''{0}''", file);
            } else if (!storedRevision.equals(latestRevision)) {
                LOGGER.log(Level.FINER,
                        "stored revision {0} for ''{1}'' does not match latest revision {2}",
                        new Object[]{storedRevision, file, rev});
            } else {
                // read from the cache
                annotation = readAnnotation(file);
            }
        }

        if (annotation != null) {
            if (fileAnnotationCacheHits != null) {
                fileAnnotationCacheHits.increment();
            }
        } else {
            if (fileAnnotationCacheMisses != null) {
                fileAnnotationCacheMisses.increment();
            }
            LOGGER.log(Level.FINEST, "annotation cache miss for ''{0}'' in revision {1}",
                    new Object[]{file, rev});
            return null;
        }

        return annotation;
    }

    private void writeCache(AnnotationData annotationData, File outfile) throws IOException {
        ObjectMapper mapper = new SmileMapper();
        mapper.writeValue(outfile, annotationData);
    }

    public void store(File file, Annotation annotation) throws CacheException {
        if (annotation.getRevision() == null || annotation.getRevision().isEmpty()) {
            throw new CacheException(String.format("annotation for ''%s'' does not contain revision", file));
        }

        File cacheFile;
        try {
            cacheFile = getCachedFile(file);
        } catch (CacheException e) {
            LOGGER.log(Level.FINER, e.getMessage());
            return;
        }

        File dir = cacheFile.getParentFile();
        // calling isDirectory() twice to prevent a race condition
        if (!dir.isDirectory() && !dir.mkdirs() && !dir.isDirectory()) {
            throw new CacheException("Unable to create cache directory '" + dir + "'.");
        }

        Statistics statistics = new Statistics();
        try {
            writeCache(annotation.annotationData, cacheFile);
            statistics.report(LOGGER, Level.FINEST, String.format("wrote annotation for '%s'", file),
                    "cache.annotation.file.store.latency");
        } catch (IOException e) {
            LOGGER.log(Level.WARNING, "failed to write annotation to cache", e);
        }
    }

    public void clear(RepositoryInfo repository) {
        CacheUtil.clearCacheDir(repository, this);
    }

    @Override
    public void optimize() {
        // nothing to do
    }

    @Override
    public boolean supportsRepository(Repository repository) {
        // all repositories are supported
        return true;
    }

    @Override
    public String getCacheDirName() {
        return ANNOTATION_CACHE_DIR_NAME;
    }

    @Override
    public String getInfo() {
        return getClass().getSimpleName();
    }
}

1	/*
2	* CDDL HEADER START
3	*
4	* The contents of this file are subject to the terms of the
5	* Common Development and Distribution License (the "License").
6	* You may not use this file except in compliance with the License.
7	*
8	* See LICENSE.txt included in this distribution for the specific
9	* language governing permissions and limitations under the License.
10	*
11	* When distributing Covered Code, include this CDDL HEADER in each
12	* file and include the License file at LICENSE.txt.
13	* If applicable, add the following below this CDDL HEADER, with the
14	* fields enclosed by brackets "[]" replaced with your own identifying
15	* information: Portions Copyright [yyyy] [name of copyright owner]
16	*
17	* CDDL HEADER END
18	*/
19
20	/*
21	* Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
22	*/
23	package org.opengrok.indexer.history;
24
25	import com.fasterxml.jackson.core.JsonToken;
26	import com.fasterxml.jackson.databind.ObjectMapper;
27	import com.fasterxml.jackson.dataformat.smile.SmileFactory;
28	import com.fasterxml.jackson.dataformat.smile.SmileParser;
29	import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
30	import io.micrometer.core.instrument.Counter;
31	import io.micrometer.core.instrument.MeterRegistry;
32	import org.jetbrains.annotations.Nullable;
33	import org.jetbrains.annotations.VisibleForTesting;
34	import org.opengrok.indexer.Metrics;
35	import org.opengrok.indexer.logger.LoggerFactory;
36	import org.opengrok.indexer.util.Statistics;
37
38	import java.io.File;
39	import java.io.IOException;
40	import java.util.logging.Level;
41	import java.util.logging.Logger;
42
43	public class FileAnnotationCache extends AbstractCache implements AnnotationCache {	1✔
44
45	private static final Logger LOGGER = LoggerFactory.getLogger(FileAnnotationCache.class);	1✔
46
47	private Counter fileAnnotationCacheHits;
48	private Counter fileAnnotationCacheMisses;
49
50	private static final String ANNOTATION_CACHE_DIR_NAME = "annotationcache";
51
52	public void initialize() {
53	MeterRegistry meterRegistry = Metrics.getRegistry();	1✔
54	if (meterRegistry != null) {	1✔
55	fileAnnotationCacheHits = Counter.builder("cache.annotation.file.get").	1✔
56	description("file annotation cache hits").	1✔
57	tag("what", "hits").	1✔
58	register(meterRegistry);	1✔
59	fileAnnotationCacheMisses = Counter.builder("cache.annotation.file.get").	1✔
60	description("file annotation cache misses").	1✔
61	tag("what", "miss").	1✔
62	register(meterRegistry);	1✔
63	}
64	}	1✔
65
66	/**
67	* Read serialized {@link AnnotationData} from a file and create {@link Annotation} instance out of it.
68	*/
69	static Annotation readCache(File file) throws IOException {
70	ObjectMapper mapper = new SmileMapper();	1✔
71	return new Annotation(mapper.readValue(file, AnnotationData.class));	1✔
72	}
73
74	/**
75	* Retrieve revision from the cache for given file. This is done in a fashion that keeps I/O low.
76	* Assumes that {@link AnnotationData#revision} is serialized in the cache file as the first member.
77	* @param file source root file
78	* @return revision from the cache file or {@code null}
79	* @throws CacheException on error
80	*/
81	@Nullable
82	String getRevision(File file) throws CacheException {
83	File cacheFile;
84	try {
85	cacheFile = getCachedFile(file);	1✔
86	} catch (CacheException e) {	×
87	throw new CacheException("failed to get annotation cache file", e);	×
88	}	1✔
89
90	SmileFactory factory = new SmileFactory();	1✔
91	try (SmileParser parser = factory.createParser(cacheFile)) {	1✔
92	parser.nextToken();	1✔
93	while (parser.getCurrentToken() != null) {	1✔
94	if (parser.getCurrentToken().equals(JsonToken.FIELD_NAME)) {	1✔
95	break;	1✔
96	}
97	parser.nextToken();	1✔
98	}
99
100	if (parser.getCurrentName().equals("revision")) {	1✔
101	parser.nextToken();	1✔
102	if (!parser.getCurrentToken().equals(JsonToken.VALUE_STRING)) {	1✔
103	LOGGER.log(Level.WARNING, "the value of the ''revision'' field in ''{0}'' is not string",	×
104	cacheFile);
105	return null;	×
106	}
107	return parser.getValueAsString();	1✔
108	} else {
109	LOGGER.log(Level.WARNING, "the first serialized field is not ''revision'' in ''{0}''", cacheFile);	×
110	return null;	×
111	}
112	} catch (IOException e) {	1✔
113	throw new CacheException(e);	1✔
114	}
115	}
116
117	@Override
118	public String getCacheFileSuffix() {
119	return "";	1✔
120	}
121
122	@VisibleForTesting
123	Annotation readAnnotation(File file) throws CacheException {
124	File cacheFile;
125	try {
126	cacheFile = getCachedFile(file);	1✔
127	} catch (CacheException e) {	×
128	LOGGER.log(Level.WARNING, "failed to get annotation cache file", e);	×
129	return null;	×
130	}	1✔
131
132	try {
133	Statistics statistics = new Statistics();	1✔
134	Annotation annotation = readCache(cacheFile);	1✔
135	statistics.report(LOGGER, Level.FINEST, String.format("deserialized annotation from cache for '%s'", file));	1✔
136	return annotation;	1✔
137	} catch (IOException e) {	1✔
138	throw new CacheException(String.format("failed to read annotation cache for '%s'", file), e);	1✔
139	}
140	}
141
142	/**
143	* This is potentially expensive operation as the cache entry has to be retrieved from disk
144	* in order to tell whether it is stale or not.
145	* @param file source file
146	* @return indication whether the cache entry is fresh
147	*/
148	public boolean isUpToDate(File file) {
149	try {
150	return get(file, null) != null;	×
151	} catch (CacheException e) {	×
152	return false;	×
153	}
154	}
155
156	public Annotation get(File file, @Nullable String rev) throws CacheException {
157	Annotation annotation = null;	1✔
158	String latestRevision = LatestRevisionUtil.getLatestRevision(file);	1✔
159	if (rev == null \|\| (latestRevision != null && latestRevision.equals(rev))) {	1✔
160	/*
161	* Double check that the cached annotation is not stale by comparing the stored revision
162	* with revision to be fetched.
163	* This should be more robust than the file time stamp based check performed by history cache,
164	* at the expense of having to read some content from the annotation cache.
165	*/
166	final String storedRevision = getRevision(file);	1✔
167	/*
168	* Even though store() does not allow to store annotation with null revision, the check
169	* should be present to catch weird cases of someone not using the store() or general badness.
170	*/
171	if (storedRevision == null) {	1✔
172	LOGGER.log(Level.FINER, "no stored revision in annotation cache for ''{0}''", file);	×
173	} else if (!storedRevision.equals(latestRevision)) {	1✔
174	LOGGER.log(Level.FINER,	1✔
175	"stored revision {0} for ''{1}'' does not match latest revision {2}",
176	new Object[]{storedRevision, file, rev});
177	} else {
178	// read from the cache
179	annotation = readAnnotation(file);	1✔
180	}
181	}
182
183	if (annotation != null) {	1✔
184	if (fileAnnotationCacheHits != null) {	1✔
185	fileAnnotationCacheHits.increment();	1✔
186	}
187	} else {
188	if (fileAnnotationCacheMisses != null) {	1✔
189	fileAnnotationCacheMisses.increment();	×
190	}
191	LOGGER.log(Level.FINEST, "annotation cache miss for ''{0}'' in revision {1}",	1✔
192	new Object[]{file, rev});
193	return null;	1✔
194	}
195
196	return annotation;	1✔
197	}
198
199	private void writeCache(AnnotationData annotationData, File outfile) throws IOException {
200	ObjectMapper mapper = new SmileMapper();	1✔
201	mapper.writeValue(outfile, annotationData);	1✔
202	}	1✔
203
204	public void store(File file, Annotation annotation) throws CacheException {
205	if (annotation.getRevision() == null \|\| annotation.getRevision().isEmpty()) {	1✔
206	throw new CacheException(String.format("annotation for ''%s'' does not contain revision", file));	×
207	}
208
209	File cacheFile;
210	try {
211	cacheFile = getCachedFile(file);	1✔
212	} catch (CacheException e) {	×
213	LOGGER.log(Level.FINER, e.getMessage());	×
214	return;	×
215	}	1✔
216
217	File dir = cacheFile.getParentFile();	1✔
218	// calling isDirectory() twice to prevent a race condition
219	if (!dir.isDirectory() && !dir.mkdirs() && !dir.isDirectory()) {	1✔
220	throw new CacheException("Unable to create cache directory '" + dir + "'.");	×
221	}
222
223	Statistics statistics = new Statistics();	1✔
224	try {
225	writeCache(annotation.annotationData, cacheFile);	1✔
226	statistics.report(LOGGER, Level.FINEST, String.format("wrote annotation for '%s'", file),	1✔
227	"cache.annotation.file.store.latency");
228	} catch (IOException e) {	×
229	LOGGER.log(Level.WARNING, "failed to write annotation to cache", e);	×
230	}	1✔
231	}	1✔
232
233	public void clear(RepositoryInfo repository) {
234	CacheUtil.clearCacheDir(repository, this);	×
235	}	×
236
237	@Override
238	public void optimize() {
239	// nothing to do
240	}	×
241
242	@Override
243	public boolean supportsRepository(Repository repository) {
244	// all repositories are supported
245	return true;	×
246	}
247
248	@Override
249	public String getCacheDirName() {
250	return ANNOTATION_CACHE_DIR_NAME;	1✔
251	}
252
253	@Override
254	public String getInfo() {
255	return getClass().getSimpleName();	1✔
256	}
257	}

oracle / opengrok / #3650

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous