25864933270

Committed 14 May 2026 02:12PM UTC coverage: 71.207% (+21.1%) from 50.078%

Build # 25864933270

Build Type

Pull #7778

github

Committed by

web-flow

Commit Message

Merge df1bcae95 into f54a3ba0c

Pull Request Pull Request #7778: Fix: close FileInputStream in loadLanguage to prevent file handle leak

Coverage Stats

3466 of 5577 branches covered (62.15%)

Branch coverage included in aggregate %.

3 of 5 new or added lines in 1 file covered. (60.0%)

10027 of 13372 relevant lines covered (74.99%)

3.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.47

/main/src/com/google/refine/commands/lang/LoadLanguageCommand.java

/*******************************************************************************
 * Copyright (C) 2018, OpenRefine contributors
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 ******************************************************************************/

package com.google.refine.commands.lang;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map.Entry;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import edu.mit.simile.butterfly.ButterflyModule;

import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.commands.Command;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;

public class LoadLanguageCommand extends Command {

    public LoadLanguageCommand() {
        super();
    }

    @Override
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        doPost(request, response);
    }

    /**
     * POST is supported but does not actually change any state so we do not add CSRF protection to it. This ensures
     * existing extensions will not have to be updated to add a CSRF token to their requests (2019-11-10)
     */

    @Override
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        String modname = request.getParameter("module");
        if (modname == null) {
            modname = "core";
        }

        // Suggested languages from request, if given...
        String[] langs = request.getParameterValues("lang");
        if (langs == null) {
            langs = new String[] {};
        }

        // Always replace suggested with user preference language, if available...
        PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
        if (ps != null) {
            String strLang = (String) ps.get("userLang");
            // If user preference language exists...
            if (!(strLang == null || strLang.isEmpty())) {

                // CORRECTOR...
                // TODO: This code may be removed sometime after the 3.7 release has been circulated.
                if ("jp".equals(strLang)) {
                    strLang = "ja";
                    ps.put("userLang", strLang);
                }
                // End CORRECTOR

                langs = new String[] { strLang };
            }
        }

        // Add default English language as least favored...
        if (langs.length == 0 || langs[langs.length - 1] != "en") {
            langs = Arrays.copyOf(langs, langs.length + 1);
            langs[langs.length - 1] = "en";
        }

        ObjectNode translations = null;
        String bestLang = null;

        // Process from least favored to best language...
        for (int i = langs.length - 1; i >= 0; i--) {
            if (langs[i] == null) continue;

            ObjectNode json = LoadLanguageCommand.loadLanguage(this.servlet, modname, langs[i]);
            if (json == null) continue;

            bestLang = langs[i];
            if (translations == null) {
                translations = json;
            } else {
                translations = LoadLanguageCommand.mergeLanguages(json, translations);
            }
        }

        if (translations != null) {
            try {
                ObjectNode node = ParsingUtilities.mapper.createObjectNode();
                node.set("dictionary", translations);
                node.set("lang", new TextNode(bestLang));
                Command.respondJSON(response, node);
            } catch (IOException e) {
                logger.error("Error writing language labels to response stream");
                Command.respondException(response, e);
            }
        } else {
            logger.error("Failed to load any language files");
            Command.respondException(response, new IOException("No language files"));
        }
    }

    static ObjectNode loadLanguage(RefineServlet servlet, String strModule, String strLang)
            throws UnsupportedEncodingException {
        ButterflyModule module = servlet.getModule(strModule);
        String strLangFile = "translation-" + strLang + ".json";
        String strMessage = "[" + strModule + ":" + strLangFile + "]";
        File langsDir = new File(module.getPath(), "langs");
        File langFile = new File(langsDir, strLangFile);
        if (!langFile.toPath().normalize().toAbsolutePath().startsWith(langsDir.toPath().normalize().toAbsolutePath())) {
            logger.error("Security: Attempt to escape the langs directory to read another file");
            return null;
        }
        try (FileInputStream fisLang = new FileInputStream(langFile);
                Reader reader = new BufferedReader(new InputStreamReader(fisLang, "UTF-8"))) {
            return ParsingUtilities.mapper.readValue(reader, ObjectNode.class);
        } catch (FileNotFoundException e) {
            // Could be normal if we've got a list of languages as fallbacks
            logger.info("Language file " + strMessage + " not found");
            logger.debug("Exception details: " + e.getMessage());
        } catch (SecurityException e) {
            logger.error("Language file " + strMessage + " cannot be read (security)", e);
        } catch (Exception e) {
            logger.error("Language file " + strMessage + " cannot be read (io)", e);
        }
        return null;
    }

    /**
     * Update the language content to the preferred language, server-side
     * 
     * @param preferred
     *            the JSON translation for the preferred language
     * @param fallback
     *            the JSON translation for the fallback language
     * @return a JSON object where values are from the preferred language if available, and the fallback language
     *         otherwise
     */
    static ObjectNode mergeLanguages(ObjectNode preferred, ObjectNode fallback) {
        ObjectNode results = ParsingUtilities.mapper.createObjectNode();
        Iterator<Entry<String, JsonNode>> iterator = fallback.fields();
        while (iterator.hasNext()) {
            Entry<String, JsonNode> entry = iterator.next();
            String code = entry.getKey();
            JsonNode value = preferred.get(code); // ...new value
            if (value == null) {
                value = entry.getValue(); // ...reuse existing value
            }
            results.set(code, value);
        }
        return results;
    }
}

1	/*******************************************************************************
2	* Copyright (C) 2018, OpenRefine contributors
3	* All rights reserved.
4	*
5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions are met:
7	*
8	* 1. Redistributions of source code must retain the above copyright notice,
9	* this list of conditions and the following disclaimer.
10	*
11	* 2. Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	*
15	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
19	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25	* POSSIBILITY OF SUCH DAMAGE.
26	******************************************************************************/
27
28	package com.google.refine.commands.lang;
29
30	import java.io.BufferedReader;
31	import java.io.File;
32	import java.io.FileInputStream;
33	import java.io.FileNotFoundException;
34	import java.io.IOException;
35	import java.io.InputStreamReader;
36	import java.io.Reader;
37	import java.io.UnsupportedEncodingException;
38	import java.util.Arrays;
39	import java.util.Iterator;
40	import java.util.Map.Entry;
41
42	import javax.servlet.ServletException;
43	import javax.servlet.http.HttpServletRequest;
44	import javax.servlet.http.HttpServletResponse;
45
46	import com.fasterxml.jackson.databind.JsonNode;
47	import com.fasterxml.jackson.databind.node.ObjectNode;
48	import com.fasterxml.jackson.databind.node.TextNode;
49	import edu.mit.simile.butterfly.ButterflyModule;
50
51	import com.google.refine.ProjectManager;
52	import com.google.refine.RefineServlet;
53	import com.google.refine.commands.Command;
54	import com.google.refine.preference.PreferenceStore;
55	import com.google.refine.util.ParsingUtilities;
56
57	public class LoadLanguageCommand extends Command {
58
59	public LoadLanguageCommand() {
60	super();	2✔
61	}	1✔
62
63	@Override
64	public void doGet(HttpServletRequest request, HttpServletResponse response)
65	throws ServletException, IOException {
66	doPost(request, response);	×
67	}	×
68
69	/**
70	* POST is supported but does not actually change any state so we do not add CSRF protection to it. This ensures
71	* existing extensions will not have to be updated to add a CSRF token to their requests (2019-11-10)
72	*/
73
74	@Override
75	public void doPost(HttpServletRequest request, HttpServletResponse response)
76	throws ServletException, IOException {
77	String modname = request.getParameter("module");	4✔
78	if (modname == null) {	2!
79	modname = "core";	×
80	}
81
82	// Suggested languages from request, if given...
83	String[] langs = request.getParameterValues("lang");	4✔
84	if (langs == null) {	2✔
85	langs = new String[] {};	3✔
86	}
87
88	// Always replace suggested with user preference language, if available...
89	PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();	3✔
90	if (ps != null) {	2!
91	String strLang = (String) ps.get("userLang");	5✔
92	// If user preference language exists...
93	if (!(strLang == null \|\| strLang.isEmpty())) {	2!
94
95	// CORRECTOR...
96	// TODO: This code may be removed sometime after the 3.7 release has been circulated.
97	if ("jp".equals(strLang)) {	×
98	strLang = "ja";	×
99	ps.put("userLang", strLang);	×
100	}
101	// End CORRECTOR
102
103	langs = new String[] { strLang };	×
104	}
105	}
106
107	// Add default English language as least favored...
108	if (langs.length == 0 \|\| langs[langs.length - 1] != "en") {	11!
109	langs = Arrays.copyOf(langs, langs.length + 1);	8✔
110	langs[langs.length - 1] = "en";	7✔
111	}
112
113	ObjectNode translations = null;	2✔
114	String bestLang = null;	2✔
115
116	// Process from least favored to best language...
117	for (int i = langs.length - 1; i >= 0; i--) {	9✔
118	if (langs[i] == null) continue;	4!
119
120	ObjectNode json = LoadLanguageCommand.loadLanguage(this.servlet, modname, langs[i]);	8✔
121	if (json == null) continue;	3✔
122
123	bestLang = langs[i];	4✔
124	if (translations == null) {	2✔
125	translations = json;	3✔
126	} else {
127	translations = LoadLanguageCommand.mergeLanguages(json, translations);	4✔
128	}
129	}
130
131	if (translations != null) {	2!
132	try {
133	ObjectNode node = ParsingUtilities.mapper.createObjectNode();	3✔
134	node.set("dictionary", translations);	5✔
135	node.set("lang", new TextNode(bestLang));	8✔
136	Command.respondJSON(response, node);	3✔
137	} catch (IOException e) {	×
138	logger.error("Error writing language labels to response stream");	×
139	Command.respondException(response, e);	×
140	}	1✔
141	} else {
142	logger.error("Failed to load any language files");	×
143	Command.respondException(response, new IOException("No language files"));	×
144	}
145	}	1✔
146
147	static ObjectNode loadLanguage(RefineServlet servlet, String strModule, String strLang)
148	throws UnsupportedEncodingException {
149	ButterflyModule module = servlet.getModule(strModule);	4✔
150	String strLangFile = "translation-" + strLang + ".json";	3✔
151	String strMessage = "[" + strModule + ":" + strLangFile + "]";	4✔
152	File langsDir = new File(module.getPath(), "langs");	7✔
153	File langFile = new File(langsDir, strLangFile);	6✔
154	if (!langFile.toPath().normalize().toAbsolutePath().startsWith(langsDir.toPath().normalize().toAbsolutePath())) {	10✔
155	logger.error("Security: Attempt to escape the langs directory to read another file");	3✔
156	return null;	2✔
157	}
158	try (FileInputStream fisLang = new FileInputStream(langFile);	5✔
159	Reader reader = new BufferedReader(new InputStreamReader(fisLang, "UTF-8"))) {	9✔
160	return ParsingUtilities.mapper.readValue(reader, ObjectNode.class);	8✔
161	} catch (FileNotFoundException e) {	1✔
162	// Could be normal if we've got a list of languages as fallbacks
163	logger.info("Language file " + strMessage + " not found");	4✔
164	logger.debug("Exception details: " + e.getMessage());	5✔
165	} catch (SecurityException e) {	×
166	logger.error("Language file " + strMessage + " cannot be read (security)", e);	×
NEW 167	} catch (Exception e) {	×
NEW 168	logger.error("Language file " + strMessage + " cannot be read (io)", e);	×
169	}	1✔
170	return null;	2✔
171	}
172
173	/**
174	* Update the language content to the preferred language, server-side
175	*
176	* @param preferred
177	* the JSON translation for the preferred language
178	* @param fallback
179	* the JSON translation for the fallback language
180	* @return a JSON object where values are from the preferred language if available, and the fallback language
181	* otherwise
182	*/
183	static ObjectNode mergeLanguages(ObjectNode preferred, ObjectNode fallback) {
184	ObjectNode results = ParsingUtilities.mapper.createObjectNode();	3✔
185	Iterator<Entry<String, JsonNode>> iterator = fallback.fields();	3✔
186	while (iterator.hasNext()) {	3✔
187	Entry<String, JsonNode> entry = iterator.next();	4✔
188	String code = entry.getKey();	4✔
189	JsonNode value = preferred.get(code); // ...new value	4✔
190	if (value == null) {	2✔
191	value = entry.getValue(); // ...reuse existing value	4✔
192	}
193	results.set(code, value);	5✔
194	}	1✔
195	return results;	2✔
196	}
197	}

OpenRefine / OpenRefine / 25864933270

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous