3169

pending completion

Build # 3169

Build Type

Pull #929

circleci

Committed by

stefan-kolb

Commit Message

<a href="https://github.com/JabRef/jabref/commit/<a class=hub.com/JabRef/jabref/commit/<a class="double-link" href="https://git"><a class=hub.com/JabRef/jabref/commit/<a class="double-link" href="https://git"><a class=hub.com/JabRef/jabref/commit/<a class="double-link" href="https://git"><a class=hub.com/JabRef/jabref/commit/943e5a8eec712184eebf3dd3a029dd7a0aef447d">943e5a8ee">&lt;a href=&quot;https://github.com/JabRef/jabref/commit/</a><a class="double-link" href="https://github.com/JabRef/jabref/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JabRef/jabref/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JabRef/jabref/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JabRef/jabref/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JabRef/jabref/commit/943e5a8eec712184eebf3dd3a029dd7a0aef447d">943e5a8ee</a><a href="https://github.com/JabRef/jabref/commit/943e5a8eec712184eebf3dd3a029dd7a0aef447d">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JabRef/jabref/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JabRef/jabref/commit/943e5a8eec712184eebf3dd3a029dd7a0aef447d&quot;&gt;943e5a8ee&lt;/a&gt;&lt;a href=&quot;https://github.com/JabRef/jabref/commit/943e5a8eec712184eebf3dd3a029dd7a0aef447d&quot;&gt;&amp;quot;&amp;g... (continued)

Pull Request Pull Request #929: "Find unlinked files" freezes when using for PDF

Run Details

50 of 50 new or added lines in 2 files covered. (100.0%)

12163 of 47524 relevant lines covered (25.59%)

0.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/net/sf/jabref/importer/fileformat/PdfContentImporter.java

package net.sf.jabref.importer.fileformat;

import net.sf.jabref.importer.ImportInspector;
import net.sf.jabref.importer.OutputPrinter;
import net.sf.jabref.importer.fetcher.DOItoBibTeXFetcher;
import net.sf.jabref.logic.util.DOI;
import net.sf.jabref.model.entry.BibEntry;
import net.sf.jabref.model.entry.BibtexEntryTypes;
import net.sf.jabref.model.entry.EntryType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.util.PDFTextStripper;

import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * PdfContentImporter parses data of the first page of the PDF and creates a BibTeX entry.
 * <p>
 * Currently, Springer and IEEE formats are supported.
 * <p>
 * Integrating XMP support is future work
 */
public class PdfContentImporter extends ImportFormat {
    private static final Log LOGGER = LogFactory.getLog(PdfContentImporter.class);

    private static final Pattern YEAR_EXTRACT_PATTERN = Pattern.compile("\\d{4}");

    private static final DOItoBibTeXFetcher doiToBibTeXFetcher = new DOItoBibTeXFetcher();

    // input paragraphs into several lines
    private String[] paragraphs;

    // current index in paragraphs
    private int i;

    // curent "line" in paragraphs.
    // sometimes, a "line" is several lines in paragraphs
    private String curString;

    private String year;

    @Override
    public boolean isRecognizedFormat(InputStream in) throws IOException {
        return false;
    }

    /**
     * Removes all non-letter characters at the end
     * <p>
     * EXCEPTION: a closing bracket is NOT removed
     *
     * @param input
     * @return TODO Additionally replace multiple subsequent spaces by one space
     */
    private static String removeNonLettersAtEnd(String input) {
        input = input.trim();
        if (input.isEmpty()) {
            return input;
        }
        char lastC = input.charAt(input.length() - 1);
        while (!Character.isLetter(lastC) && (lastC != ')')) {
            // if there is an asterix, a dot or something else at the end: remove it
            input = input.substring(0, input.length() - 1);
            if (input.isEmpty()) {
                break;
            } else {
                lastC = input.charAt(input.length() - 1);
            }
        }
        return input;
    }

    private static String streamlineNames(String names) {
        // TODO: replace with AuthorsFormatter?!
        String res;
        // supported formats:
        //   Matthias Schrepfer1, Johannes Wolf1, Jan Mendling1, and Hajo A. Reijers2
        if (names.contains(",")) {
            String[] splitNames = names.split(",");
            res = "";
            boolean isFirst = true;
            for (String splitName : splitNames) {
                String curName = removeNonLettersAtEnd(splitName);
                if (curName.indexOf("and") == 0) {
                    // skip possible ands between names
                    curName = curName.substring(3).trim();
                } else {
                    int posAnd = curName.indexOf(" and ");
                    if (posAnd >= 0) {
                        String nameBefore = curName.substring(0, posAnd);
                        // cannot be first name as "," is contained in the string
                        res = res.concat(" and ").concat(removeNonLettersAtEnd(nameBefore));
                        curName = curName.substring(posAnd + 5);
                    }
                }

                if (!"".equals(curName)) {
                    if ("et al.".equalsIgnoreCase(curName)) {
                        curName = "others";
                    }
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        res = res.concat(" and ");
                    }
                    res = res.concat(curName);
                }
            }
        } else {
            // assumption: names separated by space

            String[] splitNames = names.split(" ");
            if (splitNames.length == 0) {
                // empty names... something was really wrong...
                return "";
            }

            boolean workedOnFirstOrMiddle = false;
            boolean isFirst = true;
            int i = 0;
            res = "";
            do {
                if (workedOnFirstOrMiddle) {
                    // last item was a first or a middle name
                    // we have to check whether we are on a middle name
                    // if not, just add the item as last name and add an "and"
                    if (splitNames[i].contains(".")) {
                        // we found a middle name
                        res = res.concat(splitNames[i]).concat(" ");
                    } else {
                        // last name found
                        res = res.concat(removeNonLettersAtEnd(splitNames[i]));

                        if (!splitNames[i].isEmpty() && Character.isLowerCase(splitNames[i].charAt(0))) {
                            // it is probably be "van", "vom", ...
                            // we just rely on the fact that these things are written in lower case letters
                            // do NOT finish name
                            res = res.concat(" ");
                        } else {
                            // finish this name
                            workedOnFirstOrMiddle = false;
                        }
                    }
                } else {
                    if ("and".equalsIgnoreCase(splitNames[i])) {
                        // do nothing, just increment i at the end of this iteration
                    } else {
                        if (isFirst) {
                            isFirst = false;
                        } else {
                            res = res.concat(" and ");
                        }
                        if ("et".equalsIgnoreCase(splitNames[i]) && (splitNames.length > (i + 1))
                                && "al.".equalsIgnoreCase(splitNames[i + 1])) {
                            res = res.concat("others");
                            break;
                        } else {
                            res = res.concat(splitNames[i]).concat(" ");
                            workedOnFirstOrMiddle = true;
                        }
                    }
                }
                i++;
            } while (i < splitNames.length);
        }
        return res;
    }

    private static String streamlineTitle(String title) {
        return removeNonLettersAtEnd(title);
    }

    @Override
    public List<BibEntry> importEntries(InputStream in, OutputPrinter status) throws IOException {
        final ArrayList<BibEntry> result = new ArrayList<>(1);

        try (PDDocument document = PDDocument.load(in)) {
            if (document.isEncrypted()) {
                LOGGER.error("Encrypted documents are not supported");
                return result;
            }

            String firstPageContents = getFirstPageContents(document);

            Optional<DOI> doi = DOI.findInText(firstPageContents);
            if (doi.isPresent()) {
                ImportInspector inspector = new ImportInspector() {

                    @Override
                    public void toFront() {
                        // Do nothing
                    }

                    @Override
                    public void setProgress(int current, int max) {
                        // Do nothing
                    }

                    @Override
                    public void addEntry(BibEntry entry) {
                        // add the entry to the result object
                        result.add(entry);
                    }
                };

                doiToBibTeXFetcher.processQuery(doi.get().getDOI(), inspector, status);
                if (!result.isEmpty()) {
                    return result;
                }
            }

            // idea: split[] contains the different lines
            // blocks are separated by empty lines
            // treat each block
            //   or do special treatment at authors (which are not broken)
            //   therefore, we do a line-based and not a block-based splitting
            // i points to the current line
            // curString (mostly) contains the current block
            //   the different lines are joined into one and thereby separated by " "
            paragraphs = firstPageContents.split(System.lineSeparator());

            proceedToNextNonEmptyLine();
            if (i >= paragraphs.length) {
                // PDF could not be parsed or is empty
                // return empty list
                return result;
            }

            curString = paragraphs[i];
            i = i + 1;

            String author;
            String editor = null;
            String abstractT = null;
            String keywords = null;
            String title;
            String conference = null;
            String DOI = null;
            String series = null;
            String volume = null;
            String number = null;
            String pages = null;
            // year is a class variable as the method extractYear() uses it;
            String publisher = null;

            EntryType type = BibtexEntryTypes.INPROCEEDINGS;
            if (curString.length() > 4) {
                // special case: possibly conference as first line on the page
                extractYear();
                if (curString.contains("Conference")) {
                    fillCurStringWithNonEmptyLines();
                    conference = curString;
                    curString = "";
                } else {
                    // e.g. Copyright (c) 1998 by the Genetics Society of America
                    // future work: get year using RegEx
                    String lower = curString.toLowerCase();
                    if (lower.contains("copyright")) {
                        fillCurStringWithNonEmptyLines();
                        publisher = curString;
                        curString = "";
                    }
                }
            }

            // start: title
            fillCurStringWithNonEmptyLines();
            title = streamlineTitle(curString);
            curString = "";
            //i points to the next non-empty line

            // after title: authors
            author = null;
            while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {
                // author names are unlikely to be paragraphs among different lines
                // treat them line by line
                curString = streamlineNames(paragraphs[i]);
                if (author == null) {
                    author = curString;
                } else {
                    if ("".equals(curString)) {
                        // if paragraphs[i] is "and" then "" is returned by streamlineNames -> do nothing
                    } else {
                        author = author.concat(" and ").concat(curString);
                    }
                }
                i++;
            }
            curString = "";
            i++;

            // then, abstract and keywords follow
            while (i < paragraphs.length) {
                curString = paragraphs[i];
                if ((curString.length() >= "Abstract".length()) && "Abstract".equalsIgnoreCase(curString.substring(0, "Abstract".length()))) {
                    if (curString.length() == "Abstract".length()) {
                        // only word "abstract" found -- skip line
                        curString = "";
                    } else {
                        curString = curString.substring("Abstract".length() + 1).trim().concat(System.lineSeparator());
                    }
                    i++;
                    // fillCurStringWithNonEmptyLines() cannot be used as that uses " " as line separator
                    // whereas we need linebreak as separator
                    while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {
                        curString = curString.concat(paragraphs[i]).concat(System.lineSeparator());
                        i++;
                    }
                    abstractT = curString;
                    i++;
                } else if ((curString.length() >= "Keywords".length()) && "Keywords".equalsIgnoreCase(curString.substring(0, "Keywords".length()))) {
                    if (curString.length() == "Keywords".length()) {
                        // only word "Keywords" found -- skip line
                        curString = "";
                    } else {
                        curString = curString.substring("Keywords".length() + 1).trim();
                    }
                    i++;
                    fillCurStringWithNonEmptyLines();
                    keywords = removeNonLettersAtEnd(curString);
                } else {
                    String lower = curString.toLowerCase();

                    int pos = lower.indexOf("technical");
                    if (pos >= 0) {
                        type = BibtexEntryTypes.TECHREPORT;
                        pos = curString.trim().lastIndexOf(' ');
                        if (pos >= 0) {
                            // assumption: last character of curString is NOT ' '
                            //   otherwise pos+1 leads to an out-of-bounds exception
                            number = curString.substring(pos + 1);
                        }
                    }

                    i++;
                    proceedToNextNonEmptyLine();
                }
            }

            i = paragraphs.length - 1;

            // last block: DOI, detailed information
            // sometimes, this information is in the third last block etc...
            // therefore, read until the beginning of the file

            while (i >= 0) {
                readLastBlock();
                // i now points to the block before or is -1
                // curString contains the last block, separated by " "

                extractYear();

                int pos = curString.indexOf("(Eds.)");
                if ((pos >= 0) && (publisher == null)) {
                    // looks like a Springer last line
                    // e.g: A. Persson and J. Stirna (Eds.): PoEM 2009, LNBIP 39, pp. 161-175, 2009.
                    publisher = "Springer";
                    editor = streamlineNames(curString.substring(0, pos - 1));
                    curString = curString.substring(pos + "(Eds.)".length() + 2); //+2 because of ":" after (Eds.) and the subsequent space
                    String[] springerSplit = curString.split(", ");
                    if (springerSplit.length >= 4) {
                        conference = springerSplit[0];

                        String seriesData = springerSplit[1];
                        int lastSpace = seriesData.lastIndexOf(' ');
                        series = seriesData.substring(0, lastSpace);
                        volume = seriesData.substring(lastSpace + 1);

                        pages = springerSplit[2].substring(4);

                        if (springerSplit[3].length() >= 4) {
                            year = springerSplit[3].substring(0, 4);
                        }
                    }
                } else {
                    if (DOI == null) {
                        pos = curString.indexOf("DOI");
                        if (pos < 0) {
                            pos = curString.indexOf("doi");
                        }
                        if (pos >= 0) {
                            pos += 3;
                            char delimiter = curString.charAt(pos);
                            if ((delimiter == ':') || (delimiter == ' ')) {
                                pos++;
                            }
                            int nextSpace = curString.indexOf(' ', pos);
                            if (nextSpace > 0) {
                                DOI = curString.substring(pos, nextSpace);
                            } else {
                                DOI = curString.substring(pos);
                            }
                        }
                    }

                    if ((publisher == null) && curString.contains("IEEE")) {
                        // IEEE has the conference things at the end
                        publisher = "IEEE";

                        // year is extracted by extractYear
                        // otherwise, we could it determine as follows:
                        // String yearStr = curString.substring(curString.length()-4);
                        // if (isYear(yearStr)) {
                        //        year = yearStr;
                        // }

                        if (conference == null) {
                            pos = curString.indexOf('$');
                            if (pos > 0) {
                                // we found the price
                                // before the price, the ISSN is stated
                                // skip that
                                pos -= 2;
                                while ((pos >= 0) && (curString.charAt(pos) != ' ')) {
                                    pos--;
                                }
                                if (pos > 0) {
                                    conference = curString.substring(0, pos);
                                }
                            }
                        }
                    }
                }
            }

            BibEntry entry = new BibEntry();
            entry.setType(type);

            if (author != null) {
                entry.setField("author", author);
            }
            if (editor != null) {
                entry.setField("editor", editor);
            }
            if (abstractT != null) {
                entry.setField("abstract", abstractT);
            }
            if (keywords != null) {
                entry.setField("keywords", keywords);
            }
            if (title != null) {
                entry.setField("title", title);
            }
            if (conference != null) {
                entry.setField("booktitle", conference);
            }
            if (DOI != null) {
                entry.setField("doi", DOI);
            }
            if (series != null) {
                entry.setField("series", series);
            }
            if (volume != null) {
                entry.setField("volume", volume);
            }
            if (number != null) {
                entry.setField("number", number);
            }
            if (pages != null) {
                entry.setField("pages", pages);
            }
            if (year != null) {
                entry.setField("year", year);
            }
            if (publisher != null) {
                entry.setField("publisher", publisher);
            }

            entry.setField("review", firstPageContents);

            result.add(entry);
        } catch (IOException e) {
            LOGGER.error("Could not load document", e);
        }
        return result;
    }

    private String getFirstPageContents(PDDocument document) throws IOException {
        PDFTextStripper stripper = new PDFTextStripper();

        stripper.setStartPage(1);
        stripper.setEndPage(1);
        stripper.setSortByPosition(true);
        stripper.setParagraphEnd(System.lineSeparator());
        StringWriter writer = new StringWriter();
        stripper.writeText(document, writer);

        return writer.toString();
    }

    /**
     * Extract the year out of curString (if it is not yet defined)
     */
    private void extractYear() {
        if (year != null) {
            return;
        }

        Matcher m = YEAR_EXTRACT_PATTERN.matcher(curString);
        if (m.find()) {
            year = curString.substring(m.start(), m.end());
        }

    }

    /**
     * PDFTextStripper normally does NOT produce multiple empty lines
     * (besides at strange PDFs). These strange PDFs are handled here:
     * proceed to next non-empty line
     */
    private void proceedToNextNonEmptyLine() {
        while ((i < paragraphs.length) && "".equals(paragraphs[i].trim())) {
            i++;
        }
    }

    /**
     * Fill curString with lines until "" is found
     * No trailing space is added
     * i is advanced to the next non-empty line (ignoring white space)
     * <p>
     * Lines containing only white spaces are ignored,
     * but NOT considered as ""
     * <p>
     * Uses GLOBAL variables paragraphs, curLine, i
     */
    private void fillCurStringWithNonEmptyLines() {
        // ensure that curString does not end with " "
        curString = curString.trim();
        while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {
            String curLine = paragraphs[i].trim();
            if (!"".equals(curLine)) {
                if (!curString.isEmpty()) {
                    // insert separating space if necessary
                    curString = curString.concat(" ");
                }
                curString = curString.concat(paragraphs[i]);
            }
            i++;
        }

        proceedToNextNonEmptyLine();
    }

    /**
     * resets curString
     * curString now contains the last block (until "" reached)
     * Trailing space is added
     * <p>
     * invariant before/after: i points to line before the last handled block
     */
    private void readLastBlock() {
        while ((i >= 0) && "".equals(paragraphs[i].trim())) {
            i--;
        }
        // i is now at the end of a block

        int end = i;

        // find beginning
        while ((i >= 0) && !"".equals(paragraphs[i])) {
            i--;
        }
        // i is now the line before the beginning of the block
        // this fulfills the invariant

        curString = "";
        for (int j = i + 1; j <= end; j++) {
            curString = curString.concat(paragraphs[j].trim());
            if (j != end) {
                curString = curString.concat(" ");
            }
        }
    }

    @Override
    public String getFormatName() {
        return "PDFcontent";
    }

}

1	package net.sf.jabref.importer.fileformat;
2
3	import net.sf.jabref.importer.ImportInspector;
4	import net.sf.jabref.importer.OutputPrinter;
5	import net.sf.jabref.importer.fetcher.DOItoBibTeXFetcher;
6	import net.sf.jabref.logic.util.DOI;
7	import net.sf.jabref.model.entry.BibEntry;
8	import net.sf.jabref.model.entry.BibtexEntryTypes;
9	import net.sf.jabref.model.entry.EntryType;
10	import org.apache.commons.logging.Log;
11	import org.apache.commons.logging.LogFactory;
12	import org.apache.pdfbox.pdmodel.PDDocument;
13	import org.apache.pdfbox.util.PDFTextStripper;
14
15	import java.io.IOException;
16	import java.io.InputStream;
17	import java.io.StringWriter;
18	import java.util.ArrayList;
19	import java.util.List;
20	import java.util.Optional;
21	import java.util.regex.Matcher;
22	import java.util.regex.Pattern;
23
24	/**
25	* PdfContentImporter parses data of the first page of the PDF and creates a BibTeX entry.
26	* <p>
27	* Currently, Springer and IEEE formats are supported.
28	* <p>
29	* Integrating XMP support is future work
30	*/
31	public class PdfContentImporter extends ImportFormat {	×
32	private static final Log LOGGER = LogFactory.getLog(PdfContentImporter.class);	×
33
34	private static final Pattern YEAR_EXTRACT_PATTERN = Pattern.compile("\\d{4}");	×
35
36	private static final DOItoBibTeXFetcher doiToBibTeXFetcher = new DOItoBibTeXFetcher();	×
37
38	// input paragraphs into several lines
39	private String[] paragraphs;
40
41	// current index in paragraphs
42	private int i;
43
44	// curent "line" in paragraphs.
45	// sometimes, a "line" is several lines in paragraphs
46	private String curString;
47
48	private String year;
49
50	@Override
51	public boolean isRecognizedFormat(InputStream in) throws IOException {
52	return false;	×
53	}
54
55	/**
56	* Removes all non-letter characters at the end
57	* <p>
58	* EXCEPTION: a closing bracket is NOT removed
59	*
60	* @param input
61	* @return TODO Additionally replace multiple subsequent spaces by one space
62	*/
63	private static String removeNonLettersAtEnd(String input) {
64	input = input.trim();	×
65	if (input.isEmpty()) {	×
66	return input;	×
67	}
68	char lastC = input.charAt(input.length() - 1);	×
69	while (!Character.isLetter(lastC) && (lastC != ')')) {	×
70	// if there is an asterix, a dot or something else at the end: remove it
71	input = input.substring(0, input.length() - 1);	×
72	if (input.isEmpty()) {	×
73	break;	×
74	} else {
75	lastC = input.charAt(input.length() - 1);	×
76	}
77	}
78	return input;	×
79	}
80
81	private static String streamlineNames(String names) {
82	// TODO: replace with AuthorsFormatter?!
83	String res;
84	// supported formats:
85	// Matthias Schrepfer1, Johannes Wolf1, Jan Mendling1, and Hajo A. Reijers2
86	if (names.contains(",")) {	×
87	String[] splitNames = names.split(",");	×
88	res = "";	×
89	boolean isFirst = true;	×
90	for (String splitName : splitNames) {	×
91	String curName = removeNonLettersAtEnd(splitName);	×
92	if (curName.indexOf("and") == 0) {	×
93	// skip possible ands between names
94	curName = curName.substring(3).trim();	×
95	} else {
96	int posAnd = curName.indexOf(" and ");	×
97	if (posAnd >= 0) {	×
98	String nameBefore = curName.substring(0, posAnd);	×
99	// cannot be first name as "," is contained in the string
100	res = res.concat(" and ").concat(removeNonLettersAtEnd(nameBefore));	×
101	curName = curName.substring(posAnd + 5);	×
102	}
103	}
104
105	if (!"".equals(curName)) {	×
106	if ("et al.".equalsIgnoreCase(curName)) {	×
107	curName = "others";	×
108	}
109	if (isFirst) {	×
110	isFirst = false;	×
111	} else {
112	res = res.concat(" and ");	×
113	}
114	res = res.concat(curName);	×
115	}
116	}
117	} else {	×
118	// assumption: names separated by space
119
120	String[] splitNames = names.split(" ");	×
121	if (splitNames.length == 0) {	×
122	// empty names... something was really wrong...
123	return "";	×
124	}
125
126	boolean workedOnFirstOrMiddle = false;	×
127	boolean isFirst = true;	×
128	int i = 0;	×
129	res = "";	×
130	do {
131	if (workedOnFirstOrMiddle) {	×
132	// last item was a first or a middle name
133	// we have to check whether we are on a middle name
134	// if not, just add the item as last name and add an "and"
135	if (splitNames[i].contains(".")) {	×
136	// we found a middle name
137	res = res.concat(splitNames[i]).concat(" ");	×
138	} else {
139	// last name found
140	res = res.concat(removeNonLettersAtEnd(splitNames[i]));	×
141
142	if (!splitNames[i].isEmpty() && Character.isLowerCase(splitNames[i].charAt(0))) {	×
143	// it is probably be "van", "vom", ...
144	// we just rely on the fact that these things are written in lower case letters
145	// do NOT finish name
146	res = res.concat(" ");	×
147	} else {
148	// finish this name
149	workedOnFirstOrMiddle = false;	×
150	}
151	}
152	} else {
153	if ("and".equalsIgnoreCase(splitNames[i])) {	×
154	// do nothing, just increment i at the end of this iteration
155	} else {
156	if (isFirst) {	×
157	isFirst = false;	×
158	} else {
159	res = res.concat(" and ");	×
160	}
161	if ("et".equalsIgnoreCase(splitNames[i]) && (splitNames.length > (i + 1))	×
162	&& "al.".equalsIgnoreCase(splitNames[i + 1])) {	×
163	res = res.concat("others");	×
164	break;	×
165	} else {
166	res = res.concat(splitNames[i]).concat(" ");	×
167	workedOnFirstOrMiddle = true;	×
168	}
169	}
170	}
171	i++;	×
172	} while (i < splitNames.length);	×
173	}
174	return res;	×
175	}
176
177	private static String streamlineTitle(String title) {
178	return removeNonLettersAtEnd(title);	×
179	}
180
181	@Override
182	public List<BibEntry> importEntries(InputStream in, OutputPrinter status) throws IOException {
183	final ArrayList<BibEntry> result = new ArrayList<>(1);	×
184
185	try (PDDocument document = PDDocument.load(in)) {	×
186	if (document.isEncrypted()) {	×
187	LOGGER.error("Encrypted documents are not supported");	×
188	return result;	×
189	}
190
191	String firstPageContents = getFirstPageContents(document);	×
192
193	Optional<DOI> doi = DOI.findInText(firstPageContents);	×
194	if (doi.isPresent()) {	×
195	ImportInspector inspector = new ImportInspector() {	×
196
197	@Override
198	public void toFront() {
199	// Do nothing
200	}	×
201
202	@Override
203	public void setProgress(int current, int max) {
204	// Do nothing
205	}	×
206
207	@Override
208	public void addEntry(BibEntry entry) {
209	// add the entry to the result object
210	result.add(entry);	×
211	}	×
212	};
213
214	doiToBibTeXFetcher.processQuery(doi.get().getDOI(), inspector, status);	×
215	if (!result.isEmpty()) {	×
216	return result;	×
217	}
218	}
219
220	// idea: split[] contains the different lines
221	// blocks are separated by empty lines
222	// treat each block
223	// or do special treatment at authors (which are not broken)
224	// therefore, we do a line-based and not a block-based splitting
225	// i points to the current line
226	// curString (mostly) contains the current block
227	// the different lines are joined into one and thereby separated by " "
228	paragraphs = firstPageContents.split(System.lineSeparator());	×
229
230	proceedToNextNonEmptyLine();	×
231	if (i >= paragraphs.length) {	×
232	// PDF could not be parsed or is empty
233	// return empty list
234	return result;	×
235	}
236
237	curString = paragraphs[i];	×
238	i = i + 1;	×
239
240	String author;
241	String editor = null;	×
242	String abstractT = null;	×
243	String keywords = null;	×
244	String title;
245	String conference = null;	×
246	String DOI = null;	×
247	String series = null;	×
248	String volume = null;	×
249	String number = null;	×
250	String pages = null;	×
251	// year is a class variable as the method extractYear() uses it;
252	String publisher = null;	×
253
254	EntryType type = BibtexEntryTypes.INPROCEEDINGS;	×
255	if (curString.length() > 4) {	×
256	// special case: possibly conference as first line on the page
257	extractYear();	×
258	if (curString.contains("Conference")) {	×
259	fillCurStringWithNonEmptyLines();	×
260	conference = curString;	×
261	curString = "";	×
262	} else {
263	// e.g. Copyright (c) 1998 by the Genetics Society of America
264	// future work: get year using RegEx
265	String lower = curString.toLowerCase();	×
266	if (lower.contains("copyright")) {	×
267	fillCurStringWithNonEmptyLines();	×
268	publisher = curString;	×
269	curString = "";	×
270	}
271	}
272	}
273
274	// start: title
275	fillCurStringWithNonEmptyLines();	×
276	title = streamlineTitle(curString);	×
277	curString = "";	×
278	//i points to the next non-empty line
279
280	// after title: authors
281	author = null;	×
282	while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {	×
283	// author names are unlikely to be paragraphs among different lines
284	// treat them line by line
285	curString = streamlineNames(paragraphs[i]);	×
286	if (author == null) {	×
287	author = curString;	×
288	} else {
289	if ("".equals(curString)) {	×
290	// if paragraphs[i] is "and" then "" is returned by streamlineNames -> do nothing
291	} else {
292	author = author.concat(" and ").concat(curString);	×
293	}
294	}
295	i++;	×
296	}
297	curString = "";	×
298	i++;	×
299
300	// then, abstract and keywords follow
301	while (i < paragraphs.length) {	×
302	curString = paragraphs[i];	×
303	if ((curString.length() >= "Abstract".length()) && "Abstract".equalsIgnoreCase(curString.substring(0, "Abstract".length()))) {	×
304	if (curString.length() == "Abstract".length()) {	×
305	// only word "abstract" found -- skip line
306	curString = "";	×
307	} else {
308	curString = curString.substring("Abstract".length() + 1).trim().concat(System.lineSeparator());	×
309	}
310	i++;	×
311	// fillCurStringWithNonEmptyLines() cannot be used as that uses " " as line separator
312	// whereas we need linebreak as separator
313	while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {	×
314	curString = curString.concat(paragraphs[i]).concat(System.lineSeparator());	×
315	i++;	×
316	}
317	abstractT = curString;	×
318	i++;	×
319	} else if ((curString.length() >= "Keywords".length()) && "Keywords".equalsIgnoreCase(curString.substring(0, "Keywords".length()))) {	×
320	if (curString.length() == "Keywords".length()) {	×
321	// only word "Keywords" found -- skip line
322	curString = "";	×
323	} else {
324	curString = curString.substring("Keywords".length() + 1).trim();	×
325	}
326	i++;	×
327	fillCurStringWithNonEmptyLines();	×
328	keywords = removeNonLettersAtEnd(curString);	×
329	} else {
330	String lower = curString.toLowerCase();	×
331
332	int pos = lower.indexOf("technical");	×
333	if (pos >= 0) {	×
334	type = BibtexEntryTypes.TECHREPORT;	×
335	pos = curString.trim().lastIndexOf(' ');	×
336	if (pos >= 0) {	×
337	// assumption: last character of curString is NOT ' '
338	// otherwise pos+1 leads to an out-of-bounds exception
339	number = curString.substring(pos + 1);	×
340	}
341	}
342
343	i++;	×
344	proceedToNextNonEmptyLine();	×
345	}	×
346	}
347
348	i = paragraphs.length - 1;	×
349
350	// last block: DOI, detailed information
351	// sometimes, this information is in the third last block etc...
352	// therefore, read until the beginning of the file
353
354	while (i >= 0) {	×
355	readLastBlock();	×
356	// i now points to the block before or is -1
357	// curString contains the last block, separated by " "
358
359	extractYear();	×
360
361	int pos = curString.indexOf("(Eds.)");	×
362	if ((pos >= 0) && (publisher == null)) {	×
363	// looks like a Springer last line
364	// e.g: A. Persson and J. Stirna (Eds.): PoEM 2009, LNBIP 39, pp. 161-175, 2009.
365	publisher = "Springer";	×
366	editor = streamlineNames(curString.substring(0, pos - 1));	×
367	curString = curString.substring(pos + "(Eds.)".length() + 2); //+2 because of ":" after (Eds.) and the subsequent space	×
368	String[] springerSplit = curString.split(", ");	×
369	if (springerSplit.length >= 4) {	×
370	conference = springerSplit[0];	×
371
372	String seriesData = springerSplit[1];	×
373	int lastSpace = seriesData.lastIndexOf(' ');	×
374	series = seriesData.substring(0, lastSpace);	×
375	volume = seriesData.substring(lastSpace + 1);	×
376
377	pages = springerSplit[2].substring(4);	×
378
379	if (springerSplit[3].length() >= 4) {	×
380	year = springerSplit[3].substring(0, 4);	×
381	}
382	}
383	} else {	×
384	if (DOI == null) {	×
385	pos = curString.indexOf("DOI");	×
386	if (pos < 0) {	×
387	pos = curString.indexOf("doi");	×
388	}
389	if (pos >= 0) {	×
390	pos += 3;	×
391	char delimiter = curString.charAt(pos);	×
392	if ((delimiter == ':') \|\| (delimiter == ' ')) {	×
393	pos++;	×
394	}
395	int nextSpace = curString.indexOf(' ', pos);	×
396	if (nextSpace > 0) {	×
397	DOI = curString.substring(pos, nextSpace);	×
398	} else {
399	DOI = curString.substring(pos);	×
400	}
401	}
402	}
403
404	if ((publisher == null) && curString.contains("IEEE")) {	×
405	// IEEE has the conference things at the end
406	publisher = "IEEE";	×
407
408	// year is extracted by extractYear
409	// otherwise, we could it determine as follows:
410	// String yearStr = curString.substring(curString.length()-4);
411	// if (isYear(yearStr)) {
412	// year = yearStr;
413	// }
414
415	if (conference == null) {	×
416	pos = curString.indexOf('$');	×
417	if (pos > 0) {	×
418	// we found the price
419	// before the price, the ISSN is stated
420	// skip that
421	pos -= 2;	×
422	while ((pos >= 0) && (curString.charAt(pos) != ' ')) {	×
423	pos--;	×
424	}
425	if (pos > 0) {	×
426	conference = curString.substring(0, pos);	×
427	}
428	}
429	}
430	}
431	}
432	}	×
433
434	BibEntry entry = new BibEntry();	×
435	entry.setType(type);	×
436
437	if (author != null) {	×
438	entry.setField("author", author);	×
439	}
440	if (editor != null) {	×
441	entry.setField("editor", editor);	×
442	}
443	if (abstractT != null) {	×
444	entry.setField("abstract", abstractT);	×
445	}
446	if (keywords != null) {	×
447	entry.setField("keywords", keywords);	×
448	}
449	if (title != null) {	×
450	entry.setField("title", title);	×
451	}
452	if (conference != null) {	×
453	entry.setField("booktitle", conference);	×
454	}
455	if (DOI != null) {	×
456	entry.setField("doi", DOI);	×
457	}
458	if (series != null) {	×
459	entry.setField("series", series);	×
460	}
461	if (volume != null) {	×
462	entry.setField("volume", volume);	×
463	}
464	if (number != null) {	×
465	entry.setField("number", number);	×
466	}
467	if (pages != null) {	×
468	entry.setField("pages", pages);	×
469	}
470	if (year != null) {	×
471	entry.setField("year", year);	×
472	}
473	if (publisher != null) {	×
474	entry.setField("publisher", publisher);	×
475	}
476
477	entry.setField("review", firstPageContents);	×
478
479	result.add(entry);	×
480	} catch (IOException e) {	×
481	LOGGER.error("Could not load document", e);	×
482	}	×
483	return result;	×
484	}
485
486	private String getFirstPageContents(PDDocument document) throws IOException {
487	PDFTextStripper stripper = new PDFTextStripper();	×
488
489	stripper.setStartPage(1);	×
490	stripper.setEndPage(1);	×
491	stripper.setSortByPosition(true);	×
492	stripper.setParagraphEnd(System.lineSeparator());	×
493	StringWriter writer = new StringWriter();	×
494	stripper.writeText(document, writer);	×
495
496	return writer.toString();	×
497	}
498
499	/**
500	* Extract the year out of curString (if it is not yet defined)
501	*/
502	private void extractYear() {
503	if (year != null) {	×
504	return;	×
505	}
506
507	Matcher m = YEAR_EXTRACT_PATTERN.matcher(curString);	×
508	if (m.find()) {	×
509	year = curString.substring(m.start(), m.end());	×
510	}
511
512	}	×
513
514	/**
515	* PDFTextStripper normally does NOT produce multiple empty lines
516	* (besides at strange PDFs). These strange PDFs are handled here:
517	* proceed to next non-empty line
518	*/
519	private void proceedToNextNonEmptyLine() {
520	while ((i < paragraphs.length) && "".equals(paragraphs[i].trim())) {	×
521	i++;	×
522	}
523	}	×
524
525	/**
526	* Fill curString with lines until "" is found
527	* No trailing space is added
528	* i is advanced to the next non-empty line (ignoring white space)
529	* <p>
530	* Lines containing only white spaces are ignored,
531	* but NOT considered as ""
532	* <p>
533	* Uses GLOBAL variables paragraphs, curLine, i
534	*/
535	private void fillCurStringWithNonEmptyLines() {
536	// ensure that curString does not end with " "
537	curString = curString.trim();	×
538	while ((i < paragraphs.length) && !"".equals(paragraphs[i])) {	×
539	String curLine = paragraphs[i].trim();	×
540	if (!"".equals(curLine)) {	×
541	if (!curString.isEmpty()) {	×
542	// insert separating space if necessary
543	curString = curString.concat(" ");	×
544	}
545	curString = curString.concat(paragraphs[i]);	×
546	}
547	i++;	×
548	}	×
549
550	proceedToNextNonEmptyLine();	×
551	}	×
552
553	/**
554	* resets curString
555	* curString now contains the last block (until "" reached)
556	* Trailing space is added
557	* <p>
558	* invariant before/after: i points to line before the last handled block
559	*/
560	private void readLastBlock() {
561	while ((i >= 0) && "".equals(paragraphs[i].trim())) {	×
562	i--;	×
563	}
564	// i is now at the end of a block
565
566	int end = i;	×
567
568	// find beginning
569	while ((i >= 0) && !"".equals(paragraphs[i])) {	×
570	i--;	×
571	}
572	// i is now the line before the beginning of the block
573	// this fulfills the invariant
574
575	curString = "";	×
576	for (int j = i + 1; j <= end; j++) {	×
577	curString = curString.concat(paragraphs[j].trim());	×
578	if (j != end) {	×
579	curString = curString.concat(" ");	×
580	}
581	}
582	}	×
583
584	@Override
585	public String getFormatName() {
586	return "PDFcontent";	×
587	}
588
589	}

JabRef / jabref / 3169

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous