#3964

Committed 15 Mar 2024 04:46PM UTC coverage: 48.658% (-0.003%) from 48.661%

Build # #3964

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #4859 from adangel/doc-release-notes-7

[doc] Doc release notes 7

Run Details

32366 of 72822 branches covered (44.45%)

Branch coverage included in aggregate %.

4 of 42 new or added lines in 4 files covered. (9.52%)

2 existing lines in 1 file now uncovered.

68103 of 133657 relevant lines covered (50.95%)

0.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/pmd-doc/src/main/java/net/sourceforge/pmd/doc/internal/DeadLinksChecker.java

/**
 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
 */

package net.sourceforge.pmd.doc.internal;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import net.sourceforge.pmd.internal.util.IOUtil;

/**
 * Checks links to local pages for non-existing link-targets.
 */
public class DeadLinksChecker {
    private static final Logger LOG = LoggerFactory.getLogger(DeadLinksChecker.class);

    private static final String CHECK_EXTERNAL_LINKS_PROPERTY = "pmd.doc.checkExternalLinks";
    private static final boolean CHECK_EXTERNAL_LINKS = Boolean.parseBoolean(System.getProperty(CHECK_EXTERNAL_LINKS_PROPERTY));

    // Markdown-Link: something in []'s followed by something in ()'s
    // ignoring an optional prefix "{{ baseurl }}"
    private static final Pattern LOCAL_LINK_PATTERN = Pattern.compile("(!)?\\[.*?]\\((?:\\{\\{\\s*baseurl\\s*\\}\\})?(.*?)\\)");

    // Markdown permalink-header and captions
    private static final Pattern MD_HEADER_PERMALINK = Pattern.compile("permalink:\\s*(.*)");
    private static final Pattern MD_CAPTION = Pattern.compile("^##+\\s+(.*)$", Pattern.MULTILINE);

    // list of link targets, where the link detection doesn't work
    private static final Pattern EXCLUDED_LINK_TARGETS = Pattern.compile(
        "^pmd_userdocs_cli_reference\\.html.*" // anchors in the CLI reference are a plain HTML include
    );

    // the link is actually pointing to a file in the pmd project
    private static final String LOCAL_FILE_PREFIX = "https://github.com/pmd/pmd/blob/master/";

    // don't check links to PMD bugs/issues/pull-requests and some other sites (performance optimization)
    private static final List<String> IGNORED_URL_PREFIXES = Collections.unmodifiableList(Arrays.asList(
        "https://github.com/pmd/pmd/issues/",
        "https://github.com/pmd/pmd/pull/",
        "https://sourceforge.net/p/pmd/bugs/",
        "https://pmd.github.io/",
        "https://openjdk.org/jeps" // very slow...
    ));

    // prevent checking the same link multiple times
    private final Map<String, CompletableFuture<String>> urlResponseCache = new ConcurrentHashMap<>();

    private final ExecutorService executorService = Executors.newCachedThreadPool();


    public void checkDeadLinks(Path rootDirectory) throws InterruptedException {
        final Path pagesDirectory = rootDirectory.resolve("docs/pages");
        final Path docsDirectory = rootDirectory.resolve("docs");

        if (!Files.isDirectory(pagesDirectory)) {
            // docsDirectory is implicitly checked by this statement too
            LOG.error("can't check for dead links, didn't find \"pages\" directory at: {}", pagesDirectory);
            System.exit(1);
        }

        // read all .md-files in the pages directory
        final List<Path> mdFiles = listMdFiles(pagesDirectory);

        // Stores file path to the future deadlinks. If a future evaluates to null, the link is not dead
        final Map<Path, List<Future<String>>> fileToDeadLinks = new HashMap<>();
        // make a list of all valid link targets
        final Set<String> htmlPages = extractLinkTargets(mdFiles);

        // scan all .md-files for dead local links
        int scannedFiles = 0;
        int foundExternalLinks = 0;
        int checkedExternalLinks = 0;

        for (Path mdFile : mdFiles) {
            final String pageContent = fileToString(mdFile);
            scannedFiles++;

            // iterate line-by-line for better reporting the line numbers
            final String[] lines = pageContent.split("\r?\n");
            for (int index = 0; index < lines.length; index++) {
                final String line = lines[index];
                final int lineNo = index + 1;

                final Matcher matcher = LOCAL_LINK_PATTERN.matcher(line);
                linkCheck:
                while (matcher.find()) {
                    final String linkText = matcher.group();
                    final boolean isImageLink = matcher.group(1) != null;
                    final String linkTarget = matcher.group(2);
                    boolean linkOk;

                    if (linkTarget.charAt(0) == '/') {
                        // links must never start with / - they must be relative or start with https?//...
                        linkOk = false;
                    } else if (linkTarget.startsWith(LOCAL_FILE_PREFIX)) {
                        String localLinkPart = linkTarget.substring(LOCAL_FILE_PREFIX.length());
                        if (localLinkPart.contains("#")) {
                            localLinkPart = localLinkPart.substring(0, localLinkPart.indexOf('#'));
                        }

                        final Path localFile = rootDirectory.resolve(localLinkPart);
                        linkOk = Files.isRegularFile(localFile);
                        if (!linkOk) {
                            LOG.warn("local file not found: {}", localFile);
                            LOG.warn("  linked by: {}", linkTarget);
                        }

                    } else if (linkTarget.startsWith("http://") || linkTarget.startsWith("https://")) {
                        foundExternalLinks++;

                        if (!CHECK_EXTERNAL_LINKS) {
                            LOG.debug("ignoring check of external url: {}", linkTarget);
                            continue;
                        }

                        for (String ignoredUrlPrefix : IGNORED_URL_PREFIXES) {
                            if (linkTarget.startsWith(ignoredUrlPrefix)) {
                                LOG.debug("not checking link: {}", linkTarget);
                                continue linkCheck;
                            }
                        }

                        checkedExternalLinks++;
                        linkOk = true;

                        Future<String> futureMessage =
                            getCachedFutureResponse(linkTarget)
                                .thenApply(errorMessage -> errorMessage != null ? String.format("%8d: %s (%s)", lineNo, linkText, errorMessage) : null);

                        addDeadLink(fileToDeadLinks, mdFile, futureMessage);

                    } else {
                        // ignore local anchors
                        if (linkTarget.startsWith("#")) {
                            continue;
                        }

                        // ignore some pages where automatic link detection doesn't work
                        if (EXCLUDED_LINK_TARGETS.matcher(linkTarget).matches()) {
                            continue;
                        }

                        if (isImageLink) {
                            Path localResource = docsDirectory.resolve(linkTarget);
                            linkOk = Files.exists(localResource);
                        } else {
                            linkOk = linkTarget.isEmpty() || htmlPages.contains(linkTarget);
                        }

                        // maybe a local file
                        if (!linkOk) {
                            Path localResource = docsDirectory.resolve(linkTarget);
                            linkOk = Files.exists(localResource);
                        }
                    }

                    if (!linkOk) {
                        RunnableFuture<String> futureTask = new FutureTask<>(() -> String.format("%8d: %s", lineNo, linkText));
                        // execute this task immediately in this thread.
                        // External links are checked by another executor and don't end up here.
                        futureTask.run();
                        addDeadLink(fileToDeadLinks, mdFile, futureTask);
                    }
                }
            }
        }

        executorService.shutdown();
        LOG.info("Checking {} external links now...", checkedExternalLinks);
        Map<Path, List<String>> joined = joinFutures(fileToDeadLinks);

        LOG.info("Scanned {} files for dead links.", scannedFiles);
        LOG.info("  Found {} external links, {} of those where checked.", foundExternalLinks, checkedExternalLinks);

        if (!CHECK_EXTERNAL_LINKS) {
            LOG.info("External links weren't checked, set -D" + CHECK_EXTERNAL_LINKS_PROPERTY + "=true to enable it.");
        }

        if (joined.isEmpty()) {
            LOG.info("No errors found!");
        } else {
            LOG.warn("Found dead link(s):");
            for (Path file : joined.keySet()) {
                System.err.println(rootDirectory.relativize(file));
                joined.get(file).forEach(System.err::println);
            }
            throw new AssertionError("Dead links detected");
        }
    }


    private Map<Path, List<String>> joinFutures(Map<Path, List<Future<String>>> map) {
        Map<Path, List<String>> joined = new HashMap<>();

        for (Path p : map.keySet()) {

            List<String> evaluatedResult = map.get(p).stream()
                                              .map(f -> {
                                                  try {
                                                      return f.get();
                                                  } catch (InterruptedException | ExecutionException e) {
                                                      e.printStackTrace();
                                                      return null;
                                                  }
                                              })
                                              .filter(Objects::nonNull)
                                              .sorted(Comparator.naturalOrder())
                                              .collect(Collectors.toList());

            if (!evaluatedResult.isEmpty()) {
                joined.put(p, evaluatedResult);
            }
        }
        return joined;
    }


    private void addDeadLink(Map<Path, List<Future<String>>> fileToDeadLinks, Path file, Future<String> line) {
        fileToDeadLinks.computeIfAbsent(file, k -> new ArrayList<>()).add(line);
    }


    private Set<String> extractLinkTargets(List<Path> mdFiles) {
        final Set<String> htmlPages = new HashSet<>();
        for (Path mdFile : mdFiles) {
            final String pageContent = fileToString(mdFile);

            // extract the permalink header field
            final Matcher permalinkMatcher = MD_HEADER_PERMALINK.matcher(pageContent);
            if (!permalinkMatcher.find()) {
                continue;
            }

            final String pageUrl = permalinkMatcher.group(1)
                                                   .replaceAll("^/+", ""); // remove the leading "/"

            // add the root page
            htmlPages.add(pageUrl);

            // add all captions as anchors
            final Matcher captionMatcher = MD_CAPTION.matcher(pageContent);
            while (captionMatcher.find()) {
                final String anchor = captionMatcher.group(1)
                                                    .toLowerCase(Locale.ROOT)
                                                    .replaceAll("'|\\.", "") // remove all apostrophes and dots
                                                    .replaceAll("[^a-z0-9_]+", "-"); // replace all non-alphanumeric characters with dashes

                htmlPages.add(pageUrl + "#" + anchor);
            }
        }
        return htmlPages;
    }


    private List<Path> listMdFiles(Path pagesDirectory) {
        try (Stream<Path> stream = Files.walk(pagesDirectory)) {
            return stream
                 .filter(Files::isRegularFile)
                 .filter(path -> path.toString().endsWith(".md"))
                 .collect(Collectors.toList());
        } catch (IOException ex) {
            throw new RuntimeException("error listing files in " + pagesDirectory, ex);
        }
    }


    private String fileToString(Path mdFile) {
        try (InputStream inputStream = Files.newInputStream(mdFile)) {
            return IOUtil.readToString(inputStream, StandardCharsets.UTF_8);
        } catch (IOException ex) {
            throw new RuntimeException("error reading " + mdFile, ex);
        }
    }


    private CompletableFuture<String> getCachedFutureResponse(String url) {
        if (urlResponseCache.containsKey(url)) {
            CompletableFuture<String> cachedFuture = urlResponseCache.get(url);
            if (cachedFuture.isDone()) {
                try {
                    LOG.debug("response: HTTP {} (CACHED) on {}", cachedFuture.get(100, TimeUnit.MILLISECONDS), url);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                } catch (ExecutionException e) {
                    LOG.info("response failed: (CACHED) on {}", url);
                } catch (TimeoutException e) {
                    // actually, this shouldn't happen, as we checked with isDone() before
                    LOG.info("response future timeout: (CACHED) on {}", url);
                }
            }
            return cachedFuture;
        } else {
            // process asynchronously
            CompletableFuture<String> futureResponse = CompletableFuture.supplyAsync(() -> computeHttpResponse(url), executorService);
            urlResponseCache.put(url, futureResponse);
            return futureResponse;
        }
    }

    // limit parallel requests to avoid 429 Too Many Requests
    private Semaphore semaphore = new Semaphore(3);

    private String computeHttpResponse(String url) {
        try {
            semaphore.acquire();
            // logging to see something is going on...
            LOG.info("Checking {} now...", url);

            final HttpURLConnection httpUrlConnection = (HttpURLConnection) new URL(url).openConnection();
            httpUrlConnection.setRequestMethod("HEAD");
            httpUrlConnection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(60));
            httpUrlConnection.setReadTimeout((int) TimeUnit.SECONDS.toMillis(60));
            httpUrlConnection.connect();
            int responseCode = httpUrlConnection.getResponseCode();

            String response = "HTTP " + responseCode;
            if (httpUrlConnection.getHeaderField("Location") != null) {
                response += ", Location: " + httpUrlConnection.getHeaderField("Location");
            }
            LOG.debug("response: {} on {}", response, url);

            // everything above 400 is an error
            if (responseCode >= 400) {
                LOG.debug("response failure: {} on {}", responseCode, url);
                return "HTTP " + responseCode + " " + httpUrlConnection.getResponseMessage();
            }

            // success (HTTP 2xx) or redirection (HTTP 3xx) is ok
            return null; // no error
        } catch (IOException | InterruptedException ex) {
            LOG.debug("response: {} on {} : {}", ex.getClass().getName(), url, ex.getMessage());
            return ex.getClass().getName() + ": " + ex.getMessage();
        } finally {
            semaphore.release();
        }
    }


    public static void main(String[] args) throws IOException, InterruptedException {
        if (args.length != 1) {
            System.err.println("Wrong arguments!");
            System.err.println();
            System.err.println("java " + DeadLinksChecker.class.getSimpleName() + " <project base directory>");
            System.exit(1);
        }
        final Path rootDirectory = Paths.get(args[0]).resolve("..").toRealPath();

        DeadLinksChecker deadLinksChecker = new DeadLinksChecker();
        deadLinksChecker.checkDeadLinks(rootDirectory);
    }



}

1	/**
2	* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3	*/
4
5	package net.sourceforge.pmd.doc.internal;
6
7	import java.io.IOException;
8	import java.io.InputStream;
9	import java.net.HttpURLConnection;
10	import java.net.URL;
11	import java.nio.charset.StandardCharsets;
12	import java.nio.file.Files;
13	import java.nio.file.Path;
14	import java.nio.file.Paths;
15	import java.util.ArrayList;
16	import java.util.Arrays;
17	import java.util.Collections;
18	import java.util.Comparator;
19	import java.util.HashMap;
20	import java.util.HashSet;
21	import java.util.List;
22	import java.util.Locale;
23	import java.util.Map;
24	import java.util.Objects;
25	import java.util.Set;
26	import java.util.concurrent.CompletableFuture;
27	import java.util.concurrent.ConcurrentHashMap;
28	import java.util.concurrent.ExecutionException;
29	import java.util.concurrent.ExecutorService;
30	import java.util.concurrent.Executors;
31	import java.util.concurrent.Future;
32	import java.util.concurrent.FutureTask;
33	import java.util.concurrent.RunnableFuture;
34	import java.util.concurrent.Semaphore;
35	import java.util.concurrent.TimeUnit;
36	import java.util.concurrent.TimeoutException;
37	import java.util.regex.Matcher;
38	import java.util.regex.Pattern;
39	import java.util.stream.Collectors;
40	import java.util.stream.Stream;
41
42	import org.slf4j.Logger;
43	import org.slf4j.LoggerFactory;
44
45	import net.sourceforge.pmd.internal.util.IOUtil;
46
47	/**
48	* Checks links to local pages for non-existing link-targets.
49	*/
50	public class DeadLinksChecker {	×
51	private static final Logger LOG = LoggerFactory.getLogger(DeadLinksChecker.class);	×
52
53	private static final String CHECK_EXTERNAL_LINKS_PROPERTY = "pmd.doc.checkExternalLinks";
54	private static final boolean CHECK_EXTERNAL_LINKS = Boolean.parseBoolean(System.getProperty(CHECK_EXTERNAL_LINKS_PROPERTY));	×
55
56	// Markdown-Link: something in []'s followed by something in ()'s
57	// ignoring an optional prefix "{{ baseurl }}"
58	private static final Pattern LOCAL_LINK_PATTERN = Pattern.compile("(!)?\\[.?]\\((?:\\{\\{\\sbaseurl\\s\\}\\})?(.?)\\)");	×
59
60	// Markdown permalink-header and captions
61	private static final Pattern MD_HEADER_PERMALINK = Pattern.compile("permalink:\\s(.)");	×
62	private static final Pattern MD_CAPTION = Pattern.compile("^##+\\s+(.*)$", Pattern.MULTILINE);	×
63
64	// list of link targets, where the link detection doesn't work
65	private static final Pattern EXCLUDED_LINK_TARGETS = Pattern.compile(	×
66	"^pmd_userdocs_cli_reference\\.html.*" // anchors in the CLI reference are a plain HTML include
67	);
68
69	// the link is actually pointing to a file in the pmd project
70	private static final String LOCAL_FILE_PREFIX = "https://github.com/pmd/pmd/blob/master/";
71
72	// don't check links to PMD bugs/issues/pull-requests and some other sites (performance optimization)
73	private static final List<String> IGNORED_URL_PREFIXES = Collections.unmodifiableList(Arrays.asList(	×
74	"https://github.com/pmd/pmd/issues/",
75	"https://github.com/pmd/pmd/pull/",
76	"https://sourceforge.net/p/pmd/bugs/",
77	"https://pmd.github.io/",
78	"https://openjdk.org/jeps" // very slow...
79	));
80
81	// prevent checking the same link multiple times
NEW 82	private final Map<String, CompletableFuture<String>> urlResponseCache = new ConcurrentHashMap<>();	×
83
84	private final ExecutorService executorService = Executors.newCachedThreadPool();	×
85
86
87	public void checkDeadLinks(Path rootDirectory) throws InterruptedException {
88	final Path pagesDirectory = rootDirectory.resolve("docs/pages");	×
89	final Path docsDirectory = rootDirectory.resolve("docs");	×
90
91	if (!Files.isDirectory(pagesDirectory)) {	×
92	// docsDirectory is implicitly checked by this statement too
93	LOG.error("can't check for dead links, didn't find \"pages\" directory at: {}", pagesDirectory);	×
94	System.exit(1);	×
95	}
96
97	// read all .md-files in the pages directory
98	final List<Path> mdFiles = listMdFiles(pagesDirectory);	×
99
100	// Stores file path to the future deadlinks. If a future evaluates to null, the link is not dead
101	final Map<Path, List<Future<String>>> fileToDeadLinks = new HashMap<>();	×
102	// make a list of all valid link targets
103	final Set<String> htmlPages = extractLinkTargets(mdFiles);	×
104
105	// scan all .md-files for dead local links
106	int scannedFiles = 0;	×
107	int foundExternalLinks = 0;	×
108	int checkedExternalLinks = 0;	×
109
110	for (Path mdFile : mdFiles) {	×
111	final String pageContent = fileToString(mdFile);	×
112	scannedFiles++;	×
113
114	// iterate line-by-line for better reporting the line numbers
115	final String[] lines = pageContent.split("\r?\n");	×
116	for (int index = 0; index < lines.length; index++) {	×
117	final String line = lines[index];	×
118	final int lineNo = index + 1;	×
119
120	final Matcher matcher = LOCAL_LINK_PATTERN.matcher(line);	×
121	linkCheck:
122	while (matcher.find()) {	×
123	final String linkText = matcher.group();	×
124	final boolean isImageLink = matcher.group(1) != null;	×
125	final String linkTarget = matcher.group(2);	×
126	boolean linkOk;
127
128	if (linkTarget.charAt(0) == '/') {	×
129	// links must never start with / - they must be relative or start with https?//...
130	linkOk = false;	×
131	} else if (linkTarget.startsWith(LOCAL_FILE_PREFIX)) {	×
132	String localLinkPart = linkTarget.substring(LOCAL_FILE_PREFIX.length());	×
133	if (localLinkPart.contains("#")) {	×
134	localLinkPart = localLinkPart.substring(0, localLinkPart.indexOf('#'));	×
135	}
136
137	final Path localFile = rootDirectory.resolve(localLinkPart);	×
138	linkOk = Files.isRegularFile(localFile);	×
139	if (!linkOk) {	×
140	LOG.warn("local file not found: {}", localFile);	×
141	LOG.warn(" linked by: {}", linkTarget);	×
142	}
143
144	} else if (linkTarget.startsWith("http://") \|\| linkTarget.startsWith("https://")) {	×
145	foundExternalLinks++;	×
146
147	if (!CHECK_EXTERNAL_LINKS) {	×
148	LOG.debug("ignoring check of external url: {}", linkTarget);	×
149	continue;	×
150	}
151
152	for (String ignoredUrlPrefix : IGNORED_URL_PREFIXES) {	×
153	if (linkTarget.startsWith(ignoredUrlPrefix)) {	×
154	LOG.debug("not checking link: {}", linkTarget);	×
155	continue linkCheck;	×
156	}
157	}	×
158
159	checkedExternalLinks++;	×
160	linkOk = true;	×
161
162	Future<String> futureMessage =	×
163	getCachedFutureResponse(linkTarget)	×
NEW 164	.thenApply(errorMessage -> errorMessage != null ? String.format("%8d: %s (%s)", lineNo, linkText, errorMessage) : null);	×
165
166	addDeadLink(fileToDeadLinks, mdFile, futureMessage);	×
167
168	} else {	×
169	// ignore local anchors
170	if (linkTarget.startsWith("#")) {	×
171	continue;	×
172	}
173
174	// ignore some pages where automatic link detection doesn't work
175	if (EXCLUDED_LINK_TARGETS.matcher(linkTarget).matches()) {	×
176	continue;	×
177	}
178
179	if (isImageLink) {	×
180	Path localResource = docsDirectory.resolve(linkTarget);	×
181	linkOk = Files.exists(localResource);	×
182	} else {	×
183	linkOk = linkTarget.isEmpty() \|\| htmlPages.contains(linkTarget);	×
184	}
185
186	// maybe a local file
187	if (!linkOk) {	×
188	Path localResource = docsDirectory.resolve(linkTarget);	×
189	linkOk = Files.exists(localResource);	×
190	}
191	}
192
193	if (!linkOk) {	×
194	RunnableFuture<String> futureTask = new FutureTask<>(() -> String.format("%8d: %s", lineNo, linkText));	×
195	// execute this task immediately in this thread.
196	// External links are checked by another executor and don't end up here.
197	futureTask.run();	×
198	addDeadLink(fileToDeadLinks, mdFile, futureTask);	×
199	}
200	}	×
201	}
202	}	×
203
204	executorService.shutdown();	×
NEW 205	LOG.info("Checking {} external links now...", checkedExternalLinks);	×
NEW 206	Map<Path, List<String>> joined = joinFutures(fileToDeadLinks);	×
207
208	LOG.info("Scanned {} files for dead links.", scannedFiles);	×
209	LOG.info(" Found {} external links, {} of those where checked.", foundExternalLinks, checkedExternalLinks);	×
210
211	if (!CHECK_EXTERNAL_LINKS) {	×
212	LOG.info("External links weren't checked, set -D" + CHECK_EXTERNAL_LINKS_PROPERTY + "=true to enable it.");	×
213	}
214
215	if (joined.isEmpty()) {	×
UNCOV 216	LOG.info("No errors found!");	×
217	} else {
218	LOG.warn("Found dead link(s):");	×
219	for (Path file : joined.keySet()) {	×
NEW 220	System.err.println(rootDirectory.relativize(file));	×
NEW 221	joined.get(file).forEach(System.err::println);	×
222	}	×
223	throw new AssertionError("Dead links detected");	×
224	}
225	}	×
226
227
228	private Map<Path, List<String>> joinFutures(Map<Path, List<Future<String>>> map) {
229	Map<Path, List<String>> joined = new HashMap<>();	×
230
231	for (Path p : map.keySet()) {	×
232
233	List<String> evaluatedResult = map.get(p).stream()	×
234	.map(f -> {	×
235	try {
236	return f.get();	×
237	} catch (InterruptedException \| ExecutionException e) {	×
238	e.printStackTrace();	×
239	return null;	×
240	}
241	})
242	.filter(Objects::nonNull)	×
243	.sorted(Comparator.naturalOrder())	×
244	.collect(Collectors.toList());	×
245
246	if (!evaluatedResult.isEmpty()) {	×
247	joined.put(p, evaluatedResult);	×
248	}
249	}	×
250	return joined;	×
251	}
252
253
254	private void addDeadLink(Map<Path, List<Future<String>>> fileToDeadLinks, Path file, Future<String> line) {
255	fileToDeadLinks.computeIfAbsent(file, k -> new ArrayList<>()).add(line);	×
256	}	×
257
258
259	private Set<String> extractLinkTargets(List<Path> mdFiles) {
260	final Set<String> htmlPages = new HashSet<>();	×
261	for (Path mdFile : mdFiles) {	×
262	final String pageContent = fileToString(mdFile);	×
263
264	// extract the permalink header field
265	final Matcher permalinkMatcher = MD_HEADER_PERMALINK.matcher(pageContent);	×
266	if (!permalinkMatcher.find()) {	×
267	continue;	×
268	}
269
270	final String pageUrl = permalinkMatcher.group(1)	×
271	.replaceAll("^/+", ""); // remove the leading "/"	×
272
273	// add the root page
274	htmlPages.add(pageUrl);	×
275
276	// add all captions as anchors
277	final Matcher captionMatcher = MD_CAPTION.matcher(pageContent);	×
278	while (captionMatcher.find()) {	×
279	final String anchor = captionMatcher.group(1)	×
280	.toLowerCase(Locale.ROOT)	×
NEW 281	.replaceAll("'\|\\.", "") // remove all apostrophes and dots	×
NEW 282	.replaceAll("[^a-z0-9_]+", "-"); // replace all non-alphanumeric characters with dashes	×
283
284	htmlPages.add(pageUrl + "#" + anchor);	×
285	}	×
286	}	×
287	return htmlPages;	×
288	}
289
290
291	private List<Path> listMdFiles(Path pagesDirectory) {
292	try (Stream<Path> stream = Files.walk(pagesDirectory)) {	×
293	return stream	×
294	.filter(Files::isRegularFile)	×
295	.filter(path -> path.toString().endsWith(".md"))	×
296	.collect(Collectors.toList());	×
297	} catch (IOException ex) {	×
298	throw new RuntimeException("error listing files in " + pagesDirectory, ex);	×
299	}
300	}
301
302
303	private String fileToString(Path mdFile) {
304	try (InputStream inputStream = Files.newInputStream(mdFile)) {	×
305	return IOUtil.readToString(inputStream, StandardCharsets.UTF_8);	×
306	} catch (IOException ex) {	×
307	throw new RuntimeException("error reading " + mdFile, ex);	×
308	}
309	}
310
311
312	private CompletableFuture<String> getCachedFutureResponse(String url) {
313	if (urlResponseCache.containsKey(url)) {	×
NEW 314	CompletableFuture<String> cachedFuture = urlResponseCache.get(url);	×
NEW 315	if (cachedFuture.isDone()) {	×
316	try {
NEW 317	LOG.debug("response: HTTP {} (CACHED) on {}", cachedFuture.get(100, TimeUnit.MILLISECONDS), url);	×
NEW 318	} catch (InterruptedException e) {	×
NEW 319	Thread.currentThread().interrupt();	×
NEW 320	} catch (ExecutionException e) {	×
NEW 321	LOG.info("response failed: (CACHED) on {}", url);	×
NEW 322	} catch (TimeoutException e) {	×
323	// actually, this shouldn't happen, as we checked with isDone() before
NEW 324	LOG.info("response future timeout: (CACHED) on {}", url);	×
NEW 325	}	×
326	}
NEW 327	return cachedFuture;	×
328	} else {
329	// process asynchronously
NEW 330	CompletableFuture<String> futureResponse = CompletableFuture.supplyAsync(() -> computeHttpResponse(url), executorService);	×
331	urlResponseCache.put(url, futureResponse);	×
332	return futureResponse;	×
333	}
334	}
335
336	// limit parallel requests to avoid 429 Too Many Requests
NEW 337	private Semaphore semaphore = new Semaphore(3);	×
338
339	private String computeHttpResponse(String url) {
340	try {
NEW 341	semaphore.acquire();	×
342	// logging to see something is going on...
NEW 343	LOG.info("Checking {} now...", url);	×
344
NEW 345	final HttpURLConnection httpUrlConnection = (HttpURLConnection) new URL(url).openConnection();	×
NEW 346	httpUrlConnection.setRequestMethod("HEAD");	×
NEW 347	httpUrlConnection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(60));	×
NEW 348	httpUrlConnection.setReadTimeout((int) TimeUnit.SECONDS.toMillis(60));	×
NEW 349	httpUrlConnection.connect();	×
NEW 350	int responseCode = httpUrlConnection.getResponseCode();	×
351
352	String response = "HTTP " + responseCode;	×
NEW 353	if (httpUrlConnection.getHeaderField("Location") != null) {	×
NEW 354	response += ", Location: " + httpUrlConnection.getHeaderField("Location");	×
355	}
UNCOV 356	LOG.debug("response: {} on {}", response, url);	×
357
358	// everything above 400 is an error
NEW 359	if (responseCode >= 400) {	×
NEW 360	LOG.debug("response failure: {} on {}", responseCode, url);	×
NEW 361	return "HTTP " + responseCode + " " + httpUrlConnection.getResponseMessage();	×
362	}
363
364	// success (HTTP 2xx) or redirection (HTTP 3xx) is ok
NEW 365	return null; // no error	×
NEW 366	} catch (IOException \| InterruptedException ex) {	×
367	LOG.debug("response: {} on {} : {}", ex.getClass().getName(), url, ex.getMessage());	×
NEW 368	return ex.getClass().getName() + ": " + ex.getMessage();	×
369	} finally {
NEW 370	semaphore.release();	×
371	}
372	}
373
374
375	public static void main(String[] args) throws IOException, InterruptedException {
376	if (args.length != 1) {	×
377	System.err.println("Wrong arguments!");	×
378	System.err.println();	×
379	System.err.println("java " + DeadLinksChecker.class.getSimpleName() + " <project base directory>");	×
380	System.exit(1);	×
381	}
382	final Path rootDirectory = Paths.get(args[0]).resolve("..").toRealPath();	×
383
384	DeadLinksChecker deadLinksChecker = new DeadLinksChecker();	×
385	deadLinksChecker.checkDeadLinks(rootDirectory);	×
386	}	×
387
388
389
390	}

pmd / pmd / #3964

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous