loveapple / web-crawler / 37

Committed 14 Jul 2025 04:41AM UTC coverage: 74.359% (+25.1%) from 49.231%

Build # 37

Build Type

push

circleci

Committed by Chunli Hao

Commit Message

add enum for entiry

Run Details

15 of 22 new or added lines in 6 files covered. (68.18%)

58 of 78 relevant lines covered (74.36%)

0.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.31

/src/main/java/com/happinesea/webcrawler/config/CrawlerComponents.java

package com.happinesea.webcrawler.config;


import java.util.Iterator;
import java.util.List;

import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestTemplate;

import com.happinesea.webcrawler.model.WebPage;

@Configuration
public class CrawlerComponents {

    @Bean
    public ItemReader<String> urlReader() {
        List<String> urls = List.of(
            "https://httpbin.org/get",
            "https://httpbin.org/headers"
        );

        if (urls.isEmpty()) {
            throw new IllegalStateException("URLリストが空です。Crawlerを実行できません。");
        }

        Iterator<String> iterator = urls.iterator();
       // return () -> iterator.hasNext() ? iterator.next() : null;
        return () ->null;
    }

    @Bean
    public ItemProcessor<String, WebPage> webCrawlerProcessor(RestTemplate restTemplate) {
        return url -> {
                // 取得内容URL列表，进行各个内容DB插入处理，如果出现异常，跳过进行下一步处理
            try {
                String content = restTemplate.getForObject(url, String.class);
                return new WebPage(url, content);
            } catch (Exception e) {
                return new WebPage(url, "[Error] " + e.getClass().getSimpleName() + ": " + e.getMessage());
            }
        };
    }

    @Bean
    public ItemWriter<WebPage> webPageWriter() {
            // 最后写入处理
        return items -> items.forEach(page -> System.out.println("[Crawled] " + page));
    }

    @Bean
    public RestTemplate restTemplate() {
        return new RestTemplate();
    }
}

1	package com.happinesea.webcrawler.config;
2
3
4	import java.util.Iterator;
5	import java.util.List;
6
7	import org.springframework.batch.item.ItemProcessor;
8	import org.springframework.batch.item.ItemReader;
9	import org.springframework.batch.item.ItemWriter;
10	import org.springframework.context.annotation.Bean;
11	import org.springframework.context.annotation.Configuration;
12	import org.springframework.web.client.RestTemplate;
13
14	import com.happinesea.webcrawler.model.WebPage;
15
16	@Configuration
17	public class CrawlerComponents {	1✔
18
19	@Bean
20	public ItemReader<String> urlReader() {
21	List<String> urls = List.of(	1✔
22	"https://httpbin.org/get",
23	"https://httpbin.org/headers"
24	);
25
26	if (urls.isEmpty()) {	1✔
NEW 27	throw new IllegalStateException("URLリストが空です。Crawlerを実行できません。");	×
28	}
29
30	Iterator<String> iterator = urls.iterator();	1✔
31	// return () -> iterator.hasNext() ? iterator.next() : null;
32	return () ->null;	1✔
33	}
34
35	@Bean
36	public ItemProcessor<String, WebPage> webCrawlerProcessor(RestTemplate restTemplate) {
37	return url -> {	1✔
38	// 取得内容URL列表，进行各个内容DB插入处理，如果出现异常，跳过进行下一步处理
39	try {
40	String content = restTemplate.getForObject(url, String.class);	1✔
41	return new WebPage(url, content);	1✔
42	} catch (Exception e) {	1✔
43	return new WebPage(url, "[Error] " + e.getClass().getSimpleName() + ": " + e.getMessage());	1✔
44	}
45	};
46	}
47
48	@Bean
49	public ItemWriter<WebPage> webPageWriter() {
50	// 最后写入处理
51	return items -> items.forEach(page -> System.out.println("[Crawled] " + page));	1✔
52	}
53
54	@Bean
55	public RestTemplate restTemplate() {
56	return new RestTemplate();	1✔
57	}
58	}

loveapple / web-crawler / 37

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous