loveapple / web-crawler / 25

Committed 07 Jul 2025 05:20AM UTC coverage: 87.5% (+37.5%) from 50.0%

Build # 25

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2 from loveapple/20250701_ci

add batch template

Run Details

27 of 29 new or added lines in 4 files covered. (93.1%)

28 of 32 relevant lines covered (87.5%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.82

/src/main/java/com/happinesea/webcrawler/config/CrawlerComponents.java

package com.happinesea.webcrawler.config;


import java.util.Iterator;
import java.util.List;

import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestTemplate;

import com.happinesea.webcrawler.model.WebPage;

@Configuration
public class CrawlerComponents {

    @Bean
    public ItemReader<String> urlReader() {
        List<String> urls = List.of(
                "https://httpbin.org/get",
                "https://httpbin.org/headers"
        );
        Iterator<String> iterator = urls.iterator();
        return () -> iterator.hasNext() ? iterator.next() : null;
    }

    @Bean
    public ItemProcessor<String, WebPage> webCrawlerProcessor(RestTemplate restTemplate) {
        return url -> {
            try {
                String content = restTemplate.getForObject(url, String.class);
                return new WebPage(url, content);
            } catch (Exception e) {
                return new WebPage(url, "[Error] " + e.getClass().getSimpleName() + ": " + e.getMessage());
            }
        };
    }

    @Bean
    public ItemWriter<WebPage> webPageWriter() {
        return items -> items.forEach(page -> System.out.println("[Crawled] " + page));
    }

    @Bean
    public RestTemplate restTemplate() {
        return new RestTemplate();
    }
}

1	package com.happinesea.webcrawler.config;
2
3
4	import java.util.Iterator;
5	import java.util.List;
6
7	import org.springframework.batch.item.ItemProcessor;
8	import org.springframework.batch.item.ItemReader;
9	import org.springframework.batch.item.ItemWriter;
10	import org.springframework.context.annotation.Bean;
11	import org.springframework.context.annotation.Configuration;
12	import org.springframework.web.client.RestTemplate;
13
14	import com.happinesea.webcrawler.model.WebPage;
15
16	@Configuration
17	public class CrawlerComponents {	1✔
18
19	@Bean
20	public ItemReader<String> urlReader() {
21	List<String> urls = List.of(	1✔
22	"https://httpbin.org/get",
23	"https://httpbin.org/headers"
24	);
25	Iterator<String> iterator = urls.iterator();	1✔
26	return () -> iterator.hasNext() ? iterator.next() : null;	1✔
27	}
28
29	@Bean
30	public ItemProcessor<String, WebPage> webCrawlerProcessor(RestTemplate restTemplate) {
31	return url -> {	1✔
32	try {
33	String content = restTemplate.getForObject(url, String.class);	1✔
34	return new WebPage(url, content);	1✔
NEW 35	} catch (Exception e) {	×
NEW 36	return new WebPage(url, "[Error] " + e.getClass().getSimpleName() + ": " + e.getMessage());	×
37	}
38	};
39	}
40
41	@Bean
42	public ItemWriter<WebPage> webPageWriter() {
43	return items -> items.forEach(page -> System.out.println("[Crawled] " + page));	1✔
44	}
45
46	@Bean
47	public RestTemplate restTemplate() {
48	return new RestTemplate();	1✔
49	}
50	}

loveapple / web-crawler / 25

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous