loveapple / web-crawler / 48

Committed 30 Jul 2025 10:52AM UTC coverage: 78.916% (-21.1%) from 100.0%

Build # 48

Build Type

push

circleci

Committed by Chunli Hao

Commit Message

add test case for CrawlerComponents.java

Run Details

25 of 46 new or added lines in 3 files covered. (54.35%)

131 of 166 relevant lines covered (78.92%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/src/main/java/com/happinesea/webcrawler/config/CrawlerComponents.java

package com.happinesea.webcrawler.config;

import java.util.ArrayList;
import java.util.List;

import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.support.SynchronizedItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import com.happinesea.webcrawler.Const.ProcessStatus;
import com.happinesea.webcrawler.ContentsParser;
import com.happinesea.webcrawler.NotFoundContentsException;
import com.happinesea.webcrawler.entity.SiteCategory;
import com.happinesea.webcrawler.entity.SiteContents;
import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
import com.happinesea.webcrawler.service.SiteContentsService;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

@Configuration
@Slf4j
@Data
public class CrawlerComponents {

        @Autowired
        private SiteContentsService siteContentsService;

        @Autowired
        private ContentsParser contentsParser;

        @Bean
        public ItemReader<SiteInfoProcessPool> siteInfoProcessReader() {
                List<SiteInfoProcessPool> aliveList = siteContentsService.findAliveProcess();
                return new SynchronizedItemReader<>(new ListItemReader<>(aliveList));
        }

        @Bean
        public ItemProcessor<SiteInfoProcessPool, SiteInfoProcessPool> siteInfoProcessor() {
                return process -> {
                        siteContentsService.changSiteInfoProcess2Processing(process); // 状態を更新

                        try {
                                SiteCategory category = process.getSiteCategory();

                                // コンテンツ一覧を取得
                                List<SiteContents> contentsList = contentsParser.loadCategoryContentsList(category);
                                if (contentsList == null || contentsList.isEmpty()) {
                                        log.warn("No contents found for category: {}", category.getCategoryUrl());
                                        siteContentsService.changSiteInfoProcess2Fail(process);
                                        return process;
                                }

                                // 各コンテンツの詳細を取得
                                List<SiteContents> loadedContents = new ArrayList<>();
                                for (SiteContents contents : contentsList) {
                                        try {
                                                SiteContents fullContents = contentsParser.loadContents(contents);
                                                loadedContents.add(fullContents);
                                        } catch (NotFoundContentsException e) {
                                                log.warn("Content not found: {}", contents.getUrl(), e);
                                        }
                                }

                                // DBに保存（重複除外）
                                siteContentsService.bulkInsertIfNotExists(loadedContents);
                                return process;
                        } catch (Exception e) {
                                log.warn("Error processing siteInfoProcessPool: {}", process.getSiteInfoProcessId(), e);
                                return siteContentsService.changSiteInfoProcess2Fail(process);
                        }
                };
        }

        @Bean
        public ItemWriter<SiteInfoProcessPool> siteInfoProcessWriter() {
                return chunk -> siteContentsService.saveAllProcessPools(chunk.getItems());
        }
}

1	package com.happinesea.webcrawler.config;
2
3	import java.util.ArrayList;
4	import java.util.List;
5
6	import org.springframework.batch.item.ItemProcessor;
7	import org.springframework.batch.item.ItemReader;
8	import org.springframework.batch.item.ItemWriter;
9	import org.springframework.batch.item.support.ListItemReader;
10	import org.springframework.batch.item.support.SynchronizedItemReader;
11	import org.springframework.beans.factory.annotation.Autowired;
12	import org.springframework.context.annotation.Bean;
13	import org.springframework.context.annotation.Configuration;
14
15	import com.happinesea.webcrawler.Const.ProcessStatus;
16	import com.happinesea.webcrawler.ContentsParser;
17	import com.happinesea.webcrawler.NotFoundContentsException;
18	import com.happinesea.webcrawler.entity.SiteCategory;
19	import com.happinesea.webcrawler.entity.SiteContents;
20	import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
21	import com.happinesea.webcrawler.service.SiteContentsService;
22
23	import lombok.Data;
24	import lombok.extern.slf4j.Slf4j;
25
26	@Configuration
27	@Slf4j	1✔
28	@Data
29	public class CrawlerComponents {
30
31	@Autowired
32	private SiteContentsService siteContentsService;
33
34	@Autowired
35	private ContentsParser contentsParser;
36
37	@Bean
38	public ItemReader<SiteInfoProcessPool> siteInfoProcessReader() {
39	List<SiteInfoProcessPool> aliveList = siteContentsService.findAliveProcess();	1✔
40	return new SynchronizedItemReader<>(new ListItemReader<>(aliveList));	1✔
41	}
42
43	@Bean
44	public ItemProcessor<SiteInfoProcessPool, SiteInfoProcessPool> siteInfoProcessor() {
45	return process -> {	1✔
46	siteContentsService.changSiteInfoProcess2Processing(process); // 状態を更新	1✔
47
48	try {
49	SiteCategory category = process.getSiteCategory();	1✔
50
51	// コンテンツ一覧を取得
52	List<SiteContents> contentsList = contentsParser.loadCategoryContentsList(category);	1✔
53	if (contentsList == null \|\| contentsList.isEmpty()) {	1✔
NEW 54	log.warn("No contents found for category: {}", category.getCategoryUrl());	×
NEW 55	siteContentsService.changSiteInfoProcess2Fail(process);	×
NEW 56	return process;	×
57	}
58
59	// 各コンテンツの詳細を取得
60	List<SiteContents> loadedContents = new ArrayList<>();	1✔
61	for (SiteContents contents : contentsList) {	1✔
62	try {
63	SiteContents fullContents = contentsParser.loadContents(contents);	1✔
64	loadedContents.add(fullContents);	1✔
NEW 65	} catch (NotFoundContentsException e) {	×
NEW 66	log.warn("Content not found: {}", contents.getUrl(), e);	×
67	}	1✔
68	}	1✔
69
70	// DBに保存（重複除外）
71	siteContentsService.bulkInsertIfNotExists(loadedContents);	1✔
72	return process;	1✔
73	} catch (Exception e) {	1✔
74	log.warn("Error processing siteInfoProcessPool: {}", process.getSiteInfoProcessId(), e);	1✔
75	return siteContentsService.changSiteInfoProcess2Fail(process);	1✔
76	}
77	};
78	}
79
80	@Bean
81	public ItemWriter<SiteInfoProcessPool> siteInfoProcessWriter() {
82	return chunk -> siteContentsService.saveAllProcessPools(chunk.getItems());	1✔
83	}
84	}

loveapple / web-crawler / 48

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous