loveapple / web-crawler / 33

Committed 06 Aug 2025 07:52AM UTC coverage: 96.795%. Remained the same

Build # 33

Build Type

push

circleci

Committed by Chunli Hao

Commit Message

add post count limit

Run Details

151 of 156 relevant lines covered (96.79%)

0.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/src/main/java/com/happinesea/webcrawler/config/CrawlerComponents.java

package com.happinesea.webcrawler.config;

import java.util.ArrayList;
import java.util.List;

import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.support.SynchronizedItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import com.happinesea.webcrawler.ContentsParser;
import com.happinesea.webcrawler.NotFoundContentsException;
import com.happinesea.webcrawler.entity.SiteCategory;
import com.happinesea.webcrawler.entity.SiteContents;
import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
import com.happinesea.webcrawler.service.SiteContentsService;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

@Configuration
@Slf4j
@Data
public class CrawlerComponents {

        @Autowired
        private SiteContentsService siteContentsService;

        @Autowired
        private ContentsParser contentsParser;

        @Bean
        public ItemReader<SiteInfoProcessPool> siteInfoProcessReader() {
                List<SiteInfoProcessPool> aliveList = siteContentsService.findAliveProcess();
                return new SynchronizedItemReader<>(new ListItemReader<>(aliveList));
        }

        @Bean
        public ItemProcessor<SiteInfoProcessPool, SiteInfoProcessPool> siteInfoProcessor() {
                return process -> {
                        siteContentsService.changSiteInfoProcess2Processing(process); // 状態を更新

                        try {
                                SiteCategory category = process.getSiteCategory();

                                // コンテンツ一覧を取得
                                List<SiteContents> contentsList = contentsParser.loadCategoryContentsList(category);
                                if (contentsList == null || contentsList.isEmpty()) {
                                        log.warn("No contents found for category: {}", category.getCategoryUrl());
                                        siteContentsService.changSiteInfoProcess2Fail(process);
                                        return process;
                                }

                                // 各コンテンツの詳細を取得
                                List<SiteContents> loadedContents = new ArrayList<>();
                                for (SiteContents contents : contentsList) {
                                        try {
                                                SiteContents fullContents = contentsParser.loadContents(contents);
                                                loadedContents.add(fullContents);
                                        } catch (NotFoundContentsException e) {
                                                log.warn("Content not found: {}", contents.getUrl(), e);
                                        }
                                }

                                // DBに保存（重複除外）
                                siteContentsService.bulkInsertIfNotExists(loadedContents);
                                return process;
                        } catch (Exception e) {
                                log.warn("Error processing siteInfoProcessPool: {}", process.getSiteInfoProcessId(), e);
                                return siteContentsService.changSiteInfoProcess2Fail(process);
                        }
                };
        }

        @Bean
        public ItemWriter<SiteInfoProcessPool> siteInfoProcessWriter() {
                return chunk -> siteContentsService.saveAllProcessPools(chunk.getItems());
        }
}

1	package com.happinesea.webcrawler.config;
2
3	import java.util.ArrayList;
4	import java.util.List;
5
6	import org.springframework.batch.item.ItemProcessor;
7	import org.springframework.batch.item.ItemReader;
8	import org.springframework.batch.item.ItemWriter;
9	import org.springframework.batch.item.support.ListItemReader;
10	import org.springframework.batch.item.support.SynchronizedItemReader;
11	import org.springframework.beans.factory.annotation.Autowired;
12	import org.springframework.context.annotation.Bean;
13	import org.springframework.context.annotation.Configuration;
14
15	import com.happinesea.webcrawler.ContentsParser;
16	import com.happinesea.webcrawler.NotFoundContentsException;
17	import com.happinesea.webcrawler.entity.SiteCategory;
18	import com.happinesea.webcrawler.entity.SiteContents;
19	import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
20	import com.happinesea.webcrawler.service.SiteContentsService;
21
22	import lombok.Data;
23	import lombok.extern.slf4j.Slf4j;
24
25	@Configuration
26	@Slf4j	1✔
27	@Data
28	public class CrawlerComponents {
29
30	@Autowired
31	private SiteContentsService siteContentsService;
32
33	@Autowired
34	private ContentsParser contentsParser;
35
36	@Bean
37	public ItemReader<SiteInfoProcessPool> siteInfoProcessReader() {
38	List<SiteInfoProcessPool> aliveList = siteContentsService.findAliveProcess();	1✔
39	return new SynchronizedItemReader<>(new ListItemReader<>(aliveList));	1✔
40	}
41
42	@Bean
43	public ItemProcessor<SiteInfoProcessPool, SiteInfoProcessPool> siteInfoProcessor() {
44	return process -> {	1✔
45	siteContentsService.changSiteInfoProcess2Processing(process); // 状態を更新	1✔
46
47	try {
48	SiteCategory category = process.getSiteCategory();	1✔
49
50	// コンテンツ一覧を取得
51	List<SiteContents> contentsList = contentsParser.loadCategoryContentsList(category);	1✔
52	if (contentsList == null \|\| contentsList.isEmpty()) {	1✔
53	log.warn("No contents found for category: {}", category.getCategoryUrl());	×
54	siteContentsService.changSiteInfoProcess2Fail(process);	×
55	return process;	×
56	}
57
58	// 各コンテンツの詳細を取得
59	List<SiteContents> loadedContents = new ArrayList<>();	1✔
60	for (SiteContents contents : contentsList) {	1✔
61	try {
62	SiteContents fullContents = contentsParser.loadContents(contents);	1✔
63	loadedContents.add(fullContents);	1✔
64	} catch (NotFoundContentsException e) {	×
65	log.warn("Content not found: {}", contents.getUrl(), e);	×
66	}	1✔
67	}	1✔
68
69	// DBに保存（重複除外）
70	siteContentsService.bulkInsertIfNotExists(loadedContents);	1✔
71	return process;	1✔
72	} catch (Exception e) {	1✔
73	log.warn("Error processing siteInfoProcessPool: {}", process.getSiteInfoProcessId(), e);	1✔
74	return siteContentsService.changSiteInfoProcess2Fail(process);	1✔
75	}
76	};
77	}
78
79	@Bean
80	public ItemWriter<SiteInfoProcessPool> siteInfoProcessWriter() {
81	return chunk -> siteContentsService.saveAllProcessPools(chunk.getItems());	1✔
82	}
83	}

loveapple / web-crawler / 33

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous