loveapple / web-crawler / 39

Committed 15 Jul 2025 04:03AM UTC coverage: 75.532% (-20.6%) from 96.154%

Build # 39

Build Type

push

circleci

Committed by Chunli Hao

Commit Message

support multi thread

Run Details

21 of 37 new or added lines in 3 files covered. (56.76%)

6 existing lines in 2 files now uncovered.

71 of 94 relevant lines covered (75.53%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.15

/src/main/java/com/happinesea/webcrawler/config/BatchConfig.java

package com.happinesea.webcrawler.config;

import java.util.List;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.transaction.PlatformTransactionManager;

import com.happinesea.webcrawler.Const.ProcessStatus;
import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
import com.happinesea.webcrawler.repository.SiteInfoProcessRepository;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@Configuration
public class BatchConfig {
    @Autowired
    private SiteInfoProcessRepository siteInfoProcessRepository;

    @Bean
    public Job crawlJob(JobRepository jobRepository, Step crawlStep) {
        return new JobBuilder("crawlJob", jobRepository)
                .start(crawlStep)
                .build();
    }

    @Bean
    public Step crawlStep(JobRepository jobRepository,
                          PlatformTransactionManager transactionManager,
                          CrawlerComponents crawlerComponents) {

        List<SiteInfoProcessPool> targetList =
                siteInfoProcessRepository.findByProcessStatusNot(ProcessStatus.PROCESSING);

        if (targetList.isEmpty()) {
            log.info("処理対象の SiteInfoProcess が存在しないため、ジョブを正常終了します。");

            return new StepBuilder("emptyStep", jobRepository)
                    .tasklet((contribution, chunkContext) -> RepeatStatus.FINISHED, transactionManager)
                    .build();
        }

        // データがある場合だけ TaskExecutor を初期化
        int threadCount = targetList.size();
        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
        taskExecutor.setCorePoolSize(threadCount);
        taskExecutor.setMaxPoolSize(threadCount);
        taskExecutor.setQueueCapacity(0);
        taskExecutor.setThreadNamePrefix("crawler-");
        taskExecutor.initialize();

        return new StepBuilder("crawlStep", jobRepository)
                .<SiteInfoProcessPool, SiteInfoProcessPool>chunk(5, transactionManager)
                .reader(new ListItemReader<>(targetList))
                .processor(crawlerComponents.siteInfoProcessor())
                .writer(crawlerComponents.siteInfoProcessWriter())
                .taskExecutor(taskExecutor)
                .build();
    }

}


1	package com.happinesea.webcrawler.config;
2
3	import java.util.List;
4
5	import org.springframework.batch.core.Job;
6	import org.springframework.batch.core.Step;
7	import org.springframework.batch.core.configuration.annotation.StepScope;
8	import org.springframework.batch.core.job.builder.JobBuilder;
9	import org.springframework.batch.core.repository.JobRepository;
10	import org.springframework.batch.core.step.builder.StepBuilder;
11	import org.springframework.batch.item.support.ListItemReader;
12	import org.springframework.batch.repeat.RepeatStatus;
13	import org.springframework.beans.factory.annotation.Autowired;
14	import org.springframework.beans.factory.annotation.Value;
15	import org.springframework.context.annotation.Bean;
16	import org.springframework.context.annotation.Configuration;
17	import org.springframework.core.task.TaskExecutor;
18	import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
19	import org.springframework.transaction.PlatformTransactionManager;
20
21	import com.happinesea.webcrawler.Const.ProcessStatus;
22	import com.happinesea.webcrawler.entity.SiteInfoProcessPool;
23	import com.happinesea.webcrawler.repository.SiteInfoProcessRepository;
24
25	import lombok.extern.slf4j.Slf4j;
26
27	@Slf4j	1✔
28	@Configuration
29	public class BatchConfig {	1✔
30	@Autowired
31	private SiteInfoProcessRepository siteInfoProcessRepository;
32
33	@Bean
34	public Job crawlJob(JobRepository jobRepository, Step crawlStep) {
35	return new JobBuilder("crawlJob", jobRepository)	1✔
36	.start(crawlStep)	1✔
37	.build();	1✔
38	}
39
40	@Bean
41	public Step crawlStep(JobRepository jobRepository,
42	PlatformTransactionManager transactionManager,
43	CrawlerComponents crawlerComponents) {
44
45	List<SiteInfoProcessPool> targetList =	1✔
46	siteInfoProcessRepository.findByProcessStatusNot(ProcessStatus.PROCESSING);	1✔
47
48	if (targetList.isEmpty()) {	1✔
49	log.info("処理対象の SiteInfoProcess が存在しないため、ジョブを正常終了します。");	1✔
50
51	return new StepBuilder("emptyStep", jobRepository)	1✔
52	.tasklet((contribution, chunkContext) -> RepeatStatus.FINISHED, transactionManager)	1✔
53	.build();	1✔
54	}
55
56	// データがある場合だけ TaskExecutor を初期化
NEW 57	int threadCount = targetList.size();	×
NEW 58	ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();	×
NEW 59	taskExecutor.setCorePoolSize(threadCount);	×
NEW 60	taskExecutor.setMaxPoolSize(threadCount);	×
NEW 61	taskExecutor.setQueueCapacity(0);	×
NEW 62	taskExecutor.setThreadNamePrefix("crawler-");	×
NEW 63	taskExecutor.initialize();	×
64
NEW 65	return new StepBuilder("crawlStep", jobRepository)	×
NEW 66	.<SiteInfoProcessPool, SiteInfoProcessPool>chunk(5, transactionManager)	×
NEW 67	.reader(new ListItemReader<>(targetList))	×
NEW 68	.processor(crawlerComponents.siteInfoProcessor())	×
NEW 69	.writer(crawlerComponents.siteInfoProcessWriter())	×
NEW 70	.taskExecutor(taskExecutor)	×
NEW 71	.build();	×
72	}
73
74	}
75

loveapple / web-crawler / 39

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous