20829376976

Committed 08 Jan 2026 07:37PM UTC coverage: 70.494% (+0.7%) from 69.833%

Build # 20829376976

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #157 from broadinstitute/dp-bbnorm

Add rmdup_bbnorm_bam for alignment-free read deduplication using BBNorm

Run Details

185 of 199 new or added lines in 3 files covered. (92.96%)

6 existing lines in 2 files now uncovered.

4580 of 6497 relevant lines covered (70.49%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.46

/tools/bbmap.py

'''
    Tool wrapper for the BBMap aligner and related tools.
'''

import logging
import os
import os.path
import shutil
import subprocess

import util.file
import tools
import tools.samtools
import tools.picard

TOOL_NAME = 'bbmap.sh'

_log = logging.getLogger(__name__)  # pylint: disable=invalid-name

class BBMapTool(tools.Tool):
    '''Tool wrapper for the BBMap aligner and related tools.'''

    def __init__(self, install_methods=None):
        if install_methods is None:
            install_methods = [tools.PrexistingUnixCommand(shutil.which(TOOL_NAME), require_executability=True)]
        super(BBMapTool, self).__init__(install_methods=install_methods)

    def _get_tool_version(self):
        self.tool_version = subprocess.check_output([os.path.join(os.path.dirname(self.install_and_get_path()), 'bbversion.sh')]).decode('UTF-8').strip()

    def execute(self, tool, **kwargs):  # pylint: disable=arguments-differ
        tool_dir = os.path.dirname(self.install_and_get_path())
        tool_cmd = [os.path.join(tool_dir, tool)] + \
                   ['{}={}'.format('in' if arg=='in_' else arg,
                                   (val is True and 't') or (val is False and 'f') or val)
                    for arg, val in kwargs.items()]
        _log.debug('Running BBMap tool: %s', ' '.join(tool_cmd))
        subprocess.check_call(tool_cmd)

    def align(self, inBam, refFasta, outBam, min_qual=0, nodisk=True, JVMmemory=None, **kwargs):
        with tools.samtools.SamtoolsTool().bam2fq_tmp(inBam) as (in1, in2), \
             util.file.tmp_dir('_bbmap_align') as t_dir:
            tmp_bam = os.path.join(t_dir, 'bbmap_out.bam')
            self.execute(tool='bbmap.sh', in1=in1, in2=in2, ref=refFasta, out=tmp_bam, nodisk=nodisk, **kwargs)
            
            # Samtools filter (optional)
            if min_qual:
                tmp_bam2 = os.path.join(tdir, 'bbmap.filtered.bam')
                cmd = [samtools.install_and_get_path(), 'view', '-b', '-S', '-1', '-q', str(min_qual), tmp_bam]
                _log.debug('%s > %s', ' '.join(cmd), tmp_bam2)
                with open(tmp_bam2, 'wb') as outf:
                    util.misc.run_and_save(cmd, outf=outf)
                os.unlink(tmp_bam)
                tmp_bam = tmp_bam2

            # Picard SortSam
            sorter = tools.picard.SortSamTool()
            sorter.execute(
                tmp_bam,
                outBam,
                sort_order='coordinate',
                picardOptions=['CREATE_INDEX=true', 'VALIDATION_STRINGENCY=SILENT'],
                JVMmemory=JVMmemory
            )

    def bbnorm(self, inFastq, outFastq, tmpdir=None, target=None, k=None,
               passes=None, mindepth=0, threads=None, memory=None):
        """
        Run bbnorm for read normalization/deduplication.

        Args:
            inFastq: Input FASTQ file (interleaved for paired-end, or single-end)
            outFastq: Output FASTQ file (same format as input)
            tmpdir: Temporary directory for multipass processing
            target: Target normalization depth (default: bbnorm default of 100)
            k: Kmer length (default: bbnorm default of 31)
            passes: Number of passes (default: bbnorm default of 2)
            mindepth: Min kmer depth threshold (default: 0 to include all)
            threads: Number of threads (default: auto)
            memory: Java memory allocation string (e.g., "4g")

        Note: bbnorm auto-detects interleaved vs single-end format.
        """
        tool_dir = os.path.dirname(self.install_and_get_path())
        tool_cmd = [os.path.join(tool_dir, 'bbnorm.sh'), '-eoom']

        if memory:
            tool_cmd.append('-Xmx{}'.format(memory))

        # Build arguments
        tool_cmd.append('in={}'.format(inFastq))
        tool_cmd.append('out={}'.format(outFastq))
        tool_cmd.append('mindepth={}'.format(mindepth))

        if tmpdir is not None:
            tool_cmd.append('tmpdir={}'.format(tmpdir))
        if target is not None:
            tool_cmd.append('target={}'.format(target))
        if k is not None:
            tool_cmd.append('k={}'.format(k))
        if passes is not None:
            tool_cmd.append('passes={}'.format(passes))
        if threads is not None:
            tool_cmd.append('threads={}'.format(threads))

        _log.debug('Running bbnorm: %s', ' '.join(tool_cmd))
        subprocess.check_call(tool_cmd)


1	'''
2	Tool wrapper for the BBMap aligner and related tools.
3	'''
4
5	import logging	1✔
6	import os	1✔
7	import os.path	1✔
8	import shutil	1✔
9	import subprocess	1✔
10
11	import util.file	1✔
12	import tools	1✔
13	import tools.samtools	1✔
14	import tools.picard	1✔
15
16	TOOL_NAME = 'bbmap.sh'	1✔
17
18	_log = logging.getLogger(__name__) # pylint: disable=invalid-name	1✔
19
20	class BBMapTool(tools.Tool):	1✔
21	'''Tool wrapper for the BBMap aligner and related tools.'''
22
23	def __init__(self, install_methods=None):	1✔
24	if install_methods is None:	1✔
25	install_methods = [tools.PrexistingUnixCommand(shutil.which(TOOL_NAME), require_executability=True)]	1✔
26	super(BBMapTool, self).__init__(install_methods=install_methods)	1✔
27
28	def _get_tool_version(self):	1✔
29	self.tool_version = subprocess.check_output([os.path.join(os.path.dirname(self.install_and_get_path()), 'bbversion.sh')]).decode('UTF-8').strip()	×
30
31	def execute(self, tool, **kwargs): # pylint: disable=arguments-differ	1✔
32	tool_dir = os.path.dirname(self.install_and_get_path())	1✔
33	tool_cmd = [os.path.join(tool_dir, tool)] + \	1✔
34	['{}={}'.format('in' if arg=='in_' else arg,
35	(val is True and 't') or (val is False and 'f') or val)
36	for arg, val in kwargs.items()]
37	_log.debug('Running BBMap tool: %s', ' '.join(tool_cmd))	1✔
38	subprocess.check_call(tool_cmd)	1✔
39
40	def align(self, inBam, refFasta, outBam, min_qual=0, nodisk=True, JVMmemory=None, **kwargs):	1✔
41	with tools.samtools.SamtoolsTool().bam2fq_tmp(inBam) as (in1, in2), \	1✔
42	util.file.tmp_dir('_bbmap_align') as t_dir:
43	tmp_bam = os.path.join(t_dir, 'bbmap_out.bam')	1✔
44	self.execute(tool='bbmap.sh', in1=in1, in2=in2, ref=refFasta, out=tmp_bam, nodisk=nodisk, **kwargs)	1✔
45
46	# Samtools filter (optional)
47	if min_qual:	1✔
48	tmp_bam2 = os.path.join(tdir, 'bbmap.filtered.bam')	×
49	cmd = [samtools.install_and_get_path(), 'view', '-b', '-S', '-1', '-q', str(min_qual), tmp_bam]	×
50	_log.debug('%s > %s', ' '.join(cmd), tmp_bam2)	×
51	with open(tmp_bam2, 'wb') as outf:	×
52	util.misc.run_and_save(cmd, outf=outf)	×
53	os.unlink(tmp_bam)	×
54	tmp_bam = tmp_bam2	×
55
56	# Picard SortSam
57	sorter = tools.picard.SortSamTool()	1✔
58	sorter.execute(	1✔
59	tmp_bam,
60	outBam,
61	sort_order='coordinate',
62	picardOptions=['CREATE_INDEX=true', 'VALIDATION_STRINGENCY=SILENT'],
63	JVMmemory=JVMmemory
64	)
65
66	def bbnorm(self, inFastq, outFastq, tmpdir=None, target=None, k=None,	1✔
67	passes=None, mindepth=0, threads=None, memory=None):
68	"""
69	Run bbnorm for read normalization/deduplication.
70
71	Args:
72	inFastq: Input FASTQ file (interleaved for paired-end, or single-end)
73	outFastq: Output FASTQ file (same format as input)
74	tmpdir: Temporary directory for multipass processing
75	target: Target normalization depth (default: bbnorm default of 100)
76	k: Kmer length (default: bbnorm default of 31)
77	passes: Number of passes (default: bbnorm default of 2)
78	mindepth: Min kmer depth threshold (default: 0 to include all)
79	threads: Number of threads (default: auto)
80	memory: Java memory allocation string (e.g., "4g")
81
82	Note: bbnorm auto-detects interleaved vs single-end format.
83	"""
84	tool_dir = os.path.dirname(self.install_and_get_path())	1✔
85	tool_cmd = [os.path.join(tool_dir, 'bbnorm.sh'), '-eoom']	1✔
86
87	if memory:	1✔
88	tool_cmd.append('-Xmx{}'.format(memory))	1✔
89
90	# Build arguments
91	tool_cmd.append('in={}'.format(inFastq))	1✔
92	tool_cmd.append('out={}'.format(outFastq))	1✔
93	tool_cmd.append('mindepth={}'.format(mindepth))	1✔
94
95	if tmpdir is not None:	1✔
96	tool_cmd.append('tmpdir={}'.format(tmpdir))	1✔
97	if target is not None:	1✔
98	tool_cmd.append('target={}'.format(target))	1✔
99	if k is not None:	1✔
NEW 100	tool_cmd.append('k={}'.format(k))	×
101	if passes is not None:	1✔
NEW 102	tool_cmd.append('passes={}'.format(passes))	×
103	if threads is not None:	1✔
104	tool_cmd.append('threads={}'.format(threads))	1✔
105
106	_log.debug('Running bbnorm: %s', ' '.join(tool_cmd))	1✔
107	subprocess.check_call(tool_cmd)	1✔
108

broadinstitute / viral-core / 20829376976

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous