#990

Committed 09 Mar 2025 01:19PM UTC coverage: 3.111% (-0.02%) from 3.134%

Build # #990

Build Type

push

Committed by

Commit Message

perf: optimize duplicate file search for large file sets

Improve performance when searching large number of files by:
1. Pre-filtering files by size before MD5 calculation
2. Using thread pool for parallel processing
3. Adding progress bar for better user feedback
4. Only calculating MD5 for potential duplicate groups

Run Details

0 of 74 new or added lines in 1 file covered. (0.0%)

3 existing lines in 1 file now uncovered.

236 of 7587 relevant lines covered (3.11%)

0.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/edu/jiangxin/apktoolbox/file/duplicate/DuplicateSearchPanel.java

package edu.jiangxin.apktoolbox.file.duplicate;

import edu.jiangxin.apktoolbox.utils.DateUtils;
import edu.jiangxin.apktoolbox.swing.extend.FileListPanel;
import edu.jiangxin.apktoolbox.swing.extend.EasyPanel;
import edu.jiangxin.apktoolbox.utils.Constants;
import edu.jiangxin.apktoolbox.utils.FileUtils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;

import javax.swing.*;
import javax.swing.table.DefaultTableModel;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.io.*;
import java.util.List;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;

public class DuplicateSearchPanel extends EasyPanel {

    @Serial
    private static final long serialVersionUID = 1L;

    private JTabbedPane tabbedPane;

    private JPanel optionPanel;

    private FileListPanel fileListPanel;

    private JCheckBox isFileNameChecked;
    private JCheckBox isMD5Checked;
    private JCheckBox isModifiedTimeChecked;

    private JCheckBox isHiddenFileSearched;
    private JCheckBox isRecursiveSearched;
    private JTextField suffixTextField;

    private JPanel resultPanel;

    private JTable resultTable;

    private DefaultTableModel resultTableModel;

    private JButton searchButton;
    private JButton cancelButton;

    private JProgressBar progressBar;

    private JMenuItem openDirMenuItem;
    private JMenuItem deleteFileMenuItem;
    private JMenuItem deleteFilesInSameDirMenuItem;
    private JMenuItem deleteFilesInSameDirRecursiveMenuItem;

    private Thread searchThread;

    final private Map<String, List<File>> duplicateFileGroupMap = new HashMap<>();


    @Override
    public void initUI() {
        tabbedPane = new JTabbedPane();
        add(tabbedPane);

        createOptionPanel();
        tabbedPane.addTab("Option", null, optionPanel, "Show Search Options");

        createResultPanel();
        tabbedPane.addTab("Result", null, resultPanel, "Show Search Result");
    }

    private void createOptionPanel() {
        optionPanel = new JPanel();
        optionPanel.setLayout(new BoxLayout(optionPanel, BoxLayout.Y_AXIS));

        fileListPanel = new FileListPanel();

        JPanel checkOptionPanel = new JPanel();
        checkOptionPanel.setLayout(new BoxLayout(checkOptionPanel, BoxLayout.X_AXIS));
        checkOptionPanel.setBorder(BorderFactory.createTitledBorder("Check Options"));

        JCheckBox isSizeChecked = new JCheckBox("Size");
        isSizeChecked.setSelected(true);
        isSizeChecked.setEnabled(false);
        isFileNameChecked = new JCheckBox("Filename");
        isMD5Checked = new JCheckBox("MD5");
        isModifiedTimeChecked = new JCheckBox("Last Modified Time");
        checkOptionPanel.add(isSizeChecked);
        checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        checkOptionPanel.add(isFileNameChecked);
        checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        checkOptionPanel.add(isMD5Checked);
        checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        checkOptionPanel.add(isModifiedTimeChecked);
        checkOptionPanel.add(Box.createHorizontalGlue());

        JPanel searchOptionPanel = new JPanel();
        searchOptionPanel.setLayout(new BoxLayout(searchOptionPanel, BoxLayout.X_AXIS));
        searchOptionPanel.setBorder(BorderFactory.createTitledBorder("Search Options"));

        isHiddenFileSearched = new JCheckBox("Hidden Files");
        isRecursiveSearched = new JCheckBox("Recursive");
        isRecursiveSearched.setSelected(true);
        JLabel suffixLabel = new JLabel("Suffix: ");
        suffixTextField = new JTextField();
        suffixTextField.setToolTipText("an array of extensions, ex. {\"java\",\"xml\"}. If this parameter is empty, all files are returned.");
        searchOptionPanel.add(isHiddenFileSearched);
        searchOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        searchOptionPanel.add(isRecursiveSearched);
        searchOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        searchOptionPanel.add(suffixLabel);
        searchOptionPanel.add(suffixTextField);
        searchOptionPanel.add(Box.createHorizontalGlue());

        JPanel operationPanel = new JPanel();
        operationPanel.setLayout(new BoxLayout(operationPanel, BoxLayout.X_AXIS));
        operationPanel.setBorder(BorderFactory.createTitledBorder("Operations"));

        JPanel buttonPanel = new JPanel();
        buttonPanel.setLayout(new BoxLayout(buttonPanel, BoxLayout.X_AXIS));
        
        searchButton = new JButton("Search");
        cancelButton = new JButton("Cancel");
        searchButton.addActionListener(new OperationButtonActionListener());
        cancelButton.addActionListener(new OperationButtonActionListener());
        operationPanel.add(searchButton);
        operationPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        operationPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));
        operationPanel.add(cancelButton);
        operationPanel.add(Box.createHorizontalGlue());

        progressBar = new JProgressBar();
        progressBar.setStringPainted(true);
        progressBar.setString("Ready");

        optionPanel.add(fileListPanel);
        optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));
        optionPanel.add(checkOptionPanel);
        optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));
        optionPanel.add(searchOptionPanel);
        optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));
        optionPanel.add(operationPanel);
                optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));
        optionPanel.add(progressBar);
    }

    private void createResultPanel() {
        resultPanel = new JPanel();
        resultPanel.setLayout(new BoxLayout(resultPanel, BoxLayout.Y_AXIS));

        resultTableModel = new DuplicateFilesTableModel(new Vector<>(), DuplicateFilesConstants.COLUMN_NAMES);
        resultTable = new JTable(resultTableModel);

        resultTable.setDefaultRenderer(Vector.class, new DuplicateFilesTableCellRenderer());

        for (int i = 0; i < resultTable.getColumnCount(); i++) {
            resultTable.getColumn(resultTable.getColumnName(i)).setCellRenderer(new DuplicateFilesTableCellRenderer());
        }

        resultTable.addMouseListener(new MyMouseListener());

        resultTable.setSelectionMode(ListSelectionModel.SINGLE_INTERVAL_SELECTION);

        JScrollPane scrollPane = new JScrollPane(resultTable);
        resultPanel.add(scrollPane);
    }

    public String getComparedKey(File file) {
        StringBuilder sb = new StringBuilder();
        sb.append("[Size][");
        sb.append(DigestUtils.md5Hex(String.valueOf(file.length())));
        sb.append("]");
        
        if (isFileNameChecked.isSelected()) {
            sb.append("[Filename][");
            sb.append(DigestUtils.md5Hex(file.getName()));
            sb.append("]");
        }
        if (isMD5Checked.isSelected()) {
            sb.append("[MD5][");
            try (InputStream is = new FileInputStream(file)) {
                sb.append(DigestUtils.md5Hex(is));
            } catch (FileNotFoundException e) {
                logger.error("getComparedKey FileNotFoundException");
            } catch (IOException e) {
                logger.error("getComparedKey IOException");
            }
            sb.append("]");
        }
        if (isModifiedTimeChecked.isSelected()) {
            sb.append("[ModifiedTime][");
            sb.append(DigestUtils.md5Hex(String.valueOf(file.lastModified())));
            sb.append("]");
        }
        logger.info("path: " + file.getAbsolutePath() + ", key: " + sb);
        return sb.toString();
    }

    class MyMouseListener extends MouseAdapter {
        @Override
        public void mouseReleased(MouseEvent e) {
            super.mouseReleased(e);
            int r = resultTable.rowAtPoint(e.getPoint());
            if (r >= 0 && r < resultTable.getRowCount()) {
                resultTable.setRowSelectionInterval(r, r);
            } else {
                resultTable.clearSelection();
            }
            int rowIndex = resultTable.getSelectedRow();
            if (rowIndex < 0) {
                return;
            }
            if (e.isPopupTrigger() && e.getComponent() instanceof JTable) {
                JPopupMenu popupmenu = new JPopupMenu();
                MyMenuActionListener menuActionListener = new MyMenuActionListener();

                openDirMenuItem = new JMenuItem("Open parent folder of this file");
                openDirMenuItem.addActionListener(menuActionListener);
                popupmenu.add(openDirMenuItem);

                deleteFileMenuItem = new JMenuItem("Delete this duplicate file");
                deleteFileMenuItem.addActionListener(menuActionListener);
                popupmenu.add(deleteFileMenuItem);

                deleteFilesInSameDirMenuItem = new JMenuItem("Delete these duplicate files in the same directory");
                deleteFilesInSameDirMenuItem.addActionListener(menuActionListener);
                popupmenu.add(deleteFilesInSameDirMenuItem);

                deleteFilesInSameDirRecursiveMenuItem = new JMenuItem("Delete these duplicate files in the same directory(Recursive)");
                deleteFilesInSameDirRecursiveMenuItem.addActionListener(menuActionListener);
                popupmenu.add(deleteFilesInSameDirRecursiveMenuItem);

                popupmenu.show(e.getComponent(), e.getX(), e.getY());
            }
        }
    }

    class MyMenuActionListener implements ActionListener {
        @Override
        public void actionPerformed(ActionEvent actionEvent) {
            Object source = actionEvent.getSource();
            if (source.equals(openDirMenuItem)) {
                onOpenDir();
            } else if (source.equals(deleteFileMenuItem)) {
                onDeleteFile();
            } else if (source.equals(deleteFilesInSameDirMenuItem)) {
                onDeleteFilesInSameDir();
            } else if (source.equals(deleteFilesInSameDirRecursiveMenuItem)) {
                onDeleteFilesInSameDirRecursive();
            } else {
                logger.error("invalid source");
            }
        }

        private void onOpenDir() {
            int rowIndex = resultTable.getSelectedRow();
            String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();
            File parent = new File(parentPath);
            if (parent.isDirectory()) {
                try {
                    Desktop.getDesktop().open(parent);
                } catch (IOException e) {
                    logger.error("open parent failed: " + parent.getPath());
                }
            }
        }

        private void onDeleteFile() {
            int rowIndex = resultTable.getSelectedRow();
            String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();
            String name = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_NAME).getModelIndex()).toString();
            File selectedFile = new File(parentPath, name);
            String key = getComparedKey(selectedFile);
            List<File> files = duplicateFileGroupMap.get(key);
            for (File file : files) {
                if (!selectedFile.equals(file)) {
                    continue;
                }
                files.remove(file);
                boolean isSuccessful = file.delete();
                logger.info("delete file: " + file.getAbsolutePath() + ", result: " + isSuccessful);
                break;
            }
            resultTableModel.setRowCount(0);
            showResult();
        }

        private void onDeleteFilesInSameDir() {
            int rowIndex = resultTable.getSelectedRow();
            String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();
            for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {
                List<File> duplicateFileGroup = entry.getValue();
                for (File duplicateFile : duplicateFileGroup) {
                    String parentPathTmp = duplicateFile.getParent();
                    if (Objects.equals(parentPath, parentPathTmp)) {
                        duplicateFileGroup.remove(duplicateFile);
                        boolean isSuccessful = duplicateFile.delete();
                        logger.info("delete file: " + duplicateFile.getAbsolutePath() + ", result: " + isSuccessful);
                        break;
                    }
                }
            }
            resultTableModel.setRowCount(0);
            showResult();
        }

        private void onDeleteFilesInSameDirRecursive() {
            int rowIndex = resultTable.getSelectedRow();
            String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();
            for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {
                List<File> duplicateFileGroup = entry.getValue();
                for (File duplicateFile : duplicateFileGroup) {
                    String parentPathTmp = duplicateFile.getParent();
                    if (Objects.equals(parentPath, parentPathTmp) || FilenameUtils.directoryContains(parentPath, parentPathTmp)) {
                        duplicateFileGroup.remove(duplicateFile);
                        boolean isSuccessful = duplicateFile.delete();
                        logger.info("delete file: " + duplicateFile.getAbsolutePath() + ", result: " + isSuccessful);
                        break;
                    }
                }
            }
            resultTableModel.setRowCount(0);
            showResult();
        }
    }

    class OperationButtonActionListener implements ActionListener {
        @Override
        public void actionPerformed(ActionEvent e) {
            Object source = e.getSource();
            if (source.equals(searchButton)) {
                String[] extensions = null;
                if (StringUtils.isNotEmpty(suffixTextField.getText())) {
                    extensions = suffixTextField.getText().split(",");
                }
                searchThread = new SearchThread(extensions, isRecursiveSearched.isSelected(), isHiddenFileSearched.isSelected(), duplicateFileGroupMap);
                searchThread.start();
            } else if (source.equals(cancelButton)) {
                if (searchThread.isAlive()) {
                    searchThread.interrupt();
                }
            }

        }
    }

    private void showResult() {
        SwingUtilities.invokeLater(() -> {
            int groupIndex = 0;
            for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {
                List<File> duplicateFileGroup = entry.getValue();
                if (duplicateFileGroup.size() < 2) {
                    continue;
                }
                groupIndex++;
                for (File duplicateFile : duplicateFileGroup) {
                    Vector<Object> rowData = getRowVector(groupIndex, duplicateFile);
                    resultTableModel.addRow(rowData);
                }
            }
            tabbedPane.setSelectedIndex(1);
        });
    }

    private Vector<Object> getRowVector(int groupIndex, File file) {
        Vector<Object> rowData = new Vector<>();
        rowData.add(groupIndex);
        rowData.add(file.getParent());
        rowData.add(file.getName());
        rowData.add(FilenameUtils.getExtension(file.getName()));
        rowData.add(FileUtils.sizeOfInHumanFormat(file));
        rowData.add(DateUtils.millisecondToHumanFormat(file.lastModified()));
        return rowData;
    }

    class SearchThread extends Thread {
        private final ExecutorService executorService;
        private final AtomicInteger processedFiles = new AtomicInteger(0);
        private int totalFiles = 0;
        private final String[] extensions;
        private final boolean isRecursiveSearched;
        private final boolean isHiddenFileSearched;
        private final Map<String, List<File>> duplicateFileGroupMap;

        public SearchThread(String[] extensions, boolean isRecursiveSearched, boolean isHiddenFileSearched, Map<String, List<File>> duplicateFileGroupMap) {
            super();
            this.extensions = extensions;
            this.isRecursiveSearched = isRecursiveSearched;
            this.isHiddenFileSearched = isHiddenFileSearched;
            this.duplicateFileGroupMap = duplicateFileGroupMap;
            this.executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());

            SwingUtilities.invokeLater(() -> {
                progressBar.setValue(0);
                progressBar.setString("Starting search...");
            });
        }

        @Override
        public void run() {
            try {
                duplicateFileGroupMap.clear();
                SwingUtilities.invokeLater(() -> {
                    resultTableModel.setRowCount(0);
                });

                List<File> fileList = fileListPanel.getFileList();
                Set<File> fileSet = new TreeSet<>(fileList);
                for (File file : fileList) {
                    fileSet.addAll(org.apache.commons.io.FileUtils.listFiles(file, extensions, isRecursiveSearched));
                }

                // 1. Group files by size first
                Map<Long, List<File>> sizeGroups = new HashMap<>();
                for (File file : fileSet) {
                    if (Thread.currentThread().isInterrupted()) {
                        return;
                    }
                    if (file.isHidden() && !isHiddenFileSearched) {
                        continue;
                    }
                    sizeGroups.computeIfAbsent(file.length(), k -> new ArrayList<>()).add(file);
                }

                // 2. Only process groups with duplicate sizes
                List<Future<?>> futures = new ArrayList<>();
                totalFiles = fileSet.size();
                updateProgress();

                for (Map.Entry<Long, List<File>> entry : sizeGroups.entrySet()) {
                    if (entry.getValue().size() > 1) { // Only process groups with duplicates
                        futures.add(executorService.submit(() -> {
                            processFileGroup(entry.getValue());
                            return null;
                        }));
                    } else {
                        // Count single files directly
                        incrementProcessedFiles();
                    }
                }

                // Wait for all tasks to complete
                for (Future<?> future : futures) {
                    future.get();
                }

                showResult();
            } catch (Exception e) {
                logger.error("Search failed", e);
                SwingUtilities.invokeLater(() -> progressBar.setString("Search failed"));
            } finally {
                executorService.shutdown();
            }
        }

        private void processFileGroup(List<File> files) {
            Map<String, List<File>> groupMap = new HashMap<>();
            for (File file : files) {
                if (Thread.currentThread().isInterrupted()) {
                    return;
                }
                String key = getComparedKey(file);
                groupMap.computeIfAbsent(key, k -> new ArrayList<>()).add(file);
                incrementProcessedFiles();
            }

            // Merge results to main map
            synchronized (duplicateFileGroupMap) {
                for (Map.Entry<String, List<File>> entry : groupMap.entrySet()) {
                    if (entry.getValue().size() > 1) {
                        duplicateFileGroupMap.put(entry.getKey(), entry.getValue());
                    }
                }
            }
        }

        private void incrementProcessedFiles() {
            processedFiles.incrementAndGet();
            updateProgress();
        }

        private void updateProgress() {
            if (totalFiles > 0) {
                SwingUtilities.invokeLater(() -> {
                    int processed = processedFiles.get();
                    int percentage = (int) ((processed * 100.0) / totalFiles);
                    progressBar.setValue(percentage);
                    progressBar.setString(String.format("Processing: %d/%d files (%d%%)", 
                        processed, totalFiles, percentage));
                });
            }
        }
    }
}

1	package edu.jiangxin.apktoolbox.file.duplicate;
2
3	import edu.jiangxin.apktoolbox.utils.DateUtils;
4	import edu.jiangxin.apktoolbox.swing.extend.FileListPanel;
5	import edu.jiangxin.apktoolbox.swing.extend.EasyPanel;
6	import edu.jiangxin.apktoolbox.utils.Constants;
7	import edu.jiangxin.apktoolbox.utils.FileUtils;
8	import org.apache.commons.codec.digest.DigestUtils;
9	import org.apache.commons.io.FilenameUtils;
10	import org.apache.commons.lang3.StringUtils;
11
12	import javax.swing.*;
13	import javax.swing.table.DefaultTableModel;
14	import java.awt.*;
15	import java.awt.event.ActionEvent;
16	import java.awt.event.ActionListener;
17	import java.awt.event.MouseAdapter;
18	import java.awt.event.MouseEvent;
19	import java.io.*;
20	import java.util.List;
21	import java.util.*;
22	import java.util.concurrent.ExecutorService;
23	import java.util.concurrent.Executors;
24	import java.util.concurrent.Future;
25	import java.util.concurrent.atomic.AtomicInteger;
26
27	public class DuplicateSearchPanel extends EasyPanel {	×
28
29	@Serial
30	private static final long serialVersionUID = 1L;
31
32	private JTabbedPane tabbedPane;
33
34	private JPanel optionPanel;
35
36	private FileListPanel fileListPanel;
37
38	private JCheckBox isFileNameChecked;
39	private JCheckBox isMD5Checked;
40	private JCheckBox isModifiedTimeChecked;
41
42	private JCheckBox isHiddenFileSearched;
43	private JCheckBox isRecursiveSearched;
44	private JTextField suffixTextField;
45
46	private JPanel resultPanel;
47
48	private JTable resultTable;
49
50	private DefaultTableModel resultTableModel;
51
52	private JButton searchButton;
53	private JButton cancelButton;
54
55	private JProgressBar progressBar;
56
57	private JMenuItem openDirMenuItem;
58	private JMenuItem deleteFileMenuItem;
59	private JMenuItem deleteFilesInSameDirMenuItem;
60	private JMenuItem deleteFilesInSameDirRecursiveMenuItem;
61
62	private Thread searchThread;
63
64	final private Map<String, List<File>> duplicateFileGroupMap = new HashMap<>();	×
65
66
67	@Override
68	public void initUI() {
69	tabbedPane = new JTabbedPane();	×
70	add(tabbedPane);	×
71
72	createOptionPanel();	×
73	tabbedPane.addTab("Option", null, optionPanel, "Show Search Options");	×
74
75	createResultPanel();	×
76	tabbedPane.addTab("Result", null, resultPanel, "Show Search Result");	×
77	}	×
78
79	private void createOptionPanel() {
80	optionPanel = new JPanel();	×
81	optionPanel.setLayout(new BoxLayout(optionPanel, BoxLayout.Y_AXIS));	×
82
83	fileListPanel = new FileListPanel();	×
84
85	JPanel checkOptionPanel = new JPanel();	×
86	checkOptionPanel.setLayout(new BoxLayout(checkOptionPanel, BoxLayout.X_AXIS));	×
87	checkOptionPanel.setBorder(BorderFactory.createTitledBorder("Check Options"));	×
88
NEW 89	JCheckBox isSizeChecked = new JCheckBox("Size");	×
90	isSizeChecked.setSelected(true);	×
91	isSizeChecked.setEnabled(false);	×
92	isFileNameChecked = new JCheckBox("Filename");	×
93	isMD5Checked = new JCheckBox("MD5");	×
94	isModifiedTimeChecked = new JCheckBox("Last Modified Time");	×
95	checkOptionPanel.add(isSizeChecked);	×
96	checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
97	checkOptionPanel.add(isFileNameChecked);	×
98	checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
99	checkOptionPanel.add(isMD5Checked);	×
100	checkOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
101	checkOptionPanel.add(isModifiedTimeChecked);	×
102	checkOptionPanel.add(Box.createHorizontalGlue());	×
103
104	JPanel searchOptionPanel = new JPanel();	×
105	searchOptionPanel.setLayout(new BoxLayout(searchOptionPanel, BoxLayout.X_AXIS));	×
106	searchOptionPanel.setBorder(BorderFactory.createTitledBorder("Search Options"));	×
107
108	isHiddenFileSearched = new JCheckBox("Hidden Files");	×
109	isRecursiveSearched = new JCheckBox("Recursive");	×
110	isRecursiveSearched.setSelected(true);	×
111	JLabel suffixLabel = new JLabel("Suffix: ");	×
112	suffixTextField = new JTextField();	×
113	suffixTextField.setToolTipText("an array of extensions, ex. {\"java\",\"xml\"}. If this parameter is empty, all files are returned.");	×
114	searchOptionPanel.add(isHiddenFileSearched);	×
115	searchOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
116	searchOptionPanel.add(isRecursiveSearched);	×
117	searchOptionPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
118	searchOptionPanel.add(suffixLabel);	×
119	searchOptionPanel.add(suffixTextField);	×
120	searchOptionPanel.add(Box.createHorizontalGlue());	×
121
122	JPanel operationPanel = new JPanel();	×
123	operationPanel.setLayout(new BoxLayout(operationPanel, BoxLayout.X_AXIS));	×
124	operationPanel.setBorder(BorderFactory.createTitledBorder("Operations"));	×
125
NEW 126	JPanel buttonPanel = new JPanel();	×
NEW 127	buttonPanel.setLayout(new BoxLayout(buttonPanel, BoxLayout.X_AXIS));	×
128
129	searchButton = new JButton("Search");	×
130	cancelButton = new JButton("Cancel");	×
131	searchButton.addActionListener(new OperationButtonActionListener());	×
132	cancelButton.addActionListener(new OperationButtonActionListener());	×
133	operationPanel.add(searchButton);	×
134	operationPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
135	operationPanel.add(Box.createHorizontalStrut(Constants.DEFAULT_X_BORDER));	×
136	operationPanel.add(cancelButton);	×
137	operationPanel.add(Box.createHorizontalGlue());	×
138
NEW 139	progressBar = new JProgressBar();	×
NEW 140	progressBar.setStringPainted(true);	×
NEW 141	progressBar.setString("Ready");	×
142
143	optionPanel.add(fileListPanel);	×
144	optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));	×
145	optionPanel.add(checkOptionPanel);	×
146	optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));	×
147	optionPanel.add(searchOptionPanel);	×
148	optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));	×
149	optionPanel.add(operationPanel);	×
NEW 150	optionPanel.add(Box.createVerticalStrut(Constants.DEFAULT_Y_BORDER));	×
NEW 151	optionPanel.add(progressBar);	×
UNCOV 152	}	×
153
154	private void createResultPanel() {
155	resultPanel = new JPanel();	×
156	resultPanel.setLayout(new BoxLayout(resultPanel, BoxLayout.Y_AXIS));	×
157
158	resultTableModel = new DuplicateFilesTableModel(new Vector<>(), DuplicateFilesConstants.COLUMN_NAMES);	×
159	resultTable = new JTable(resultTableModel);	×
160
161	resultTable.setDefaultRenderer(Vector.class, new DuplicateFilesTableCellRenderer());	×
162
163	for (int i = 0; i < resultTable.getColumnCount(); i++) {	×
164	resultTable.getColumn(resultTable.getColumnName(i)).setCellRenderer(new DuplicateFilesTableCellRenderer());	×
165	}
166
167	resultTable.addMouseListener(new MyMouseListener());	×
168
169	resultTable.setSelectionMode(ListSelectionModel.SINGLE_INTERVAL_SELECTION);	×
170
171	JScrollPane scrollPane = new JScrollPane(resultTable);	×
172	resultPanel.add(scrollPane);	×
173	}	×
174
175	public String getComparedKey(File file) {
176	StringBuilder sb = new StringBuilder();	×
177	sb.append("[Size][");	×
178	sb.append(DigestUtils.md5Hex(String.valueOf(file.length())));	×
179	sb.append("]");	×
180
181	if (isFileNameChecked.isSelected()) {	×
182	sb.append("[Filename][");	×
183	sb.append(DigestUtils.md5Hex(file.getName()));	×
184	sb.append("]");	×
185	}
186	if (isMD5Checked.isSelected()) {	×
187	sb.append("[MD5][");	×
188	try (InputStream is = new FileInputStream(file)) {	×
189	sb.append(DigestUtils.md5Hex(is));	×
190	} catch (FileNotFoundException e) {	×
191	logger.error("getComparedKey FileNotFoundException");	×
192	} catch (IOException e) {	×
193	logger.error("getComparedKey IOException");	×
194	}	×
195	sb.append("]");	×
196	}
197	if (isModifiedTimeChecked.isSelected()) {	×
198	sb.append("[ModifiedTime][");	×
199	sb.append(DigestUtils.md5Hex(String.valueOf(file.lastModified())));	×
200	sb.append("]");	×
201	}
202	logger.info("path: " + file.getAbsolutePath() + ", key: " + sb);	×
203	return sb.toString();	×
204	}
205
206	class MyMouseListener extends MouseAdapter {	×
207	@Override
208	public void mouseReleased(MouseEvent e) {
209	super.mouseReleased(e);	×
210	int r = resultTable.rowAtPoint(e.getPoint());	×
211	if (r >= 0 && r < resultTable.getRowCount()) {	×
212	resultTable.setRowSelectionInterval(r, r);	×
213	} else {
214	resultTable.clearSelection();	×
215	}
216	int rowIndex = resultTable.getSelectedRow();	×
217	if (rowIndex < 0) {	×
218	return;	×
219	}
220	if (e.isPopupTrigger() && e.getComponent() instanceof JTable) {	×
221	JPopupMenu popupmenu = new JPopupMenu();	×
222	MyMenuActionListener menuActionListener = new MyMenuActionListener();	×
223
224	openDirMenuItem = new JMenuItem("Open parent folder of this file");	×
225	openDirMenuItem.addActionListener(menuActionListener);	×
226	popupmenu.add(openDirMenuItem);	×
227
228	deleteFileMenuItem = new JMenuItem("Delete this duplicate file");	×
229	deleteFileMenuItem.addActionListener(menuActionListener);	×
230	popupmenu.add(deleteFileMenuItem);	×
231
232	deleteFilesInSameDirMenuItem = new JMenuItem("Delete these duplicate files in the same directory");	×
233	deleteFilesInSameDirMenuItem.addActionListener(menuActionListener);	×
234	popupmenu.add(deleteFilesInSameDirMenuItem);	×
235
236	deleteFilesInSameDirRecursiveMenuItem = new JMenuItem("Delete these duplicate files in the same directory(Recursive)");	×
237	deleteFilesInSameDirRecursiveMenuItem.addActionListener(menuActionListener);	×
238	popupmenu.add(deleteFilesInSameDirRecursiveMenuItem);	×
239
240	popupmenu.show(e.getComponent(), e.getX(), e.getY());	×
241	}
242	}	×
243	}
244
245	class MyMenuActionListener implements ActionListener {	×
246	@Override
247	public void actionPerformed(ActionEvent actionEvent) {
248	Object source = actionEvent.getSource();	×
249	if (source.equals(openDirMenuItem)) {	×
250	onOpenDir();	×
251	} else if (source.equals(deleteFileMenuItem)) {	×
252	onDeleteFile();	×
253	} else if (source.equals(deleteFilesInSameDirMenuItem)) {	×
254	onDeleteFilesInSameDir();	×
255	} else if (source.equals(deleteFilesInSameDirRecursiveMenuItem)) {	×
256	onDeleteFilesInSameDirRecursive();	×
257	} else {
258	logger.error("invalid source");	×
259	}
260	}	×
261
262	private void onOpenDir() {
263	int rowIndex = resultTable.getSelectedRow();	×
264	String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();	×
265	File parent = new File(parentPath);	×
266	if (parent.isDirectory()) {	×
267	try {
268	Desktop.getDesktop().open(parent);	×
269	} catch (IOException e) {	×
270	logger.error("open parent failed: " + parent.getPath());	×
271	}	×
272	}
273	}	×
274
275	private void onDeleteFile() {
276	int rowIndex = resultTable.getSelectedRow();	×
277	String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();	×
278	String name = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_NAME).getModelIndex()).toString();	×
279	File selectedFile = new File(parentPath, name);	×
280	String key = getComparedKey(selectedFile);	×
281	List<File> files = duplicateFileGroupMap.get(key);	×
282	for (File file : files) {	×
283	if (!selectedFile.equals(file)) {	×
284	continue;	×
285	}
286	files.remove(file);	×
287	boolean isSuccessful = file.delete();	×
288	logger.info("delete file: " + file.getAbsolutePath() + ", result: " + isSuccessful);	×
289	break;	×
290	}
291	resultTableModel.setRowCount(0);	×
292	showResult();	×
293	}	×
294
295	private void onDeleteFilesInSameDir() {
296	int rowIndex = resultTable.getSelectedRow();	×
297	String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();	×
298	for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {	×
299	List<File> duplicateFileGroup = entry.getValue();	×
300	for (File duplicateFile : duplicateFileGroup) {	×
301	String parentPathTmp = duplicateFile.getParent();	×
302	if (Objects.equals(parentPath, parentPathTmp)) {	×
303	duplicateFileGroup.remove(duplicateFile);	×
304	boolean isSuccessful = duplicateFile.delete();	×
305	logger.info("delete file: " + duplicateFile.getAbsolutePath() + ", result: " + isSuccessful);	×
306	break;	×
307	}
308	}	×
309	}	×
310	resultTableModel.setRowCount(0);	×
311	showResult();	×
312	}	×
313
314	private void onDeleteFilesInSameDirRecursive() {
315	int rowIndex = resultTable.getSelectedRow();	×
316	String parentPath = resultTableModel.getValueAt(rowIndex, resultTable.getColumn(DuplicateFilesConstants.COLUMN_NAME_FILE_PARENT).getModelIndex()).toString();	×
317	for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {	×
318	List<File> duplicateFileGroup = entry.getValue();	×
319	for (File duplicateFile : duplicateFileGroup) {	×
320	String parentPathTmp = duplicateFile.getParent();	×
321	if (Objects.equals(parentPath, parentPathTmp) \|\| FilenameUtils.directoryContains(parentPath, parentPathTmp)) {	×
322	duplicateFileGroup.remove(duplicateFile);	×
323	boolean isSuccessful = duplicateFile.delete();	×
324	logger.info("delete file: " + duplicateFile.getAbsolutePath() + ", result: " + isSuccessful);	×
325	break;	×
326	}
327	}	×
328	}	×
329	resultTableModel.setRowCount(0);	×
330	showResult();	×
331	}	×
332	}
333
334	class OperationButtonActionListener implements ActionListener {	×
335	@Override
336	public void actionPerformed(ActionEvent e) {
337	Object source = e.getSource();	×
338	if (source.equals(searchButton)) {	×
339	String[] extensions = null;	×
340	if (StringUtils.isNotEmpty(suffixTextField.getText())) {	×
341	extensions = suffixTextField.getText().split(",");	×
342	}
343	searchThread = new SearchThread(extensions, isRecursiveSearched.isSelected(), isHiddenFileSearched.isSelected(), duplicateFileGroupMap);	×
344	searchThread.start();	×
345	} else if (source.equals(cancelButton)) {	×
346	if (searchThread.isAlive()) {	×
347	searchThread.interrupt();	×
348	}
349	}
350
351	}	×
352	}
353
354	private void showResult() {
355	SwingUtilities.invokeLater(() -> {	×
356	int groupIndex = 0;	×
357	for (Map.Entry<String, List<File>> entry : duplicateFileGroupMap.entrySet()) {	×
358	List<File> duplicateFileGroup = entry.getValue();	×
359	if (duplicateFileGroup.size() < 2) {	×
360	continue;	×
361	}
362	groupIndex++;	×
363	for (File duplicateFile : duplicateFileGroup) {	×
364	Vector<Object> rowData = getRowVector(groupIndex, duplicateFile);	×
365	resultTableModel.addRow(rowData);	×
366	}	×
367	}	×
368	tabbedPane.setSelectedIndex(1);	×
369	});	×
370	}	×
371
372	private Vector<Object> getRowVector(int groupIndex, File file) {
373	Vector<Object> rowData = new Vector<>();	×
374	rowData.add(groupIndex);	×
375	rowData.add(file.getParent());	×
376	rowData.add(file.getName());	×
377	rowData.add(FilenameUtils.getExtension(file.getName()));	×
378	rowData.add(FileUtils.sizeOfInHumanFormat(file));	×
379	rowData.add(DateUtils.millisecondToHumanFormat(file.lastModified()));	×
380	return rowData;	×
381	}
382
383	class SearchThread extends Thread {
384	private final ExecutorService executorService;
NEW 385	private final AtomicInteger processedFiles = new AtomicInteger(0);	×
NEW 386	private int totalFiles = 0;	×
387	private final String[] extensions;
388	private final boolean isRecursiveSearched;
389	private final boolean isHiddenFileSearched;
390	private final Map<String, List<File>> duplicateFileGroupMap;
391
392	public SearchThread(String[] extensions, boolean isRecursiveSearched, boolean isHiddenFileSearched, Map<String, List<File>> duplicateFileGroupMap) {	×
393	super();	×
394	this.extensions = extensions;	×
395	this.isRecursiveSearched = isRecursiveSearched;	×
396	this.isHiddenFileSearched = isHiddenFileSearched;	×
397	this.duplicateFileGroupMap = duplicateFileGroupMap;	×
NEW 398	this.executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());	×
399
NEW 400	SwingUtilities.invokeLater(() -> {	×
NEW 401	progressBar.setValue(0);	×
NEW 402	progressBar.setString("Starting search...");	×
NEW 403	});	×
UNCOV 404	}	×
405
406	@Override
407	public void run() {
408	try {
NEW 409	duplicateFileGroupMap.clear();	×
NEW 410	SwingUtilities.invokeLater(() -> {	×
NEW 411	resultTableModel.setRowCount(0);	×
NEW 412	});	×
413
NEW 414	List<File> fileList = fileListPanel.getFileList();	×
NEW 415	Set<File> fileSet = new TreeSet<>(fileList);	×
NEW 416	for (File file : fileList) {	×
NEW 417	fileSet.addAll(org.apache.commons.io.FileUtils.listFiles(file, extensions, isRecursiveSearched));	×
NEW 418	}	×
419
420	// 1. Group files by size first
NEW 421	Map<Long, List<File>> sizeGroups = new HashMap<>();	×
NEW 422	for (File file : fileSet) {	×
NEW 423	if (Thread.currentThread().isInterrupted()) {	×
NEW 424	return;	×
425	}
NEW 426	if (file.isHidden() && !isHiddenFileSearched) {	×
NEW 427	continue;	×
428	}
NEW 429	sizeGroups.computeIfAbsent(file.length(), k -> new ArrayList<>()).add(file);	×
NEW 430	}	×
431
432	// 2. Only process groups with duplicate sizes
NEW 433	List<Future<?>> futures = new ArrayList<>();	×
NEW 434	totalFiles = fileSet.size();	×
NEW 435	updateProgress();	×
436
NEW 437	for (Map.Entry<Long, List<File>> entry : sizeGroups.entrySet()) {	×
NEW 438	if (entry.getValue().size() > 1) { // Only process groups with duplicates	×
NEW 439	futures.add(executorService.submit(() -> {	×
NEW 440	processFileGroup(entry.getValue());	×
NEW 441	return null;	×
442	}));
443	} else {
444	// Count single files directly
NEW 445	incrementProcessedFiles();	×
446	}
NEW 447	}	×
448
449	// Wait for all tasks to complete
NEW 450	for (Future<?> future : futures) {	×
NEW 451	future.get();	×
NEW 452	}	×
453
NEW 454	showResult();	×
NEW 455	} catch (Exception e) {	×
NEW 456	logger.error("Search failed", e);	×
NEW 457	SwingUtilities.invokeLater(() -> progressBar.setString("Search failed"));	×
458	} finally {
NEW 459	executorService.shutdown();	×
460	}
NEW 461	}	×
462
463	private void processFileGroup(List<File> files) {
NEW 464	Map<String, List<File>> groupMap = new HashMap<>();	×
NEW 465	for (File file : files) {	×
466	if (Thread.currentThread().isInterrupted()) {	×
NEW 467	return;	×
468	}
NEW 469	String key = getComparedKey(file);	×
NEW 470	groupMap.computeIfAbsent(key, k -> new ArrayList<>()).add(file);	×
NEW 471	incrementProcessedFiles();	×
NEW 472	}	×
473
474	// Merge results to main map
NEW 475	synchronized (duplicateFileGroupMap) {	×
NEW 476	for (Map.Entry<String, List<File>> entry : groupMap.entrySet()) {	×
NEW 477	if (entry.getValue().size() > 1) {	×
NEW 478	duplicateFileGroupMap.put(entry.getKey(), entry.getValue());	×
479	}
480	}	×
481	}	×
NEW 482	}	×
483
484	private void incrementProcessedFiles() {
NEW 485	processedFiles.incrementAndGet();	×
NEW 486	updateProgress();	×
NEW 487	}	×
488
489	private void updateProgress() {
NEW 490	if (totalFiles > 0) {	×
NEW 491	SwingUtilities.invokeLater(() -> {	×
NEW 492	int processed = processedFiles.get();	×
NEW 493	int percentage = (int) ((processed * 100.0) / totalFiles);	×
NEW 494	progressBar.setValue(percentage);	×
NEW 495	progressBar.setString(String.format("Processing: %d/%d files (%d%%)",	×
NEW 496	processed, totalFiles, percentage));	×
NEW 497	});	×
498	}
UNCOV 499	}	×
500	}
501	}

jiangxincode / ApkToolBoxGUI / #990

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous