#349

Committed 17 Jul 2025 09:52PM UTC coverage: 69.883% (+0.3%) from 69.604%

Build # #349

Build Type

push

Committed by

Commit Message

refactor: move BIP39 key producer to dedicated class, add coverage tests, update README with usage examples

Run Details

70 of 76 new or added lines in 10 files covered. (92.11%)

1 existing line in 1 file now uncovered.

1434 of 2052 relevant lines covered (69.88%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.16

/src/main/java/net/ladenthin/bitcoinaddressfinder/persistence/lmdb/LMDBPersistence.java

// @formatter:off
/**
 * Copyright 2020 Bernard Ladenthin bernard.ladenthin@gmail.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
// @formatter:on
package net.ladenthin.bitcoinaddressfinder.persistence.lmdb;

import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import net.ladenthin.bitcoinaddressfinder.persistence.Persistence;
import net.ladenthin.bitcoinaddressfinder.persistence.PersistenceUtils;
import org.lmdbjava.CursorIterable;
import org.lmdbjava.Dbi;
import org.lmdbjava.Env;
import org.lmdbjava.EnvFlags;
import org.lmdbjava.KeyRange;
import org.lmdbjava.Txn;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLongArray;
import net.ladenthin.bitcoinaddressfinder.ByteBufferUtility;
import net.ladenthin.bitcoinaddressfinder.ByteConversion;
import net.ladenthin.bitcoinaddressfinder.KeyUtility;
import net.ladenthin.bitcoinaddressfinder.SeparatorFormat;
import net.ladenthin.bitcoinaddressfinder.configuration.CAddressFileOutputFormat;
import net.ladenthin.bitcoinaddressfinder.configuration.CLMDBConfigurationReadOnly;
import net.ladenthin.bitcoinaddressfinder.configuration.CLMDBConfigurationWrite;
import org.apache.commons.codec.binary.Hex;
import org.bitcoinj.base.Coin;
import org.bitcoinj.base.LegacyAddress;
import org.lmdbjava.BufferProxy;
import org.lmdbjava.ByteBufferProxy;

import static org.lmdbjava.DbiFlags.MDB_CREATE;
import static org.lmdbjava.Env.create;
import org.lmdbjava.EnvInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class LMDBPersistence implements Persistence {

    private static final String DB_NAME_HASH160_TO_COINT = "hash160toCoin";
    private static final int DB_COUNT = 1;
    
    private final Logger logger = LoggerFactory.getLogger(LMDBPersistence.class);

    private final PersistenceUtils persistenceUtils;
    private final CLMDBConfigurationWrite lmdbConfigurationWrite;
    private final CLMDBConfigurationReadOnly lmdbConfigurationReadOnly;
    private final KeyUtility keyUtility;
    private Env<ByteBuffer> env;
    private Dbi<ByteBuffer> lmdb_h160ToAmount;
    private long increasedCounter = 0;
    private long increasedSum = 0;
    private BloomFilter<byte[]> addressBloomFilter = null;


    public LMDBPersistence(CLMDBConfigurationWrite lmdbConfigurationWrite, PersistenceUtils persistenceUtils) {
        this.lmdbConfigurationReadOnly = null;
        this.lmdbConfigurationWrite = lmdbConfigurationWrite;
        this.persistenceUtils = persistenceUtils;
        this.keyUtility = new KeyUtility(persistenceUtils.network, new ByteBufferUtility(true));
    }

    public LMDBPersistence(CLMDBConfigurationReadOnly lmdbConfigurationReadOnly, PersistenceUtils persistenceUtils) {
        this.lmdbConfigurationReadOnly = lmdbConfigurationReadOnly;
        lmdbConfigurationWrite = null;
        this.persistenceUtils = persistenceUtils;
        this.keyUtility = new KeyUtility(persistenceUtils.network, new ByteBufferUtility(true));
    }
    
    @Override
    public void init() {
        if (lmdbConfigurationWrite != null) {
            initWritable();
        } else if (lmdbConfigurationReadOnly != null) {
            initReadOnly();
        } else {
            throw new IllegalArgumentException("Neither write nor read-only configuration provided.");
        }
        
        
        logStatsIfConfigured(true);
    }
    
    public void buildAddressBloomFilter() {
        logger.info("##### BEGIN: buildAddressBloomFilter #####");
        // Attention: slow!
        long count = count();

        BloomFilter<byte[]> filter = BloomFilter.create(Funnels.byteArrayFunnel(), count, lmdbConfigurationReadOnly.bloomFilterFpp);
        long inserted = 0;

        try (Txn<ByteBuffer> txn = env.txnRead()) {
            try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {
                for (CursorIterable.KeyVal<ByteBuffer> kv : iterable) {
                    ByteBuffer key = kv.key();
                    byte[] keyBytes = new byte[key.remaining()];
                    key.get(keyBytes);
                    key.rewind();
                    filter.put(keyBytes);
                    inserted++;
                }
            }
        }

        addressBloomFilter = filter;
        long size = getApproximateSizeBytes(filter);
        logger.info("Inserted {} addresses into BloomFilter with size of {}", inserted, formatSize(size));
        logger.info("##### END: buildAddressBloomFilter #####");
    }

    public void unloadBloomFilter() {
        addressBloomFilter = null;
    }
    
    private void initReadOnly() {
        BufferProxy<ByteBuffer> bufferProxy = getBufferProxyByUseProxyOptimal(lmdbConfigurationReadOnly.useProxyOptimal);
        env = create(bufferProxy).setMaxDbs(DB_COUNT).open(new File(lmdbConfigurationReadOnly.lmdbDirectory), EnvFlags.MDB_RDONLY_ENV, EnvFlags.MDB_NOLOCK);
        lmdb_h160ToAmount = env.openDbi(DB_NAME_HASH160_TO_COINT);
        
        if (lmdbConfigurationReadOnly.useBloomFilter) {
            buildAddressBloomFilter();
        }
    }

    private void initWritable() {
        // -Xmx10G -XX:MaxDirectMemorySize=5G
        // We always need an Env. An Env owns a physical on-disk storage file. One
        // Env can store many different databases (ie sorted maps).
        File lmdbDirectory = new File(lmdbConfigurationWrite.lmdbDirectory);
        lmdbDirectory.mkdirs();
        
        BufferProxy<ByteBuffer> bufferProxy = getBufferProxyByUseProxyOptimal(lmdbConfigurationWrite.useProxyOptimal);
        
        env = create(bufferProxy)
                // LMDB also needs to know how large our DB might be. Over-estimating is OK.
                .setMapSize(new ByteConversion().mibToBytes(lmdbConfigurationWrite.initialMapSizeInMiB))
                // LMDB also needs to know how many DBs (Dbi) we want to store in this Env.
                .setMaxDbs(DB_COUNT)
                // Now let's open the Env. The same path can be concurrently opened and
                // used in different processes, but do not open the same path twice in
                // the same process at the same time.
                
                //https://github.com/kentnl/CHI-Driver-LMDB
                .open(lmdbDirectory, EnvFlags.MDB_NOSYNC, EnvFlags.MDB_NOMETASYNC, EnvFlags.MDB_WRITEMAP, EnvFlags.MDB_MAPASYNC);
        // We need a Dbi for each DB. A Dbi roughly equates to a sorted map. The
        // MDB_CREATE flag causes the DB to be created if it doesn't already exist.
        lmdb_h160ToAmount = env.openDbi(DB_NAME_HASH160_TO_COINT, MDB_CREATE);
    }

    /**
     * https://github.com/lmdbjava/lmdbjava/wiki/Buffers
     *
     * @param useProxyOptimal
     * @return
     */
    private BufferProxy<ByteBuffer> getBufferProxyByUseProxyOptimal(boolean useProxyOptimal) {
        if (useProxyOptimal) {
            return ByteBufferProxy.PROXY_OPTIMAL;
        } else {
            return ByteBufferProxy.PROXY_SAFE;
        }
    }
    
    private void logStatsIfConfigured(boolean onInit) {
        if (isLoggingEnabled(lmdbConfigurationWrite, onInit) || isLoggingEnabled(lmdbConfigurationReadOnly, onInit)) {
            logStats();
        }
    }

    private boolean isLoggingEnabled(CLMDBConfigurationReadOnly config, boolean onInit) {
        return config != null && (onInit ? config.logStatsOnInit : config.logStatsOnClose);
    }

    @Override
    public void close() {
        logStatsIfConfigured(false);
        lmdb_h160ToAmount.close();
        env.close();
    }
    
    @Override
    public boolean isClosed() {
        return env.isClosed();
    }

    @Override
    public Coin getAmount(ByteBuffer hash160) {
        try (Txn<ByteBuffer> txn = env.txnRead()) {
            ByteBuffer byteBuffer = lmdb_h160ToAmount.get(txn, hash160);
            return getCoinFromByteBuffer(byteBuffer);
        }
    }
    
    private Coin getCoinFromByteBuffer(ByteBuffer byteBuffer) {
        if (byteBuffer == null || byteBuffer.capacity() == 0) {
            return Coin.ZERO;
        }
        return Coin.valueOf(byteBuffer.getLong());
    }

    @Override
    public boolean containsAddress(ByteBuffer hash160) {
        if (lmdbConfigurationReadOnly.disableAddressLookup) {
            return false;
        }
        
        byte[] hash160AsByteArray = new byte[hash160.remaining()];
        hash160.get(hash160AsByteArray);
        hash160.rewind();
        
        // Use Bloom filter if available for fast pre-check
        if (addressBloomFilter != null) {
            boolean mightContain = addressBloomFilter.mightContain(hash160AsByteArray);
            if (!mightContain) {
                return false; // definitely not present
            }
            // Possibly in DB, proceed to verify
        }
        
        // Perform LMDB lookup (always happens if no Bloom filter is present)
        try (Txn<ByteBuffer> txn = env.txnRead()) {
            ByteBuffer byteBuffer = lmdb_h160ToAmount.get(txn, hash160);
            return byteBuffer != null;
        }
    }

    @Override
    public void writeAllAmountsToAddressFile(File file, CAddressFileOutputFormat addressFileOutputFormat, AtomicBoolean shouldRun) throws IOException {
        try (Txn<ByteBuffer> txn = env.txnRead()) {
            try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {
                try (FileWriter writer = new FileWriter(file)) {
                    for (final CursorIterable.KeyVal<ByteBuffer> kv : iterable) {
                        if (!shouldRun.get()) {
                            return;
                        }
                        ByteBuffer addressAsByteBuffer = kv.key();
                        if(logger.isTraceEnabled()) {
                            String hexFromByteBuffer = new ByteBufferUtility(false).getHexFromByteBuffer(addressAsByteBuffer);
                            logger.trace("Process address: " + hexFromByteBuffer);
                        }
                        LegacyAddress address = keyUtility.byteBufferToAddress(addressAsByteBuffer);
                        final String line;
                        switch(addressFileOutputFormat) {
                            case HexHash:
                                line = Hex.encodeHexString(address.getHash()) + System.lineSeparator();
                                break;
                            case FixedWidthBase58BitcoinAddress:
                                line = String.format("%-34s", address.toBase58()) + System.lineSeparator();
                                break;
                            case DynamicWidthBase58BitcoinAddressWithAmount:
                                ByteBuffer value = kv.val();
                                Coin coin = getCoinFromByteBuffer(value);
                                line = address.toBase58() + SeparatorFormat.COMMA.getSymbol() + coin.getValue() + System.lineSeparator();
                                break;
                            default:
                                throw new IllegalArgumentException("Unknown addressFileOutputFormat: " + addressFileOutputFormat);
                        }
                        writer.write(line);
                    }
                }
            }
        }
    }

    @Override
    public void putAllAmounts(Map<ByteBuffer, Coin> amounts) throws IOException {
        for (Map.Entry<ByteBuffer, Coin> entry : amounts.entrySet()) {
            ByteBuffer hash160 = entry.getKey();
            Coin coin = entry.getValue();
            putNewAmount(hash160, coin);
        }
    }

    @Override
    public void changeAmount(ByteBuffer hash160, Coin amountToChange) {
        Coin valueInDB = getAmount(hash160);
        Coin toWrite = valueInDB.add(amountToChange);
        putNewAmount(hash160, toWrite);
    }

    @Override
    public void putNewAmount(ByteBuffer hash160, Coin amount) {
        putNewAmountWithAutoIncrease(hash160, amount);
    }
    
    /**
     * If an {@link org.lmdbjava.Env.MapFullException} was thrown during a put. The map might be increased if configured.
     * The increase value needs to be high enough. Otherwise the next put fails nevertheless.
     */
    private void putNewAmountWithAutoIncrease(ByteBuffer hash160, Coin amount) {
        try {
            putNewAmountUnsafe(hash160, amount);
        } catch (org.lmdbjava.Env.MapFullException e) {
            if (lmdbConfigurationWrite.increaseMapAutomatically == true) {
                increaseDatabaseSize(new ByteConversion().mibToBytes(lmdbConfigurationWrite.increaseSizeInMiB));
                /**
                 * It is possible that the exception will be thrown again, in this case increaseSizeInMiB should be changed and it's a configuration issue.
                 * See {@link CLMDBConfigurationWrite#increaseSizeInMiB}.
                 */
                putNewAmountUnsafe(hash160, amount);
            } else {
                throw e;
            }
        }
    }
    
    private void putNewAmountUnsafe(ByteBuffer hash160, Coin amount) {
        try (Txn<ByteBuffer> txn = env.txnWrite()) {
            if (lmdbConfigurationWrite.deleteEmptyAddresses && amount.isZero()) {
                lmdb_h160ToAmount.delete(txn, hash160);
            } else {
                long amountAsLong = amount.longValue();
                if (lmdbConfigurationWrite.useStaticAmount) {
                    amountAsLong = lmdbConfigurationWrite.staticAmount;
                }
                lmdb_h160ToAmount.put(txn, hash160, persistenceUtils.longToByteBufferDirect(amountAsLong));
            }
            txn.commit();
        }
    }

    @Override
    public Coin getAllAmountsFromAddresses(List<ByteBuffer> hash160s) {
        Coin allAmounts = Coin.ZERO;
        for (ByteBuffer hash160 : hash160s) {
            allAmounts = allAmounts.add(getAmount(hash160));
        }
        return allAmounts;
    }

    @Override
    public long count() {
        long count = 0;
        try (Txn<ByteBuffer> txn = env.txnRead()) {
            try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {
                for (final CursorIterable.KeyVal<ByteBuffer> kv : iterable) {
                    count++;
                }
            }
        }
        return count;
    }

    @Override
    public long getDatabaseSize() {
        EnvInfo info = env.info();
        return info.mapSize;
    }

    @Override
    public void increaseDatabaseSize(long toIncrease) {
        increasedCounter++;
        increasedSum += toIncrease;
        long newSize = getDatabaseSize() + toIncrease;
        env.setMapSize(newSize);
    }

    @Override
    public long getIncreasedCounter() {
        return increasedCounter;
    }

    @Override
    public long getIncreasedSum() {
        return increasedSum;
    }
    
    @Override
    public void logStats() {
        logger.info("##### BEGIN: LMDB stats #####");
        logger.info("... this may take a lot of time ...");
        logger.info("DatabaseSize: " + new ByteConversion().bytesToMib(getDatabaseSize()) + " MiB");
        logger.info("IncreasedCounter: " + getIncreasedCounter());
        logger.info("IncreasedSum: " + new ByteConversion().bytesToMib(getIncreasedSum()) + " MiB");
        logger.info("Stat: " + env.stat());
        // Attention: slow!
        long count = count();
        logger.info("LMDB contains " + count + " unique entries.");
        logger.info("##### END: LMDB stats #####");
    }

    public static long getApproximateSizeBytes(BloomFilter<?> bloomFilter) {
        try {
            // Access private field: bits
            Field bitsField = BloomFilter.class.getDeclaredField("bits");
            bitsField.setAccessible(true);
            Object bits = bitsField.get(bloomFilter);

            // Access internal AtomicLongArray: data
            Field dataField = bits.getClass().getDeclaredField("data");
            dataField.setAccessible(true);
            AtomicLongArray data = (AtomicLongArray) dataField.get(bits);

            return data.length() * Long.BYTES; // 8 bytes per long
        } catch (Exception e) {
            throw new RuntimeException("Failed to estimate BloomFilter size", e);
        }
    }

    public static String formatSize(long sizeInBytes) {
        if (sizeInBytes >= 1024 * 1024) {
            return String.format("%.2f MB", sizeInBytes / 1024.0 / 1024.0);
        } else if (sizeInBytes >= 1024) {
            return String.format("%.2f KB", sizeInBytes / 1024.0);
        } else {
            return sizeInBytes + " bytes";
        }
    }
}

1	// @formatter:off
2	/**
3	* Copyright 2020 Bernard Ladenthin bernard.ladenthin@gmail.com
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	*/
18	// @formatter:on
19	package net.ladenthin.bitcoinaddressfinder.persistence.lmdb;
20
21	import com.google.common.hash.BloomFilter;
22	import com.google.common.hash.Funnels;
23	import net.ladenthin.bitcoinaddressfinder.persistence.Persistence;
24	import net.ladenthin.bitcoinaddressfinder.persistence.PersistenceUtils;
25	import org.lmdbjava.CursorIterable;
26	import org.lmdbjava.Dbi;
27	import org.lmdbjava.Env;
28	import org.lmdbjava.EnvFlags;
29	import org.lmdbjava.KeyRange;
30	import org.lmdbjava.Txn;
31
32	import java.io.File;
33	import java.io.FileWriter;
34	import java.io.IOException;
35	import java.lang.reflect.Field;
36	import java.nio.ByteBuffer;
37	import java.util.List;
38	import java.util.Map;
39	import java.util.concurrent.atomic.AtomicBoolean;
40	import java.util.concurrent.atomic.AtomicLongArray;
41	import net.ladenthin.bitcoinaddressfinder.ByteBufferUtility;
42	import net.ladenthin.bitcoinaddressfinder.ByteConversion;
43	import net.ladenthin.bitcoinaddressfinder.KeyUtility;
44	import net.ladenthin.bitcoinaddressfinder.SeparatorFormat;
45	import net.ladenthin.bitcoinaddressfinder.configuration.CAddressFileOutputFormat;
46	import net.ladenthin.bitcoinaddressfinder.configuration.CLMDBConfigurationReadOnly;
47	import net.ladenthin.bitcoinaddressfinder.configuration.CLMDBConfigurationWrite;
48	import org.apache.commons.codec.binary.Hex;
49	import org.bitcoinj.base.Coin;
50	import org.bitcoinj.base.LegacyAddress;
51	import org.lmdbjava.BufferProxy;
52	import org.lmdbjava.ByteBufferProxy;
53
54	import static org.lmdbjava.DbiFlags.MDB_CREATE;
55	import static org.lmdbjava.Env.create;
56	import org.lmdbjava.EnvInfo;
57	import org.slf4j.Logger;
58	import org.slf4j.LoggerFactory;
59
60	public class LMDBPersistence implements Persistence {
61
62	private static final String DB_NAME_HASH160_TO_COINT = "hash160toCoin";
63	private static final int DB_COUNT = 1;
64
65	private final Logger logger = LoggerFactory.getLogger(LMDBPersistence.class);	1✔
66
67	private final PersistenceUtils persistenceUtils;
68	private final CLMDBConfigurationWrite lmdbConfigurationWrite;
69	private final CLMDBConfigurationReadOnly lmdbConfigurationReadOnly;
70	private final KeyUtility keyUtility;
71	private Env<ByteBuffer> env;
72	private Dbi<ByteBuffer> lmdb_h160ToAmount;
73	private long increasedCounter = 0;	1✔
74	private long increasedSum = 0;	1✔
75	private BloomFilter<byte[]> addressBloomFilter = null;	1✔
76
77
78	public LMDBPersistence(CLMDBConfigurationWrite lmdbConfigurationWrite, PersistenceUtils persistenceUtils) {	1✔
79	this.lmdbConfigurationReadOnly = null;	1✔
80	this.lmdbConfigurationWrite = lmdbConfigurationWrite;	1✔
81	this.persistenceUtils = persistenceUtils;	1✔
82	this.keyUtility = new KeyUtility(persistenceUtils.network, new ByteBufferUtility(true));	1✔
83	}	1✔
84
85	public LMDBPersistence(CLMDBConfigurationReadOnly lmdbConfigurationReadOnly, PersistenceUtils persistenceUtils) {	1✔
86	this.lmdbConfigurationReadOnly = lmdbConfigurationReadOnly;	1✔
87	lmdbConfigurationWrite = null;	1✔
88	this.persistenceUtils = persistenceUtils;	1✔
89	this.keyUtility = new KeyUtility(persistenceUtils.network, new ByteBufferUtility(true));	1✔
90	}	1✔
91
92	@Override
93	public void init() {
94	if (lmdbConfigurationWrite != null) {	1✔
95	initWritable();	1✔
96	} else if (lmdbConfigurationReadOnly != null) {	1✔
97	initReadOnly();	1✔
98	} else {
99	throw new IllegalArgumentException("Neither write nor read-only configuration provided.");	×
100	}
101
102
103	logStatsIfConfigured(true);	1✔
104	}	1✔
105
106	public void buildAddressBloomFilter() {
107	logger.info("##### BEGIN: buildAddressBloomFilter #####");	1✔
108	// Attention: slow!
109	long count = count();	1✔
110
111	BloomFilter<byte[]> filter = BloomFilter.create(Funnels.byteArrayFunnel(), count, lmdbConfigurationReadOnly.bloomFilterFpp);	1✔
112	long inserted = 0;	1✔
113
114	try (Txn<ByteBuffer> txn = env.txnRead()) {	1✔
115	try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {	1✔
116	for (CursorIterable.KeyVal<ByteBuffer> kv : iterable) {	1✔
117	ByteBuffer key = kv.key();	1✔
118	byte[] keyBytes = new byte[key.remaining()];	1✔
119	key.get(keyBytes);	1✔
120	key.rewind();	1✔
121	filter.put(keyBytes);	1✔
122	inserted++;	1✔
123	}	1✔
124	}
125	}
126
127	addressBloomFilter = filter;	1✔
128	long size = getApproximateSizeBytes(filter);	1✔
129	logger.info("Inserted {} addresses into BloomFilter with size of {}", inserted, formatSize(size));	1✔
130	logger.info("##### END: buildAddressBloomFilter #####");	1✔
131	}	1✔
132
133	public void unloadBloomFilter() {
134	addressBloomFilter = null;	×
135	}	×
136
137	private void initReadOnly() {
138	BufferProxy<ByteBuffer> bufferProxy = getBufferProxyByUseProxyOptimal(lmdbConfigurationReadOnly.useProxyOptimal);	1✔
139	env = create(bufferProxy).setMaxDbs(DB_COUNT).open(new File(lmdbConfigurationReadOnly.lmdbDirectory), EnvFlags.MDB_RDONLY_ENV, EnvFlags.MDB_NOLOCK);	1✔
140	lmdb_h160ToAmount = env.openDbi(DB_NAME_HASH160_TO_COINT);	1✔
141
142	if (lmdbConfigurationReadOnly.useBloomFilter) {	1✔
143	buildAddressBloomFilter();	1✔
144	}
145	}	1✔
146
147	private void initWritable() {
148	// -Xmx10G -XX:MaxDirectMemorySize=5G
149	// We always need an Env. An Env owns a physical on-disk storage file. One
150	// Env can store many different databases (ie sorted maps).
151	File lmdbDirectory = new File(lmdbConfigurationWrite.lmdbDirectory);	1✔
152	lmdbDirectory.mkdirs();	1✔
153
154	BufferProxy<ByteBuffer> bufferProxy = getBufferProxyByUseProxyOptimal(lmdbConfigurationWrite.useProxyOptimal);	1✔
155
156	env = create(bufferProxy)	1✔
157	// LMDB also needs to know how large our DB might be. Over-estimating is OK.
158	.setMapSize(new ByteConversion().mibToBytes(lmdbConfigurationWrite.initialMapSizeInMiB))	1✔
159	// LMDB also needs to know how many DBs (Dbi) we want to store in this Env.
160	.setMaxDbs(DB_COUNT)	1✔
161	// Now let's open the Env. The same path can be concurrently opened and
162	// used in different processes, but do not open the same path twice in
163	// the same process at the same time.
164
165	//https://github.com/kentnl/CHI-Driver-LMDB
166	.open(lmdbDirectory, EnvFlags.MDB_NOSYNC, EnvFlags.MDB_NOMETASYNC, EnvFlags.MDB_WRITEMAP, EnvFlags.MDB_MAPASYNC);	1✔
167	// We need a Dbi for each DB. A Dbi roughly equates to a sorted map. The
168	// MDB_CREATE flag causes the DB to be created if it doesn't already exist.
169	lmdb_h160ToAmount = env.openDbi(DB_NAME_HASH160_TO_COINT, MDB_CREATE);	1✔
170	}	1✔
171
172	/**
173	* https://github.com/lmdbjava/lmdbjava/wiki/Buffers
174	*
175	* @param useProxyOptimal
176	* @return
177	*/
178	private BufferProxy<ByteBuffer> getBufferProxyByUseProxyOptimal(boolean useProxyOptimal) {
179	if (useProxyOptimal) {	1✔
180	return ByteBufferProxy.PROXY_OPTIMAL;	1✔
181	} else {
182	return ByteBufferProxy.PROXY_SAFE;	×
183	}
184	}
185
186	private void logStatsIfConfigured(boolean onInit) {
187	if (isLoggingEnabled(lmdbConfigurationWrite, onInit) \|\| isLoggingEnabled(lmdbConfigurationReadOnly, onInit)) {	1✔
188	logStats();	1✔
189	}
190	}	1✔
191
192	private boolean isLoggingEnabled(CLMDBConfigurationReadOnly config, boolean onInit) {
193	return config != null && (onInit ? config.logStatsOnInit : config.logStatsOnClose);	1✔
194	}
195
196	@Override
197	public void close() {
198	logStatsIfConfigured(false);	1✔
199	lmdb_h160ToAmount.close();	1✔
200	env.close();	1✔
201	}	1✔
202
203	@Override
204	public boolean isClosed() {
205	return env.isClosed();	1✔
206	}
207
208	@Override
209	public Coin getAmount(ByteBuffer hash160) {
210	try (Txn<ByteBuffer> txn = env.txnRead()) {	1✔
211	ByteBuffer byteBuffer = lmdb_h160ToAmount.get(txn, hash160);	1✔
212	return getCoinFromByteBuffer(byteBuffer);	1✔
213	}
214	}
215
216	private Coin getCoinFromByteBuffer(ByteBuffer byteBuffer) {
217	if (byteBuffer == null \|\| byteBuffer.capacity() == 0) {	1✔
218	return Coin.ZERO;	1✔
219	}
220	return Coin.valueOf(byteBuffer.getLong());	1✔
221	}
222
223	@Override
224	public boolean containsAddress(ByteBuffer hash160) {
225	if (lmdbConfigurationReadOnly.disableAddressLookup) {	1✔
226	return false;	×
227	}
228
229	byte[] hash160AsByteArray = new byte[hash160.remaining()];	1✔
230	hash160.get(hash160AsByteArray);	1✔
231	hash160.rewind();	1✔
232
233	// Use Bloom filter if available for fast pre-check
234	if (addressBloomFilter != null) {	1✔
235	boolean mightContain = addressBloomFilter.mightContain(hash160AsByteArray);	1✔
236	if (!mightContain) {	1✔
237	return false; // definitely not present	1✔
238	}
239	// Possibly in DB, proceed to verify
240	}
241
242	// Perform LMDB lookup (always happens if no Bloom filter is present)
243	try (Txn<ByteBuffer> txn = env.txnRead()) {	1✔
244	ByteBuffer byteBuffer = lmdb_h160ToAmount.get(txn, hash160);	1✔
245	return byteBuffer != null;	1✔
246	}
247	}
248
249	@Override
250	public void writeAllAmountsToAddressFile(File file, CAddressFileOutputFormat addressFileOutputFormat, AtomicBoolean shouldRun) throws IOException {
251	try (Txn<ByteBuffer> txn = env.txnRead()) {	1✔
252	try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {	1✔
253	try (FileWriter writer = new FileWriter(file)) {	1✔
254	for (final CursorIterable.KeyVal<ByteBuffer> kv : iterable) {	1✔
255	if (!shouldRun.get()) {	1✔
256	return;	×
257	}
258	ByteBuffer addressAsByteBuffer = kv.key();	1✔
259	if(logger.isTraceEnabled()) {	1✔
260	String hexFromByteBuffer = new ByteBufferUtility(false).getHexFromByteBuffer(addressAsByteBuffer);	×
261	logger.trace("Process address: " + hexFromByteBuffer);	×
262	}
263	LegacyAddress address = keyUtility.byteBufferToAddress(addressAsByteBuffer);	1✔
264	final String line;
265	switch(addressFileOutputFormat) {	1✔
266	case HexHash:
267	line = Hex.encodeHexString(address.getHash()) + System.lineSeparator();	1✔
268	break;	1✔
269	case FixedWidthBase58BitcoinAddress:
270	line = String.format("%-34s", address.toBase58()) + System.lineSeparator();	1✔
271	break;	1✔
272	case DynamicWidthBase58BitcoinAddressWithAmount:
273	ByteBuffer value = kv.val();	1✔
274	Coin coin = getCoinFromByteBuffer(value);	1✔
275	line = address.toBase58() + SeparatorFormat.COMMA.getSymbol() + coin.getValue() + System.lineSeparator();	1✔
276	break;	1✔
277	default:
278	throw new IllegalArgumentException("Unknown addressFileOutputFormat: " + addressFileOutputFormat);	×
279	}
280	writer.write(line);	1✔
281	}	1✔
282	}	×
283	}	×
284	}	×
285	}	1✔
286
287	@Override
288	public void putAllAmounts(Map<ByteBuffer, Coin> amounts) throws IOException {
289	for (Map.Entry<ByteBuffer, Coin> entry : amounts.entrySet()) {	×
290	ByteBuffer hash160 = entry.getKey();	×
291	Coin coin = entry.getValue();	×
292	putNewAmount(hash160, coin);	×
293	}	×
294	}	×
295
296	@Override
297	public void changeAmount(ByteBuffer hash160, Coin amountToChange) {
298	Coin valueInDB = getAmount(hash160);	×
299	Coin toWrite = valueInDB.add(amountToChange);	×
300	putNewAmount(hash160, toWrite);	×
301	}	×
302
303	@Override
304	public void putNewAmount(ByteBuffer hash160, Coin amount) {
305	putNewAmountWithAutoIncrease(hash160, amount);	1✔
306	}	1✔
307
308	/**
309	* If an {@link org.lmdbjava.Env.MapFullException} was thrown during a put. The map might be increased if configured.
310	* The increase value needs to be high enough. Otherwise the next put fails nevertheless.
311	*/
312	private void putNewAmountWithAutoIncrease(ByteBuffer hash160, Coin amount) {
313	try {
314	putNewAmountUnsafe(hash160, amount);	1✔
315	} catch (org.lmdbjava.Env.MapFullException e) {	1✔
316	if (lmdbConfigurationWrite.increaseMapAutomatically == true) {	1✔
317	increaseDatabaseSize(new ByteConversion().mibToBytes(lmdbConfigurationWrite.increaseSizeInMiB));	1✔
318	/**
319	* It is possible that the exception will be thrown again, in this case increaseSizeInMiB should be changed and it's a configuration issue.
320	* See {@link CLMDBConfigurationWrite#increaseSizeInMiB}.
321	*/
322	putNewAmountUnsafe(hash160, amount);	1✔
323	} else {
324	throw e;	1✔
325	}
326	}	1✔
327	}	1✔
328
329	private void putNewAmountUnsafe(ByteBuffer hash160, Coin amount) {
330	try (Txn<ByteBuffer> txn = env.txnWrite()) {	1✔
331	if (lmdbConfigurationWrite.deleteEmptyAddresses && amount.isZero()) {	1✔
332	lmdb_h160ToAmount.delete(txn, hash160);	×
333	} else {
334	long amountAsLong = amount.longValue();	1✔
335	if (lmdbConfigurationWrite.useStaticAmount) {	1✔
336	amountAsLong = lmdbConfigurationWrite.staticAmount;	1✔
337	}
338	lmdb_h160ToAmount.put(txn, hash160, persistenceUtils.longToByteBufferDirect(amountAsLong));	1✔
339	}
340	txn.commit();	1✔
341	}
342	}	1✔
343
344	@Override
345	public Coin getAllAmountsFromAddresses(List<ByteBuffer> hash160s) {
346	Coin allAmounts = Coin.ZERO;	×
347	for (ByteBuffer hash160 : hash160s) {	×
348	allAmounts = allAmounts.add(getAmount(hash160));	×
349	}	×
350	return allAmounts;	×
351	}
352
353	@Override
354	public long count() {
355	long count = 0;	1✔
356	try (Txn<ByteBuffer> txn = env.txnRead()) {	1✔
357	try (CursorIterable<ByteBuffer> iterable = lmdb_h160ToAmount.iterate(txn, KeyRange.all())) {	1✔
358	for (final CursorIterable.KeyVal<ByteBuffer> kv : iterable) {	1✔
359	count++;	1✔
360	}	1✔
361	}
362	}
363	return count;	1✔
364	}
365
366	@Override
367	public long getDatabaseSize() {
368	EnvInfo info = env.info();	1✔
369	return info.mapSize;	1✔
370	}
371
372	@Override
373	public void increaseDatabaseSize(long toIncrease) {
374	increasedCounter++;	1✔
375	increasedSum += toIncrease;	1✔
376	long newSize = getDatabaseSize() + toIncrease;	1✔
377	env.setMapSize(newSize);	1✔
378	}	1✔
379
380	@Override
381	public long getIncreasedCounter() {
382	return increasedCounter;	1✔
383	}
384
385	@Override
386	public long getIncreasedSum() {
387	return increasedSum;	1✔
388	}
389
390	@Override
391	public void logStats() {
392	logger.info("##### BEGIN: LMDB stats #####");	1✔
393	logger.info("... this may take a lot of time ...");	1✔
394	logger.info("DatabaseSize: " + new ByteConversion().bytesToMib(getDatabaseSize()) + " MiB");	1✔
395	logger.info("IncreasedCounter: " + getIncreasedCounter());	1✔
396	logger.info("IncreasedSum: " + new ByteConversion().bytesToMib(getIncreasedSum()) + " MiB");	1✔
397	logger.info("Stat: " + env.stat());	1✔
398	// Attention: slow!
399	long count = count();	1✔
400	logger.info("LMDB contains " + count + " unique entries.");	1✔
401	logger.info("##### END: LMDB stats #####");	1✔
402	}	1✔
403
404	public static long getApproximateSizeBytes(BloomFilter<?> bloomFilter) {
405	try {
406	// Access private field: bits
407	Field bitsField = BloomFilter.class.getDeclaredField("bits");	1✔
408	bitsField.setAccessible(true);	1✔
409	Object bits = bitsField.get(bloomFilter);	1✔
410
411	// Access internal AtomicLongArray: data
412	Field dataField = bits.getClass().getDeclaredField("data");	1✔
413	dataField.setAccessible(true);	1✔
414	AtomicLongArray data = (AtomicLongArray) dataField.get(bits);	1✔
415
416	return data.length() * Long.BYTES; // 8 bytes per long	1✔
NEW 417	} catch (Exception e) {	×
NEW 418	throw new RuntimeException("Failed to estimate BloomFilter size", e);	×
419	}
420	}
421
422	public static String formatSize(long sizeInBytes) {
423	if (sizeInBytes >= 1024 * 1024) {	1✔
NEW 424	return String.format("%.2f MB", sizeInBytes / 1024.0 / 1024.0);	×
425	} else if (sizeInBytes >= 1024) {	1✔
NEW 426	return String.format("%.2f KB", sizeInBytes / 1024.0);	×
427	} else {
428	return sizeInBytes + " bytes";	1✔
429	}
430	}
431	}

bernardladenthin / BitcoinAddressFinder / #349

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous