evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/exist-core/src/main/java/org/exist/storage/blob/BlobStoreDumpTool.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 */
package org.exist.storage.blob;

import org.exist.start.CompatibleJavaVersionCheck;
import org.exist.start.StartException;
import org.exist.util.SystemExitCodes;
import org.exist.util.crypto.digest.DigestType;
import se.softhouse.jargo.Argument;
import se.softhouse.jargo.ArgumentException;
import se.softhouse.jargo.CommandLineParser;
import se.softhouse.jargo.ParsedArguments;

import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;

import static java.nio.file.StandardOpenOption.READ;
import static org.exist.storage.blob.BlobStoreImpl.BLOB_STORE_HEADER_LEN;
import static org.exist.storage.blob.BlobStoreImpl.BLOB_STORE_MAGIC_NUMBER;
import static org.exist.storage.blob.BlobStoreImpl.REFERENCE_COUNT_LEN;
import static se.softhouse.jargo.Arguments.enumArgument;
import static se.softhouse.jargo.Arguments.fileArgument;
import static se.softhouse.jargo.Arguments.helpArgument;

/**
 * Simple tool for dumping the content
 * of the Blob Store in CSV format for the
 * purposes of debugging or visualisation.
 *
 * @author <a href="mailto:adam@evolvedbinary.com">Adam Retter</a>
 */
public class BlobStoreDumpTool {

    private static final Argument<?> helpArg = helpArgument("-h", "--help");
    private static final Argument<DigestType> digestArg = enumArgument(DigestType.class, "-d", "--digest")
            .description("The digest type used by the Blob Store")
            .defaultValue(DigestType.BLAKE_256)
            .build();
    private static final Argument<Path> persistentFileArg = fileArgument()
            .description("Path to the Blob Store persistent file e.g. blob.dbx")
            .required()
            .transform(java.io.File::toPath)
            .build();

    public static void main(final String args[]) throws IOException {
        try {
            CompatibleJavaVersionCheck.checkForCompatibleJavaVersion();

            final ParsedArguments arguments = CommandLineParser
                    .withArguments(digestArg, persistentFileArg)
                    .andArguments(helpArg)
                    .parse(args);

            final DigestType digestType = arguments.get(digestArg);
            final Path persistentFile = arguments.get(persistentFileArg);

            final PrintStream printStream = System.out;

            dump(digestType, persistentFile, printStream);
        } catch (final StartException e) {
            if (e.getMessage() != null && !e.getMessage().isEmpty()) {
                System.err.println(e.getMessage());
            }
            System.exit(e.getErrorCode());
        } catch(final ArgumentException e) {
            System.out.println(e.getMessageAndUsage());
            System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);
        }
    }

    /**
     * Dumps the content of the Blob Store persistent file.
     *
     * @param digestType the digest type used by the Blob Store
     * @param persistentFile the Blob Store persistent file to dump
     * @param printStream the stream to dump the Blob Store to
     *
     * @throws IOException if an I/O error occurs whilst reading the Blob Store's persistent file.
     */
    public static void dump(final DigestType digestType, final Path persistentFile, final PrintStream printStream)
            throws IOException {
        final int digestBytesLen = digestType.getDigestLengthBytes();

        final ByteBuffer buffer = ByteBuffer.allocate(digestBytesLen + REFERENCE_COUNT_LEN);

        try (final SeekableByteChannel channel = Files.newByteChannel(persistentFile, READ)) {

            // dump the file header details
            dumpFileHeader(printStream, buffer, persistentFile, channel);
            printStream.println("Digest Type: " + digestType.name());
            printStream.println();

            // write a CSV table header
            printStream.println("blobId,referenceCount");

            buffer.clear();

            final byte[] id = new byte[digestBytesLen];

            while (channel.read(buffer) > -1) {
                buffer.flip();
                buffer.get(id);
                final BlobId blobId = new BlobId(id);
                final int count = buffer.getInt();

                printStream.println(blobId + "," + count);

                buffer.clear();
            }
        }
    }

    /**
     * Dump the File Header of the Blob Store
     *
     * @param printStream the stream to dump the Blob Store to
     * @param buffer a buffer to use
     * @param persistentFile the Blob Store persistent file to dump
     * @param channel the open channel for reading the {@code persistentFile}.
     *
     * @throws IOException if an I/O error occurs whilst reading the Blob Store's persistent file.
     */
    private static void dumpFileHeader(final PrintStream printStream, final ByteBuffer buffer, final Path persistentFile,
            final SeekableByteChannel channel) throws IOException {
        buffer.clear();
        buffer.limit(BLOB_STORE_HEADER_LEN);

        channel.read(buffer);

        buffer.flip();

        final boolean validMagic =
                buffer.get() == BLOB_STORE_MAGIC_NUMBER[0]
                        && buffer.get() == BLOB_STORE_MAGIC_NUMBER[1]
                        && buffer.get() == BLOB_STORE_MAGIC_NUMBER[2]
                        && buffer.get() == BLOB_STORE_MAGIC_NUMBER[3];

        if (!validMagic) {
            throw new IOException("File was not recognised as a valid Elemental Blob Store: "
                    + persistentFile.toAbsolutePath().toString());
        }

        // check the version of the blob store format
        final short storedVersion = buffer.getShort();
        printStream.println("*** Blob Store file format version: " + storedVersion);

        // TODO(AR) validate that we can read this format version
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*/
24	package org.exist.storage.blob;
25
26	import org.exist.start.CompatibleJavaVersionCheck;
27	import org.exist.start.StartException;
28	import org.exist.util.SystemExitCodes;
29	import org.exist.util.crypto.digest.DigestType;
30	import se.softhouse.jargo.Argument;
31	import se.softhouse.jargo.ArgumentException;
32	import se.softhouse.jargo.CommandLineParser;
33	import se.softhouse.jargo.ParsedArguments;
34
35	import java.io.IOException;
36	import java.io.PrintStream;
37	import java.nio.ByteBuffer;
38	import java.nio.channels.SeekableByteChannel;
39	import java.nio.file.Files;
40	import java.nio.file.Path;
41
42	import static java.nio.file.StandardOpenOption.READ;
43	import static org.exist.storage.blob.BlobStoreImpl.BLOB_STORE_HEADER_LEN;
44	import static org.exist.storage.blob.BlobStoreImpl.BLOB_STORE_MAGIC_NUMBER;
45	import static org.exist.storage.blob.BlobStoreImpl.REFERENCE_COUNT_LEN;
46	import static se.softhouse.jargo.Arguments.enumArgument;
47	import static se.softhouse.jargo.Arguments.fileArgument;
48	import static se.softhouse.jargo.Arguments.helpArgument;
49
50	/**
51	* Simple tool for dumping the content
52	* of the Blob Store in CSV format for the
53	* purposes of debugging or visualisation.
54	*
55	* @author <a href="mailto:adam@evolvedbinary.com">Adam Retter</a>
56	*/
57	public class BlobStoreDumpTool {	×
58
59	private static final Argument<?> helpArg = helpArgument("-h", "--help");	×
60	private static final Argument<DigestType> digestArg = enumArgument(DigestType.class, "-d", "--digest")	×
61	.description("The digest type used by the Blob Store")	×
62	.defaultValue(DigestType.BLAKE_256)	×
63	.build();	×
64	private static final Argument<Path> persistentFileArg = fileArgument()	×
65	.description("Path to the Blob Store persistent file e.g. blob.dbx")	×
66	.required()	×
67	.transform(java.io.File::toPath)	×
68	.build();	×
69
70	public static void main(final String args[]) throws IOException {
71	try {
72	CompatibleJavaVersionCheck.checkForCompatibleJavaVersion();	×
73
74	final ParsedArguments arguments = CommandLineParser	×
75	.withArguments(digestArg, persistentFileArg)	×
76	.andArguments(helpArg)	×
77	.parse(args);	×
78
79	final DigestType digestType = arguments.get(digestArg);	×
80	final Path persistentFile = arguments.get(persistentFileArg);	×
81
82	final PrintStream printStream = System.out;	×
83
84	dump(digestType, persistentFile, printStream);	×
85	} catch (final StartException e) {	×
86	if (e.getMessage() != null && !e.getMessage().isEmpty()) {	×
87	System.err.println(e.getMessage());	×
88	}
89	System.exit(e.getErrorCode());	×
90	} catch(final ArgumentException e) {	×
91	System.out.println(e.getMessageAndUsage());	×
92	System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);	×
93	}
94	}	×
95
96	/**
97	* Dumps the content of the Blob Store persistent file.
98	*
99	* @param digestType the digest type used by the Blob Store
100	* @param persistentFile the Blob Store persistent file to dump
101	* @param printStream the stream to dump the Blob Store to
102	*
103	* @throws IOException if an I/O error occurs whilst reading the Blob Store's persistent file.
104	*/
105	public static void dump(final DigestType digestType, final Path persistentFile, final PrintStream printStream)
106	throws IOException {
107	final int digestBytesLen = digestType.getDigestLengthBytes();	×
108
109	final ByteBuffer buffer = ByteBuffer.allocate(digestBytesLen + REFERENCE_COUNT_LEN);	×
110
111	try (final SeekableByteChannel channel = Files.newByteChannel(persistentFile, READ)) {	×
112
113	// dump the file header details
114	dumpFileHeader(printStream, buffer, persistentFile, channel);	×
115	printStream.println("Digest Type: " + digestType.name());	×
116	printStream.println();	×
117
118	// write a CSV table header
119	printStream.println("blobId,referenceCount");	×
120
121	buffer.clear();	×
122
123	final byte[] id = new byte[digestBytesLen];	×
124
125	while (channel.read(buffer) > -1) {	×
126	buffer.flip();	×
127	buffer.get(id);	×
128	final BlobId blobId = new BlobId(id);	×
129	final int count = buffer.getInt();	×
130
131	printStream.println(blobId + "," + count);	×
132
133	buffer.clear();	×
134	}
135	}
136	}	×
137
138	/**
139	* Dump the File Header of the Blob Store
140	*
141	* @param printStream the stream to dump the Blob Store to
142	* @param buffer a buffer to use
143	* @param persistentFile the Blob Store persistent file to dump
144	* @param channel the open channel for reading the {@code persistentFile}.
145	*
146	* @throws IOException if an I/O error occurs whilst reading the Blob Store's persistent file.
147	*/
148	private static void dumpFileHeader(final PrintStream printStream, final ByteBuffer buffer, final Path persistentFile,
149	final SeekableByteChannel channel) throws IOException {
150	buffer.clear();	×
151	buffer.limit(BLOB_STORE_HEADER_LEN);	×
152
153	channel.read(buffer);	×
154
155	buffer.flip();	×
156
157	final boolean validMagic =	×
158	buffer.get() == BLOB_STORE_MAGIC_NUMBER[0]	×
159	&& buffer.get() == BLOB_STORE_MAGIC_NUMBER[1]	×
160	&& buffer.get() == BLOB_STORE_MAGIC_NUMBER[2]	×
161	&& buffer.get() == BLOB_STORE_MAGIC_NUMBER[3];	×
162
163	if (!validMagic) {	×
164	throw new IOException("File was not recognised as a valid Elemental Blob Store: "	×
165	+ persistentFile.toAbsolutePath().toString());	×
166	}
167
168	// check the version of the blob store format
169	final short storedVersion = buffer.getShort();	×
170	printStream.println("*** Blob Store file format version: " + storedVersion);	×
171
172	// TODO(AR) validate that we can read this format version
173	}	×
174	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous