evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/exist-core/src/main/java/org/exist/backup/ExportMain.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 *
 * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
 *       The original license header is included below.
 *
 * =====================================================================
 *
 * eXist-db Open Source Native XML Database
 * Copyright (C) 2001 The eXist-db Authors
 *
 * info@exist-db.org
 * http://www.exist-db.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.backup;

import org.exist.EXistException;
import org.exist.security.PermissionDeniedException;
import org.exist.start.CompatibleJavaVersionCheck;
import org.exist.start.StartException;
import org.exist.storage.BrokerPool;
import org.exist.storage.DBBroker;
import org.exist.storage.txn.Txn;
import org.exist.util.Configuration;
import org.exist.util.DatabaseConfigurationException;
import org.exist.util.OSUtil;
import org.exist.util.SystemExitCodes;
import org.exist.xquery.TerminatedException;
import se.softhouse.jargo.Argument;
import se.softhouse.jargo.ArgumentException;
import se.softhouse.jargo.CommandLineParser;
import se.softhouse.jargo.ParsedArguments;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Optional;

import static org.exist.util.ArgumentUtil.getBool;
import static org.exist.util.ArgumentUtil.getOpt;
import static se.softhouse.jargo.Arguments.*;


public class ExportMain {

    /* general arguments */
    private static final Argument<?> helpArg = helpArgument("-h", "--help");
    private static final Argument<Boolean> verboseArg = optionArgument("-v", "--verbose")
            .description("print processed resources to stdout")
            .defaultValue(false)
            .build();

    /* control arguments */
    private static final Argument<Boolean> noCheckArg = optionArgument("-n", "--nocheck")
            .description("do not run a consistency check. Just export the data.")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> checkDocsArg = optionArgument("-s", "--check-docs")
            .description("scan every document to find errors in the the nodes stored (costs time)")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> directAccessArg = optionArgument("-D", "--direct")
            .description("use an (even more) direct access to the db, bypassing some index structures")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> exportArg = optionArgument("-x", "--export")
            .description("export database contents while preserving as much data as possible")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> noExportArg = optionArgument("--no-export")
            .description("do not export the database contents, overrides argument --export")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> incrementalArg = optionArgument("-i", "--incremental")
            .description("create incremental backup (use with --export|-x)")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> zipArg = optionArgument("-z", "--zip")
            .description("write output to a ZIP instead of a file system directory")
            .defaultValue(false)
            .build();
    private static final Argument<Boolean> noZipArg = optionArgument("--no-zip")
            .description("do not zip the output, overrides argument --zip")
            .defaultValue(false)
            .build();

    /* export parameters */
    private static final Argument<File> configArg = fileArgument("-c", "--config")
            .description("the database configuration (conf.xml) file to use for launching the db.")
            .build();
    private static final Argument<File> outputDirArg = fileArgument("-d", "--dir")
            .description("the directory to which all output will be written.")
            .defaultValue(Paths.get("export").toAbsolutePath().toFile())
            .build();

    protected static BrokerPool startDB(final Optional<Path> configFile) {
        try {
            final Configuration config;

            if (configFile.isPresent()) {
                config = new Configuration(configFile.get().toAbsolutePath().toString(), Optional.empty());
            } else {
                config = new Configuration();
            }
            config.setProperty(BrokerPool.PROPERTY_EXPORT_ONLY, Boolean.TRUE);
            BrokerPool.configure(1, 5, config);
            return (BrokerPool.getInstance());
        } catch (final DatabaseConfigurationException | EXistException e) {
            System.err.println("ERROR: Failed to open database: " + e.getMessage());
        }
        return (null);
    }


    @SuppressWarnings("unchecked")
    public static void main(final String[] args) {
        try {
            CompatibleJavaVersionCheck.checkForCompatibleJavaVersion();

            final ParsedArguments arguments = CommandLineParser
                    .withArguments(noCheckArg, checkDocsArg, directAccessArg, exportArg, noExportArg, incrementalArg, zipArg, noZipArg)
                    .andArguments(configArg, outputDirArg)
                    .andArguments(helpArg, verboseArg)
                    .programName("export" + (OSUtil.IS_WINDOWS ? ".bat" : ".sh"))
                    .parse(args);

            process(arguments);
        } catch (final StartException e) {
            if (e.getMessage() != null && !e.getMessage().isEmpty()) {
                System.err.println(e.getMessage());
            }
            System.exit(e.getErrorCode());
        } catch (final ArgumentException e) {
            System.out.println(e.getMessageAndUsage());
            System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);

        }
    }

    private static void process(final ParsedArguments arguments) {
        final boolean verbose = getBool(arguments, verboseArg);

        final boolean noCheck = getBool(arguments, noCheckArg);
        final boolean checkDocs = getBool(arguments, checkDocsArg);
        final boolean direct = getBool(arguments, directAccessArg);
        boolean export = getBool(arguments, exportArg);
        final boolean noExport = getBool(arguments, noExportArg);
        if (noExport) {
            export = false;
        }
        final boolean incremental = getBool(arguments, incrementalArg);
        boolean zip = getBool(arguments, zipArg);
        final boolean noZip = getBool(arguments, noZipArg);
        if (noZip) {
            zip = false;
        }

        final Optional<Path> dbConfig = getOpt(arguments, configArg).map(File::toPath);
        final Path exportTarget = arguments.get(outputDirArg).toPath();

        final BrokerPool pool = startDB(dbConfig);

        if (pool == null) {
            System.exit(SystemExitCodes.CATCH_ALL_GENERAL_ERROR_EXIT_CODE);
        }
        int retval = 0; // return value

        try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()));
                final Txn transaction = pool.getTransactionManager().beginTransaction()) {

            List<ErrorReport> errors = null;

            if (!noCheck) {
                final ConsistencyCheck checker = new ConsistencyCheck(broker, transaction, direct, checkDocs);
                errors = checker.checkAll(new CheckCallback());
            }

            if (errors != null && !errors.isEmpty()) {
                System.err.println("ERRORS FOUND.");
                retval = 1;
            } else {
                System.out.println("No errors.");
            }

            if (export) {
                if (!Files.exists(exportTarget)) {
                    Files.createDirectories(exportTarget);
                } else if(!Files.isDirectory(exportTarget)) {
                    System.err.println("Output dir already exists and is a file: " + exportTarget.toAbsolutePath().toString());
                    System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);
                }
                final SystemExport sysexport = new SystemExport(broker, transaction, new Callback(verbose), null, direct);
                sysexport.export(exportTarget.toAbsolutePath().toString(), incremental, zip, errors);
            }

            transaction.commit();

        } catch (final EXistException e) {
            System.err.println("ERROR: Failed to retrieve database broker: " + e.getMessage());
            retval = SystemExitCodes.NO_BROKER_EXIT_CODE;
        } catch (final TerminatedException e) {
            System.err.println("WARN: Export was terminated by db.");
            retval = SystemExitCodes.TERMINATED_EARLY_EXIT_CODE;
        } catch (final PermissionDeniedException pde) {
            System.err.println("ERROR: Failed to retrieve database data: " + pde.getMessage());
            retval = SystemExitCodes.PERMISSION_DENIED_EXIT_CODE;
        } catch (final IOException ioe) {
            System.err.println("ERROR: Failed to retrieve database data: " + ioe.getMessage());
            retval = SystemExitCodes.IO_ERROR_EXIT_CODE;
        } finally {
            BrokerPool.stopAll(false);
        }
        System.exit(retval);
    }

    private static class Callback implements SystemExport.StatusCallback {

        private boolean verbose = false;

        public Callback(final boolean verbose) {
            this.verbose = verbose;
        }

        @Override
        public void startCollection(final String path) {
            if (verbose) {
                System.out.println("Entering collection " + path + " ...");
            }
        }

        @Override
        public void startDocument(final String name, final int count, final int docsCount) {
            if (verbose) {
                System.out.println("Writing document " + name + " [" + (count + 1) + " of " + docsCount + ']');
            }
        }

        @Override
        public void error(final String message, final Throwable exception) {
            System.err.println(message);

            if (exception != null) {
                exception.printStackTrace();
            }
        }
    }


    private static class CheckCallback implements org.exist.backup.ConsistencyCheck.ProgressCallback {
        @Override
        public void startDocument(final String name, final int current, final int count) {
        }

        @Override
        public void startCollection(final String path) {
        }

        @Override
        public void error(final ErrorReport error) {
            System.out.println(error.toString());
        }
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*
24	* NOTE: Parts of this file contain code from 'The eXist-db Authors'.
25	* The original license header is included below.
26	*
27	* =====================================================================
28	*
29	* eXist-db Open Source Native XML Database
30	* Copyright (C) 2001 The eXist-db Authors
31	*
32	* info@exist-db.org
33	* http://www.exist-db.org
34	*
35	* This library is free software; you can redistribute it and/or
36	* modify it under the terms of the GNU Lesser General Public
37	* License as published by the Free Software Foundation; either
38	* version 2.1 of the License, or (at your option) any later version.
39	*
40	* This library is distributed in the hope that it will be useful,
41	* but WITHOUT ANY WARRANTY; without even the implied warranty of
42	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
43	* Lesser General Public License for more details.
44	*
45	* You should have received a copy of the GNU Lesser General Public
46	* License along with this library; if not, write to the Free Software
47	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
48	*/
49	package org.exist.backup;
50
51	import org.exist.EXistException;
52	import org.exist.security.PermissionDeniedException;
53	import org.exist.start.CompatibleJavaVersionCheck;
54	import org.exist.start.StartException;
55	import org.exist.storage.BrokerPool;
56	import org.exist.storage.DBBroker;
57	import org.exist.storage.txn.Txn;
58	import org.exist.util.Configuration;
59	import org.exist.util.DatabaseConfigurationException;
60	import org.exist.util.OSUtil;
61	import org.exist.util.SystemExitCodes;
62	import org.exist.xquery.TerminatedException;
63	import se.softhouse.jargo.Argument;
64	import se.softhouse.jargo.ArgumentException;
65	import se.softhouse.jargo.CommandLineParser;
66	import se.softhouse.jargo.ParsedArguments;
67
68	import java.io.File;
69	import java.io.IOException;
70	import java.nio.file.Files;
71	import java.nio.file.Path;
72	import java.nio.file.Paths;
73	import java.util.List;
74	import java.util.Optional;
75
76	import static org.exist.util.ArgumentUtil.getBool;
77	import static org.exist.util.ArgumentUtil.getOpt;
78	import static se.softhouse.jargo.Arguments.*;
79
80
81	public class ExportMain {	×
82
83	/* general arguments */
84	private static final Argument<?> helpArg = helpArgument("-h", "--help");	×
85	private static final Argument<Boolean> verboseArg = optionArgument("-v", "--verbose")	×
86	.description("print processed resources to stdout")	×
87	.defaultValue(false)	×
88	.build();	×
89
90	/* control arguments */
91	private static final Argument<Boolean> noCheckArg = optionArgument("-n", "--nocheck")	×
92	.description("do not run a consistency check. Just export the data.")	×
93	.defaultValue(false)	×
94	.build();	×
95	private static final Argument<Boolean> checkDocsArg = optionArgument("-s", "--check-docs")	×
96	.description("scan every document to find errors in the the nodes stored (costs time)")	×
97	.defaultValue(false)	×
98	.build();	×
99	private static final Argument<Boolean> directAccessArg = optionArgument("-D", "--direct")	×
100	.description("use an (even more) direct access to the db, bypassing some index structures")	×
101	.defaultValue(false)	×
102	.build();	×
103	private static final Argument<Boolean> exportArg = optionArgument("-x", "--export")	×
104	.description("export database contents while preserving as much data as possible")	×
105	.defaultValue(false)	×
106	.build();	×
107	private static final Argument<Boolean> noExportArg = optionArgument("--no-export")	×
108	.description("do not export the database contents, overrides argument --export")	×
109	.defaultValue(false)	×
110	.build();	×
111	private static final Argument<Boolean> incrementalArg = optionArgument("-i", "--incremental")	×
112	.description("create incremental backup (use with --export\|-x)")	×
113	.defaultValue(false)	×
114	.build();	×
115	private static final Argument<Boolean> zipArg = optionArgument("-z", "--zip")	×
116	.description("write output to a ZIP instead of a file system directory")	×
117	.defaultValue(false)	×
118	.build();	×
119	private static final Argument<Boolean> noZipArg = optionArgument("--no-zip")	×
120	.description("do not zip the output, overrides argument --zip")	×
121	.defaultValue(false)	×
122	.build();	×
123
124	/* export parameters */
125	private static final Argument<File> configArg = fileArgument("-c", "--config")	×
126	.description("the database configuration (conf.xml) file to use for launching the db.")	×
127	.build();	×
128	private static final Argument<File> outputDirArg = fileArgument("-d", "--dir")	×
129	.description("the directory to which all output will be written.")	×
130	.defaultValue(Paths.get("export").toAbsolutePath().toFile())	×
131	.build();	×
132
133	protected static BrokerPool startDB(final Optional<Path> configFile) {
134	try {
135	final Configuration config;
136
137	if (configFile.isPresent()) {	×
138	config = new Configuration(configFile.get().toAbsolutePath().toString(), Optional.empty());	×
139	} else {	×
140	config = new Configuration();	×
141	}
142	config.setProperty(BrokerPool.PROPERTY_EXPORT_ONLY, Boolean.TRUE);	×
143	BrokerPool.configure(1, 5, config);	×
144	return (BrokerPool.getInstance());	×
145	} catch (final DatabaseConfigurationException \| EXistException e) {	×
146	System.err.println("ERROR: Failed to open database: " + e.getMessage());	×
147	}
148	return (null);	×
149	}
150
151
152	@SuppressWarnings("unchecked")
153	public static void main(final String[] args) {
154	try {
155	CompatibleJavaVersionCheck.checkForCompatibleJavaVersion();	×
156
157	final ParsedArguments arguments = CommandLineParser	×
158	.withArguments(noCheckArg, checkDocsArg, directAccessArg, exportArg, noExportArg, incrementalArg, zipArg, noZipArg)	×
159	.andArguments(configArg, outputDirArg)	×
160	.andArguments(helpArg, verboseArg)	×
161	.programName("export" + (OSUtil.IS_WINDOWS ? ".bat" : ".sh"))	×
162	.parse(args);	×
163
164	process(arguments);	×
165	} catch (final StartException e) {	×
166	if (e.getMessage() != null && !e.getMessage().isEmpty()) {	×
167	System.err.println(e.getMessage());	×
168	}
169	System.exit(e.getErrorCode());	×
170	} catch (final ArgumentException e) {	×
171	System.out.println(e.getMessageAndUsage());	×
172	System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);	×
173
174	}
175	}	×
176
177	private static void process(final ParsedArguments arguments) {
178	final boolean verbose = getBool(arguments, verboseArg);	×
179
180	final boolean noCheck = getBool(arguments, noCheckArg);	×
181	final boolean checkDocs = getBool(arguments, checkDocsArg);	×
182	final boolean direct = getBool(arguments, directAccessArg);	×
183	boolean export = getBool(arguments, exportArg);	×
184	final boolean noExport = getBool(arguments, noExportArg);	×
185	if (noExport) {	×
186	export = false;	×
187	}
188	final boolean incremental = getBool(arguments, incrementalArg);	×
189	boolean zip = getBool(arguments, zipArg);	×
190	final boolean noZip = getBool(arguments, noZipArg);	×
191	if (noZip) {	×
192	zip = false;	×
193	}
194
195	final Optional<Path> dbConfig = getOpt(arguments, configArg).map(File::toPath);	×
196	final Path exportTarget = arguments.get(outputDirArg).toPath();	×
197
198	final BrokerPool pool = startDB(dbConfig);	×
199
200	if (pool == null) {	×
201	System.exit(SystemExitCodes.CATCH_ALL_GENERAL_ERROR_EXIT_CODE);	×
202	}
203	int retval = 0; // return value	×
204
205	try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()));	×
206	final Txn transaction = pool.getTransactionManager().beginTransaction()) {	×
207
208	List<ErrorReport> errors = null;	×
209
210	if (!noCheck) {	×
211	final ConsistencyCheck checker = new ConsistencyCheck(broker, transaction, direct, checkDocs);	×
212	errors = checker.checkAll(new CheckCallback());	×
213	}
214
215	if (errors != null && !errors.isEmpty()) {	×
216	System.err.println("ERRORS FOUND.");	×
217	retval = 1;	×
218	} else {	×
219	System.out.println("No errors.");	×
220	}
221
222	if (export) {	×
223	if (!Files.exists(exportTarget)) {	×
224	Files.createDirectories(exportTarget);	×
225	} else if(!Files.isDirectory(exportTarget)) {	×
226	System.err.println("Output dir already exists and is a file: " + exportTarget.toAbsolutePath().toString());	×
227	System.exit(SystemExitCodes.INVALID_ARGUMENT_EXIT_CODE);	×
228	}
229	final SystemExport sysexport = new SystemExport(broker, transaction, new Callback(verbose), null, direct);	×
230	sysexport.export(exportTarget.toAbsolutePath().toString(), incremental, zip, errors);	×
231	}
232
233	transaction.commit();	×
234
235	} catch (final EXistException e) {	×
236	System.err.println("ERROR: Failed to retrieve database broker: " + e.getMessage());	×
237	retval = SystemExitCodes.NO_BROKER_EXIT_CODE;	×
238	} catch (final TerminatedException e) {	×
239	System.err.println("WARN: Export was terminated by db.");	×
240	retval = SystemExitCodes.TERMINATED_EARLY_EXIT_CODE;	×
241	} catch (final PermissionDeniedException pde) {	×
242	System.err.println("ERROR: Failed to retrieve database data: " + pde.getMessage());	×
243	retval = SystemExitCodes.PERMISSION_DENIED_EXIT_CODE;	×
244	} catch (final IOException ioe) {	×
245	System.err.println("ERROR: Failed to retrieve database data: " + ioe.getMessage());	×
246	retval = SystemExitCodes.IO_ERROR_EXIT_CODE;	×
247	} finally {
248	BrokerPool.stopAll(false);	×
249	}
250	System.exit(retval);	×
251	}	×
252
253	private static class Callback implements SystemExport.StatusCallback {
254
255	private boolean verbose = false;	×
256
257	public Callback(final boolean verbose) {	×
258	this.verbose = verbose;	×
259	}	×
260
261	@Override
262	public void startCollection(final String path) {
263	if (verbose) {	×
264	System.out.println("Entering collection " + path + " ...");	×
265	}
266	}	×
267
268	@Override
269	public void startDocument(final String name, final int count, final int docsCount) {
270	if (verbose) {	×
271	System.out.println("Writing document " + name + " [" + (count + 1) + " of " + docsCount + ']');	×
272	}
273	}	×
274
275	@Override
276	public void error(final String message, final Throwable exception) {
277	System.err.println(message);	×
278
279	if (exception != null) {	×
280	exception.printStackTrace();	×
281	}
282	}	×
283	}
284
285
286	private static class CheckCallback implements org.exist.backup.ConsistencyCheck.ProgressCallback {
287	@Override
288	public void startDocument(final String name, final int current, final int count) {
289	}	×
290
291	@Override
292	public void startCollection(final String path) {
293	}	×
294
295	@Override
296	public void error(final ErrorReport error) {
297	System.out.println(error.toString());	×
298	}	×
299	}
300	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous