evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.43

/exist-core/src/main/java/org/exist/xmldb/RemoteRestoreService.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 *
 * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
 *       The original license header is included below.
 *
 * =====================================================================
 *
 * eXist-db Open Source Native XML Database
 * Copyright (C) 2001 The eXist-db Authors
 *
 * info@exist-db.org
 * http://www.exist-db.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.xmldb;

import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
import org.exist.util.FileUtils;
import org.exist.util.Leasable;
import org.exist.xmlrpc.RpcAPI;
import org.xmldb.api.base.Collection;
import org.xmldb.api.base.ErrorCodes;
import org.xmldb.api.base.XMLDBException;

import javax.annotation.Nullable;
import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import static org.exist.util.FileUtils.withUnixSep;
import static org.exist.xmldb.RemoteCollection.MAX_UPLOAD_CHUNK;

public class RemoteRestoreService implements EXistRestoreService {

    private final Leasable<XmlRpcClient> leasableXmlRpcClient;
    private final RemoteCallSite remoteCallSite;

    /**
     * Constructor for DatabaseInstanceManagerImpl.
     *
     * @param leasableXmlRpcClient the leasable instance of a the XML RPC client
     * @param remoteCallSite the remote call site
     */
    public RemoteRestoreService(final Leasable<XmlRpcClient> leasableXmlRpcClient, final RemoteCallSite remoteCallSite) {
        this.leasableXmlRpcClient = leasableXmlRpcClient;
        this.remoteCallSite = remoteCallSite;
    }

    @Override
    public String getName() {
        return "RestoreService";
    }

    @Override
    public String getVersion() {
        return "1.0";
    }

    @Override
    public void restore(final String backup, @Nullable final String newAdminPassword,
            final RestoreServiceTaskListener restoreListener, final boolean overwriteApps) throws XMLDBException {
        final Path backupPath = Paths.get(backup).normalize().toAbsolutePath();
        if (!Files.exists(backupPath)) {
            throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Backup does not exist: " + backupPath);
        }

        final String remoteFileName;
        final String backupFileName = FileUtils.fileName(backupPath);
        if (backupFileName.endsWith(".zip")) {
            remoteFileName = uploadBackupFile(backupPath, restoreListener);
        } else if (Files.isDirectory(backupPath)) {
            final Path tmpZipFile = zipBackupDir(backupPath, restoreListener);
            try {
                remoteFileName = uploadBackupFile(tmpZipFile, restoreListener);
            } finally {
                FileUtils.deleteQuietly(tmpZipFile);
            }
        } else if (backupFileName.equals("__contents__.xml")) {
            final Path tmpZipFile = zipBackupDir(backupPath.getParent(), restoreListener);
            try {
                remoteFileName = uploadBackupFile(tmpZipFile, restoreListener);
            } finally {
                FileUtils.deleteQuietly(tmpZipFile);
            }
        } else {
            throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Backup does not appear to be an Elemental backup");
        }

        final String restoreTaskHandle;
        List<Object> params = new ArrayList<>();
        params.add(newAdminPassword);
        params.add(remoteFileName);
        params.add(overwriteApps);
        restoreTaskHandle = (String) remoteCallSite.execute("restore", params);

        // has the admin password changed?
        try (Leasable<XmlRpcClient>.Lease xmlRpcClientLease = leasableXmlRpcClient.lease()){
            final XmlRpcClientConfigImpl config = (XmlRpcClientConfigImpl) xmlRpcClientLease.get().getClientConfig();
            final String currentPassword = config.getBasicPassword();
            if (newAdminPassword != null && !currentPassword.equals(newAdminPassword)) {
                config.setBasicPassword(newAdminPassword);
            }
            try {
                Thread.sleep(3000);
            } catch (final InterruptedException e) {
                // restore interrupt status
                Thread.currentThread().interrupt();

                throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e);
            }
        }

        // now we need to poll for results...
        boolean finished = false;
        params = new ArrayList<>();
        params.add(restoreTaskHandle);
        while (!finished) {
            final List<String> events = new ArrayList<>();
            final Object[] results = (Object[]) remoteCallSite.execute("getRestoreTaskEvents", params);
            if (results != null) {
                for (final Object result : results) {
                    events.add((String)result);
                }
            }

            for (final String event : events) {

                // dispatch event to the listener
                switch (RpcAPI.RestoreTaskEvent.fromCode(event.charAt(0))) {
                    case STARTED:
                        restoreListener.started(Long.parseLong(event.substring(1)));
                        break;

                    case PROCESSING_DESCRIPTOR:
                        restoreListener.processingDescriptor(event.substring(1));
                        break;

                    case CREATED_COLLECTION:
                        restoreListener.createdCollection(event.substring(1));
                        break;

                    case RESTORED_RESOURCE:
                        restoreListener.restoredResource(event.substring(1));
                        break;

                    case SKIP_RESOURCES:
                        final int sep = event.indexOf('@');
                        final String strCount = event.substring(1, sep);
                        final String message = event.substring(sep + 1);
                        restoreListener.skipResources(message, Long.valueOf(strCount));

                    case INFO:
                        restoreListener.info(event.substring(1));
                        break;

                    case WARN:
                        restoreListener.warn(event.substring(1));
                        break;

                    case ERROR:
                        restoreListener.error(event.substring(1));
                        break;

                    case FINISHED:
                        restoreListener.finished();
                        finished = true;
                        break;
                }

                if (finished) {
                    break; // exit the for loop! we are done...
                }
            }

            // before looping... sleep a bit, if we got zero events sleep longer as the server is likely busy restoring something large
            if (!finished) {
                try {
                    if (!events.isEmpty()) {
                        Thread.sleep(1500);
                    } else {
                        Thread.sleep(3000);
                    }
                } catch (final InterruptedException e) {
                    // restore interrupt status
                    Thread.currentThread().interrupt();

                    throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e);
                }
            }

            // stop looping on finished event or on exception
        }
    }

    private String uploadBackupFile(final Path backupZipFile, final RestoreServiceTaskListener restoreListener) throws XMLDBException {
        try {
            final long backupZipFileSize = Files.size(backupZipFile);

            restoreListener.startedTransfer(backupZipFileSize);

            String fileName = null;
            final byte[] chunk = new byte[(int) Math.min(backupZipFileSize, MAX_UPLOAD_CHUNK)];
            try (final InputStream is = new BufferedInputStream(Files.newInputStream(backupZipFile))) {
                int len = -1;
                while ((len = is.read(chunk)) > -1) {
                    final List<Object> params = new ArrayList<>(4);
                    if (fileName != null) {
                        params.add(fileName);
                    }
                    params.add(chunk);
                    params.add(len);
                    fileName = (String) remoteCallSite.execute("upload", params);

                    restoreListener.transferred(len);
                }
            }

            restoreListener.finishedTransfer();

            return fileName;
        } catch (final IOException e) {
            throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Unable to upload backup file: " + e.getMessage());
        }
    }

    private Path zipBackupDir(final Path dir, final RestoreServiceTaskListener restoreListener) throws XMLDBException {
        restoreListener.startedZipForTransfer(FileUtils.sizeQuietly(dir));
        try {
            final Path zipFile = Files.createTempFile("remote-restore-service", "zip");
            try (final OutputStream fos = new BufferedOutputStream(Files.newOutputStream(zipFile));
                 final ZipOutputStream zos = new ZipOutputStream(fos)) {
                Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                        final Path zipEntryPath = dir.relativize(file);
                        final String zipEntryName = withUnixSep(zipEntryPath.toString());
                        zos.putNextEntry(new ZipEntry(zipEntryName));
                        final long written = Files.copy(file, zos);
                        zos.closeEntry();

                        restoreListener.addedFileToZipForTransfer(written);

                        return FileVisitResult.CONTINUE;
                    }
                });
            }
            restoreListener.finishedZipForTransfer();
            return zipFile;
        } catch (final IOException e) {
            throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Unable to zip backup dir: " + e.getMessage());
        }
    }

    @Override
    public void setCollection(final Collection collection) {
    }

    @Override
    public String getProperty(final String s) {
        return null;
    }

    @Override
    public String getProperty(String name, String defaultValue) throws XMLDBException {
        return defaultValue;
    }

    @Override
    public void setProperty(final String s, final String s1) {
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*
24	* NOTE: Parts of this file contain code from 'The eXist-db Authors'.
25	* The original license header is included below.
26	*
27	* =====================================================================
28	*
29	* eXist-db Open Source Native XML Database
30	* Copyright (C) 2001 The eXist-db Authors
31	*
32	* info@exist-db.org
33	* http://www.exist-db.org
34	*
35	* This library is free software; you can redistribute it and/or
36	* modify it under the terms of the GNU Lesser General Public
37	* License as published by the Free Software Foundation; either
38	* version 2.1 of the License, or (at your option) any later version.
39	*
40	* This library is distributed in the hope that it will be useful,
41	* but WITHOUT ANY WARRANTY; without even the implied warranty of
42	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
43	* Lesser General Public License for more details.
44	*
45	* You should have received a copy of the GNU Lesser General Public
46	* License along with this library; if not, write to the Free Software
47	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
48	*/
49	package org.exist.xmldb;
50
51	import org.apache.xmlrpc.client.XmlRpcClient;
52	import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
53	import org.exist.util.FileUtils;
54	import org.exist.util.Leasable;
55	import org.exist.xmlrpc.RpcAPI;
56	import org.xmldb.api.base.Collection;
57	import org.xmldb.api.base.ErrorCodes;
58	import org.xmldb.api.base.XMLDBException;
59
60	import javax.annotation.Nullable;
61	import java.io.*;
62	import java.nio.file.*;
63	import java.nio.file.attribute.BasicFileAttributes;
64	import java.util.ArrayList;
65	import java.util.List;
66	import java.util.zip.ZipEntry;
67	import java.util.zip.ZipOutputStream;
68
69	import static org.exist.util.FileUtils.withUnixSep;
70	import static org.exist.xmldb.RemoteCollection.MAX_UPLOAD_CHUNK;
71
72	public class RemoteRestoreService implements EXistRestoreService {
73
74	private final Leasable<XmlRpcClient> leasableXmlRpcClient;
75	private final RemoteCallSite remoteCallSite;
76
77	/**
78	* Constructor for DatabaseInstanceManagerImpl.
79	*
80	* @param leasableXmlRpcClient the leasable instance of a the XML RPC client
81	* @param remoteCallSite the remote call site
82	*/
83	public RemoteRestoreService(final Leasable<XmlRpcClient> leasableXmlRpcClient, final RemoteCallSite remoteCallSite) {	1✔
84	this.leasableXmlRpcClient = leasableXmlRpcClient;	1✔
85	this.remoteCallSite = remoteCallSite;	1✔
86	}	1✔
87
88	@Override
89	public String getName() {
90	return "RestoreService";	×
91	}
92
93	@Override
94	public String getVersion() {
95	return "1.0";	×
96	}
97
98	@Override
99	public void restore(final String backup, @Nullable final String newAdminPassword,
100	final RestoreServiceTaskListener restoreListener, final boolean overwriteApps) throws XMLDBException {
101	final Path backupPath = Paths.get(backup).normalize().toAbsolutePath();	1✔
102	if (!Files.exists(backupPath)) {	1!
103	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Backup does not exist: " + backupPath);	×
104	}
105
106	final String remoteFileName;
107	final String backupFileName = FileUtils.fileName(backupPath);	1✔
108	if (backupFileName.endsWith(".zip")) {	1✔
109	remoteFileName = uploadBackupFile(backupPath, restoreListener);	1✔
110	} else if (Files.isDirectory(backupPath)) {	1!
111	final Path tmpZipFile = zipBackupDir(backupPath, restoreListener);	×
112	try {
113	remoteFileName = uploadBackupFile(tmpZipFile, restoreListener);	×
114	} finally {	×
115	FileUtils.deleteQuietly(tmpZipFile);	×
116	}
117	} else if (backupFileName.equals("__contents__.xml")) {	1!
118	final Path tmpZipFile = zipBackupDir(backupPath.getParent(), restoreListener);	1✔
119	try {
120	remoteFileName = uploadBackupFile(tmpZipFile, restoreListener);	1✔
121	} finally {	1✔
122	FileUtils.deleteQuietly(tmpZipFile);	1✔
123	}
124	} else {
125	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Backup does not appear to be an Elemental backup");	×
126	}
127
128	final String restoreTaskHandle;
129	List<Object> params = new ArrayList<>();	1✔
130	params.add(newAdminPassword);	1✔
131	params.add(remoteFileName);	1✔
132	params.add(overwriteApps);	1✔
133	restoreTaskHandle = (String) remoteCallSite.execute("restore", params);	1✔
134
135	// has the admin password changed?
136	try (Leasable<XmlRpcClient>.Lease xmlRpcClientLease = leasableXmlRpcClient.lease()){	1✔
137	final XmlRpcClientConfigImpl config = (XmlRpcClientConfigImpl) xmlRpcClientLease.get().getClientConfig();	1✔
138	final String currentPassword = config.getBasicPassword();	1✔
139	if (newAdminPassword != null && !currentPassword.equals(newAdminPassword)) {	1!
140	config.setBasicPassword(newAdminPassword);	1✔
141	}
142	try {
143	Thread.sleep(3000);	1✔
144	} catch (final InterruptedException e) {	1✔
145	// restore interrupt status
146	Thread.currentThread().interrupt();	×
147
148	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e);	×
149	}
150	}
151
152	// now we need to poll for results...
153	boolean finished = false;	1✔
154	params = new ArrayList<>();	1✔
155	params.add(restoreTaskHandle);	1✔
156	while (!finished) {	1✔
157	final List<String> events = new ArrayList<>();	1✔
158	final Object[] results = (Object[]) remoteCallSite.execute("getRestoreTaskEvents", params);	1✔
159	if (results != null) {	1!
160	for (final Object result : results) {	1✔
161	events.add((String)result);	1✔
162	}
163	}
164
165	for (final String event : events) {	1!
166
167	// dispatch event to the listener
168	switch (RpcAPI.RestoreTaskEvent.fromCode(event.charAt(0))) {	1!
169	case STARTED:
170	restoreListener.started(Long.parseLong(event.substring(1)));	1✔
171	break;	1✔
172
173	case PROCESSING_DESCRIPTOR:
174	restoreListener.processingDescriptor(event.substring(1));	1✔
175	break;	1✔
176
177	case CREATED_COLLECTION:
178	restoreListener.createdCollection(event.substring(1));	1✔
179	break;	1✔
180
181	case RESTORED_RESOURCE:
182	restoreListener.restoredResource(event.substring(1));	1✔
183	break;	1✔
184
185	case SKIP_RESOURCES:
186	final int sep = event.indexOf('@');	×
187	final String strCount = event.substring(1, sep);	×
188	final String message = event.substring(sep + 1);	×
189	restoreListener.skipResources(message, Long.valueOf(strCount));	×
190
191	case INFO:
192	restoreListener.info(event.substring(1));	×
193	break;	×
194
195	case WARN:
196	restoreListener.warn(event.substring(1));	1✔
197	break;	1✔
198
199	case ERROR:
200	restoreListener.error(event.substring(1));	×
201	break;	×
202
203	case FINISHED:
204	restoreListener.finished();	1✔
205	finished = true;	1✔
206	break;
207	}
208
209	if (finished) {	1✔
210	break; // exit the for loop! we are done...	1✔
211	}
212	}
213
214	// before looping... sleep a bit, if we got zero events sleep longer as the server is likely busy restoring something large
215	if (!finished) {	1!
216	try {
217	if (!events.isEmpty()) {	×
218	Thread.sleep(1500);	×
219	} else {	×
220	Thread.sleep(3000);	×
221	}
222	} catch (final InterruptedException e) {	×
223	// restore interrupt status
224	Thread.currentThread().interrupt();	×
225
226	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e);	×
227	}
228	}
229
230	// stop looping on finished event or on exception
231	}
232	}	1✔
233
234	private String uploadBackupFile(final Path backupZipFile, final RestoreServiceTaskListener restoreListener) throws XMLDBException {
235	try {
236	final long backupZipFileSize = Files.size(backupZipFile);	1✔
237
238	restoreListener.startedTransfer(backupZipFileSize);	1✔
239
240	String fileName = null;	1✔
241	final byte[] chunk = new byte[(int) Math.min(backupZipFileSize, MAX_UPLOAD_CHUNK)];	1✔
242	try (final InputStream is = new BufferedInputStream(Files.newInputStream(backupZipFile))) {	1✔
243	int len = -1;	1✔
244	while ((len = is.read(chunk)) > -1) {	1✔
245	final List<Object> params = new ArrayList<>(4);	1✔
246	if (fileName != null) {	1!
247	params.add(fileName);	×
248	}
249	params.add(chunk);	1✔
250	params.add(len);	1✔
251	fileName = (String) remoteCallSite.execute("upload", params);	1✔
252
253	restoreListener.transferred(len);	1✔
254	}
255	}
256
257	restoreListener.finishedTransfer();	1✔
258
259	return fileName;	1✔
260	} catch (final IOException e) {	×
261	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Unable to upload backup file: " + e.getMessage());	×
262	}
263	}
264
265	private Path zipBackupDir(final Path dir, final RestoreServiceTaskListener restoreListener) throws XMLDBException {
266	restoreListener.startedZipForTransfer(FileUtils.sizeQuietly(dir));	1✔
267	try {
268	final Path zipFile = Files.createTempFile("remote-restore-service", "zip");	1✔
269	try (final OutputStream fos = new BufferedOutputStream(Files.newOutputStream(zipFile));	1✔
270	final ZipOutputStream zos = new ZipOutputStream(fos)) {	1✔
271	Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {	1✔
272	@Override
273	public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
274	final Path zipEntryPath = dir.relativize(file);	1✔
275	final String zipEntryName = withUnixSep(zipEntryPath.toString());	1✔
276	zos.putNextEntry(new ZipEntry(zipEntryName));	1✔
277	final long written = Files.copy(file, zos);	1✔
278	zos.closeEntry();	1✔
279
280	restoreListener.addedFileToZipForTransfer(written);	1✔
281
282	return FileVisitResult.CONTINUE;	1✔
283	}
284	});
285	}
286	restoreListener.finishedZipForTransfer();	1✔
287	return zipFile;	1✔
288	} catch (final IOException e) {	×
289	throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "Unable to zip backup dir: " + e.getMessage());	×
290	}
291	}
292
293	@Override
294	public void setCollection(final Collection collection) {
295	}	×
296
297	@Override
298	public String getProperty(final String s) {
299	return null;	×
300	}
301
302	@Override
303	public String getProperty(String name, String defaultValue) throws XMLDBException {
304	return defaultValue;	×
305	}
306
307	@Override
308	public void setProperty(final String s, final String s1) {
309	}	×
310	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous